sec_edgar 0.0.1

data/lib/sec_edgar/ownership_document.rb

@@ -0,0 +1,121 @@
+module SecEdgar
+  TRANSACTION_CODES = {
+    'P' => 'Open market or private purchase of non-derivative or derivative security',
+    'S' => 'Open market or private sale of non-derivative or derivative security6',
+    'V' => 'Transaction voluntarily reported earlier than required',
+    'A' => 'Grant, award or other acquisition pursuant to Rule 16b-3(d)',
+    'D' => 'Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)',
+    'F' => 'Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise',
+    'I' => 'Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer',
+    'M' => 'Exercise or conversion of derivative security exempted pursuant to Rule 16b-3',
+    'C' => 'Conversion of derivative security',
+    'E' => 'Expiration of short derivative position',
+    'H' => 'Expiration (or cancellation) of long derivative position with value received',
+    'O' => 'Exercise of out-of-the-money derivative security',
+    'X' => 'Exercise of in-the-money or at-the-money derivative security',
+    'G' => 'Bona fide gift',
+    'L' => 'Small acquisition under Rule 16a-6',
+    'W' => 'Acquisition or disposition by will or the laws of descent and distribution',
+    'Z' => 'Deposit into or withdrawal from voting trust',
+    'J' => 'Other acquisition or disposition (describe transaction)',
+    'K' => 'Transaction in equity swap or instrument with similar characteristics',
+    'U' => 'Disposition pursuant to a tender of shares in a change of control transaction'
+  }
+
+  OwnershipDocument = Struct.new(
+    :schema_version,
+    :document_type,
+    :period_of_report,
+    :not_subject_to_section_16,
+    :issuer_name,
+    :issuer_cik,
+    :issuer_trading_symbol,
+    :owner_cik,
+    :owner_name,
+    :owner_address,
+    :is_director,
+    :is_officer,
+    :is_ten_percent_owner,
+    :is_other,
+    :officer_title,
+    :transactions,
+    :derivative_transactions,
+    :footnotes
+  ) do
+
+    def sum_shares
+      @sum ||= begin
+        sum = 0
+        all_trans.each do |t|
+          if t.acquired_or_disposed_code == 'A'
+            sum += t.shares
+          else
+            sum -= t.shares
+          end
+        end
+        sum
+      end
+    end
+
+    def dollar_volume
+      @dollar_volume ||= begin
+        sum = 0
+        all_trans.each do |t|
+          if t.acquired_or_disposed_code == 'A'
+            sum += t.shares * t.price_per_share
+          else
+            sum -= t.shares * t.price_per_share
+          end
+        end
+        sum
+      end
+    end
+
+    def total_shares
+      @total ||= all_trans.map(&:shares).inject(:+)
+    end
+
+    def all_trans
+      trans + derivative_trans
+    end
+
+    def trans
+      transactions || []
+    end
+
+    def derivative_trans
+      derivative_transactions || []
+    end
+
+    SecEdgar::TRANSACTION_CODES.keys.each do |code|
+      define_method("per_code_#{ code.downcase }".to_sym) do
+        return 0 if total_shares == 0
+        all_trans.select do |t|
+          t.code.upcase == code
+        end.tap do |t|
+          return 0 if t.empty?
+        end.map do |t|
+          t.shares
+        end.inject(:+) * 100.0 / total_shares
+      end
+    end
+
+    def sum_shares_after
+      return all_trans.map(&:shares_after).max if sum_shares > 0
+      trans.map(&:shares_after).min || 0
+    end
+
+    def as_json(*)
+      tran_codes = {}
+
+      SecEdgar::TRANSACTION_CODES.keys.each do |code|
+        name = "per_code_#{ code.downcase }".to_sym
+        tran_codes[name] = send(name)
+      end
+
+      super
+        .merge(sum_shares: sum_shares, sum_shares_after: sum_shares_after)
+        .merge(tran_codes)
+    end
+  end
+end

data/lib/sec_edgar/poll.rb

@@ -0,0 +1,132 @@
+require 'fileutils'
+
+module SecEdgar
+  PollFiling = Struct.new(
+      :cik,
+      :company_name,
+      :type,
+      :date_str,
+      :ftp_link,
+      :client,
+      :title,
+      :term,
+      :file_id) do
+    def date
+      Date.parse(date_str)
+    end
+
+    def link
+      "http://www.sec.gov/Archives/#{ ftp_link }"
+    end
+
+    def content(&error_blk)
+      @content ||= client.fetch(ftp_link, nil)
+    rescue RestClient::ResourceNotFound => e
+      puts "404 Resource Not Found: Bad link #{ self.link }"
+      if block_given?
+        error_blk.call(e, self)
+      else
+        raise e
+      end
+    end
+  end
+
+  class Poll
+    attr_reader :client, :year, :quarter, :local_base
+
+    def initialize(year = 1993, qtr = 1, client = SecEdgar::FtpClient.instance, local_base = "./assets")
+      @client = client
+      @year = year
+      @quarter = qtr
+      @local_base = local_base
+    end
+
+    def self.all(from_year, to_year, &blk)
+      polls_by_year(from_year, to_year).each do |poll|
+        poll.form4s.each do |filing|
+          blk.call(filing)
+        end
+      end
+    end
+
+    def self.polls_by_year(from_year, to_year)
+      polls = []
+      (from_year..to_year).each do |yr|
+        (1..4).each do |qtr|
+          polls << self.new(yr, qtr)
+        end
+      end
+      polls
+    end
+
+    def to_s
+      "<Poll y: #{ year } q: #{ quarter }>"
+    end
+
+    def index
+      unless File.exist?(local_file)
+        puts "fetching from sec #{ to_s }..."
+        fetch
+      end
+      File.read(local_file)
+    end
+
+    def form4s
+      filings.select do |line|
+        line =~ form4_regexp
+      end.map do |f|
+        begin
+          PollFiling.new(*f.split("|"), client)
+        rescue => e
+          puts line
+          raise e
+        end
+      end
+    end
+
+    def filings
+      index.scrub.split("\n").select do |line|
+        line =~ filing_regexp
+      end
+    end
+
+    def local_file
+      "#{ local_dir }/master.idx"
+    end
+
+    private
+
+    def form4_regexp
+      /^(\d+\|.*\|4\|.*.txt)$/i
+    end
+
+    def filing_regexp
+      /^(\d+\|.*\|.*.txt)$/i
+    end
+
+    def remote
+      "/edgar/full-index/#{ year }/QTR#{ quarter }/master.idx"
+    end
+
+    def local_dir
+      path = "#{ local_base }/#{  year.to_s }/QTR#{ quarter }"
+      FileUtils.mkdir_p(path)
+      path
+    end
+
+    def fetch
+      client.fetch(remote, local_file)
+    rescue => e
+      puts "SecEdgar::Poll#fetch_quarter failed for #{ year }, QTR#{ quarter }"
+      puts "local: #{ local_file }"
+      puts "remote: #{ remote }"
+      puts e.message
+    ensure
+      verify_download
+    end
+
+    def verify_download
+      File.delete(local_file) unless File.size?(local_file)
+    end
+  end
+end

data/lib/sec_edgar/sec_uri.rb

@@ -0,0 +1,102 @@
+# encoding: UTF-8
+
+module SecEdgar
+  class SecURI
+    attr_accessor :host, :scheme, :path, :query_values
+
+    def self.browse_edgar_uri(args = nil)
+      build_with_path('/browse-edgar', args)
+    end
+
+    def self.for_date(date)
+      instance = SecURI.new
+      instance.scheme = 'ftp'
+      instance.host = 'ftp.sec.gov'
+      instance.path = "edgar/daily-index/#{ date.to_sec_uri_format }"
+      instance
+    end
+
+    def self.ownership_display_uri(args = nil)
+      build_with_path('/own-disp', args)
+    end
+
+    def self.build_with_path(path, args)
+      instance = SecURI.new
+      instance.path += path
+      return instance if args.nil?
+      options = send("handle_#{ args.class.to_s.underscore }_args", args)
+      instance.query_values = options
+      instance
+    end
+
+    def self.handle_string_args(string_arg)
+      options = {}
+      begin Float(string_arg)
+        options[:CIK] = string_arg
+      rescue
+        if string_arg.length <= 4
+          options[:CIK] = string_arg
+        else
+          options[:company] = string_arg.gsub(/[(,?!\''"":.)]/, '')
+        end
+      end
+      options
+    end
+
+    private_class_method :handle_string_args
+
+    def self.handle_hash_args(hash_arg)
+      options = hash_arg
+      if hash_arg[:symbol] || hash_arg[:cik]
+        options[:CIK] = (hash_arg[:symbol] || hash_arg[:cik])
+        return options
+      end
+      options[:company] = company_name_from_hash_args(hash_arg)
+      options
+    end
+
+    private_class_method :handle_hash_args
+
+    def self.company_name_from_hash_args(args)
+      return "#{ args[:last] } #{ args[:first] }" if args[:first] && args[:last]
+      return args[:name].gsub(/[(,?!\''"":.)]/, '') if args[:name]
+    end
+
+    private_class_method :company_name_from_hash_args
+
+    def initialize
+      self.host = 'www.sec.gov'
+      self.scheme = 'http'
+      self.path = 'cgi-bin'
+    end
+
+    def []=(key, value)
+      query_values[key] = value
+      self
+    end
+
+    def output_atom
+      query_values.merge!(output: 'atom')
+      self
+    end
+
+    def to_s
+      uri.to_s
+    end
+
+    def to_str
+      to_s
+    end
+
+    private
+
+    def uri
+      Addressable::URI.new(
+        host: host,
+        scheme: scheme,
+        path: path,
+        query_values: query_values
+      )
+    end
+  end
+end

data/lib/sec_edgar/transaction.rb

@@ -0,0 +1,20 @@
+module SecEdgar
+  Transaction = Struct.new(
+    :acquired_or_disposed_code,
+    :nature_of_ownership,
+    :code,
+    :shares,
+    :security_title,
+    :direct_or_indirect_code,
+    :form_type,
+    :equity_swap_involved,
+    :transaction_date,
+    :shares_after,
+    :price_per_share
+  ) do
+    def holdings_change
+      return 0 if shares_after == 0
+      shares.fdiv(shares_after)
+    end
+  end
+end

data/lib/sec_edgar/utils.rb

@@ -0,0 +1,28 @@
+class String
+  def blank?
+    /^\s*$/.match(self)
+  end
+
+  def underscore
+    self.gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase
+  end
+end
+
+class Date
+  def quarter
+    ((month / 3.0) - 0.1).floor + 1
+  end
+
+  def to_sec_uri_format
+    today = Date.today
+    if today.quarter == quarter && today.year == year
+      "company.#{ strftime("%Y%m%d") }.idx"
+    else
+      "#{ year }/QTR#{ quarter }/company.#{ strftime("%Y%m%d") }.idx"
+    end
+  end
+end

data/lib/sec_edgar.rb

@@ -0,0 +1,26 @@
+module SecEdgar; end
+
+require 'nokogiri'
+require 'singleton'
+require 'net/ftp'
+require 'date'
+require 'addressable/uri'
+require 'open-uri'
+require 'rest-client'
+require 'rss'
+
+require_relative './sec_edgar/utils'
+require_relative './sec_edgar/sec_uri'
+require_relative './sec_edgar/entity'
+require_relative './sec_edgar/basic_stats'
+require_relative './sec_edgar/address'
+require_relative './sec_edgar/ftp_client'
+require_relative './sec_edgar/filing'
+require_relative './sec_edgar/footnote'
+require_relative './sec_edgar/derivative_transaction'
+require_relative './sec_edgar/transaction'
+require_relative './sec_edgar/filing_persister'
+require_relative './sec_edgar/filing_parser'
+require_relative './sec_edgar/officer_title'
+require_relative './sec_edgar/ownership_document'
+require_relative './sec_edgar/poll'

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: sec_edgar
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- CJ Avilla
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.8
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+description: Tool for querying the SEC database
+email: cjavilla@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/assets/1996/QTR1/master.idx
+- lib/assets/1996/QTR2/master.idx
+- lib/assets/1996/QTR3/master.idx
+- lib/assets/1996/QTR4/master.idx
+- lib/sec_edgar.rb
+- lib/sec_edgar/address.rb
+- lib/sec_edgar/basic_stats.rb
+- lib/sec_edgar/derivative_transaction.rb
+- lib/sec_edgar/entity.rb
+- lib/sec_edgar/filing.rb
+- lib/sec_edgar/filing_parser.rb
+- lib/sec_edgar/filing_persister.rb
+- lib/sec_edgar/filing_updater.rb
+- lib/sec_edgar/footnote.rb
+- lib/sec_edgar/ftp_client.rb
+- lib/sec_edgar/nil_ownership_document.rb
+- lib/sec_edgar/officer_title.rb
+- lib/sec_edgar/ownership4Document.xsd.xml
+- lib/sec_edgar/ownershipDocumentCommon.xsd.xml
+- lib/sec_edgar/ownership_document.rb
+- lib/sec_edgar/poll.rb
+- lib/sec_edgar/sec_uri.rb
+- lib/sec_edgar/transaction.rb
+- lib/sec_edgar/utils.rb
+homepage: http://rubygems.org/gems/sec_edgar
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: SEC filings finder
+test_files: []