sec_edgar 0.0.1

data/lib/sec_edgar/address.rb

@@ -0,0 +1,3 @@
+module SecEdgar
+  Address = Struct.new(:street1, :street2, :city, :state, :zip, :state_description)
+end

data/lib/sec_edgar/basic_stats.rb

@@ -0,0 +1,26 @@
+module SecEdgar
+  module BasicStats
+    extend self
+
+    def precision(tp, fp)
+      tp.fdiv(tp + fp)
+    end
+
+    def recall(tp, fn)
+      tp.fdiv(tp + fn)
+    end
+
+    def f_beta(tp, fp, fn, beta)
+      p = precision(tp, fp)
+      r = recall(tp, fn)
+      b2 = beta ** 2
+      (1 + b2) * ((p * r).fdiv((b2 * p) + r))
+    end
+
+    def f1(tp, fp, fn)
+      p = precision(tp, fp)
+      r = recall(tp, fn)
+      2 * ((p * r).fdiv(p + r))
+    end
+  end
+end

data/lib/sec_edgar/derivative_transaction.rb

@@ -0,0 +1,21 @@
+module SecEdgar
+  DerivativeTransaction = Struct.new(
+    :acquired_or_disposed_code,
+    :conversion_or_exercise_price,
+    :deemed_execution_date,
+    :exercise_date,
+    :expiration_date,
+    :underlying_security_title,
+    :underlying_security_shares,
+    :nature_of_ownership,
+    :code,
+    :shares,
+    :security_title,
+    :direct_or_indirect_code,
+    :form_type,
+    :equity_swap_involved,
+    :transaction_date,
+    :shares_after,
+    :price_per_share
+  )
+end

data/lib/sec_edgar/entity.rb

@@ -0,0 +1,74 @@
+# encoding: UTF-8
+module SecEdgar
+  class Entity
+    COLUMNS = [
+      :cik,
+      :name,
+      :mailing_address,
+      :business_address,
+      :assigned_sic,
+      :assigned_sic_desc,
+      :assigned_sic_href,
+      :assitant_director,
+      :cik_href,
+      :formerly_names,
+      :state_location,
+      :state_location_href,
+      :state_of_incorporation
+    ]
+
+    attr_accessor(*COLUMNS)
+
+    def initialize(entity)
+      COLUMNS.each do |column|
+        instance_variable_set("@#{ column }", entity[column.to_s])
+      end
+    end
+
+    def filings
+      Filing.find(@cik)
+    end
+
+    def transactions
+      Transaction.find(@cik)
+    end
+
+    def self.query(url)
+      RestClient.get(url) do |response, request, result, &block|
+        case response.code
+        when 200
+          return response
+        else
+          response.return!(request, result, &block)
+        end
+      end
+    end
+
+    # def self.find(entity_args)
+    #   temp = {}
+    #   temp[:url] = SecURI.browse_edgar_uri(entity_args)
+    #   temp[:url][:action] = :getcompany
+    #   response = query(temp[:url].output_atom.to_s)
+    #   document = Nokogiri::HTML(response)
+    #   xml = document.xpath("//feed/company-info")
+    #   Entity.new(parse(xml))
+    # end
+    #
+    # def self.parse(xml)
+    #   content = Hash.from_xml(xml.to_s)
+    #   if content['company_info'].present?
+    #     content = content['company_info']
+    #     content['name'] = content.delete('conformed_name')
+    #     if content['formerly_names'].present?
+    #       content['formerly_names'] = content.delete('formerly_names')['names']
+    #     end
+    #     content['addresses']['address'].each do |address|
+    #       content["#{address['type']}_address"] = address
+    #     end
+    #     return content
+    #   else
+    #     return {}
+    #   end
+    # end
+  end
+end

data/lib/sec_edgar/filing.rb

@@ -0,0 +1,171 @@
+# encoding: UTF-8
+
+module SecEdgar
+  class Filing
+    COLUMNS = [:cik, :title, :summary, :link, :term, :date, :file_id]
+
+    attr_accessor(*COLUMNS)
+
+    def initialize(filing)
+      COLUMNS.each do |column|
+        instance_variable_set("@#{ column }", filing[column])
+      end
+    end
+
+    def self.fetch(uri, &blk)
+      open(uri) do |rss|
+        parse_rss(rss, &blk)
+      end
+    end
+
+    def self.recent(options = {}, &blk)
+      start = options.fetch(:start, 0)
+      count = options.fetch(:count, 100)
+      limit = options.fetch(:limit, 100)
+      limited_count = [limit - start, count].min
+      fetch(uri_for_recent(start, limited_count), &blk)
+      start += count
+      return if start >= limit
+      recent({ start: start, count: count, limit: limit }, &blk)
+    rescue OpenURI::HTTPError => e
+      puts e
+      return
+    end
+
+    def self.for_cik(cik, options = {}, &blk)
+      start = options.fetch(:start, 0)
+      count = options.fetch(:count, 100)
+      limit = options.fetch(:limit, 100)
+      fetch(uri_for_cik(cik, start, count), &blk)
+      start += count
+      return if start >= limit
+      for_cik(cik, { start: start, count: count, limit: limit }, &blk)
+    rescue OpenURI::HTTPError
+      return
+    end
+
+    def self.for_date(date, &blk)
+      ftp = Net::FTP.new('ftp.sec.gov')
+      ftp.login
+      file_name = ftp.nlst("edgar/daily-index/#{ date.to_sec_uri_format }*")[0]
+      ftp.close
+      open("ftp://ftp.sec.gov/#{ file_name }") do |file|
+        if file_name[-2..-1] == 'gz'
+          gz_reader = Zlib::GzipReader.new(file)
+          gz_reader.rewind
+          filings_for_index(gz_reader).each(&blk)
+        else
+          filings_for_index(file).each(&blk)
+        end
+      end
+    rescue Net::FTPTempError
+    end
+
+    def self.filings_for_index(index)
+      [].tap do |filings|
+        content_section = false
+        index.each_line do |row|
+          content_section = true if row.include?('-------------')
+          next if !content_section || row.include?('------------')
+          filing = filing_for_index_row(row)
+          filings << filing unless filing.nil?
+        end
+      end
+    end
+
+    def self.filing_for_index_row(row)
+      data = row.split(/   /).reject(&:blank?).map(&:strip)
+      data = row.split(/  /).reject(&:blank?).map(&:strip) if data.count == 4
+      data[1].gsub!('/ADV', '')
+      data.delete_at(1) if data[1][0] == '/'
+      return nil unless Regexp.new(/\d{8}/).match(data[3])
+      unless data[4][0..3] == 'http'
+        data[4] = "http://www.sec.gov/Archives/#{ data[4] }"
+      end
+      Filing.new(
+        term: data[1],
+        cik: data[2],
+        date: Date.parse(data[3]),
+        link: data[4]
+      )
+    end
+
+    def self.uri_for_recent(start = 0, count = 100)
+      SecURI.browse_edgar_uri(
+        action: :getcurrent,
+        owner: :include,
+        output: :atom,
+        start: start,
+        count: count
+      )
+    end
+
+    def self.uri_for_cik(cik, start = 0, count = 100)
+      SecURI.browse_edgar_uri(
+        action: :getcompany,
+        owner: :include,
+        output: :atom,
+        start: start,
+        count: count,
+        CIK: cik
+      )
+    end
+
+    def self.parse_rss(rss, &blk)
+      feed = RSS::Parser.parse(rss, false)
+      feed.entries.each do |entry|
+        filing = Filing.new(
+          cik: entry.title.content.match(/\((\w{10})\)/)[1],
+          file_id: entry.id.content.split('=').last,
+          term:  entry.category.term,
+          title: entry.title.content,
+          summary: entry.summary.content,
+          date: DateTime.parse(entry.updated.content.to_s),
+          link: entry.link.href.gsub('-index.htm', '.txt')
+        )
+        blk.call(filing)
+      end
+    end
+
+    # def self.find(cik, start = 0, count = 80)
+    #   temp = {}
+    #   temp[:url] = SecURI.browse_edgar_uri({cik: cik})
+    #   temp[:url][:action] = :getcompany
+    #   temp[:url][:start] = start
+    #   temp[:url][:count] = count
+    #   response = Entity.query(temp[:url].output_atom.to_s)
+    #   document = Nokogiri::HTML(response)
+    #   parse(cik, document)
+    # end
+
+    def self.parse(cik, document)
+      filings = []
+      if document.xpath('//content').to_s.length > 0
+        document.xpath('//content').each do |e|
+          if e.xpath('//content/accession-nunber').to_s.length > 0
+            content = Hash.from_xml(e.to_s)['content']
+            content[:cik] = cik
+            content[:file_id] = content.delete('accession_nunber')
+            content[:date] = content.delete('filing_date')
+            content[:link] = content.delete('filing_href')
+            content[:term] = content.delete('filing_type')
+            content[:title] = content.delete('form_name')
+            filings << Filing.new(content)
+          end
+        end
+      end
+      filings
+    end
+
+    def content(&error_blk)
+      @content ||= RestClient.get(self.link)
+    rescue RestClient::ResourceNotFound => e
+      puts "404 Resource Not Found: Bad link #{ self.link }"
+      if block_given?
+        error_blk.call(e, self)
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/sec_edgar/filing_parser.rb

@@ -0,0 +1,216 @@
+module SecEdgar
+  class FilingParser
+    attr_reader :xsd_uri
+
+    def initialize(filing)
+      @xsd_uri = 'lib/sec4/ownership4Document.xsd.xml'
+      @filing = filing
+      @content = filing.content
+    end
+
+    def content
+      if @content.start_with?('-----BEGIN PRIVACY-ENHANCED MESSAGE-----')
+        @content = strip_privacy_wrapper(@content)
+      end
+      @content
+    end
+
+    def doc
+      @doc ||= OwnershipDocument.new
+    end
+
+    def parse(&error_blk)
+      if block_given? && !xml_valid?
+        error_blk.call(xml_errors)
+        puts "Error: returning NilObjectDocument #{ @filing.link }"
+        return NilOwnershipDocument.new
+      end
+
+      footnotes | transactions | derivative_transactions # eager init
+      parse_doc(xml_doc)
+      parse_issuer(xml_doc.xpath('//issuer'))
+      parse_owner(xml_doc.xpath('//reportingOwner'))
+      parse_non_derivative_table(xml_doc.xpath('//nonDerivativeTable'))
+      parse_derivative_table(xml_doc.xpath('//derivativeTable'))
+      parse_footnotes(xml_doc.xpath('//footnotes'))
+      doc
+    end
+
+    def footnotes
+      doc.footnotes ||= []
+    end
+
+    def transactions
+      doc.transactions ||= []
+    end
+
+    def derivative_transactions
+      doc.derivative_transactions ||= []
+    end
+
+    def xml_filing
+      Nokogiri::XML(xml_doc.to_xml)
+    end
+
+    private
+
+    def strip_privacy_wrapper(raw)
+      lines = raw.split("\n")
+      lines.shift until lines.first.start_with?('<SEC-DOCUMENT>')
+      lines.pop if lines.last.start_with?('-----END PRIVACY-ENHANCED MESSAGE--')
+      lines.join("\n")
+    end
+
+    def parse_transaction(el)
+      transaction = Transaction.new
+      transaction.security_title = el.xpath('securityTitle').text.strip
+      transaction.transaction_date = Date.parse(el.xpath('transactionDate').text)
+
+      parse_transaction_amounts(transaction, el.xpath('transactionAmounts'))
+      parse_transaction_ownership_nature(transaction, el.xpath('ownershipNature'))
+      parse_transaction_coding(transaction, el.xpath('transactionCoding'))
+
+      # post transaction amounts
+      transaction.shares_after = el
+        .xpath('postTransactionAmounts/sharesOwnedFollowingTransaction/value')
+        .text
+        .to_f
+      transactions << transaction
+    end
+
+    def parse_footnote(el)
+      footnote = Footnote.new
+      footnote.content = el.text.strip
+      footnote.id = el.attribute("id").value
+      footnotes << footnote
+    end
+
+    def parse_derivative_transaction(el)
+      transaction = DerivativeTransaction.new
+
+      transaction.security_title = el.xpath('securityTitle').text.strip
+      transaction.transaction_date = Date.parse(el.xpath('transactionDate').text)
+      transaction.conversion_or_exercise_price = el.xpath('conversionOrExercisePrice').text.to_f
+
+      unless (expiration_date = el.xpath('expirationDate/value').text).blank?
+        transaction.expiration_date = Date.parse(expiration_date)
+      end
+
+      unless (exercise_date = el.xpath('exerciseDate/value').text).blank?
+        transaction.exercise_date = Date.parse(exercise_date)
+      end
+
+      parse_transaction_amounts(transaction, el.xpath('transactionAmounts'))
+      parse_transaction_ownership_nature(transaction, el.xpath('ownershipNature'))
+      parse_transaction_coding(transaction, el.xpath('transactionCoding'))
+      parse_transaction_underlying(transaction, el.xpath('underlyingSecurity'))
+
+      # post transaction amounts
+      transaction.shares_after = el
+        .xpath('postTransactionAmounts/sharesOwnedFollowingTransaction/value')
+        .text
+        .to_f
+      derivative_transactions << transaction
+    end
+
+    def parse_transaction_underlying(transaction, el)
+      transaction.underlying_security_title = el.xpath('underlyingSecurityTitle/value').text
+      transaction.underlying_security_shares =
+        el.xpath('underlyingSecurityShares/value').text.to_f
+    end
+
+    def parse_non_derivative_table(el)
+      el.xpath('//nonDerivativeTransaction').each do |transaction_el|
+        parse_transaction(transaction_el)
+      end
+    end
+
+    def parse_derivative_table(el)
+      el.xpath('//derivativeTransaction').each do |transaction_el|
+        parse_derivative_transaction(transaction_el)
+      end
+    end
+
+    def parse_footnotes(el)
+      el.xpath('//footnote').each do |note_el|
+        parse_footnote(note_el)
+      end
+    end
+
+    def parse_transaction_amounts(transaction, el)
+      transaction.acquired_or_disposed_code =
+        el.xpath('transactionAcquiredDisposedCode/value').text
+      transaction.shares =
+        el.xpath('transactionShares/value').text.to_f
+      transaction.price_per_share =
+        el.xpath('transactionPricePerShare/value').text.to_f
+    end
+
+    def parse_transaction_coding(transaction, el)
+      transaction.code = el.xpath('transactionCode').text
+      transaction.form_type = el.xpath('transactionFormType').text
+      transaction.equity_swap_involved = el.xpath('equitySwapInvolved').text == '1'
+    end
+
+    def parse_transaction_ownership_nature(transaction, el)
+      transaction.nature_of_ownership = el.xpath('natureOfOwnership/value').text
+      transaction.direct_or_indirect_code = el.xpath('directOrIndirectOwnership/value').text
+    end
+
+    def parse_doc(el)
+      doc.schema_version = el.xpath('//schemaVersion').text
+      doc.document_type = el.xpath('//documentType').text
+      doc.not_subject_to_section_16 =
+        (el.xpath('//notSubjectToSection16').text == '1' ||
+         el.xpath('//notSubjectToSection16').text == 'true')
+      doc.period_of_report = Date.parse(el.xpath('//periodOfReport').text)
+    rescue ArgumentError => e
+      puts "parse_doc error: #{ el.inspect }"
+      puts e
+      raise e
+    end
+
+    def parse_issuer(el)
+      doc.issuer_cik = el.xpath('//issuerCik').text
+      doc.issuer_name = el.xpath('//issuerName').text
+      doc.issuer_trading_symbol = el.xpath('//issuerTradingSymbol').text
+    end
+
+    def parse_owner(el)
+      doc.owner_cik = el.xpath('//rptOwnerCik').text
+      doc.owner_name = el.xpath('//rptOwnerName').text
+      doc.is_director = el.xpath('//isDirector').text == '1'
+      doc.is_ten_percent_owner = el.xpath('//isTenPercentOwner').text == '1'
+      doc.is_other = el.xpath('//isOther').text == '1'
+      doc.is_officer =
+        (el.xpath('//isOfficer').text == '1' ||
+         el.xpath('//isOfficer').text.downcase == 'true')
+      doc.officer_title = el.xpath('//officerTitle').text
+
+      address = Address.new
+      address.street1 = el.xpath('//rptOwnerStreet1').text
+      address.street2 = el.xpath('//rptOwnerStreet2').text
+      address.city = el.xpath('//rptOwnerCity').text
+      address.state = el.xpath('//rptOwnerState').text
+      address.zip = el.xpath('//rptOwnerZipCode').text
+      address.state_description = el.xpath('//rptOwnerStateDescription').text
+      doc.owner_address = address
+    end
+
+    def xml_doc
+      Nokogiri::XML(content).xpath('//ownershipDocument')
+    end
+
+    def xml_schema
+      @schema ||= Nokogiri::XML::Schema(IO.read(xsd_uri))
+    end
+
+    def xml_valid?
+      xml_errors.empty? && !content.blank?
+    end
+
+    def xml_errors
+      @errors ||= xml_schema.validate(xml_filing)
+    end
+  end
+end

data/lib/sec_edgar/filing_persister.rb

@@ -0,0 +1,53 @@
+# module SecEdgar
+#   class FilingPersister
+#     attr_reader :filing
+#
+#     def initialize(filing)
+#       @filing = filing
+#     end
+#
+#     def doc
+#       @doc ||= RawFiling.for_filing(filing).parsed
+#     end
+#
+#     def persist!
+#       return form if form = Form4.find_by(link: filing.link)
+#       doc_json = doc.to_json
+#       return if doc_json == '{}'
+#
+#       form = Form4.new(
+#         cik: filing.cik,
+#         title: filing.title,
+#         link: filing.link,
+#         term: filing.term,
+#         date: filing.date,
+#         file_id: filing.file_id,
+#         dollar_volume: doc.dollar_volume,
+#         document: { d: doc_json }
+#       )
+#       form.company = Company.where(cik: doc.issuer_cik).first_or_initialize
+#       form.company.update_attributes(
+#         name: doc.issuer_name,
+#         ticker: doc.issuer_trading_symbol.upcase
+#       )
+#
+#       form.insider = Insider.where(cik: doc.owner_cik).first_or_initialize
+#       form.insider.update_attributes(
+#         name: doc.owner_name[0, 254]
+#       )
+#
+#       form.day_traded_price = form.company.price_on(form.date)
+#       form.day_traded_volume = form.company.volume_on(form.date)
+#       form.plus_3_months_price = form.company.price_on(form.date + 3.months)
+#       form.plus_6_months_price = form.company.price_on(form.date + 6.months)
+#       form.plus_12_months_price = form.company.price_on(form.date + 12.months)
+#       # need to find a more detailed data source
+#       # form.price_to_earnings = Company.price_to_earnings_on(form.date)
+#       # form.price_to_book = Company.price_to_book_on(form.date)
+#
+#       form.doc = doc
+#       form.save!
+#       form
+#     end
+#   end
+# end

data/lib/sec_edgar/filing_updater.rb

@@ -0,0 +1,52 @@
+module SecEdgar
+  class FilingUpdater
+    attr_reader :form
+
+    def initialize(form)
+      @form = form
+    end
+
+    def doc
+      @doc ||= RawFiling.for_form(form).parsed
+    end
+
+    def update
+      doc_json = doc.to_json
+      return if doc_json == '{}'
+
+      form.update({
+        dollar_volume: doc.dollar_volume,
+        document: { d: doc_json }
+      })
+
+      unless form.company
+        form.company = Company.where(cik: doc.issuer_cik).first_or_initialize
+        form.company.update_attributes(
+          name: doc.issuer_name,
+          ticker: doc.issuer_trading_symbol.upcase
+        )
+      end
+
+      unless form.insider
+        form.insider = Insider.where(cik: doc.owner_cik).first_or_initialize
+        form.insider.update_attributes(
+          name: doc.owner_name[0, 254]
+        )
+      end
+
+      dt = Date.parse(form.date)
+      form.day_traded_price = form.company.price_on(dt)
+      form.day_traded_volume = form.company.volume_on(dt)
+      form.plus_3_months_price = form.company.price_on(dt + 3.months)
+      form.plus_6_months_price = form.company.price_on(dt + 6.months)
+      form.plus_12_months_price = form.company.price_on(dt + 12.months)
+      # need to find a more detailed data source
+      # form.price_to_earnings = Company.price_to_earnings_on(form.date)
+      # form.price_to_book = Company.price_to_book_on(form.date)
+
+      form.doc = doc
+      form.save!
+      form
+    end
+  end
+end

data/lib/sec_edgar/footnote.rb

@@ -0,0 +1,3 @@
+module SecEdgar
+  Footnote = Struct.new(:id, :content)
+end

data/lib/sec_edgar/ftp_client.rb

@@ -0,0 +1,35 @@
+module SecEdgar
+  class FtpClient
+    include Singleton
+
+    def initialize
+      @ftp = Net::FTP.new
+      @ftp.passive = true
+      connect
+      login
+
+      at_exit do
+        puts "Closing Sec4::FtpClient..."
+        @ftp.close
+      end
+    end
+
+    def fetch(remote_url, local_url)
+      @ftp.getbinaryfile(remote_url, local_url)
+    end
+
+    def connect
+      @ftp.connect('ftp.sec.gov', 21)
+    rescue => e
+      puts "Sec4::FtpClient connection failed"
+      puts e.message
+    end
+
+    def login
+      @ftp.login
+    rescue => e
+      puts "Sec4::FtpClient login failed"
+      puts e.message
+    end
+  end
+end

data/lib/sec_edgar/nil_ownership_document.rb

@@ -0,0 +1,39 @@
+module Sec4
+  class NilOwnershipDocument
+    def document_type
+      '4'
+    end
+
+    def issuer_trading_symbol
+      'NULL'
+    end
+
+    def is_director
+      false
+    end
+
+    def is_officer
+      false
+    end
+
+    def is_owner
+      false
+    end
+
+    def is_ten_percent_owner
+      false
+    end
+
+    def officer_title
+      ''
+    end
+
+    def transactions
+      []
+    end
+
+    def derivative_transactions
+      []
+    end
+  end
+end