sec_query 1.1.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 10cc642633f4e01fb1bebd0d1f611766e3f93a69
-  data.tar.gz: 2f50d5d488d8504299e2fd01f852e76b784aec2a
+SHA256:
+  metadata.gz: e05c826068c7e3b70b62043aa111e83bd01118130116a9db4d6e7991ea923231
+  data.tar.gz: 38465104422bc0096f3c4a432f0b6801da6282e87c8f7e5e0e6468b7606f5d30
 SHA512:
-  metadata.gz: 4f5846cda290b9c3f2182460842e9263c1a2d60bbb5ae07d2abce512ce15d6f47220acaac721777af46b38b487115708db9652d4e911b618a965bb1baaea9fa6
-  data.tar.gz: 0053b07879bf113fb79127bcac37decc9b6a8569f12404597e81827940fe9923110515e31f074f2ef61056560e350c6e4195016746ebe4d6e34e5bbb47c634af
+  metadata.gz: 538a894e0910434e9ed11c604485342919d6d9de42da73b3d86b2da57e1cd7b7b4b04bdeb51f405e5c1d5b7ff0e01eed6ae8e9eaa919c73ccc7af98af102a44f
+  data.tar.gz: 46ea472a4334afca91affa041ba5ee2bbc5879b9009c7ab2d27e05e2f884c056e77146e9ca88dbb9785c8774f4662b6c178e6995986f2a22a898307d754ef8ef

data/.github/workflows/ruby.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -4,6 +4,8 @@
 .bundle
 .config
 .yardoc
+.ruby-gemset
+.ruby-version
 Gemfile.lock
 InstalledFiles
 _yardoc

data/Gemfile

@@ -1,4 +1,4 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in sec_query.gemspec
 gemspec

data/README.md

@@ -6,6 +6,10 @@ Look-up an Entity - person or company - by Central Index Key (CIK), stock symbol
 
 Additionally retrieve some, or all, Relationships, Transactions and Filings as recorded by the SEC.
 
+## Note: 9/13/16, SEC.GOV embraces SSL!
+
+On or before Septmember 13th, 2016, the SEC.gov updated their site to use SSL (Huzzah!). Version 1.2.0 addresses this change. All versions less than 1.2.0, will cease to function as expected. Update immediately.
+
 ## Installation
 
 To install the 'sec_query' Ruby Gem run the following command at the terminal prompt.
@@ -24,9 +28,29 @@ If running 'sec_query' from the command prompt in irb:
 
 ## Functionality
 
-### FIND COMPANY:
+### Entity:
+
+An Sec::Entity instance contains the following attributes:
+
+* cik
+* name
+* mailing_address
+* business_adddress
+* assigned_sic
+* assigned_sic_desc
+* assigned_sic_href
+* assitant_director
+* cik_href
+* formerly_name
+* state_location
+* state_location_href
+* state_of_incorporation
+
+#### Class Methods
+
+##### .find 
 
-#### By Stock Symbol:
+###### By Stock Symbol:
 
 `SecQuery::Entity.find("aapl")`
 
@@ -34,7 +58,7 @@ Or:
 
 `SecQuery::Entity.find({:symbol=> "aapl"})`
 
-#### By Name:
+###### By Name:
 
 `SecQuery::Entity.find("Apple, Inc.")`
 
@@ -42,7 +66,7 @@ Or:
 
 `SecQuery::Entity.find({:name=> "Apple, Inc."})`
 
-#### By Central Index Key, CIK:
+######  Central Index Key, CIK
 
 `SecQuery::Entity.find( "0000320193")`
 
@@ -50,7 +74,7 @@ Or:
 
 `SecQuery::Entity.find({:cik=> "0000320193"})`
 
-#### FIND PERSON:
+###### By First, Middle and Last Name:
 
 By First, Middle and Last Name.
 
@@ -58,61 +82,59 @@ By First, Middle and Last Name.
 
 Middle initial or name is optional, but helps when there are multiple results for First and Last Name.
 
-### RELATIONSHIPS, TRANSACTIONS, FILINGS
-
-To return everything - All Relationships, Transactions and Filings - that the SEC Edgar system has stored on a company or person, do any of the following commands (They all do the same thing.):
-
-`SecQuery::Entity.find("AAPL",  true)`
-
-`SecQuery::Entity.find("AAPL",  true, true, true)`
-
-`SecQuery::Entity.find("AAPL", {:relationships=> true, :transactions=> true, :filings=>true})`
-
-`SecQuery::Entity.find("AAPL", :relationships=> true, :transactions=> true, :filings=>true)`
+#### Instance Methods
 
-`SecQuery::Entity.find("AAPL", :relationships=> true, :transactions=> {:start=> 0, :count=> 80}, :filings=>{:start=> 0, :count=> 80})`
+##### .filings
 
-You may also limit either the transactions or filings by adding the :limit to the transaction or filing arguements.
+Returns a list of Sec::Filing instances for an Sec::Entity
 
-For example,
+### SecQuery::Filing
 
-`SecQuery::Entity.find("AAPL", :relationships=> true, :transactions=> {:start=> 0, :count=>20, :limit=> 20}, :filings=>{:start=> 0, :count=> 20, :limit=> 20})`
+SecQuery::Filing instance may contains the following attributes:
 
-The above query will only return the last 20 transactions and filings.  This is helpful when querying companies that may have thousands or tens of thousands of transactions or filings.
+* cik
+* title
+* symmary
+* link
+* term
+* date
+* file_id
+* detail
 
-## Classes
+#### Class Methods
 
-This gem contains four classes - Entity, Relationship, Transaction and Filing.  Each Class contains the listed fields. (Everything I could parse out of the query results.)
+##### .recent
 
-* Entity
+Find recent filings:
 
-`:first, :middle, :last, :name, :symbol, :cik, :url, :type, :sic, :location, :state_of_inc, :formerly, :mailing_address, :business_address, :relationships, :transactions, :filings`
-
-* Relationship
-
-`:name, :position, :date, :cik`
-
-* Transaction
-
-`:filing_number, :code, :date, :reporting_owner, :form, :type, :modes, :shares, :price, :owned, :number, :owner_cik, :security_name, :deemed, :exercise, :nature, :derivative, :underlying_1, :exercised,	:underlying_2, :expires, :underlying_3`
-
-* Filing
+```
+filings = []
+SecQuery::Filing.recent(start: 0, count: 10, limit: 10) do |filing|
+  filings.push filing
+end
+```
 
-`:cik, :title, :summary, :link, :term, :date, :file_id`
+Requires a block. Returns the most recent filings. Use start, count and limit to iterate through recent filings.
 
-Filings can are fetched a few different ways. Here are some of the supported
-methods:
+### SecQuery::FilingDetail
+Represents the detail page for a given filing. 
+Ex: [Filing Detail page](https://www.sec.gov/Archives/edgar/data/320193/000032019317000070/0000320193-17-000070-index.htm) of Apple's Annual Report from 2017
 
-```rb
-# prints the links for the most recent filings
-SecQuery::Filing.recent do |filing|
-  p filing.link
-end
+#### Instance Methods
+* link
+* filing_date
+* accepted_date
+* period_of_report
+* sec_access_number
+* document_count
+* format_files
+* data_files
 
-# prints all of the links for cik 0000704051 (LEGG MASON, INC.)
-SecQuery::Filing.for_cik('0000704051') do |filing|
-  p filing.link
-end
+#### Class Methods
+##### .fetch
+```
+appl_10k_details_url = 'https://www.sec.gov/Archives/edgar/data/320193/000032019317000070/0000320193-17-000070-index.htm'
+filing_detail = SecQuery::FilingDetail.fetch(appl_10k_details_url)
 ```
 
 ## To Whom It May Concern at the SEC

data/Rakefile

@@ -1 +1,9 @@
-require 'bundler/gem_tasks'
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task test: :spec
+
+task default: :test
+

data/lib/sec_query.rb

@@ -4,7 +4,6 @@
 require 'active_support/all'
 require 'addressable/uri'
 require 'open-uri'
-require 'net/ftp'
 require 'rest-client'
 require 'rss'
 require 'nokogiri'
@@ -13,7 +12,6 @@ require 'rubygems'
 # internal
 require 'sec_query/entity'
 require 'sec_query/filing'
-require 'sec_query/relationship'
+require 'sec_query/filing_detail'
 require 'sec_query/sec_uri'
-require 'sec_query/transaction'
 require 'sec_query/version'

data/lib/sec_query/entity.rb

@@ -15,12 +15,8 @@ module SecQuery
       end
     end
 
-    def filings
-      Filing.find(@cik)
-    end
-
-    def transactions
-      Transaction.find(@cik)
+    def filings(args={})
+      Filing.find(@cik, 0, 80, args)
     end
 
     def self.query(url)
@@ -53,7 +49,7 @@ module SecQuery
           content['formerly_names'] = content.delete('formerly_names')['names']
         end
         content['addresses']['address'].each do |address|
-          content["#{address['type']}_address"] = address
+          content["#{address['type']}_address"] = address unless address.nil?
         end
         return content
       else

data/lib/sec_query/filing.rb

@@ -14,9 +14,13 @@ module SecQuery
       end
     end
 
+    def detail
+      @detail ||= FilingDetail.fetch(@link)
+    end
+
     def self.fetch(uri, &blk)
-      open(uri) do |rss|
-        parse_rss(rss, &blk)
+      RestClient::Request.execute(method: :get, url: uri.to_s, timeout: 10) do |response, request, result, &block|
+        parse_rss(response.body, &blk)
       end
     end
 
@@ -46,20 +50,10 @@ module SecQuery
     end
 
     def self.for_date(date, &blk)
-      ftp = Net::FTP.new('ftp.sec.gov')
-      ftp.login
-      file_name = ftp.nlst("edgar/daily-index/#{ date.to_sec_uri_format }*")[0]
-      ftp.close
-      open("ftp://ftp.sec.gov/#{ file_name }") do |file|
-        if file_name[-2..-1] == 'gz'
-          gz_reader = Zlib::GzipReader.new(file)
-          gz_reader.rewind
-          filings_for_index(gz_reader).each(&blk)
-        else
-          filings_for_index(file).each(&blk)
-        end
+      url = SecURI.for_date(date).to_s
+      RestClient::Request.execute(method: :get, url: url, timeout: 10) do |response, request, result, &block|
+        filings_for_index(response.body).each(&blk)
       end
-    rescue Net::FTPTempError
     end
 
     def self.filings_for_index(index)
@@ -82,7 +76,7 @@ module SecQuery
       return nil unless Regexp.new(/\d{8}/).match(data[3])
       return nil if data[4] == nil
       unless data[4][0..3] == 'http'
-        data[4] = "http://www.sec.gov/Archives/#{ data[4] }"
+        data[4] = "https://www.sec.gov/Archives/#{ data[4] }"
       end
       begin
         Date.parse(data[3])
@@ -134,22 +128,29 @@ module SecQuery
       end
     end
 
-    def self.find(cik, start = 0, count = 80)
+    def self.find(cik, start = 0, count = 80, args={})
       temp = {}
       temp[:url] = SecURI.browse_edgar_uri({cik: cik})
       temp[:url][:action] = :getcompany
       temp[:url][:start] = start
       temp[:url][:count] = count
+      args.each {|k,v| temp[:url][k]=v }
+      
       response = Entity.query(temp[:url].output_atom.to_s)
       document = Nokogiri::HTML(response)
       parse(cik, document)
     end
 
+    def self.last(cik, args = {})
+      filings = find(cik, 0, 1, args)
+      filings.is_a?(Array) ? filings.first : nil
+    end
+
     def self.parse(cik, document)
       filings = []
       if document.xpath('//content').to_s.length > 0
         document.xpath('//content').each do |e|
-          if e.xpath('//content/accession-nunber').to_s.length > 0
+          if e.xpath('//content/accession-number').to_s.length > 0
             content = Hash.from_xml(e.to_s)['content']
             content[:cik] = cik
             content[:file_id] = content.delete('accession_nunber')

data/lib/sec_query/filing_detail.rb

@@ -0,0 +1,68 @@
+# encoding: UTF-8
+
+module SecQuery
+  # => SecQuery::FilingDetail
+  # SecQuery::FilingDetail requests and parses Filing Detail for any given SecQuery::Filing
+  class FilingDetail
+    COLUMNS = [:link, :filing_date, :accepted_date, :period_of_report, :sec_access_number, :document_count, :format_files, :data_files]
+
+    attr_accessor(*COLUMNS)
+
+    def initialize(filing_detail)
+      COLUMNS.each do |column|
+        instance_variable_set("@#{ column }", filing_detail[column])
+      end
+    end
+
+    def self.fetch(uri)
+      response = RestClient::Request.execute(method: :get, url: uri.to_s.gsub('http:', 'https:'), timeout: 10)
+      document = Nokogiri::HTML(response.body)
+      filing_date = document.xpath('//*[@id="formDiv"]/div[2]/div[1]/div[2]').text
+      accepted_date = document.xpath('//*[@id="formDiv"]/div[2]/div[1]/div[4]').text
+      period_of_report = document.xpath('//*[@id="formDiv"]/div[2]/div[2]/div[2]').text
+      sec_access_number = document.xpath('//*[@id="secNum"]/text()').text.strip
+      document_count = document.xpath('//*[@id="formDiv"]/div[2]/div[1]/div[6]').text.to_i
+      format_files_table = document.xpath("//table[@summary='Document Format Files']")
+      data_files_table = document.xpath("//table[@summary='Data Files']")
+
+      format_files = (parsed = parse_files(format_files_table)) && (parsed || [])
+      data_files = (parsed = parse_files(data_files_table)) && (parsed || [])
+
+      new({uri: uri,
+           filing_date: filing_date,
+           accepted_date: accepted_date,
+           period_of_report: period_of_report,
+           sec_access_number: sec_access_number,
+           document_count: document_count,
+           format_files: format_files,
+           data_files: data_files})
+    end
+
+    def self.parse_files(format_files_table)
+      # get table headers
+      headers = []
+      format_files_table.xpath('//th').each do |th|
+        headers << th.text
+      end
+
+      # get table rows
+      rows = []
+      format_files_table.xpath('//tr').each_with_index do |row, i|
+        rows[i] = {}
+        row.xpath('td').each_with_index do |td, j|
+          if td.children.first && td.children.first.name == 'a'
+            relative_url = td.children.first.attributes.first[1].value
+            rows[i][headers[j]] = {
+                'link' => "https://www.sec.gov#{relative_url}",
+                'text' => td.text.gsub(/\A\p{Space}*/, '')
+            }
+          else
+            rows[i][headers[j]] = td.text.gsub(/\A\p{Space}*/, '')
+          end
+        end
+      end
+
+      rows.reject(&:empty?)
+    end
+  end
+end

data/lib/sec_query/sec_uri.rb

@@ -25,9 +25,9 @@ module SecQuery
 
     def self.for_date(date)
       instance = SecURI.new
-      instance.scheme = 'ftp'
-      instance.host = 'ftp.sec.gov'
-      instance.path = "edgar/daily-index/#{ date.to_sec_uri_format }"
+      instance.host = 'www.sec.gov'
+      instance.scheme = 'https'
+      instance.path = "Archives/edgar/daily-index/#{ date.to_sec_uri_format }"
       instance
     end
 
@@ -49,7 +49,7 @@ module SecQuery
       begin Float(string_arg)
         options[:CIK] = string_arg
       rescue
-        if string_arg.length <= 4
+        if string_arg.length <= 5
           options[:CIK] = string_arg
         else
           options[:company] = string_arg.gsub(/[(,?!\''"":.)]/, '')
@@ -81,7 +81,7 @@ module SecQuery
 
     def initialize
       self.host = 'www.sec.gov'
-      self.scheme = 'http'
+      self.scheme = 'https'
       self.path = 'cgi-bin'
     end