ecfs 0.2.2 → 0.3.0

data/lib/ecfs.rb

@@ -5,6 +5,9 @@ require "ecfs/filing"
 require "ecfs/proceedings_query"
 require "ecfs/proceeding"
 require "ecfs/document"
+require "ecfs/error"
+require "ecfs/too_many_filings_error"
+require "ecfs/bulk_filings_query"
 
 module ECFS
 end

data/lib/ecfs/bulk_filings_query.rb

@@ -0,0 +1,64 @@
+module ECFS
+  class BulkFilingsQuery
+    attr_accessor :results
+
+    def initialize(docket_number)
+      @docket_number = docket_number
+    end
+
+    def get
+      make_query(@docket_number)
+    end
+
+    def get_midpoint(start_date, end_date)
+      start_date       = start_date.split("/")
+      start_date_month = start_date[0].to_i
+      start_date_day   = start_date[1].to_i
+      start_date_year  = start_date[2].to_i
+      docket_started   = Date.new(start_date_year, start_date_month, start_date_day)
+
+      e = end_date.split("/").map(&:to_i).reverse
+      difference      = (Date.new(e[0], e[2], e[1]) - docket_started).to_i
+      half_difference = difference/2
+      midpoint        = (docket_started + half_difference).to_s.gsub("-", "/")
+
+      midpoint.split("/").reverse.join("/")
+    end
+
+    def make_query(docket_number, posted_after=false, posted_before=false)
+
+      if @results.nil?
+        @results = []
+      end
+
+      query = ECFS::Filing.query.tap do |q|
+        q.docket_number = docket_number
+        q.posted_after  = posted_after  if posted_after
+        q.posted_before = posted_before if posted_before
+      end
+
+      begin
+        @results << query.get
+      rescue ECFS::TooManyFilingsError
+        unless posted_after
+          posted_after = ECFS::Proceeding.find(docket_number)["date_created"] 
+        end
+        unless posted_before
+          today = Date.today.to_s.split("-")
+          posted_before = today[1] + "/" + today[2] + "/" + today[0]
+        end
+        midpoint = get_midpoint(posted_after, posted_before).split("/")
+        formatted_midpoint = midpoint[1] + "/" + midpoint[0] + "/" + midpoint[2]
+
+        make_query(docket_number, formatted_midpoint, posted_before)
+        make_query(docket_number, posted_after, formatted_midpoint)
+      end
+
+      @results.flatten!
+
+      @results
+    end
+
+  end
+
+end

data/lib/ecfs/error.rb

@@ -0,0 +1,4 @@
+module ECFS
+  class Error < StandardError
+  end
+end

data/lib/ecfs/filings_query.rb

@@ -2,6 +2,7 @@ require "pp"
 require "mechanize"
 require "spreadsheet"
 require "ecfs/spreadsheet_parser"
+require "ecfs/too_many_filings_error"
 require "pry"
 
 module ECFS
@@ -31,7 +32,7 @@ module ECFS
         "bureau_id_number"            => "bureauIdentificationNumber",
         "report_number"               => "reportNumber",
         "submission_type_id"          => "submissionTypeId",
-        "exparte"                     => "__checkbox_exParte"
+        "exparte"                     => "__checkbox_exParte",
       }
     end
 
@@ -69,12 +70,30 @@ module ECFS
       end
     end
 
+    def page_contains_ecfs_error_message?(page)
+      xpath = "//*[@id='yui-main']/div/table/tbody/tr[2]/td/span[1]"
+      text = page.search(xpath).text.gsub(/\s+/, " ").strip
+      if text == "Retrieved the 10,000 most recent records. To view older records narrow your search criteria."
+        result = true
+      else
+        result = false
+      end
+
+      result
+    end
+
     def download_spreadsheet!
       agent = mechanize_agent
-      link_text = "\r\n    \t    \t    \tExport to Excel file\r\n    \t        \t"
-      link = agent.get(url).link_with(:text => link_text)
+      page = agent.get(url)
+
+      if page_contains_ecfs_error_message?(page)
+        raise ECFS::TooManyFilingsError.new
+      else
+        link_text = "\r\n    \t    \t    \tExport to Excel file\r\n    \t        \t"
+        link = page.link_with(:text => link_text)
 
-      @rows = agent.click(link).rows
+        @rows = agent.click(link).rows
+      end
     end
   end
 end

data/lib/ecfs/proceedings_query.rb

@@ -12,7 +12,7 @@ module ECFS
         "bureau_code"    => "bureauCode",
         "subject"        => "subject",
         "bureau_id"      => "bureauIdentificationNumber",
-        "applicant"      => "application",
+        "applicant"      => "applicant",
         "filed_by"       => "filedBy",
         "recent_filings" => "__checkbox_recentFilingsRequired",
         "open_status"    => "openStatus",

data/lib/ecfs/too_many_filings_error.rb

@@ -0,0 +1,6 @@
+require "ecfs/error"
+
+module ECFS
+  class TooManyFilingsError < ECFS::Error
+  end
+end

data/lib/ecfs/util.rb

@@ -12,4 +12,5 @@ module ECFS
       "#{chunks[1]}-#{chunks[0]}-#{chunks[2]}"
     end
   end
+
 end

data/lib/ecfs/version.rb

@@ -1,3 +1,3 @@
 module ECFS
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/test/helper.rb

@@ -2,10 +2,17 @@ require 'rubygems'
 require 'bundler'
 require 'vcr'
 
+require 'minitest/unit'
+require 'minitest/autorun'
+require 'minitest/pride'
+
+require 'pry-rescue/minitest'
+
 VCR.configure do |c|
   c.cassette_library_dir = 'fixtures/vcr_cassettes'
   c.hook_into :webmock
   c.allow_http_connections_when_no_cassette = true
+  c.default_cassette_options = { :record => :new_episodes}
 end
 
 begin

data/test/test_filing.rb

@@ -1,10 +1,10 @@
 require "helper"
 require "pp"
 
-class TestFiling < Test::Unit::TestCase
+class TestFiling < MiniTest::Unit::TestCase
 
   def test_search
-    VCR.use_cassette('test_filings_query_test_get') do
+    VCR.use_cassette('main_cassette') do
       filings = ECFS::Filing.query.tap do |q|
         q.docket_number = "12-375" 
       end.get
@@ -29,24 +29,22 @@ class TestFiling < Test::Unit::TestCase
 
       assert_equal nil, filing.documents
 
-      VCR.use_cassette('test_document') do
-        filing.fetch_documents!
-        documents = filing.documents
-        document = documents.first
-
-        assert_equal Array, documents.class
-        assert_equal ECFS::Document, document.class
-        assert_equal String, document.full_text.class
-        assert_equal Array, document.pages.class
-        page = document.pages.first
-        assert_equal ECFS::Document::Page, page.class
-        assert_equal String, page.text.class
-        assert_equal Fixnum, page.page_number.class
-
-        #VCR.use_cassette('test_proceedings_query_test_get_proceeding_info') do
-        #  binding.pry
-        #end
-      end
+      filing.fetch_documents!
+      documents = filing.documents
+      document = documents.first
+
+      assert_equal Array, documents.class
+      assert_equal ECFS::Document, document.class
+      assert_equal String, document.full_text.class
+      assert_equal Array, document.pages.class
+      page = document.pages.first
+      assert_equal ECFS::Document::Page, page.class
+      assert_equal String, page.text.class
+      assert_equal Fixnum, page.page_number.class
+
+      #VCR.use_cassette('test_proceedings_query_test_get_proceeding_info') do
+      #  binding.pry
+      #end
     end
   end
 

data/test/test_filings_query.rb

@@ -1,7 +1,7 @@
 require "helper"
 require "pp"
 
-class TestFilingsQuery < Test::Unit::TestCase
+class TestFilingsQuery < MiniTest::Unit::TestCase
   
   def test_add_constraint
     filings_query = ECFS::FilingsQuery.new
@@ -31,7 +31,7 @@ class TestFilingsQuery < Test::Unit::TestCase
   end
 
   def test_get
-    VCR.use_cassette('test_filings_query_test_get') do
+    VCR.use_cassette('main_cassette') do
       filings_query = ECFS::FilingsQuery.new
       filings_query.eq("docket_number", "12-375")
       rows = filings_query.get

data/test/test_large_proceeding.rb

@@ -0,0 +1,43 @@
+require "helper"
+require "pp"
+
+class TestLargeProceeding < MiniTest::Unit::TestCase
+
+  attr_accessor :results
+
+  def test_throws_error
+    VCR.use_cassette('bulk_cassette') do
+      docket_number = "09-191"
+      query = ECFS::Filing.query.tap do |q|
+        q.docket_number = docket_number
+      end
+      assert_raises ECFS::TooManyFilingsError do
+        results = query.get
+      end
+    end
+  end
+
+  def test_bulk_filings_query
+    VCR.use_cassette('bulk_cassette') do
+      docket_number = "11-109"
+      query = ECFS::BulkFilingsQuery.new(docket_number)
+      filings = query.get
+      proceeding = ECFS::Proceeding.find(docket_number)
+
+      assert_equal proceeding["total_filings"].to_i, filings.length
+    end
+  end
+
+  #def test_bulk_filings_query_xxl
+  #  VCR.use_cassette('bulk_cassette') do
+  #    docket_number = "09-191"
+  #    query = ECFS::BulkFilingsQuery.new(docket_number)
+  #    filings = query.get
+  #    proceeding = ECFS::Proceeding.find(docket_number)
+
+  #    assert_equal proceeding["total_filings"].to_i, filings.length
+  #  end
+  #end
+
+
+end

data/test/test_proceeding.rb

@@ -1,10 +1,10 @@
 require "helper"
 require "pp"
 
-class TestProceeding < Test::Unit::TestCase
+class TestProceeding < MiniTest::Unit::TestCase
   
   def test_find
-    VCR.use_cassette('test_proceedings_query_test_get_proceeding_info') do
+    VCR.use_cassette('main_cassette') do
       proceeding = ECFS::Proceeding.find("12-375")
       %w[
         bureau_name subject date_created status
@@ -19,7 +19,7 @@ class TestProceeding < Test::Unit::TestCase
   end
 
   def test_search
-    VCR.use_cassette('test_get_proceeding_search_results') do
+    VCR.use_cassette('main_cassette') do
 
       proceedings = ECFS::Proceeding.query.tap do |q|
         q.bureau_code = "WC"
@@ -40,28 +40,23 @@ class TestProceeding < Test::Unit::TestCase
       next_proceedings = proceedings.next
       assert_equal ECFS::Proceeding::ResultSet, next_proceedings.class
 
-      VCR.use_cassette('test_filings_query_test_get') do
-        prison_phones = proceedings["results"].select {|p| p["docket_number"] == "12-375"}.first
-        prison_phones.fetch_filings!
+      prison_phones = proceedings["results"].select {|p| p["docket_number"] == "12-375"}.first
+      prison_phones.fetch_filings!
 
-        assert_equal Array, prison_phones["filings"].class
-      end
-
-      VCR.use_cassette('test_proceedings_query_test_get_proceeding_info') do
-        prison_phones = proceedings["results"].select {|p| p["docket_number"] == "12-375"}.first
-        assert_equal ECFS::Proceeding, prison_phones.class
-        fetched = prison_phones.fetch_info!
-        assert_equal ECFS::Proceeding, prison_phones.class
-        %w[
-          bureau_name subject date_created status
-          total_filings filings_in_last_30_days
-        ].each do |key|
-          assert prison_phones.keys.include?(key)
-        end
+      assert_equal Array, prison_phones["filings"].class
 
+      prison_phones = proceedings["results"].select {|p| p["docket_number"] == "12-375"}.first
+      assert_equal ECFS::Proceeding, prison_phones.class
+      fetched = prison_phones.fetch_info!
+      assert_equal ECFS::Proceeding, prison_phones.class
+      %w[
+        bureau_name subject date_created status
+        total_filings filings_in_last_30_days
+      ].each do |key|
+        assert prison_phones.keys.include?(key)
       end
+
     end
   end
 
-
 end

data/test/test_proceedings_query.rb

@@ -1,7 +1,7 @@
 require "helper"
 require "pp"
 
-class TestProceedingsQuery < Test::Unit::TestCase
+class TestProceedingsQuery < MiniTest::Unit::TestCase
 
   def test_add_constraint
     proceedings_query = ECFS::ProceedingsQuery.new
@@ -31,7 +31,7 @@ class TestProceedingsQuery < Test::Unit::TestCase
   end
 
   def test_get_proceeding_info
-    VCR.use_cassette('test_proceedings_query_test_get_proceeding_info') do
+    VCR.use_cassette('main_cassette') do
       proceedings_query = ECFS::ProceedingsQuery.new
       proceedings_query.eq("docket_number", "12-375")
       results = proceedings_query.get
@@ -45,7 +45,7 @@ class TestProceedingsQuery < Test::Unit::TestCase
   end
 
   def test_search_proceedings
-    VCR.use_cassette('test_get_proceeding_search_results') do
+    VCR.use_cassette('main_cassette') do
       proceedings_query = ECFS::ProceedingsQuery.new
       proceedings_query.eq("bureau_code", "WC")
       proceedings_query.eq("page_number", "1")

metadata

@@ -1,68 +1,60 @@
 --- !ruby/object:Gem::Specification
 name: ecfs
 version: !ruby/object:Gem::Version
-  version: 0.2.2
-  prerelease: 
+  version: 0.3.0
 platform: ruby
 authors:
 - Alan deLevie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-01 00:00:00.000000000 Z
+date: 2014-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -70,73 +62,92 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.9.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+- !ruby/object:Gem::Dependency
+  name: pry-rescue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: ECFS provides a set of utilities for scraping FCC rulemakings
@@ -146,68 +157,68 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - ecfs.gemspec
+- fixtures/vcr_cassettes/bulk_cassette.yml
+- fixtures/vcr_cassettes/main_cassette.yml
 - fixtures/vcr_cassettes/test_document.yml
 - fixtures/vcr_cassettes/test_filings_query_test_get.yml
 - fixtures/vcr_cassettes/test_get_proceeding_search_results.yml
 - fixtures/vcr_cassettes/test_get_proceeding_search_results_test_proceedings_query_test_get_proceeding_info.yml
 - fixtures/vcr_cassettes/test_proceedings_query_test_get_proceeding_info.yml
 - lib/ecfs.rb
+- lib/ecfs/bulk_filings_query.rb
 - lib/ecfs/document.rb
+- lib/ecfs/error.rb
 - lib/ecfs/filing.rb
 - lib/ecfs/filings_query.rb
 - lib/ecfs/proceeding.rb
 - lib/ecfs/proceedings_query.rb
 - lib/ecfs/query.rb
 - lib/ecfs/spreadsheet_parser.rb
+- lib/ecfs/too_many_filings_error.rb
 - lib/ecfs/util.rb
 - lib/ecfs/version.rb
 - test/helper.rb
 - test/test_filing.rb
 - test/test_filings_query.rb
+- test/test_large_proceeding.rb
 - test/test_proceeding.rb
 - test/test_proceedings_query.rb
 homepage: http://github.com/adelevie/ecfs
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -550046902955849839
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -550046902955849839
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: ECFS helps you obtain comments and other filings from the FCC's Electronic
   Comment Filing System
 test_files:
 - test/helper.rb
 - test/test_filing.rb
 - test/test_filings_query.rb
+- test/test_large_proceeding.rb
 - test/test_proceeding.rb
 - test/test_proceedings_query.rb