ecfs 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95767cb5a02a676bcb02c3a28999ad717d37e754
-  data.tar.gz: f7e2085bcd71d228b15cdaaebbb52f6356bc1a0e
+  metadata.gz: fa5da38c0c5348fa0c845d28c0f9c2516cf33bb3
+  data.tar.gz: 30aed783337f289d994d490d26240bb43b6266e4
 SHA512:
-  metadata.gz: 0fda14c85e1e0376cd8046a67441e6d72d0b02a2d2f1abbba6548deb8fa0d972657613449b6a11aa1b59a1a0ee49ae842de0d2c26dd9f063fd60257b991dd52a
-  data.tar.gz: 7b45427927b845289db6b7b17d159c39c5203c60c9bdaa4d1aa553418ef92607b548b8c7e462666c83d42747cf42db175e443fc6185a575a48fb10624858dc49
+  metadata.gz: e91e0878401399171a6f346d72049a3d3c71975d2a6ca1671e7775a8f49e5a7bfac62e84765a7ec27b500e95599c2a8df77e684fe3e4885ee2e0a1dc9446432e
+  data.tar.gz: 0fd4b2971174f2c3317016d85282a056b046ea3f06702349b92de6b5f50804ddf35591aa0e41f10f70e260fbfe28d009e2e2147cf3e8fd1eaa73b8ab960d4d41

data/README.md

@@ -256,6 +256,23 @@ p filings.first
 }
 ```
 
+Options:
+
+```ruby
+filings = ECFS::SolrScrapeQuery.new.tap do |q|
+  q.docket_number = '12-83'
+  
+  # a minimum date, inclusive. mm/dd/yyyy
+  q.received_min_date = '03/30/2012'
+  
+  # an after_scrape block
+  q.after_scrape = Proc.new do |filings|
+    p "Fetched asynchronyously: #{filings.length}"
+  end
+  # This is handy for large scrapes.
+end.get
+```
+
 ### Daily Releases
 
 This feature parses these types of pages: http://transition.fcc.gov/Daily_Releases/Daily_Business/2014/db0917/.

data/lib/ecfs/solr_scrape_query.rb

@@ -7,8 +7,9 @@ module ECFS
   class SolrScrapeQuery
     attr_accessor :docket_number
     attr_accessor :received_min_date
+    attr_accessor :after_scrape
     
-    def filings_from_docket_number(docket_number, start=0, received_min_date=nil)
+    def filings_from_docket_number(docket_number, start=0, received_min_date=nil, after_scrape=nil)
       url = "http://apps.fcc.gov/ecfs/solr/search?sort=dateRcpt&proceeding=#{docket_number}&dir=asc&start=#{start}"
       
       if received_min_date
@@ -46,6 +47,10 @@ module ECFS
         }
       end
       
+      if after_scrape
+        after_scrape.call(filings)
+      end
+            
       return filings, total
     end
     
@@ -72,7 +77,7 @@ module ECFS
       url = "http://apps.fcc.gov/ecfs/solr/search?sort=dateRcpt&proceeding=#{@docket_number}&dir=asc&start=0"
       filings = []
       
-      first_page_of_filings, total = filings_from_docket_number(@docket_number, 0, @received_min_date)
+      first_page_of_filings, total = filings_from_docket_number(@docket_number, 0, @received_min_date, @after_scrape)
       
       pages = (total.to_f/20.0).ceil.to_i.times.map {|n| n*20} # divide, round up, then map *20
       pages.shift
@@ -80,7 +85,7 @@ module ECFS
       filings.concat first_page_of_filings
       
       pages.each do |page|
-        filings.concat filings_from_docket_number(@docket_number, page, @received_min_date)[0]
+        filings.concat filings_from_docket_number(@docket_number, page, @received_min_date, @after_scrape)[0]
       end
       
       filings.each do |filing|

data/lib/ecfs/version.rb

@@ -1,3 +1,3 @@
 module ECFS
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/test/test_solr_scrape.rb

@@ -33,5 +33,39 @@ class TestSolrScrape < MiniTest::Unit::TestCase
       assert filing_date > min_date
     end
   end
+  
+  class FakeArrayThing
+    def initialize
+      @filings = []
+    end
+    
+    def concat(filings)
+      @filings.concat(filings)
+    end
+    
+    def filings
+      @filings
+    end
+  end
+  
+  def test_after_scrape
+    VCR.use_cassette('solr_cassette') do
+      
+      @fake_array_thing = FakeArrayThing.new
+      
+      filings = ECFS::SolrScrapeQuery.new.tap do |q|
+        q.docket_number = '12-83'
+        q.after_scrape = Proc.new do |filings|
+          @fake_array_thing.concat(filings)
+        end
+      end.get
+      
+      assert filings.first.is_a?(Hash)
+      assert filings.first.has_key?('docket_number')
+      assert filings.first.has_key?('citation')
+      
+      assert_equal filings.length, @fake_array_thing.filings.length
+    end
+  end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ecfs
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Alan deLevie