ecfs 0.3.3 → 0.3.4

data/lib/ecfs.rb

@@ -9,6 +9,7 @@ require "ecfs/error"
 require "ecfs/too_many_filings_error"
 require "ecfs/bulk_filings_query"
 require "ecfs/daily_releases_query"
+require "ecfs/solr_scrape_query"
 
 module ECFS
 end

data/lib/ecfs/solr_scrape_query.rb

@@ -0,0 +1,73 @@
+require "pp"
+require "pry"
+require "mechanize"
+
+module ECFS
+
+  class SolrScrapeQuery
+    attr_accessor :docket_number
+
+    def initialize
+    end
+    
+    def filings_from_docket_number(docket_number, start=0)
+      url = "http://apps.fcc.gov/ecfs/solr/search?sort=dateRcpt&proceeding=#{docket_number}&dir=asc&start=#{start}"
+      
+      agent = Mechanize.new
+      page = agent.get(url)
+      
+      total = page.search('div').find {|div| div.text.start_with?("Showing results")}.text.split('of ')[1].to_i
+      table = page.search('div.dataTable table').first
+      
+      rows = table.search('tr')
+      rows.shift
+      
+      filings = rows.map do |row|
+        columns = row.search('td')
+      
+        proceeding     = columns[0].text.strip
+        name_of_filer  = columns[1].text.strip
+        date_recieved  = columns[2].text.strip
+        type_of_filing = columns[3].text.strip
+        pages          = columns[4].text.strip.to_i
+              
+        id = columns[1].search('a').first.attributes['href'].value.split('?id=')[1]
+        url = "http://apps.fcc.gov/ecfs/comment/view?id=#{id}"
+        
+        {
+          'docket_number' => proceeding,
+          'name_of_filer' => name_of_filer,
+          'type_of_filing' => type_of_filing,
+          'url' => url,
+          'date_recieved' => date_recieved,
+          'pages' => pages
+        }
+      end
+      
+      return filings, total
+    end
+    
+    def get(fetch_document_urls=false)
+      url = "http://apps.fcc.gov/ecfs/solr/search?sort=dateRcpt&proceeding=#{@docket_number}&dir=asc&start=0"
+      filings = []
+      
+      first_page_of_filings, total = filings_from_docket_number(@docket_number, 0)
+      
+      pages = (total.to_f/20.0).ceil.to_i.times.map {|n| n*20} # divide, round up, then map *20
+      pages.shift
+      
+      filings.concat first_page_of_filings
+      
+      pages.each do |page|
+        filings.concat filings_from_docket_number(@docket_number, page)[0]
+      end
+      
+      if fetch_document_urls
+        p "pretending to fetch some urls"
+      end
+
+      filings
+    end
+
+  end
+end

data/lib/ecfs/version.rb

@@ -1,3 +1,3 @@
 module ECFS
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

data/test/test_solr_scrape.rb

@@ -0,0 +1,18 @@
+require "helper"
+require "pp"
+require "pry"
+
+class TestSolrScrape < MiniTest::Unit::TestCase
+
+  def test_synopsis
+    VCR.use_cassette('main_cassette') do
+      filings = ECFS::SolrScrapeQuery.new.tap do |q|
+        q.docket_number = '12-83'
+      end.get
+      
+      assert filings.first.is_a?(Hash)
+      assert filings.first.has_key?('docket_number')
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ecfs
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Alan deLevie
@@ -196,6 +196,7 @@ files:
 - lib/ecfs/proceeding.rb
 - lib/ecfs/proceedings_query.rb
 - lib/ecfs/query.rb
+- lib/ecfs/solr_scrape_query.rb
 - lib/ecfs/spreadsheet_parser.rb
 - lib/ecfs/too_many_filings_error.rb
 - lib/ecfs/util.rb
@@ -207,6 +208,7 @@ files:
 - test/test_large_proceeding.rb
 - test/test_proceeding.rb
 - test/test_proceedings_query.rb
+- test/test_solr_scrape.rb
 homepage: http://github.com/adelevie/ecfs
 licenses:
 - MIT
@@ -240,3 +242,4 @@ test_files:
 - test/test_large_proceeding.rb
 - test/test_proceeding.rb
 - test/test_proceedings_query.rb
+- test/test_solr_scrape.rb