ecfs 0.2.0 → 0.2.1

data/lib/ecfs/filings_query.rb

@@ -54,14 +54,8 @@ module ECFS
     end
 
     def get
-      rows = download_spreadsheet.rows
-      if @typecast_results
-        return rows.map do |row|
-          row_to_filing(row)
-        end
-      else
-        return rows
-      end
+      download_spreadsheet!
+      @typecast_results ? @rows.map {|row| row_to_filing(row)} : @rows
     end
 
     def row_to_filing(row)
@@ -69,20 +63,18 @@ module ECFS
     end
 
     def mechanize_agent
-      agent = Mechanize.new
-      agent.follow_meta_refresh = true
-      agent.pluggable_parser["application/vnd.ms-excel"] = ECFS::SpreadsheetParser
-
-      agent
+      Mechanize.new.tap do |agent|
+        agent.follow_meta_refresh = true
+        agent.pluggable_parser["application/vnd.ms-excel"] = ECFS::SpreadsheetParser
+      end
     end
 
-    def download_spreadsheet
-      agent = self.mechanize_agent
-      page = agent.get(self.url)
+    def download_spreadsheet!
+      agent = mechanize_agent
       link_text = "\r\n    \t    \t    \tExport to Excel file\r\n    \t        \t"
-      link = page.link_with(:text => link_text)
-      
-      agent.click(link)
+      link = agent.get(url).link_with(:text => link_text)
+
+      @rows = agent.click(link).rows
     end
   end
 end

data/lib/ecfs/proceeding.rb

@@ -16,7 +16,7 @@ module ECFS
     def self.find(docket_number)
       query.tap do |q|
         q.eq("docket_number", docket_number)
-      end.get
+      end.get.merge!({"docket_number" => docket_number})
     end
 
     def fetch_info!

data/lib/ecfs/proceedings_query.rb

@@ -51,8 +51,8 @@ module ECFS
         # if docket_number is given along with other constraints, the other constraints will be ignored.
         warn "Constraints other than `docket_number` will be ignored." if @constraints.keys.length > 1
         
-        return scrape_proceedings_page unless @typecast_results
-        results = ECFS::Proceeding.new(scrape_proceedings_page)
+        return scrape_proceeding_page! unless @typecast_results
+        results = ECFS::Proceeding.new(scrape_proceeding_page!)
       else
         return scrape_results_page unless @typecast_results
         results = ECFS::Proceeding::ResultSet.new(scrape_results_page)
@@ -61,85 +61,136 @@ module ECFS
       results
     end
 
-    def mechanize_page
-      Mechanize.new.get(self.url)
-    end
 
     private
 
-    def scrape_proceedings_page
-      page = self.mechanize_page
-
-      container = []
-      page.search("div").select do |d| 
-        d.attributes["class"].nil? == false
-      end.select do |d|
-        d.attributes["class"].text == "wwgrp"
-      end.each do |node|
-        node.search("span").each do |span|
-          search = span.search("label")
-          pair = []
-          if search.length > 0
-            key = search.first.children.first.text.lstrip.rstrip.split(":")[0].gsub(" ", "_").downcase
-            pair << key
-          else
-            value = span.text.lstrip.rstrip
-            value.gsub!(",", "") if value.is_a?(String)
-            pair << value
-          end
-          container << pair
+    def mechanize_page
+      Mechanize.new.get(url)
+    end
+
+    def scrape_proceeding_page!
+      container_to_hash do
+        mechanize_page.search("div").select do |div| 
+          div.attributes["class"].nil? == false
+        end.select do |div|
+          div.attributes["class"].text == "wwgrp"
+        end.map do |node|
+          search_node(node)
         end
       end
+    end
+
+    def container_to_hash(&block)
       hash = {}
-      container.each_slice(2) do |chunk|
-        hash.merge!({chunk[0][0] => chunk[1][0]})
+      block.call.flatten!.each_slice(2) do |chunk|
+        hash[chunk[0]] = chunk[1]
       end
 
-      hash["date_created"] = format_date(hash["date_created"])
-
       hash
     end
 
-    def scrape_results_page
-      page = self.mechanize_page
+    def search_node(node)
+      node.search("span").map do |span|
+        search = span.search("label")
+        key_or_value_from_search_and_span(search, span)
+      end
+    end
+
+    def key_or_value_from_search_and_span(search, span)
+      search.length > 0 ? key_from_search(search) : value_from_span(span)
+    end
+
+    def key_from_search(search)
+      format_key_text(search.first.children.first.text)
+    end
+
+    def format_key_text(key_text)
+      key_text.lstrip!.rstrip!
+      key_text = key_text.split(":")[0]
+      key_text.gsub!(" ", "_")
+      key_text.downcase!
+    end
+
+    def value_from_span(span)
+      value = text_from_span(span)
+      value.gsub!(",", "") if value.is_a?(String)
+
+      value
+    end
+
+    def text_from_span(span)
+      span.text.lstrip.rstrip
+    end
 
-      total_pages = page.link_with(:text => "Last").attributes.first[1].split("pageNumber=")[1].gsub(",","").to_i
-      banner      = page.search("//*[@id='yui-main']/div/div[2]/table/tbody/tr[2]/td/span[1]").text.lstrip.rstrip.split("Modify Search")[0].rstrip.split  
-      first       = banner[1].gsub(",","").to_i
-      last        = banner[3].gsub(",","").to_i
-      total       = banner[5].gsub(",","").to_i
-      table_rows  = page.search("//*[@id='yui-main']/div/div[2]/table/tbody/tr[2]/td/table/tbody").children
-      results     = table_rows.map { |row| row_to_proceeding(row) }
+    def scrape_results_page
+      page   = mechanize_page
+      banner = extract_banner_from_page(page)
 
       {
         "constraints"   => @constraints,
-        "fcc_url"       => self.url,
-        "current_page"  => self.constraints["page_number"].gsub(",","").to_i,
-        "total_pages"   => total_pages,
-        "first_result"  => first,
-        "last_result"   => last,
-        "total_results" => total,
-        "results"       => results
+        "fcc_url"       => url,
+        "current_page"  => current_page,
+        "total_pages"   => total_pages_from_page(page),
+        "first_result"  => first_from_banner(banner),
+        "last_result"   => last_from_banner(banner),
+        "total_results" => total_from_banner(banner),
+        "results"       => proceedings_from_page(page)
       }
     end
 
-    def row_to_proceeding(row)
-      hash = row_to_hash(row)
+    def current_page
+      self.constraints["page_number"].gsub(",","").to_i
+    end
 
-      ECFS::Proceeding.new(hash)
+    def proceedings_from_page(page)
+      extract_table_rows_from_page(page).map do |row|
+        row_to_proceeding(row)
+      end
     end
 
-    def row_to_hash(row)
-      bureau                  = bureau_from_row(row)
-      subject                 = subject_from_row(row)
-      docket_number           = docket_number_from_row(row)
-      filings_in_last_30_days = filings_in_last_30_days_from_row(row)
+    def extract_table_rows_from_page(page)
+      xpath = "//*[@id='yui-main']/div/div[2]/table/tbody/tr[2]/td/table/tbody"
+      page.search(xpath).children
+    end
+
+    def first_from_banner(banner)
+      extract_from_banner(banner, 1)
+    end
+
+    def last_from_banner(banner)
+      extract_from_banner(banner, 3)
+    end
+
+    def total_from_banner(banner)
+      extract_from_banner(banner, 5)
+    end
+
+    def extract_banner_from_page(page)
+      xpath = "//*[@id='yui-main']/div/div[2]/table/tbody/tr[2]/td/span[1]"
+      page.search(xpath).text.tap do |t|
+        t.lstrip!
+        t.rstrip!
+      end.split("Modify Search")[0].rstrip.split  
+    end
 
+    def extract_from_banner(banner, index)
+      banner[index].gsub(",", "").to_i
+    end
+
+    def total_pages_from_page(page)
+      page.link_with(:text => "Last").attributes.first[1].split("pageNumber=")[1].gsub(",","").to_i
+    end
+
+    def row_to_proceeding(row)
+      ECFS::Proceeding.new(row_to_hash(row))
+    end
+
+    def row_to_hash(row)
       {
-        "docket_number"           => docket_number,
-        "bureau"                  => bureau,
-        "subject"                 => subject,
-        "filings_in_last_30_days" => filings_in_last_30_days
+        "docket_number"           => docket_number_from_row(row),
+        "bureau"                  => bureau_from_row(row),
+        "subject"                 => subject_from_row(row),
+        "filings_in_last_30_days" => filings_in_last_30_days_from_row(row)
       }
     end
 
@@ -158,6 +209,7 @@ module ECFS
     def filings_in_last_30_days_from_row(row)
       row.children[6].children.first.text.lstrip.rstrip.to_i
     end
+    #####
 
   end
 end

data/lib/ecfs/spreadsheet_parser.rb

@@ -11,62 +11,78 @@ module ECFS
     def initialize(uri = nil, response = nil, body = nil, code = nil)
       super(uri, response, body, code)
       @body = body
-      extract_rows!
-      format_rows!
+      @rows = formatted_rows
     end
 
     private
 
-    def extract_rows!
-      book = Spreadsheet.open(StringIO.new(@body))
-      sheet1 = book.worksheet 0
-      @rows = []
+    def file
+      StringIO.new(@body)
+    end
+
+    def book
+      Spreadsheet.open(file)
+    end
+
+    def sheet
+      book.worksheet(0)
+    end
+
+    def unformatted_rows
+      my_rows = []
       first = false
-      sheet1.each do |row|
-        @rows << row if first
+      sheet.each do |row|
+        my_rows << row if first
         first = true
       end
 
-      @rows
+      my_rows
     end
 
-    def format_rows!
-      @rows.map! do |row|
-        urls = []  
-        indices = (7..row.length-1).to_a
-        indices.each do |i|
-          text = row[i].data.split("id=")[1]
-          urls << "http://apps.fcc.gov/ecfs/document/view?id=#{extract_filing_id(text)}"
-        end
-
-        {
-          "name_of_filer"  => row[1],
-          "docket_number"  => row[0],
-          "lawfirm_name"   => row[2],
-          "date_received"  => format_date(row[3]),
-          "date_posted"    => format_date(row[4]),
-          "exparte"        => format_exparte(row[5]),
-          "type_of_filing" => row[6],
-          "document_urls"  => urls
-        }
+    def formatted_rows
+      unformatted_rows.map do |row|
+        row_to_hash(row)
       end
     end
 
-    def format_exparte(my_bool)
-      return true  if my_bool == "Y"
-      return false if my_bool == "N"
-      return nil
+    def row_to_hash(row)
+      {
+        "name_of_filer"  => row[1],
+        "docket_number"  => row[0],
+        "lawfirm_name"   => row[2],
+        "date_received"  => format_iso_date(row[3]),
+        "date_posted"    => format_iso_date(row[4]),
+        "exparte"        => bool_from_exparte(row[5]),
+        "type_of_filing" => row[6],
+        "document_urls"  => urls_from_row(row)
+      }
+    end
+
+    def urls_from_row(row)
+      indices = (7..row.length-1).to_a
+      
+      indices.map do |index|
+        extract_url_from_row_and_index(row, index)
+      end
+    end
+
+    def extract_url_from_row_and_index(row, index)
+      text = row[index].data.split("id=")[1]
+      
+      "http://apps.fcc.gov/ecfs/document/view?id=#{extract_filing_id(text)}"
     end
 
     def extract_filing_id(txt)
       re1='(\\d+)'
       re=(re1)
       m = Regexp.new(re, Regexp::IGNORECASE)
-      if m.match(txt)
-        int1 = m.match(txt)[1]
-        return int1
-      end
+
+      m.match(txt)[1]
+    end
+
+    def bool_from_exparte(y_or_n)
+      {"Y" => true, "N" => false}[y_or_n]
     end
 
-  end # end class
-end # end module
+  end
+end

data/lib/ecfs/util.rb

@@ -1,10 +1,15 @@
 module ECFS
   module Util
-    def format_date(date)
+    def format_iso_date(date)
       # input format 12/22/1988
       chunks = date.split("/")
       new_date = "#{chunks[2]}-#{chunks[0]}-#{chunks[1]}" # "22-12-1988"
       "#{new_date}T00:00:00.000Z" # dumb hack
     end
+
+    def iso_date_to_simple_date(iso_date)
+      chunks = iso_date.split("T")[0].split("-")
+      "#{chunks[1]}-#{chunks[0]}-#{chunks[2]}"
+    end
   end
 end

data/lib/ecfs/version.rb

@@ -1,3 +1,3 @@
 module ECFS
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/test/test_filing.rb

@@ -42,6 +42,10 @@ class TestFiling < Test::Unit::TestCase
         assert_equal ECFS::Document::Page, page.class
         assert_equal String, page.text.class
         assert_equal Fixnum, page.page_number.class
+
+        #VCR.use_cassette('test_proceedings_query_test_get_proceeding_info') do
+        #  binding.pry
+        #end
       end
     end
   end

data/test/test_proceeding.rb

@@ -9,6 +9,7 @@ class TestProceeding < Test::Unit::TestCase
       %w[
         bureau_name subject date_created status
         total_filings filings_in_last_30_days
+        docket_number
       ].each do |key|
         assert proceeding.keys.include?(key)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ecfs
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-24 00:00:00.000000000 Z
+date: 2013-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler