ecfs 0.5.1 → 0.6.0

data/Rakefile

@@ -1,11 +1 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
-
-require 'rake/testtask'
-
-Rake::TestTask.new do |t|
- t.libs << 'test'
-end
-
-desc "Run tests"
-task :default => :test
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ecfs"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ecfs.gemspec

@@ -8,27 +8,30 @@ Gem::Specification.new do |spec|
   spec.version       = ECFS::VERSION
   spec.authors       = ["Alan deLevie"]
   spec.email         = ["adelevie@gmail.com"]
-  spec.description   = %q{ECFS provides a set of utilities for scraping FCC rulemakings}
-  spec.summary       = %q{ECFS helps you obtain comments and other filings from the FCC's Electronic Comment Filing System}
-  spec.homepage      = "http://github.com/adelevie/ecfs"
+
+  spec.summary       = %q{Scraper for the FCC's Electronic Comment Filing System}
+  spec.description   = %q{Provides Ruby-based access to the FCC's Electronic Comment Filing System}
+  spec.homepage      = "https://github.com/adelevie/ecfs"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "webmock", "1.9.0"
-  spec.add_development_dependency "minitest", "~> 4.7.3"
-  spec.add_development_dependency "pry-rescue"
-  spec.add_development_dependency "m", "~> 1.3.1"
-
+  #if spec.respond_to?(:metadata)
+  #  spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com' to prevent pushes to rubygems.org, or delete to allow pushes to any server."
+  #end
 
-  spec.add_dependency "pdf-reader"
+  spec.add_dependency "nokogiri"
   spec.add_dependency "pry"
-  spec.add_dependency "mechanize"
-  spec.add_dependency "spreadsheet"
+  spec.add_dependency "unirest"
+  spec.add_dependency "rubyzip"
+  spec.add_dependency "open_uri_redirections"
+
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "bundler"#, "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "vcr"
 end

data/lib/ecfs.rb

@@ -1,15 +1,211 @@
-require "ecfs/version"
-require "ecfs/query"
-require "ecfs/filings_query"
-require "ecfs/filing"
-require "ecfs/proceedings_query"
-require "ecfs/proceeding"
-require "ecfs/document"
-require "ecfs/error"
-require "ecfs/too_many_filings_error"
-require "ecfs/bulk_filings_query"
-require "ecfs/daily_releases_query"
-require "ecfs/solr_scrape_query"
+require 'ecfs/version'
+require 'nokogiri'
+require 'open-uri'
+require 'pry'
+require 'net/http'
+require 'uri'
+require 'unirest'
+require 'zip'
+require 'open_uri_redirections'
+require 'fileutils'
 
 module ECFS
-end
+  module Util
+    SIGNALS = [
+      'E.g.', 'Accord', 'See', 'See also', 'Cf.',
+      'Compare', 'Contra', 'But see', 'But cf.',
+      'See generally'
+    ].map {|s| "#{s} Id."} << 'Id.'
+
+    def self.get_footnotes(url: nil, id_tree: false)
+      # hacky 'temp' file
+      rando = (rand * 1000000000000000000).to_i
+      FileUtils.mkdir_p "tmp/#{rando}"
+      path = "tmp/#{rando}/document.doc.zip"
+
+      open(path, 'wb', allow_redirections: :all) do |file|
+        file << open(url, allow_redirections: :all).read
+        `unzip #{path} -d tmp/#{rando}`
+      end
+
+      xml = File.open("tmp/#{rando}/word/footnotes.xml").read
+      doc = Nokogiri::XML(xml)
+
+      footnotes = doc.children[0].children[3..-1]
+
+      my_footnotes = footnotes.to_ary.map do |fn|
+        {
+          index: fn.attributes['id'].value.to_i - 1,
+          text: fn.text.strip
+        }
+      end
+
+      # compute the tree of id. citations
+      if id_tree
+        my_footnotes.each {|fn| fn[:ids] = []}
+        my_footnotes.each {|fn| fn[:id] = false}
+        ids = my_footnotes.select {|fn| fn[:text].start_with?(*ECFS::Util::SIGNALS)}
+        ids.each {|id| id[:id] = true}
+
+        my_footnotes.each do |fn|
+          if fn[:id] == true
+            parent_idx = fn[:index]-1
+            my_footnotes.find {|fn| fn[:index] == parent_idx}[:ids] << fn
+          end
+        end
+
+        my_footnotes = send_ids_to_parent(my_footnotes)
+
+      end
+
+      FileUtils.rm_rf("tmp/#{rando}")
+
+      my_footnotes
+    end
+
+    private
+
+    # if a footnote is an id and has ids, we send its ids to its parent
+    # these footnotes are reflected as parents, but are actually siblings
+    # so we call these ptsbs (parents that should be siblings). <3 software.
+    def self.send_ids_to_parent(footnotes)
+      ptsbs_array = footnotes.select {|fn| fn[:id] == true && fn[:ids].length > 0}
+      if ptsbs_array.empty?
+        return footnotes
+      else
+        ptsbs_array.each do |ptsbs|
+          parent_idx = ptsbs[:index]-1
+          footnotes.find {|fn| fn[:index] == parent_idx}[:ids].concat(ptsbs[:ids])
+          ptsbs[:ids] = []
+        end
+        self.send_ids_to_parent(footnotes)
+      end
+    end
+  end
+
+  module EDOCS
+    def self.search(docket: nil, da: nil, fcc: nil, report: nil, file: nil, fcc_rcd_vol: nil, fcc_rcd_page: nil)
+      uri = URI.parse("https://apps.fcc.gov/edocs_public/Query.do?mode=advanced&rpt=cond")
+      params = {
+        'fccNo' => fcc,
+        'daNo' => da,
+        'fileNo' => file,
+        'docket' => docket,
+        'reportNo' => report,
+        'fccRecordVol' => fcc_rcd_vol,
+        'fccRecordPage' => fcc_rcd_page
+      }
+      params.reject! {|_k,v| v.nil?}
+
+      url = 'https://apps.fcc.gov/edocs_public/Query.do?mode=advance&rpt=cond'
+      response = Unirest.post url, parameters: params
+      doc = Nokogiri::HTML(response.raw_body)
+
+      tables = doc.css('table.tableWithOutBorder').children.css('table.tableWithOutBorder')
+      results = tables[2].css('table.tableWithBorder')
+
+      results.map do |result|
+        links = result.search('a').to_a
+        links.shift
+        links = links.map do |link|
+          path = link.attributes["href"].value
+
+          "https://apps.fcc.gov/edocs_public/#{path}"
+        end
+
+        word = links.select {|link| link.end_with?('.doc', '.docx')}
+        pdf = links.select {|link| link.end_with?('.pdf')}
+        txt = links.select {|link| link.end_with?('.txt')}
+
+        rows = result.search('tr')
+
+        {
+          title: rows[0].text.strip,
+          released: rows[1].text.strip.split(': ')[1],
+          description: rows[2].text.strip.split('Description: ')[1],
+          word: word,
+          pdf: pdf,
+          txt: txt
+        }.reject {|_k,v| v.nil?}
+      end
+    end
+  end
+
+  module Proceedings
+    def self.search(docket: nil)
+      url = "http://apps.fcc.gov/ecfs/proceeding/view?name=#{docket}"
+      response = Unirest.get url
+      doc = Nokogiri::HTML(response.raw_body)
+      table = doc.search('table.dataTable').first
+      rows = table.search('div.wwgrp')
+
+      proceeding = {}
+      rows.each do |row|
+        key = row.search('span')[0].text.strip
+        key.gsub!(" ", "")
+        key.gsub!(":", "")
+        key.downcase!
+        value = row.search('span')[1].text.strip
+        proceeding[key.to_sym] = value
+      end
+
+      proceeding
+    end
+  end
+
+  module Filings
+    ATTRS = [
+      :docket, :filer, :lawfirm, :received,
+      :posted, :exparte, :type, :pages
+    ]
+
+    def self.get_document_links(url: url)
+      doc = Nokogiri::HTML(open(url))
+      xpath = "//*[@id=\"documents.link\"]"
+      links = doc.xpath(xpath).search('a')
+
+      links.map do |link|
+        id = link.attributes["href"].value.split('?id=')[1]
+
+        "http://apps.fcc.gov/ecfs/document/view?id=#{id}"
+      end
+    end
+
+    def self.search(docket: nil, size: 1000, start: 0, order: 'asc')
+      url = "http://apps.fcc.gov/ecfs/comment_search_solr/doSearch?proceeding=#{docket}&dir=#{order}&start=#{start}&size=#{size}"
+      doc = Nokogiri::HTML(open(url))
+      xpath = "//*[@id='yui-main']/div/div[4]"
+      table = doc.xpath(xpath).children[1]
+      rows = table.search('tr')
+      rows.shift
+
+      filings = []
+      rows.each do |row|
+        row_hash = {}
+        cols = row.search('td')
+
+        cols.each_with_index do |col, i|
+          attribute = ECFS::Filings::ATTRS[i]
+          row_hash[attribute] = col.text.strip
+
+          # get the url
+          if attribute == :filer
+            path = col.search('a').first.attributes["href"].value
+            id = path.split('?id=')[1]
+            url = "http://apps.fcc.gov/ecfs/comment/view?id=#{id}"
+            row_hash[:url] = url
+          end
+        end
+
+        # cast dates and int
+        row_hash[:received] = DateTime.parse(row_hash[:received]).to_s
+        row_hash[:posted] = DateTime.parse(row_hash[:posted]).to_s
+        row_hash[:pages] = row_hash[:pages].to_i
+
+        filings << row_hash
+      end
+
+      filings
+    end
+  end
+end

data/lib/ecfs/version.rb

@@ -1,3 +1,3 @@
 module ECFS
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,37 +1,23 @@
 --- !ruby/object:Gem::Specification
 name: ecfs
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Alan deLevie
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2015-01-19 00:00:00.000000000 Z
+date: 2015-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
-- !ruby/object:Gem::Dependency
-  name: rake
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -39,13 +25,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: vcr
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -53,41 +39,41 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: unirest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 4.7.3
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 4.7.3
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-rescue
+  name: open_uri_redirections
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -95,27 +81,27 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: m
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: pdf-reader
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -123,27 +109,27 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '10.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -151,20 +137,20 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: spreadsheet
+  name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ECFS provides a set of utilities for scraping FCC rulemakings
+description: Provides Ruby-based access to the FCC's Electronic Comment Filing System
 email:
 - adelevie@gmail.com
 executables: []
@@ -172,11 +158,16 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
+- ".ruby-version"
 - ".travis.yml"
+- CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/console
+- bin/setup
 - ecfs.gemspec
 - fixtures/vcr_cassettes/bulk_cassette.yml
 - fixtures/vcr_cassettes/main_cassette.yml
@@ -202,15 +193,7 @@ files:
 - lib/ecfs/too_many_filings_error.rb
 - lib/ecfs/util.rb
 - lib/ecfs/version.rb
-- test/helper.rb
-- test/test_daily_releases.rb
-- test/test_filing.rb
-- test/test_filings_query.rb
-- test/test_large_proceeding.rb
-- test/test_proceeding.rb
-- test/test_proceedings_query.rb
-- test/test_solr_scrape.rb
-homepage: http://github.com/adelevie/ecfs
+homepage: https://github.com/adelevie/ecfs
 licenses:
 - MIT
 metadata: {}
@@ -233,14 +216,5 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: ECFS helps you obtain comments and other filings from the FCC's Electronic
-  Comment Filing System
-test_files:
-- test/helper.rb
-- test/test_daily_releases.rb
-- test/test_filing.rb
-- test/test_filings_query.rb
-- test/test_large_proceeding.rb
-- test/test_proceeding.rb
-- test/test_proceedings_query.rb
-- test/test_solr_scrape.rb
+summary: Scraper for the FCC's Electronic Comment Filing System
+test_files: []