fech-search 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a6af2516a3e0869ff3cdb0d4ff69d0574557cd74
+  data.tar.gz: 53b16020e56f3a70011df1044ec868f915f653be
+SHA512:
+  metadata.gz: 7be2bf91e0b29e3faa985a133bbd5e2a7327c59bca89eb80bd127c701a21b2a50fadbe22f77dd54bb4ed8f5ae5a11cd41e26c9167c822eb9e7677514501e72d0
+  data.tar.gz: 3ef5499dbb83778c98d575a562bd2f1efa4389829ad91c084811ffea33dda063d7b5f269396c171ac6c6381b3711e71cd186b5de6600b00e363cf58135b5cb6a

data/.gitignore

@@ -0,0 +1,16 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fech-search.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,30 @@
+PATH
+  remote: .
+  specs:
+    fech-search (0.0.1)
+      fech (= 1.4.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.4)
+    fastercsv (1.5.5)
+    fech (1.4.1)
+      fastercsv
+      people
+    people (0.2.1)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fech-search!
+  rspec (= 2.13.0)

data/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2013, The Huffington Post
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,124 @@
+# Fech-Search
+
+Fech-Search is a plugin for [Fech](http://nytimes.github.io/Fech/) that adds an interface for searching for electronic filings submitted to the Federal Election Commission. Where Fech provides a way to download and parse filings, Fech-Search provides users with a way to find the filings they're interested in.
+
+Fech-Search is essentially a Ruby wrapper around the [electronic filing search form on the FEC's website](http://www.fec.gov/finance/disclosure/efile_search.shtml).
+
+## Usage
+
+### Example
+
+    require 'fech-search'
+
+Perform a search for form F3P filings (report of receipts and disbursements) submitted by Romney for President:
+
+    search = Fech::Search.new(:committee_name => "Romney for President", :form_type => "F3P")
+
+The search is performed when `Fech::Search.new` is called. You can then access the results of the search with `search.results`, which is simply an array of `Fech::SearchResult` objects:
+
+    results = search.results
+    results.size
+    => 100
+
+Each `Fech::SearchResult` object has the following attributes:
+
+- amended_by
+- committee_id
+- committee_name
+- date_filed
+- date_format
+- description
+- filing_id
+- form_type
+- period
+
+You can now work with the results as you would any Ruby array.
+
+Remove any filings that have been amended:
+
+    results.select! { |r| r.amended_by.nil? }
+    results.size
+    => 41
+
+Limit to filings covering the last six months of 2012:
+
+    results.select! { |r| r.period[:from] >= Date.new(2012, 7, 1) && r.period[:to] <= Date.new(2012, 12, 31) }
+    results.size
+    => 6
+
+Create a `Fech::Filing` object from one of the results and download the filing data:
+
+    filing = results.first.filing.download
+
+You now have access to the same filing object and methods as if you had created it directly with [Fech](http://nytimes.github.io/Fech/).
+
+To initialize the `Fech::Filing` object with parameters, pass them as arguments to the `SearchResult#filing` method:
+
+    filing = results.first.filing(:csv_parser => Fech::CsvDoctor)
+
+Get information from the filing:
+
+    filing.summary[:col_a_total_receipts]
+    => "4747984.49"
+
+    filing.summary[:col_b_total_receipts]
+    => "10617838.18"
+
+### Search parameters
+
+The following search parameters are available:
+
+- `:committee_id` (The nine-character committee ID assigned by the FEC)
+    - examples: "C00499202", "C00130187"
+- `:committee_name`
+    - examples: "Restore Our Future", "Obama for America"
+- `:state` (Two-character state abbreviation)
+    - examples: "MA", "FL"
+- `:party` (Three-character party abbreviation)
+    - examples: "REP", "DEM"
+- `:committee_type` (One-character committee type. [See a list of committee types](http://www.fec.gov/finance/disclosure/metadata/CommitteeTypeCodes.shtml))
+    - examples: "H", "P"
+- `:report_type` ([See a list of report types](http://www.fec.gov/finance/disclosure/metadata/ReportTypeCodes.shtml))
+    - examples: "M4", "Q1"
+- `:date` (A Ruby Date object)
+- `:form_type` ([See the FEC's electronic filing search for a list of form types](http://www.fec.gov/finance/disclosure/efile_search.shtml))
+    - examples: "F3", "F24"
+  
+Any number of these parameters may be used. However, the FEC's search functionality has some limitations:
+
+- All other parameters are ignored when `:committee_id` is used.
+- `:form_type` cannot be used by itself; another parameter must be used with it.
+
+An `ArgumentError` will be raised if either of these is violated with `Fech::Search.new`.
+
+__Note:__ Overly broad searches can be slow, so you should make your search as specific as possible.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fech-search'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fech-search
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Authors
+
+- Aaron Bycoffe, bycoffe@huffingtonpost.com
+
+## Copyright
+
+Copyright © 2013 The Huffington Post. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/fech-search.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/fech-search/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Aaron Bycoffe"]
+  gem.email         = ["bycoffe@huffingtonpost.com"]
+  gem.description   = %q{A Fech plugin for searching electronic FEC filings}
+  gem.summary       = %q{Fech-Search provides an interface for searching for electronic filings submitted to the FEC, and accessing the data in those filings using Fech}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fech-search"
+  gem.require_paths = ["lib"]
+  gem.version       = Fech::Search::VERSION
+
+  gem.add_dependency "fech", "~> 1.4.1"
+
+  gem.add_development_dependency "rspec", "~> 2.13.0"
+end

data/lib/fech-search.rb

@@ -0,0 +1,4 @@
+require "fech"
+require "fech-search/version"
+require "fech-search/search"
+require "fech-search/search_result"

data/lib/fech-search/search.rb

@@ -0,0 +1,187 @@
+require 'net/http'
+
+module Fech
+
+  # Fech::Search is an interface for the FEC's electronic filing search
+  # (http://www.fec.gov/finance/disclosure/efile_search.shtml)
+  class Search
+
+    # @param [Hash] search_params a hash of parameters to be
+    # passed to the search form.
+    def initialize(search_params={})
+      @search_params = validate_params(make_params(search_params))
+      @search_url = 'http://query.nictusa.com/cgi-bin/dcdev/forms/'
+      @response = search
+    end
+
+    # Convert the search parameters passed to @initialize to use
+    # the format and keys needed for the form submission.
+    # @return [Hash]
+    def make_params(search_params)
+      {
+        'comid' => search_params[:committee_id] || '',
+        'name' => search_params[:committee_name] || '',
+        'state' => search_params[:state] || '',
+        'party' => search_params[:party] || '',
+        'type' => search_params[:committee_type] || '',
+        'rpttype' => search_params[:report_type] || '',
+        'date' => search_params[:date] ? search_params[:date].strftime('%m/%d/%Y') : '',
+        'frmtype' => search_params[:form_type] || ''
+      }
+    end
+
+    def validate_params(params)
+      raise ArgumentError, "At least one search parameter must be given" if params.values.all? { |x| x.empty? }
+      nonempty_keys = params.select { |k, v| !v.empty? }.keys
+      raise ArgumentError, ":committee_id cannot be used with other search parameters" if nonempty_keys.include?("comid") && nonempty_keys.size > 1
+      raise ArgumentError, ":form_type must be used with at least one other search parameter" if nonempty_keys.include?("frmtype") && nonempty_keys.size == 1
+      params
+    end
+
+    # Performs the search of the FEC's electronic filing database.
+    def search
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.read_timeout = 5000
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request.set_form_data(@search_params)
+      @response = http.request(request)
+    end
+
+    # A parsed URI for the search
+    def uri
+      uri = URI.parse(@search_url)
+    end
+
+    def body
+      @response ? @response.body : nil
+    end
+
+    # The results page is formatted differently depending
+    # on whether the search includes a date. Use the correct
+    # method for parsing the results depending on whether
+    # a date was used in the search. Will return an array of
+    # results if called directly, or will yield the results one
+    # by one if a block is passed.
+    def results(&block)
+      if @search_params['date'] != ''
+        results_from_date_search(&block)
+      else
+        results_from_nondate_search(&block)
+      end
+    end
+
+    # Parse the results from a search that does not include a date.
+    # Will return an array of results if called directly, or will
+    # yield the results one by one if a block is passed.
+    def results_from_nondate_search(&block)
+      parsed_results = []
+      regex = /<DT>(.*?)<P/m
+      match = body.match regex
+      return [] if match.nil?
+      content = match[1]
+      committee_sections = content.split(/<DT>/)
+      committee_sections.each do |section|
+        data = parse_committee_section(section)
+        data.each do |result|
+          search_result = SearchResult.new(result)
+
+          if block_given?
+            yield search_result
+          else
+            parsed_results << search_result
+          end
+        end
+      end
+      block_given? ? nil : parsed_results
+    end
+
+    # For results of a search that does not include a date, parse
+    # the section giving information on the committee that submitted
+    # the filing.
+    # @param [String] section
+    def parse_committee_section(section)
+      data = []
+      section.gsub!(/^<BR>/, '')
+      rows = section.split(/\n/)
+      committee_data = parse_committee_row(rows.first)
+      rows[1..-1].each do |row|
+        data << committee_data.merge(parse_filing_row(row))
+      end
+      data
+    end
+
+    # Parse the results from a search that includes a date.
+    # Will return an array of results if called directly, or will
+    # yield the results one by one if a block is passed.
+    def results_from_date_search(&block)
+      parsed_results = []
+      dl = body.match(/<DL>(.*?)<BR><P/m)[0]
+      rows = dl.split('<DT>')[1..-1].map { |row| row.split("\n") }
+      rows.each do |committee, *filings|
+        committee = parse_committee_row(committee)
+        filings.each do |filing|
+          next if filing == "<BR><P"
+          data = committee.merge(parse_filing_row(filing))
+          search_result = SearchResult.new(data)
+          if block_given?
+            yield search_result
+          else
+            parsed_results << search_result
+          end
+        end
+      end
+      block_given? ? nil : parsed_results
+    end
+
+    # For results of a search that includes a date, parse
+    # the portion of the results with information on the
+    # committee that submitted the filing.
+    # @param [String] row
+    # @return [Hash] the committee name and ID
+    def parse_committee_row(row)
+      regex = /
+              '>
+              (.*?)
+              \s-\s
+              (C\d{8})
+              /x
+      match = row.match regex
+      {:committee_name => match[1], :committee_id => match[2]}
+    end
+
+    # Parse a result row with information on the filing itself.
+    # @param [String] row
+    # @return [Hash] the filing ID, form type, period, date filed, description
+    # and, optionally, the filing that amended this filing.
+    def parse_filing_row(row)
+      regex = /
+              FEC-(\d+)
+              \s
+              Form
+              \s
+              (F.*?)
+              \s\s-\s
+              (period\s([-\/\d]+),\s)?
+              filed
+              \s
+              ([\/\d]+)
+              \s
+              (-\s
+               (.*?)
+               ($|<BR>.*?FEC-(\d+))
+              )?
+              /x
+      match = row.match regex
+      {:filing_id => match[1],
+       :form_type => match[2],
+       :period => match[4],
+       :date_filed => match[5],
+       :description => match[7],
+       :amended_by => match[9]
+      }
+    end
+
+  end
+
+end
+

data/lib/fech-search/search_result.rb

@@ -0,0 +1,41 @@
+module Fech
+
+  # Fech:SearchResult is a class representing a search result
+  # from Fech::Search.
+  class SearchResult
+    attr_reader :committee_name, :committee_id, :filing_id, :form_type, :period, :date_filed, :description, :amended_by
+
+    # @param [Hash] attrs The attributes of the search result.
+    def initialize(attrs)
+      @date_format = '%m/%d/%Y'
+
+      @committee_name = attrs[:committee_name]
+      @committee_id   = attrs[:committee_id]
+      @filing_id      = attrs[:filing_id]
+      @form_type      = attrs[:form_type]
+      @period         = parse_period(attrs[:period])
+      @date_filed     = Date.strptime(attrs[:date_filed], @date_format)
+      @description    = attrs[:description]
+      @amended_by     = attrs[:amended_by]
+    end
+
+    # Parse the string representing a filing period.
+    # @param [String] period a string representing a filing period
+    # @return [Hash, nil] a hash representing the start and end
+    # of a filing period.
+    def parse_period(period)
+      return if period.nil?
+      from, to = period.split('-')
+      from = Date.strptime(from, @date_format)
+      to = Date.strptime(to, @date_format)
+      {:from => from, :to => to}
+    end
+
+    # The Fech filing object for this search result
+    # @return [Fech::Filing]
+    def filing(opts={})
+      Fech::Filing.new(self.filing_id, opts)
+    end
+  end
+
+end

data/lib/fech-search/version.rb

@@ -0,0 +1,5 @@
+module Fech
+  class Search
+    VERSION = "0.0.1"
+  end
+end

data/spec/search_result_spec.rb

@@ -0,0 +1,46 @@
+require 'fech-search'
+
+describe Fech::SearchResult do
+
+  before do
+    @date_search = Fech::Search.new(:date => Date.new(2013, 4, 20))
+    @date_results = @date_search.results
+    @date_result = @date_results.first
+    @committee_search = Fech::Search.new(:committee_id => "C00431171")
+    @committee_results = @committee_search.results
+    @committee_result = @committee_results.first
+  end
+
+  context "creating" do
+
+    it "should have valid dates for period" do
+      from = @committee_result.period[:from]
+      from.class.should == Date
+      to = @committee_result.period[:to]
+      to.class.should == Date
+      to.should >= from
+    end
+
+  end
+
+  context "filing" do
+
+    it "should make filing accessible" do
+      @date_result.filing.class.should == Fech::Filing
+      @committee_result.filing.class.should == Fech::Filing
+    end
+
+    it "should create filing object for correct filing" do
+      filing = @committee_result.filing
+      filing.filing_id.should == @committee_result.filing_id
+    end
+
+    it "should pass arguments to Fech::Filing#new" do
+      filing = @date_result.filing(:download_dir => "/tmp")
+      filing.download_dir.should == "/tmp"
+    end
+
+  end
+
+
+end

data/spec/search_spec.rb

@@ -0,0 +1,51 @@
+require 'fech-search'
+
+describe Fech::Search do
+
+  context "creating" do
+
+    it "should raise ArgumentError if no parameters are given" do
+      expect { Fech::Search.new }.to raise_error(ArgumentError)
+    end
+
+    it "should raise ArgumentError if :committee_id is given with another parameter" do
+      expect { Fech::Search.new(:committee_id => "C00431171", :date => Date.new(2013, 5, 16)) }.to raise_error(ArgumentError)
+    end
+
+    it "should raise ArgumentError if :form_type is given without another parameter" do
+      expect { Fech::Search.new(:form_type => "F3") }.to raise_error(ArgumentError)
+    end
+
+    it "should not raise an error if :committee_id is given by itself" do
+      expect { Fech::Search.new(:committee_id => "C00431171") }.to_not raise_error(ArgumentError)
+    end
+
+    it "should not raise an error if :form_type is given with another parameter" do
+      expect { Fech::Search.new(:form_type => "F3", :date => Date.new(2013, 5, 29)) }.to_not raise_error(ArgumentError)
+    end
+
+  end
+
+  context "results" do
+
+    it "should return an array of results from committee search" do
+      search = Fech::Search.new(:committee_id => "C00431171")
+      search.results.class.should == Array
+      search.results[0].class.should == Fech::SearchResult
+    end
+
+    it "should return an array of results from date search" do
+      search = Fech::Search.new(:date => Date.new(2013, 5, 29))
+      search.results.class.should == Array
+      search.results[0].class.should == Fech::SearchResult
+    end
+
+    it "should return an array of results from date and report_type search" do
+      search = Fech::Search.new(:date => Date.new(2013, 5, 29), :report_type => "M4")
+      search.results.class.should == Array
+      search.results[0].class.should == Fech::SearchResult
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: fech-search
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Aaron Bycoffe
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fech
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.13.0
+description: A Fech plugin for searching electronic FEC filings
+email:
+- bycoffe@huffingtonpost.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- fech-search.gemspec
+- lib/fech-search.rb
+- lib/fech-search/search.rb
+- lib/fech-search/search_result.rb
+- lib/fech-search/version.rb
+- spec/search_result_spec.rb
+- spec/search_spec.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Fech-Search provides an interface for searching for electronic filings submitted
+  to the FEC, and accessing the data in those filings using Fech
+test_files:
+- spec/search_result_spec.rb
+- spec/search_spec.rb
+has_rdoc: