fech 0.1.0

data/.gitignore

@@ -0,0 +1,7 @@
+sources/rows/*
+pkg/*
+.bundle
+*.gem
+.yardoc/*
+doc/*
+rendered_maps.rb

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--profile

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fech.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,49 @@
+PATH
+  remote: .
+  specs:
+    fech (0.1.0)
+      fastercsv
+      people (~> 0.2.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    ZenTest (4.5.0)
+    autotest (4.4.6)
+      ZenTest (>= 4.4.1)
+    columnize (0.3.2)
+    diff-lcs (1.1.2)
+    fastercsv (1.5.4)
+    linecache (0.43)
+    mocha (0.9.12)
+    people (0.2.1)
+    rcov (0.9.9)
+    rdoc (3.9.2)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
+    yard (0.7.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  autotest
+  bundler
+  fech!
+  mocha
+  rcov
+  rdoc
+  rspec (~> 2.6)
+  ruby-debug
+  yard

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2011 The New York Times Company
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this library except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.rdoc

@@ -0,0 +1,178 @@
+
+  ______   ______     ______     __
+ /\  ___\ /\  ___\   /\  ___\   /\ \___
+ \ \  __\ \ \  __\   \ \ \____  \ \  __ \
+  \ \_\    \ \_____\  \ \_____\  \ \_\ \_\
+   \/_/     \/_____/   \/_____/   \/_/\/_/
+
+Fech provides a layer of abstraction in parsing electronic presidential campaign filings from the Federal Election Commission. It lets you access filing attributes the same way regardless of filing version, and works as a framework for cleaning and filing data.
+
+== Installation
+
+Install Fech as a gem:
+
+  gem install Fech
+  
+For use in a Rails 3 application, put the following in your Gemfile:
+
+  gem 'Fech'
+
+then issue the 'bundle install' command.
+
+== Getting Started
+
+Start by creating a Filing object that corresponds to any electronic filing from the FEC. You'll then have to download the file before parsing it:
+
+  filing = Fech::Filing.new(723604)
+  filing.download
+
+* Pass in the FEC filing id
+* Optional: specify the :download_dir on initialization to set where filings are stored. Otherwise, they'll go into a temp folder on your filesystem.
+
+=== Summary Data
+
+To get summary data for the filing (various aggregate amounts, stats about the filing):
+
+  filing.summary
+  #=> {:coverage_from_date=>"20110101", :coverage_from_date=>"20110301", ... }
+
+Returns a named hash of all attributes available for the filing. (Which fields are available can vary depending on the filing version number.)
+
+=== Accessing specific line items
+
+To grab every row in the filing of a certain type (all Schedule A items, for example):
+
+  filing.rows_like(/^sa/)
+  #=> [{:transaction_id=>"SA17.XXXX", :contribution_date>"20110101" ... } ... ]
+
+This will return an array of hashes, one hash for each row found in the filing whose line number matches the regular expression you passed in. (SA17s and SA18s would both be returned in this example). You can also pass in strings for exact matches.
+
+When given a block, .rows_like will yield a single hash at a time:
+
+  filing.rows_like(/^sa/) do |contribution|
+    contribution.transaction_id
+    #=> {:transaction_id=>"SA17.XXXX", :contribution_date>"20110101" ... }
+  end
+
+== Usage
+
+=== Accessing specific fields
+
+By default, .rows_like will process every field in the matched rows (some rows have 200+ fields). You can speed up performance significantly by asking for just the subset of fields you need.
+
+  filing.rows_like(/^sa/, :include => [:transaction_id]) do |contribution|
+    contribution
+    #=> {:transaction_id=>"SA17.XXXX"}
+  end
+
+=== Raw data
+
+If you want to access the raw arrays of row data, pass :raw => true to .rows_like or any of its shortcuts:
+
+  filing.contributions(:raw => true)
+  #=> [["SA17A", "C00XXXXXX", "SA17.XXXX", nil, nil, "IND" ... ], ... ]
+  
+  filing.contributions(:raw => true) do |row|
+    #row => ["SA17A", "C00XXXXX", "SA17.XXXX", nil, nil, "IND" ... ]
+  end
+
+The source maps for individual row types and versions may also be accessed directly:
+
+  Fech::Filing.map_for("sa")
+  Fech::Filing.map_for(/sa/, :version => 6.1)
+  #=> [:form_type, :filer_committee_id_number, :transaction_id ... ]
+
+You can then bypass some of the overhead of Fech if you're building something more targeted.
+
+=== Converting / Preprocessing data
+
+For performing bulk actions on specific types of fields, you can register "translations" which will manipulate specific data under certain conditions.
+
+An example: dates within filings are formatted as YYYYMMDD. To automatically convert all Schedule B :expenditure_date values to native Ruby Dates:
+
+  filing.translate do |t|
+    t.convert(:row => /^sb/, :field => :expenditure_date) { |v| Date.parse(v) }
+  end
+
+The block you give .convert will be given the original value of that field when it is run. After you run .convert, any time you parse a row beginning with "SB", the :expenditure_date value will be a Date object.
+
+The :field parameter can also be a regular expression:
+
+  filing.translate do |t|
+    t.convert(:row => /^f3p/, :field => /^coverage_.*_date/) { |v| Date.parse(v) }
+  end
+
+Now, both :coverage_from_date and :coverage_through_date will be automatically cast to dates.
+
+You can leave off any or all of the parameters (row, field, version) for more broad adoption of the translation.
+
+=== Derived Fields
+
+You may want to perform the same calculation on many rows of data (contributions minus refunds to create a net value, for example). This can be done without cluttering up your more app-specific parsing code by using a .combine translation. The translation blocks you pass .combine receive the entire row as their context, not just a single value. The :field parameter becomes what you want the new value to be named.
+
+  filing.translate do |t|
+    t.combine(:row => :f3pn, :field => :net_individual_contributions) do |row|
+      contribs = row.col_a_individual_contribution_total.to_f
+      refunds = row.col_a_total_contributions_refunds.to_f
+      contribs - refunds
+    end
+  end
+
+In this example, every parsed Schedule A row would contain an attribute not found in the original filing data - :net_individual_contributions - which contains the result of the calculation above. The values used to construct combinations will have already been run through any .convert translations you've specified.
+
+=== Built-in translations
+
+There are two sets of translations that come with Fech for some of the more common needs:
+ * Breaking apart names into their component parts and joining them back together, depending on which the filing provides
+ * Converting date field strings to Ruby Date objects
+
+You can mix these translations into your parser when you create it:
+
+  filing = Fech::Filing.new(723604, :translate => [:names, :dates])
+
+Just be aware that as you add more translations, the parsing will start to take slightly longer (although having translations that aren't used will not slow it down).
+
+=== Aliasing fields
+
+You can allow any field (converted, combined, or untranslated) to have an arbitrary number of aliases. For example, you could alias the F3P line's :col_a_6_cash_on_hand_beginning_period to the more manageable :cash_beginning
+
+  filing.translate do |t|
+    t.alias :cash_beginning, :col_a_cash_on_hand_beginning_period, :f3p
+  end
+  
+  filing.summary.cash_beginning == filing.summary.col_a_cash_on_hand_beginning_period.
+  #=> true
+
+We found it useful to be able to access attributes using the name the fields in our database they correspond to.
+
+== Warnings
+
+Filings can contain data that is incomplete or wrong: contributions might be in excess of legal limits, or data may have just been entered incorrectly. While some of these mistakes are corrected in later filing amendments, you should be aware that the data is not perfect. Fech will only return data as accurate as the source.
+
+When filings get very large, be careful not to perform operations that attempt to transform many rows in memory.
+
+== Supported row types and versions
+
+The following row types are currently supported from filing version 3 through 7.0:
+* F3P (Summaries)
+* F3PS
+* F3S
+* F3P31 (Items to be Liquidated)
+* SA (Contributions)
+* SB (Expenditures)
+* SC (Loans)
+* SC1
+* SC2
+* SD (Debts & Obligations)
+* SE (Independent Expenditures)
+* SF (Coordinated Expenditures)
+
+== Authors
+
+Michael Strickland, michael.strickland@nytimes.com
+
+Evan Carmi
+
+== Copyright
+
+Copyright (c) 2011 The New York Times Company. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,3 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+Dir.glob('tasks/*.rake').each { |r| import r }

data/autotest/discover.rb

@@ -0,0 +1 @@
+Autotest.add_discovery { "rspec2" }

data/fech.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "fech/version"
+
+Gem::Specification.new do |s|
+  s.name        = "fech"
+  s.version     = Fech::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Michael Strickland", "Evan Carmi"]
+  s.email       = ["michael.c.strickland@gmail.com"]
+  s.homepage    = "http://github.com/nytimes/fech"
+  s.summary     = %q{Ruby library for parsing FEC filings.}
+  s.description = %q{A Ruby library for interacting with electronic filings from the Federal Election Commission.}
+
+  s.rubyforge_project = "fech"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency "fastercsv"
+  s.add_dependency "people", "~> 0.2.1"
+  s.add_development_dependency "rspec", "~> 2.6"
+  s.add_development_dependency "mocha"
+  s.add_development_dependency "autotest"
+  s.add_development_dependency "ruby-debug"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rcov"
+  s.add_development_dependency "rdoc"
+  s.add_development_dependency "yard"
+end

data/lib/fech.rb

@@ -0,0 +1,13 @@
+require 'fech/filing'
+require 'fech/rendered_maps'
+require 'fech/mappings'  
+require 'fech/default_translations'
+require 'fech/translator'
+require 'fech/mapped'
+require 'fech/fech_utils'
+require 'fech/map_generator'
+
+module Fech
+  extend FechUtils
+  DEFAULT_VERSION = "7.0"
+end

data/lib/fech/default_translations.rb

@@ -0,0 +1,135 @@
+module Fech
+  
+  # Stores sets of build-in translations that can be mixed in to a Fech::Filing.
+  # Contains functions that accept a Translator, and add arbitrary translations
+  # to it. The public function names should correspond to the key used to mix it in.
+  #
+  # filing = Fech::Filing.new(XXXXXX, :translate => [:names, :dates])
+  class DefaultTranslations
+
+    # The five bits that make up a name, and their labels in the People gem
+    NAME_BITS = [:prefix, :first_name, :middle_name, :last_name, :suffix]
+    PEOPLE_BITS = [:title, :first, :middle, :last, :suffix]
+    
+    attr_reader :t
+    
+    def initialize(translator)
+      @t = translator
+    end
+    
+    # Splits composite names into its component parts, and combines those parts
+    # into composites where appropriate. Assumes that the canonical names of the
+    # fields follow the pattern:
+    #   * FIELD_name         - "Mr. John Charles Smith Sr."
+    #   * FIELD_prefix       - "Mr."
+    #   * FIELD_first_name   - "John"
+    #   * FIELD_middle_name  - "Charles"
+    #   * FIELD_last_name    - "Smith"
+    #   * FIELD_suffix       - "Sr."
+    def names
+      
+      # COMBINE split names into composite names for these rows
+      composites = [
+        {:row => :sa,     :version => /^[6-7]/,   :field => [:contributor, :donor_candidate]},
+        {:row => :sb,     :version => /^[6-7]/,   :field => [:payee, :beneficiary_candidate]},
+        {:row => :sc,     :version => /^[6-7]/,   :field => [:lender, :lender_candidate]},
+        {:row => :sc1,    :version => /^[6-7]/,   :field => [:treasurer, :authorized]},
+        {:row => :sc2,    :version => /^[6-7]/,   :field => :guarantor},
+        {:row => :sd,     :version => /^[6-7]/,   :field => :creditor},
+        {:row => :se,     :version => /^[6-7]/,   :field => [:payee, :candidate]},
+        {:row => :sf,     :version => /^[6-7]/,   :field => [:payee, :payee_candidate]},
+        {:row => :f3p,    :version => /^[6-7]/,   :field => :treasurer},
+        {:row => :f3p31,  :version => /^[6-7]/,   :field => :contributor},
+      ]
+      # SPLIT composite names into component parts for these rows
+      components = [
+        {:row => :sa,     :version => /^3|(5.0)/, :field => :contributor},
+        {:row => :sa,     :version => /^[3-5]/,   :field => :donor_candidate},
+        {:row => :sb,     :version => /^3|(5.0)/, :field => :payee},
+        {:row => :sb,     :version => /^[3-5]/,   :field => :beneficiary_candidate},
+        {:row => :sc,     :version => /^[3-5]/,   :field => [:lender, :lender_candidate]},
+        {:row => :sc1,    :version => /^[3-5]/,   :field => [:treasurer, :authorized]},
+        {:row => :sc2,    :version => /^[3-5]/,   :field => :guarantor},
+        {:row => :sd,     :version => /^[3-5]/,   :field => :creditor},
+        {:row => :se,     :version => /^[3-5]/,   :field => [:payee, :cadidate]},
+        {:row => :sf,     :version => /^[3-5]/,   :field => [:payee, :payee_candidate]},
+        {:row => :f3p,    :version => /^[3-5]/,   :field => :treasurer},
+        {:row => :f3p31,  :version => /^[3-5]/,   :field => :contributor},
+      ]
+      
+      composites.each { |c| combine_components_into_name(c) }
+      components.each { |c| split_name_into_components(c) }
+      
+    end
+    
+    # Converts everything that looks like an FEC-formatted date to a
+    # native Ruby Date object.
+    def dates
+      t.convert do |value|
+        if /^\d{8}$/.match(value).nil?
+          value
+        else
+          Date.parse(value)
+        end
+      end
+    end
+    
+    private
+    
+    # Turns "Allred^Ann^Mrs.^III" into "Mrs. Ann Allred III"
+    def self.fix_carrot_names(name)
+      name = name.split("^").reverse
+      # move the suffix to the beginning
+      name.push name.shift if name.size > 3
+      name.join(" ")
+    end
+    
+    # Create a Translation for the given row, version named as :field
+    def combine_components_into_name(composite)
+      raise ArgumentError, "Must pass a :row, :version AND :field" if composite.nil?
+      composite[:field] = [composite[:field]] unless composite[:field].is_a?(Array)
+      
+      composite[:field].each do |field|
+        t.combine(:row => composite[:row], :version => composite[:version],
+                  :field => "#{field}_name") do |row|
+                  
+          # Gather each name_bit from the parsed row, and join it into one value
+          bits = NAME_BITS.collect do |field_name|
+            row.send("#{field}_#{field_name}".to_sym)
+          end
+          bits.compact.join(" ")
+        end
+      end
+    end
+    
+    # Create a Translation for all five name bits, that will strip
+    # out its respective bit from an already-populate composite name field.
+    def split_name_into_components(component)
+      raise ArgumentError, "Must pass a :row, :version AND :field" if component.nil?
+      component[:field] = [component[:field]] unless component[:field].is_a?(Array)
+      
+      component[:field].each do |field|
+        NAME_BITS.zip(PEOPLE_BITS).each do |field_name, people_name|
+          t.combine(:row => component[:row], :version => component[:version],
+                    :field => "#{field}_#{field_name}") do |row|
+          
+            # Grab the original, composite name
+            name = row.send("#{field}_name")
+          
+            unless name.nil?
+              # Fix various name formatting errors
+              name = self.class.fix_carrot_names(name) unless name.index("^").nil?
+          
+              # Extract just the component you want
+              (Fech::Translator::NAME_PARSER.parse(name)[people_name] || "").strip
+            else
+              nil
+            end
+          end
+        end
+      end
+    end
+    
+  end
+  
+end