myl-fech 1.0.3

data/.gitignore

@@ -0,0 +1,7 @@
+sources/rows/*
+pkg/*
+.bundle
+*.gem
+.yardoc/*
+doc/*
+

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--profile

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fech.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,48 @@
+PATH
+  remote: .
+  specs:
+    fech (0.9.10)
+      fastercsv
+      people
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    columnize (0.3.6)
+    diff-lcs (1.1.2)
+    fastercsv (1.5.4)
+    linecache (0.43)
+    mocha (0.9.12)
+    people (0.2.1)
+    rake (0.8.7)
+    rcov (0.9.9)
+    rdoc (3.9.2)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
+    yard (0.7.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  fech!
+  linecache (= 0.43)
+  mocha
+  rake (= 0.8.7)
+  rcov
+  rdoc
+  rspec (~> 2.6)
+  ruby-debug
+  yard

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2011 The New York Times Company
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this library except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.rdoc

@@ -0,0 +1,82 @@
+  ______   ______     ______     __
+ /\  ___\ /\  ___\   /\  ___\   /\ \___
+ \ \  __\ \ \  __\   \ \ \____  \ \  __ \
+  \ \_\    \ \_____\  \ \_____\  \ \_\ \_\
+   \/_/     \/_____/   \/_____/   \/_/\/_/
+
+Fech makes it easy to parse electronic campaign finance filings[http://www.fec.gov/finance/disclosure/efile_search.shtml] by candidates, parties and political action committees from the Federal Election Commission. It lets you access filing attributes the same way regardless of filing version, and works as a framework for cleaning and filing data. Fech is an open source project of The New York Times, but contributions from anyone interested in working with F.E.C. filings are greatly appreciated.
+
+
+NOTE: You're looking at the Make Your Laws fork of New York Times' Fech gem. It has bugfixes and features that NYT's doesn't. Take a look at the commit history / network graph.
+
+Latest (myl-fech) version: 1.0.3
+
+
+Fech is tested under Ruby versions 1.8.7, 1.9.2 and 1.9.3.
+
+== Documentation
+
+Can be found at Fech's Github page[http://nytimes.github.com/Fech/].
+
+== News
+
+* June 16, 2012: Version 1.0.1 released. Bug-fix for older Form 2 support.
+* April 11, 2012: Version 1.0.0 released! Support for Ruby 1.9.3 added, all form types supported.
+* April 9, 2012: Version 1.0.0.rc1 released. Release candidate with backwards-incompatible change (renaming zip attribute to zip_code).
+* March 29, 2012: Version 0.9.10 released. Bug-fix for Form 24 in versions 6.4 and 7.0.
+* March 28, 2012: Version 0.9.9 released. Bug-fix to add support for F3XA form type, support for Schedule H and L.
+* March 23, 2012: Version 0.9.6 and 0.9.5 released. Bug-fixes for F6 mappings.
+* March 10, 2012: Version 0.9.4 released. Added support for F6 filings.
+* March 8, 2012: Version 0.9.3 released. Bug-fix for F2 & F24 mappings.
+* Feb. 29, 2012: Version 0.9.2 released. Bug-fix for F3 mappings, added filing comparison class.
+* Feb. 21, 2012: Versions 0.9.0, 0.9.1 released. Added support for alternative CSV Parsers, F4 filings.
+* Feb. 19, 2012: Version 0.8.2 released. Added layouts for F1M and F2 filings.
+* Feb. 15, 2012: Version 0.8.1 released. Bug-fix to support F3 termination filings.
+* Feb. 13, 2012: Version 0.8.0 released. Layouts for form 3 and form 1 added, for parsing House candidate committee filings.
+* Feb. 11, 2012: Version 0.7.0 released. Layouts for form 9 added, for parsing electioneering communications filings.
+* Jan. 28, 2012: Version 0.6.0 released. Added support for quoted fields.
+* Nov. 22, 2011: Version 0.5.0 released. Layouts for form 3X added, for parsing filings from non-candidate committees.
+* Nov. 13, 2011: Version 0.3.0 released. Layouts for forms 24, 5, 56 and 57 added, for parsing independent expenditure filings.
+* Nov. 4, 2011: Version 0.2.1 released. Bug-fix release to address a problem with the :include option for selecting only certain columns. Thanks to Aaron Bycoffe for the report and patch.
+
+== Installation
+
+Install Fech as a gem:
+
+  gem install myl-fech
+  
+For use in a Rails 3 application, put the following in your Gemfile:
+
+  gem 'myl-fech', :require => 'fech'
+
+then issue the 'bundle install' command. Fech has been tested under Ruby 1.8.7, 1.9.2 and JRuby 1.6.3.
+
+== How to contribute
+
+Fech's goal is to provide support for all electronically filed forms and their row types, so contributors can pick an form type that is currently unsupported and try to implement it, or to improve an existing implementation. For entirely new form types, please include specs showing successful parsing of the summary. A good way to start is to look at the mappings for a similar form type. Please note that Fech currently commits to supporting the F.E.C.'s filing formats back to version 3.0, where applicable.
+
+Bug reports and feature requests are welcomed by submitting an Issue. Please be advised that development is focused on parsing filings, which may contain errors or be improperly filed, and not on providing wrappers or helper methods for working with filings in another context.
+
+To get started, fork the repo, make your additions or changes and send a pull request.
+
+== Pronunciation guide
+
+It's "fetch", with a soft "ch" sound. There are no other acceptable pronunciations.
+
+== Authors
+
+Michael Strickland, michael.strickland@nytimes.com
+
+Evan Carmi, evan@ecarmi.org
+
+Aaron Bycoffe, bycoffe@huffingtonpost.com
+
+Derek Willis, dwillis@nytimes.com
+
+Daniel Pritchett, daniel@sharingatwork.com
+
+Sai, github@saizai.com
+
+== Copyright
+
+Copyright (c) 2012 The New York Times Company & Sai. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,3 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+Dir.glob('tasks/*.rake').each { |r| import r }

data/autotest/discover.rb

@@ -0,0 +1 @@
+Autotest.add_discovery { "rspec2" }

data/fech.gemspec

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "fech/version"
+
+Gem::Specification.new do |s|
+  s.name        = "myl-fech"
+  s.version     = Fech::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Michael Strickland", "Evan Carmi", "Aaron Bycoffe", "Derek Willis", "Sai"]
+  s.email       = ["dwillis@gmail.com"]
+  s.homepage    = "http://github.com/MakeYourLaws/fech"
+  s.summary     = %q{Ruby library for parsing FEC filings.}
+  s.description = %q{A Ruby library for interacting with electronic filings from the Federal Election Commission.}
+
+  s.rubyforge_project = "fech"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency "fastercsv"
+  s.add_dependency "people"
+  if RUBY_VERSION < "1.9"
+    s.add_development_dependency "linecache", "0.43"
+    s.add_development_dependency "ruby-debug"
+    s.add_development_dependency "iconv"
+  end
+  if RUBY_VERSION >= "1.9"
+    s.add_development_dependency "ruby-debug19"
+    s.add_development_dependency "linecache19"
+  end 
+  s.add_development_dependency "rake", "0.8.7"
+  s.add_development_dependency "rspec", "~> 2.6"
+  s.add_development_dependency "mocha"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rcov"
+  s.add_development_dependency "rdoc"
+  s.add_development_dependency "yard"
+end

data/lib/fech/comparison.rb

@@ -0,0 +1,36 @@
+module Fech
+  # Fech::Comparison takes two Filing objects and does comparisons on them,
+  # checking for differences between an original and amended filing, or 
+  # two filings covering the same period in different years.
+  class Comparison
+    attr_accessor :filing_1, :filing_2
+    
+    # Create a new Comparison object by passing in two Filing objects
+    # Filing objects need to be downloaded first
+    # f1 = Fech::Filing.new(767437)
+    # f1.download
+    # f2 = Fech::Filing.new(751798)
+    # f2.download
+    # comparison = Fech::Comparison.new(f1, f2)
+    # comparison.summary
+    def initialize(filing_1, filing_2, opts={})
+      @filing_1     = filing_1
+      @filing_2     = filing_2
+    end
+    
+    # compares summary of this filing with summary of an earlier
+    # or later version of the filing, returning a Fech::Mapped hash
+    # of mapped fields whose values have changed. based on rails' hash diff:
+    # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/core_ext/hash/diff.rb
+    def summary
+      @filing_1.summary.delete_if { |k, v| @filing_2.summary[k] == v }.merge!(@filing_2.summary.dup.delete_if { |k, v| @filing_1.summary.has_key?(k) })
+    end
+    
+    # compares a schedule of itemized records from one filing to another
+    # returns an array of records that are new or have changed.
+    def schedule(schedule)
+      @filing_1.rows_like(schedule.to_sym) - @filing_2.rows_like(schedule.to_sym)
+    end
+    
+  end
+end

data/lib/fech/csv.rb

@@ -0,0 +1,70 @@
+require 'csv'       if     RUBY_VERSION > '1.9'
+require 'fastercsv' unless RUBY_VERSION > '1.9'
+
+# Fech::Csv is a wrapper that provides simple CSV handling consistency
+# between Ruby 1.8 and Ruby 1.9.
+#
+# 1.8's "FasterCSV" library has been brought into the Ruby core as of 1.9
+# as "CSV".  The methods are the same so no overloading should be needed.
+module Fech
+  begin
+    # Ruby 1.9 and up compatibility
+    class Csv < CSV
+    end
+  rescue
+    # 1.8 compatibility
+    class Csv < FasterCSV
+    end
+  end
+
+  class Csv
+
+    # Loads a given file and parses it into an array, line by line.
+    # Basic wrapper around FasterCSV.foreach
+    # @param [String] file_path location of the filing on the file system
+    # @options opts passed through to FasterCSV
+    def self.parse_row(file_path, opts)
+      foreach(file_path, opts) { |row| yield row }
+    end
+
+  end
+
+  class CsvDoctor < Fech::Csv
+
+    # Skips FasterCSV's whole-file wrapper, and passes each line in
+    # the file to a function that will parse it individually.
+    def self.parse_row(file_path, opts)
+      opts.reject! {|k,v| ![:col_sep, :quote_char].include?(k)}
+
+      File.open(file_path, 'r').each do |line|
+        # Skip empty lines
+        next if line.strip.empty?
+        yield safe_line(line, opts)
+      end
+    end
+
+    # Tries to parse the line with FasterCSV.parse_line and the given
+    # quote_char. If this fails, try again with the "\0" quote_char.
+    # @param [String] line the file's row to parse
+    # @options opts :quote_char the quote_char to try initially
+    def self.safe_line(line, opts)
+      begin
+        parse_line(line, opts)
+      rescue Fech::Csv::MalformedCSVError
+        row = parse_line(line, opts.merge(:quote_char => "\0"))
+        row.map! { |val| safe_value(val) }
+      end
+    end
+
+    # Removes extraneous quotes from values.
+    def self.safe_value(val)
+      return val unless val.is_a?(String)
+      begin
+        parse_line(val).first
+      rescue Fech::Csv::MalformedCSVError
+        val
+      end
+    end
+
+  end
+end

data/lib/fech/default_translations.rb

@@ -0,0 +1,133 @@
+module Fech
+  
+  # Stores sets of build-in translations that can be mixed in to a Fech::Filing.
+  # Contains functions that accept a Translator, and add arbitrary translations
+  # to it. The public function names should correspond to the key used to mix it in.
+  #
+  # filing = Fech::Filing.new(XXXXXX, :translate => [:names, :dates])
+  class DefaultTranslations
+
+    # The five bits that make up a name, and their labels in the People gem
+    NAME_BITS = [:prefix, :first_name, :middle_name, :last_name, :suffix]
+    PEOPLE_BITS = [:title, :first, :middle, :last, :suffix]
+    
+    attr_reader :t
+    
+    def initialize(translator)
+      @t = translator
+    end
+    
+    # Splits composite names into its component parts, and combines those parts
+    # into composites where appropriate. Assumes that the canonical names of the
+    # fields follow the pattern:
+    #   * FIELD_name         - "Mr. John Charles Smith Sr."
+    #   * FIELD_prefix       - "Mr."
+    #   * FIELD_first_name   - "John"
+    #   * FIELD_middle_name  - "Charles"
+    #   * FIELD_last_name    - "Smith"
+    #   * FIELD_suffix       - "Sr."
+    def names
+      
+      # COMBINE split names into composite names for these rows
+      composites = [
+        {:row => :sa,     :version => /^[6-7]/,   :field => [:contributor, :donor_candidate]},
+        {:row => :sb,     :version => /^[6-7]/,   :field => [:payee, :beneficiary_candidate]},
+        {:row => :sc,     :version => /^[6-7]/,   :field => [:lender, :lender_candidate]},
+        {:row => :sc1,    :version => /^[6-7]/,   :field => [:treasurer, :authorized]},
+        {:row => :sc2,    :version => /^[6-7]/,   :field => :guarantor},
+        {:row => :sd,     :version => /^[6-7]/,   :field => :creditor},
+        {:row => :se,     :version => /^[6-7]/,   :field => [:payee, :candidate]},
+        {:row => :sf,     :version => /^[6-7]/,   :field => [:payee, :payee_candidate]},
+        {:row => :f3p,    :version => /^[6-7]/,   :field => :treasurer},
+        {:row => :f3p31,  :version => /^[6-7]/,   :field => :contributor},
+      ]
+      # SPLIT composite names into component parts for these rows
+      components = [
+        {:row => :sa,     :version => /^3|(5.0)/, :field => :contributor},
+        {:row => :sa,     :version => /^[3-5]/,   :field => :donor_candidate},
+        {:row => :sb,     :version => /^3|(5.0)/, :field => :payee},
+        {:row => :sb,     :version => /^[3-5]/,   :field => :beneficiary_candidate},
+        {:row => :sc,     :version => /^[3-5]/,   :field => [:lender, :lender_candidate]},
+        {:row => :sc1,    :version => /^[3-5]/,   :field => [:treasurer, :authorized]},
+        {:row => :sc2,    :version => /^[3-5]/,   :field => :guarantor},
+        {:row => :sd,     :version => /^[3-5]/,   :field => :creditor},
+        {:row => :se,     :version => /^[3-5]/,   :field => [:payee, :cadidate]},
+        {:row => :sf,     :version => /^[3-5]/,   :field => [:payee, :payee_candidate]},
+        {:row => :f3p,    :version => /^[3-5]/,   :field => :treasurer},
+        {:row => :f3p31,  :version => /^[3-5]/,   :field => :contributor},
+      ]
+      
+      composites.each { |c| combine_components_into_name(c) }
+      components.each { |c| split_name_into_components(c) }
+      
+    end
+    
+    # Converts everything that looks like an FEC-formatted date to a
+    # native Ruby Date object.
+    def dates
+      # only convert fields whose name is date* or *_date*
+      # lots of other things might be 8 digits, and we have to exclude eg 'candidate'
+      t.convert :field => /(^|_)date/ do |value|
+        Date.parse(value) rescue value
+      end
+    end
+    
+    private
+    
+    # Turns "Allred^Ann^Mrs.^III" into "Mrs. Ann Allred III"
+    def self.fix_carrot_names(name)
+      name = name.split("^").reverse
+      # move the suffix to the beginning
+      name.push name.shift if name.size > 3
+      name.join(" ")
+    end
+    
+    # Create a Translation for the given row, version named as :field
+    def combine_components_into_name(composite)
+      raise ArgumentError, "Must pass a :row, :version AND :field" if composite.nil?
+      composite[:field] = [composite[:field]] unless composite[:field].is_a?(Array)
+      
+      composite[:field].each do |field|
+        t.combine(:row => composite[:row], :version => composite[:version],
+                  :field => "#{field}_name") do |row|
+                  
+          # Gather each name_bit from the parsed row, and join it into one value
+          bits = NAME_BITS.collect do |field_name|
+            row.send("#{field}_#{field_name}".to_sym)
+          end
+          bits.compact.join(" ")
+        end
+      end
+    end
+    
+    # Create a Translation for all five name bits, that will strip
+    # out its respective bit from an already-populate composite name field.
+    def split_name_into_components(component)
+      raise ArgumentError, "Must pass a :row, :version AND :field" if component.nil?
+      component[:field] = [component[:field]] unless component[:field].is_a?(Array)
+      
+      component[:field].each do |field|
+        NAME_BITS.zip(PEOPLE_BITS).each do |field_name, people_name|
+          t.combine(:row => component[:row], :version => component[:version],
+                    :field => "#{field}_#{field_name}") do |row|
+          
+            # Grab the original, composite name
+            name = row.send("#{field}_name")
+          
+            unless name.nil?
+              # Fix various name formatting errors
+              name = self.class.fix_carrot_names(name) unless name.index("^").nil?
+          
+              # Extract just the component you want
+              (Fech::Translator::NAME_PARSER.parse(name)[people_name] || "").strip
+            else
+              nil
+            end
+          end
+        end
+      end
+    end
+    
+  end
+  
+end

data/lib/fech/fech_utils.rb

@@ -0,0 +1,76 @@
+# Contains helper functions and static variables used by various
+# Fech classes.
+module FechUtils
+  
+  # All supported row types pointed to regular expressions that will correctly
+  # match that row type in the wild. If multiple matches exist, Fech will match
+  # the longest regex pattern found.
+  ROW_TYPES = {
+    :hdr   => /^hdr$/i,
+    :f1    => /^f1/i,
+    :f1m   => /(^f1m[^a|n])/i,
+    :f2    => /(^f2$)|(^f2[^4])/i,
+    :f24   => /(^f24$)|(^f24[an])/i,
+    :f3    => /^f3[a|n|t]/i,
+    :f3l   => /^f3l[a|n]/i,
+    :f3p   => /(^f3p$)|(^f3p[^s|3])/i,
+    :f3s   => /^f3s/i,
+    :f3p31 => /^f3p31/i,
+    :f3ps  => /^f3ps/i,
+    :f3x   => /(^f3x$)|(^f3x[ant])/i,
+    :f4    => /^f4[na]/i,
+    :f5    => /^f5[na]/i,
+    :f56   => /^f56/i,
+    :f57   => /^f57/i,
+    :f6    => /(^f6$)|(^f6[an])/i,
+    :f65   => /^f65/i,
+    :f7    => /^f7[na]/i,
+    :f76   => /^f76/i,
+    :f9    => /^f9/i,
+    :f91   => /^f91/i,
+    :f92   => /^f92/i,
+    :f93   => /^f93/i,
+    :f94   => /^f94/i,
+    :f99   => /^f99/i,
+    :h1    => /^h1/i,
+    :h2    => /^h2/i,
+    :h3    => /^h3/i,
+    :h4    => /^h4/i,
+    :h5    => /^h5/i,
+    :h6    => /^h6/i,
+    :sa    => /^sa/i,
+    :sb    => /^sb/i,
+    :sc    => /^sc[^1-2]/i,
+    :sc1   => /^sc1/i,
+    :sc2   => /^sc2/i,
+    :sd    => /^sd/i,
+    :se    => /^se/i,
+    :sf    => /^sf/i,
+    :sl    => /^sl/i,
+    :text  => /^text/i,
+  }
+  
+  # Converts symbols and strings to Regexp objects for use in regex-keyed maps.
+  # Assumes that symbols should be matched literally, strings unanchored.
+  # @param [String,Symbol,Regexp] label the object to convert to a Regexp
+  def regexify(label)
+    if label.is_a?(Regexp)
+      Regexp.new(label.source, Regexp::IGNORECASE)
+    elsif label.is_a?(Symbol)
+      if ROW_TYPES.keys.include?(label)
+        ROW_TYPES[label]
+      else
+        Regexp.new("^#{label.to_s}$", Regexp::IGNORECASE)
+      end
+    else
+      Regexp.new(Regexp.escape(label.to_s), Regexp::IGNORECASE)
+    end
+  end
+  
+  # get symbol (eg :f1) based on the literal row type returned (eg 'f1nwhatever')
+  # this is used to pick which table to store it in, since it's one table per base type (not per equivalent variant)
+  def base_type row_type
+    ROW_TYPES.sort{|x,y| y[1].to_s.length <=> x[1].to_s.length }.find{|k,v| k if row_type =~ v }.first
+  end
+
+end