fech-ftp 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b1921cd5fbeae654fe3ce6a142914b6c20cce674
+  data.tar.gz: 3390d96ba890e335478fc4f68968f7cb1ee383ee
+SHA512:
+  metadata.gz: dc3c7e63d9c5f0d158e3c83872129f47ce2c79f72d50145f43c61dd50826a1850107b1b68c246e161739c4b17d10456c273d0ed356fcc2619a3f2c6227872628
+  data.tar.gz: 0ba4c5ffb8dc5078525b8b6acd41066abcfdabdd2b9fe4540183c2c25c71508ac55d3e5c1b9cbfabc879b8c558c069cb86d520157d559d3a336d2162b44392df

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in FecFTP.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Derek Willis
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# fech-ftp
+
+A Ruby library for retrieving and parsing [FTP data downloads](http://www.fec.gov/finance/disclosure/ftp_download.shtml) from the Federal Election Commission. While fech-ftp provides an API to some transaction data (contributions from a committee to a candidate and contributions between two committees), its main purpose is to provide a simple interface to the "[committee master](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryCommitteeMaster.shtml)" and "[candidate master](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryCandidateMaster.shtml)" files, with the ultimate goal of providing a way to connect individual transactions parsed by [Fech](https://github.com/NYTimes/Fech) with their canonical recipients.
+
+fech-ftp is tested under Ruby 2.0.0, 2.1.X and 2.2.X.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fech-ftp'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fech-ftp
+
+## Usage
+
+Fech-FTP can be used by itself or in combination with Fech. To retrieve canonical details about candidates for an election cycle, pass the cycle into the following constructor:
+
+```ruby
+require 'fech-ftp'
+cands = Fech::Candidate.detail(2014)
+cands.first # => {:candidate_id=>"H0AK00097", :candidate_name=>"COX, JOHN ROBERT", :party=>"REP", :election_year=>"2012", :office_state=>"AK", :office=>"H", :district=>"00", :incumbent_challenger_status=>"C", :candidate_status=>"N", :committee_id=>"C00525261", :street_one=>"PO BOX 1092", :street_two=>"", :city=>"ANCHOR POINT", :state=>"AK", :zipcode=>"995561092"}
+```
+
+If you want to have the data transferred into an csv, add the property `format: :csv`, like so:
+
+```ruby
+Fech::Candidate.detail(2014, format: :csv)
+```
+
+You can specify the location that the FEC zip files opened by fech-ftp are downloaded by adding the property `location`, with an absolute path to a directory that must include a trailing slash, like so:
+
+```ruby
+Fech::Candidate.detail(2014, location: "/tmp/fec/")
+```
+
+If you are using the [Sequel Gem](https://github.com/jeremyevans/sequel), you can pass in the DB table object as the `connection` property:
+
+```ruby
+Fech::Candidate.detail(2014, format: :db, connection: DB[:candidates])
+```
+
+Please note that it assumes the table object's columns == header properties for the data that gets passed in. Otherwise, an exception will be thrown.
+To get around this, you can provide the `connection` property as an array, with the traditional `DB` constant value being the in the first element, followed by the table name:
+
+```ruby
+Fech::Candidate.detail(2014, format: :db, connection: [<DATABASE OBJECT>, :candidates])
+```
+
+The table will automatically be created, and then will be populated with the selected dataset. Also please note that it assumes there are no foreign keys. To add them, please follow the Sequel documentation guidelines for adding/altering foreign keys.
+
+`Fech::Candidate` and `Fech::Committee` each have multiple datasets available and can be accessed using the corresponding method:
+
+`Candidate` responds to `summary_current, summary_all, detail`
+`Committee` responds to `linkage, summary_all, detail`
+
+The following have only a single dataset but will respond to any of the above methods:
+
+```ruby
+Fech::CandidateContribution
+Fech::IndividualContribution
+Fech::CommitteeContribution
+```
+
+There are additional classes representing [PAC contributions to candidates](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryContributionstoCandidates.shtml) (`CandidateContribution`) and [transactions involving two committees](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryCommitteetoCommittee.shtml) (`CommitteeContribution`). Be advised that both of the FTP files loaded by these classes are large and can take minutes to parse. They are appropriately used for background processing or data loading purposes, not for providing a live API. Individual Contributions in particular runs in excess of 1-2 million rows of data (~ 200mb)
+
+## Contributing
+
+1. Fork it ( http://github.com/dwillis/fech-ftp/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/test_*.rb"
+end

data/fech-ftp.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fech-ftp/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fech-ftp"
+  spec.version       = Fech::Ftp::VERSION
+  spec.authors       = ["Derek Willis"]
+  spec.email         = ["dwillis@gmail.com"]
+  spec.summary       = %q{A Ruby interface for FTP data from the Federal Election Commission.}
+  spec.description   = %q{Retrieve and parse summary and detailed federal campaign finance data.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+  spec.add_dependency "fech"
+  spec.add_dependency "remote_table"
+  spec.add_dependency "american_date"
+  spec.add_dependency "sqlite3"
+  spec.add_dependency "sequel"
+  spec.add_dependency "rubyzip"
+  spec.add_dependency "activesupport"
+end

data/lib/fech-ftp/candidate.rb

@@ -0,0 +1,42 @@
+module Fech
+  class Candidate < Table
+    HEADERS = {
+      detail: {
+        file: 'cn',
+        headers: [
+          :candidate_id, :candidate_name, :party, :election_year,
+          :office_state, :office, :district, :incumbent_challenger_status,
+          :candidate_status, :committee_id, :street_one, :street_two, :city,
+          :state, :zipcode
+        ]
+      },
+      summary_current: {
+        file: 'webl',
+        headers: [
+          :candidate_id, :candidate_name, :status, :party_code, :party,
+          :total_receipts, :transfers_from_authorized, :total_disbursements,
+          :transfers_to_authorized, :beginning_cash, :ending_cash, :candidate_contributions,
+          :candidate_loans, :other_loans, :candidate_loan_repayments, :other_loan_repayments,
+          :debts_owed_by, :total_individual_contributions, :office_state, :district,
+          :special_election_status, :primary_election_status, :runoff_election_status,
+          :general_election_status, :general_election_percent, :pac_contributions,
+          :party_contributions, :coverage_end_date, :individual_refunds, :committee_refunds
+        ]
+      },
+      summary_all: {
+        file: 'weball',
+        headers: [
+          :candidate_id, :candidate_name, :status, :party_code, :party,
+          :total_receipts, :transfers_from_authorized, :total_disbursements,
+          :transfers_to_authorized, :beginning_cash, :ending_cash, :candidate_contributions,
+          :candidate_loans, :other_loans, :candidate_loan_repayments, :other_loan_repayments,
+          :debts_owed_by, :total_individual_contributions, :office_state, :district,
+          :special_election_status, :primary_election_status, :runoff_election_status,
+          :general_election_status, :general_election_percent, :pac_contributions,
+          :party_contributions, :coverage_end_date, :individual_refunds, :committee_refunds
+        ]
+      }
+    }
+  end
+end
+

data/lib/fech-ftp/candidate_contribution.rb

@@ -0,0 +1,13 @@
+module Fech
+  class CandidateContribution < Table
+    HEADERS = {
+      file: "oth",
+      headers: [
+        :committee_id, :amendment, :report_type, :primary_general, :microfilm,
+        :transaction_type, :entity_type, :name, :city, :state, :zipcode,
+        :employer, :occupation, :transaction_date, :amount, :other_committee_id, :candidate_id,
+        :transaction_id, :filing_id, :memo_code, :memo_text, :fec_record_number
+      ]
+    }
+  end
+end

data/lib/fech-ftp/committee.rb

@@ -0,0 +1,35 @@
+module Fech
+  class Committee < Table
+    HEADERS = {
+      detail: {
+        file: 'cm',
+        headers: [
+          :committee_id, :committee_name, :treasurer, :street_one,
+          :street_two,:city, :state, :zipcode, :designation, :type, :party,
+          :filing_frequency, :category, :connected_organization, :candidate_id
+        ]
+      },
+      summary_all: {
+        file: 'webk',
+        headers: [
+          :committee_id, :committee_name, :type, :designation, :filing_frequency,
+          :total_receipts, :transfers_from_affiliates, :individual_contributions,
+          :pac_contributions, :candidate_contributions, :candidate_loans,
+          :total_loans_received, :total_disbursements, :transfers_to_affiliates,
+          :individual_refunds, :committee_refunds, :candidate_loan_repayments,
+          :loan_repayments, :beginning_year_cash, :ending_cash, :debts_owed_by,
+          :nonfederal_transfers_received, :contributions_to_committees,
+          :independent_expenditures, :party_coordinated_expenditures,
+          :nonfederal_share_expenditures, :coverage_end_date
+        ]
+      },
+      linkage: {
+        file: 'ccl',
+        headers: [
+          :committee_id, :candidate_election_year, :election_year,
+          :candidate_id, :type, :designation, :linkage_id
+        ]
+      }
+    }
+  end
+end

data/lib/fech-ftp/committee_contribution.rb

@@ -0,0 +1,15 @@
+module Fech
+  class CommitteeContribution < Table
+    HEADERS = {
+      detail: {
+        file: "oth",
+        headers: [
+          :committee_id, :amendment, :report_type, :primary_general, :microfilm,
+          :transaction_type, :entity_type, :name, :city, :state, :zipcode,
+          :employer, :occupation, :transaction_date, :amount, :other_committee_id,
+          :transaction_id, :filing_id, :memo_code, :memo_text, :fec_record_number
+        ]
+      }
+    }
+  end
+end

data/lib/fech-ftp/individual_contribution.rb

@@ -0,0 +1,13 @@
+module Fech
+  class IndividualContribution < Table
+    HEADERS = {
+      file: 'indiv',
+      headers: [
+        :committee_id, :amendment_indicator, :report_type, :election_type,
+        :image_number, :trans_type, :entity_type, :name, :city, :state, :zip,
+        :employer, :occupation, :trans_date, :trans_amount, :other_id, :trans_id,
+        :filing_id, :memo_cd, :memo_txt, :fec_record_number
+      ]
+    }
+  end
+end

data/lib/fech-ftp/table.rb

@@ -0,0 +1,128 @@
+module Fech
+  class Table
+    def initialize(cycle, opts={})
+      @cycle    = cycle
+      @headers  = opts[:headers]
+      @file     = opts[:file]
+      @format   = opts[:format]
+      @location = opts[:location]
+      @receiver = opts[:connection] || receiver
+      @parser   = parser
+    end
+
+    def receiver
+      if @format == :csv
+        CSV.open("#{@file}#{@cycle.to_s[2..3]}.csv", 'a+', headers: @headers, write_headers: true)
+      else
+        []
+      end
+    end
+
+    def retrieve_data
+      fetch_file { |row| enter_row(row) }
+      return @receiver
+    end
+
+    def enter_row(row)
+      case @format
+      when :db
+        table_exist? ? @receiver << row : create_table(row)
+      when :csv
+        @receiver << row.values
+      else
+        @receiver << row
+      end
+    end
+
+    # the @receiver obj is the database itself.
+    # This assumes the table needs to be created.
+
+    def table_exist?
+      @receiver.respond_to? :columns
+    end
+
+    def create_table(row)
+      db, table = @receiver
+      table = table.to_s.pluralize.to_sym
+      db.create_table(table) { primary_key :id }
+
+      row.each do |k,v|
+        v = v.nil? ? String : v.class
+        db.alter_table table do
+          add_column k, v
+        end
+      end
+
+      @receiver = db[table]
+      @receiver << row
+    end
+
+    def fetch_file(&blk)
+      zip_file = "#{@file}#{@cycle.to_s[2..3]}.zip"
+      Net::FTP.open("ftp.fec.gov") do |ftp|
+        ftp.login
+        ftp.chdir("./FEC/#{@cycle}")
+        begin
+          ftp.get(zip_file, "./#{zip_file}")
+        rescue Net::FTPPermError
+          raise 'File not found - please try the other methods'
+        end
+      end
+
+      unzip(zip_file, &blk)
+    end
+
+    def parser
+      @headers.map.with_index do |h,i|
+        if h.to_s =~ /cash|amount|contributions|total|loan|transfer|debts|refund|expenditure/
+          [h, ->(line) { line[i].to_f }]
+        elsif h == :filing_id
+          [h, ->(line) { line[i].to_i }]
+        elsif h.to_s =~ /_date/
+          [h, ->(line) { parse_date(line[i]) }]
+        else
+          [h, ->(line) { line[i] }]
+        end
+      end
+    end
+
+    def format_row(line)
+      hash = {}
+      line = line.encode('UTF-8', invalid: :replace, replace: ' ').chomp.split("|")
+
+      @parser.each { |k,blk| hash[k] = blk.call(line) }
+
+      return hash
+    end
+
+    def parse_date(date)
+      if date == '' && table_exist?
+        if table_exist?
+          return Date.new(@cycle, 1,1)
+        else
+          return ''
+        end
+      end
+
+      if date.length == 8
+        Date.strptime(date, "%m%d%Y")
+      else
+        Date.parse(date)
+      end
+    end
+
+    def unzip(zip_file, &blk)
+      Zip::File.open(zip_file) do |zip|
+        zip.each do |entry|
+          path = @location.nil? ? entry.name : @location + entry.name
+          entry.extract(path) if !File.file?(path)
+          File.delete(zip_file)
+          File.foreach(path) do |row|
+            blk.call(format_row(row))
+          end
+          File.delete(path)
+        end
+      end
+    end
+  end
+end

data/lib/fech-ftp/table_methods.rb

@@ -0,0 +1,16 @@
+module Fech
+  class Table
+    class_eval do
+      [:detail, :summary_all, :linkage, :summary_current].each do |meth|
+        action = lambda do |cycle, opts={}|
+          attrs = self::HEADERS[meth] || self::HEADERS
+          attrs.merge!(opts)
+          table = new(cycle, attrs)
+          table.retrieve_data
+        end
+
+        define_singleton_method(meth, action)
+      end
+    end
+  end
+end

data/lib/fech-ftp/version.rb

@@ -0,0 +1,5 @@
+module Fech
+  class Ftp
+    VERSION = "0.1.0"
+  end
+end

data/lib/fech-ftp.rb

@@ -0,0 +1,28 @@
+require 'fech'
+require "fech-ftp/version"
+require "fech-ftp/table"
+require "fech-ftp/committee"
+require "fech-ftp/candidate"
+require "fech-ftp/candidate_contribution"
+require "fech-ftp/committee_contribution"
+require "fech-ftp/individual_contribution"
+require "fech-ftp/table_methods"
+require "net/ftp"
+require 'zip'
+require 'active_support'
+require 'active_support/deprecation'
+require 'remote_table'
+require 'american_date'
+require 'sqlite3'
+require 'sequel'
+require 'csv'
+
+module Fech
+  class Utilities
+    def self.superpacs
+      url = "http://www.fec.gov/press/press2011/ieoc_alpha.shtml"
+      t = RemoteTable.new url, :row_xpath => '//table/tr', :column_xpath => 'td', :encoding => 'windows-1252', :headers => %w{ row_id committee_id committee_name filing_frequency}
+      t.entries
+    end
+  end
+end

data/test/cm.txt

@@ -0,0 +1,7 @@
+C00008896|OPEIU LOCAL 153 "VOTE" (VOICE OF THE ELECTORATE) COMMITTEE|RICHARD J LANIGAN|265 WEST 14TH ST.||NEW YORK|NY|10011|U|Q||Q|L|EMPLOYEES INT'L UNION; OFFICE & PROF.|
+C00009282|NORFOLK SOUTHERN CORPORATION GOOD GOVERNMENT FUND|LEDOUX, MARQUE|ONE CONSTITUTION AVE NE||WASHINGTON|DC|20002|B|Q|UNK|M|C|NORFOLK SOUTHERN CORPORATION|
+C00009357|THE RECORDING INDUSTRY ASSOCIATION OF AMERICA POLITICAL ACTION COMMITTEE|GLAZIER, MITCH|1025 F STREET NW|10TH FLOOR|WASHINGTON|DC|20004|B|Q||Q|T|THE RECORDING INDUSTRY ASSOCIATION OF AMERICA, INC.|
+C00009423|LAND O'LAKES, INC., PAC|GRABOW, KAREN|P.O. BOX 64101||ST. PAUL|MN|55164|B|Q||Q|V|LAND O'LAKES, INC.|
+C00009456|3RD CONGRESSIONAL DISTRICT DEMOCRATIC PARTY OF WISCONSIN|UGLAND, GERALD DAVID MR.|2450 RIVER BEND ROAD||PLOVER|WI|544672728|U|Y|DEM|Q||DEMOCRATIC PARTY OF WISCONSIN|
+C00009597|NATIONAL COUNCIL OF COAL LESSORS INC PAC|CRAWFORD, THOMAS|101 CONSTITUTION AVENUE NW|SUITE 900|WASHINGTON|DC|20001|U|Q||A|T|COAL LESSORS INC; NAT'L COUNCIL OF|
+C00277525|JERRY CARROLL "S" COMMITTEE|JERRY CARROLL|PO BOX 9079||STOCKTON|CA|95208|P|S|IND|A|||S2CA00591

data/test/cn.txt

@@ -0,0 +1,10 @@
+H0AK00097|COX, JOHN ROBERT|REP|2012|AK|H|00|C|N|C00525261|PO BOX 1092||ANCHOR POINT|AK|995561092
+H0AL02087|ROBY, MARTHA|REP|2014|AL|H|02|I|C|C00462143|3260 BANKHEAD AVE||MONTGOMERY|AL|361062448
+H0AL05049|CRAMER, ROBERT E "BUD" JR|DEM|2008|AL|H|05|I|P|C00239038|PO BOX 2621||HUNTSVILLE|AL|35804
+H0AL05163|BROOKS, MO|REP|2014|AL|H|05|I|C|C00464149|7610 FOXFIRE DR.||HUNTSVILLE|AL|35802
+H0AL05189|SHEPARD, TAZEWELL|DEM|2010|AL|H|05|C|P|C00477026|303 WILLIAMS AVE|SUITE 1311|HUNTSVILLE|AL|35801
+H0AL06088|COOKE, STANLEY KYLE|REP|2010|AL|H|06|C|N|C00464222|723 CHERRY BROOK ROAD||KIMBERLY|AL|35091
+H0AL07086|SEWELL, TERRI A.|DEM|2014|AL|H|07|I|C|C00458976|P.O. BOX 1964||BIRMINGHAM|AL|35201
+H0AL07094|HILLIARD, EARL FREDERICK JR|DEM|2010|AL|H|07|O|P|C00460410|PO BOX 12804||BIRMINGHAM|AL|35202
+H0AL07177|CHAMBERLAIN, DON|REP|2012|AL|H|07|C|P|C00482059|512 LAPSLEY ST||SELMA|AL|36701
+H0AR01083|CRAWFORD, ERIC ALAN RICK|REP|2014|AR|H|01|I|C|C00462374|34 CR 455||JONESBORO|AR|72404

data/test/itcont.txt

@@ -0,0 +1,9 @@
+C00492371|N|M2|P|11990223949|15|IND|BIRDWELL, BRAD A.|CYPRESS|TX|77429|BIRDWELL|CONTRACTOR|01232011|500||SA17A.4153|714660|||4021720111136021419
+C00492371|N|M2|P|11990223955|15|IND|OVERMAN, JOE E.|PACOFIC|MT|63069|ET SECURITY|PILOT|01232011|250||SA17A.4151|714660|||4021720111136021438
+C00410266|N|M2||11930387712|15|IND|HIGGINS, THOMAS|MALVERN|PA|19355|THE VANGUARD GROUP|ACCOUNTANT|01042011|600||AC9DB5EBA79C54DED937|714732|||4031620111137408276
+C00410266|N|M2||11930387715|15|IND|NORRIS, ELIZABETH A|MALVERN|PA|19355||INFORMATION REQUESTED|01132011|4000||AD2BAAB7CE6234AA585D|714732|||4031620111137408284
+C00410266|N|M2||11930387715|15|IND|NORRIS, JAMES|MALVERN|PA|19355|THE VANGUARD GROUP|MANAGER|01042011|4000||A1B7AADE1665546EB8AF|714732|||4031620111137408285
+C00410266|N|M2||11930387716|15|IND|POLLOCK, RANDALL|MALVERN|PA|19355|VANGUARD GROUP|INVESTMENT COMPANY EXECUTIVE|01122011|250||AC184E2EB48E14A4991F|714732|||4031620111137408288
+C00410266|N|M2||11930387717|15|IND|REED, GLENN|MALVERN|PA|19355|VANGUARD|INVESTMENT COMPANY EXECUTIVE|01042011|4000||AC8B80FC4AF714F49911|714732|||4031620111137408289
+C00410266|N|M2||11930387720|15|IND|WHITAKER, BARRY|MALVERN|PA|19355|VANGUARD|INVESTMENT COMPANY EXECUTIVE|01102011|700||A2F0D333DFECA4369933|714732|||4031620111137408298
+C00109546|N|M2||11990225012|15|IND|BOWERS, KIMBERLY S|SAN ANTONIO|TX|78249|VALERO SERVICES, INC.|EVP & GENERAL COUNSEL|01312011|205||INCA281987|714749|||4031620111137409615

data/test/test_candidate.rb

@@ -0,0 +1,27 @@
+require 'fech-ftp'
+require 'minitest/autorun'
+
+class TestCandidate < MiniTest::Test
+  def setup
+    detail_file = File.readlines('test/cn.txt').to_a
+    candidate_detail = Fech::Candidate.new(2012, headers: Fech::Candidate::HEADERS[:detail][:headers])
+
+    summary_file = File.readlines('test/webl.txt').to_a
+    candidate_summary = Fech::Candidate.new(2012, headers: Fech::Candidate::HEADERS[:summary_current][:headers])
+
+    @detail_row = candidate_detail.format_row(detail_file[3])
+    @summary_row = candidate_summary.format_row(summary_file[3])
+  end
+
+  def test_candidate_id_loads_properly
+    assert @detail_row.fetch(:candidate_id) == "H0AL05163"
+  end
+
+  def test_candidate_dollar_amts_are_floats
+    assert_kind_of Float, @summary_row.fetch(:total_receipts)
+  end
+
+  def test_that_summary_creates_dates
+    assert_equal Date.parse('11/27/2013'), @summary_row.fetch(:coverage_end_date)
+  end
+end

data/test/test_committee.rb

@@ -0,0 +1,36 @@
+require 'fech-ftp'
+require 'minitest/autorun'
+
+class TestCommittee < MiniTest::Test
+  def setup
+    db = Sequel.sqlite # synonymous with Sequel.connect; test assumes the table was not created yet.
+    detail_file = File.readlines('test/cm.txt').to_a
+    detail_head = Fech::Committee::HEADERS[:detail][:headers]
+
+    cm_detail   = Fech::Committee.new(2012, headers: detail_head)
+    @db_test = Fech::Committee.new(2012, headers: detail_head, format: :db, connection: [db, :committee])
+
+    summary_file = File.readlines('test/webk.txt').to_a
+    cm_summary   = Fech::Committee.new(2012, headers: Fech::Committee::HEADERS[:summary_all][:headers])
+
+    @summary_row = cm_summary.format_row(summary_file[1])
+    @detail_row  = cm_detail.format_row(detail_file[1])
+  end
+
+  def test_that_db_option_is_formatted_correctly
+    @db_test.create_table(@summary_row)
+    assert_equal @db_test.table_exist?, true
+  end
+
+  def test_that_committee_detail_is_properly_loaded
+    assert_equal "NORFOLK SOUTHERN CORPORATION GOOD GOVERNMENT FUND", @detail_row.fetch(:committee_name)
+  end
+
+  def test_that_summary_creates_dates
+    assert_equal Date.parse('06/30/2013'), @summary_row.fetch(:coverage_end_date)
+  end
+
+  def test_that_amount_cols_are_floats
+    assert_kind_of Float, @summary_row.fetch(:total_receipts)
+  end
+end

data/test/test_individual.rb

@@ -0,0 +1,27 @@
+require 'fech-ftp'
+require 'minitest/autorun'
+
+class TestIndividual < Minitest::Test
+  def setup
+    indiv_file = File.readlines('test/itcont.txt').to_a
+    indiv_table = Fech::IndividualContribution.new(2012, headers: Fech::IndividualContribution::HEADERS[:headers])
+
+    @row = indiv_table.format_row(indiv_file[0])
+  end
+
+  def test_dollar_amts_are_floats
+    assert_kind_of Float, @row.fetch(:trans_amount)
+  end
+
+  def test_filing_id_are_integers
+    assert_kind_of Integer, @row.fetch(:filing_id)
+  end
+
+  def test_date_parses_properly
+    assert_equal Date.strptime('01232011', "%m%d%Y"), @row.fetch(:trans_date)
+  end
+
+  def test_fec_record_number_matches
+    assert_equal "4021720111136021419", @row.fetch(:fec_record_number)
+  end
+end

data/test/webk.txt

@@ -0,0 +1,9 @@
+C00494757|AMERICA'S FUTURE FUND PAC|N|D|Q|2200|0|2000|200|0|0|0|40002.5|0|0|0|0|0|38504.58|702.08|0|0|5000|0|0|0|09/30/2013
+C00492025|AMERICAN DEFENSE AND MILITARY PAC (ADAM PAC)|N|D|Q|12500|0|0|12500|0|0|0|8757.67|0|0|0|0|0|7841.79|11584.12|0|0|5000|0|0|0|06/30/2013
+C00553271|AMERICAN LEADERSHIP NOW (ALN PAC)|N|D|Q||||||||||||||||||||||
+C00495150|AMERICAN LIBERTY AND NATION PAC (ALAN PAC)|N|D|Q|39200|34200|0|5000|0|0|0|17109.98|0|0|0|0|0|6787.9|28877.92|0|0|17000|0|0|0|06/30/2013
+C00439521|AMERICAN SECURITY PAC|N|D|Q|25000|0|0|25000|0|0|0|25522.06|0|0|0|0|0|3241.23|2719.17|0|0|2500|0|0|0|09/30/2013
+C00527846|ANN MARIE PAC|N|D|Q|0|0|0|0|0|0|0|3307.75|0|0|0|0|0|10703.71|7395.96|0|0|0|0|0|0|06/30/2013
+C00531764|ANN PAC|N|D|Q|67045|0|15045|52000|0|0|0|17636.26|0|0|0|0|0|1000|50408.74|0|0|2000|0|0|0|09/30/2013
+C00540062|ANNIEPAC|N|D|Q|5000|0|0|5000|0|0|0|415|0|0|0|0|0|25|4610|0|0|0|0|0|0|06/30/2013
+C00524777|APAC|N|D|Q|0|0|0|0|0|0|0|10151.53|0|0|0|0|0|10262.8|111.27|0|0|0|0|0|0|06/30/2013

data/test/webl.txt

@@ -0,0 +1,17 @@
+H2AK00101|MOORE, MATTHEW EDWARD|C|1|DEM|15749.5|0|13376.95|0|0|2372.55|0|12000|0|0|0|12100|3741.5|AK|00||||||0|0|09/30/2013|0|0
+H4AK00123|DUNBAR, FORREST|C|1|DEM|21805.01|0|3225|0|0|18580.01|0|0|0|0|0|0|21805.01|AK|00||||||0|0|09/30/2013|0|0
+H6AK00045|YOUNG, DONALD E|I|2|REP|300676.14|0|176268.29|0|508299.46|632707.31|0|0|0|0|0|0|210896.02|AK|00||||||88828.14|0|09/30/2013|1000|0
+H4AL01156|LEFLORE, BURTON|O|1|DEM|12581|0|11276|0|0|1304|4932|0|0|0|0|3081|12581|AL|01||||||0|0|11/27/2013|0|0
+H2AL01077|BONNER, JOSIAH ROBINS JR.|I|2|REP|85681.86|0|207065.54|0|157209.22|35825.54|0|0|0|0|0|0|37650|AL|01||||||48000|0|09/30/2013|750|0
+H2AL01176|YOUNG, LARRY DEAN JR|O|2|REP|260046.54|0|226818.07|0|0|33228.47|0|174500|0|75000|0|99500|85546.54|AL|01||||||0|0|10/16/2013|0|0
+H4AL01123|BYRNE, BRADLEY ROBERTS|I|2|REP|1119799.1|0|983582.56|0|0|136216.54|0|0|0|0|0|75063.68|708698.11|AL|01|W|||||411100.99|0|11/27/2013|3000|2500
+H4AL01131|JAMES, JESSICA|C|2|REP|10113|0|10113|0|0|0|0|0|0|0|0|0|10113|AL|01||||||0|0|12/31/2013|0|0
+H4AL01149|FINCHER, CHAD|O|2|REP|158282.35|0|153133.28|0|0|5149.07|0|10000|0|0|0|10000|138635|AL|01||||||9500|0|09/30/2013|300|0
+H4AL01172|GRIFFITH, PRESTON WELLS III|O|2|REP|209780.99|0|184705.42|0|0|25075.57|0|0|0|0|0|19422.81|196780.99|AL|01||||||13000|0|09/30/2013|0|0
+H4AL01180|POWE, SHARON L|O|2|REP|5450|0|4931|0|5000|519|0|0|0|0|0|0|5450|AL|01||||||0|0|09/12/2013|0|0
+H2AL01192|RAILEY, CURTIS MONROE|C|3|IND|0|0|1150.74|0|2368.57|1217.83|0|0|0|0|0|10000|0|AL|01||||||0|0|06/30/2013|0|0
+H0AL02087|ROBY, MARTHA|I|2|REP|497600.35|31.25|202341.17|0|186496.35|481755.53|0|0|0|0|0|0|250003.52|AL|02||||||243188.7|0|09/30/2013|25|1000
+H2AL03032|ROGERS, MICHAEL|I|2|REP|308853.99|0|257847.29|0|292048.29|343054.99|0|0|0|0|0|0|91328.35|AL|03||||||215390|1000|09/30/2013|0|0
+H2AL07165|NORRIS, PHILLIP DWIGHT|C|2|REP|100|0|50|0|0|50|0|0|0|0|0|0|100|AL|04||||||0|0|12/31/2013|0|0
+H6AL04098|ADERHOLT, ROBERT B. REP.|I|2|REP|504485.15|0|327299.7|0|152881.82|330067.27|0|0|0|0|0|0|192597.05|AL|04||||||311128.84|0|09/30/2013|0|0
+H0AL05163|BROOKS, MO|I|2|REP|188145|0|55273.51|0|499658.59|632530.08|0|0|0|0|0|0|85045|AL|05||||||103100|0|09/30/2013|0|0

metadata

@@ -0,0 +1,215 @@
+--- !ruby/object:Gem::Specification
+name: fech-ftp
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Derek Willis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fech
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: remote_table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: american_date
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Retrieve and parse summary and detailed federal campaign finance data.
+email:
+- dwillis@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fech-ftp.gemspec
+- lib/fech-ftp.rb
+- lib/fech-ftp/candidate.rb
+- lib/fech-ftp/candidate_contribution.rb
+- lib/fech-ftp/committee.rb
+- lib/fech-ftp/committee_contribution.rb
+- lib/fech-ftp/individual_contribution.rb
+- lib/fech-ftp/table.rb
+- lib/fech-ftp/table_methods.rb
+- lib/fech-ftp/version.rb
+- test/cm.txt
+- test/cn.txt
+- test/itcont.txt
+- test/test_candidate.rb
+- test/test_committee.rb
+- test/test_individual.rb
+- test/webk.txt
+- test/webl.txt
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A Ruby interface for FTP data from the Federal Election Commission.
+test_files:
+- test/cm.txt
+- test/cn.txt
+- test/itcont.txt
+- test/test_candidate.rb
+- test/test_committee.rb
+- test/test_individual.rb
+- test/webk.txt
+- test/webl.txt