fech-ftp 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 74b4ad9ebf4adbe59ec021a13408871e0180a6fe
-  data.tar.gz: 2d5a44faf28e8e4bc12d0fa351c0b61651fa17f8
+  metadata.gz: 2497302b726593b2adf7596e8129f7ae35d06a29
+  data.tar.gz: d19312acbe195d7a9f6fb5174b0fb7085b7ac969
 SHA512:
-  metadata.gz: bf7a38d8dd0906992baf77bbe19666b477945a1209895cb8158b07b43fc9db68fba6e386e0a7856e53be830e1f9f0cccea7bbe6b40781def329e135cf11d4392
-  data.tar.gz: 8ed8360b64583bfe81ca70846b60f864e1e4bfe87cf0aaf9208a5cb851d0c6a94ef575b9afdff6a7a278602cb12320baa1ee88a5ea6fa943799707b26b2fe2aa
+  metadata.gz: ed07c1f566fc9f617486c823d38b59012c02aab1b11337eb60f69afd75c6c21aa9472bce7a382982d875d220474edb0fec9e093caeae03497719ff0ed1e1b2b7
+  data.tar.gz: 3c585a2dde2d9c1d841f309627d8aae66d749fb329dc829dd970452ed26235a1585ee9e142be889f9c3c4efd74456117e2bbe6180675aeb2321b8bd262d4e55b

data/README.md

@@ -1,8 +1,10 @@
 # fech-ftp
 
-A Ruby library for retrieving and parsing [FTP data downloads](http://www.fec.gov/finance/disclosure/ftp_download.shtml) from the Federal Election Commission. While fech-ftp provides an API to some transaction data (contributions from a committee to a candidate and contributions between two committees), its main purpose is to provide a simple interface to the "[committee master](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryCommitteeMaster.shtml)" and "[candidate master](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryCandidateMaster.shtml)" files, with the ultimate goal of providing a way to connect individual transactions parsed by [Fech](https://github.com/NYTimes/Fech) with their canonical recipients.
+A Ruby library for retrieving and parsing [bulk data downloads](http://classic.fec.gov/finance/disclosure/ftp_download.shtml) from the Federal Election Commission. While fech-ftp provides an API to some transaction data (contributions from a committee to a candidate and contributions between two committees), its main purpose is to provide a simple interface to the "[committee master](http://classic.fec.gov/finance/disclosure/metadata/DataDictionaryCommitteeMaster.shtml)" and "[candidate master](http://classic.fec.gov/finance/disclosure/metadata/DataDictionaryCandidateMaster.shtml)" files, with the ultimate goal of providing a way to connect individual transactions parsed by [Fech](https://github.com/NYTimes/Fech) with their canonical recipients.
 
-fech-ftp is tested under Ruby 2.0.0, 2.1.X and 2.2.X.
+fech-ftp is tested under Ruby 2.0.0, 2.1.X, 2.2.X, and 2.3.X.
+
+Ironically, the FEC announced in late 2017 that it would be moving its bulk data files from an FTP server to S3. This library has been updated to work with that setup, so it performs exactly zero FTP requests.
 
 ## Installation
 
@@ -24,33 +26,33 @@ Fech-FTP can be used by itself or in combination with Fech. To retrieve canonica
 
 ```ruby
 require 'fech-ftp'
-cands = Fech::Candidate.detail(2014)
-cands.first # => {:candidate_id=>"H0AK00097", :candidate_name=>"COX, JOHN ROBERT", :party=>"REP", :election_year=>"2012", :office_state=>"AK", :office=>"H", :district=>"00", :incumbent_challenger_status=>"C", :candidate_status=>"N", :committee_id=>"C00525261", :street_one=>"PO BOX 1092", :street_two=>"", :city=>"ANCHOR POINT", :state=>"AK", :zipcode=>"995561092"}
+cands = Fech::Candidate.detail(2018)
+cands.first # => {:candidate_id=>"H0AL02087", :candidate_name=>"ROBY, MARTHA", :party=>"REP", :election_year=>"2018", :office_state=>"AL", :office=>"H", :district=>"02", :incumbent_challenger_status=>"I", :candidate_status=>"C", :committee_id=>"C00462143", :street_one=>"3260 BANKHEAD AVE", :street_two=>"", :city=>"MONTGOMERY", :state=>"AL", :zipcode=>"361062448"}
 ```
 
 If you want to have the data transferred into an csv, add the property `format: :csv`, like so:
 
 ```ruby
-Fech::Candidate.detail(2014, format: :csv)
+Fech::Candidate.detail(2018, format: :csv)
 ```
 
 You can specify the location that the FEC zip files opened by fech-ftp are downloaded by adding the property `location`, with an absolute path to an _existing_ directory that must include a trailing slash, like so:
 
 ```ruby
-Fech::Candidate.detail(2014, location: "/tmp/fec/")
+Fech::Candidate.detail(2018, location: "/tmp/fec/")
 ```
 
 If you are using the [Sequel Gem](https://github.com/jeremyevans/sequel), you can pass in the DB table object as the `connection` property:
 
 ```ruby
-Fech::Candidate.detail(2014, format: :db, connection: DB[:candidates])
+Fech::Candidate.detail(2018, format: :db, connection: DB[:candidates])
 ```
 
 Please note that it assumes the table object's columns == header properties for the data that gets passed in. Otherwise, an exception will be thrown.
 To get around this, you can provide the `connection` property as an array, with the traditional `DB` constant value being the in the first element, followed by the table name:
 
 ```ruby
-Fech::Candidate.detail(2014, format: :db, connection: [<DATABASE OBJECT>, :candidates])
+Fech::Candidate.detail(2018, format: :db, connection: [<DATABASE OBJECT>, :candidates])
 ```
 
 The table will automatically be created, and then will be populated with the selected dataset. Also please note that it assumes there are no foreign keys. To add them, please follow the Sequel documentation guidelines for adding/altering foreign keys.
@@ -68,7 +70,12 @@ Fech::IndividualContribution
 Fech::CommitteeContribution
 ```
 
-There are additional classes representing [PAC contributions to candidates](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryContributionstoCandidates.shtml) (`CandidateContribution`) and [transactions involving two committees](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryCommitteetoCommittee.shtml) (`CommitteeContribution`). Be advised that both of the FTP files loaded by these classes are large and can take minutes to parse. They are appropriately used for background processing or data loading purposes, not for providing a live API. Individual Contributions in particular runs in excess of 1-2 million rows of data (~ 200mb)
+There are additional classes representing [PAC contributions to candidates](http://classic.fec.gov/finance/disclosure/metadata/DataDictionaryContributionstoCandidates.shtml) (`CandidateContribution`) and [transactions involving two committees](http://classic.fec.gov/finance/disclosure/metadata/DataDictionaryCommitteetoCommittee.shtml) (`CommitteeContribution`). Be advised that both of the FTP files loaded by these classes are large and can take minutes to parse. They are appropriately used for background processing or data loading purposes, not for providing a live API. Individual Contributions in particular runs in excess of 1-2 million rows of data (~ 200mb)
+
+## Authors
+
+* [Derek Willis](https://github.com/dwillis)
+* [Matt Long](https://github.com/wismer)
 
 ## Contributing
 

data/lib/fech-ftp.rb

@@ -7,7 +7,7 @@ require "fech-ftp/candidate_contribution"
 require "fech-ftp/committee_contribution"
 require "fech-ftp/individual_contribution"
 require "fech-ftp/table_methods"
-require "net/ftp"
+require "net/http"
 require 'zip'
 require 'active_support'
 require 'active_support/deprecation'
@@ -20,7 +20,7 @@ require 'csv'
 module Fech
   class Utilities
     def self.superpacs
-      url = "http://www.fec.gov/press/press2011/ieoc_alpha.shtml"
+      url = "http://classic.fec.gov/press/press2011/ieoc_alpha.shtml"
       t = RemoteTable.new url, :row_xpath => '//table/tr', :column_xpath => 'td', :encoding => 'windows-1252', :headers => %w{ row_id committee_id committee_name filing_frequency}
       t.entries
     end

data/lib/fech-ftp/table.rb

@@ -1,5 +1,7 @@
 module Fech
   class Table
+    AWS_URL = "https://cg-519a459a-0ea3-42c2-b7bc-fa1143481f74.s3-us-gov-west-1.amazonaws.com/bulk-downloads"
+
     def initialize(cycle, opts={})
       @cycle    = cycle
       @headers  = opts[:headers]
@@ -59,19 +61,13 @@ module Fech
     end
 
     def fetch_file(&blk)
-      zip_file = "#{@file}#{@cycle.to_s[2..3]}.zip"
-      Net::FTP.open("ftp.fec.gov") do |ftp|
-        ftp.passive = true if @passive
-        ftp.login
-        ftp.chdir("./FEC/#{@cycle}")
-        begin
-          ftp.get(zip_file, "./#{zip_file}")
-        rescue Net::FTPPermError
-          raise 'File not found - please try the other methods'
-        end
-      end
+      filename = "#{@file}#{@cycle.to_s[2..3]}.zip"
+      uri = URI("#{AWS_URL}/#{@cycle}/#{filename}")
+      response = Net::HTTP.get_response(uri)
 
-      unzip(zip_file, &blk)
+      if response.code == '200'
+        unzip(response.body, &blk)
+      end
     end
 
     def parser
@@ -114,15 +110,14 @@ module Fech
     end
 
     def unzip(zip_file, &blk)
-      Zip::File.open(zip_file) do |zip|
+      Zip::File.open_buffer(zip_file) do |zip|
         zip.each do |entry|
           path = @location.nil? ? entry.name : @location + entry.name
           entry.extract(path) if !File.file?(path)
-          File.delete(zip_file)
+
           File.foreach(path) do |row|
             blk.call(format_row(row))
           end
-          File.delete(path)
         end
       end
     end

data/lib/fech-ftp/version.rb

@@ -1,5 +1,5 @@
 module Fech
   class Ftp
-    VERSION = "0.1.1"
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fech-ftp
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Derek Willis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-13 00:00:00.000000000 Z
+date: 2017-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: A Ruby interface for FTP data from the Federal Election Commission.
@@ -213,4 +213,3 @@ test_files:
 - test/test_individual.rb
 - test/webk.txt
 - test/webl.txt
-has_rdoc: