fech 0.1.0

data/spec/fech_utils_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe FechUtils do
+
+  describe ".regexify" do
+    
+    it "should convert symbols to anchored regular expressions" do
+      regex = Fech.regexify(:date_coverage_from)
+      regex.should == /^date_coverage_from$/i
+    end
+    
+    it "should convert strings to unanchored regular expressions" do
+      regex = Fech.regexify("date_coverage_from")
+      regex.should == /date_coverage_from/i
+    end
+    
+    it "should produce case-insensitive regular expressions" do
+      regex = Fech.regexify(:date_coverage_from)
+      regex.match("Date_coverage_FROM").size.should == 1
+    end
+    
+    it "should return a custom regex if passed a row symbol defined in ROW_TYPES" do
+      regex = Fech.regexify(:f3p)
+      regex.should == FechUtils::ROW_TYPES[:f3p]
+    end
+    
+  end
+
+end

data/spec/filing_spec.rb

@@ -0,0 +1,251 @@
+require 'spec_helper'
+
+describe Fech::Filing do
+  
+  before do
+    @filing = Fech::Filing.new(723604)
+    @filing.stubs(:file_path).returns(File.join(File.dirname(__FILE__), 'data', '723604.fec'))
+  end
+  
+  describe "#filing_version" do
+    
+    it "should return the correct filing version" do
+      @filing.send(:filing_version).should == "7.0"
+    end
+    
+    it "should parse the file only once" do
+      @filing.expects(:parse_filing_version).once.returns("7.0")
+      @filing.send(:filing_version)
+      @filing.send(:filing_version)
+    end
+    
+  end
+  
+  describe ".hash_zip" do
+    
+    it "should zip the given keys and values into a hash" do
+      @filing = Fech::Filing.new(723604)
+      keys = [:one,:three,:two]
+      values = [1, 3, 2]
+      @filing.hash_zip(keys, values).should == {:one => 1, :two => 2, :three => 3}
+    end
+    
+  end
+  
+  describe "#summary" do
+    
+    it "should return the mapped summary row" do
+      sum = @filing.summary
+      sum.should be_a_kind_of(Hash)
+      sum[:form_type].should == "F3PN"
+    end
+  end
+  
+  describe "#mappings" do
+    
+    it "should create a new Mappings instance with correct version" do
+      @filing.send(:mappings).version.should == "7.0"
+    end
+    
+    it "should memoize itself" do
+      mapping = Fech::Mappings.new
+      Fech::Mappings.expects(:new).once.returns(mapping)
+      @filing.send(:mappings)
+      @filing.send(:mappings)
+    end
+    
+  end
+  
+  describe "#rows_like" do
+    
+    it "should return only rows matching the specified regex" do
+      @filing.rows_like(/^sa/).size.should == 1
+      @filing.rows_like(/^s/).size.should == 2
+      @filing.rows_like(/^sc/).size.should == 0
+    end
+    
+    it "should return an array if no block is given" do
+      @filing.rows_like(/^s/).class.should == Array
+    end
+    
+    it "should return empty array if no matches found" do
+      @filing.rows_like(/^sc/).should == []
+    end
+    
+    it "should yield hashes of row values if passed a block" do
+      @filing.rows_like(/^sa/) do |c|
+        c.should be_a_kind_of(Hash)
+      end
+    end
+    
+    it "should allow case-insensitive string input" do
+      @filing.rows_like("Sa17a").size.should be > 0
+    end
+    
+  end
+  
+  describe "#parse_row?" do
+    
+    before do
+      f = open(@filing.file_path, 'r')
+      f.readline
+      @row = f.readline.split(@filing.delimiter)
+      f.close
+    end
+    
+    it "should return the mapped row" do
+      @filing.send(:parse_row?, @row).should be_a_kind_of(Hash)
+    end
+    
+    describe "when :parse_if is specified" do
+      
+      it "should return the mapped row if :parse_if matches row type" do
+        @filing.send(:parse_row?, @row, {:parse_if => /^f3p/}).should be_a_kind_of(Hash)
+      end
+      
+      it "should return false if row was skipped" do
+        @filing.send(:parse_row?, @row, {:parse_if => /^sa/}).should == false
+      end
+      
+    end
+    
+    it "should return the raw row data if :raw is true" do
+      @filing.send(:parse_row?, @row, {:raw => true}).class.should == Array
+    end
+    
+  end
+  
+  describe "#map_for" do
+    
+    it "should return the correct map for given row type" do
+      map = @filing.map_for(/sa/)
+      map.class.should == Array
+      map.first.should == :form_type
+    end
+    
+    it "should raise error if no map is found" do
+      lambda { @filing.map_for(/sz/) }.should raise_error
+    end
+    
+  end
+  
+  describe ".map_for" do
+    
+    it "should return the correct map for given row type" do
+      map = Fech::Filing.map_for(/sa/)
+      map.class.should == Array
+      map.first.should == :form_type
+    end
+    
+    it "should raise error if no map is found" do
+      lambda { Fech::Filing.map_for(/sz/) }.should raise_error
+    end
+    
+    it "should allow choice of version" do
+      v7 = Fech::Filing.map_for(/sa/, :version => 7.0)
+      v6 = Fech::Filing.map_for(/sa/, :version => 6.1)
+      v6.should_not == v7
+    end
+    
+  end
+  
+  describe "#map" do
+    
+    before do
+      f = open(@filing.file_path, 'r')
+      f.readline
+      @f3p_row = f.readline.split(@filing.delimiter)
+      @sa_row = f.readline.split(@filing.delimiter)
+      f.close
+    end
+    
+    it "should map the data in row to named values according to row_map" do
+      row_map = @filing.send(:mappings).for_row(@sa_row.first)
+      mapped = @filing.send(:map, @sa_row)
+      mapped.should be_a_kind_of(Hash)
+      
+      mapped[:form_type].should == "SA17A"
+      mapped[:contributor_state].should == "SC"
+    end
+    
+    it "should perform conversion translations" do
+      row_map = @filing.send(:mappings).for_row(@sa_row.first)
+      @filing.translate do |t|
+        t.convert(:row => @sa_row.first, :field => :contribution_date) do |v|
+          Date.parse(v)
+        end
+      end
+      mapped = @filing.send(:map, @sa_row)
+      mapped[:contribution_date].should == Date.parse("20110322")
+    end
+    
+    it "should perform combination translations" do
+      @filing.translate do |t|
+        t.combine(:row => @f3p_row.first, :field => :net_individual_contributions) do |row|
+          row[:col_a_17_a_iii_individual_contribution_total].to_f - row[:col_a_28_a_individuals].to_f
+        end
+      end
+      mapped = @filing.send(:map, @f3p_row)
+      mapped[:net_individual_contributions].should == mapped[:col_a_17_a_iii_individual_contribution_total].to_f - mapped[:col_a_28_a_individuals].to_f
+    end
+    
+    it "should return only field asked for if :include was specified" do
+      fields = [:form_type, :filer_committee_id_number, :transaction_id]
+      mapped = @filing.send(:map, @sa_row, :include => fields)
+      fields.each do |field|
+        mapped[field].should_not be_nil
+      end
+      mapped.size.should == 3
+    end
+    
+  end
+  
+  describe "amendments" do
+    
+    before do
+      @filing = Fech::Filing.new(723604)
+      @filing.stubs(:file_path).returns(File.join(File.dirname(__FILE__), 'data', '723604.fec'))
+    end
+    
+    describe "for non-amending filings" do
+      
+      describe "#amendment?" do
+        it "should return false" do
+          @filing.stubs(:header).returns({:report_id => nil})
+          @filing.amendment?.should == false
+        end
+      end
+      
+    end
+    
+    describe "#amends" do
+      
+      it "should return nil for filings without a report_id in HDR" do
+        @filing.stubs(:header).returns({:report_id => nil})
+        @filing.amends.should == nil
+      end
+  
+      it "should return a filing_id for filings with a report_id in HDR" do
+        @filing.stubs(:header).returns({:report_id => "723603"})
+        @filing.amends.should == "723603"
+      end
+      
+    end
+    
+    describe "#amendment?" do
+  
+      it "should return false for filings without a report_id in HDR" do
+        @filing.stubs(:header).returns({:report_id => nil})
+        @filing.amendment?.should == false
+      end
+  
+      it "should return true for filings with a report_id in HDR" do
+        @filing.stubs(:header).returns({:report_id => "723603"})
+        @filing.amendment?.should == true
+      end
+    
+    end
+    
+  end
+  
+end

data/spec/map_generator_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe Fech::MapGenerator do
+  
+  describe ".generate_row_map_from_file" do
+    
+    before do
+      @mg = Fech::MapGenerator.new
+      @row = "sa"
+      @source_dir = File.join(File.dirname(__FILE__), 'sources', @row + '.csv')
+      Fech::MapGenerator.stubs(:row_map_file).returns(@source_dir)
+    end
+    
+    it "should not raise error on bad data" do
+      lambda {
+        Fech::MapGenerator.generate_row_map_from_file(@source_dir, @row)
+      }.should_not raise_error
+    end
+    
+    it "should skip rows without a canonical name" do
+      row_map = Fech::MapGenerator.generate_row_map_from_file(@source_dir, @row)
+      row_map["^7"].should == [:form_type, :date_coverage_from, :aggregate]
+    end
+    
+    it "should skip columns without a version_type header" do
+      row_map = Fech::MapGenerator.generate_row_map_from_file(@source_dir, @row)
+      row_map.keys.should =~ ["^7", "^6", "^[2-5]"]
+    end
+    
+    it "should not add items where there is no index" do
+      row_map = Fech::MapGenerator.generate_row_map_from_file(@source_dir, @row)
+      row_map["^6"][1].should be_nil
+    end
+    
+  end
+  
+ describe ".convert_header_file_to_row_files" do
+    before do
+      @source_dir = File.join(File.dirname(__FILE__), 'sources')
+    end
+    it "should not raise error" do
+      Fech::MapGenerator.convert_header_file_to_row_files(@source_dir)
+      lambda {
+        Fech::MapGenerator.convert_header_file_to_row_files(@source_dir)
+      }.should_not raise_error
+    end
+  end
+
+end

data/spec/mapped_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe Fech::Mapped do
+  
+  before do
+    @f = Fech::Filing.new(723604)
+    @f.stubs(:file_path).returns(File.join(File.dirname(__FILE__), 'data', '723604.fec'))
+  end
+  
+  describe "@filing" do
+    
+    it "should get set to the calling filing" do
+      header = @f.header
+      header.should be_a_kind_of Fech::Mapped
+      header.filing.should == @f
+    end
+    
+  end
+  
+  describe "[]" do
+    
+    it "should obey aliases in the filing's translator" do
+      @f.translate do |t|
+        t.alias :version, :fec_version, "hdr"
+      end
+      
+      header = @f.header
+      header[:version].should == header[:fec_version]
+    end
+    
+    it "should detect the most recent alias first" do
+      @f.translate do |t|
+        t.alias :version, :fec_version, "hdr"
+        t.alias :version, :report_id, "hdr"
+      end
+      
+      header = @f.header
+      header[:version].should_not == header[:fec_version]
+      header[:version].should == header[:report_id]
+    end
+    
+  end
+  
+end

data/spec/mappings_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe Fech::Mappings do
+  
+  describe "#for_row" do
+    
+    before do
+      @mappings = Fech::Mappings.new
+    end
+    
+    it "should return the correct row_map" do
+      @mappings.for_row("sa").should == @mappings.map["^sa"]["^7.0|6.4"]
+      @mappings.for_row("f3p31").should_not == @mappings.for_row("f3p")
+    end
+    
+    it "should use a greedy match on the row type, matching most complete available option" do
+      @mappings.for_row("f3p31").should == @mappings.map[FechUtils::ROW_TYPES[:f3p31].source]["^7.0|6.4|6.3|6.2|6.1"]
+      @mappings.for_row("f3p").should == @mappings.map[FechUtils::ROW_TYPES[:f3p].source]["^7.0"]
+    end
+    
+  end
+  
+  describe ".key_by_regex" do
+    
+    before do
+      @hash = {
+        "^foo$" => :foo,
+        "bar" => :bar
+      }
+    end
+    
+    it "should match a key by regexp" do
+      Fech::Mappings.key_by_regex(@hash, "foo").should == :foo
+      Fech::Mappings.key_by_regex(@hash, "bar").should == :bar
+      Fech::Mappings.key_by_regex(@hash, "foobar").should == :bar
+    end
+    
+    it "raise error if key not found" do
+      expect {
+        Fech::Mappings.key_by_regex(@hash, "oof")
+      }.to raise_error
+    end
+    
+  end
+  
+end

data/spec/sources/F3P.csv

@@ -0,0 +1 @@
+canonical,^7.0,,^6.4|6.3|6.2|6.1,,^5.3|5.2,,^5.1|5.0|3,

data/spec/sources/SchA.csv

@@ -0,0 +1 @@
+canonical,^7.0|6.4,,^6.3|6.2,,^6.1,,^5.3,,^5.2,,^5.1,,^5.0,,^3,

data/spec/sources/SchB.csv

@@ -0,0 +1 @@
+canonical,^7.0|6.4,,^6.3|6.2,,^6.1,,^5.3,,^5.2|5.1,,^5.0,,^3,

data/spec/sources/SchC.csv

@@ -0,0 +1 @@
+canonical,^7.0|6.4|6.3|6.2,,^6.1,,^5.3,,^5.2|5.1,,^5.0|3,

data/spec/sources/headers/3.csv

@@ -0,0 +1 @@
+Header,Record Type,EF Type,FEC Ver,Soft Name,Soft Ver,Name Delim,Rpt ID,Rpt Number,HDRcomment ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

data/spec/sources/headers/5.0.csv

@@ -0,0 +1 @@
+F1,FORM TYPE,FILER FEC CMTE ID ,,COMMITTEE NAME,STREET 1,STREET 2,CITY,STATE,ZIP,,DATE (Submitted),,CHG OF COMMITTEE NAME,CHG OF ADDRESS,,5. COMMITTEE TYPE,,5. FEC CANDIDATE ID NUMBER,5. CANDIDATE NAME,5. CAN/OFFICE ,5. CAN/STATE,5. CAN/DIST,5. PARTY CODE,5. PARTY TYPE,,6. FEC COMMITTEE ID NUMBER,6. COMMITTEE NAME (Affiliated),6. STREET 1,6. STREET 2,6. CITY,6. STATE,6. ZIP,6. RELATIONSHIP (w/ Above Cmte),6. ORGANIZATION TYPE,,7. IND/NAME  (Custodian Name),7. STREET 1,7. STREET 2,7. CITY,7. STATE,7. ZIP,7. TITLE,7. TELEPHONE,,8. IND/NAME  (Treasurer),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,,8. IND/NAME  (Designated Agent),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,,9. IND/NAME  (Bank/Depository),9. STREET 1,9. STREET 2,9. CITY,9. STATE,9. ZIP,,NAME/TREASURER (as signed),DATE  (Signed),,COMMITTEE EMAIL,COMMITTEE WEB URL,,COMMITTEE FAX NUMBER,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

data/spec/sources/headers/5.1.csv

@@ -0,0 +1 @@
+F1,FORM TYPE,FILER FEC CMTE ID ,,COMMITTEE NAME,STREET 1,STREET 2,CITY,STATE,ZIP,,DATE (Submitted),,CHG OF COMMITTEE NAME,CHG OF ADDRESS,,5. COMMITTEE TYPE,,5. FEC CANDIDATE ID NUMBER,5. CANDIDATE NAME,5. CAN/OFFICE ,5. CAN/STATE,5. CAN/DIST,5. PARTY CODE,5. PARTY TYPE,,6. FEC COMMITTEE ID NUMBER,6. COMMITTEE NAME (Affiliated),6. STREET 1,6. STREET 2,6. CITY,6. STATE,6. ZIP,6. RELATIONSHIP (w/ Above Cmte),6. ORGANIZATION TYPE,,7. IND/NAME  (Custodian Name),7. STREET 1,7. STREET 2,7. CITY,7. STATE,7. ZIP,7. TITLE,7. TELEPHONE,,8. IND/NAME  (Treasurer),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,,8. IND/NAME  (Designated Agent),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,,9. IND/NAME  (Bank/Depository),9. STREET 1,9. STREET 2,9. CITY,9. STATE,9. ZIP,,NAME/TREASURER (as signed),DATE  (Signed),,COMMITTEE EMAIL,COMMITTEE WEB URL,,COMMITTEE FAX NUMBER,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

data/spec/sources/headers/5.2.csv

@@ -0,0 +1 @@
+F1,FORM TYPE,FILER FEC CMTE ID ,COMMITTEE NAME,STREET 1,STREET 2,CITY,STATE,ZIP,DATE (Submitted),CHG OF COMMITTEE NAME,CHG OF ADDRESS,5. COMMITTEE TYPE,5. FEC CANDIDATE ID NUMBER,5. CANDIDATE NAME,5. CAN/OFFICE ,5. CAN/STATE,5. CAN/DIST,5. PARTY CODE,5. PARTY TYPE,6. FEC COMMITTEE ID NUMBER,6. COMMITTEE NAME (Affiliated),6. STREET 1,6. STREET 2,6. CITY,6. STATE,6. ZIP,6. RELATIONSHIP (w/ Above Cmte),6. ORGANIZATION TYPE,7. IND/NAME  (Custodian Name),7. STREET 1,7. STREET 2,7. CITY,7. STATE,7. ZIP,7. TITLE,7. TELEPHONE,8. IND/NAME  (Treasurer),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,8. IND/NAME  (Designated Agent),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,9. IND/NAME  (Bank/Depository),9. STREET 1,9. STREET 2,9. CITY,9. STATE,9. ZIP,NAME/TREASURER (as signed),DATE  (Signed),COMMITTEE EMAIL,COMMITTEE WEB URL,COMMITTEE FAX NUMBER,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

data/spec/sources/headers/5.3.csv

@@ -0,0 +1 @@
+F1,FORM TYPE,FILER FEC CMTE ID ,,COMMITTEE NAME,STREET 1,STREET 2,CITY,STATE,ZIP,,DATE (Submitted),,CHG OF COMMITTEE NAME,CHG OF ADDRESS,,5. COMMITTEE TYPE,,5. FEC CANDIDATE ID NUMBER,5. CANDIDATE NAME,5. CAN/OFFICE ,5. CAN/STATE,5. CAN/DIST,5. PARTY CODE,5. PARTY TYPE,,6. FEC COMMITTEE ID NUMBER,6. COMMITTEE NAME (Affiliated),6. STREET 1,6. STREET 2,6. CITY,6. STATE,6. ZIP,6. RELATIONSHIP (w/ Above Cmte),6. ORGANIZATION TYPE,,7. IND/NAME  (Custodian Name),7. STREET 1,7. STREET 2,7. CITY,7. STATE,7. ZIP,7. TITLE,7. TELEPHONE,,8. IND/NAME  (Treasurer),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,,8. IND/NAME  (Designated Agent),8. STREET 1,8. STREET 2,8. CITY,8. STATE,8. ZIP,8. TITLE,8. TELEPHONE,,9. IND/NAME  (Bank/Depository),9. STREET 1,9. STREET 2,9. CITY,9. STATE,9. ZIP,,NAME/TREASURER (as signed),DATE  (Signed),,COMMITTEE EMAIL,COMMITTEE WEB URL,,COMMITTEE FAX NUMBER,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,