offensivepolitics-fechell 0.1.0

data/README.rdoc

@@ -0,0 +1,25 @@
+= FECHell
+
+Parsing a loading electronically filed fundraising reports from the Federal Election Commission. FECHell uses format definition files from Watchdog.net (http://watchdog.net/data/crawl/fec/electronic/headers/). 
+
+== Install
+
+sudo gem install offensivepolitics-fechell
+
+== Usage
+
+  require 'rubygems'
+  require 'fechell'
+
+  h = FECHell.new
+
+  h.load('path_to.fec') do |line|
+	  schedule = line[0]
+	  values = line[1]
+	  puts "line is a schedule #{schedule} with keys #{values.keys.join(',')}"
+  end
+
+== Todo
+ - Support v3.0 electronic filing
+ - Better error checking on failed schedule identification.
+ - Convert Watchdog.net CSV files into ruby classes

data/lib/defs/3.00.csv

@@ -0,0 +1 @@
+F1;FORM TYPE;FILER FEC CMTE ID ;COMMITTEE NAME;STREET 1;STREET 2;CITY;STATE;ZIP;DATE (Submitted);CHG OF COMMITTEE NAME;CHG OF ADDRESS;5. COMMITTEE TYPE;5. FEC CANDIDATE ID NUMBER;5. CANDIDATE NAME;5. CAN/OFFICE ;5. CAN/STATE;5. CAN/DIST;5. PARTY CODE;5. PARTY TYPE;6. FEC COMMITTEE ID NUMBER;6. COMMITTEE NAME (Affiliated);6. STREET 1;6. STREET 2;6. CITY;6. STATE;6. ZIP;6. RELATIONSHIP (w/ Above Cmte);6. ORGANIZATION TYPE;7. IND/NAME  (Custodian Name);7. STREET 1;7. STREET 2;7. CITY;7. STATE;7. ZIP;7. TITLE;7. TELEPHONE;8. IND/NAME  (Treasurer);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;8. IND/NAME  (Designated Agent);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;9. IND/NAME  (Bank/Depository);9. STREET 1;9. STREET 2;9. CITY;9. STATE;9. ZIP;NAME/TREASURER (as signed);DATE  (Signed);;COMMITTEE EMAIL;COMMITTEE WEB URL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

data/lib/defs/5.00.csv

@@ -0,0 +1 @@
+F1;FORM TYPE;FILER FEC CMTE ID ;;COMMITTEE NAME;STREET 1;STREET 2;CITY;STATE;ZIP;;DATE (Submitted);;CHG OF COMMITTEE NAME;CHG OF ADDRESS;;5. COMMITTEE TYPE;;5. FEC CANDIDATE ID NUMBER;5. CANDIDATE NAME;5. CAN/OFFICE ;5. CAN/STATE;5. CAN/DIST;5. PARTY CODE;5. PARTY TYPE;;6. FEC COMMITTEE ID NUMBER;6. COMMITTEE NAME (Affiliated);6. STREET 1;6. STREET 2;6. CITY;6. STATE;6. ZIP;6. RELATIONSHIP (w/ Above Cmte);6. ORGANIZATION TYPE;;7. IND/NAME  (Custodian Name);7. STREET 1;7. STREET 2;7. CITY;7. STATE;7. ZIP;7. TITLE;7. TELEPHONE;;8. IND/NAME  (Treasurer);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;;8. IND/NAME  (Designated Agent);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;;9. IND/NAME  (Bank/Depository);9. STREET 1;9. STREET 2;9. CITY;9. STATE;9. ZIP;;NAME/TREASURER (as signed);DATE  (Signed);;COMMITTEE EMAIL;COMMITTEE WEB URL;;COMMITTEE FAX NUMBER;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

data/lib/defs/5.1.csv

@@ -0,0 +1 @@
+F1;FORM TYPE;FILER FEC CMTE ID ;;COMMITTEE NAME;STREET 1;STREET 2;CITY;STATE;ZIP;;DATE (Submitted);;CHG OF COMMITTEE NAME;CHG OF ADDRESS;;5. COMMITTEE TYPE;;5. FEC CANDIDATE ID NUMBER;5. CANDIDATE NAME;5. CAN/OFFICE ;5. CAN/STATE;5. CAN/DIST;5. PARTY CODE;5. PARTY TYPE;;6. FEC COMMITTEE ID NUMBER;6. COMMITTEE NAME (Affiliated);6. STREET 1;6. STREET 2;6. CITY;6. STATE;6. ZIP;6. RELATIONSHIP (w/ Above Cmte);6. ORGANIZATION TYPE;;7. IND/NAME  (Custodian Name);7. STREET 1;7. STREET 2;7. CITY;7. STATE;7. ZIP;7. TITLE;7. TELEPHONE;;8. IND/NAME  (Treasurer);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;;8. IND/NAME  (Designated Agent);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;;9. IND/NAME  (Bank/Depository);9. STREET 1;9. STREET 2;9. CITY;9. STATE;9. ZIP;;NAME/TREASURER (as signed);DATE  (Signed);;COMMITTEE EMAIL;COMMITTEE WEB URL;;COMMITTEE FAX NUMBER;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

data/lib/defs/5.2.csv

@@ -0,0 +1 @@
+F1;FORM TYPE;FILER FEC CMTE ID ;COMMITTEE NAME;STREET 1;STREET 2;CITY;STATE;ZIP;DATE (Submitted);CHG OF COMMITTEE NAME;CHG OF ADDRESS;5. COMMITTEE TYPE;5. FEC CANDIDATE ID NUMBER;5. CANDIDATE NAME;5. CAN/OFFICE ;5. CAN/STATE;5. CAN/DIST;5. PARTY CODE;5. PARTY TYPE;6. FEC COMMITTEE ID NUMBER;6. COMMITTEE NAME (Affiliated);6. STREET 1;6. STREET 2;6. CITY;6. STATE;6. ZIP;6. RELATIONSHIP (w/ Above Cmte);6. ORGANIZATION TYPE;7. IND/NAME  (Custodian Name);7. STREET 1;7. STREET 2;7. CITY;7. STATE;7. ZIP;7. TITLE;7. TELEPHONE;8. IND/NAME  (Treasurer);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;8. IND/NAME  (Designated Agent);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;9. IND/NAME  (Bank/Depository);9. STREET 1;9. STREET 2;9. CITY;9. STATE;9. ZIP;NAME/TREASURER (as signed);DATE  (Signed);COMMITTEE EMAIL;COMMITTEE WEB URL;COMMITTEE FAX NUMBER;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

data/lib/defs/5.3.csv

@@ -0,0 +1 @@
+F1;FORM TYPE;FILER FEC CMTE ID ;COMMITTEE NAME;STREET 1;STREET 2;CITY;STATE;ZIP;DATE (Submitted);CHG OF COMMITTEE NAME;CHG OF ADDRESS;5. COMMITTEE TYPE;5. FEC CANDIDATE ID NUMBER;5. CANDIDATE NAME;5. CAN/OFFICE ;5. CAN/STATE;5. CAN/DIST;5. PARTY CODE;5. PARTY TYPE;6. FEC COMMITTEE ID NUMBER;6. COMMITTEE NAME (Affiliated);6. STREET 1;6. STREET 2;6. CITY;6. STATE;6. ZIP;6. RELATIONSHIP (w/ Above Cmte);6. ORGANIZATION TYPE;7. IND/NAME  (Custodian Name);7. STREET 1;7. STREET 2;7. CITY;7. STATE;7. ZIP;7. TITLE;7. TELEPHONE;8. IND/NAME  (Treasurer);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;8. IND/NAME  (Designated Agent);8. STREET 1;8. STREET 2;8. CITY;8. STATE;8. ZIP;8. TITLE;8. TELEPHONE;9. IND/NAME  (Bank/Depository);9. STREET 1;9. STREET 2;9. CITY;9. STATE;9. ZIP;NAME/TREASURER (as signed);DATE  (Signed);COMMITTEE EMAIL;COMMITTEE WEB URL;COMMITTEE FAX NUMBER;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

data/lib/defs/6.1.csv

@@ -0,0 +1 @@
+F1;FORM TYPE;FILER COMMITTEE ID NUMBER;CHANGE OF COMMITTEE NAME;COMMITTEE NAME;CHANGE OF ADDRESS;STREET 1;STREET 2;CITY;STATE;ZIP;COMMITTEE EMAIL;COMMITTEE WEB URL;COMMITTEE FAX NUMBER;SUBMISSION DATE;SIGNATURE LAST NAME;SIGNATURE FIRST NAME;SIGNATURE MIDDLE NAME;SIGNATURE PREFIX;SIGNATURE SUFFIX;DATE SIGNED;5. COMMITTEE TYPE;5. CANDIDATE ID NUMBER;5. CANDIDATE LAST NAME;5. CANDIDATE FIRST NAME;5. CANDIDATE MIDDLE NAME;5. CANDIDATE PREFIX;5. CANDIDATE SUFFIX;5. CANDIDATE OFFICE ;5. CANDIDATE STATE;5. CANDIDATE DISTRICT;5. PARTY CODE;5. PARTY TYPE;7. CUSTODIAN LAST NAME;7. CUSTODIAN FIRST NAME;7. CUSTODIAN MIDDLE NAME;7. CUSTODIAN PREFIX;7. CUSTODIAN SUFFIX;7. CUSTODIAN STREET 1;7. CUSTODIAN STREET 2;7. CUSTODIAN CITY;7. CUSTODIAN STATE;7. CUSTODIAN ZIP;7. CUSTODIAN TITLE;7. CUSTODIAN TELEPHONE;8. TREASURER LAST NAME;8. TREASURER FIRST NAME;8. TREASURER MIDDLE NAME;8. TREASURER PREFIX;8. TREASURER SUFFIX;8. TREASURER STREET 1;8. TREASURER STREET 2;8. TREASURER CITY;8. TREASURER STATE;8. TREASURER ZIP;8. TREASURER TITLE;8. TREASURER TELEPHONE;6. AFFILIATED CMTTE ID NUM;6. AFFILIATED CMTTE NAME;6. AFFILIATED STREET 1;6. AFFILIATED STREET 2;6. AFFILIATED CITY;6. AFFILIATED STATE;6. AFFILIATED ZIP;"6. RELATIONSHIP (with Filing Committe named in field #4)";6. ORGANIZATION TYPE;8. AGENT LAST NAME;8. AGENT FIRST NAME;8. AGENT MIDDLE NAME;8. AGENT PREFIX;8. AGENT SUFFIX;8. AGENT STREET 1;8. AGENT STREET 2;8. AGENT CITY;8. AGENT STATE;8. AGENT ZIP;8. AGENT TITLE;8. AGENT TELEPHONE;9. a) BANK NAME;9. a) BANK STREET 1;9. a) BANK STREET 2;9. a) BANK CITY;9. a) BANK STATE;9. a) BANK ZIP;9. b) BANK NAME;9. b) BANK STREET 1;9. b) BANK STREET 2;9. b) BANK CITY;9. b) BANK STATE;9. b) BANK ZIP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

data/lib/defs/6.2.csv

@@ -0,0 +1 @@
+F1;FORM TYPE;FILER COMMITTEE ID NUMBER;CHANGE OF COMMITTEE NAME;COMMITTEE NAME;CHANGE OF ADDRESS;STREET 1;STREET 2;CITY;STATE;ZIP;COMMITTEE EMAIL;COMMITTEE WEB URL;COMMITTEE FAX NUMBER;SUBMISSION DATE;SIGNATURE LAST NAME;SIGNATURE FIRST NAME;SIGNATURE MIDDLE NAME;SIGNATURE PREFIX;SIGNATURE SUFFIX;DATE SIGNED;5. COMMITTEE TYPE;5. CANDIDATE ID NUMBER;5. CANDIDATE LAST NAME;5. CANDIDATE FIRST NAME;5. CANDIDATE MIDDLE NAME;5. CANDIDATE PREFIX;5. CANDIDATE SUFFIX;5. CANDIDATE OFFICE ;5. CANDIDATE STATE;5. CANDIDATE DISTRICT;5. PARTY CODE;5. PARTY TYPE;5 (e). ORGANIZATION TYPE;5 (f). LEADERSHIP PAC;6. AFFILIATED CMTTE ID NUM;6. AFFILIATED CMTTE NAME;6. AFFILIATED STREET 1;6. AFFILIATED STREET 2;6. AFFILIATED CITY;6. AFFILIATED STATE;6. AFFILIATED ZIP;6. AFFILIATED RELATIONSHIP CODE (with Filing Committe named in field #4);7. CUSTODIAN LAST NAME;7. CUSTODIAN FIRST NAME;7. CUSTODIAN MIDDLE NAME;7. CUSTODIAN PREFIX;7. CUSTODIAN SUFFIX;7. CUSTODIAN STREET 1;7. CUSTODIAN STREET 2;7. CUSTODIAN CITY;7. CUSTODIAN STATE;7. CUSTODIAN ZIP;7. CUSTODIAN TITLE;7. CUSTODIAN TELEPHONE;8. TREASURER LAST NAME;8. TREASURER FIRST NAME;8. TREASURER MIDDLE NAME;8. TREASURER PREFIX;8. TREASURER SUFFIX;8. TREASURER STREET 1;8. TREASURER STREET 2;8. TREASURER CITY;8. TREASURER STATE;6. RELATIONSHIP (with Filing Committe named in field #4);8. TREASURER ZIP;6. ORGANIZATION TYPE;8. TREASURER TITLE;8. TREASURER TELEPHONE;8. AGENT LAST NAME;8. AGENT FIRST NAME;8. AGENT MIDDLE NAME;8. AGENT PREFIX;8. AGENT SUFFIX;8. AGENT STREET 1;8. AGENT STREET 2;8. AGENT CITY;8. AGENT STATE;8. AGENT ZIP;8. AGENT TITLE;8. AGENT TELEPHONE;9. a) BANK NAME;9. a) BANK STREET 1;9. a) BANK STREET 2;9. a) BANK CITY;9. a) BANK STATE;9. a) BANK ZIP;9. b) BANK NAME;9. b) BANK STREET 1;9. b) BANK STREET 2;9. b) BANK CITY;9. b) BANK STATE;9. b) BANK ZIP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

data/lib/fechell.rb

@@ -0,0 +1,196 @@
+require "rubygems"
+require "fastercsv"
+
+class FECHell
+
+	@@modules = nil
+
+	SEPERATORS = {"3.00" => ",",
+								"5.00" => ",",
+								"5.1" => ",",
+								"5.2" => ",",
+								"5.3" => ",",
+								"6.1" => "\x1e",
+								"6.2" => "\x1e"
+							 }
+				
+	def self.load_modules
+  	@@modules = {}
+		Dir["#{File.dirname(__FILE__)}/defs/*.csv"].each do |filename|
+			version = File.basename(filename,".csv")
+			
+			formats = {}
+
+			FasterCSV.open(filename,"r",:col_sep => ';').each do |line|
+				schedule = line[0].gsub(' ','').gsub('Sch','S')
+				formats[schedule] = line[1..-1]				
+			end
+
+			@@modules[version] = formats
+
+		end
+	end
+
+	def initialize
+		FECHell.load_modules unless @@modules.nil? == false
+	end
+
+
+	def peek_format(data)
+
+		header = {}
+		parts_raw = []
+		first_line = []
+		form_type = ''
+
+		lines = []
+
+		begin
+
+			if data.is_a?(Array) then
+				lines = data[0..1]
+			else
+				if data.is_a?(StringIO) then
+					file = data
+				elsif data.is_a?(String)
+					file = File.open(data,"r")
+				end
+				lines[0] = file.readline
+				lines[1] = file.readline
+				file.close
+			end
+			
+			line = lines[0]
+
+			seperator = guess_seperator(lines[0])
+			parts_raw = FasterCSV.parse(line,:col_sep => seperator)[0]
+
+			header[:record_type] = parts_raw[0]
+			header[:ef_type] = parts_raw[1]
+			header[:fec_version] = parts_raw[2]
+			header[:software_name] = parts_raw[3]
+			header[:software_version] = parts_raw[4]
+			header[:report_id] = parts_raw[5]
+			header[:report_number] = parts_raw[6]
+
+			line = lines[1]
+			parts_raw = FasterCSV.parse(line,:col_sep => seperator)[0]
+
+			form_type = parts_raw[0]
+			parts_raw.shift
+			
+		rescue FasterCSV::MalformedCSVError
+			header = {}
+			parts_raw = []
+			first_line = []
+			form_type = ''
+		end	
+		return seperator,header,form_type,parts_raw
+	end
+
+	def guess_seperator(first_line)
+		seperator = ","
+		if first_line.index(28).nil? == false then
+			seperator = "\x1c"
+		end
+
+		seperator
+	end
+
+	def fields_for(version,schedule)
+		@@modules[version][schedule.upcase]
+	end
+
+	def guess_schedule(version,full_schedule)
+		schedules = []
+
+		# 1st we look for an exact match
+		# otherwise we look for a partial match, taking into account appended returns
+		full_schedule.upcase!
+
+		if @@modules[version].keys.index(full_schedule)
+			schedules << full_schedule
+		else
+			@@modules[version].keys.each do |key|
+				if full_schedule[0...1] == 'F' # F[number] can have an 'N','A' at the end
+					regex = "^#{key}[ANT]?"
+				else # TEXT or Schedules. S[Letter][numbers]
+					regex = "^#{key}"
+				end
+				if full_schedule.match(regex)
+					schedules << key
+				end
+			end
+		end
+
+    # return the longest match first
+		schedules.sort { |l,r| r.size <=> l.size }
+	end
+
+	def header_lines(filename)
+		seperator,header,form_type,elements = peek_format(filename)	
+
+		csv = FasterCSV.open(filename,"r",:col_sep => seperator)
+		
+		schedules = guess_schedule(header[:fec_version],form_type)
+		
+		line = csv.readline
+		line = csv.readline
+		schedule = line[0]
+		schedules1,values1 = process_line(header[:fec_version],schedule,line)
+		
+		csv.close
+		return header[:fec_version],form_type,schedules1[0],values1
+
+	end
+
+	def process_line(fec_version,schedule,line)
+		guesses = guess_schedule(fec_version,schedule)		
+
+		offsets = @@modules[fec_version][guesses[0]]
+		values = {}
+		index =0
+		offsets.each do |offset|
+			values[offset] = line[index]
+			index = index + 1
+		end
+
+		return guesses,values
+	end
+
+	def process(filename,options)
+			
+			seperator,header,form_type,elements = peek_format(filename)		
+
+			schedules = guess_schedule(header[:fec_version],form_type)
+			if schedules.size == 0 then
+				puts "ERROR: #{filename} - type was #{form_type} we found nothing"
+			end
+			offsets = @@modules[header[:fec_version]][schedules[0]]
+			begin 
+				FasterCSV.open(filename,:col_sep => seperator).each do |line|
+					next if line.nil?
+					next if line.size == 0
+					sch = line[0]
+
+					next unless sch.match('^[STF]')
+
+					guesses,values = process_line(header[:fec_version],sch,line)
+
+					next if guesses.nil? || guesses.size == 0
+					next if values.nil?
+
+					if options[:ignore_schedules]
+						match_str = "^#{options[:ignore_schedules].join('|')}"
+						next if sch.match(match_str)
+					end
+
+					yield [guesses[0],values]
+
+				end
+			rescue FasterCSV::MalformedCSVError
+				puts "malformed content in file #{filename}"					
+			end			
+	end
+
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification 
+name: offensivepolitics-fechell
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Jason Holt
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-04-25 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Parse electronically filed FEC reports.
+email: jjh@offensivepolitics.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- lib/defs/3.00.csv
+- lib/defs/5.00.csv
+- lib/defs/5.1.csv
+- lib/defs/5.2.csv
+- lib/defs/5.3.csv
+- lib/defs/6.1.csv
+- lib/defs/6.2.csv
+- lib/fechell.rb
+- README.rdoc
+has_rdoc: true
+homepage: http://offensivepolitics.net/fechell
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Fechell
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+  version: 
+requirements: []
+
+rubyforge_project: fechell
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Parse electronically filed FEC reports.
+test_files: []
+