kevintyll-ssn_validator 1.0.0 → 1.0.1

data/History.txt

@@ -14,5 +14,11 @@
 * 1 major enhancement:
   * Added the death master file to determine if the ssn belongs to the deceased.
 
-* 1 minor enhancement:
+* 1 bug fix:
   * Fixed bug where an error was thrown if the ssn area has not been assigned yet.
+
+== 1.0.1 2009-04-23
+
+* 2 bug fixes:
+  * Fixed dmf mysql load error when single quote is in a data file record.
+  * Fixed ssn_validator:update_data rake task that broke on the last build.

data/Manifest.txt

@@ -25,3 +25,5 @@ test/test_death_master_file_loader.rb
 test/mocks/test/death_master_file_loader.rb
 test/files/test_dmf_initial_load.txt
 test/files/test_dmf_update_load.txt
+test/files/test_dmf_funky_data_load.txt
+test/files/valid_csv_from_funky_data_file.txt

data/lib/ssn_validator/models/death_master_file_loader.rb

@@ -19,7 +19,7 @@ class DeathMasterFileLoader
     raise(ArgumentError, "as_of not specified") unless @file_as_of
     max_as_of = DeathMasterFile.maximum(:as_of)
     raise(ArgumentError, "A more recent file has already been processed.  DB as_of date #{max_as_of}") if  max_as_of && (max_as_of >= @file_as_of.to_date)
-   
+
     if File.exists?(@file_path_or_url)
       @download_file = File.open(@file_path_or_url)
     elsif URI.parse(@file_path_or_url).kind_of?(URI::HTTP)
@@ -45,19 +45,19 @@ class DeathMasterFileLoader
 
   def get_file_from_web
     uri = URI.parse(@file_path_or_url)
-    
+
     request = Net::HTTP::Get.new(uri.request_uri)
     request.basic_auth(SsnValidator::Ntis.user_name,SsnValidator::Ntis.password)
 
     http = Net::HTTP.new(uri.host, uri.port)
     http.use_ssl = (uri.port == 443)
     http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-    
+
     response = http.request(request)
-    
+
     raise(ArgumentError, "Invalid URL: #{@file_path_or_url}") if response.kind_of?(Net::HTTPNotFound)
     raise(ArgumentError, "Authorization Required: Invalid username or password.  Set the variables SsnValidator::Ntis.user_name and SsnValidator::Ntis.password in your environment.rb file.") if response.kind_of?(Net::HTTPUnauthorized)
-    
+
     return response.body
   end
 
@@ -81,7 +81,7 @@ class DeathMasterFileLoader
   # Processes 28 million rows in 23 minutes. Input file 2.6GB output: 2.9GB.
   # Used to convert a packed fixed-length file into csv for mysql import.
   def convert_file_to_csv
-    #packed_file = File.open(file_path)
+
     csv_file = Tempfile.new("dmf")    # create temp file for converted csv formmat.
 
 
@@ -98,37 +98,37 @@ class DeathMasterFileLoader
         @delete_ssns << attributes_hash[:social_security_number]
       else
         # empty field for id to be generated by mysql.
-        newline = "''," +
+        newline = "``," +
           # social_security_number
-        "'#{attributes_hash[:social_security_number]}'," +
+        "`#{attributes_hash[:social_security_number]}`," +
           # last_name
-        "'#{attributes_hash[:last_name]}'," +
+        "`#{attributes_hash[:last_name]}`," +
           # name_suffix
-        "'#{attributes_hash[:name_suffix]}'," +
+        "`#{attributes_hash[:name_suffix]}`," +
           # first_name
-        "'#{attributes_hash[:first_name]}'," +
+        "`#{attributes_hash[:first_name]}`," +
           # middle_name
-        "'#{attributes_hash[:middle_name]}'," +
+        "`#{attributes_hash[:middle_name]}`," +
           # verify_proof_code
-        "'#{attributes_hash[:verify_proof_code]}'," +
+        "`#{attributes_hash[:verify_proof_code]}`," +
           # date_of_death - need YYYY-MM-DD.
-        "'#{attributes_hash[:date_of_death]}'," +
+        "`#{attributes_hash[:date_of_death]}`," +
           # date_of_birth - need YYYY-MM-DD.
-        "'#{attributes_hash[:date_of_birth]}'," +
+        "`#{attributes_hash[:date_of_birth]}`," +
           # state_of_residence - must be code between 01 and 65 or else nil.
-        "'#{attributes_hash[:state_of_residence]}'," +
+        "`#{attributes_hash[:state_of_residence]}`," +
           # last_known_zip_residence
-        "'#{attributes_hash[:last_known_zip_residence]}'," +
+        "`#{attributes_hash[:last_known_zip_residence]}`," +
           # last_known_zip_payment
-        "'#{attributes_hash[:last_known_zip_payment]}'," +
+        "`#{attributes_hash[:last_known_zip_payment]}`," +
           # created_at
-        "'#{timenow}'," +
+        "`#{timenow}`," +
           # updated_at
-        "'#{timenow}'," +
+        "`#{timenow}`," +
           # as_of
-        "'#{attributes_hash[:as_of]}'" +"\n"
+        "`#{attributes_hash[:as_of]}`" +"\n"
 
-        csv_file.puts newline
+        csv_file.syswrite newline
         puts "#{i} records processed." if (i % 25000 == 0) && (i > 0)
       end
     end
@@ -201,9 +201,9 @@ class DeathMasterFileLoader
 
     #This will insert new records, and replace records with existing ssns.
     #This only works because there is a unique index on social_security_number.
-    mysql_command = <<TEXT
-LOAD DATA LOCAL INFILE '#{csv_file.path}' REPLACE INTO TABLE death_master_files FIELDS TERMINATED BY ',' ENCLOSED BY "'" LINES TERMINATED BY '\n';
-TEXT
+    mysql_command = <<-TEXT
+    LOAD DATA LOCAL INFILE '#{csv_file.path}' REPLACE INTO TABLE death_master_files FIELDS TERMINATED BY ',' ENCLOSED BY "`" LINES TERMINATED BY '\n';
+    TEXT
 
     DeathMasterFile.connection.execute(mysql_command)
     puts "Mysql import complete."

data/lib/ssn_validator.rb

@@ -12,5 +12,5 @@ require 'rake'
 import "#{File.dirname(__FILE__)}/tasks/ssn_validator.rake"
 
 module SsnValidator
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

data/lib/tasks/ssn_validator.rake

@@ -2,7 +2,7 @@
 namespace :ssn_validator do
   desc "Loads the current file from http://www.socialsecurity.gov/employer/ssns/highgroup.txt if it hasn't already been loaded."
   task :update_data => :environment do
-    SsnHighGroupCode.load_current_high_group_codes_file
+    SsnHighGroupCodeLoader.load_current_high_group_codes_file
   end
 
   namespace :death_master_file do

data/test/files/test_dmf_funky_data_load.txt

@@ -0,0 +1,6 @@
+A772781978BAD DEATH DAY           ANTHONY        GABRIEL        P031020090130200910
+A772783025BAD DEATH MONTH         JUAN                          P000220091101191010
+A772786197BAD BIRTH YEAR          ISAIAH                        P0202200901100009FO
+A772786123QUOTED MIDDLE NAME      ISAIAH         '              P0202200901102009FO
+A342786123QUOTED'NAME             ISAIAH                        P0202200901102009FO
+A344566123QUOTED NAME'            ISAIAH                        P0202200901102009FO

data/test/files/valid_csv_from_funky_data_file.txt

@@ -0,0 +1,6 @@
+``,`772781978`,`BAD DEATH DAY`,``,`ANTHONY`,`GABRIEL`,`P`,`2009-03-10`,`2009-01-30`,`10`,``,``,`2009-04-23 13:59:45`,`2009-04-23 13:59:45`,`2009-11-30`
+``,`772783025`,`BAD DEATH MONTH`,``,`JUAN`,``,`P`,``,`1910-11-01`,`10`,``,``,`2009-04-23 13:59:45`,`2009-04-23 13:59:45`,`2009-11-30`
+``,`772786197`,`BAD BIRTH YEAR`,``,`ISAIAH`,``,`P`,`2009-02-02`,`0009-01-10`,``,``,``,`2009-04-23 13:59:45`,`2009-04-23 13:59:45`,`2009-11-30`
+``,`772786123`,`QUOTED MIDDLE NAME`,``,`ISAIAH`,`'`,`P`,`2009-02-02`,`2009-01-10`,``,``,``,`2009-04-23 13:59:45`,`2009-04-23 13:59:45`,`2009-11-30`
+``,`342786123`,`QUOTED'NAME`,``,`ISAIAH`,``,`P`,`2009-02-02`,`2009-01-10`,``,``,``,`2009-04-23 13:59:45`,`2009-04-23 13:59:45`,`2009-11-30`
+``,`344566123`,`QUOTED NAME'`,``,`ISAIAH`,``,`P`,`2009-02-02`,`2009-01-10`,``,``,``,`2009-04-23 13:59:45`,`2009-04-23 13:59:45`,`2009-11-30`

data/test/mocks/test/death_master_file_loader.rb

@@ -2,6 +2,11 @@ require 'ssn_validator/models/death_master_file_loader'
 
 class DeathMasterFileLoader
 
+  #Gives access to the private convert_file_to_csv method
+  def create_csv_file
+    convert_file_to_csv
+  end
+
   def get_file_from_web
     case @file_path_or_url
     when /MA\d\d\d\d\d\d/ #these are the valid urls I want to mock a response to.

data/test/test_death_master_file_loader.rb

@@ -78,8 +78,36 @@ class TestDeathMasterFileLoader < Test::Unit::TestCase
     assert DeathMasterFile.find_by_as_of((initial_run += 1.month).to_s(:db))
     assert DeathMasterFile.find_by_as_of((initial_run += 1.month).to_s(:db))
     assert DeathMasterFile.find_by_as_of((initial_run += 1.month).to_s(:db))
+  end
 
+  def test_should_load_file_with_funky_data
+    DeathMasterFileLoader.new(File.dirname(__FILE__) + '/files/test_dmf_funky_data_load.txt','2009-04-01').load_file
+    assert DeathMasterFile.count == 6
+    DeathMasterFile.find(:all).each do |dmf|
+      # this is more of an issue with the bulk mysql load, but since, I'm using
+      #sqlight3 for it's in memory db, I can't test the mysql load.
+      assert_not_nil dmf.as_of
+    end
+  end
 
+  def test_should_create_csv_file_for_mysql_import
+    #since, I'm using sqlight3 for it's in memory db, I can't test the mysql load
+    #but I can test the csv file creation.
+    loader = DeathMasterFileLoader.new(File.dirname(__FILE__) + '/files/test_dmf_funky_data_load.txt','2009-11-30')
+    csv = loader.create_csv_file #this method was created in the mock only to call the private convert_file_to_csv method
+    correctly_formatted_csv = File.open(File.dirname(__FILE__) + '/files/valid_csv_from_funky_data_file.txt')
+    #csv is open and at the end of the file, so we need to reopen it so we can read the lines.
+    generated_file = File.open(csv.path).readlines
+    #compare the values of each csv line, with the correctly formated "control file"
+    correctly_formatted_csv.each_with_index do |line,i|
+      csv_line = generated_file[i]
+      correctly_formatted_record_array = line.split(',')
+      csv_record_array = csv_line.split(',')
+      comparable_value_indices = (0..11).to_a << 14 #skip indices 12 and 13, they are the created_at and updated_at fields, they will never match.
+      comparable_value_indices.each do |i|
+        assert_equal correctly_formatted_record_array[i], csv_record_array[i]
+      end
+    end
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: kevintyll-ssn_validator
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors: 
 - Kevin Tyll
@@ -83,3 +83,5 @@ test_files:
 - test/mocks/test/death_master_file_loader.rb
 - test/files/test_dmf_initial_load.txt
 - test/files/test_dmf_update_load.txt
+- test/files/test_dmf_funky_data_load.txt
+- test/files/valid_csv_from_funky_data_file.txt