ofac 1.1.11

data/lib/ofac/models/ofac_sdn.rb

@@ -0,0 +1,5 @@
+require 'activerecord'
+
+class OfacSdn < ActiveRecord::Base
+
+end

data/lib/ofac/models/ofac_sdn_loader.rb

@@ -0,0 +1,305 @@
+require 'net/http'
+require 'activerecord'
+require 'active_record/connection_adapters/mysql_adapter'
+
+class OfacSdnLoader
+
+
+  #Loads the most recent file from http://www.treas.gov/offices/enforcement/ofac/sdn/delimit/index.shtml
+  def self.load_current_sdn_file
+    puts "Reloading OFAC sdn data"
+    puts "Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn"
+    #get the 3 data files
+    sdn = Tempfile.new('sdn')
+    sdn.write(Net::HTTP.get(URI.parse('http://www.treas.gov/offices/enforcement/ofac/sdn/delimit/sdn.pip')))
+    sdn.rewind
+    address = Tempfile.new('sdn')
+    address.write(Net::HTTP.get(URI.parse('http://www.treas.gov/offices/enforcement/ofac/sdn/delimit/add.pip')))
+    address.rewind
+    alt = Tempfile.new('sdn')
+    alt.write(Net::HTTP.get(URI.parse('http://www.treas.gov/offices/enforcement/ofac/sdn/delimit/alt.pip')))
+    alt.rewind
+    
+    if OfacSdn.connection.kind_of?(ActiveRecord::ConnectionAdapters::MysqlAdapter)
+      puts "Converting file to csv format for Mysql import.  This could take several minutes."
+
+      csv_file = convert_to_flattened_csv(sdn, address, alt)
+
+      bulk_mysql_update(csv_file)
+    else
+      active_record_file_load(sdn, address, alt)
+    end
+
+    sdn.close
+    @address.close
+    @alt.close
+  end
+
+
+  private
+
+  #convert the file's null value to an empty string
+  #and removes " chars.
+  def self.clean_file_string(line)
+    line.gsub!(/-0-(\s)?/,'')
+    line.gsub!(/\n/,'')
+    line.gsub(/\"/,'')
+  end
+
+  #split the line into an array
+  def self.convert_line_to_array(line)
+    clean_file_string(line).split('|') unless line.nil?
+  end
+
+  #return an 2 arrays of the records matching the sdn primary key
+  #1 array of address records and one array of alt records
+  def self.foreign_key_records(sdn_id)
+    address_records = []
+    alt_records = []
+
+    #the first element in each array is the primary and foreign keys
+    #we are denormalizing the data
+    if @current_address_hash && @current_address_hash[:id] == sdn_id
+      address_records << @current_address_hash
+      loop do
+        @current_address_hash = address_text_to_hash(@address.gets)
+        if @current_address_hash && @current_address_hash[:id] == sdn_id
+          address_records << @current_address_hash
+        else
+          break
+        end
+      end
+    end
+
+    if @current_alt_hash && @current_alt_hash[:id] == sdn_id
+      alt_records << @current_alt_hash
+      loop do
+        @current_alt_hash = alt_text_to_hash(@alt.gets)
+        if @current_alt_hash && @current_alt_hash[:id] == sdn_id
+          alt_records << @current_alt_hash
+        else
+          break
+        end
+      end
+    end
+    return address_records, alt_records
+  end
+
+  def self.sdn_text_to_hash(line)
+    unless line.nil?
+      value_array = convert_line_to_array(line)
+      {:id => value_array[0],
+        :name => value_array[1],
+        :sdn_type => value_array[2],
+        :program => value_array[3],
+        :title => value_array[4],
+        :vessel_call_sign => value_array[5],
+        :vessel_type => value_array[6],
+        :vessel_tonnage => value_array[7],
+        :gross_registered_tonnage => value_array[8],
+        :vessel_flag => value_array[9],
+        :vessel_owner => value_array[10],
+        :remarks => value_array[11]
+      }
+    end
+  end
+
+  def self.address_text_to_hash(line)
+    unless line.nil?
+      value_array = convert_line_to_array(line)
+      {:id => value_array[0],
+       :address => value_array[2],
+       :city => value_array[3],
+       :country => value_array[4],
+       :address_remarks => value_array[5]
+      }
+    end
+  end
+
+  def self.alt_text_to_hash(line)
+    unless line.nil?
+      value_array = convert_line_to_array(line)
+      {:id => value_array[0],
+       :alternate_identity_type => value_array[2],
+       :alternate_identity_name => value_array[3],
+       :alternate_identity_remarks => value_array[4]
+      }
+    end
+  end
+
+  def self.convert_hash_to_mysql_import_string(record_hash)
+    # empty field for id to be generated by mysql.
+    new_line = "``|" +
+      #    :name
+    "`#{record_hash[:name]}`|" +
+      #    :sdn_type
+    "`#{record_hash[:sdn_type]}`|" +
+      #    :program
+    "`#{record_hash[:program]}`|" +
+      #    :title
+    "`#{record_hash[:title]}`|" +
+      #    :vessel_call_sign
+    "`#{record_hash[:vessel_call_sign]}`|" +
+      #    :vessel_type
+    "`#{record_hash[:vessel_type]}`|" +
+      #    :vessel_tonnage
+    "`#{record_hash[:vessel_tonnage]}`|" +
+      #    :gross_registered_tonnage
+    "`#{record_hash[:gross_registered_tonnage]}`|" +
+      #    :vessel_flag
+    "`#{record_hash[:vessel_flag]}`|" +
+      #    :vessel_owner
+    "`#{record_hash[:vessel_owner]}`|" +
+      #    :remarks
+    "`#{record_hash[:remarks]}`|" +
+    #    :address
+    "`#{record_hash[:address]}`|" +
+    #    :city
+    "`#{record_hash[:city]}`|" +
+    #    :country
+    "`#{record_hash[:country]}`|" +
+    #    :address_remarks
+    "`#{record_hash[:address_remarks]}`|" +
+    #    :alternate_identity_type
+    "`#{record_hash[:alternate_identity_type]}`|" +
+    #    :alternate_identity_name
+    "`#{record_hash[:alternate_identity_name]}`|" +
+    #    :alternate_identity_remarks
+    "`#{record_hash[:alternate_identity_remarks]}`|" +
+    #:created_at
+    "`#{Time.now.to_s(:db)}`|" +
+      # updated_at
+    "`#{Time.now.to_s(:db)}`" + "\n"
+
+    new_line
+  end
+
+  def self.convert_to_flattened_csv(sdn_file, address_file, alt_file)
+    @address = address_file
+    @alt = alt_file
+
+    csv_file = Tempfile.new("ofac")    # create temp file for converted csv format.
+    #get the first line from the address and alt files
+    @current_address_hash = address_text_to_hash(@address.gets)
+    @current_alt_hash = alt_text_to_hash(@alt.gets)
+
+    start = Time.now
+    
+    sdn_file.each_with_index do |line, i|
+      
+      #initialize the address and alt atributes to empty strings
+      address_attributes = address_text_to_hash("|||||")
+      alt_attributes = alt_text_to_hash("||||")
+
+      sdn_attributes = sdn_text_to_hash(line)
+
+      #get the foreign key records for this sdn
+      address_records, alt_records = foreign_key_records(sdn_attributes[:id])
+
+      if address_records.empty?
+        #no matching address records, so initialized blank values will be used.
+        if alt_records.empty?
+          #no matching address records, so initialized blank values will be used.
+          csv_file.syswrite(convert_hash_to_mysql_import_string(sdn_attributes.merge(address_attributes).merge(alt_attributes)))
+        else
+          alt_records.each do |alt|
+            csv_file.syswrite(convert_hash_to_mysql_import_string(sdn_attributes.merge(address_attributes).merge(alt)))
+          end
+        end
+      else
+        address_records.each do |address|
+          if alt_records.empty?
+            #no matching address records, so initialized blank values will be used.
+            csv_file.syswrite(convert_hash_to_mysql_import_string(sdn_attributes.merge(address).merge(alt_attributes)))
+          else
+            alt_records.each do |alt|
+              csv_file.syswrite(convert_hash_to_mysql_import_string(sdn_attributes.merge(address).merge(alt)))
+            end
+          end
+        end
+      end
+      puts "#{i} records processed." if (i % 1000 == 0) && (i > 0)
+    end
+    puts "File conversion ran for #{(Time.now - start) / 60} minutes."
+    return csv_file
+  end
+
+  def self.active_record_file_load(sdn_file, address_file, alt_file)
+    @address = address_file
+    @alt = alt_file
+
+    #OFAC data is a complete list, so we have to dump and load
+    OfacSdn.delete_all
+
+    #get the first line from the address and alt files
+    @current_address_hash = address_text_to_hash(@address.gets)
+    @current_alt_hash = alt_text_to_hash(@alt.gets)
+    attributes = {}
+    sdn_file.each_with_index do |line, i|
+
+      #initialize the address and alt atributes to empty strings
+      address_attributes = address_text_to_hash("|||||")
+      alt_attributes = alt_text_to_hash("||||")
+
+      sdn_attributes = sdn_text_to_hash(line)
+     
+      #get the foreign key records for this sdn
+      address_records, alt_records = foreign_key_records(sdn_attributes[:id])
+
+      if address_records.empty?
+        #no matching address records, so initialized blank values will be used.
+        if alt_records.empty?
+          #no matching address records, so initialized blank values will be used.
+          attributes = sdn_attributes.merge(address_attributes).merge(alt_attributes)
+          attributes.delete(:id)
+          OfacSdn.create(attributes)
+        else
+          alt_records.each do |alt|
+            attributes = sdn_attributes.merge(address_attributes).merge(alt)
+            attributes.delete(:id)
+            OfacSdn.create(attributes)
+          end
+        end
+      else
+        address_records.each do |address|
+          if alt_records.empty?
+            #no matching address records, so initialized blank values will be used.
+            attributes = sdn_attributes.merge(address).merge(alt_attributes)
+            attributes.delete(:id)
+            OfacSdn.create(attributes)
+          else
+            alt_records.each do |alt|
+              attributes = sdn_attributes.merge(address).merge(alt)
+              attributes.delete(:id)
+              OfacSdn.create(attributes)
+            end
+          end
+        end
+      end
+      
+      puts "#{i} records processed." if (i % 5000 == 0) && (i > 0)
+    end
+  end
+
+  # For mysql, use:
+  # LOAD DATA LOCAL INFILE 'ssdm1.csv' INTO TABLE death_master_files FIELDS TERMINATED BY '|' ENCLOSED BY "`" LINES TERMINATED BY '\n';
+  # This is a much faster way of loading large amounts of data into mysql.  For information on the LOAD DATA command
+  # see http://dev.mysql.com/doc/refman/5.1/en/load-data.html
+  def self.bulk_mysql_update(csv_file)
+    puts "Deleting all records in ofac_sdn..."
+
+    #OFAC data is a complete list, so we have to dump and load
+    OfacSdn.delete_all
+
+    puts "Importing into Mysql..."
+    
+    mysql_command = <<-TEXT
+    LOAD DATA LOCAL INFILE '#{csv_file.path}' REPLACE INTO TABLE ofac_sdns FIELDS TERMINATED BY '|' ENCLOSED BY "`" LINES TERMINATED BY '\n';
+    TEXT
+
+    OfacSdn.connection.execute(mysql_command)
+    puts "Mysql import complete."
+
+  end
+
+end

data/lib/ofac/ofac_match.rb

@@ -0,0 +1,139 @@
+class OfacMatch
+
+  attr_reader :possible_hits
+
+  #Intialize a Match object with a record hash of fields you want to match on.
+  #Each key in the hash, also has a data hash value for the weight, token, and type.
+  #
+  #    match = Ofac::Match.new({:name => {:weight => 10, :token => 'Kevin Tyll'},
+  #        :city   => {:weight => 40, :token => 'Clearwater',     },
+  #        :address => {:weight => 40, :token => '1234 Park St.',    },
+  #        :zip    => {:weight => 10, :token => '33759', :type => :number}})
+  #
+  # data hash keys:
+  #   * <tt>data[:weight]</tt> - value to apply to the score if there is a match (Default is 100/number of keys in the record hash)
+  #   * <tt>data[:token]</tt> - string to match
+  #   * <tt>data[:match]</tt> - set from records hash
+  #   * <tt>data[:score]</tt> - output field
+  #   * <tt>data[:type]</tt> - the type of match that should be performed (valid values are +:sound+ | +:number+) (Default is +:sound+)
+  def initialize(stats={})
+    @possible_hits = []
+    @stats = stats.dup
+    weight = 100
+    weight = 100 / @stats.length if @stats.length > 0
+    @stats.each_value do |data|
+      data[:weight] ||= weight
+      data[:match]  ||= ''
+      data[:type]   ||= :sound
+      data[:score]  ||= 0
+      data[:token]  = data[:token].to_s.upcase
+    end
+  end
+
+  # match_records is an array of hashes.
+  #
+  # The hash keys must match the record hash keys set when initialized.
+  #
+  # score will return the highest score of all the records that
+  # are sent in match_records.
+  def score(match_records)
+    score_results = Array.new
+    unless match_records.empty?
+      #place the match_records information
+      #into our @stats hash
+      match_records.each do |match|
+        match.each do |key, value|
+          @stats[key.to_sym][:match] = value.to_s.upcase
+        end
+        record_score = calculate_record
+        score_results.push(record_score)
+        @possible_hits << match.merge(:score => record_score) if record_score > 0
+      end
+      score = score_results.max #take max score
+    end
+    @possible_hits.uniq!
+    score ||= 0
+  end
+
+  private
+
+
+  # calculate the score for this record
+  # comparing the token to the match fields in the @stats hash
+  # and storing the score into the record
+  def calculate_record
+    score = 0
+    unless @stats.nil?
+      #need to make sure we check the name first, since city and address don't
+      #get added to the score unless there is a name match
+      [:name,:city,:address].each do |field|
+        data = @stats[field]
+        if (data[:token].blank?)
+          value = 0 #token is blank can't be sure of a match if nothing to match against
+        else
+          if (data[:match].blank?)
+            value = 0 #token has value match is blank
+          else
+            #token and match both have values
+            if (data[:type] == :number)
+              value = data[:token] == data[:match] ? 1 : 0
+            else
+              #first see if there is an exact match
+              value = data[:match].split('|').include?(data[:token]) ? 1 : 0
+
+              unless value > 0
+                #do a sounds like with the data as given to see if we get a match
+                #if match on sounds_like, only give .75 of the weight.
+                data[:match].split('|').each do |separate_value|
+                  if data[:token].ofac_sounds_like(separate_value,false)
+                    value = 0.75
+                    break
+                  else
+                    value = 0
+                  end
+                end
+
+              end
+                
+              #if no match, then break the data down and see if we can find matches on the
+              #individual words
+              unless value > 0
+                token_data = data[:token].gsub(/\W/,'|')
+                token_array = token_data.split('|')
+                token_array.delete('')
+
+                match_data = data[:match].gsub(/\W/,'|')
+                match_array = match_data.split('|')
+                match_array.delete('')
+
+                value = 0
+                partial_weight = 1/token_array.length.to_f
+                
+                token_array.each do |partial_token|
+                  #first see if we get an exact match of the partial
+                  if success = match_array.include?(partial_token)
+                    value += partial_weight
+                  end
+                  unless success
+                    #if this for :address or :city
+                    #and there is no match at all, subtract 10% of the weight from :name score
+                    unless field == :name
+                      value -= partial_weight * 0.1
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+        data[:score] = data[:weight] * value
+        score += data[:score]
+        break if field == :name && data[:score] == 0
+      end
+
+    end
+    score.round
+  end
+
+end
+