free_zipcode_data 1.0.6 → 1.2.0

data/lib/free_zipcode_data/db_table.rb

@@ -8,6 +8,7 @@ module FreeZipcodeData
     ISSUE_URL = 'https://github.com/midwire/free_zipcode_data/issues/new'
 
     attr_reader :database, :tablename
+
     @@progressbar = nil
 
     def initialize(database:, tablename:)
@@ -23,6 +24,18 @@ module FreeZipcodeData
 
     private
 
+    def logger
+      Logger.instance
+    end
+
+    def warn_once(message)
+      @warned_messages ||= {}
+      return if @warned_messages[message]
+
+      logger.warn(message)
+      @warned_messages[message] = true
+    end
+
     def country_lookup_table
       @country_lookup_table ||=
         begin
@@ -33,9 +46,9 @@ module FreeZipcodeData
 
     def select_first(sql)
       rows = database.execute(sql)
-      rows[0].nil? ? nil : rows[0].first
-    rescue SQLite3::SQLException => err
-      raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
+      rows[0]&.first
+    rescue SQLite3::SQLException => e
+      raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
     end
 
     def get_country_id(country)
@@ -43,20 +56,54 @@ module FreeZipcodeData
       select_first(sql)
     end
 
-    def get_state_id(state_abbr, state_name)
-      sql = "SELECT id FROM states
-        WHERE abbr = '#{state_abbr}' OR name = '#{escape_single_quotes(state_name)}'"
+    # Look up a state ID scoped to a country, trying progressively less specific
+    # criteria: (1) abbr + name + country, (2) abbr + country, (3) name + country.
+    # Returns nil if no match is found.
+    def get_state_id(country, state_abbr, state_name)
+      escaped_country = escape_single_quotes(country)
+      return nil if escaped_country.empty?
+
+      escaped_abbr = escape_single_quotes(state_abbr)
+      escaped_name = escape_single_quotes(state_name)
+      country_cond = "c.alpha2 = '#{escaped_country}'"
+      # Most specific lookup: abbr + name + country
+      res = find_state_where("s.abbr = '#{escaped_abbr}'", "s.name = '#{escaped_name}'", country_cond)
+      return res if res
+
+      # Fallback: abbr + country only
+      res = find_state_where("s.abbr = '#{escaped_abbr}'", country_cond)
+      if res
+        logger.verbose("State fallback: abbr '#{state_abbr}' + country '#{country}' (name mismatch)")
+        return res
+      end
+      # Fallback: name + country only
+      res = find_state_where("s.name = '#{escaped_name}'", country_cond)
+      if res
+        logger.verbose("State fallback: name '#{state_name}' + country '#{country}' (abbr mismatch)")
+        return res
+      end
+      logger.warn("State lookup failed: abbr='#{state_abbr}', name='#{state_name}', country='#{country}'")
+      nil
+    end
+
+    def find_state_where(*conditions)
+      sql = <<-SQL
+        SELECT s.id FROM states s
+        INNER JOIN countries c ON s.country_id = c.id
+        WHERE #{conditions.join(' AND ')}
+      SQL
       select_first(sql)
     end
 
     def get_county_id(county)
       return nil if county.nil?
+
       sql = "SELECT id FROM counties WHERE name = '#{escape_single_quotes(county)}'"
       select_first(sql)
     end
 
     def escape_single_quotes(string)
-      string&.gsub(/[']/, '\'\'') || ''
+      string&.gsub('\'', '\'\'') || ''
     end
   end
 end

data/lib/free_zipcode_data/logger.rb

@@ -13,27 +13,23 @@ module FreeZipcodeData
       @log_provider = provider
     end
 
-    def log_exception(e, data = {})
-      msg = "EXCEPTION : #{e.class.name} : #{e.message}"
+    def log_exception(error, data = {})
+      msg = "EXCEPTION : #{error.class.name} : #{error.message}"
       msg += "\n data : #{data.inspect}" if data && !data.empty?
-      msg += "\n  #{e.backtrace[0, 6].join("\n  ")}"
+      msg += "\n  #{error.backtrace[0, 6].join("\n  ")}"
       log_provider.error(msg)
     end
 
-    def method_missing(meth, *args, &block)
+    def method_missing(meth, *, &)
       if log_provider.respond_to?(meth)
-        log_provider.send(meth, *args, &block)
+        log_provider.send(meth, *, &)
       else
         super
       end
     end
 
-    def respond_to?(meth, include_private = false)
-      if log_provider.respond_to?(meth)
-        true
-      else
-        super
-      end
+    def respond_to_missing?(meth, include_private = false)
+      log_provider.respond_to?(meth) || super
     end
 
     def verbose(msg)
@@ -43,7 +39,7 @@ module FreeZipcodeData
     private
 
     def default_logger
-      logger = ::Logger.new(STDOUT)
+      logger = ::Logger.new($stdout)
       logger.formatter = proc do |_, _, _, msg|
         "#{msg}\n"
       end

data/lib/free_zipcode_data/runner.rb

@@ -58,9 +58,9 @@ module FreeZipcodeData
     private
 
     def initialize_table(table_sym, database)
-      tablename = options["#{table_sym}_tablename".to_sym]
+      tablename = options[:"#{table_sym}_tablename"]
       logger.verbose("Initializing #{table_sym} table: '#{tablename}'...")
-      klass = instance_eval("#{titleize(table_sym)}Table", __FILE__, __LINE__)
+      klass = FreeZipcodeData.const_get(:"#{titleize(table_sym)}Table")
       table = klass.new(
         database: database.conn,
         tablename: tablename

data/lib/free_zipcode_data/state_table.rb

@@ -17,21 +17,27 @@ module FreeZipcodeData
 
       ndx = <<-SQL
         CREATE UNIQUE INDEX "main"."unique_state"
-        ON #{tablename} (abbr, country_id COLLATE NOCASE ASC);
+        ON #{tablename} (abbr COLLATE NOCASE ASC, country_id);
       SQL
       database.execute_batch(ndx)
 
       ndx = <<-SQL
         CREATE UNIQUE INDEX "main"."state_name"
-        ON #{tablename} (name COLLATE NOCASE ASC);
+        ON #{tablename} (name COLLATE NOCASE ASC, country_id);
       SQL
       database.execute_batch(ndx)
     end
 
     def write(row)
-      return nil unless row[:short_state]
+      return nil unless synthesize_state(row)
+
       row[:state] = 'Marshall Islands' if row[:short_state] == 'MH' && row[:state].nil?
       country_id = get_country_id(row[:country])
+      unless country_id
+        warn_once("Country '#{row[:country]}' not found in countries table, skipping state")
+        return nil
+      end
+
       sql = <<-SQL
         INSERT INTO states (abbr, name, country_id)
         VALUES ('#{row[:short_state]}',
@@ -41,11 +47,37 @@ module FreeZipcodeData
       SQL
       begin
         database.execute(sql)
-      rescue SQLite3::ConstraintException
-        # Swallow duplicates
+      rescue SQLite3::ConstraintException => e
+        unless e.message.include?('UNIQUE')
+          raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
+        end
+      rescue StandardError => e
+        raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
       end
 
       update_progress
     end
+
+    private
+
+    # Synthesize state from country for stateless countries.
+    # Mutates the row hash so downstream Kiba destinations (CountyTable, ZipcodeTable)
+    # see the synthesized short_state and state values.
+    def synthesize_state(row)
+      if row[:short_state].nil? || row[:short_state] == ''
+        country_entry = country_lookup_table[row[:country]]
+        unless country_entry
+          warn_once(
+            "Cannot synthesize state for country '#{row[:country]}': " \
+            'not in country_lookup_table'
+          )
+          return false
+        end
+
+        row[:short_state] = row[:country]
+        row[:state] = country_entry[:name]
+      end
+      row[:short_state]
+    end
   end
 end

data/lib/free_zipcode_data/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FreeZipcodeData
-  VERSION = '1.0.6'.freeze
+  VERSION = '1.2.0'
 end

data/lib/free_zipcode_data/zipcode_table.rb

@@ -29,7 +29,15 @@ module FreeZipcodeData
     def write(row)
       return nil unless row[:postal_code]
 
-      state_id = get_state_id(row[:short_state], row[:state])
+      state_id = get_state_id(row[:country], row[:short_state], row[:state])
+      unless state_id
+        logger.verbose(
+          "Skipping zipcode '#{row[:postal_code]}': no state found for " \
+          "abbr='#{row[:short_state]}', country='#{row[:country]}'"
+        )
+        return nil
+      end
+
       city_name = escape_single_quotes(row[:city])
 
       sql = <<-SQL
@@ -45,10 +53,12 @@ module FreeZipcodeData
 
       begin
         database.execute(sql)
-      rescue SQLite3::ConstraintException => _err
-        # there are some duplicates - swallow them
-      rescue StandardError => err
-        raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
+      rescue SQLite3::ConstraintException => e
+        unless e.message.include?('UNIQUE')
+          raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
+        end
+      rescue StandardError => e
+        raise "Please file an issue at #{ISSUE_URL}: [#{e}] -> SQL: [#{sql}]"
       end
 
       update_progress

data/lib/free_zipcode_data.rb

@@ -14,16 +14,16 @@ module FreeZipcodeData
     ENV.fetch('APP_ENV', 'development')
   end
 
-  #:nocov:
+  # :nocov:
   def self.config_file(filename = '.free_zipcode_data.yml')
     return root.join('spec', 'fixtures', filename) if current_environment == 'test'
 
-    home = ENV.fetch('HOME')
+    home = Dir.home
     file = ENV.fetch('FZD_CONFIG_FILE', File.join(home, '.free_zipcode_data.yml'))
     FileUtils.touch(file)
     file
   end
-  #:nocov:
+  # :nocov:
 
   def self.os
     if RUBY_PLATFORM.match?(/cygwin|mswin|mingw|bccwin|wince|emx/)

data/lib/tasks/version.rake

@@ -6,8 +6,6 @@ require 'fileutils'
 
 # rubocop:disable Metrics/BlockLength
 namespace :version do
-  PROJECT_ROOT = File.expand_path(FileUtils.pwd).freeze
-  PROJECT_NAME = ENV['PROJECT_NAME'] || File.basename(PROJECT_ROOT)
 
   desc 'Write changes to the CHANGELOG'
   task :changes do
@@ -23,7 +21,7 @@ namespace :version do
 
   desc 'Increment the patch version and write changes to the changelog'
   task :bump_patch do
-    exit unless check_branch_and_warn
+    exit unless check_branch_and_warn?
     major, minor, patch = read_version
     patch = patch.to_i + 1
     write_version_file([major, minor, patch])
@@ -36,7 +34,7 @@ namespace :version do
 
   desc 'Increment the minor version and write changes to the changelog'
   task :bump_minor do
-    exit unless check_branch_and_warn
+    exit unless check_branch_and_warn?
     major, minor, _patch = read_version
     minor = minor.to_i + 1
     patch = 0
@@ -47,7 +45,7 @@ namespace :version do
 
   desc 'Increment the major version and write changes to the changelog'
   task :bump_major do
-    exit unless check_branch_and_warn
+    exit unless check_branch_and_warn?
     major, _minor, _patch = read_version
     major = major.to_i + 1
     minor = 0
@@ -59,19 +57,27 @@ namespace :version do
 
   private
 
+  def project_root
+    @project_root ||= File.expand_path(FileUtils.pwd).freeze
+  end
+
+  def project_name
+    @project_name ||= ENV['PROJECT_NAME'] || File.basename(project_root)
+  end
+
   def version_file_path
-    split = PROJECT_NAME.split('-')
-    "#{PROJECT_ROOT}/lib/#{split.join('/')}/version.rb"
+    split = project_name.split('-')
+    "#{project_root}/lib/#{split.join('/')}/version.rb"
   end
 
   def module_name
-    case PROJECT_NAME
+    case project_name
     when /-/
-      PROJECT_NAME.split('-').map(&:capitalize).join('::')
+      project_name.split('-').map(&:capitalize).join('::')
     when /_/
-      PROJECT_NAME.split('_').map(&:capitalize).join
+      project_name.split('_').map(&:capitalize).join
     else
-      PROJECT_NAME.capitalize
+      project_name.capitalize
     end
   end
 
@@ -79,13 +85,13 @@ namespace :version do
     silence_warnings do
       load version_file_path
     end
-    text = eval("#{module_name}::VERSION")
+    text = module_name.split('::').inject(Object) { |mod, name| mod.const_get(name) }::VERSION
     text.split('.')
   end
 
   def write_version_file(version_array)
     version = version_array.join('.')
-    new_version = %(  VERSION = '#{version}'.freeze)
+    new_version = %(  VERSION = '#{version}')
     lines = File.readlines(version_file_path)
     File.open(version_file_path, 'w') do |f|
       lines.each do |line|
@@ -100,19 +106,17 @@ namespace :version do
 
   def update_readme_version_strings
     version_string = read_version.join('.')
-    readme = open('README.md').read
+    readme = File.read('README.md')
     regex = /^\*\*Version: [0-9.]+\*\*$/i
     return nil unless readme =~ regex
 
-    File.open('README.md', 'w') do |f|
-      f.write(readme.gsub(regex, "**Version: #{version_string}**"))
-    end
+    File.write('README.md', readme.gsub(regex, "**Version: #{version_string}**"))
   end
 
   def changelog
     return @changelog_path if @changelog_path
 
-    @changelog_path = File.join(PROJECT_ROOT, 'CHANGELOG')
+    @changelog_path = File.join(project_root, 'CHANGELOG')
     FileUtils.touch(@changelog_path)
     @changelog_path
   end
@@ -159,16 +163,15 @@ namespace :version do
     STRING
   end
 
-  def check_branch_and_warn
+  def check_branch_and_warn?
     return true unless current_branch == 'master'
 
     puts(branch_warning_message)
-    while (line = $stdin.gets.chomp)
-      return true if line =~ /[yY]/
+    line = $stdin.gets.chomp
+    return true if line =~ /[yY]/
 
-      puts 'Aborting version bump.'
-      return false
-    end
+    puts 'Aborting version bump.'
+    false
   end
 
   def launch_editor(file)

data/spec/etl/csv_source_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'etl/csv_source'
+
+RSpec.describe CsvSource do
+  let(:fixture_csv) { File.join(FreeZipcodeData.root, 'spec', 'fixtures', 'test_data.csv') }
+
+  describe '#initialize' do
+    it 'stores the filename and options' do
+      source = described_class.new(filename: fixture_csv)
+      expect(source.filename).to eq(fixture_csv)
+      expect(source.headers).to be true
+      expect(source.delimeter).to eq("\t")
+    end
+
+    it 'accepts custom delimiter and quote char' do
+      source = described_class.new(filename: fixture_csv, delimeter: ',', quote_char: '"')
+      expect(source.delimeter).to eq(',')
+      expect(source.quote_char).to eq('"')
+    end
+  end
+
+  # CsvSource implements only #each (Kiba source protocol), not Enumerable
+  # rubocop:disable Style/MapIntoArray
+  describe '#each' do
+    it 'yields each row as a hash with symbolized keys' do
+      source = described_class.new(filename: fixture_csv, delimeter: ',', quote_char: '"')
+      rows = []
+      source.each { |row| rows << row }
+
+      expect(rows.length).to eq(6)
+      expect(rows.first).to be_a(Hash)
+      expect(rows.first.keys).to include(:country, :postal_code, :city)
+    end
+
+    it 'parses the correct data from each row' do
+      source = described_class.new(filename: fixture_csv, delimeter: ',', quote_char: '"')
+      rows = []
+      source.each { |row| rows << row }
+
+      first = rows.first
+      expect(first[:country]).to eq('US')
+      expect(first[:postal_code]).to eq('10001')
+      expect(first[:city]).to eq('New York')
+      expect(first[:short_state]).to eq('NY')
+    end
+
+    it 'handles rows from multiple countries' do
+      source = described_class.new(filename: fixture_csv, delimeter: ',', quote_char: '"')
+      countries = []
+      source.each { |row| countries << row[:country] }
+
+      expect(countries.uniq.sort).to eq(%w[CA GB US])
+    end
+  end
+  # rubocop:enable Style/MapIntoArray
+end

data/spec/etl/free_zipcode_data_job_spec.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'kiba'
+require 'etl/free_zipcode_data_job'
+
+RSpec.describe ETL::FreeZipcodeDataJob do
+  let(:db) { create_test_database(line_count: 6) }
+  let(:fixture_csv) { File.join(FreeZipcodeData.root, 'spec', 'fixtures', 'test_data.csv') }
+  let(:logger) { FreeZipcodeData::Logger.instance }
+  let(:string_io) { StringIO.new }
+  let(:options) do
+    OpenStruct.new(
+      country_tablename: 'countries',
+      state_tablename: 'states',
+      county_tablename: 'counties',
+      zipcode_tablename: 'zipcodes',
+      verbose: false
+    )
+  end
+
+  before do
+    FreeZipcodeData::Options.instance.initialize_hash(options)
+    logger.log_provider = Logger.new(string_io)
+  end
+
+  describe '.setup' do
+    it 'returns a Kiba job definition' do
+      job = described_class.setup(fixture_csv, db, logger, options)
+      expect(job).not_to be_nil
+    end
+  end
+
+  describe 'full ETL pipeline' do
+    before do
+      # Build all tables
+      FreeZipcodeData::CountryTable.new(database: db, tablename: 'countries').build
+      FreeZipcodeData::StateTable.new(database: db, tablename: 'states').build
+      FreeZipcodeData::CountyTable.new(database: db, tablename: 'counties').build
+      FreeZipcodeData::ZipcodeTable.new(database: db, tablename: 'zipcodes').build
+
+      job = described_class.setup(fixture_csv, db, logger, options)
+      Kiba.run(job)
+    end
+
+    it 'populates the countries table' do
+      rows = db.execute('SELECT alpha2 FROM countries ORDER BY alpha2')
+      expect(rows.flatten).to include('CA', 'GB', 'US')
+    end
+
+    it 'populates the states table' do
+      rows = db.execute('SELECT abbr FROM states ORDER BY abbr')
+      abbrs = rows.flatten
+      expect(abbrs).to include('CA', 'IL', 'NY')
+    end
+
+    it 'populates the counties table' do
+      rows = db.execute('SELECT name FROM counties ORDER BY name')
+      names = rows.flatten
+      expect(names).to include('Cook', 'Los Angeles', 'New York')
+    end
+
+    it 'populates the zipcodes table' do
+      rows = db.execute('SELECT code FROM zipcodes ORDER BY code')
+      codes = rows.flatten
+      expect(codes).to include('10001', '60601', '90210')
+    end
+
+    it 'links zipcodes to states' do
+      rows = db.execute(<<-SQL)
+        SELECT z.code, s.abbr
+        FROM zipcodes z
+        JOIN states s ON CAST(z.state_id AS INTEGER) = s.id
+        WHERE z.code = '60601'
+      SQL
+      expect(rows[0]).to eq(%w[60601 IL])
+    end
+
+    it 'links states to countries' do
+      rows = db.execute(<<-SQL)
+        SELECT s.abbr, c.alpha2
+        FROM states s
+        JOIN countries c ON s.country_id = c.id
+        WHERE s.abbr = 'NY'
+      SQL
+      expect(rows[0]).to eq(%w[NY US])
+    end
+
+    it 'stores geocode data for zipcodes' do
+      rows = db.execute("SELECT lat, lon FROM zipcodes WHERE code = '10001'")
+      lat = rows[0][0].to_f
+      lon = rows[0][1].to_f
+      expect(lat).to be_within(0.01).of(40.7484)
+      expect(lon).to be_within(0.01).of(-73.9967)
+    end
+
+    it 'scopes duplicate state abbreviations by country' do
+      us_ny = db.execute(<<-SQL)
+        SELECT s.id, s.name, c.alpha2
+        FROM states s
+        JOIN countries c ON s.country_id = c.id
+        WHERE s.abbr = 'NY' AND c.alpha2 = 'US'
+      SQL
+      ca_ny = db.execute(<<-SQL)
+        SELECT s.id, s.name, c.alpha2
+        FROM states s
+        JOIN countries c ON s.country_id = c.id
+        WHERE s.abbr = 'NY' AND c.alpha2 = 'CA'
+      SQL
+      expect(us_ny.length).to eq(1)
+      expect(ca_ny.length).to eq(1)
+      expect(us_ny[0][0]).not_to eq(ca_ny[0][0])
+      expect(us_ny[0][1]).to eq('New York')
+      expect(ca_ny[0][1]).to eq('Northern York')
+    end
+
+    it 'links cross-country zipcodes to the correct state' do
+      us_zip = db.execute(<<-SQL)
+        SELECT z.code, s.name, c.alpha2
+        FROM zipcodes z
+        JOIN states s ON CAST(z.state_id AS INTEGER) = s.id
+        JOIN countries c ON s.country_id = c.id
+        WHERE z.code = '10001'
+      SQL
+      ca_zip = db.execute(<<-SQL)
+        SELECT z.code, s.name, c.alpha2
+        FROM zipcodes z
+        JOIN states s ON CAST(z.state_id AS INTEGER) = s.id
+        JOIN countries c ON s.country_id = c.id
+        WHERE z.code = 'K0A'
+      SQL
+      expect(us_zip[0]).to eq(['10001', 'New York', 'US'])
+      expect(ca_zip[0]).to eq(['K0A', 'Northern York', 'CA'])
+    end
+  end
+end

data/spec/fixtures/.free_zipcode_data.yml

@@ -0,0 +1 @@
+---

data/spec/fixtures/US.txt

@@ -0,0 +1,5 @@
+US	10001	New York	New York	NY	New York	061	Manhattan	MN	40.7484	-73.9967	4
+US	90210	Beverly Hills	California	CA	Los Angeles	037		LA	34.0901	-118.4065	4
+US	60601	Chicago	Illinois	IL	Cook	031		CK	41.8819	-87.6278	4
+CA	H2X	Montreal	Quebec	QC			Montreal		45.5088	-73.5878	4
+GB	SW1A	London	England	ENG	Westminster		City of Westminster		51.5014	-0.1419	1

data/spec/fixtures/test_data.csv

@@ -0,0 +1,7 @@
+COUNTRY,POSTAL_CODE,CITY,STATE,SHORT_STATE,COUNTY,SHORT_COUNTY,COMMUNITY,SHORT_COMMUNITY,LATITUDE,LONGITUDE,ACCURACY
+US,10001,New York,New York,NY,New York,061,Manhattan,MN,40.7484,-73.9967,4
+US,90210,Beverly Hills,California,CA,Los Angeles,037,,LA,34.0901,-118.4065,4
+US,60601,Chicago,Illinois,IL,Cook,031,,CK,41.8819,-87.6278,4
+CA,H2X,Montreal,Quebec,QC,,,Montreal,,45.5088,-73.5878,4
+CA,K0A,Almonte,Northern York,NY,Lanark,LNK,,,45.2260,-76.1840,4
+GB,SW1A,London,England,ENG,Westminster,,City of Westminster,,51.5014,-0.1419,1

data/spec/fixtures/test_data.txt

@@ -0,0 +1,5 @@
+US	10001	New York	New York	NY	New York	061	Manhattan	MN	40.7484	-73.9967	4
+US	90210	Beverly Hills	California	CA	Los Angeles	037		LA	34.0901	-118.4065	4
+US	60601	Chicago	Illinois	IL	Cook	031		CK	41.8819	-87.6278	4
+CA	H2X	Montreal	Quebec	QC			Montreal		45.5088	-73.5878	4
+GB	SW1A	London	England	ENG	Westminster		City of Westminster		51.5014	-0.1419	1