free_zipcode_data 1.0.1

data/free_zipcode_data.gemspec

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'free_zipcode_data/version'
+
+# rubocop:disable Metrics/BlockLength
+Gem::Specification.new do |spec|
+  spec.name          = 'free_zipcode_data'
+  spec.version       = FreeZipcodeData::VERSION
+  spec.authors       = ['Chris Blackburn', 'Chris McKnight']
+  spec.email         = ['87a1779b@opayq.com', 'fixme@mcknight.bogus']
+  spec.summary       = 'Free US and world-wide postal codes in SQLite and CSV format'
+  spec.description   = <<~STRING
+    Free US and world-wide postal codes in SQLite and CSV format.
+    Automated zipcode/postal code aggregation and processing for any needs.
+  STRING
+  spec.homepage      = 'https://github.com/midwire/free_zipcode_data'
+  spec.license       = 'MIT'
+
+  spec.required_ruby_version = '>= 2.3.0'
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'pry-nav', '~> 0.2'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.7'
+  spec.add_development_dependency 'rubocop', '~> 0.55'
+  spec.add_development_dependency 'ruby-prof', '~> 0.17'
+  spec.add_development_dependency 'simplecov', '~> 0.16'
+
+  spec.add_runtime_dependency 'colored', '~> 1.2'
+  spec.add_runtime_dependency 'kiba', '~> 2.0'
+  spec.add_runtime_dependency 'ruby-progressbar', '~> 1.9'
+  spec.add_runtime_dependency 'rubyzip', '~> 1.2'
+  spec.add_runtime_dependency 'sqlite3', '~> 1.3'
+  spec.add_runtime_dependency 'trollop', '~> 2.1'
+end
+# rubocop:enable Metrics/BlockLength

data/lib/etl/common.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative 'csv_source'
+require_relative '../free_zipcode_data/country_table'
+require_relative '../free_zipcode_data/state_table'
+require_relative '../free_zipcode_data/county_table'
+require_relative '../free_zipcode_data/zipcode_table'
+
+def show_me
+  transform do |row|
+    ap row
+    row
+  end
+end
+
+def limit(count)
+  count = Integer(count || -1)
+  return if count == -1
+  transform do |row|
+    @counter ||= 0
+    @counter += 1
+    @counter > count ? nil : row
+  end
+end

data/lib/etl/csv_source.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+class CsvSource
+  attr_reader :filename, :delimeter, :quote_char, :headers
+
+  def initialize(filename:, headers: true, delimeter: "\t", quote_char: '"')
+    @filename   = filename
+    @headers    = headers
+    @delimeter  = delimeter
+    @quote_char = quote_char
+  end
+
+  def each
+    CSV.open(filename,
+      col_sep: delimeter,
+      headers: headers,
+      header_converters: :symbol,
+      quote_char: quote_char) do |csv|
+      csv.each do |row|
+        yield(row.to_hash)
+      end
+    end
+  end
+end

data/lib/etl/free_zipcode_data_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative 'common'
+
+module ETL
+  module FreeZipcodeDataJob
+    module_function
+
+    def setup(country_file, database, logger, options)
+      Kiba.parse do
+        pre_process do
+          logger.info("Processing '#{country_file}' data, please be patient...")
+        end
+
+        source CsvSource, filename: country_file, quote_char: '"', delimeter: ','
+
+        destination FreeZipcodeData::CountryTable,
+          database: database,
+          tablename: options[:country_tablename]
+
+        destination FreeZipcodeData::StateTable,
+          database: database,
+          tablename: options[:state_tablename]
+
+        destination FreeZipcodeData::CountyTable,
+          database: database,
+          tablename: options[:county_tablename]
+
+        destination FreeZipcodeData::ZipcodeTable,
+          database: database,
+          tablename: options[:zipcode_tablename]
+
+        post_process do
+          logger.verbose('Finished generating table data...')
+        end
+      end
+    end
+  end
+end

data/lib/free_zipcode_data/country_table.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative 'db_table'
+
+module FreeZipcodeData
+  class CountryTable < DbTable
+    def build
+      schema = <<-SQL
+        create table #{tablename} (
+          id integer not null primary key,
+          alpha2 varchar(2) not null,
+          alpha3 varchar(3),
+          iso varchar(3),
+          name varchar(255) not null
+        )
+      SQL
+      database.execute_batch(schema)
+
+      ndx = <<-SQL
+        CREATE UNIQUE INDEX "main"."unique_country_alpha2"
+        ON #{tablename} (alpha2 COLLATE NOCASE ASC);
+      SQL
+      database.execute_batch(ndx)
+    end
+
+    def write(row)
+      country_hash = country_lookup_table[row[:country]]
+
+      sql = <<-SQL
+        INSERT INTO countries (alpha2, alpha3, iso, name)
+        VALUES ('#{row[:country]}',
+          '#{country_hash[:alpha3]}',
+          '#{country_hash[:iso]}',
+          '#{country_hash[:name]}')
+      SQL
+
+      begin
+        database.execute(sql)
+      rescue SQLite3::ConstraintException
+        # Swallow duplicates
+      end
+
+      update_progress
+    end
+  end
+end

data/lib/free_zipcode_data/county_table.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative 'db_table'
+
+module FreeZipcodeData
+  class CountyTable < DbTable
+    def build
+      schema = <<-SQL
+        create table #{tablename} (
+          id integer not null primary key,
+          state_id integer,
+          abbr varchar(255),
+          name varchar(255),
+          county_seat varchar(255)
+        )
+      SQL
+      database.execute_batch(schema)
+
+      ndx = <<-SQL
+        CREATE UNIQUE INDEX "main"."unique_county"
+        ON #{tablename} (state_id, abbr, name COLLATE NOCASE ASC);
+      SQL
+      database.execute_batch(ndx)
+    end
+
+    def write(row)
+      return nil unless row[:county]
+      state_id = get_state_id(row[:short_state], row[:state])
+      return nil unless state_id
+
+      sql = <<-SQL
+        INSERT INTO counties (state_id, abbr, name)
+        VALUES ('#{state_id}',
+          '#{row[:short_county]}',
+          '#{escape_single_quotes(row[:county])}'
+        )
+      SQL
+
+      begin
+        database.execute(sql)
+      rescue SQLite3::ConstraintException
+        # swallow duplicates
+      rescue StandardError => err
+        raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
+      end
+
+      update_progress
+    end
+  end
+end

data/lib/free_zipcode_data/data_source.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'open-uri'
+require 'zip'
+
+module FreeZipcodeData
+  class DataSource
+    BASE_URL = 'http://download.geonames.org/export/zip'
+
+    attr_reader :country, :options
+
+    def initialize(country = nil)
+      @country = country
+      @options = Options.instance.hash
+      @logger  = Logger.instance
+    end
+
+    def download
+      return nil if !options.clobber && File.exist?(zipfile_path)
+      FileUtils.mkdir_p(options.work_dir)
+      @logger.info("Downloading: #{zipfile} from GeoNames...")
+      open(zipfile_path, 'wb') do |file|
+        file << open("#{BASE_URL}/#{zipfile}").read
+      end
+    end
+
+    def datafile
+      @datafile ||= begin
+        datafile_with_headers
+      end
+    end
+
+    private
+
+    def zipfile
+      @zipfile ||= begin
+        filename = country.nil? ? 'allCountries' : country.upcase
+        filename += '.zip' unless filename =~ /\.zip$/
+        filename
+      end
+    end
+
+    def zipfile_path
+      @zipfile_path ||= File.join(options.work_dir, zipfile)
+    end
+
+    def unzipped_datafile
+      @unzipped_datafile ||= begin
+        country_file = nil
+        Zip::File.open(zipfile_path) do |zip|
+          zip.each do |entry|
+            next if entry.name =~ /readme/i
+            country_file = File.join(options.work_dir, entry.name)
+            if File.exist?(country_file)
+              if options[:clobber]
+                Zip.on_exists_proc = true
+                Logger.instance.verbose("Extracting: #{zipfile}...")
+                entry.extract(country_file)
+              end
+            else
+              Logger.instance.verbose("Extracting: #{zipfile}...")
+              entry.extract(country_file)
+            end
+            break
+          end
+        end
+        country_file
+      end
+    end
+
+    def datafile_with_headers
+      filename = "#{unzipped_datafile}.csv"
+      if File.exist?(filename) && !options[:clobber]
+        @logger.verbose("File: #{filename} already exists, skipping...")
+        return filename
+      end
+      @logger.verbose("Preparing: #{filename} for processing...")
+      CSV.open(filename, 'w') do |outfile|
+        outfile << %w[COUNTRY POSTAL_CODE CITY STATE SHORT_STATE COUNTY SHORT_COUNTY COMMUNITY SHORT_COMMUNITY LATITUDE LONGITUDE ACCURACY]
+        CSV.foreach(unzipped_datafile, headers: false, col_sep: "\t", quote_char: '|') do |row|
+          outfile << row
+        end
+      end
+      filename
+    end
+  end
+end

data/lib/free_zipcode_data/db_table.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'ruby-progressbar'
+
+module FreeZipcodeData
+  class DbTable
+    ISSUE_URL = 'https://github.com/midwire/free_zipcode_data/issues/new'
+
+    attr_reader :database, :tablename
+    @@progressbar = nil
+
+    def initialize(database:, tablename:)
+      @database  = database
+      @tablename = tablename
+      lc = select_first('SELECT value FROM meta where name = "line_count"')
+      @@progressbar = ProgressBar.create(total: lc.to_i * 4, format: '%t: |%B| %e')
+    end
+
+    def update_progress
+      @@progressbar.increment
+    end
+
+    private
+
+    def country_lookup_table
+      @country_lookup_table ||= YAML.load_file('country_lookup_table.yml')
+    end
+
+    def select_first(sql)
+      rows = database.execute(sql)
+      rows[0].nil? ? nil : rows[0].first
+    rescue SQLite3::SQLException => err
+      raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
+    end
+
+    def get_country_id(country)
+      sql = "SELECT id FROM countries WHERE alpha2 = '#{country}'"
+      select_first(sql)
+    end
+
+    def get_state_id(state_abbr, state_name)
+      sql = "SELECT id FROM states
+        WHERE abbr = '#{state_abbr}' OR name = '#{escape_single_quotes(state_name)}'"
+      select_first(sql)
+    end
+
+    def get_county_id(county)
+      return nil if county.nil?
+      sql = "SELECT id FROM counties WHERE name = '#{escape_single_quotes(county)}'"
+      select_first(sql)
+    end
+
+    def escape_single_quotes(string)
+      string&.gsub(/[']/, '\'\'') || ''
+    end
+  end
+end

data/lib/free_zipcode_data/logger.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require 'logger'
+
+module FreeZipcodeData
+  class Logger
+    include Singleton
+
+    attr_accessor :log_provider
+
+    def initialize(provider = default_logger)
+      @log_provider = provider
+    end
+
+    def log_exception(e, data = {})
+      msg = "EXCEPTION : #{e.class.name} : #{e.message}"
+      msg += "\n data : #{data.inspect}" if data && !data.empty?
+      msg += "\n  #{e.backtrace[0, 6].join("\n  ")}"
+      log_provider.error(msg)
+    end
+
+    def method_missing(meth, *args, &block)
+      if log_provider.respond_to?(meth)
+        log_provider.send(meth, *args, &block)
+      else
+        super
+      end
+    end
+
+    def respond_to?(meth, include_private = false)
+      if log_provider.respond_to?(meth)
+        true
+      else
+        super
+      end
+    end
+
+    def verbose(msg)
+      info(msg) if options&.verbose
+    end
+
+    private
+
+    def default_logger
+      logger = ::Logger.new(STDOUT)
+      logger.formatter = proc do |_, _, _, msg|
+        "#{msg}\n"
+      end
+      logger
+    end
+
+    def options
+      Options.instance.hash
+    end
+  end
+end

data/lib/free_zipcode_data/options.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module FreeZipcodeData
+  class Options
+    include Singleton
+
+    def initialize_hash(hash)
+      @@_options = hash
+    end
+
+    def [](key)
+      @@_options[key]
+    end
+
+    def hash
+      @@_options
+    end
+  end
+end