free_zipcode_data 1.0.1

data/lib/free_zipcode_data/runner.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require 'colored'
+require 'trollop'
+require 'kiba'
+
+require_relative '../etl/free_zipcode_data_job'
+
+require 'pry' if ENV.fetch('APP_ENV', '') == 'development'
+
+module FreeZipcodeData
+  # rubocop:disable Metrics/ClassLength
+  class Runner
+    attr_accessor :logger, :options
+
+    # Make a singleton but allow the class to be instantiated for easier testing
+    def self.instance
+      @instance || new
+    end
+
+    def initialize
+      @logger = Logger.instance
+    end
+
+    def start
+      start_time = Time.now
+      opt = FreeZipcodeData::Options.instance
+      opt.initialize_hash(collect_args)
+      @options = opt.hash
+
+      logger.info("Starting FreeZipcodeData v#{VERSION}...".green)
+
+      datasource = DataSource.new(options.country)
+      datasource.download
+
+      db_file = File.join(options.work_dir, 'free_zipcode_data.sqlite3')
+      database = SqliteRam.new(db_file)
+      configure_meta(database.conn, datasource.datafile)
+
+      %i[country state county zipcode].each { |t| initialize_table(t, database) }
+
+      extract_transform_load(datasource, database)
+
+      logger.info("Saving database to disk '#{db_file}'...")
+      database.save_to_disk
+
+      if options.generate_files
+        logger.info('Generating .csv files...')
+        database.dump_tables(options.work_dir)
+      end
+
+      elapsed = Time.at(Time.now - start_time).utc.strftime('%H:%M:%S')
+      logger.info("Processed #{datasource_line_count} zipcodes in [#{elapsed}].".yellow)
+    end
+
+    private
+
+    def initialize_table(table_sym, database)
+      tablename = options["#{table_sym}_tablename".to_sym]
+      logger.verbose("Initializing #{table_sym} table: '#{tablename}'...")
+      klass = instance_eval("#{titleize(table_sym)}Table", __FILE__, __LINE__)
+      table = klass.new(
+        database: database.conn,
+        tablename: tablename
+      )
+      table.build
+    end
+
+    def datasource_line_count(filename)
+      @datasource_line_count ||= begin
+        count = File.foreach(filename).inject(0) { |c, _line| c + 1 }
+        logger.verbose("Processing #{count} zipcodes in '#{filename}'...")
+        count
+      end
+    end
+
+    def configure_meta(database, datasource)
+      schema = <<-SQL
+        create table meta (
+          id integer not null primary key,
+          name varchar(255),
+          value varchar(255)
+        )
+      SQL
+      database.execute_batch(schema)
+
+      sql = <<-SQL
+        INSERT INTO meta (name, value)
+        VALUES ('line_count', #{datasource_line_count(datasource)})
+      SQL
+      database.execute(sql)
+    end
+
+    def extract_transform_load(datasource, database)
+      job = ETL::FreeZipcodeDataJob.setup(
+        datasource.datafile,
+        database.conn,
+        logger,
+        options
+      )
+      Kiba.run(job)
+    end
+
+    # rubocop:disable Metrics/BlockLength
+    # rubocop:disable Metrics/MethodLength
+    def collect_args
+      Trollop.options do
+        opt(
+          :work_dir,
+          'REQUIRED: Specify your work/build directory, where the SQLite and .csv files will be built',
+          type: :string, required: true, short: '-w'
+        )
+        opt(
+          :country,
+          'Specify the country code for processing, or all countries if not specified',
+          type: :string, required: false, short: '-f'
+        )
+        opt(
+          :generate_files,
+          'Generate CSV files: [counties.csv, states.csv, countries.csv, zipcodes.csv]',
+          type: :boolean, required: false, short: '-g', default: false
+        )
+        opt(
+          :country_tablename,
+          'Specify the name for the `countries` table',
+          type: :string, required: false, default: 'countries'
+        )
+        opt(
+          :state_tablename,
+          'Specify the name for the `states` table',
+          type: :string, required: false, default: 'states'
+        )
+        opt(
+          :county_tablename,
+          'Specify the name for the `counties` table',
+          type: :string, required: false, default: 'counties'
+        )
+        opt(
+          :zipcode_tablename,
+          'Specify the name for the `zipcodes` table',
+          type: :string, required: false, default: 'zipcodes'
+        )
+        opt(
+          :clobber,
+          'Overwrite existing files',
+          type: :boolean, required: false, short: '-c', default: false
+        )
+        opt(
+          :dry_run,
+          'Do not actually move or copy files',
+          type: :boolean, required: false, short: '-d',
+          default: false
+        )
+        opt(
+          :verbose,
+          'Be verbose with output',
+          type: :boolean, required: false, short: '-v',
+          default: false
+        )
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+    # rubocop:enable Metrics/BlockLength
+
+    def titleize(string)
+      ret = string.to_s.dup
+      ret[0] = ret[0].capitalize
+      ret
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/free_zipcode_data/sqlite_ram.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'sqlite3'
+require 'csv'
+
+# Open a SQlite DB, work with it in-memory and save back to disk
+class SqliteRam
+  attr_reader :filename, :conn
+
+  def initialize(sqlite_filename)
+    @filename = sqlite_filename
+    @ram_db   = SQLite3::Database.new(':memory:')
+    @file_db  = SQLite3::Database.new(sqlite_filename)
+    @conn     = @ram_db
+  end
+
+  def save_to_disk
+    backup = SQLite3::Backup.new(@file_db, 'main', @ram_db, 'main')
+    backup.step(-1)
+    backup.finish
+  end
+
+  def dump_tables(path)
+    tables = conn.execute('select name from sqlite_master where type = "table"')
+    sql = nil
+    tables.each do |table_array|
+      table = table_array.first
+      headers_sql = "pragma table_info('#{table}')"
+      header = conn.execute(headers_sql).map { |e| e[1] }
+      CSV.open(File.join(path, "#{table}.csv"), 'w') do |csv|
+        csv << header
+        sql = "select * from #{table}"
+        conn.execute(sql).each do |row_array|
+          csv << row_array
+        end
+      end
+    end
+  end
+end

data/lib/free_zipcode_data/state_table.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative 'db_table'
+
+module FreeZipcodeData
+  class StateTable < DbTable
+    def build
+      schema = <<-SQL
+        create table #{tablename} (
+          id integer not null primary key,
+          country_id integer not null,
+          abbr varchar(2) not null,
+          name varchar(255)
+        )
+      SQL
+      database.execute_batch(schema)
+
+      ndx = <<-SQL
+        CREATE UNIQUE INDEX "main"."unique_state"
+        ON #{tablename} (abbr, country_id COLLATE NOCASE ASC);
+      SQL
+      database.execute_batch(ndx)
+
+      ndx = <<-SQL
+        CREATE UNIQUE INDEX "main"."state_name"
+        ON #{tablename} (name COLLATE NOCASE ASC);
+      SQL
+      database.execute_batch(ndx)
+    end
+
+    def write(row)
+      return nil unless row[:short_state]
+      row[:state] = 'Marshall Islands' if row[:short_state] == 'MH' && row[:state].nil?
+      country_id = get_country_id(row[:country])
+      sql = <<-SQL
+        INSERT INTO states (abbr, name, country_id)
+        VALUES ('#{row[:short_state]}',
+          '#{escape_single_quotes(row[:state])}',
+          #{country_id}
+        )
+      SQL
+      begin
+        database.execute(sql)
+      rescue SQLite3::ConstraintException
+        # Swallow duplicates
+      end
+
+      update_progress
+    end
+  end
+end

data/lib/free_zipcode_data/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module FreeZipcodeData
+  VERSION = '1.0.1'.freeze
+end

data/lib/free_zipcode_data/zipcode_table.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative 'db_table'
+
+module FreeZipcodeData
+  class ZipcodeTable < DbTable
+    def build
+      schema = <<-SQL
+        create table #{tablename} (
+          id integer not null primary key,
+          code varchar(10) not null,
+          state_id integer,
+          city varchar(255),
+          area_code varchar(3),
+          lat float,
+          lon float,
+          accuracy varchar(8)
+        )
+      SQL
+      database.execute_batch(schema)
+
+      ndx = <<-SQL
+        CREATE UNIQUE INDEX "main"."unique_zipcode"
+        ON #{tablename} (state_id, code, city COLLATE NOCASE ASC);
+      SQL
+      database.execute_batch(ndx)
+    end
+
+    def write(row)
+      return nil unless row[:postal_code]
+
+      state_id = get_state_id(row[:short_state], row[:state])
+      city_name = escape_single_quotes(row[:city])
+
+      sql = <<-SQL
+        INSERT INTO zipcodes (code, state_id, city, lat, lon, accuracy)
+        VALUES ('#{row[:postal_code]}',
+          '#{state_id}',
+          '#{city_name}',
+          '#{row[:latitude]}',
+          '#{row[:longitude]}',
+          '#{row[:accuracy]}'
+        )
+      SQL
+
+      begin
+        database.execute(sql)
+      rescue SQLite3::ConstraintException => _err
+        # there are some duplicates - swallow them
+      rescue StandardError => err
+        raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
+      end
+
+      update_progress
+    end
+  end
+end

data/lib/free_zipcode_data.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'readline'
+
+require 'free_zipcode_data/version'
+
+module FreeZipcodeData
+  def self.root
+    Pathname.new(File.dirname(__FILE__)).parent
+  end
+
+  def self.current_environment
+    ENV.fetch('APP_ENV', 'development')
+  end
+
+  #:nocov:
+  def self.config_file(filename = '.free_zipcode_data.yml')
+    return root.join('spec', 'fixtures', filename) if current_environment == 'test'
+    home = ENV.fetch('HOME')
+    file = ENV.fetch('FZD_CONFIG_FILE', File.join(home, '.free_zipcode_data.yml'))
+    FileUtils.touch(file)
+    file
+  end
+  #:nocov:
+
+  def self.os
+    if RUBY_PLATFORM.match?(/cygwin|mswin|mingw|bccwin|wince|emx/)
+      :retarded
+    else
+      :normal
+    end
+  end
+
+  autoload :CountryTable, 'free_zipcode_data/country_table'
+  autoload :StateTable,   'free_zipcode_data/state_table'
+  autoload :CountyTable,  'free_zipcode_data/county_table'
+  autoload :ZipcodeTable, 'free_zipcode_data/zipcode_table'
+  autoload :DataSource,   'free_zipcode_data/data_source'
+  autoload :Logger,       'free_zipcode_data/logger'
+  autoload :Options,      'free_zipcode_data/options'
+  autoload :Settings,     'free_zipcode_data/settings'
+  autoload :SqliteRam,    'free_zipcode_data/sqlite_ram'
+end

data/lib/tasks/version.rake

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require 'rake'
+require 'readline'
+require 'fileutils'
+
+# rubocop:disable Metrics/BlockLength
+namespace :version do
+  PROJECT_ROOT = File.expand_path(FileUtils.pwd).freeze
+  PROJECT_NAME = ENV['PROJECT_NAME'] || File.basename(PROJECT_ROOT)
+
+  desc 'Write changes to the CHANGELOG'
+  task :changes do
+    text = ask('CHANGELOG Entry:')
+    text.insert(
+      0,
+      "*#{read_version.join('.')}* (#{Time.now.strftime('%B %d, %Y')})\n\n"
+    )
+    text << "\n"
+    prepend_changelog(text)
+    launch_editor(changelog)
+  end
+
+  desc 'Increment the patch version and write changes to the changelog'
+  task :bump_patch do
+    exit unless check_branch_and_warn
+    major, minor, patch = read_version
+    patch = patch.to_i + 1
+    write_version_file([major, minor, patch])
+    update_readme_version_strings
+    Rake::Task['version:changes'].invoke
+  end
+
+  desc 'Alias for :bump_patch'
+  task bump: :bump_patch
+
+  desc 'Increment the minor version and write changes to the changelog'
+  task :bump_minor do
+    exit unless check_branch_and_warn
+    major, minor, _patch = read_version
+    minor = minor.to_i + 1
+    patch = 0
+    write_version_file([major, minor, patch])
+    update_readme_version_strings
+    Rake::Task['version:changes'].invoke
+  end
+
+  desc 'Increment the major version and write changes to the changelog'
+  task :bump_major do
+    exit unless check_branch_and_warn
+    major, _minor, _patch = read_version
+    major = major.to_i + 1
+    minor = 0
+    patch = 0
+    write_version_file([major, minor, patch])
+    update_readme_version_strings
+    Rake::Task['version:changes'].invoke
+  end
+
+  private
+
+  def version_file_path
+    split = PROJECT_NAME.split('-')
+    "#{PROJECT_ROOT}/lib/#{split.join('/')}/version.rb"
+  end
+
+  def module_name
+    if PROJECT_NAME =~ /-/
+      PROJECT_NAME.split('-').map(&:capitalize).join('::')
+    elsif PROJECT_NAME =~ /_/
+      PROJECT_NAME.split('_').map(&:capitalize).join
+    else
+      PROJECT_NAME.capitalize
+    end
+  end
+
+  def read_version
+    silence_warnings do
+      load version_file_path
+    end
+    text = eval("#{module_name}::VERSION")
+    text.split('.')
+  end
+
+  def write_version_file(version_array)
+    version = version_array.join('.')
+    new_version = %(  VERSION = '#{version}'.freeze)
+    lines = File.readlines(version_file_path)
+    File.open(version_file_path, 'w') do |f|
+      lines.each do |line|
+        if line =~ /VERSION/
+          f.write("#{new_version}\n")
+        else
+          f.write(line)
+        end
+      end
+    end
+  end
+
+  def update_readme_version_strings
+    version_string = read_version.join('.')
+    readme = open('README.md').read
+    regex = /^\*\*Version: [0-9\.]+\*\*$/i
+    return nil unless readme =~ regex
+    File.open('README.md', 'w') do |f|
+      f.write(readme.gsub(regex, "**Version: #{version_string}**"))
+    end
+  end
+
+  def changelog
+    return @changelog_path if @changelog_path
+    @changelog_path = File.join(PROJECT_ROOT, 'CHANGELOG')
+    FileUtils.touch(@changelog_path)
+    @changelog_path
+  end
+
+  def prepend_changelog(text_array)
+    old = File.read(changelog).to_s.chomp
+    text_array.push(old)
+    File.open(changelog, 'w') do |f|
+      text_array.flatten.each do |line|
+        f.puts(line)
+      end
+    end
+  end
+
+  # rubocop:disable Lint/AssignmentInCondition
+  def ask(message)
+    response = []
+    puts message
+    puts 'Hit <Control>-D when finished:'
+    while line = Readline.readline('* ', false)
+      response << "* #{line.chomp}" unless line.nil?
+    end
+    response
+  end
+  # rubocop:enable Lint/AssignmentInCondition
+
+  def current_branch
+    `git symbolic-ref --short HEAD`.chomp
+  end
+
+  def branch_warning_message
+    <<~STRING
+      You typically do not want to bump versions on the 'master' branch
+      unless you plan to rebase or back-merge into 'develop'.
+
+      If you don't care or don't know what I'm talking about just enter 'y'
+      and continue.
+
+      Optionally, you can hit 'n' to abort and switch your branch to 'develop'
+      or whatever branch you use for development, bump the version, merge to
+      'master' then 'rake release'.
+
+      Do you really want to bump the version on your 'master' branch? (y/n)
+    STRING
+  end
+
+  def check_branch_and_warn
+    return true unless current_branch == 'master'
+    puts(branch_warning_message)
+    while (line = $stdin.gets.chomp)
+      return true if line =~ /[yY]/
+      puts 'Aborting version bump.'
+      return false
+    end
+  end
+
+  def launch_editor(file)
+    system("#{ENV['EDITOR']} #{file}") if ENV['EDITOR']
+  end
+
+  def silence_warnings
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    yield
+  ensure
+    $VERBOSE = original_verbosity
+  end
+end
+# rubocop:enable Metrics/BlockLength