geocode_records 0.1.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f9481740c101b666c731e1071a7d019b3e5f580
-  data.tar.gz: c2f36a4a3908b26fdfab3fcba6f07892a8193ae3
+  metadata.gz: 1cfba62c3b057b69bc8180fe3d09d7fa0a09c931
+  data.tar.gz: 9ab706ef30253318866feb80a9d2fa5ee947ace7
 SHA512:
-  metadata.gz: 7a3189caf4b574054a078989e2cc5b23ebd0773525eee869c7e2da1ac6ef7a6257a8104c3d5b857d952a7cd16f2315796210474f0e45c54a5e79d554730fc503
-  data.tar.gz: 7410d0746fbd76029a0cd573804abe6bd5a553e46c1a8d2a84415b3fb8090da7ca7bec25a3a8888b0edb248cdc99385b7bc60cf7c981e05de03d071df81edf48
+  metadata.gz: 2a99ba57d347697a74160599861696897b564bc8f528dcccab87eccb0c5127a9743380c769e15ef66a6d94fb9dea693a526a1cd97ac86183169d778a5b3a254b
+  data.tar.gz: c10936fe5170800b77bc186099666a9dadd3e909074c9af39956519b90a1d5f864d929cbac964047ddbbfc7def9ad05bb149c7cdff66f5bc5f12fcb44437f665

data/.gitignore

@@ -14,3 +14,4 @@
 mkmf.log
 /node_modules/
 log/*
+package-lock.json

data/CHANGELOG

@@ -1,3 +1,16 @@
+1.0.0 / 2018-01-16
+
+* Breaking changes
+
+  * Totally change the API
+  * Require smartystreets 1.8.2
+  * Require pgloader to be in your path
+  * Require psql to be in your path
+
+* Enhancements
+
+  * Way faster for big files
+
 0.1.5 / 2017-10-15
 
 * Enhancements

data/README.md

@@ -1,31 +1,7 @@
 # GeocodeRecords
 
-TODO: Write a gem description
+As long as you do very specific things... quickly re-geocode tables.
 
-## Installation
+## Copyright
 
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'geocode_records'
-```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install geocode_records
-
-## Usage
-
-TODO: Write usage instructions here
-
-## Contributing
-
-1. Fork it ( https://github.com/[my-github-username]/geocode_records/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+Copyright 2018 Seamus Abshere

data/geocode_records.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Seamus Abshere"]
   spec.email         = ["seamus@abshere.net"]
   spec.summary       = %q{Geocode an ActiveRecord::Relation with node_smartystreets}
-  spec.description   = %q{A quick way to re-geocode a table}
+  spec.description   = %q{A quick way to re-geocode a table. Requires 2 binaries, so YMMV.}
   spec.homepage      = "https://github.com/seamusabshere/geocode_records"
   spec.license       = "MIT"
 
@@ -17,18 +17,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
-
-  spec.add_runtime_dependency 'activerecord', '>=4.1.9'
+  
   spec.add_runtime_dependency 'activesupport'
-  spec.add_runtime_dependency 'pg'
-  spec.add_runtime_dependency 'attr_extras'
-  spec.add_runtime_dependency 'zaru'
-  spec.add_runtime_dependency 'upsert'
-  spec.add_runtime_dependency 'pasqual'
   
+  spec.add_development_dependency 'activerecord', '>=4.1.9'
+  spec.add_development_dependency 'pg', '~>0.21'
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "pry"
-
 end

data/lib/geocode_records/dump_sql_to_csv.rb

@@ -1,18 +1,38 @@
-class DumpSqlToCsv
-  attr_private :sql
-  attr_private :pasqual
+class GeocodeRecords
+  class DumpSqlToCsv
+    attr_reader :database_url
+    attr_reader :glob
+    attr_reader :table_name
+    attr_reader :subquery
 
-  def initialize(pasqual, sql, ignored_options = {})
-    @pasqual = pasqual
-    @sql = sql
-  end
+    def initialize(
+      database_url:,
+      glob:,
+      table_name: nil,
+      subquery: nil)
+      @database_url = database_url
+      @glob = glob
+      @table_name = table_name
+      @subquery = subquery
+    end
 
-  def path
-    @path = Dir::Tmpname.create(sql[0,64].delete('"').gsub(/\W/,'_').squeeze) {}
+    def perform
+      memo = GeocodeRecords.new_tmp_path(subquery || table_name)
+      GeocodeRecords.run_sql(
+        database_url,
+        "\\copy (#{sql}) TO '#{memo}' DELIMITER ',' CSV HEADER" 
+      )
+      memo
+    end
 
-    pasqual.command "\\copy (#{sql}) TO '#{@path}' DELIMITER ',' CSV HEADER"
+    private
 
-    @path
+    def sql
+      @sql ||= unless glob
+        "SELECT id, house_number_and_street, city, state, regexp_replace(postcode, '.0$', '') AS postcode FROM #{subquery ? "(#{subquery}) t1" : table_name} WHERE city IS NOT NULL OR postcode IS NOT NULL"
+      else
+        "SELECT id, glob FROM #{subquery ? "(#{subquery}) t1" : table_name} WHERE (city IS NULL AND postcode IS NULL) AND glob IS NOT NULL"
+      end
+    end
   end
-
 end

data/lib/geocode_records/geocode_csv.rb

@@ -1,41 +1,62 @@
-require 'tmpdir'
-require 'fileutils'
-require 'csv'
-require 'shellwords'
-require 'zaru'
+require 'json'
 
-require 'geocode_records/smarty_streets'
-
-# copied from hotdog/app/services/file_geocoder.rb with seamus variations
 class GeocodeRecords
   class GeocodeCsv
+    attr_reader :path
     attr_reader :glob
     attr_reader :include_invalid
 
-    def initialize(input_path, options = {})
-      @input_path = input_path
-      options ||= {}
-      @glob = options[:glob]
-      @include_invalid = options[:include_invalid]
-      @mutex = Mutex.new
+    REQUIRED_SMARTYSTREETS_VERSION = Gem::Version.new('1.8.2')
+    COLUMN_DEFINITION = {
+      delivery_line_1: true,
+      components: {
+        primary_number: true,
+        secondary_number: true,
+        city_name: true,
+        default_city_name: true,
+        state_abbreviation: true,
+        zipcode: true
+      },
+      metadata: {
+        latitude: true,
+        longitude: true
+      }
+    }
+
+    def initialize(
+      path:,
+      glob:,
+      include_invalid:
+    )
+      @path = path
+      @glob = glob
+      @include_invalid = include_invalid
     end
 
-    def path
-      return if @path
-      @mutex.synchronize do
-        return if @path
-        geocode
-        recode
-        @path = @recoded_path
+    def perform
+      return unless File.size(path) > 32
+      memo = GeocodeRecords.new_tmp_path File.basename("geocoded-#{path}")
+      args = [
+        smartystreets_bin_path,
+        '-i', path,
+        '-o', memo,
+        '--quiet',
+        '--auth-id', ENV.fetch('SMARTY_STREETS_AUTH_ID'),
+        '--auth-token', ENV.fetch('SMARTY_STREETS_AUTH_TOKEN'),
+        '--column-definition', JSON.dump(COLUMN_DEFINITION),
+      ]
+      if include_invalid
+        args += [ '--include-invalid' ]
       end
+      input_map.each do |ss, local|
+        args += [ "--#{ss}-col", local.to_s ]
+      end
+      GeocodeRecords.system(*args)
+      memo
     end
 
     private
 
-    attr_private :input_path
-    attr_private :geocoded_path
-    attr_private :recoded_path
-
     def input_map
       @input_map ||= if glob
         { 'street' => 'glob' }
@@ -47,55 +68,18 @@ class GeocodeRecords
       end
     end
 
-    def geocode
-      @geocoded_path = Dir::Tmpname.create(Zaru.sanitize!(input_path + '.geocode')) {}
-      args = [
-        '-i', input_path,
-        '-o', geocoded_path,
-        '--auth-id', ENV.fetch('SMARTY_STREETS_AUTH_ID'),
-        '--auth-token', ENV.fetch('SMARTY_STREETS_AUTH_TOKEN')
-      ]
-      if include_invalid
-        args += [ '--include-invalid' ]
-      end
-      input_map.each do |ss, local|
-        args += [ "--#{ss}-col", local.to_s ]
-      end
-      SmartyStreets.run *args
-      raise "Geocoding failed on #{input_path.inspect} with args #{Shellwords.join(args)}" unless $?.success?
-    end
-
-    def recode
-      @recoded_path = Dir::Tmpname.create(Zaru.sanitize!(input_path + '.recode')) {}
-      File.open(@recoded_path, 'w') do |f|
-        f.write output_columns.to_csv
-        CSV.foreach(@geocoded_path, headers: true) do |geocoded_row|
-          f.write recode_columns.map { |k| geocoded_row[k] }.to_csv
+    def smartystreets_bin_path
+      @smartystreets_bin_path ||= begin
+        memo = [
+          'node_modules/.bin/smartystreets',
+          `which smartystreets`.chomp
+        ].compact.detect do |path|
+          File.exist? path
         end
-      end
-      File.unlink @geocoded_path
-    end
-
-    def output_columns
-      @output_columns ||= (File.open(input_path) { |f| CSV.parse_line(f.gets) } + RECODE_MAP.keys).uniq
-    end
-
-    # no street yet - street_name, street_suffix
-    RECODE_MAP = {
-      'house_number_and_street' => 'ss_delivery_line_1',
-      'house_number' => 'ss_primary_number',
-      'unit_number' => 'ss_secondary_number',
-      'city' => 'ss_city_name',
-      'state' => 'ss_state_abbreviation',
-      'postcode' => 'ss_zipcode',
-      'latitude' => 'ss_latitude',
-      'longitude' => 'ss_longitude',
-      'default_city' => 'ss_default_city_name',
-    }.freeze
-
-    def recode_columns
-      @recode_columns ||= output_columns.map do |output_k|
-        RECODE_MAP[output_k] || output_k
+        raise "can't find smartystreets bin" unless memo
+        version = Gem::Version.new `#{memo} -V`.chomp
+        raise "smartystreets #{version} too old" unless version >= REQUIRED_SMARTYSTREETS_VERSION
+        memo
       end
     end
   end

data/lib/geocode_records/update_table_from_csv.rb

@@ -1,36 +1,130 @@
 require 'csv'
-require 'upsert'
 
 class GeocodeRecords
   class UpdateTableFromCsv
-    MAX_INT = 2**31 - 1
-    attr_private :connection
-    attr_private :table_name
-    attr_private :csv_path
-    attr_private :upsert
-    def initialize(connection, table_name, csv_path, ignored_options = {})
-      @upsert = Upsert.new connection, table_name
-      @csv_path = csv_path
+    CREATE_TABLE_SQL = (<<-SQL).gsub('      ', '').freeze
+      CREATE TABLE $TMP_TABLE_NAME (
+        id uuid primary key,
+        ss_delivery_line_1 text,
+        ss_primary_number text,
+        ss_secondary_number text,
+        ss_city_name text,
+        ss_state_abbreviation text,
+        ss_zipcode text,
+        ss_latitude float,
+        ss_longitude float,
+        ss_default_city_name text
+      )
+    SQL
+
+    PGLOADER_CONFIG = <<-SQL
+      LOAD CSV
+      FROM '$PATH'
+      (
+        $INPUT_COLUMNS
+      )
+      INTO $DATABASE_URL?$TMP_TABLE_NAME
+      (
+        id,
+        ss_delivery_line_1,
+        ss_primary_number,
+        ss_secondary_number,
+        ss_city_name,
+        ss_state_abbreviation,
+        ss_zipcode,
+        ss_latitude,
+        ss_longitude,
+        ss_default_city_name
+      )
+      WITH
+        skip header = 1,
+        fields optionally enclosed by '"',
+        fields escaped by double-quote,
+        fields terminated by ','
+      SET client_encoding to 'utf8';
+    SQL
+
+    UPDATE_TABLE_SQL = (<<-SQL).gsub('      ', '').freeze
+      UPDATE $TABLE_NAME AS target
+      SET
+        house_number_and_street = src.ss_delivery_line_1,
+        house_number = CASE WHEN LENGTH(src.ss_primary_number) > 7 THEN NULL ELSE src.ss_primary_number::int END,
+        unit_number = src.ss_secondary_number,
+        city = COALESCE(src.ss_default_city_name, src.ss_city_name),
+        state = src.ss_state_abbreviation,
+        postcode = src.ss_zipcode,
+        latitude = src.ss_latitude,
+        longitude = src.ss_longitude
+      FROM $TMP_TABLE_NAME AS src
+      WHERE
+            target.id = src.id
+        AND src.ss_zipcode IS NOT NULL
+    SQL
+
+    attr_reader :database_url
+    attr_reader :table_name
+    attr_reader :path
+
+    def initialize(
+      database_url:,
+      table_name:,
+      path:
+    )
+      @database_url = database_url
+      @table_name = table_name
+      @path = path
     end
+
     def perform
-      count = 0
-      CSV.foreach(csv_path, headers: true) do |row|
-        next unless row['postcode']
-        row = row.to_hash
-        if hn = row['house_number']
-          row['house_number'] = hn.to_i
-          next if row['house_number'] > MAX_INT
-        end
-        if default_city = row.delete('default_city')
-          row['city'] = default_city
-        end
-        selector = { id: row.delete('id') }
-        setter = row
-        upsert.row selector, setter
-        # $stderr.write "U#{count}..." if count % 1000 == 0
-        count += 1
+      return unless File.size(path) > 32
+      tmp_table_name = create_tmp_table
+      begin
+        load_csv_into_tmp_table tmp_table_name
+        update_original_table tmp_table_name
+      ensure
+        delete_tmp_table tmp_table_name
+      end
+    end
+
+    def create_tmp_table
+      memo = "geocode_records_#{table_name}_#{rand(999999)}"
+      GeocodeRecords.run_sql(
+        database_url,
+        CREATE_TABLE_SQL.sub('$TMP_TABLE_NAME', memo)
+      )
+      memo
+    end
+
+    def load_csv_into_tmp_table(tmp_table_name)
+      pg_loader_config_path = GeocodeRecords.new_tmp_path('pgloader')
+      File.open(pg_loader_config_path, 'w') { |f| f.write PGLOADER_CONFIG.sub('$INPUT_COLUMNS', input_columns.join(',')).sub('$DATABASE_URL', database_url).sub('$TMP_TABLE_NAME', tmp_table_name).sub('$PATH', path) }
+      GeocodeRecords.system(
+        'pgloader',
+        # '--debug',
+        '--quiet',
+        pg_loader_config_path
+      )
+      File.unlink pg_loader_config_path
+    end
+
+    def update_original_table(tmp_table_name)
+      GeocodeRecords.run_sql(
+        database_url,
+        UPDATE_TABLE_SQL.sub('$TMP_TABLE_NAME', tmp_table_name).sub('$TABLE_NAME', table_name)
+      )
+    end
+
+    def delete_tmp_table(tmp_table_name)
+      GeocodeRecords.run_sql(
+        database_url,
+        "DROP TABLE IF EXISTS #{tmp_table_name}"
+      )
+    end
+
+    def input_columns
+      CSV.parse_line(File.open(path) { |f| f.gets }).map do |col|
+        "#{col} [NULL IF BLANKS]"
       end
-      count
     end
   end
 end

data/lib/geocode_records/version.rb

@@ -1,3 +1,3 @@
 class GeocodeRecords
-  VERSION = '0.1.5'
+  VERSION = '1.0.0'
 end

data/lib/geocode_records.rb

@@ -1,77 +1,95 @@
-require 'active_record'
 require 'active_support'
 require 'active_support/core_ext'
-require 'attr_extras'
-require 'pasqual'
+require 'tmpdir'
+require 'shellwords'
+require 'fileutils'
 
 require_relative 'geocode_records/version'
 require_relative 'geocode_records/dump_sql_to_csv'
 require_relative 'geocode_records/geocode_csv'
 require_relative 'geocode_records/update_table_from_csv'
-require_relative 'geocode_records/smarty_streets'
 
 class GeocodeRecords
+  class << self
+    def new_tmp_path(hint)
+      Dir::Tmpname.create(hint[0,64].delete('"').gsub(/\W/,'_').squeeze) {}
+    end
 
-  attr_reader :records
-  attr_reader :options
-  def initialize(records, options = {})
-    records.is_a?(ActiveRecord::Relation) or raise(ArgumentError, "expected AR::Relation, got #{records.class}")
-    @options = (options || {}).symbolize_keys
-    @records = records
-  end
-  
-  def perform
-    SmartyStreets.check_compatible!
+    def system(*args)
+      result = Kernel.system(*args)
+      unless result
+        raise "failed command:\n#{Shellwords.join args}"
+      end
+      nil
+    end
 
-    if records.count > 0
-      # $stderr.puts "GeocodeRecords: #{records.count} to go!"
-      ungeocoded_path = DumpSqlToCsv.new(pasqual, to_sql, options).path
-      geocoded_path = GeocodeCsv.new(ungeocoded_path, options).path
-      UpdateTableFromCsv.new(connection, table_name, geocoded_path, options).perform
-      set_the_geom
-      File.unlink geocoded_path
-      File.unlink ungeocoded_path
+    def run_sql(database_url, sql)
+      system(
+        'psql',
+        database_url,
+        '-v', 'ON_ERROR_STOP=on',
+        # '--echo-all',
+        '--quiet',
+        '--no-psqlrc',
+        '--pset', 'pager=off',
+        '--command', sql
+      )
     end
   end
 
-  private
+  attr_reader :database_url
+  attr_reader :table_name
 
-  def glob
-    !!options[:glob]
-  end
+  # optional
+  attr_reader :include_invalid
+  attr_reader :subquery
 
-  def set_the_geom
-    records.update_all <<-SQL
-      the_geom              = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326),
-      the_geom_webmercator  = ST_Transform(ST_SetSRID(ST_MakePoint(longitude, latitude), 4326), 3857)
-    SQL
+   def initialize(
+    database_url:,
+    table_name:,
+    subquery: nil,
+    include_invalid: nil
+  )
+    @database_url = database_url
+    @table_name = table_name
+    @subquery = subquery
+    @include_invalid = include_invalid
   end
-
-  def to_sql
-    c = connection
-    c.unprepared_statement do
-      if glob
-        c.to_sql records.select('id', 'glob').where.not(glob: nil).arel, records.bind_values
-      else
-        c.to_sql records.select('id', 'house_number_and_street', 'house_number', 'unit_number', 'city', 'state', "regexp_replace(postcode, '.0$', '') AS postcode").where('city IS NOT NULL OR postcode IS NOT NULL').arel, records.bind_values
-      end
-    end
+  
+  def perform
+    geocode glob: false
+    geocode glob: true
   end
 
-  def connection
-    records.connection
-  end
+  private
 
-  def table_name
-    @table_name = begin
-      memo = options[:table_name]
-      memo ||= records.table_name if records.respond_to?(:table_name)
-      memo ||= records.engine.table_name
-      memo
+  def geocode(glob:)
+    ungeocoded_path = nil
+    geocoded_path = nil
+    begin
+      ungeocoded_path = DumpSqlToCsv.new(
+        database_url: database_url,
+        table_name: table_name,
+        subquery: subquery,
+        glob: glob
+      ).perform
+      unless File.size(ungeocoded_path) > 32
+        $stderr.puts "No records found for #{table_name} #{subquery}, skipping"
+        return
+      end
+      geocoded_path = GeocodeCsv.new(
+        path: ungeocoded_path,
+        glob: glob,
+        include_invalid: include_invalid
+      ).perform
+      UpdateTableFromCsv.new(
+        database_url: database_url,
+        table_name: table_name,
+        path: geocoded_path
+      ).perform
+    ensure
+      FileUtils.rm_f geocoded_path if geocoded_path
+      FileUtils.rm_f ungeocoded_path if ungeocoded_path
     end
   end
-
-  def pasqual
-    @pasqual ||= Pasqual.for ENV.fetch('DATABASE_URL')
-  end
 end

data/package.json

@@ -2,6 +2,6 @@
   "name": "rolodeck",
   "private": true,
   "dependencies": {
-    "smartystreets": "^1.7.2"
+    "smartystreets": "^1.8.2"
   }
 }

data/spec/geocode_records_spec.rb

@@ -1,132 +1,85 @@
 require 'spec_helper'
 
-dbname = 'geocode_records_test'
-ENV['DATABASE_URL'] = "postgresql://127.0.0.1/#{dbname}"
-
-unless ENV['FAST'] == 'true'
-  psql = Pasqual.for ENV['DATABASE_URL']
-  psql.dropdb rescue nil
-  psql.createdb
-  psql.command 'CREATE EXTENSION postgis'
-  psql.command <<-SQL
-  CREATE TABLE homes (
-    id serial primary key,
-    the_geom geometry(Geometry,4326),
-    the_geom_webmercator geometry(Geometry,3857),
-    house_number_and_street text,
-    house_number int,
-    unit_number text,
-    city text,
-    state text,
-    postcode text,
-    latitude float,
-    longitude float
-  )
-  SQL
-  psql.command <<-SQL
-  CREATE TABLE glob_homes (
-    id serial primary key,
-    glob text,
-    the_geom geometry(Geometry,4326),
-    the_geom_webmercator geometry(Geometry,3857),
-    house_number_and_street text,
-    house_number int,
-    unit_number text,
-    city text,
-    state text,
-    postcode text,
-    latitude float,
-    longitude float
-  )
-  SQL
-end
-
-require 'active_record'
-ActiveRecord::Base.establish_connection
-
-require 'logger'
-require 'fileutils'
-FileUtils.mkdir_p 'log'
-logger = Logger.new 'log/test.log'
-ActiveRecord::Base.logger = logger
-
-class Home < ActiveRecord::Base
-end
-class GlobHome < ActiveRecord::Base
-end
-
 describe GeocodeRecords do
-  it 'has a version number' do
-    expect(GeocodeRecords::VERSION).not_to be nil
-  end
-
-  it "geocodes an AR::Relation" do
+  subject { GeocodeRecords.new(database_url: ENV.fetch('DATABASE_URL'), table_name: 'homes').perform }
+  
+  it "geocodes" do
     home = Home.create! house_number_and_street: '1038 e deyton st', postcode: '53703'
-    GeocodeRecords.new(Home.all).perform
+    subject
     home.reload
     expect(home.house_number_and_street).to eq('1038 E Dayton St')
   end
 
-  it "geocodes an AR::Relation with just a glob" do
-    home = GlobHome.create! glob: '1038 e dayton st, madison, wi 53703'
-    GeocodeRecords.new(GlobHome.all, glob: true).perform
+  it "geocodes glob" do
+    home = Home.create! glob: '1038 e dayton st, madison, wi 53703'
+    subject
     home.reload
     expect(home.house_number_and_street).to eq('1038 E Dayton St')
     expect(home.postcode).to eq('53703')
   end
 
+  it "geocodes by sql" do
+    home = Home.create! house_number_and_street: '1038 e deyton st', postcode: '53703', foo: 'bar'
+    home_ignored = Home.create! house_number_and_street: '1038 e deyton st', postcode: '53703'
+    GeocodeRecords.new(database_url: ENV.fetch('DATABASE_URL'), table_name: 'homes', subquery: %{SELECT * FROM homes WHERE foo = 'bar'}).perform  
+    home.reload
+    home_ignored.reload
+    expect(home.latitude).to be_present
+    expect(home_ignored.latitude).to be_nil
+  end
+
   it "doesn't break on float-format postcode" do
     home = Home.create! house_number_and_street: '1038 e deyton st', postcode: '53703.0'
-    GeocodeRecords.new(Home.all).perform
+    subject
     home.reload
     expect(home.house_number_and_street).to eq('1038 E Dayton St')
   end
 
   it "doesn't break on unzeropadded postcode" do
     home = Home.create! house_number_and_street: '36 main st', postcode: '5753'
-    GeocodeRecords.new(Home.all).perform
+    subject
     home.reload
     expect(home.house_number_and_street).to eq('36 Main St')
   end
 
   it "doesn't break on unzeropadded float-format postcode" do
     home = Home.create! house_number_and_street: '36 main st', postcode: '5753.0'
-    GeocodeRecords.new(Home.all).perform
+    subject
     home.reload
     expect(home.house_number_and_street).to eq('36 Main St')
   end
 
   it "doesn't break on zip-4" do
     home = Home.create! house_number_and_street: '1038 e dayton st', postcode: '53703-2428'
-    GeocodeRecords.new(Home.all).perform
+    subject
     home.reload
     expect(home.house_number_and_street).to eq('1038 E Dayton St')
   end
 
   it "accepts city and state only" do
     home = Home.create! house_number_and_street: '1038 e dayton st', city: 'madison', state: 'wisconsin'
-    GeocodeRecords.new(Home.all).perform
+    subject
     home.reload
     expect(home.house_number_and_street).to eq('1038 E Dayton St')
   end
 
   it "allows invalid" do
     home = Home.create! house_number_and_street: '1039 e dayton st', city: 'madison', state: 'wisconsin'
-    GeocodeRecords.new(Home.all, include_invalid: true).perform
+    GeocodeRecords.new(database_url: ENV.fetch('DATABASE_URL'), table_name: 'homes', include_invalid: true).perform
     home.reload
     expect(home.house_number_and_street).to eq('1039 E Dayton St')
   end
 
   it "overwrites unit" do
     home = Home.create! house_number_and_street: '123 n blount st apt 403', city: 'madison', state: 'wisconsin'
-    GeocodeRecords.new(Home.all, include_invalid: true).perform
+    GeocodeRecords.new(database_url: ENV.fetch('DATABASE_URL'), table_name: 'homes', include_invalid: true).perform
     home.reload
     expect(home.house_number_and_street).to eq('123 N Blount St Unit 403')
   end
 
   it "overwrites city name with default_city_name" do
     home = Home.create! house_number_and_street: '7333 Bay Bridge Rd', city: 'eastvale', state: 'ca'
-    GeocodeRecords.new(Home.all, include_invalid: true).perform
+    GeocodeRecords.new(database_url: ENV.fetch('DATABASE_URL'), table_name: 'homes', include_invalid: true).perform
     home.reload
     expect(home.city).to eq('Corona')
   end
@@ -134,7 +87,7 @@ describe GeocodeRecords do
   describe 'known issues' do
     it "doesn't fix float-format postcode on records that it can't geocode" do
       home = Home.create! house_number_and_street: 'gibberish', postcode: '53703.0'
-      GeocodeRecords.new(Home.all).perform
+      subject
       home.reload
       expect(home.house_number_and_street).to eq('gibberish')
       expect(home.postcode).to eq('53703.0')
@@ -142,7 +95,7 @@ describe GeocodeRecords do
 
     it "doesn't fix unzeropadded postcode on records that it can't geocode" do
       home = Home.create! house_number_and_street: 'gibberish', postcode: '5753'
-      GeocodeRecords.new(Home.all).perform
+      subject
       home.reload
       expect(home.house_number_and_street).to eq('gibberish')
       expect(home.postcode).to eq('5753')

data/spec/spec_helper.rb

@@ -2,3 +2,62 @@ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'geocode_records'
 
 require 'pry'
+
+dbname = 'geocode_records_test'
+ENV['DATABASE_URL'] = "postgresql://127.0.0.1:#{ENV['PGPORT'] || 5432}/#{dbname}"
+
+unless ENV['FAST'] == 'true'
+  GeocodeRecords.system('createdb', ENV.fetch('DATABASE_URL')) rescue nil
+  GeocodeRecords.run_sql(
+    ENV.fetch('DATABASE_URL'),
+    'CREATE EXTENSION IF NOT EXISTS postgis'
+  )
+  GeocodeRecords.run_sql(
+    ENV.fetch('DATABASE_URL'),
+    'DROP TABLE IF EXISTS homes'
+  )
+  sql = <<-SQL
+    CREATE TABLE homes (
+      id uuid primary key,
+      the_geom geometry(Geometry,4326),
+      the_geom_webmercator geometry(Geometry,3857),
+      glob text,
+      house_number_and_street text,
+      house_number int,
+      unit_number text,
+      city text,
+      state text,
+      postcode text,
+      latitude float,
+      longitude float,
+      foo text
+    )
+  SQL
+  GeocodeRecords.run_sql(
+    ENV.fetch('DATABASE_URL'),
+    sql
+  )
+end
+
+require 'active_record'
+ActiveRecord::Base.establish_connection
+
+require 'logger'
+require 'fileutils'
+FileUtils.mkdir_p 'log'
+logger = Logger.new 'log/test.log'
+ActiveRecord::Base.logger = logger
+
+require 'securerandom'
+class Home < ActiveRecord::Base
+  self.primary_key = 'id'
+  before_create do
+    self.id ||= SecureRandom.uuid
+  end
+end
+
+RSpec.configure do |config|
+  config.before :each do |example|
+    Home.delete_all
+  end
+end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: geocode_records
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 1.0.0
 platform: ruby
 authors:
 - Seamus Abshere
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-15 00:00:00.000000000 Z
+date: 2018-01-16 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.1.9
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.1.9
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -39,75 +25,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pg
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: attr_extras
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: zaru
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: upsert
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: 4.1.9
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.1.9
 - !ruby/object:Gem::Dependency
-  name: pasqual
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '0.21'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.21'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -164,7 +108,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: A quick way to re-geocode a table
+description: A quick way to re-geocode a table. Requires 2 binaries, so YMMV.
 email:
 - seamus@abshere.net
 executables:
@@ -185,11 +129,9 @@ files:
 - lib/geocode_records.rb
 - lib/geocode_records/dump_sql_to_csv.rb
 - lib/geocode_records/geocode_csv.rb
-- lib/geocode_records/smarty_streets.rb
 - lib/geocode_records/update_table_from_csv.rb
 - lib/geocode_records/version.rb
 - package.json
-- spec/geocode_records/smarty_streets_spec.rb
 - spec/geocode_records_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/seamusabshere/geocode_records
@@ -212,11 +154,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Geocode an ActiveRecord::Relation with node_smartystreets
 test_files:
-- spec/geocode_records/smarty_streets_spec.rb
 - spec/geocode_records_spec.rb
 - spec/spec_helper.rb

data/lib/geocode_records/smarty_streets.rb

@@ -1,38 +0,0 @@
-class GeocodeRecords
-
-  module SmartyStreets
-
-    VERSION = '1.7.2'
-
-    def self.bin_path
-      @bin_path ||= if File.exist?('node_modules/.bin/smartystreets')
-        'node_modules/.bin/smartystreets'
-      else
-        'smartystreets'
-      end
-    end
-
-    def self.check_compatible!
-      raise "smartystreets >= #{VERSION} is required" unless SmartyStreets.compatible?
-    end
-
-    def self.compatible?
-      output = run_with_output('-V')
-      current_version = Gem::Version.new output.chomp
-      min_version = Gem::Version.new VERSION
-      current_version >= min_version
-    end
-
-    def self.run(*args)
-      shargs = Shellwords.join(args)
-      system "#{bin_path} #{shargs}"
-    end
-    
-    def self.run_with_output(*args)
-      shargs = Shellwords.join(args)
-      `#{bin_path} #{shargs}`
-    end
-
-  end
-
-end

data/spec/geocode_records/smarty_streets_spec.rb

@@ -1,39 +0,0 @@
-require 'spec_helper'
-
-require 'geocode_records/smarty_streets'
-
-describe GeocodeRecords::SmartyStreets do
-
-  describe '.bin_path' do
-    subject { described_class.bin_path }
-
-    it { is_expected.to eq 'node_modules/.bin/smartystreets' }
-  end
-
-  describe '.compatible?' do
-    before { allow(described_class).to receive(:run_with_output).and_return("#{version}\n") }
-
-    subject { described_class.compatible? }
-
-    context 'v1.3.1' do
-      let(:version) { '1.3.1' }
-
-      it { is_expected.to be false }
-    end
-
-    context 'v1.7.2' do
-      let(:version) { '1.7.2' }
-
-      it { is_expected.to be true }
-    end
-    
-  end
-
-  describe '.run_with_output' do
-    subject { described_class.run_with_output '-V' }
-
-    it { is_expected.to match /\d+\.\d+.\d+/ }
-  end
-
-end
-