really-broken-geocoder 1.5.1

data/lib/geocoder/stores/mongo_mapper.rb

@@ -0,0 +1,13 @@
+require 'geocoder/stores/base'
+require 'geocoder/stores/mongo_base'
+
+module Geocoder::Store
+  module MongoMapper
+    include Base
+    include MongoBase
+
+    def self.included(base)
+      MongoBase.included_by_model(base)
+    end
+  end
+end

data/lib/geocoder/stores/mongoid.rb

@@ -0,0 +1,13 @@
+require 'geocoder/stores/base'
+require 'geocoder/stores/mongo_base'
+
+module Geocoder::Store
+  module Mongoid
+    include Base
+    include MongoBase
+
+    def self.included(base)
+      MongoBase.included_by_model(base)
+    end
+  end
+end

data/lib/geocoder/version.rb

@@ -0,0 +1,3 @@
+module Geocoder
+  VERSION = "1.5.1"
+end

data/lib/hash_recursive_merge.rb

@@ -0,0 +1,74 @@
+# 
+# = Hash Recursive Merge
+# 
+# Merges a Ruby Hash recursively, Also known as deep merge.
+# Recursive version of Hash#merge and Hash#merge!.
+# 
+# Category::    Ruby
+# Package::     Hash
+# Author::      Simone Carletti <weppos@weppos.net>
+# Copyright::   2007-2008 The Authors
+# License::     MIT License
+# Link::        http://www.simonecarletti.com/
+# Source::      http://gist.github.com/gists/6391/
+#
+module HashRecursiveMerge
+
+  #
+  # Recursive version of Hash#merge!
+  # 
+  # Adds the contents of +other_hash+ to +hsh+, 
+  # merging entries in +hsh+ with duplicate keys with those from +other_hash+.
+  # 
+  # Compared with Hash#merge!, this method supports nested hashes.
+  # When both +hsh+ and +other_hash+ contains an entry with the same key,
+  # it merges and returns the values from both arrays.
+  # 
+  #    h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+  #    h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
+  #    h1.rmerge!(h2)   #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+  #    
+  # Simply using Hash#merge! would return
+  # 
+  #    h1.merge!(h2)    #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+  # 
+  def rmerge!(other_hash)
+    merge!(other_hash) do |key, oldval, newval|
+      oldval.class == self.class ? oldval.rmerge!(newval) : newval
+    end
+  end
+
+  #
+  # Recursive version of Hash#merge
+  # 
+  # Compared with Hash#merge!, this method supports nested hashes.
+  # When both +hsh+ and +other_hash+ contains an entry with the same key,
+  # it merges and returns the values from both arrays.
+  # 
+  # Compared with Hash#merge, this method provides a different approch
+  # for merging nasted hashes.
+  # If the value of a given key is an Hash and both +other_hash+ abd +hsh
+  # includes the same key, the value is merged instead replaced with
+  # +other_hash+ value.
+  # 
+  #    h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+  #    h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
+  #    h1.rmerge(h2)    #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+  #    
+  # Simply using Hash#merge would return
+  # 
+  #    h1.merge(h2)     #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+  # 
+  def rmerge(other_hash)
+    r = {}
+    merge(other_hash) do |key, oldval, newval|
+      r[key] = oldval.class == self.class ? oldval.rmerge(newval) : newval
+    end
+  end
+
+end
+
+
+class Hash
+  include HashRecursiveMerge
+end

data/lib/maxmind_database.rb

@@ -0,0 +1,109 @@
+require 'csv'
+require 'net/http'
+
+module Geocoder
+  module MaxmindDatabase
+    extend self
+
+    def download(package, dir = "tmp")
+      filepath = File.expand_path(File.join(dir, archive_filename(package)))
+      open(filepath, 'wb') do |file|
+        uri = URI.parse(archive_url(package))
+        Net::HTTP.start(uri.host, uri.port) do |http|
+          http.request_get(uri.path) do |resp|
+            # TODO: show progress
+            resp.read_body do |segment|
+              file.write(segment)
+            end
+          end
+        end
+      end
+    end
+
+    def insert(package, dir = "tmp")
+      data_files(package, dir).each do |filepath,table|
+        print "Resetting table #{table}..."
+        ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
+        puts "done"
+        insert_into_table(table, filepath)
+      end
+    end
+
+    def archive_filename(package)
+      p = archive_url_path(package)
+      s = !(pos = p.rindex('/')).nil? && pos + 1 || 0
+      p[s..-1]
+    end
+
+    private # -------------------------------------------------------------
+
+    def table_columns(table_name)
+      {
+        maxmind_geolite_city_blocks: %w[start_ip_num end_ip_num loc_id],
+        maxmind_geolite_city_location: %w[loc_id country region city postal_code latitude longitude metro_code area_code],
+        maxmind_geolite_country: %w[start_ip end_ip start_ip_num end_ip_num country_code country]
+      }[table_name.to_sym]
+    end
+
+    def insert_into_table(table, filepath)
+      start_time = Time.now
+      print "Loading data for table #{table}"
+      rows = []
+      columns = table_columns(table)
+      CSV.foreach(filepath, encoding: "ISO-8859-1") do |line|
+        # Some files have header rows.
+        # skip if starts with "Copyright" or "locId" or "startIpNum"
+        next if line.first.match(/[A-z]/)
+        rows << line.to_a
+        if rows.size == 10000
+          insert_rows(table, columns, rows)
+          rows = []
+          print "."
+        end
+      end
+      insert_rows(table, columns, rows) if rows.size > 0
+      puts "done (#{Time.now - start_time} seconds)"
+    end
+
+    def insert_rows(table, headers, rows)
+      value_strings = rows.map do |row|
+        "(" + row.map{ |col| sql_escaped_value(col) }.join(',') + ")"
+      end
+      q = "INSERT INTO #{table} (#{headers.join(',')}) " +
+        "VALUES #{value_strings.join(',')}"
+      ActiveRecord::Base.connection.execute(q)
+    end
+
+    def sql_escaped_value(value)
+      value.to_i.to_s == value ? value :
+        ActiveRecord::Base.connection.quote(value)
+    end
+
+    def data_files(package, dir = "tmp")
+      case package
+      when :geolite_city_csv
+        # use the last two in case multiple versions exist
+        files = Dir.glob(File.join(dir, "GeoLiteCity_*/*.csv"))[-2..-1].sort
+        Hash[*files.zip(["maxmind_geolite_city_blocks", "maxmind_geolite_city_location"]).flatten]
+      when :geolite_country_csv
+        {File.join(dir, "GeoIPCountryWhois.csv") => "maxmind_geolite_country"}
+      end
+    end
+
+    def archive_url(package)
+      base_url + archive_url_path(package)
+    end
+
+    def archive_url_path(package)
+      {
+        geolite_country_csv: "GeoIPCountryCSV.zip",
+        geolite_city_csv: "GeoLiteCity_CSV/GeoLiteCity-latest.zip",
+        geolite_asn_csv: "asnum/GeoIPASNum2.zip"
+      }[package]
+    end
+
+    def base_url
+      "http://geolite.maxmind.com/download/geoip/database/"
+    end
+  end
+end

data/lib/tasks/geocoder.rake

@@ -0,0 +1,54 @@
+require "geocoder/models/mongoid"
+
+namespace :geocode do
+  desc "Geocode all objects without coordinates."
+  task :all => :environment do
+    class_name = ENV['CLASS'] || ENV['class']
+    sleep_timer = ENV['SLEEP'] || ENV['sleep']
+    batch = ENV['BATCH'] || ENV['batch']
+    reverse = ENV['REVERSE'] || ENV['reverse']
+    limit = ENV['LIMIT'] || ENV['limit']
+    raise "Please specify a CLASS (model)" unless class_name
+    klass = class_from_string(class_name)
+    batch = (batch.to_i unless batch.nil?) || 1000
+    orm = (klass < Geocoder::Model::Mongoid) ? 'mongoid' : 'active_record'
+    reverse = false unless reverse.to_s.downcase == 'true'
+
+    geocode_record = lambda { |obj|
+      reverse ? obj.reverse_geocode : obj.geocode
+      obj.save
+      sleep(sleep_timer.to_f) unless sleep_timer.nil?
+    }
+
+    scope = reverse ? klass.not_reverse_geocoded : klass.not_geocoded
+    scope = scope.limit(limit) if limit
+    if orm == 'mongoid'
+      scope.each do |obj|
+        geocode_record.call(obj)
+      end
+    elsif orm == 'active_record'
+      if limit
+        scope.each do |obj|
+          geocode_record.call(obj)
+        end
+      else
+        scope.find_each(batch_size: batch) do |obj|
+          geocode_record.call(obj)
+        end
+      end
+    end
+
+  end
+end
+##
+# Get a class object from the string given in the shell environment.
+# Similar to ActiveSupport's +constantize+ method.
+#
+def class_from_string(class_name)
+  parts = class_name.split("::")
+  constant = Object
+  parts.each do |part|
+    constant = constant.const_get(part)
+  end
+  constant
+end

data/lib/tasks/maxmind.rake

@@ -0,0 +1,73 @@
+require 'maxmind_database'
+
+namespace :geocoder do
+  namespace :maxmind do
+    namespace :geolite do
+
+      desc "Download and load/refresh MaxMind GeoLite City data"
+      task load: [:download, :extract, :insert]
+
+      desc "Download MaxMind GeoLite City data"
+      task :download do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.download!(p, dir: ENV['DIR'] || "tmp/")
+      end
+
+      desc "Extract (unzip) MaxMind GeoLite City data"
+      task :extract do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.extract!(p, dir: ENV['DIR'] || "tmp/")
+      end
+
+      desc "Load/refresh MaxMind GeoLite City data"
+      task insert: [:environment] do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.insert!(p, dir: ENV['DIR'] || "tmp/")
+      end
+    end
+  end
+end
+
+module MaxmindTask
+  extend self
+
+  def check_for_package!
+    if %w[city country].include?(p = ENV['PACKAGE'])
+      return p
+    else
+      puts "Please specify PACKAGE=city or PACKAGE=country"
+      exit
+    end
+  end
+
+  def download!(package, options = {})
+    p = "geolite_#{package}_csv".intern
+    Geocoder::MaxmindDatabase.download(p, options[:dir])
+  end
+
+  def extract!(package, options = {})
+    begin
+      require 'zip'
+    rescue LoadError
+      puts "Please install gem: rubyzip (>= 1.0.0)"
+      exit
+    end
+    require 'fileutils'
+    p = "geolite_#{package}_csv".intern
+    archive_filename = Geocoder::MaxmindDatabase.archive_filename(p)
+    Zip::File.open(File.join(options[:dir], archive_filename)).each do |entry|
+      filepath = File.join(options[:dir], entry.name)
+      if File.exist? filepath
+        warn "File already exists (#{entry.name}), skipping"
+      else
+        FileUtils.mkdir_p(File.dirname(filepath))
+        entry.extract(filepath)
+      end
+    end
+  end
+
+  def insert!(package, options = {})
+    p = "geolite_#{package}_csv".intern
+    Geocoder::MaxmindDatabase.insert(p, options[:dir])
+  end
+end

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: really-broken-geocoder
+version: !ruby/object:Gem::Version
+  version: 1.5.1
+platform: ruby
+authors:
+- Alex Reisner
+- Lindsey Dew
+- Daniel Le Dosquet-Bergquist
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-02-05 00:00:00.000000000 Z
+dependencies: []
+description: Provides object geocoding (by street or IP address), reverse geocoding
+  (coordinates to street address), distance queries for ActiveRecord and Mongoid,
+  result caching, and more. Designed for Rails but works with Sinatra and other Rack
+  frameworks too.
+email:
+- dev@makersacademy.com
+executables:
+- geocode
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- bin/geocode
+- examples/autoexpire_cache_dalli.rb
+- examples/autoexpire_cache_redis.rb
+- examples/cache_bypass.rb
+- examples/reverse_geocode_job.rb
+- lib/generators/geocoder/config/config_generator.rb
+- lib/generators/geocoder/config/templates/initializer.rb
+- lib/generators/geocoder/maxmind/geolite_city_generator.rb
+- lib/generators/geocoder/maxmind/geolite_country_generator.rb
+- lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb
+- lib/generators/geocoder/maxmind/templates/migration/geolite_country.rb
+- lib/generators/geocoder/migration_version.rb
+- lib/geocoder.rb
+- lib/geocoder/cache.rb
+- lib/geocoder/calculations.rb
+- lib/geocoder/cli.rb
+- lib/geocoder/configuration.rb
+- lib/geocoder/configuration_hash.rb
+- lib/geocoder/esri_token.rb
+- lib/geocoder/exceptions.rb
+- lib/geocoder/ip_address.rb
+- lib/geocoder/kernel_logger.rb
+- lib/geocoder/logger.rb
+- lib/geocoder/lookup.rb
+- lib/geocoder/lookups/amap.rb
+- lib/geocoder/lookups/baidu.rb
+- lib/geocoder/lookups/baidu_ip.rb
+- lib/geocoder/lookups/ban_data_gouv_fr.rb
+- lib/geocoder/lookups/base.rb
+- lib/geocoder/lookups/bing.rb
+- lib/geocoder/lookups/db_ip_com.rb
+- lib/geocoder/lookups/dstk.rb
+- lib/geocoder/lookups/esri.rb
+- lib/geocoder/lookups/freegeoip.rb
+- lib/geocoder/lookups/geocoder_ca.rb
+- lib/geocoder/lookups/geocoder_us.rb
+- lib/geocoder/lookups/geocodio.rb
+- lib/geocoder/lookups/geoip2.rb
+- lib/geocoder/lookups/geoportail_lu.rb
+- lib/geocoder/lookups/google.rb
+- lib/geocoder/lookups/google_places_details.rb
+- lib/geocoder/lookups/google_places_search.rb
+- lib/geocoder/lookups/google_premier.rb
+- lib/geocoder/lookups/here.rb
+- lib/geocoder/lookups/ip2location.rb
+- lib/geocoder/lookups/ipapi_com.rb
+- lib/geocoder/lookups/ipdata_co.rb
+- lib/geocoder/lookups/ipinfo_io.rb
+- lib/geocoder/lookups/ipstack.rb
+- lib/geocoder/lookups/latlon.rb
+- lib/geocoder/lookups/location_iq.rb
+- lib/geocoder/lookups/mapbox.rb
+- lib/geocoder/lookups/mapquest.rb
+- lib/geocoder/lookups/maxmind.rb
+- lib/geocoder/lookups/maxmind_geoip2.rb
+- lib/geocoder/lookups/maxmind_local.rb
+- lib/geocoder/lookups/nominatim.rb
+- lib/geocoder/lookups/opencagedata.rb
+- lib/geocoder/lookups/pelias.rb
+- lib/geocoder/lookups/pickpoint.rb
+- lib/geocoder/lookups/pointpin.rb
+- lib/geocoder/lookups/postcode_anywhere_uk.rb
+- lib/geocoder/lookups/postcodes_io.rb
+- lib/geocoder/lookups/smarty_streets.rb
+- lib/geocoder/lookups/telize.rb
+- lib/geocoder/lookups/tencent.rb
+- lib/geocoder/lookups/test.rb
+- lib/geocoder/lookups/yandex.rb
+- lib/geocoder/models/active_record.rb
+- lib/geocoder/models/base.rb
+- lib/geocoder/models/mongo_base.rb
+- lib/geocoder/models/mongo_mapper.rb
+- lib/geocoder/models/mongoid.rb
+- lib/geocoder/query.rb
+- lib/geocoder/railtie.rb
+- lib/geocoder/request.rb
+- lib/geocoder/results/amap.rb
+- lib/geocoder/results/baidu.rb
+- lib/geocoder/results/baidu_ip.rb
+- lib/geocoder/results/ban_data_gouv_fr.rb
+- lib/geocoder/results/base.rb
+- lib/geocoder/results/bing.rb
+- lib/geocoder/results/db_ip_com.rb
+- lib/geocoder/results/dstk.rb
+- lib/geocoder/results/esri.rb
+- lib/geocoder/results/freegeoip.rb
+- lib/geocoder/results/geocoder_ca.rb
+- lib/geocoder/results/geocoder_us.rb
+- lib/geocoder/results/geocodio.rb
+- lib/geocoder/results/geoip2.rb
+- lib/geocoder/results/geoportail_lu.rb
+- lib/geocoder/results/google.rb
+- lib/geocoder/results/google_places_details.rb
+- lib/geocoder/results/google_places_search.rb
+- lib/geocoder/results/google_premier.rb
+- lib/geocoder/results/here.rb
+- lib/geocoder/results/ip2location.rb
+- lib/geocoder/results/ipapi_com.rb
+- lib/geocoder/results/ipdata_co.rb
+- lib/geocoder/results/ipinfo_io.rb
+- lib/geocoder/results/ipstack.rb
+- lib/geocoder/results/latlon.rb
+- lib/geocoder/results/location_iq.rb
+- lib/geocoder/results/mapbox.rb
+- lib/geocoder/results/mapquest.rb
+- lib/geocoder/results/maxmind.rb
+- lib/geocoder/results/maxmind_geoip2.rb
+- lib/geocoder/results/maxmind_local.rb
+- lib/geocoder/results/nominatim.rb
+- lib/geocoder/results/opencagedata.rb
+- lib/geocoder/results/pelias.rb
+- lib/geocoder/results/pickpoint.rb
+- lib/geocoder/results/pointpin.rb
+- lib/geocoder/results/postcode_anywhere_uk.rb
+- lib/geocoder/results/postcodes_io.rb
+- lib/geocoder/results/smarty_streets.rb
+- lib/geocoder/results/telize.rb
+- lib/geocoder/results/tencent.rb
+- lib/geocoder/results/test.rb
+- lib/geocoder/results/yandex.rb
+- lib/geocoder/sql.rb
+- lib/geocoder/stores/active_record.rb
+- lib/geocoder/stores/base.rb
+- lib/geocoder/stores/mongo_base.rb
+- lib/geocoder/stores/mongo_mapper.rb
+- lib/geocoder/stores/mongoid.rb
+- lib/geocoder/version.rb
+- lib/hash_recursive_merge.rb
+- lib/maxmind_database.rb
+- lib/tasks/geocoder.rake
+- lib/tasks/maxmind.rake
+homepage: http://www.rubygeocoder.com
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/alexreisner/geocoder
+  changelog_uri: https://github.com/alexreisner/geocoder/blob/master/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Complete geocoding solution for Ruby.
+test_files: []