geocoder 1.1.9 → 1.3.7

data/lib/geocoder/sql.rb

@@ -63,26 +63,27 @@ module Geocoder
     # http://www.beginningspatial.com/calculating_bearing_one_point_another
     #
     def full_bearing(latitude, longitude, lat_attr, lon_attr, options = {})
+      degrees_per_radian = Geocoder::Calculations::DEGREES_PER_RADIAN
       case options[:bearing] || Geocoder.config.distances
       when :linear
-        "CAST(" +
-          "DEGREES(ATAN2( " +
-            "RADIANS(#{lon_attr} - #{longitude.to_f}), " +
-            "RADIANS(#{lat_attr} - #{latitude.to_f})" +
-          ")) + 360 " +
-        "AS decimal) % 360"
+        "MOD(CAST(" +
+          "(ATAN2( " +
+            "((#{lon_attr} - #{longitude.to_f}) / #{degrees_per_radian}), " +
+            "((#{lat_attr} - #{latitude.to_f}) / #{degrees_per_radian})" +
+          ") * #{degrees_per_radian}) + 360 " +
+        "AS decimal), 360)"
       when :spherical
-        "CAST(" +
-          "DEGREES(ATAN2( " +
-            "SIN(RADIANS(#{lon_attr} - #{longitude.to_f})) * " +
-            "COS(RADIANS(#{lat_attr})), (" +
-              "COS(RADIANS(#{latitude.to_f})) * SIN(RADIANS(#{lat_attr}))" +
+        "MOD(CAST(" +
+          "(ATAN2( " +
+            "SIN( (#{lon_attr} - #{longitude.to_f}) / #{degrees_per_radian} ) * " +
+            "COS( (#{lat_attr}) / #{degrees_per_radian} ), (" +
+              "COS( (#{latitude.to_f}) / #{degrees_per_radian} ) * SIN( (#{lat_attr}) / #{degrees_per_radian})" +
             ") - (" +
-              "SIN(RADIANS(#{latitude.to_f})) * COS(RADIANS(#{lat_attr})) * " +
-              "COS(RADIANS(#{lon_attr} - #{longitude.to_f}))" +
+              "SIN( (#{latitude.to_f}) / #{degrees_per_radian}) * COS((#{lat_attr}) / #{degrees_per_radian}) * " +
+              "COS( (#{lon_attr} - #{longitude.to_f}) / #{degrees_per_radian})" +
             ")" +
-          ")) + 360 " +
-        "AS decimal) % 360"
+          ") * #{degrees_per_radian}) + 360 " +
+        "AS decimal), 360)"
       end
     end
 

data/lib/geocoder/stores/active_record.rb

@@ -18,14 +18,19 @@ module Geocoder::Store
 
         # scope: geocoded objects
         scope :geocoded, lambda {
-          where("#{geocoder_options[:latitude]} IS NOT NULL " +
-            "AND #{geocoder_options[:longitude]} IS NOT NULL")
+          where("#{table_name}.#{geocoder_options[:latitude]} IS NOT NULL " +
+            "AND #{table_name}.#{geocoder_options[:longitude]} IS NOT NULL")
         }
 
         # scope: not-geocoded objects
         scope :not_geocoded, lambda {
-          where("#{geocoder_options[:latitude]} IS NULL " +
-            "OR #{geocoder_options[:longitude]} IS NULL")
+          where("#{table_name}.#{geocoder_options[:latitude]} IS NULL " +
+            "OR #{table_name}.#{geocoder_options[:longitude]} IS NULL")
+        }
+
+        # scope: not-reverse geocoded objects
+        scope :not_reverse_geocoded, lambda {
+          where("#{table_name}.#{geocoder_options[:fetched_address]} IS NULL")
         }
 
         ##
@@ -45,7 +50,7 @@ module Geocoder::Store
             # If no lat/lon given we don't want any results, but we still
             # need distance and bearing columns so you can add, for example:
             # .order("distance")
-            select(select_clause(nil, "NULL", "NULL")).where(false_condition)
+            select(select_clause(nil, null_value, null_value)).where(false_condition)
           end
         }
 
@@ -64,7 +69,7 @@ module Geocoder::Store
               full_column_name(geocoder_options[:longitude])
             ))
           else
-            select(select_clause(nil, "NULL", "NULL")).where(false_condition)
+            select(select_clause(nil, null_value, null_value)).where(false_condition)
           end
         }
       end
@@ -107,31 +112,37 @@ module Geocoder::Store
       # * +:exclude+         - an object to exclude (used by the +nearbys+ method)
       # * +:distance_column+ - used to set the column name of the calculated distance.
       # * +:bearing_column+  - used to set the column name of the calculated bearing.
+      # * +:min_radius+      - the value to use as the minimum radius.
+      #                        ignored if database is sqlite.
+      #                        default is 0.0
       #
       def near_scope_options(latitude, longitude, radius = 20, options = {})
         if options[:units]
           options[:units] = options[:units].to_sym
         end
+        latitude_attribute = options[:latitude] || geocoder_options[:latitude]
+        longitude_attribute = options[:longitude] || geocoder_options[:longitude]
         options[:units] ||= (geocoder_options[:units] || Geocoder.config.units)
-        select_distance = options.fetch(:select_distance, true)
+        select_distance = options.fetch(:select_distance)  { true }
         options[:order] = "" if !select_distance && !options.include?(:order)
-        select_bearing = options.fetch(:select_bearing, true)
+        select_bearing = options.fetch(:select_bearing) { true }
         bearing = bearing_sql(latitude, longitude, options)
         distance = distance_sql(latitude, longitude, options)
-        distance_column = options.fetch(:distance_column, 'distance')
-        bearing_column = options.fetch(:bearing_column, 'bearing')
+        distance_column = options.fetch(:distance_column) { 'distance' }
+        bearing_column = options.fetch(:bearing_column)  { 'bearing' }
 
         b = Geocoder::Calculations.bounding_box([latitude, longitude], radius, options)
         args = b + [
-          full_column_name(geocoder_options[:latitude]),
-          full_column_name(geocoder_options[:longitude])
+          full_column_name(latitude_attribute),
+          full_column_name(longitude_attribute)
         ]
         bounding_box_conditions = Geocoder::Sql.within_bounding_box(*args)
 
         if using_sqlite?
           conditions = bounding_box_conditions
         else
-          conditions = [bounding_box_conditions + " AND #{distance} <= ?", radius]
+          min_radius = options.fetch(:min_radius, 0).to_f
+          conditions = [bounding_box_conditions + " AND (#{distance}) BETWEEN ? AND ?", min_radius, radius]
         end
         {
           :select => select_clause(options[:select],
@@ -153,8 +164,8 @@ module Geocoder::Store
         Geocoder::Sql.send(
           method_prefix + "_distance",
           latitude, longitude,
-          full_column_name(geocoder_options[:latitude]),
-          full_column_name(geocoder_options[:longitude]),
+          full_column_name(options[:latitude] || geocoder_options[:latitude]),
+          full_column_name(options[:longitude]|| geocoder_options[:longitude]),
           options
         )
       end
@@ -172,8 +183,8 @@ module Geocoder::Store
           Geocoder::Sql.send(
             method_prefix + "_bearing",
             latitude, longitude,
-            full_column_name(geocoder_options[:latitude]),
-            full_column_name(geocoder_options[:longitude]),
+            full_column_name(options[:latitude] || geocoder_options[:latitude]),
+            full_column_name(options[:longitude]|| geocoder_options[:longitude]),
             options
           )
         end
@@ -215,7 +226,18 @@ module Geocoder::Store
       end
 
       def using_sqlite?
-        connection.adapter_name.match /sqlite/i
+        connection.adapter_name.match(/sqlite/i)
+      end
+
+      def using_postgres?
+        connection.adapter_name.match(/postgres/i)
+      end
+
+      ##
+      # Use OID type when running in PosgreSQL
+      #
+      def null_value
+        using_postgres? ? 'NULL::text' : 'NULL'
       end
 
       ##
@@ -234,6 +256,17 @@ module Geocoder::Store
       end
     end
 
+    ##
+    # Get nearby geocoded objects.
+    # Takes the same options hash as the near class method (scope).
+    # Returns nil if the object is not geocoded.
+    #
+    def nearbys(radius = 20, options = {})
+      return nil unless geocoded?
+      options.merge!(:exclude => self) unless send(self.class.primary_key).nil?
+      self.class.near(self, radius, options)
+    end
+
     ##
     # Look up coordinates and assign to +latitude+ and +longitude+ attributes
     # (or other as specified in +geocoded_by+). Returns coordinates (array).

data/lib/geocoder/stores/base.rb

@@ -55,17 +55,6 @@ module Geocoder
           point, to_coordinates, options)
       end
 
-      ##
-      # Get nearby geocoded objects.
-      # Takes the same options hash as the near class method (scope).
-      # Returns nil if the object is not geocoded.
-      #
-      def nearbys(radius = 20, options = {})
-        return nil unless geocoded?
-        options.merge!(:exclude => self) unless send(self.class.primary_key).nil?
-        self.class.near(self, radius, options)
-      end
-
       ##
       # Look up coordinates and assign to +latitude+ and +longitude+ attributes
       # (or other as specified in +geocoded_by+). Returns coordinates (array).
@@ -101,7 +90,14 @@ module Geocoder
           return
         end
 
-        results = Geocoder.search(query)
+        query_options = [:lookup, :ip_lookup, :language].inject({}) do |hash, key|
+          if options.has_key?(key)
+            val = options[key]
+            hash[key] = val.respond_to?(:call) ? val.call(self) : val
+          end
+          hash
+        end
+        results = Geocoder.search(query, query_options)
 
         # execute custom block, if specified in configuration
         block_key = reverse ? :reverse_block : :geocode_block

data/lib/geocoder/stores/mongo_base.rb

@@ -11,37 +11,6 @@ module Geocoder::Store
         scope :not_geocoded, lambda {
           where(geocoder_options[:coordinates] => nil)
         }
-
-        scope :near, lambda{ |location, *args|
-          coords  = Geocoder::Calculations.extract_coordinates(location)
-
-          # no results if no lat/lon given
-          return where(:id => false) unless coords.is_a?(Array)
-
-          radius  = args.size > 0 ? args.shift : 20
-          options = args.size > 0 ? args.shift : {}
-          options[:units] ||= geocoder_options[:units]
-
-          # Use BSON::OrderedHash if Ruby's hashes are unordered.
-          # Conditions must be in order required by indexes (see mongo gem).
-          version = RUBY_VERSION.split('.').map { |i| i.to_i }
-          empty = version[0] < 2 && version[1] < 9 ? BSON::OrderedHash.new : {}
-
-          conds = empty.clone
-          field = geocoder_options[:coordinates]
-          conds[field] = empty.clone
-          conds[field]["$nearSphere"]  = coords.reverse
-
-          if radius
-            conds[field]["$maxDistance"] = \
-              Geocoder::Calculations.distance_to_radians(radius, options[:units])
-          end
-
-          if obj = options[:exclude]
-            conds[:_id.ne] = obj.id
-          end
-          where(conds)
-        }
       end
     end
 

data/lib/geocoder/version.rb

@@ -1,3 +1,3 @@
 module Geocoder
-  VERSION = "1.1.9"
+  VERSION = "1.3.7"
 end

data/lib/geocoder.rb

@@ -1,21 +1,23 @@
 require "geocoder/configuration"
+require "geocoder/logger"
+require "geocoder/kernel_logger"
 require "geocoder/query"
 require "geocoder/calculations"
 require "geocoder/exceptions"
 require "geocoder/cache"
 require "geocoder/request"
 require "geocoder/lookup"
+require "geocoder/ip_address"
 require "geocoder/models/active_record" if defined?(::ActiveRecord)
 require "geocoder/models/mongoid" if defined?(::Mongoid)
 require "geocoder/models/mongo_mapper" if defined?(::MongoMapper)
 
 module Geocoder
-  extend self
 
   ##
   # Search for information about an address or a set of coordinates.
   #
-  def search(query, options = {})
+  def self.search(query, options = {})
     query = Geocoder::Query.new(query, options) unless query.is_a?(Geocoder::Query)
     query.blank? ? [] : query.execute
   end
@@ -23,7 +25,7 @@ module Geocoder
   ##
   # Look up the coordinates of the given street or IP address.
   #
-  def coordinates(address, options = {})
+  def self.coordinates(address, options = {})
     if (results = search(address, options)).size > 0
       results.first.coordinates
     end
@@ -33,23 +35,14 @@ module Geocoder
   # Look up the address of the given coordinates ([lat,lon])
   # or IP address (string).
   #
-  def address(query, options = {})
+  def self.address(query, options = {})
     if (results = search(query, options)).size > 0
       results.first.address
     end
   end
-
-  ##
-  # The working Cache object, or +nil+ if none configured.
-  #
-  def cache
-    warn "WARNING: Calling Geocoder.cache is DEPRECATED. The #cache method now belongs to the Geocoder::Lookup object."
-    Geocoder::Lookup.get(Geocoder.config.lookup).cache
-  end
 end
 
 # load Railtie if Rails exists
 if defined?(Rails)
   require "geocoder/railtie"
-  Geocoder::Railtie.insert
 end

data/lib/maxmind_database.rb

@@ -0,0 +1,109 @@
+require 'csv'
+require 'net/http'
+
+module Geocoder
+  module MaxmindDatabase
+    extend self
+
+    def download(package, dir = "tmp")
+      filepath = File.expand_path(File.join(dir, archive_filename(package)))
+      open(filepath, 'wb') do |file|
+        uri = URI.parse(archive_url(package))
+        Net::HTTP.start(uri.host, uri.port) do |http|
+          http.request_get(uri.path) do |resp|
+            # TODO: show progress
+            resp.read_body do |segment|
+              file.write(segment)
+            end
+          end
+        end
+      end
+    end
+
+    def insert(package, dir = "tmp")
+      data_files(package, dir).each do |filepath,table|
+        print "Resetting table #{table}..."
+        ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
+        puts "done"
+        insert_into_table(table, filepath)
+      end
+    end
+
+    def archive_filename(package)
+      p = archive_url_path(package)
+      s = !(pos = p.rindex('/')).nil? && pos + 1 || 0
+      p[s..-1]
+    end
+
+    private # -------------------------------------------------------------
+
+    def table_columns(table_name)
+      {
+        maxmind_geolite_city_blocks: %w[start_ip_num end_ip_num loc_id],
+        maxmind_geolite_city_location: %w[loc_id country region city postal_code latitude longitude metro_code area_code],
+        maxmind_geolite_country: %w[start_ip end_ip start_ip_num end_ip_num country_code country]
+      }[table_name.to_sym]
+    end
+
+    def insert_into_table(table, filepath)
+      start_time = Time.now
+      print "Loading data for table #{table}"
+      rows = []
+      columns = table_columns(table)
+      CSV.foreach(filepath, encoding: "ISO-8859-1") do |line|
+        # Some files have header rows.
+        # skip if starts with "Copyright" or "locId" or "startIpNum"
+        next if line.first.match(/[A-z]/)
+        rows << line.to_a
+        if rows.size == 10000
+          insert_rows(table, columns, rows)
+          rows = []
+          print "."
+        end
+      end
+      insert_rows(table, columns, rows) if rows.size > 0
+      puts "done (#{Time.now - start_time} seconds)"
+    end
+
+    def insert_rows(table, headers, rows)
+      value_strings = rows.map do |row|
+        "(" + row.map{ |col| sql_escaped_value(col) }.join(',') + ")"
+      end
+      q = "INSERT INTO #{table} (#{headers.join(',')}) " +
+        "VALUES #{value_strings.join(',')}"
+      ActiveRecord::Base.connection.execute(q)
+    end
+
+    def sql_escaped_value(value)
+      value.to_i.to_s == value ? value :
+        ActiveRecord::Base.connection.quote(value)
+    end
+
+    def data_files(package, dir = "tmp")
+      case package
+      when :geolite_city_csv
+        # use the last two in case multiple versions exist
+        files = Dir.glob(File.join(dir, "GeoLiteCity_*/*.csv"))[-2..-1].sort
+        Hash[*files.zip(["maxmind_geolite_city_blocks", "maxmind_geolite_city_location"]).flatten]
+      when :geolite_country_csv
+        {File.join(dir, "GeoIPCountryWhois.csv") => "maxmind_geolite_country"}
+      end
+    end
+
+    def archive_url(package)
+      base_url + archive_url_path(package)
+    end
+
+    def archive_url_path(package)
+      {
+        geolite_country_csv: "GeoIPCountryCSV.zip",
+        geolite_city_csv: "GeoLiteCity_CSV/GeoLiteCity-latest.zip",
+        geolite_asn_csv: "asnum/GeoIPASNum2.zip"
+      }[package]
+    end
+
+    def base_url
+      "http://geolite.maxmind.com/download/geoip/database/"
+    end
+  end
+end

data/lib/tasks/geocoder.rake

@@ -3,12 +3,23 @@ namespace :geocode do
   task :all => :environment do
     class_name = ENV['CLASS'] || ENV['class']
     sleep_timer = ENV['SLEEP'] || ENV['sleep']
+    batch = ENV['BATCH'] || ENV['batch']
+    reverse = ENV['REVERSE'] || ENV['reverse']
     raise "Please specify a CLASS (model)" unless class_name
     klass = class_from_string(class_name)
+    batch = batch.to_i unless batch.nil?
+    reverse = false unless reverse.to_s.downcase == 'true'
 
-    klass.not_geocoded.each do |obj|
-      obj.geocode; obj.save
-      sleep(sleep_timer.to_f) unless sleep_timer.nil?
+    if reverse
+      klass.not_reverse_geocoded.find_each(batch_size: batch) do |obj|
+        obj.reverse_geocode; obj.save
+        sleep(sleep_timer.to_f) unless sleep_timer.nil?
+      end
+    else
+      klass.not_geocoded.find_each(batch_size: batch) do |obj|
+        obj.geocode; obj.save
+        sleep(sleep_timer.to_f) unless sleep_timer.nil?
+      end
     end
   end
 end

data/lib/tasks/maxmind.rake

@@ -0,0 +1,73 @@
+require 'maxmind_database'
+
+namespace :geocoder do
+  namespace :maxmind do
+    namespace :geolite do
+
+      desc "Download and load/refresh MaxMind GeoLite City data"
+      task load: [:download, :extract, :insert]
+
+      desc "Download MaxMind GeoLite City data"
+      task :download do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.download!(p, dir: ENV['DIR'] || "tmp/")
+      end
+
+      desc "Extract (unzip) MaxMind GeoLite City data"
+      task :extract do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.extract!(p, dir: ENV['DIR'] || "tmp/")
+      end
+
+      desc "Load/refresh MaxMind GeoLite City data"
+      task insert: [:environment] do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.insert!(p, dir: ENV['DIR'] || "tmp/")
+      end
+    end
+  end
+end
+
+module MaxmindTask
+  extend self
+
+  def check_for_package!
+    if %w[city country].include?(p = ENV['PACKAGE'])
+      return p
+    else
+      puts "Please specify PACKAGE=city or PACKAGE=country"
+      exit
+    end
+  end
+
+  def download!(package, options = {})
+    p = "geolite_#{package}_csv".intern
+    Geocoder::MaxmindDatabase.download(p, options[:dir])
+  end
+
+  def extract!(package, options = {})
+    begin
+      require 'zip'
+    rescue LoadError
+      puts "Please install gem: rubyzip (>= 1.0.0)"
+      exit
+    end
+    require 'fileutils'
+    p = "geolite_#{package}_csv".intern
+    archive_filename = Geocoder::MaxmindDatabase.archive_filename(p)
+    Zip::File.open(File.join(options[:dir], archive_filename)).each do |entry|
+      filepath = File.join(options[:dir], entry.name)
+      if File.exist? filepath
+        warn "File already exists (#{entry.name}), skipping"
+      else
+        FileUtils.mkdir_p(File.dirname(filepath))
+        entry.extract(filepath)
+      end
+    end
+  end
+
+  def insert!(package, options = {})
+    p = "geolite_#{package}_csv".intern
+    Geocoder::MaxmindDatabase.insert(p, options[:dir])
+  end
+end