broken-geocoder 1.3.4

data/lib/geocoder/stores/base.rb

@@ -0,0 +1,116 @@
+module Geocoder
+  module Store
+    module Base
+
+      ##
+      # Is this object geocoded? (Does it have latitude and longitude?)
+      #
+      def geocoded?
+        to_coordinates.compact.size > 0
+      end
+
+      ##
+      # Coordinates [lat,lon] of the object.
+      #
+      def to_coordinates
+        [:latitude, :longitude].map{ |i| send self.class.geocoder_options[i] }
+      end
+
+      ##
+      # Calculate the distance from the object to an arbitrary point.
+      # See Geocoder::Calculations.distance_between for ways of specifying
+      # the point. Also takes a symbol specifying the units
+      # (:mi or :km; can be specified in Geocoder configuration).
+      #
+      def distance_to(point, units = nil)
+        units ||= self.class.geocoder_options[:units]
+        return nil unless geocoded?
+        Geocoder::Calculations.distance_between(
+          to_coordinates, point, :units => units)
+      end
+
+      alias_method :distance_from, :distance_to
+
+      ##
+      # Calculate the bearing from the object to another point.
+      # See Geocoder::Calculations.distance_between for
+      # ways of specifying the point.
+      #
+      def bearing_to(point, options = {})
+        options[:method] ||= self.class.geocoder_options[:method]
+        return nil unless geocoded?
+        Geocoder::Calculations.bearing_between(
+          to_coordinates, point, options)
+      end
+
+      ##
+      # Calculate the bearing from another point to the object.
+      # See Geocoder::Calculations.distance_between for
+      # ways of specifying the point.
+      #
+      def bearing_from(point, options = {})
+        options[:method] ||= self.class.geocoder_options[:method]
+        return nil unless geocoded?
+        Geocoder::Calculations.bearing_between(
+          point, to_coordinates, options)
+      end
+
+      ##
+      # Look up coordinates and assign to +latitude+ and +longitude+ attributes
+      # (or other as specified in +geocoded_by+). Returns coordinates (array).
+      #
+      def geocode
+        fail
+      end
+
+      ##
+      # Look up address and assign to +address+ attribute (or other as specified
+      # in +reverse_geocoded_by+). Returns address (string).
+      #
+      def reverse_geocode
+        fail
+      end
+
+      private # --------------------------------------------------------------
+
+      ##
+      # Look up geographic data based on object attributes (configured in
+      # geocoded_by or reverse_geocoded_by) and handle the results with the
+      # block (given to geocoded_by or reverse_geocoded_by). The block is
+      # given two-arguments: the object being geocoded and an array of
+      # Geocoder::Result objects).
+      #
+      def do_lookup(reverse = false)
+        options = self.class.geocoder_options
+        if reverse and options[:reverse_geocode]
+          query = to_coordinates
+        elsif !reverse and options[:geocode]
+          query = send(options[:user_address])
+        else
+          return
+        end
+
+        query_options = [:lookup, :ip_lookup, :language].inject({}) do |hash, key|
+          if options.has_key?(key)
+            val = options[key]
+            hash[key] = val.respond_to?(:call) ? val.call(self) : val
+          end
+          hash
+        end
+        results = Geocoder.search(query, query_options)
+
+        # execute custom block, if specified in configuration
+        block_key = reverse ? :reverse_block : :geocode_block
+        if custom_block = options[block_key]
+          custom_block.call(self, results)
+
+        # else execute block passed directly to this method,
+        # which generally performs the "auto-assigns"
+        elsif block_given?
+          yield(self, results)
+        end
+      end
+    end
+  end
+end
+

data/lib/geocoder/stores/mongo_base.rb

@@ -0,0 +1,58 @@
+module Geocoder::Store
+  module MongoBase
+
+    def self.included_by_model(base)
+      base.class_eval do
+
+        scope :geocoded, lambda {
+          where(geocoder_options[:coordinates].ne => nil)
+        }
+
+        scope :not_geocoded, lambda {
+          where(geocoder_options[:coordinates] => nil)
+        }
+      end
+    end
+
+    ##
+    # Coordinates [lat,lon] of the object.
+    # This method always returns coordinates in lat,lon order,
+    # even though internally they are stored in the opposite order.
+    #
+    def to_coordinates
+      coords = send(self.class.geocoder_options[:coordinates])
+      coords.is_a?(Array) ? coords.reverse : []
+    end
+
+    ##
+    # Look up coordinates and assign to +latitude+ and +longitude+ attributes
+    # (or other as specified in +geocoded_by+). Returns coordinates (array).
+    #
+    def geocode
+      do_lookup(false) do |o,rs|
+        if r = rs.first
+          unless r.coordinates.nil?
+            o.__send__ "#{self.class.geocoder_options[:coordinates]}=", r.coordinates.reverse
+          end
+          r.coordinates
+        end
+      end
+    end
+
+    ##
+    # Look up address and assign to +address+ attribute (or other as specified
+    # in +reverse_geocoded_by+). Returns address (string).
+    #
+    def reverse_geocode
+      do_lookup(true) do |o,rs|
+        if r = rs.first
+          unless r.address.nil?
+            o.__send__ "#{self.class.geocoder_options[:fetched_address]}=", r.address
+          end
+          r.address
+        end
+      end
+    end
+  end
+end
+

data/lib/geocoder/stores/mongo_mapper.rb

@@ -0,0 +1,13 @@
+require 'geocoder/stores/base'
+require 'geocoder/stores/mongo_base'
+
+module Geocoder::Store
+  module MongoMapper
+    include Base
+    include MongoBase
+
+    def self.included(base)
+      MongoBase.included_by_model(base)
+    end
+  end
+end

data/lib/geocoder/stores/mongoid.rb

@@ -0,0 +1,13 @@
+require 'geocoder/stores/base'
+require 'geocoder/stores/mongo_base'
+
+module Geocoder::Store
+  module Mongoid
+    include Base
+    include MongoBase
+
+    def self.included(base)
+      MongoBase.included_by_model(base)
+    end
+  end
+end

data/lib/geocoder/version.rb

@@ -0,0 +1,3 @@
+module Geocoder
+  VERSION = "1.3.4"
+end

data/lib/hash_recursive_merge.rb

@@ -0,0 +1,74 @@
+# 
+# = Hash Recursive Merge
+# 
+# Merges a Ruby Hash recursively, Also known as deep merge.
+# Recursive version of Hash#merge and Hash#merge!.
+# 
+# Category::    Ruby
+# Package::     Hash
+# Author::      Simone Carletti <weppos@weppos.net>
+# Copyright::   2007-2008 The Authors
+# License::     MIT License
+# Link::        http://www.simonecarletti.com/
+# Source::      http://gist.github.com/gists/6391/
+#
+module HashRecursiveMerge
+
+  #
+  # Recursive version of Hash#merge!
+  # 
+  # Adds the contents of +other_hash+ to +hsh+, 
+  # merging entries in +hsh+ with duplicate keys with those from +other_hash+.
+  # 
+  # Compared with Hash#merge!, this method supports nested hashes.
+  # When both +hsh+ and +other_hash+ contains an entry with the same key,
+  # it merges and returns the values from both arrays.
+  # 
+  #    h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+  #    h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
+  #    h1.rmerge!(h2)   #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+  #    
+  # Simply using Hash#merge! would return
+  # 
+  #    h1.merge!(h2)    #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+  # 
+  def rmerge!(other_hash)
+    merge!(other_hash) do |key, oldval, newval|
+      oldval.class == self.class ? oldval.rmerge!(newval) : newval
+    end
+  end
+
+  #
+  # Recursive version of Hash#merge
+  # 
+  # Compared with Hash#merge!, this method supports nested hashes.
+  # When both +hsh+ and +other_hash+ contains an entry with the same key,
+  # it merges and returns the values from both arrays.
+  # 
+  # Compared with Hash#merge, this method provides a different approch
+  # for merging nasted hashes.
+  # If the value of a given key is an Hash and both +other_hash+ abd +hsh
+  # includes the same key, the value is merged instead replaced with
+  # +other_hash+ value.
+  # 
+  #    h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+  #    h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
+  #    h1.rmerge(h2)    #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+  #    
+  # Simply using Hash#merge would return
+  # 
+  #    h1.merge(h2)     #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+  # 
+  def rmerge(other_hash)
+    r = {}
+    merge(other_hash) do |key, oldval, newval|
+      r[key] = oldval.class == self.class ? oldval.rmerge(newval) : newval
+    end
+  end
+
+end
+
+
+class Hash
+  include HashRecursiveMerge
+end

data/lib/maxmind_database.rb

@@ -0,0 +1,109 @@
+require 'csv'
+require 'net/http'
+
+module Geocoder
+  module MaxmindDatabase
+    extend self
+
+    def download(package, dir = "tmp")
+      filepath = File.expand_path(File.join(dir, archive_filename(package)))
+      open(filepath, 'wb') do |file|
+        uri = URI.parse(archive_url(package))
+        Net::HTTP.start(uri.host, uri.port) do |http|
+          http.request_get(uri.path) do |resp|
+            # TODO: show progress
+            resp.read_body do |segment|
+              file.write(segment)
+            end
+          end
+        end
+      end
+    end
+
+    def insert(package, dir = "tmp")
+      data_files(package, dir).each do |filepath,table|
+        print "Resetting table #{table}..."
+        ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
+        puts "done"
+        insert_into_table(table, filepath)
+      end
+    end
+
+    def archive_filename(package)
+      p = archive_url_path(package)
+      s = !(pos = p.rindex('/')).nil? && pos + 1 || 0
+      p[s..-1]
+    end
+
+    private # -------------------------------------------------------------
+
+    def table_columns(table_name)
+      {
+        maxmind_geolite_city_blocks: %w[start_ip_num end_ip_num loc_id],
+        maxmind_geolite_city_location: %w[loc_id country region city postal_code latitude longitude metro_code area_code],
+        maxmind_geolite_country: %w[start_ip end_ip start_ip_num end_ip_num country_code country]
+      }[table_name.to_sym]
+    end
+
+    def insert_into_table(table, filepath)
+      start_time = Time.now
+      print "Loading data for table #{table}"
+      rows = []
+      columns = table_columns(table)
+      CSV.foreach(filepath, encoding: "ISO-8859-1") do |line|
+        # Some files have header rows.
+        # skip if starts with "Copyright" or "locId" or "startIpNum"
+        next if line.first.match(/[A-z]/)
+        rows << line.to_a
+        if rows.size == 10000
+          insert_rows(table, columns, rows)
+          rows = []
+          print "."
+        end
+      end
+      insert_rows(table, columns, rows) if rows.size > 0
+      puts "done (#{Time.now - start_time} seconds)"
+    end
+
+    def insert_rows(table, headers, rows)
+      value_strings = rows.map do |row|
+        "(" + row.map{ |col| sql_escaped_value(col) }.join(',') + ")"
+      end
+      q = "INSERT INTO #{table} (#{headers.join(',')}) " +
+        "VALUES #{value_strings.join(',')}"
+      ActiveRecord::Base.connection.execute(q)
+    end
+
+    def sql_escaped_value(value)
+      value.to_i.to_s == value ? value :
+        ActiveRecord::Base.connection.quote(value)
+    end
+
+    def data_files(package, dir = "tmp")
+      case package
+      when :geolite_city_csv
+        # use the last two in case multiple versions exist
+        files = Dir.glob(File.join(dir, "GeoLiteCity_*/*.csv"))[-2..-1].sort
+        Hash[*files.zip(["maxmind_geolite_city_blocks", "maxmind_geolite_city_location"]).flatten]
+      when :geolite_country_csv
+        {File.join(dir, "GeoIPCountryWhois.csv") => "maxmind_geolite_country"}
+      end
+    end
+
+    def archive_url(package)
+      base_url + archive_url_path(package)
+    end
+
+    def archive_url_path(package)
+      {
+        geolite_country_csv: "GeoIPCountryCSV.zip",
+        geolite_city_csv: "GeoLiteCity_CSV/GeoLiteCity-latest.zip",
+        geolite_asn_csv: "asnum/GeoIPASNum2.zip"
+      }[package]
+    end
+
+    def base_url
+      "http://geolite.maxmind.com/download/geoip/database/"
+    end
+  end
+end

data/lib/tasks/geocoder.rake

@@ -0,0 +1,38 @@
+namespace :geocode do
+  desc "Geocode all objects without coordinates."
+  task :all => :environment do
+    class_name = ENV['CLASS'] || ENV['class']
+    sleep_timer = ENV['SLEEP'] || ENV['sleep']
+    batch = ENV['BATCH'] || ENV['batch']
+    reverse = ENV['REVERSE'] || ENV['reverse']
+    raise "Please specify a CLASS (model)" unless class_name
+    klass = class_from_string(class_name)
+    batch = batch.to_i unless batch.nil?
+    reverse = false unless reverse.to_s.downcase == 'true'
+
+    if reverse
+      klass.not_reverse_geocoded.find_each(batch_size: batch) do |obj|
+        obj.reverse_geocode; obj.save
+        sleep(sleep_timer.to_f) unless sleep_timer.nil?
+      end
+    else
+      klass.not_geocoded.find_each(batch_size: batch) do |obj|
+        obj.geocode; obj.save
+        sleep(sleep_timer.to_f) unless sleep_timer.nil?
+      end
+    end
+  end
+end
+
+##
+# Get a class object from the string given in the shell environment.
+# Similar to ActiveSupport's +constantize+ method.
+#
+def class_from_string(class_name)
+  parts = class_name.split("::")
+  constant = Object
+  parts.each do |part|
+    constant = constant.const_get(part)
+  end
+  constant
+end

data/lib/tasks/maxmind.rake

@@ -0,0 +1,73 @@
+require 'maxmind_database'
+
+namespace :geocoder do
+  namespace :maxmind do
+    namespace :geolite do
+
+      desc "Download and load/refresh MaxMind GeoLite City data"
+      task load: [:download, :extract, :insert]
+
+      desc "Download MaxMind GeoLite City data"
+      task :download do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.download!(p, dir: ENV['DIR'] || "tmp/")
+      end
+
+      desc "Extract (unzip) MaxMind GeoLite City data"
+      task :extract do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.extract!(p, dir: ENV['DIR'] || "tmp/")
+      end
+
+      desc "Load/refresh MaxMind GeoLite City data"
+      task insert: [:environment] do
+        p = MaxmindTask.check_for_package!
+        MaxmindTask.insert!(p, dir: ENV['DIR'] || "tmp/")
+      end
+    end
+  end
+end
+
+module MaxmindTask
+  extend self
+
+  def check_for_package!
+    if %w[city country].include?(p = ENV['PACKAGE'])
+      return p
+    else
+      puts "Please specify PACKAGE=city or PACKAGE=country"
+      exit
+    end
+  end
+
+  def download!(package, options = {})
+    p = "geolite_#{package}_csv".intern
+    Geocoder::MaxmindDatabase.download(p, options[:dir])
+  end
+
+  def extract!(package, options = {})
+    begin
+      require 'zip'
+    rescue LoadError
+      puts "Please install gem: rubyzip (>= 1.0.0)"
+      exit
+    end
+    require 'fileutils'
+    p = "geolite_#{package}_csv".intern
+    archive_filename = Geocoder::MaxmindDatabase.archive_filename(p)
+    Zip::File.open(File.join(options[:dir], archive_filename)).each do |entry|
+      filepath = File.join(options[:dir], entry.name)
+      if File.exist? filepath
+        warn "File already exists (#{entry.name}), skipping"
+      else
+        FileUtils.mkdir_p(File.dirname(filepath))
+        entry.extract(filepath)
+      end
+    end
+  end
+
+  def insert!(package, options = {})
+    p = "geolite_#{package}_csv".intern
+    Geocoder::MaxmindDatabase.insert(p, options[:dir])
+  end
+end