geonames_local 1.0.0 → 2.0.0

data/Gemfile

@@ -3,6 +3,7 @@ gemspec # Specify  gem's dependencies in mongoid_geospatial.gemspec
 
 group :development do
   gem 'mongoid', '~> 3.0'
+  gem 'mongoid_geospatial'
   # gem 'pg'
   gem 'rgeo'
   gem 'georuby'

data/README.rdoc

@@ -1,6 +1,6 @@
 = Geonames Local
 
-Download and store (pg, mongo, tokyo) Geonames.org data.
+Downloads and store Geonames.org data locally (MongoDB, PostGIS, Tokyo).
 Making every Geoname API operation possible on your servers.
 No hit limit, fast as possible.
 
@@ -10,7 +10,7 @@ No hit limit, fast as possible.
 To use one adapter, install the corresponding gem:
 
   PostgreSQL   =>   pg
-  MongoDB      =>   mongo (optional: mongo_ext)
+  MongoDB      =>   mongoid (optional: mongo_ext)
   Tokyo        =>   tokyocabinet
 
 You will also need in your system:
@@ -19,17 +19,6 @@ You will also need in your system:
 * curl
 
 
-=== PostgreSQL
-
-Be sure to use a database based on the PostGIS template.
-
-
-=== MongoDB
-
-MongoDB 2D support is new, only mongo >= 1.3.3 mongodb gem >= 0.19.2
-http://github.com/mongodb/mongo-ruby-driver
-
-
 === Config YML
 
   geonames conf
@@ -45,26 +34,23 @@ If you are not sure your country code, use:
 
   geonames list <search>
 
+To populate the countries database for the first time use:
 
-== Relational Mapping
+  geonames -c geoconfig.yml countries
 
-When using PG, this gem will (try) to relational map Geonames
-data on your scheme.
 
-== ActiveRecord
+== Mongoid
 
-Check out lib/geonames/models/ar.rb
-
-I`m wondering the best way (and if it`s a good idea) to auto include'em on rails.
-For now, just copy as you see fit.
+So, supposing Mongoid, something like this is possible:
 
+  City.first.province.country.abbr
+  => "BR"
 
-== PostgreSQL
 
-So, supposing ActiveRecord, something like this is possible:
+== Postgis
 
-  City.first.province.country.abbr
-  => "BR"
+TBD (by someone else)
+Be sure to use a database based on the PostGIS template.
 
 
 === Migration

data/bin/geonames

@@ -3,7 +3,8 @@
 # $KCODE = "u" # -Ku
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 
-require "geonames_cli"
+require "geonames_local"
+require "geonames_local/cli"
 include Geonames
 
 Geonames::CLI.work(ARGV)

data/geonames.yml

@@ -1,7 +1,7 @@
 #
 # Geonames Local Config Example
 #
-:store: mongodb # postgres / tokyo
+:store: mongodb # postgis / tokyo
 :codes: [br] # to store all countries informations in the countries table, just write [country]
 :level: city
 :min_pop: 100000
@@ -10,7 +10,7 @@
   :geom: true
 :db:
   :host: localhost
-  :dbname: geonames_test
+  :name: geonames_test
   :user:
-  :password:
+  :pass:
   :purge: false

data/lib/geonames_local.rb

@@ -5,14 +5,9 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 # Require Libs
-require 'geonames_local/geonames'
-
-require 'geonames_local/adapters/mongodb'
-
-module Geonames
+require 'geonames_local/features/spot'
+require 'geonames_local/features/road'
+require 'geonames_local/features/zone'
 
-  Adapter = Geonames::Mongodb.new
-
-end
-
-require 'geonames_local/models/mongo'
+# Require Main
+require 'geonames_local/geonames'

data/lib/geonames_local/cli.rb

@@ -2,6 +2,14 @@
 # Geonames Local
 #
 require 'optparse'
+# Require CLI Stuff
+require 'geonames_local/geonames'
+require 'geonames_local/data/shp'
+require 'geonames_local/data/dump'
+require 'geonames_local/data/export'
+require 'geonames_local/cli'
+
+
 module Geonames
   class CLI
     def self.parse_options(argv)
@@ -113,76 +121,44 @@ BANNER
       # Do the magic! Import Geonames Data
       #
       else
-        db = load_adapter(Opt[:store])
+        load_adapter(Opt[:store])
         info "Using adapter #{Opt[:store]}.."
 
-        if Opt[:codes][0] == "country"
-          Geonames::Dump.work(Opt[:codes], :dump)
+        if argv[0] =~ /coun|nati/
+          dump = Geonames::Dump.new(:country, :dump)
+          info "\n---\nTotal #{dump.data.length} parsed."
+
+          info "Writing to DB"
+          Geonames::Models::Country.from_batch(dump.data)
         else
-          Geonames::Dump.work(Opt[:codes], :zip)
-          Geonames::Dump.work(Opt[:codes], :dump)
-        end
+          zip = Geonames::Dump.new(Opt[:codes], :zip).data
+          dump = Geonames::Dump.new(Opt[:codes], :dump).data
+          info "\n---\nTotal #{dump.size} parsed. #{zip.size} zips."
+
+          info "Join dump << zip"
+          dump = unify!(dump, zip).group_by(&:kind)
 
-        info "\n---\nTotal #{Cache[:dump].length} parsed. #{Cache[:zip].length} zips."
-        # Sync.work!
-        info "Join dump << zip"
-        unify!
-        write_to_store!(db)
+          info "Writing provinces..."
+          Geonames::Models::Province.from_batch dump[:province]
+          info "Writing cities..."
+          Geonames::Models::City.from_batch dump[:city]
+        end
       end
     end
 
     def load_adapter(name)
       begin
-        require "geonames_local/adapters/#{name}"
         require "geonames_local/models/#{name}"
-        Geonames.class_eval(name.capitalize).new(Opt[:db])
       rescue LoadError
-        puts "Can't find adapter #{name}"
+        puts "Can't find adapter for #{name}"
         stop!
       end
     end
 
-    def write_to_store!(db)
-      if Opt[:codes][0] == "country"
-        Cache[:countries] = Cache[:dump]
-        do_write(db, Cache[:countries])
-      else
-        groups = Cache[:dump].group_by(&:kind)
-
-        Cache[:provinces] = groups[:provinces]
-        Cache[:cities] = groups[:cities]
-
-        do_write(db, Cache[:provinces])
-        do_write(db, Cache[:cities])
-      end
-    end
-
-    def do_write(db, values)
-      return if values.empty?
-
-      if Opt[:codes][0] == "country"
-        key = table = "countries"
-      else
-        key = values[0].table
-      end
-      start = Time.now
-      writt = 0
-      info "\nWriting #{values.length} #{key}..."
-      values.each do |val|
-        meth = val.respond_to?(:gid) ? [val.gid] : [val.name, true]
-        unless db.find(table || val.table, *meth)
-          db.insert(table || val.table, val)
-          writt += 1
-        end
-      end
-      total = Time.now - start
-      info "#{writt} #{key} written in #{total} sec (#{(writt/total).to_i}/s)"
-    end
-
-    def unify!
+    def unify! dump, zip
       start = Time.now
-      Cache[:dump].map! do |spot|
-        if other = Cache[:zip].find { |d| d.code == spot.code }
+      dump.map! do |spot|
+        if other = zip.find { |d| d.code == spot.code }
           spot.zip = other.zip
           spot
         else
@@ -190,6 +166,7 @@ BANNER
         end
       end
       info "Done. #{(Time.now-start).to_i}s"
+      dump
     end
 
     def stop!

data/lib/geonames_local/data/dump.rb

@@ -3,23 +3,36 @@ module Geonames
     URL = "http://download.geonames.org/export/"
     TMP = "/tmp/geonames/"
 
-    def self.work(codes=:all, kind=:dump)
-      new(codes, kind)
-    end
-
     def initialize(codes, kind)
       @codes = codes
       @kind = kind
+      @data = []
       if codes.respond_to? :each
         for code in codes
-          info "\nWorking on #{kind} for #{code}"
-          file = get_file(code)
-          download file
-          uncompress file unless code == "country"
-          parse file
+          work code
         end
+      elsif codes == :country
+        countries
       end
+    end
+
+    def countries
+      info "\nDumping country database"
+      file = get_file('country')
+      download file
+      parse file
+    end
+
+    def work code
+      info "\nWorking on #{@kind} for #{code}"
+      file = get_file(code)
+      download file
+      uncompress file
+      parse file
+    end
 
+    def data
+      @data
     end
 
     def get_file(code)
@@ -43,7 +56,7 @@ module Geonames
       return if l =~ /^#|^iso/i
       if @kind == :dump
         if l =~ /^\D/
-          return Country.parse(l)
+          return l
         else
           if Opt[:level] != "all"
             return unless l =~ /ADM\d/ # ADM2 => cities
@@ -59,7 +72,7 @@ module Geonames
       File.open("/tmp/geonames/#{@kind}/#{file.gsub("zip", "txt")}") do |f|
         while line = f.gets
           if record = parse_line(line)
-            Cache[@kind] << record
+            @data << record
             red += 1
           end
         end

data/lib/geonames_local/data/shp.rb

@@ -63,7 +63,7 @@ module Geonames
     end
 
     def write
-      db = Postgres.new(Opt[:db])
+      db = Postgis.new(Opt[:db])
       Geonames::CLI.do_write(db, Cache[:zones])
       Geonames::CLI.do_write(db, reduce!)
     end

data/lib/geonames_local/features/spot.rb

@@ -11,16 +11,16 @@ module Geonames
     # = Geonames Spot
     #
     # Every geoname type will be parsed as a spot
-    def initialize(params=nil, k=nil)
+    def initialize(params = nil, kind = nil)
       return unless params.instance_of? String
-      k == :zip ? parse_zip(params) :  parse(params)
+      kind == :zip ? parse_zip(params) : parse(params)
       if @kind == :province
         @name.gsub!(/Estado d\w\s/, "")
         @abbr = get_abbr
       end
     end
 
-    # Geonames donest have province/state abbr..#fail!
+    # Geonames does not have province/state abbr..#fail!
     # This works 75% of the time in brazil heh
     def get_abbr
       s = @name.split(" ")
@@ -49,14 +49,11 @@ module Geonames
     #
     # Parse Geonames Zip Export
     def parse_zip(row)
-      # country, zip, @name, province, cc, dunno, adm1, adm2, lat, lon  = row.split(/\t/)
       # country, zip, @name, state, state_code, procince, province_code, community, community_code, lat, lon, acc  = row.split(/\t/)
-      country, zip, @name, adm1, adm1_code, adm2, adm2_code, adm3, adm3_code, lat, lon, acc  = row.split(/\t/)
+      country, zip, @name, a1, @code, a2, a2c, a3, a3c, lat, lon, acc = row.split(/\t/)
       parse_geom(lat, lon)
       # @code = adm1
-      # @kind = :city
-      @code = adm1_code
-      @kind = :cities
+      @kind = :city
       @zip = zip.split("-")[0]
     end
 
@@ -82,39 +79,17 @@ module Geonames
       Time.utc(*@up.split("-"))
     end
 
-    # For tokyo
-    def to_hash
-      { "id" => @geoname_id, "gid" => @geoname_id.to_s, "kind" => @kind.to_s,
-        "name" => @name, "ascii" => @ascii, "country" => @country,
-        "geom" => [@geom.x, @geom.y], "tz" => @tz }
-    end
-
+    # Translate geonames ADMx to models
     def human_code(code)
       case code
-        when 'ADM1' then :provinces
-        when 'ADM2', 'ADM3', 'ADM4' then :cities
+        when 'ADM1' then :province
+        when 'ADM2', 'ADM3', 'ADM4' then :city
         else :other
       end
     end
 
     class << self
 
-    def all
-      Adapter.all(@coll)
-    end
-
-    def first
-      from_hash(Adapter.first(@coll))
-    end
-
-    def find(id)
-      Adapter.find(@coll, id)
-    end
-
-    def find_by_name(name)
-      Adapter.find_by_name(@coll, name).map { |hsh| from_hash(hsh) }
-    end
-
     def nearest(x,y)
       from_hash(Adapter.find_near(@coll, x, y, 1)[0])
     end

data/lib/geonames_local/geonames.rb

@@ -1,11 +1,6 @@
-require 'rubygems'
 require 'logger'
 require 'yaml'
 
-require 'geonames_local/features/spot'
-require 'geonames_local/features/road'
-require 'geonames_local/features/zone'
-
 module Geonames
   Opt = {}
   Cache = {:dump => [], :zip => [], :roads => [], :zones => []}

data/lib/geonames_local/models/mongodb.rb

@@ -1,62 +1,180 @@
+require 'mongoid'
+require 'mongoid_geospatial'
+
+Mongoid.configure do |config|
+  #config.master = Mongo::Connection.new.db("symbolize_test")
+  config.connect_to(Opt[:db][:name])
+end
+
 module Geonames
   module Models
-    module Mongo
 
-      class City < Geonames::Spot
-        set_coll "cities"
+
+    class City < Geonames::Spot
+      include Mongoid::Document
+      include Mongoid::Geospatial
+      store_in :collection => "cities"
+
+      field :ascii, type: String
+      field :slug,  type: String
+      field :name,  type: String
+      field :abbr,  type: String
+      field :area
+      field :gid,   type: Integer
+      field :zip,   type: Integer
+      field :geom,  type: Point, spatial: true
+
+      belongs_to :province
+      belongs_to :country, index: true
+      # has_many :hoods
+
+      before_validation :set_defaults
+
+      validates :name, :country, presence: true
+      validates :name, :uniqueness => { :scope => :province_id }
+
+      index name: 1
+      index slug: 1
+      index geom: '2d'
+
+      #spatial_index :geom
+
+      scope :ordered, order_by(name: 1)
+
+      def set_defaults
+        self.country ||= province.try(:country)
+        self.slug    ||= name.try(:downcase) # don't use slugize
       end
 
-      class Country < Geonames::Spot
-        attr_accessor :code, :name, :gid, :iso, :capital, :pop
-        set_coll "countries"
+      def self.search(search, page)
+        cities = search ? where(:field => /#{search}/i) : all
+        cities.page(page)
+      end
 
-    def self.parse(row)
-      new(row)
-    end
+      def <=> other
+        self.name <=> other.name
+      end
 
-    def initialize(params)
-      parse(params)
-    end
+      def to_s
+        "#{name}/#{abbr}"
+      end
 
-    def parse
-      @iso, @iso3, @ison, @fips, @name, @capital, @area, @pop, continent, tld,
-      currency, phone, postal, langs, gid, neighbours = row.split(/\t/)
-      @code = iso
-    end
+      def self.from_batch data
+        data.each do |city|
+          info "Writing city #{city.name}"
+          next unless city.country
+          city = new.parse(city)
+          city.country = city.province.country
+          city.save
+        end
+      end
 
-    def cities
-      # qry.addcond("country", TBDQRY::QSTREQ, @code)
-    end
+      def parse(spot)
+        self.name, self.ascii = spot.name, spot.ascii
+        self.code, self.gid = spot.code, spot.gid
+        self.province = Province.find_by(code: spot.province)
+        self.geom = [spot.lon, spot.lat]
+        self.abbr = province.abbr
+        self
+      end
 
-    def to_hash
-      { "gid" => @gid.to_s, "name" => @name, "kind" => "country", "code" => @code}
     end
 
-    def export
-      [@gid, @code, @name]
-    end
 
-    def export_header
-      ["gid", "code", "name"]
-    end
-  end
+    class Province < Geonames::Spot
+      include Mongoid::Document
+      store_in :collection => "provinces"
+
+      field :gid,    type: Integer  # geonames id
+      field :code,   type: String
+      field :name,   type: String
+      field :abbr,   type: String
+      field :codes,  type: Array # phone code
+
+      belongs_to :country
+      has_many :cities
+
+      validates :name, presence: true
+      validates :country, presence: true
 
-      class Province
-    attr_accessor :code, :name, :gid
+      index name: 1
+      index codes: 1
 
-    def self.all
-      Tokyo.new.all({ :kind => "province" }).map do |c|
-        new(c)
+      scope :ordered, order_by(name: 1)
+
+      def self.from_batch data
+        data.each do |province|
+          info "Writing province #{province.name}"
+          next unless province.country
+          province = new.parse(province)
+          province.country = Country.find_by(abbr: /#{province.country}/i)
+          province.save!
+        end
+      end
+
+      def parse(spot)
+        self.code, self.name = spot.province, spot.name
+        self.gid = spot.gid
+        self
       end
+
     end
 
-    def initialize(params)
-      @code = params["code"]
-      @name = params["name"]
-      @gid = params["gid"]
+
+    class Country < Geonames::Spot
+      include Mongoid::Document
+      store_in :collection => "countries"
+
+      field :gid,    type: Integer  # geonames id
+      field :name,   type: String
+      field :abbr,   type: String
+      field :code    # optional phone/whatever code
+
+      has_many :cities
+      has_many :provinces
+
+      validates :abbr, :name, presence: true, uniqueness: true
+
+      index name: 1
+      index code: 1
+
+      scope :ordered, order_by(name: 1)
+
+
+      def parse row
+        self.abbr, @iso3, @ison, @fips, self.name, @capital, @area, @pop, continent, tld,
+        currency, phone, postal, langs, gid, neighbours = row.split(/\t/)
+        self
+      end
+
+      def self.from_batch data
+        data.each do |spot|
+          new.parse(spot).save
+        end
+      end
+
+      def to_hash
+        { "gid" => @gid.to_s, "name" => @name, "kind" => "country", "code" => @code}
+      end
+
+      def export
+        [@gid, @code, @name]
+      end
+
+      def export_header
+        ["gid", "code", "name"]
+      end
     end
 
-  end
+
+
+    class Zip
+      include Mongoid::Document
+
+      field :code
+      belongs_to :city
+
     end
+
   end
 end