geonames_local 0.3.1 → 0.5.0

data/README.rdoc

@@ -1,19 +1,23 @@
 = Geonames Local
 
-Download and store Geonames.org data.
+Download and store (pg, mongo, tokyo) Geonames.org data.
 Making every Geoname API operation possible on your servers.
 No hit limit, fast as possible.
 
 
 == Usage
 
-To use the adapter, install the corresponding gem:
+To use one adapter, install the corresponding gem:
 
   PostgreSQL   =>   pg
-  MongoDB      =>   mongodb (optional: mongo_ext)
+  MongoDB      =>   mongo (optional: mongo_ext)
   Tokyo        =>   tokyocabinet
 
-Don't forget to install *unzip* and *curl*.
+You will also need in your system:
+
+* unzip
+* curl
+
 
 === PostgreSQL
 
@@ -35,7 +39,7 @@ The file is self explanatory.
 
   geonames -c geonames.yml
 
-Work. Use -v for verbose.
+To run it. Use -v for verbose.
 
 If you are not sure your country code, use:
 
@@ -45,7 +49,14 @@ If you are not sure your country code, use:
 == Relational Mapping
 
 When using PG, this gem will (try) to relational map Geonames
-data on your scheme. Postgresql done, tokyo still need heavy work.
+data on your scheme.
+
+== ActiveRecord
+
+Check out lib/geonames/models/ar.rb
+
+I`m wondering the best way (and if it`s a good idea) to auto include'em on rails.
+For now, just copy as you see fit.
 
 
 == PostgreSQL
@@ -55,6 +66,7 @@ So, supposing ActiveRecord, something like this is possible:
   City.first.province.country.abbr
   => "BR"
 
+
 === Migration
 
 Default PG migration:
@@ -62,7 +74,7 @@ Default PG migration:
     create_table :cities do |t|
       t.references :country, :null => false
       t.references :province
-      t.string  :name, :null => false
+      t.string  :name,   :null => false
       t.point   :geom,   :srid => 4326
       t.integer :gid, :zip
     end
@@ -80,22 +92,25 @@ Default PG migration:
     end
 
     add_index :cities, :name
-    add_index :cities, :gid
     add_index :cities, :zip
     add_index :cities, :country_id
     add_index :cities, :province_id
+    add_index :cities, :gid,  :unique  => true
     add_index :cities, :geom, :spatial => true
     add_index :provinces, :name
     add_index :provinces, :abbr
-    add_index :provinces, :gid
     add_index :provinces, :country_id
+    add_index :provinces, :gid,  :unique => true
+    add_index :countries, :abbr, :unique => true
+    add_index :countries, :name, :unique => true
 
 
-== Features/Problems
+== TODO/Problems
 
 - Local relational mapping of geonames data
-- Map geonames fields to your db scheme UNDONE
+- Map geonames fields to your db scheme
 - Countries are a static yml file :/
+- IP Geonames? http://ipinfodb.com
 
 
 == Note on Patches/Pull Requests

data/Rakefile

@@ -44,20 +44,13 @@ rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
 end
 
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
+require 'rspec/core/rake_task'
+desc "Runs spec suite"
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/*_spec.rb'
+  spec.rspec_opts = ['--backtrace --colour']
 end
 
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-
-task :spec => :check_dependencies
-
 task :default => :spec
 
 require 'rake/rdoctask'

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.5.0

data/bin/geonames

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 $VERBOSE = true # -w
-$KCODE = "u" # -Ku
+# $KCODE = "u" # -Ku
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 
 require "geonames_cli"

data/geonames.yml

@@ -1,14 +1,15 @@
 #
 # Geonames Local Config Example
 #
-:store: mongodb
+:store: postgres
 :codes: [br]
 :level: city
+:min_pop: 100000
 :mapping:
   :name: name
   :geom: true
 :db:
   :host: localhost
-  :dbname: rtrac_development
+  :dbname: geonames_ar
   :user: postgres
   :password:

data/geonames_local.gemspec

@@ -1,74 +1,71 @@
 # Generated by jeweler
 # DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{geonames_local}
-  s.version = "0.3.1"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Marcos Piccinini"]
-  s.date = %q{2010-09-14}
+  s.date = %q{2011-04-24}
   s.default_executable = %q{geonames}
   s.description = %q{Dump and feed a tokyo cabinet for local geonames search}
   s.email = %q{x@nofxx.com}
   s.executables = ["geonames"]
   s.extra_rdoc_files = [
     "LICENSE",
-     "README.rdoc"
+    "README.rdoc"
   ]
   s.files = [
     ".document",
-     ".gitignore",
-     "LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "bin/geonames",
-     "geonames.yml",
-     "geonames_local.gemspec",
-     "lib/geonames_ar.rb",
-     "lib/geonames_cli.rb",
-     "lib/geonames_local.rb",
-     "lib/geonames_local/adapters/mongodb.rb",
-     "lib/geonames_local/adapters/postgres.rb",
-     "lib/geonames_local/adapters/tokyo.rb",
-     "lib/geonames_local/cli.rb",
-     "lib/geonames_local/config/codes.yml",
-     "lib/geonames_local/config/geonames.sql",
-     "lib/geonames_local/config/geonames.yml",
-     "lib/geonames_local/data/dump.rb",
-     "lib/geonames_local/data/export.rb",
-     "lib/geonames_local/data/shp.rb",
-     "lib/geonames_local/features/road.rb",
-     "lib/geonames_local/features/spot.rb",
-     "lib/geonames_local/features/zone.rb",
-     "lib/geonames_local/geonames.rb",
-     "lib/geonames_local/model/city.rb",
-     "lib/geonames_local/model/country.rb",
-     "lib/geonames_local/model/province.rb",
-     "lib/geonames_local/mongo/city.rb",
-     "lib/geonames_local/mongo/country.rb",
-     "lib/geonames_local/mongo/province.rb",
-     "spec/geonames_local/adapters/mongodb_spec.rb",
-     "spec/geonames_local/adapters/postgres_spec.rb",
-     "spec/geonames_local/adapters/tokyo_spec.rb",
-     "spec/geonames_local/cli_spec.rb",
-     "spec/geonames_local/data/cache_spec.rb",
-     "spec/geonames_local/data/shp_spec.rb",
-     "spec/geonames_local/features/road_spec.rb",
-     "spec/geonames_local/features/spot_spec.rb",
-     "spec/geonames_local/features/zone_spec.rb",
-     "spec/geonames_local/model/ar_spec.rb",
-     "spec/geonames_local/mongo/city_spec.rb",
-     "spec/geonames_local_spec.rb",
-     "spec/spec.opts",
-     "spec/spec_ar_helper.rb",
-     "spec/spec_helper.rb",
-     "spec/spec_mongo_helper.rb",
-     "task/benchmark.rb",
-     "task/benchmark_cabinet.rb"
+    "LICENSE",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/geonames",
+    "geonames.yml",
+    "geonames_local.gemspec",
+    "lib/geonames_ar.rb",
+    "lib/geonames_cli.rb",
+    "lib/geonames_local.rb",
+    "lib/geonames_local/adapters/mongodb.rb",
+    "lib/geonames_local/adapters/postgres.rb",
+    "lib/geonames_local/adapters/tokyo.rb",
+    "lib/geonames_local/cli.rb",
+    "lib/geonames_local/config/codes.yml",
+    "lib/geonames_local/config/geonames.sql",
+    "lib/geonames_local/config/geonames.yml",
+    "lib/geonames_local/data/dump.rb",
+    "lib/geonames_local/data/export.rb",
+    "lib/geonames_local/data/shp.rb",
+    "lib/geonames_local/data/sync.rb",
+    "lib/geonames_local/features/road.rb",
+    "lib/geonames_local/features/spot.rb",
+    "lib/geonames_local/features/zone.rb",
+    "lib/geonames_local/geonames.rb",
+    "lib/geonames_local/models/ar.rb",
+    "lib/geonames_local/models/mongo.rb",
+    "lib/geonames_local/models/tokyo.rb",
+    "spec/geonames_local/adapters/mongodb_spec.rb",
+    "spec/geonames_local/adapters/postgres_spec.rb",
+    "spec/geonames_local/adapters/tokyo_spec.rb",
+    "spec/geonames_local/cli_spec.rb",
+    "spec/geonames_local/data/cache_spec.rb",
+    "spec/geonames_local/data/dump_spec.rb",
+    "spec/geonames_local/data/shp_spec.rb",
+    "spec/geonames_local/features/road_spec.rb",
+    "spec/geonames_local/features/spot_spec.rb",
+    "spec/geonames_local/features/zone_spec.rb",
+    "spec/geonames_local/models/ar_spec.rb",
+    "spec/geonames_local/models/mongo_spec.rb",
+    "spec/geonames_local_spec.rb",
+    "spec/spec_ar_helper.rb",
+    "spec/spec_helper.rb",
+    "spec/spec_mongo_helper.rb",
+    "task/benchmark.rb",
+    "task/benchmark_cabinet.rb"
   ]
   s.homepage = %q{http://github.com/nofxx/geonames_local}
   s.post_install_message = %q{
@@ -94,26 +91,26 @@ http://github.com/mongodb/mongo-ruby-driver
 
 Have fun because:
 }
-  s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{Dump and feed a tokyo local geonames db}
   s.test_files = [
+    "spec/geonames_local/adapters/mongodb_spec.rb",
+    "spec/geonames_local/adapters/postgres_spec.rb",
+    "spec/geonames_local/adapters/tokyo_spec.rb",
+    "spec/geonames_local/cli_spec.rb",
+    "spec/geonames_local/data/cache_spec.rb",
+    "spec/geonames_local/data/dump_spec.rb",
+    "spec/geonames_local/data/shp_spec.rb",
+    "spec/geonames_local/features/road_spec.rb",
+    "spec/geonames_local/features/spot_spec.rb",
+    "spec/geonames_local/features/zone_spec.rb",
+    "spec/geonames_local/models/ar_spec.rb",
+    "spec/geonames_local/models/mongo_spec.rb",
+    "spec/geonames_local_spec.rb",
     "spec/spec_ar_helper.rb",
-     "spec/spec_helper.rb",
-     "spec/geonames_local/data/shp_spec.rb",
-     "spec/geonames_local/data/cache_spec.rb",
-     "spec/geonames_local/model/ar_spec.rb",
-     "spec/geonames_local/mongo/city_spec.rb",
-     "spec/geonames_local/cli_spec.rb",
-     "spec/geonames_local/adapters/postgres_spec.rb",
-     "spec/geonames_local/adapters/tokyo_spec.rb",
-     "spec/geonames_local/adapters/mongodb_spec.rb",
-     "spec/geonames_local/features/spot_spec.rb",
-     "spec/geonames_local/features/zone_spec.rb",
-     "spec/geonames_local/features/road_spec.rb",
-     "spec/spec_mongo_helper.rb",
-     "spec/geonames_local_spec.rb"
+    "spec/spec_helper.rb",
+    "spec/spec_mongo_helper.rb"
   ]
 
   if s.respond_to? :specification_version then

data/lib/geonames_ar.rb

@@ -4,11 +4,9 @@
 $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
+# Adapter = Geonames::Postgres.new(Opt[:db])
 # Require Libs
 require 'geonames_local/geonames'
 
-require 'geonames_local/model/country'
-require 'geonames_local/model/province'
-require 'geonames_local/model/city'
+require 'geonames_local/models/ar'
 #require 'geonames_local/data/shp'
-

data/lib/geonames_cli.rb

@@ -6,8 +6,9 @@ $:.unshift(File.dirname(__FILE__)) unless
 
 # Require CLI Stuff
 require 'geonames_local/geonames'
-require 'geonames_local/cli'
 require 'geonames_local/data/shp'
 require 'geonames_local/data/dump'
+require 'geonames_local/data/sync'
 require 'geonames_local/data/export'
+require 'geonames_local/cli'
 

data/lib/geonames_local/adapters/mongodb.rb

@@ -78,7 +78,8 @@ module Geonames
     # getNear command returns distance too
     # <1.9 needs OrderedHash
     def near(resource, x, y, limit=nil)
-      cmd = OrderedHash.new
+      # FIXME: Better explicity require OrderedHash?
+      cmd = Object.const_defined?("OrderedHash") ? OrderedHash.new : { }
       cmd["geoNear"] = resource
       cmd["near"] = sin_proj(x,y)
       cmd["num"] = limit if limit

data/lib/geonames_local/adapters/postgres.rb

@@ -5,27 +5,30 @@ module Geonames
     Countries = {}
     Provinces = {}
 
-    def initialize(opts) #table, addr = "localhost", port = 5432)
+    def initialize(opts={}) #table, addr = "localhost", port = 5432)
       @conn = PGconn.new(opts)
     end
 
     #
     # Get Country and Province ID from the DB
+    #
+    # Maps the FKs ids correctly for our bank
+    #
     def get_some_ids(some)
-      c = Countries[some.country] ||=
+      cid = Countries[some.country] ||=
           @conn.exec("SELECT countries.id FROM countries WHERE UPPER(countries.abbr) = UPPER('#{some.country}')")[0]["id"] rescue nil
-      c ||= write("countries", {:name => Codes[some.country.downcase.to_sym][:pt_br], :abbr => some.country })
+      cid ||= write("countries", {:name => Codes[some.country.downcase.to_sym][:pt_br], :abbr => some.country })
 
-      p = nil
-      ci = nil
+      pid = nil
+      tid = nil
       if some.kind_of? Spot
-        p = Provinces[some.province] ||= find("provinces", Cache[:provinces].
+        pid = Provinces[some.province] ||= find("provinces", Cache[:provinces].
                                        find{ |p| p.province == some.province}.gid)
       else
-        ci = find("cities", some.city)
-        p = @conn.exec("SELECT cities.province_id FROM cities WHERE cities.id = #{ci}")[0]["province_id"] rescue nil
+        tid = find("cities", some.city)
+        pid = @conn.exec("SELECT cities.province_id FROM cities WHERE cities.id = #{tid}")[0]["province_id"] rescue nil
       end
-      [c, p, ci]
+      [cid, pid, tid]
     end
 
     #
@@ -33,22 +36,25 @@ module Geonames
     def insert(table, some)
       country_id, province_id, city_id = get_some_ids(some)
       case table
-      when :cities
+      when :city
         write("cities", {:name => some.name, :country_id => country_id,
                  :geom => some.geom.as_hex_ewkb, :gid => some.gid,
                  :zip => some.zip, :province_id => province_id})
-      when :provinces
+      when :province
         write("provinces", { :name => some.name, :abbr => some.abbr,
                  :country_id => country_id, :gid => some.gid })
-      when :roads
+      when :road
         write("roads", { :name => some.name, :geom => some.geom.as_hex_ewkb, :kind => some.kind,
                  :country_id => country_id, :city_id => city_id, :province_id => province_id })
+      else
+        puts "Fail to insert #{some}"
       end
     end
 
     #
     # Find a record`s ID
-    def find(table, id, name=nil)
+    def find(kind, id, name=nil)
+      table = get_table kind
       begin
         if name
           @conn.exec("SELECT #{table}.id FROM #{table} WHERE (#{table}.name = E'#{id}')")[0]["id"]
@@ -79,6 +85,15 @@ module Geonames
       end.join(",")
     end
 
+    def get_table(kind)
+      case kind
+      when :city then "cities"
+      when :country then "countries"
+      else
+        kind.to_s + "s"
+      end
+    end
+
     #
     # Naive PG insert ORM =D
     def write(table, hsh)

data/lib/geonames_local/cli.rb

@@ -60,6 +60,11 @@ BANNER
         Opt.merge! YAML.load(File.read(Opt[:config]))
       end
 
+      # Load config/geonames.yml if there's one
+      if File.exists?(cfg = File.join("config", "geonames.yml"))
+        Opt.merge! YAML.load(File.read(cfg))
+      end
+
       if shp = Opt[:shp]
         SHP.import(shp)
         exit
@@ -94,69 +99,14 @@ BANNER
       if argv[0] =~ /csv|json/
         Geonames::Export.new(Country.all).to_csv
       else
-        db = load_adapter(Opt[:store])
         info "Using adapter #{Opt[:store]}.."
         Geonames::Dump.work(Opt[:codes], :zip) #rescue puts "Command not found: #{comm} #{@usage}"
         Geonames::Dump.work(Opt[:codes], :dump) #rescue puts "Command not found: #{comm} #{@usage}"
         info "\n---\nTotal #{Cache[:dump].length} parsed. #{Cache[:zip].length} zips."
-        info "Join dump << zip"
-        unify!
-        write_to_store!(db)
-      end
-    end
-
-    def load_adapter(name)
-      begin
-        require "geonames_local/adapters/#{name}"
-        Geonames.class_eval(name.capitalize).new(Opt[:db])
-      rescue LoadError
-        puts "Can't find adapter #{name}"
-        stop!
+        Sync.work!
       end
     end
 
-    def write_to_store!(db)
-      groups = Cache[:dump].group_by(&:kind)
-      Cache[:provinces] = groups[:provinces]
-      # ensure this order....
-      do_write(db, groups[:provinces])
-      do_write(db, groups[:cities])
-    end
-
-    def do_write(db, values)
-      return if values.empty?
-      key = values[0].table
-      start = Time.now
-      writt = 0
-      info "\nWriting #{values.length} #{key}..."
-      values.each do |val|
-        meth = val.respond_to?(:gid) ? [val.gid] : [val.name, true]
-        unless db.find(val.table, *meth)
-          db.insert(val.table, val)
-          writt += 1
-        end
-      end
-      total = Time.now - start
-      info "#{writt} #{key} written in #{total} sec (#{(writt/total).to_i}/s)"
-    end
-
-    def unify!
-      start = Time.now
-      Cache[:dump].map! do |spot|
-        if other = Cache[:zip].find { |d| d.code == spot.code }
-          spot.zip = other.zip
-          spot
-        else
-          spot
-        end
-      end
-      info "Done. #{(Time.now-start).to_i}s"
-    end
-
-    def stop!
-      puts "Closing Geonames..."
-      exit
-    end
     end
 
   end