geonames_local 0.0.1 → 0.0.3

data/.gitignore

@@ -9,6 +9,7 @@ tmtags
 *~
 \#*
 .\#*
+*flymake.rb
 
 ## VIM
 *.swp
@@ -19,3 +20,8 @@ rdoc
 pkg
 
 ## PROJECT::SPECIFIC
+*.tct
+*.pid
+*.log
+*.qgr
+*.lex

data/README.rdoc

@@ -1,9 +1,82 @@
-= geonames_local
+= Geonames Local
+
+Download and store in tokyo or postgresql Geonames.org data.
+Making every Geoname API operation possible on your servers.
+No hit limit, fast as possible.
+
+== Features/Problems
+
+- Local relational mapping of geonames data
+- Map geonames fields to your db scheme UNDONE
+- Countries are a static yml file :/
+
+== Usage
+
+  geonames conf
+
+Will generate a "geonames.yml" file on your folder.
+The file is self explanatory.
+
+  geonames -c geonames.yml
+
+Work. Use -v for verbose.
+
+If you are not sure your country code, use:
+
+  geonames list <search>
+
+
+== Relational Mapping
+
+When using PG, this gem will (try) to relational map Geonames
+data on your scheme. Postgresql done, tokyo still need heavy work.
+
+
+== PostgreSQL
+
+So, supposing ActiveRecord, something like this is possible:
+
+  City.first.province.country.abbr
+  => "BR"
+
+=== Migration
+
+Default PG migration:
+
+    create_table :cities do |t|
+      t.references :country, :null => false
+      t.references :province
+      t.string  :name, :null => false
+      t.point   :geom,   :srid => 4326
+      t.integer :gid, :zip
+    end
+
+    create_table :provinces do |t|
+      t.references :country, :null => false
+      t.string :name, :null => false
+      t.string :abbr, :limit => 2, :null => false
+      t.integer :gid
+    end
+
+    create_table :countries do |t|
+      t.string :name, :limit => 30, :null => false
+      t.string :abbr, :limit => 2,  :null => false
+    end
+
+    add_index :cities, :name
+    add_index :cities, :gid
+    add_index :cities, :zip
+    add_index :cities, :country_id
+    add_index :cities, :province_id
+    add_index :cities, :geom, :spatial => true
+    add_index :provinces, :name
+    add_index :provinces, :abbrg
+    add_index :provinces, :gid
+    add_index :provinces, :country_id
 
-Description goes here.
 
 == Note on Patches/Pull Requests
- 
+
 * Fork the project.
 * Make your feature addition or bug fix.
 * Add tests for it. This is important so I don't break it in a

data/Rakefile

@@ -11,6 +11,7 @@ begin
     gem.homepage = "http://github.com/nofxx/geonames_local"
     gem.authors = ["Marcos Piccinini"]
     gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency "tokyotyrant", ">= 1.10"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new
@@ -43,3 +44,59 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
+
+#
+# Tokyo Tyrant rake tasks
+#
+namespace :tyrant do
+  TYRANT_DB_FILE  = File.join("tyrant.tct")
+  TYRANT_PID_FILE = File.join("tyrant.pid")
+  TYRANT_LOG_FILE = File.join("tyrant.log")
+
+  desc "Start Tyrant server"
+  task :start  do
+    raise RuntimeError, "Tyrant is already running." if tyrant_running?
+    system "ttserver -pid #{TYRANT_PID_FILE} -log #{TYRANT_LOG_FILE} #{TYRANT_DB_FILE}&"
+    sleep(2)
+    if tyrant_running?
+      puts "Tyrant started successfully (pid #{tyrant_pid})."
+    else
+      puts "Failed to start tyrant push server. Check logs."
+    end
+  end
+
+  desc "Stop Tyrant server"
+  task :stop do
+    raise RuntimeError, "Tyrant isn't running." unless tyrant_running?
+    system "kill #{tyrant_pid}"
+    sleep(2)
+    if tyrant_running?
+      puts "Tyrant didn't stopped. Check the logs."
+    else
+      puts "Tyrant stopped."
+    end
+  end
+
+  desc "Restart Tyrant server"
+  task :restart => [:stop, :start]
+
+  desc "Get Tyrant Server Status"
+  task :status do
+    puts tyrant_running? ? "Tyrant running. (#{tyrant_pid})" : "Tyrant not running."
+  end
+end
+
+def tyrant_pid
+  `cat #{TYRANT_PID_FILE}`.to_i
+end
+
+def tyrant_running?
+  return false unless File.exist?(TYRANT_PID_FILE)
+  process_check = `ps -p #{tyrant_pid} | wc -l`
+  if process_check.to_i < 2
+    puts "Erasing pidfile..."
+    `rm #{TYRANT_PID_FILE}`
+  end
+  tyrant_pid
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.0.3

data/bin/geonames

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+$VERBOSE = true # -w
+$KCODE = "u" # -Ku
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require "geonames_local"
+include Geonames
+
+Geonames::CLI.work(ARGV)

data/lib/config/codes.yml

@@ -0,0 +1,24 @@
+:br:
+  :pt_br: Brasil
+  :en_us: Brazil
+:us:
+  :pt_br: Estados Unidos
+  :en_us: United States
+:cl:
+  :pt_br: Chile
+  :en_us: Chile
+:ar:
+  :pt_br: Argentina
+  :en_us: Argentina
+:ug:
+  :pt_br: Uganda
+  :en_us: Uganda
+:td:
+  :pt_br: Chad
+  :en_us: Chad
+:ru:
+  :pt_br: Rússia
+  :en_us: Russia
+:it:
+  :pt_br: Itália
+  :en_us: Italy

data/lib/config/geonames.yml

@@ -0,0 +1,14 @@
+#
+# Geonames Local Config Example
+#
+:store: pg
+:codes: [br, cl]
+:level: city
+:mapping:
+  :name: name
+  :geom: true
+:db:
+  :host: localhost
+  :dbname: db
+  :user: postgres
+  :password:

data/lib/data/geoquery.rb

@@ -0,0 +1,31 @@
+module Geonames
+  class Geoquery
+    R = 1
+
+    def self.point(tdb, x, y)
+            qry = TDBQRY::new(tdb)
+      qry.addcond("x", TDBQRY::QCNUMGE, minx.to_s())
+      qry.addcond("x", TDBQRY::QCNUMLE, maxx.to_s())
+      qry.addcond("y", TDBQRY::QCNUMGE, miny.to_s())
+      qry.addcond("y", TDBQRY::QCNUMLE, maxy.to_s())
+      qry.setorder("x", TDBQRY::QONUMASC)
+      qry.setlimit(80)
+    end
+
+
+    def self.area(tdb, minx, maxx, miny, maxy)
+      qry = TDBQRY::new(tdb)
+      qry.addcond("x", TDBQRY::QCNUMGE, minx.to_s())
+      qry.addcond("x", TDBQRY::QCNUMLE, maxx.to_s())
+      qry.addcond("y", TDBQRY::QCNUMGE, miny.to_s())
+      qry.addcond("y", TDBQRY::QCNUMLE, maxy.to_s())
+      qry.setorder("x", TDBQRY::QONUMASC)
+
+      res = qry.search
+      info res.length # number of results found
+      return res
+    end
+
+
+  end
+end

data/lib/data/postgres.rb

@@ -0,0 +1,69 @@
+require "pg"
+
+module Geonames
+  class Postgres
+    Countries = {}
+    Provinces = {}
+
+    def initialize(opts) #table, addr = "localhost", port = 5432)
+      @conn = PGconn.new(opts)
+    end
+
+    #
+    # Get Country and Province ID from the DB
+    def get_some_ids(some)
+      c = Countries[some.country] ||=
+          @conn.exec("SELECT countries.id FROM countries WHERE UPPER(countries.abbr) = UPPER('#{some.country}')")[0]["id"] rescue nil
+      c ||= write("countries", {:name => Codes[some.country.downcase.to_sym][:pt_br], :abbr => some.country })
+
+      p = Provinces[some.province] ||= find("provinces", Cache[:provinces].
+                                       find{ |p| p.province == some.province}.gid)
+      [c, p]
+    end
+
+    #
+    # Insert a record
+    def insert(some)
+      country_id, province_id = get_some_ids(some)
+      if some.kind == :cities
+        write("cities", {:name => some.name, :country_id => country_id,
+                 :geom => some.geom.as_hex_ewkb, :gid => some.gid,
+                 :zip => some.zip, :province_id => province_id})
+      else
+        write("provinces", { :name => some.name, :abbr => some.abbr,
+                 :country_id => country_id, :gid => some.gid })
+      end
+    end
+
+    #
+    # Find a record`s ID
+    def find(kind, id)
+      @conn.exec("SELECT #{kind}.id FROM #{kind} WHERE #{kind}.gid = #{id}")[0]["id"] rescue nil
+    end
+
+    #
+    # F'oo -> F''oo  (for pg)
+    def escape_name(name)
+      name.gsub("'", "''")
+    end
+
+    #
+    # Sanitize values por pg.. here until my lazyness open pg rdoc...
+    def pg_values(arr)
+      arr.map do |v|
+        case v
+        when String then "E'#{escape_name(v)}'"
+        when NilClass then 'NULL'
+        else v
+        end
+      end.join(",")
+    end
+
+    #
+    # Naive PG insert ORM =D
+    def write(table, hsh)
+      for_pg = pg_values(hsh.values)
+      @conn.exec("INSERT INTO #{table} (#{hsh.keys.join(",")}) VALUES(#{for_pg}) RETURNING id")[0]["id"]
+    end
+  end
+end

data/lib/data/tokyo.rb

@@ -0,0 +1,84 @@
+
+module Geonames
+  class Tokyo
+
+    def initialize(conn=nil, resource=nil, extra=nil)
+      if conn
+        require 'tokyotyrant'
+        resource ||= 'localhost'
+        extra ||= 1978
+        @tbl = TokyoTyrant::RDBTBL
+        @qry = TokyoTyrant::RDBQRY
+      else
+        require 'tokyocabinet'
+        resource ||= 'geonames.tct'
+        extra ||= (TokyoCabinet::TDB::OWRITER | TokyoCabinet::TDB::OCREAT)
+        @tbl = TokyoCabinet::TDB
+        @qry = TokyoCabinet::TDBQRY
+      end
+      @rdb = @tbl.new
+      @rdb.open(resource, extra)
+      set_indexes
+    end
+
+    def all(params)
+      qry = @qry.new(@rdb)
+      params.each do |k,v|
+        #qry.addcond(k.to_s, Q::QCNUMEQ, v.to_s)
+        qry.addcond(k.to_s, @qry::QCSTREQ, v.to_s)
+      end
+      qry.setorder("name", @qry::QOSTRASC)
+      qry.search.map { |id| @rdb.get(id) }
+    end
+
+    def find(id)
+      #qry = Q.new(@rdb)
+      #qry.addcond("gid", Q::QCNUMEQ, id.to_s)
+      #qry.setlimit(10)
+      #id = qry.search.pop
+      @rdb.get(id)
+    end
+
+    # def to_obj(hsh)
+    #   hsh["kind"] == "country" ? Country.new(hsh) : Spot.new(hsh)
+    # end
+
+    def write(o)
+      # pkey = @rdb.genuid
+      if @rdb.put(o.gid, o.to_hash)
+       # info "ok"
+      else
+        info "err #{@rdb.errmsg(@rdb.ecode)}"
+      end
+    end
+
+    def count
+      @qry.new(@rdb).search.length
+    end
+
+    def close
+      # close the database
+      if !@rdb.close
+        STDERR.printf("close error: %s\n", @rdb.errmsg(@rdb.ecode))
+      end
+    end
+
+    def set_indexes
+      #for index in indexes
+      # @rdb.setindex("gid", @tbl::ITOPT)
+      @rdb.setindex("kind", @tbl::ITLEXICAL)
+      @rdb.setindex("name", @tbl::ITQGRAM)
+      @rdb.setindex("country", @tbl::ITLEXICAL)
+
+      #end
+
+    end
+
+    def flush
+      @rdb.vanish
+    end
+
+
+  end
+
+end

data/lib/features/city.rb

@@ -0,0 +1,9 @@
+class City
+  attr_accessor :country, :province, :name
+
+  def self.all
+    qry.addcond(QCSTREQ, 'city')
+  end
+  
+  
+end

data/lib/features/country.rb

@@ -0,0 +1,61 @@
+module Geonames
+  class Country
+    attr_accessor :code, :name, :gid, :iso, :capital, :pop
+
+    def self.all
+      Tokyo.new.all({ :kind => "country" }).map do |c|
+        new(c)
+      end
+    end
+
+    # [0] iso alpha2
+    # [1] iso alpha3
+    # [2] iso numeric
+    # [3] fips code
+    # [4] name
+    # [5] capital
+    # [6] areaInSqKm
+    # [7] population
+    # [8] continent
+    # [9] top level domain
+    # [10] Currency code
+    # [11] Currency name
+    # [12] Phone
+    # [13] Postal Code Format
+    # [14] Postal Code Regex
+    # [15] Languages
+    # [16] Geoname id
+    # [17] Neighbours
+    # [18] Equivalent Fips Code
+    #
+    def self.parse(row)
+      new(row)
+    end
+
+    def initialize(params)
+      parse(params)
+    end
+
+    def parse
+      @iso, @iso3, @ison, @fips, @name, @capital, @area, @pop, continent, tld,
+      currency, phone, postal, langs, gid, neighbours = row.split(/\t/)
+      @code = iso
+    end
+
+    def cities
+      # qry.addcond("country", TBDQRY::QSTREQ, @code)
+    end
+
+    def to_hash
+      { "gid" => @gid.to_s, "name" => @name, "kind" => "country", "code" => @code}
+    end
+
+    def export
+      [@gid, @code, @name]
+    end
+
+    def export_header
+      ["gid", "code", "name"]
+    end
+  end
+end

data/lib/features/province.rb

@@ -0,0 +1,18 @@
+module Geonames
+  class Province
+    attr_accessor :code, :name, :gid
+
+    def self.all
+      Tokyo.new.all({ :kind => "province" }).map do |c|
+        new(c)
+      end
+    end
+
+    def initialize(params)
+      @code = params["code"]
+      @name = params["name"]
+      @gid = params["gid"]
+    end
+
+  end
+end

data/lib/features/road.rb

@@ -0,0 +1,2 @@
+  class Road
+  end

data/lib/features/spot.rb

@@ -0,0 +1,88 @@
+module Geonames
+  class Spot
+    attr_accessor :gid, :name, :ascii, :lat, :lon, :country, :kind,
+                  :code,  :pop, :tz, :geom, :province, :zip, :abbr
+    alias :x :lon
+    alias :y :lat
+    alias :geoname_id :gid
+
+    #
+    # = Geonames Spot
+    #
+    # Every geoname type will be parsed as a spot
+    def initialize(params, k)
+      return unless params.instance_of? String
+      k == :zip ? parse_zip(params) :  parse(params)
+      if @kind == :provinces
+        @name.gsub!(/Estado d\w\s/, "")
+        @abbr = get_abbr
+      end
+    end
+
+    # Geonames donest have province/state abbr..#fail!
+    # This works 75% of the time in brazil heh
+    def get_abbr
+      s = @name.split(" ")
+      if s.length > 1
+        [s[0][0].chr,s[-1][0].chr].map(&:upcase).join
+      else
+        s[0][0..1].upcase
+      end
+    end
+
+    #
+    # Parse Geonames Dump Export
+    def parse(row)
+      gid, @name, @ascii, @alternates, lat, lon, feat, kind,
+        @country, cc2, adm1, adm2, adm3, adm4, pop, ele,
+        gtop, @tz, @up = row.split(/\t/)
+
+      parse_geom(lat, lon)
+      @gid = @geoname_id = gid.to_i
+      @kind = human_code(kind)
+      @province = adm1
+      @code = adm2
+    end
+
+    #
+    # Parse Geonames Zip Export
+    def parse_zip(row)
+      country, zip, @name, province, cc, dunno, adm1, adm2, lat, lon  = row.split(/\t/)
+      parse_geom(lat, lon)
+      @code = adm1
+      @kind = :cities
+      @zip = zip.split("-")[0]
+    end
+
+    #
+    # Parse Geom to float or GeoRuby Point
+    def parse_geom(lat, lon)
+      @lat, @lon = lat.to_f, lon.to_f
+      
+      if defined?("GeoRuby")
+        @geom = GeoRuby::SimpleFeatures::Point.from_x_y(@lon, @lat)
+      end
+    end
+
+    #
+    # Parse Time
+    def updated_at
+      Time.utc(*@up.split("-"))
+    end
+
+    # For tokyo
+    def to_hash
+      { "gid" => @geoname_id.to_s, "kind" => @kind.to_s, "name" => @name, "ascii" => @ascii,
+        "lat" => @lat.to_s, "lon" => @lon.to_s, "tz" => @tz, "country" => @country }
+    end
+
+
+    def human_code(code)
+      case code
+        when 'ADM1' then :provinces
+        when 'ADM2' then :cities
+        else :other
+      end
+    end
+  end
+end

data/lib/geonames_local.rb

@@ -1,6 +1,23 @@
 require 'features/spot'
 require 'features/country'
 require 'features/city'
+require 'rubygems'
+require 'logger'
+require 'data/tokyo'
+require 'data/postgres'
+require 'work/cli'
+require 'work/dump'
+require 'work/export'
 
 module Geonames
+  Opt = {}
+  Cache = {:dump => [], :zip => []}
+  Codes = YAML.load(File.read(File.join(File.dirname(__FILE__),'config', 'codes.yml')))
+  VERSION =  File.read(File.join(File.dirname(__FILE__), '..', 'VERSION'))
+
+  def info(txt)
+    if Opt[:verbose]
+      puts(txt)
+    end
+  end
 end

data/lib/work/cli.rb

@@ -0,0 +1,144 @@
+#
+# Geonames Local
+#
+require 'optparse'
+module Geonames
+  class CLI
+
+    def self.parse_options(argv)
+      options = {}
+
+      argv.options do |opts|
+        opts.banner = <<BANNER
+Geonames Command Line Usage:
+
+geonames <country code(s)> <opts>
+
+BANNER
+        opts.on("-l", "--level LEVEL", String, "The level of logging to report" ) { |level| options[:level] = level }
+        opts.on("-d", "--dump", "Dump DB before all" ) { options[:dump] = true }
+        opts.separator ""
+        opts.separator "Config file:"
+        opts.on("-c", "--config CONFIG", String, "Geonames Config file path" ) { |file|  options[:config] = file }
+        opts.separator ""
+        opts.separator "Tyrant Options:"
+        opts.on("-t", "--tyrant", "Use tyrant" ) { options[:tyrant] = true }
+        opts.on("-s", "--server SERVER", String, "Tyrant Server URL" ) { |url|  options[:server] = url }
+        opts.on("-p", "--port PORT", Integer, "Tyrant Server Port")  { |val| options[:port] = val.to_i }
+        opts.separator ""
+        opts.separator "Common Options:"
+        opts.on("-h", "--help", "Show this message" ) { puts opts; exit }
+        opts.on("-v", "--verbose", "Turn on logging to STDOUT" ) { |bool| options[:verbose] = bool }
+        opts.on("-V", "--version", "Show version") {  puts Geonames::VERSION;  exit }
+        opts.separator ""
+        begin
+          opts.parse!
+          if argv.empty? && !options[:config]
+            puts opts
+            exit
+          end
+        rescue
+          puts opts
+          exit
+        end
+      end
+      options
+    end
+    private_class_method :parse_options
+
+    def self.work(argv)
+      trap(:INT) { stop! }
+      trap(:TERM) { stop! }
+      Opt.merge! parse_options(argv)
+
+      if Opt[:config]
+        Opt.merge! YAML.load(File.read(Opt[:config]))
+      end
+
+      if argv[0] =~ /list|codes/
+         Codes.each do |key,val|
+          str = [val.values, key.to_s].join(" ").downcase
+          if s = argv[1]
+            next unless str =~ /#{s.downcase}/
+          end
+          puts "#{val[:en_us]}: #{key}"
+        end
+        exit
+      end
+
+      if argv[0] =~ /scaff|conf/
+        fname = (argv[1] || "geonames") + ".yml"
+        if File.exist?(fname)
+          puts "File exists."
+        else
+          puts "Writing to #{fname}"
+          `cp #{File.join(File.dirname(__FILE__), '..', 'config', 'geonames.yml')} #{fname}`
+        end
+        exit
+      end
+      require "geo_ruby" if Opt[:mapping] && Opt[:mapping][:geom]
+
+      if argv[0] =~ /csv|json/
+        Geonames::Export.new(Country.all).to_csv
+      else
+        Geonames::Dump.work(Opt[:codes], :zip) #rescue puts "Command not found: #{comm} #{@usage}"
+        Geonames::Dump.work(Opt[:codes], :dump) #rescue puts "Command not found: #{comm} #{@usage}"
+        info "\n---\nTotal #{Cache[:dump].length} parsed. #{Cache[:zip].length} zips."
+        info "Join dump << zip"
+        unify!
+        write_to_store!
+      end
+    end
+
+    def self.write_to_store!
+      db = case Opt[:store].to_sym
+           when :tyrant then Geonames::Tokyo.new(Opt[:tyrant])
+           when :pg     then Geonames::Postgres.new(Opt[:db])
+           else
+             info "No store defined!"
+             exit
+           end
+
+      groups = Cache[:dump].group_by(&:kind)
+      Cache[:provinces] = groups[:provinces]
+      # ensure this order....
+      do_write(db, groups[:provinces])
+      do_write(db, groups[:cities])
+    end
+
+    def self.do_write(db, val)
+      key = val[0].kind
+      start = Time.now
+      writt = 0
+      info "\nWriting #{key}..."
+      val.each do |v|
+        unless db.find v.kind, v.gid
+          db.insert v
+          writt += 1
+        end
+      end
+      total = Time.now - start
+      info "#{writt} #{key} written in #{total} sec (#{(writt/total).to_i}/s)"
+    end
+
+    def self.unify!
+      start = Time.now
+      Cache[:dump].map! do |spot|
+        if other = Cache[:zip].find { |d| d.code == spot.code }
+          spot.zip = other.zip
+          spot
+        else
+          spot
+        end
+      end
+      info "Done. #{(Time.now-start).to_i}s"
+    end
+
+    def self.stop!
+      puts "Closing Geonames..."
+      exit
+    end
+
+  end
+
+end

data/lib/work/dump.rb

@@ -0,0 +1,76 @@
+module Geonames
+  class Dump
+    URL = "http://download.geonames.org/export/"
+    TMP = "/tmp/geonames/"
+
+    def initialize(codes, kind)
+      @codes = codes
+      @kind = kind
+      if codes.respond_to? :each
+        for code in codes
+          info "\nWorking on #{kind} for #{code}"
+          file = get_file(code)
+          download file
+          uncompress file unless code == "country"
+          parse file
+        end
+      end
+
+    end
+
+    def get_file(code)
+      code == "country" ? "countryInfo.txt" : "#{code.upcase}.zip"
+    end
+
+    def download(file)
+      Dir.mkdir(TMP) unless File.exists?(TMP)
+      Dir.mkdir(TMP + @kind.to_s) unless File.exists?(TMP + @kind.to_s)
+      fname = TMP + "#{@kind}/#{file}"
+      return if File.exists?(fname)
+      `curl #{URL}/#{@kind}/#{file} -o #{fname}`
+    end
+
+    def uncompress(file)
+      info "Uncompressing #{file}"
+      `unzip -quo /tmp/geonames/#{@kind}/#{file} -d /tmp/geonames/#{@kind}`
+    end
+
+    def parse_line(l)
+      return if l =~ /^#|^iso/i
+      if @kind == :dump
+        if l =~ /^\D/
+          Country.parse(l)
+        else
+          if Opt[:level] != "all"
+            return unless l =~ /ADM\d/ # ADM2 => cities
+          end
+        end
+      end
+      Spot.new(l, @kind)
+    end
+
+    def parse(file)
+      red = 0
+      start = Time.now
+      File.open("/tmp/geonames/#{@kind}/#{file.gsub("zip", "txt")}") do |f|
+        while line = f.gets
+          if record = parse_line(line)
+            Cache[@kind] << record
+            red += 1
+          end
+        end
+        total = Time.now - start
+        info "#{red} #{@kind} entries parsed in #{total} sec (#{(red/total).to_i}/s)"
+      end
+      rescue Errno::ENOENT => e
+      info "Failed to download #{file}, skipping."
+    end
+
+
+    def self.work(codes=:all, kind=:dump)
+      new(codes, kind)
+    end
+
+
+  end
+end

data/lib/work/export.rb

@@ -0,0 +1,21 @@
+require 'csv'
+
+module Geonames
+  class Export
+
+    def initialize(data)
+      info "Starting export.."
+      @data = data
+    end
+
+    def to_csv
+      file = "export.csv"
+      info "Writing #{file} (#{@data.length} objects)"
+      CSV.open("export.csv", 'w') do |csv|
+        csv << @data[0].export_header
+        @data.each { |o| csv << o.export }
+      end
+      info "Export done."
+    end
+  end
+end

data/spec/data/tokyo_spec.rb

@@ -0,0 +1,29 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe "Tokyo" do
+
+  it "should write down a spot" do
+    t = Geonames::Tokyo.new('localhost', 1978)
+    m = mock(Geonames::Spot, { :gid => 888, :to_hash => { "gid" => 888, "kind" => "city", "lat" => 5.5 }})
+    t.write(m)
+  end
+
+  it "should read it up" do
+    t = Geonames::Tokyo.new('localhost', 1978)
+    record = t.find(888)
+    record.should be_instance_of Geonames::Spot
+  end
+
+  it "should not duplicate" do
+    t = Geonames::Tokyo.new('localhost', 1978)
+    t.all({ :gid => 888}).length.should eql(1)
+  end
+
+  it "should return all countries" do
+    all = Geonames::Country.all
+    all.should be_instance_of Array
+    all[0].should be_instance_of Geonames::Country
+    all[0].gid.should eql(1)
+  end
+
+end

data/spec/geonames_local_spec.rb

@@ -5,7 +5,7 @@ describe "GeonamesLocal" do
 
   describe "Parsing" do
     before do
-      @spot =  Geonames::Spot.new("6319037\tMaxaranguape\tMaxaranguape\t-5.46874226086957\t-35.3565714695652\tA\tADM2\tBR\t22\t2407500\t6593\t12\tAmerica/Recife\t2006-12-17")
+      @spot =  Geonames::Spot.new("6319037\tMaxaranguape\tMaxaranguape\t\t-5.46874226086957\t-35.3565714695652\tA\tADM2\tBR\t22\t2407500\t6593\t\t12\t\t\t\tAmerica/Recife\t2006-12-17")
     end
 
     it "should parse geoid integer" do
@@ -41,6 +41,42 @@ describe "GeonamesLocal" do
     end
   end
 
+  describe "More Parseing" do
+    before do
+      @spot =  Geonames::Spot.new("3384862\tRiacho Zuza\tRiacho Zuza\t\t-9.4333333\t-37.6666667\tH\tSTMI\tBR\t\t02\t\t\t\t0\t\t241\tAmerica/Maceio\t1993-12-17\n")
+    end
+
+    it "should parse geoid integer" do
+      @spot.geoname_id.should eql(3384862)
+    end
+
+    it "should parse name" do
+      @spot.name.should eql("Riacho Zuza")
+      @spot.ascii.should eql("Riacho Zuza")
+    end
+
+    it "should parse geostuff" do
+      @spot.lat.should be_close(-9.4333333, 0.001)
+      @spot.lon.should be_close(-37.6666667, 0.001)
+    end
+
+    it "should parse spot kind" do
+      @spot.kind.should eql(:other)
+    end
+
+    it "should parse spot country" do
+      @spot.country.should eql("BR")
+    end
+
+    it "shuold parse timezone" do
+      @spot.tz.should eql("America/Maceio")
+    end
+
+    it "should parse updated_at" do
+      @spot.updated_at.should be_instance_of(Time)
+      @spot.updated_at.day.should eql(17)
+    end
+  end
 end
 
 # 6319037 Maxaranguape  Maxaranguape    -5.46874226086957 -35.3565714695652 A ADM2  BR    22  2407500     6593    12  America/Recife  2006-12-17

data/task/benchmark.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+#
+#  DB => br, cl ~ 6k objects
+#
+require 'benchmark'
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'geonames_local'
+
+# @db.flush
+
+def b1(db)
+  puts "#{db.count} Objects"
+  Benchmark.bmbm do |b|
+    b.report("All Country") { db.all({ :kind => "country" })}
+    b.report("Find by GID") { db.find(888) }
+    b.report("Find by name") { db.all({ :name => "Maxaranguape"}) }
+    b.report("Find on country") { db.all({ :country => "CL"}) }
+  end
+end
+
+print "Tyrant => "
+b1(Geonames::Tokyo.new(:tyrant))
+
+
+print "Cabinet => "
+b1(Geonames::Tokyo.new)
+

data/task/benchmark_cabinet.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+#
+#  DB => br, cl ~ 6k objects
+#
+require 'benchmark'
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'geonames_local'
+include Geonames
+# @db.flush
+
+
+def b1(db)
+  puts "#{db.count} Objects"
+  Benchmark.bmbm do |b|
+    b.report("All Country") { Country.all }
+    b.report("Find by GID") { db.find(888) }
+    b.report("Find by name") { db.all({ :name => "Maxaranguape"}) }
+    b.report("Find on country") { db.all({ :country => "CL"}) }
+  end
+end
+
+print "Cabinet => "
+b1(Tokyo.new)
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: geonames_local
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors: 
 - Marcos Piccinini
@@ -9,8 +9,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-13 00:00:00 -02:00
-default_executable: geoname
+date: 2010-01-28 00:00:00 -02:00
+default_executable: geonames
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
@@ -22,10 +22,20 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.2.9
     version: 
+- !ruby/object:Gem::Dependency 
+  name: tokyotyrant
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "1.10"
+    version: 
 description: Dump and feed a tokyo cabinet for local geonames search
 email: x@nofxx.com
 executables: 
-- geoname
+- geonames
 extensions: []
 
 extra_rdoc_files: 
@@ -37,10 +47,28 @@ files:
 - LICENSE
 - README.rdoc
 - Rakefile
+- VERSION
+- bin/geonames
+- lib/config/codes.yml
+- lib/config/geonames.yml
+- lib/data/geoquery.rb
+- lib/data/postgres.rb
+- lib/data/tokyo.rb
+- lib/features/city.rb
+- lib/features/country.rb
+- lib/features/province.rb
+- lib/features/road.rb
+- lib/features/spot.rb
 - lib/geonames_local.rb
+- lib/work/cli.rb
+- lib/work/dump.rb
+- lib/work/export.rb
+- spec/data/tokyo_spec.rb
 - spec/geonames_local_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- task/benchmark.rb
+- task/benchmark_cabinet.rb
 has_rdoc: true
 homepage: http://github.com/nofxx/geonames_local
 licenses: []
@@ -70,5 +98,6 @@ signing_key:
 specification_version: 3
 summary: Dump and feed a tokyo local geonames db
 test_files: 
-- spec/spec_helper.rb
+- spec/data/tokyo_spec.rb
 - spec/geonames_local_spec.rb
+- spec/spec_helper.rb