geonames_local 0.3.1 → 0.5.0

data/lib/geonames_local/data/dump.rb

@@ -3,6 +3,10 @@ module Geonames
     URL = "http://download.geonames.org/export/"
     TMP = "/tmp/geonames/"
 
+    def self.work(codes=:all, kind=:dump)
+      new(codes, kind)
+    end
+
     def initialize(codes, kind)
       @codes = codes
       @kind = kind
@@ -67,10 +71,6 @@ module Geonames
     end
 
 
-    def self.work(codes=:all, kind=:dump)
-      new(codes, kind)
-    end
-
   end
 
 end

data/lib/geonames_local/data/sync.rb

@@ -0,0 +1,74 @@
+module Geonames
+  class Sync
+    class << self
+
+      def work!
+        unify!
+        write_to_store!
+      end
+
+      def load_adapter(name)
+        begin
+          require "geonames_local/adapters/#{name}"
+          Geonames.class_eval(name.capitalize).new(Opt[:db])
+        rescue LoadError
+          puts "Can't find adapter #{name}"
+          stop!
+        end
+      end
+
+      def write_to_store!
+        groups = Cache[:dump].group_by(&:kind)
+        Cache[:provinces] = groups[:province]
+        # ensure this order....
+        do_write(groups[:province])
+        do_write(groups[:city])
+      end
+
+      def do_write(values)
+        return if values.empty?
+        db = load_adapter(Opt[:store])
+        key = values[0].table
+        start = Time.now
+        writt = 0
+        min_pop = Opt[:min_pop]
+        info "\nWriting #{values.length} #{key}..."
+        info "\nWriting spots with pop > #{Opt[:min_pop]} hab." if min_pop
+        values.each do |val|
+          if min_pop
+            next unless val.pop && val.pop.to_i >= min_pop
+          end
+          arg = val.respond_to?(:gid) ? [val.gid] : [val.name, true]
+          unless db.find(val.table, *arg)
+            db.insert(val.table, val)
+            writt += 1
+          end
+        end
+        total = Time.now - start
+        info "#{writt} #{key} written in #{total} sec (#{(writt/total).to_i}/s)"
+      end
+
+      def unify!
+        info "Join dump << zip"
+        start = Time.now
+        Cache[:dump].map! do |spot|
+          if other = Cache[:zip].find { |d| d.code == spot.code }
+            spot.zip = other.zip
+            spot
+          else
+            spot
+          end
+        end
+        info "Done. #{(Time.now-start).to_i}s"
+      end
+
+      def stop!
+        puts "Closing Geonames..."
+        exit
+      end
+
+    end
+
+  end
+
+end

data/lib/geonames_local/features/spot.rb

@@ -35,9 +35,8 @@ module Geonames
     # Parse Geonames Dump Export
     def parse(row)
       gid, @name, @ascii, @alternates, lat, lon, feat, kind,
-        @country, cc2, adm1, adm2, adm3, adm4, pop, ele,
-        gtop, @tz, @up = row.split(/\t/)
-
+        @country, cc2, adm1, adm2, adm3, adm4, @pop, @ele,
+        @gtop, @tz, @up = row.split(/\t/)
       parse_geom(lat, lon)
       @gid = @geoname_id = gid.to_i
       @kind = human_code(kind)
@@ -51,7 +50,7 @@ module Geonames
       country, zip, @name, province, cc, dunno, adm1, adm2, lat, lon  = row.split(/\t/)
       parse_geom(lat, lon)
       @code = adm1
-      @kind = :cities
+      @kind = :city
       @zip = zip.split("-")[0]
     end
 
@@ -62,9 +61,15 @@ module Geonames
 
       if defined?("GeoRuby")
         @geom = GeoRuby::SimpleFeatures::Point.from_x_y(@lon, @lat)
+      else
+        { :lat => @lat, :lon => @lon }
       end
     end
 
+    def alt
+      @ele || @gtop
+    end
+
     #
     # Parse Time
     def updated_at

data/lib/geonames_local/models/ar.rb

@@ -0,0 +1,50 @@
+module Geonames
+  module Models
+    module AR
+
+      class City < ActiveRecord::Base
+        attr_accessor :x, :y, :z
+
+        belongs_to :province
+        belongs_to :country
+
+        validates_presence_of :country
+        validates_presence_of :name
+
+        def abbr
+          province.try(:abbr) || country.abbr
+        end
+
+        def geom=(val)
+          self[:geom] = case val
+          when Array then Point.xy(*val)
+          else val
+          end
+        end
+
+        # Instantiate self.geom as a Point
+        def validation
+          self.country ||= province.country
+          unless !@x || !@y || @x == "" || @y == ""
+            self.geom = Point.from_x_y(@x.to_f, @y.to_f)
+          end
+        end
+
+      end
+
+      class Province < ActiveRecord::Base
+        has_many :cities
+        belongs_to :country
+      end
+
+      class Country < ActiveRecord::Base
+        attr_accessor :code, :gid, :iso, :capital, :pop
+        has_many :provinces
+        has_many :cities
+      end
+
+    end
+  end
+end
+
+

data/lib/geonames_local/{model/country.rb → models/mongo.rb}

@@ -1,33 +1,15 @@
 module Geonames
-  class Country
-    attr_accessor :code, :name, :gid, :iso, :capital, :pop
+  module Models
+    module Mongo
 
-    def self.all
-      Tokyo.new.all({ :kind => "country" }).map do |c|
-        new(c)
+      class City < Geonames::Spot
+        set_coll "cities"
       end
-    end
 
-    # [0] iso alpha2
-    # [1] iso alpha3
-    # [2] iso numeric
-    # [3] fips code
-    # [4] name
-    # [5] capital
-    # [6] areaInSqKm
-    # [7] population
-    # [8] continent
-    # [9] top level domain
-    # [10] Currency code
-    # [11] Currency name
-    # [12] Phone
-    # [13] Postal Code Format
-    # [14] Postal Code Regex
-    # [15] Languages
-    # [16] Geoname id
-    # [17] Neighbours
-    # [18] Equivalent Fips Code
-    #
+      class Country < Geonames::Spot
+        attr_accessor :code, :name, :gid, :iso, :capital, :pop
+        set_coll "countries"
+
     def self.parse(row)
       new(row)
     end
@@ -57,5 +39,24 @@ module Geonames
     def export_header
       ["gid", "code", "name"]
     end
+  end
+
+      class Province
+    attr_accessor :code, :name, :gid
+
+    def self.all
+      Tokyo.new.all({ :kind => "province" }).map do |c|
+        new(c)
+      end
+    end
+
+    def initialize(params)
+      @code = params["code"]
+      @name = params["name"]
+      @gid = params["gid"]
+    end
+
+  end
+    end
   end
 end

data/lib/geonames_local/{mongo/country.rb → models/tokyo.rb}

@@ -1,5 +1,8 @@
 module Geonames
-  class Country
+  module Models
+    module Tokyo
+
+class Country
     attr_accessor :code, :name, :gid, :iso, :capital, :pop
 
     def self.all
@@ -57,5 +60,24 @@ module Geonames
     def export_header
       ["gid", "code", "name"]
     end
+
+    end
+  end
+      class Province
+    attr_accessor :code, :name, :gid
+
+    def self.all
+      Tokyo.new.all({ :kind => "province" }).map do |c|
+        new(c)
+      end
+    end
+
+    def initialize(params)
+      @code = params["code"]
+      @name = params["name"]
+      @gid = params["gid"]
+    end
+
   end
+
 end

data/lib/geonames_local.rb

@@ -15,6 +15,4 @@ module Geonames
 
 end
 
-require 'geonames_local/mongo/country'
-require 'geonames_local/mongo/province'
-require 'geonames_local/mongo/city'
+require 'geonames_local/models/mongo'

data/spec/geonames_local/adapters/mongodb_spec.rb

@@ -1,20 +1,21 @@
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 require File.expand_path(File.dirname(__FILE__) + '/../../../lib/geonames_local/adapters/mongodb')
 
-describe Mongodb do
+describe "Mongo Models" do
 
   SPECDB = "geonames_spec"
 
-  before(:all) do
+  before() do
     Mongodb.new({:dbname => SPECDB}).purge
     @mong = Mongodb.new({:dbname => SPECDB})
+
   end
 
   def mock_spot(name)
     Spot.new("1\t#{name}\t#{name}\t\t-5.46874226086957\t-35.3565714695652\tA\tADM2\tBR\t22\t2407500\t6593\t\t12\t\t\t\tAmerica/Recife\t2006-12-17", :dump)
   end
 
-  describe "Parsing Dump" do
+  describe "Parsing dump" do
     before do
       @mock_spot = mock("Spot")
     end
@@ -37,7 +38,7 @@ describe Mongodb do
     it "should store geom with sinusoidal projection" do
       @mock_spot.should_receive(:to_hash).and_return({"id" => 8, "name" => "Sao Tome", "geom" => [5,8]})
       @mong.insert("cities", @mock_spot)
-      @mong.find("cities", 8)["geom"][0].should be_close(4.95, 0.1)
+      @mong.find("cities", 8)["geom"][0].should be_within(0.01).of(4.95)
       @mong.find("cities", 8)["geom"][1].should eql(8)
     end
 
@@ -47,7 +48,7 @@ describe Mongodb do
 
     describe "Finds" do
 
-      before(:all) do
+      before() do
         @mong.insert("cities", {"id" => 9, "name" => "Sao Paulo", "geom" => [15,15]})
         @mong.insert("cities", {"id" => 10, "name" => "Sao Tome", "geom" => [-7,-34]})
         @mong.insert("cities", {"id" => 11, "name" => "Sao Benedito", "geom" => [-9,-39]})
@@ -59,7 +60,7 @@ describe Mongodb do
 
       it "should find geo" do
         @mong.find_near("cities", -5, -35).first["name"].should eql("Sao Tome")
-        @mong.find_near("cities", -5, -35).first["geom"][0].should be_close(-5.80, 0.1)
+        @mong.find_near("cities", -5, -35).first["geom"][0].should be_within(0.1).of(-5.80,)
         @mong.find_near("cities", -5, -35).first["geom"][1].should eql(-34)
       end
 
@@ -72,8 +73,12 @@ describe Mongodb do
         @mong.find_within("cities", [[10, 10],[20, 20]]).first["name"].should eql("Sao Paulo")
       end
 
+      it "should find within tiny radius" do
+        @mong.find_within("cities", [[-6, -36], 2]).length.should eql(0)
+      end
+
       it "should find within radius" do
-        @mong.find_within("cities", [[-6, -36], 2]).length.should eql(1)
+        @mong.find_within("cities", [[-6, -36], 3]).length.should eql(1)
       end
 
       it "should find within wider radius" do
@@ -85,12 +90,12 @@ describe Mongodb do
       end
 
       it "should find geoNear" do
-        @mong.near("cities", -5, -35).first["dis"].should be_close(1.97, 0.01)
+        @mong.near("cities", -5, -35).first["dis"].should be_within(0.01).of(1.97)
         @mong.near("cities", -5, -35).first["obj"]["name"].should eql("Sao Tome")
       end
 
       it "should find geoNear" do
-        @mong.near("cities", -5, -35).first["dis"].should be_close(1.97, 0.01)
+        @mong.near("cities", -5, -35).first["dis"].should be_within(0.01).of(1.97)
         @mong.near("cities", -5, -35).first["obj"]["name"].should eql("Sao Tome")
       end
 

data/spec/geonames_local/cli_spec.rb

@@ -1,8 +1,13 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
+# Acceptance to go!
 describe CLI do
 
   describe "Parsing Dump" do
+
+    it "should fetch zips" do
+
+    end
     # before(:all)
   end
 end

data/spec/geonames_local/data/dump_spec.rb

@@ -0,0 +1,4 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+describe Dump do
+end

data/spec/geonames_local/features/spot_spec.rb

@@ -6,192 +6,272 @@ describe Spot do
 
   describe "Parsing Dump" do
 
-    before do
-      @spot = Spot.new("6319037\tMaxaranguape\tMaxaranguape\t\t-5.46874226086957\t-35.3565714695652\tA\tADM2\tBR\t22\t2407500\t6593\t\t12\t\t\t\tAmerica/Recife\t2006-12-17", :dump)
-    end
+    let(:spot) { Spot.new("6319037\tMaxaranguape\tMaxaranguape\t\t-5.46874226086957\t-35.3565714695652\tA\tADM2\tBR\t22\t2407500\t6593\t\t12\t\t\t\tAmerica/Recife\t2006-12-17", :dump) }
 
     it "should parse geoid integer" do
-      @spot.geoname_id.should eql(6319037)
-      @spot.gid.should eql(6319037)
+      spot.geoname_id.should eql(6319037)
+      spot.gid.should eql(6319037)
     end
 
     it "should parse code" do
-      @spot.code.should eql("6593")
+      spot.code.should eql("6593")
     end
 
     it "should parse province code" do
-      @spot.province.should eql("2407500")
+      spot.province.should eql("2407500")
     end
 
     it "should parse name" do
-      @spot.name.should eql("Maxaranguape")
-      @spot.ascii.should eql("Maxaranguape")
+      spot.name.should eql("Maxaranguape")
+      spot.ascii.should eql("Maxaranguape")
     end
 
     it "should parse geostuff" do
-      @spot.lat.should be_close(-5.4687, 0.001)
-      @spot.y.should be_close(-5.4687, 0.001)
-      @spot.lon.should be_close(-35.3565, 0.001)
+      spot.lat.should be_within(0.001).of(-5.4687)
+      spot.y.should be_within(0.001).of(-5.4687)
+      spot.lon.should be_within(0.001).of(-35.3565)
     end
 
     it "should parse spot kind" do
-      @spot.kind.should eql(:city)
+      spot.kind.should eql(:city)
     end
 
     it "should parse spot country" do
-      @spot.country.should eql("BR")
+      spot.country.should eql("BR")
     end
 
     it "shuold parse timezone" do
-      @spot.tz.should eql("America/Recife")
+      spot.tz.should eql("America/Recife")
     end
 
     it "should parse updated_at" do
-      @spot.updated_at.should be_instance_of(Time)
-      @spot.updated_at.day.should eql(17)
+      spot.updated_at.should be_instance_of(Time)
+      spot.updated_at.day.should eql(17)
     end
   end
 
-  describe "More Parseing" do
+  describe "More Parsing" do
 
-    before do
-      @spot =  Geonames::Spot.new("3384862\tRiacho Zuza\tRiacho Zuza\t\t-9.4333333\t-37.6666667\tH\tSTMI\tBR\t\t02\t\t\t\t0\t\t241\tAmerica/Maceio\t1993-12-17\n", :dump)
-    end
+    let(:spot) { Geonames::Spot.new("3384862\tRiacho Zuza\tRiacho Zuza\t\t-9.4333333\t-37.6666667\tH\tSTMI\tBR\t\t02\t\t\t\t0\t\t241\tAmerica/Maceio\t1993-12-17\n", :dump) }
 
     it "should parse geoid integer" do
-      @spot.geoname_id.should eql(3384862)
+      spot.geoname_id.should eql(3384862)
     end
 
     it "should parse name" do
-      @spot.name.should eql("Riacho Zuza")
-      @spot.ascii.should eql("Riacho Zuza")
+      spot.name.should eql("Riacho Zuza")
+      spot.ascii.should eql("Riacho Zuza")
     end
 
     it "should parse geostuff" do
-      @spot.lat.should be_close(-9.4333333, 0.001)
-      @spot.lon.should be_close(-37.6666667, 0.001)
+      spot.lat.should be_within(0.001).of(-9.4333333)
+      spot.lon.should be_within(0.001).of(-37.6666667)
     end
 
     it "should parse spot kind" do
-      @spot.kind.should eql(:other)
+      spot.kind.should eql(:other)
     end
 
     it "should parse spot country" do
-      @spot.country.should eql("BR")
+      spot.country.should eql("BR")
     end
 
     it "shuold parse timezone" do
-      @spot.tz.should eql("America/Maceio")
+      spot.tz.should eql("America/Maceio")
     end
 
     it "should parse updated_at" do
-      @spot.updated_at.should be_instance_of(Time)
-      @spot.updated_at.day.should eql(17)
+      spot.updated_at.should be_instance_of(Time)
+      spot.updated_at.day.should eql(17)
     end
   end
 
   describe "Parsing Province" do
 
-    before do
-      @spot =  Geonames::Spot.new("3457153\tEstado de Minas Gerais\tEstado de Minas Gerais\tMinas,Minas Geraes,Minas Gerais\t-18.0\t-44.0\tA\tADM1\tBR\tBR\t15\t\t\t\t16672613\t\t1219\tAmerica/Sao_Paulo\t2007-05-15\n", :dump)
-    end
+    let(:spot) { Geonames::Spot.new("3457153\tEstado de Minas Gerais\tEstado de Minas Gerais\tMinas,Minas Geraes,Minas Gerais\t-18.0\t-44.0\tA\tADM1\tBR\tBR\t15\t\t\t\t16672613\t\t1219\tAmerica/Sao_Paulo\t2007-05-15\n", :dump) }
 
     it "should be kind of province" do
-      @spot.kind.should eql(:province)
+      spot.kind.should eql(:province)
     end
 
     it "should parse geoid" do
-      @spot.geoname_id.should eql(3457153)
-      @spot.gid.should eql(3457153)
+    spot.geoname_id.should eql(3457153)
+      spot.gid.should eql(3457153)
     end
 
     it "should parse code" do
-      @spot.code.should be_empty
+      spot.code.should be_empty
     end
 
     it "should parse province code" do
-      @spot.province.should eql("15")
+      spot.province.should eql("15")
     end
 
     it "should create abbr" do
-      @spot.abbr.should eql("MG")
+      spot.abbr.should eql("MG")
     end
 
     it "should parse name" do
-      @spot.name.should eql("Minas Gerais")
-      @spot.ascii.should eql("Estado de Minas Gerais")
+      spot.name.should eql("Minas Gerais")
+      spot.ascii.should eql("Estado de Minas Gerais")
     end
 
     it "should parse geostuff" do
-      @spot.lat.should be_close(-18.0, 0.001)
-      @spot.lon.should be_close(-44.0, 0.001)
+      spot.lat.should be_within(0.001).of(-18.0)
+      spot.lon.should be_within(0.001).of(-44.0)
     end
 
   end
 
-  describe "Parsing Zip" do
+  describe "Parsing City" do
+
+    let(:spot) {  Spot.new "3386859\tTamboril\tTamboril\t\t-4.9931\t-40.26738\tA\tADM2\tBR\t\t06\t2313203\t\t\t25455\t\t401\tAmerica/Fortaleza\t2011-04-21" }
+
+    it "should parse name" do
+      spot.name.should eql("Tamboril")
+    end
+
+    it "should parse ascii name" do
+      spot.name.should eql("Tamboril")
+    end
+
+    it "should parse x" do
+      spot.x.should be_within(0.001).of(-40.26738)
+    end
+
+    it "should parse y" do
+      spot.y.should be_within(0.001).of(-4.9931)
+    end
 
-    before do
-      @spot = Geonames::Spot.new("BR\t76375-000\tHidrolina\tGoias\t29\t\t5209804\t\t-14.7574\t-49.3596\t\n", :zip)
+    it "should parse tz" do
+      spot.tz.should eql("America/Fortaleza")
     end
 
+    it "should parse kind" do
+      spot.kind.should eql(:city)
+    end
+
+    it "should parse country" do
+      spot.country.should eql("BR")
+    end
+
+    it "should parse province" do
+      pending
+      spot.province.should eql("C")
+    end
+
+    it "should parse pop" do
+      spot.pop.should eql("25455")
+    end
+
+  end
+
+  describe "Parsing Big City" do
+
+    let(:spot) {  Spot.new "6322846\tLondrina\tLondrina\t\t-23.58643\t-51.08739\tA\tADM2\tBR\t\t18\t4113700\t\t\t506645\t\t544\tAmerica/Sao_Paulo\t2011-04-21" }
+
+    it "should parse name" do
+      spot.name.should eql("Londrina")
+    end
+
+    it "should parse ascii name" do
+      spot.name.should eql("Londrina")
+    end
+
+    it "should parse x" do
+      spot.x.should be_within(0.001).of(-51.08739)
+    end
+
+    it "should parse y" do
+      spot.y.should be_within(0.001).of(-23.58643)
+    end
+
+    it "should parse tz" do
+      spot.tz.should eql("America/Sao_Paulo")
+    end
+
+    it "should parse kind" do
+      spot.kind.should eql(:city)
+    end
+
+    it "should parse country" do
+      spot.country.should eql("BR")
+    end
+
+    it "should parse province" do
+      pending
+      spot.province.should eql("C")
+    end
+
+    it "should parse pop" do
+      spot.pop.should eql("506645")
+    end
+
+
+  end
+
+  describe "Parsing Zip" do
+
+    let(:spot) { Geonames::Spot.new("BR\t76375-000\tHidrolina\tGoias\t29\t\t5209804\t\t-14.7574\t-49.3596\t\n", :zip) }
+
     it "should parse zip oO" do
-      @spot.zip.should eql("76375")
+      spot.zip.should eql("76375")
     end
 
     it "should be a city" do
-      @spot.kind.should eql(:cities)
+      spot.kind.should eql(:city)
     end
 
     it "should parse code" do
-      @spot.code.should eql("5209804")
+      spot.code.should eql("5209804")
     end
 
     it "should parse geoid integer" do
-      @spot.gid.should be_nil # eql(3384862)
+      spot.gid.should be_nil # eql(3384862)
     end
 
     it "should parse name" do
-      @spot.name.should eql("Hidrolina")
-      @spot.ascii.should be_nil # eql("Hidrolina")
+      spot.name.should eql("Hidrolina")
+      spot.ascii.should be_nil # eql("Hidrolina")
     end
 
-    it "should parse geostuff" do
-      @spot.lat.should be_close(-14.7574, 0.001)
-      @spot.lon.should be_close(-49.3596, 0.001)
+    it "should parse lat" do
+      spot.lat.should be_within(0.001).of(-14.7574)
+    end
+
+    it "should parse lon" do
+      spot.lon.should be_within(0.001).of(-49.3596)
     end
 
   end
 
   describe "From Hash" do
 
-    before do
-      @spot = Spot.from_hash({"id" => 9, "name" => "Sao Rock", "geom" => [15,15], "kind" => "cities", "country" => "BR", "gid" => 13232, "tz" => "America/Foo", "ascii" => "Rock"})
-    end
+    let(:spot) { Spot.from_hash({"id" => 9, "name" => "Sao Rock", "geom" => [15,15], "kind" => "cities", "country" => "BR", "gid" => 13232, "tz" => "America/Foo", "ascii" => "Rock"}) }
 
     it "should be an spot" do
-      @spot.should be_instance_of Spot
+      spot.should be_instance_of Spot
     end
 
     it "should set the name" do
-      @spot.name.should eql("Sao Rock")
+      spot.name.should eql("Sao Rock")
     end
 
     it "should set the geom" do
-      @spot.geom.should be_instance_of(GeoRuby::SimpleFeatures::Point)
-      @spot.geom.x.should eql(15)
+      spot.geom.should be_instance_of(GeoRuby::SimpleFeatures::Point)
+      spot.geom.x.should eql(15)
     end
 
     it "should set the tz" do
-      @spot.tz.should eql("America/Foo")
+      spot.tz.should eql("America/Foo")
     end
 
     it "should set the ascii" do
-      @spot.ascii.should eql("Rock")
+      spot.ascii.should eql("Rock")
     end
 
     it "should set the country abbr" do
-      @spot.country.should eql("BR")
+      spot.country.should eql("BR")
     end
 
   end