geonames_local 3.3.1 → 3.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c788cde1271fbcde5596ca18134547b68ebe2488
-  data.tar.gz: 983752fea10114e374c21e75e23b5e21808c31fe
+  metadata.gz: 3d899c1e90546accbce85fa372b534d862f81d16
+  data.tar.gz: b0978645a9ae924f268bb8783a82e2de3564a733
 SHA512:
-  metadata.gz: a7a08a957601e2663126bd6fe7c5021b46b6527e0e15a1549e8b6971b5ad688443f82c73aad92ba44e93f4e48567a6034a108c742c24d92bb472af2fb92ee369
-  data.tar.gz: eb7b597058aa178b5e8f28f1d565a9b586dd7099d3b6675b3d32bc99ca45c1fc13d2b7f3020b532d5e12a7ae0af69f71af223e95cbb10b1ba13413a161f31204
+  metadata.gz: d907b4d932d092bae9b440b31226e12d8c68d8b54e5806076dde56c7b68569f5087352903d97db250d7dc603f14418499527758823a6428f20485d8cc2bee8ab
+  data.tar.gz: fa3189ca4d1c1ae453dcd1c7e6cd8d9fc227fd2524fbff07f87e6584e1858295a5b72ec40812b27e1353c448113af571466cc581fd82647ab34ccfccd2597489

data/Gemfile

@@ -1,15 +1,18 @@
 source 'http://rubygems.org'
 gemspec # Specify  gem's dependencies in mongoid_geospatial.gemspec
 
-group :development do
-  gem 'mongoid'
-  gem 'mongoid_geospatial'
-  # gem 'pg'
+gem 'rake'
+gem 'dbf'
+
+platforms :ruby do
   gem 'rgeo'
-  gem 'georuby'
-  gem 'dbf'
-  gem 'rspec'
-  gem 'guard-rspec'
-  gem 'pry'
-  # gem 'fuubar'
 end
+
+gem 'georuby'
+
+gem 'mongoid'
+gem 'mongoid_geospatial'
+
+gem 'pry'
+gem 'rspec'
+gem 'guard-rspec'

data/Guardfile

@@ -4,6 +4,6 @@ require 'guard'
 
 guard 'rspec' do
   watch(%r{^spec/.+_spec\.rb$})
-  watch(%r{^lib/(.+)\.rb$})     { "spec" } # { |m| "spec spec/lib/#{m[1]}_spec.rb" }
-  watch('spec/spec_helper.rb')  { "spec" }
+  watch(%r{^lib/(.+)\.rb$})     { 'spec' } # { |m| "spec spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { 'spec' }
 end

data/Rakefile

@@ -1,5 +1,5 @@
 #!/usr/bin/env rake
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
 
 require 'rspec/core'
 require 'rspec/core/rake_task'
@@ -13,49 +13,48 @@ RSpec::Core::RakeTask.new(:rcov) do |spec|
   spec.rcov = true
 end
 
-task :default => :spec
+task default: :spec
 
 require 'yard'
 
-
 #
 # Tokyo Tyrant rake tasks
 #
 namespace :tyrant do
-  TYRANT_DB_FILE  = File.join("tyrant.tct")
-  TYRANT_PID_FILE = File.join("tyrant.pid")
-  TYRANT_LOG_FILE = File.join("tyrant.log")
+  TYRANT_DB_FILE  = File.join('tyrant.tct')
+  TYRANT_PID_FILE = File.join('tyrant.pid')
+  TYRANT_LOG_FILE = File.join('tyrant.log')
 
-  desc "Start Tyrant server"
+  desc 'Start Tyrant server'
   task :start  do
-    raise RuntimeError, "Tyrant is already running." if tyrant_running?
+    fail RuntimeError, 'Tyrant is already running.' if tyrant_running?
     system "ttserver -pid #{TYRANT_PID_FILE} -log #{TYRANT_LOG_FILE} #{TYRANT_DB_FILE}&"
     sleep(2)
     if tyrant_running?
       puts "Tyrant started successfully (pid #{tyrant_pid})."
     else
-      puts "Failed to start tyrant push server. Check logs."
+      puts 'Failed to start tyrant push server. Check logs.'
     end
   end
 
-  desc "Stop Tyrant server"
+  desc 'Stop Tyrant server'
   task :stop do
-    raise RuntimeError, "Tyrant isn't running." unless tyrant_running?
+    fail RuntimeError, "Tyrant isn't running." unless tyrant_running?
     system "kill #{tyrant_pid}"
     sleep(2)
     if tyrant_running?
       puts "Tyrant didn't stopped. Check the logs."
     else
-      puts "Tyrant stopped."
+      puts 'Tyrant stopped.'
     end
   end
 
-  desc "Restart Tyrant server"
-  task :restart => [:stop, :start]
+  desc 'Restart Tyrant server'
+  task restart: [:stop, :start]
 
-  desc "Get Tyrant Server Status"
+  desc 'Get Tyrant Server Status'
   task :status do
-    puts tyrant_running? ? "Tyrant running. (#{tyrant_pid})" : "Tyrant not running."
+    puts tyrant_running? ? "Tyrant running. (#{tyrant_pid})" : 'Tyrant not running.'
   end
 end
 
@@ -67,7 +66,7 @@ def tyrant_running?
   return false unless File.exist?(TYRANT_PID_FILE)
   process_check = `ps -p #{tyrant_pid} | wc -l`
   if process_check.to_i < 2
-    puts "Erasing pidfile..."
+    puts 'Erasing pidfile...'
     `rm #{TYRANT_PID_FILE}`
   end
   tyrant_pid

data/geonames.yml

@@ -3,7 +3,7 @@
 #
 :store: mongodb
 :level: city
-:nations: [br]
+:nations: [br, it, nl]
 :locales: [pt, en]
 :min_pop: 100000
 :mapping:
@@ -11,7 +11,7 @@
   :geom: true
 :db:
   :host: localhost
-  :name: baccafe_development
+  :name: geonames_test
   :user:
   :pass:
   :purge: false

data/geonames_local.gemspec

@@ -2,33 +2,32 @@
 require File.expand_path('../lib/geonames_local/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.name = %q{geonames_local}
+  gem.name = 'geonames_local'
   gem.version = Geonames::VERSION
-  gem.homepage = %q{http://github.com/nofxx/geonames_local}
+  gem.homepage = 'http://github.com/nofxx/geonames_local'
 
-  gem.authors = ["Marcos Piccinini"]
-  gem.default_executable = %q{geonames}
-  gem.description = %q{Dumps geonames data to feed a local db}
-  gem.email = %q{x@nofxx.com}
-  gem.license = "MIT"
+  gem.authors = ['Marcos Piccinini']
+  gem.default_executable = 'geonames'
+  gem.description = 'Dumps geonames data to feed a local db'
+  gem.email = 'x@nofxx.com'
+  gem.license = 'MIT'
 
-  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   gem.files         = `git ls-files`.split("\n")
   gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  gem.name          = "geonames_local"
-  gem.require_paths = ["lib"]
-  gem.summary = %q{Dumps geonames data for local usage}
-
+  gem.name          = 'geonames_local'
+  gem.require_paths = ['lib']
+  gem.summary = 'Dumps geonames data for local usage'
 
   gem.extra_rdoc_files = [
-    "MIT-LICENSE",
-    "README.md"
+    'MIT-LICENSE',
+    'README.md'
   ]
 
-  gem.add_dependency('mongoid', ['~> 3.1.0'])
+  gem.add_dependency('mongoid', ['> 3.0.0'])
   gem.add_dependency('geopolitical', ['> 0.8.1'])
 
-  gem.post_install_message = %q{
+  gem.post_install_message = '
 Geonames Local
 --------------
 
@@ -36,5 +35,6 @@ Use `geonames init` to create a config.yml file.
 Or `geonames -c config.yml` to run using a config file.
 
 Have fun!
-}
+
+'
 end

data/lib/geonames_local/cli.rb

@@ -2,13 +2,11 @@
 # Geonames Local
 #
 require 'optparse'
+require 'benchmark'
 # Require CLI Stuff
-require 'geonames_local/geonames'
 require 'geonames_local/data/shp'
 require 'geonames_local/data/dump'
 require 'geonames_local/data/export'
-require 'geonames_local/cli'
-
 
 module Geonames
   #
@@ -23,7 +21,7 @@ module Geonames
         opts.banner = 'Geonames Command Line Usage\n\n    geonames <nation code(s)> <opts>\n\n\n'
 
         opts.on('-l', '--level LEVEL', String, 'The level of logging to report') { |level| options[:level] = level }
-        opts.on('-d', '--dump', 'Dump DB before all') { options[:dump] = true }
+        opts.on('-d', '--dump', 'Dump DB before all') { options[:clean] = true }
         opts.separator ''
         opts.separator 'Config file:'
         opts.on('-c', '--config CONFIG', String, 'Geonames Config file path') { |file|  options[:config] = file }
@@ -57,14 +55,13 @@ module Geonames
     private_class_method :parse_options
 
     class << self
-
       def load_config
-        info "Loading config file..."
+        info 'Loading config file...'
         if Opt[:config]
           Opt.merge! YAML.load(File.read(Opt[:config]))
         else
           # Load config/geonames.yml if there's one
-          if File.exists?(cfg = File.join('config', 'geonames.yml'))
+          if File.exist?(cfg = File.join('config', 'geonames.yml'))
             Opt.merge! YAML.load(File.read(cfg))
           else
             fail
@@ -75,12 +72,42 @@ module Geonames
         exit
       end
 
-      # Ugly but works?
-      def work(argv)
-        info "Geopolitical Local Start!"
-
+      def trap_signals
+        puts 'Geopolitical Local Start!'
         trap(:INT) { stop! }
         trap(:TERM) { stop! }
+      end
+
+      def wrapper
+        Geonames::Models::MongoWrapper
+      end
+
+      def work_nations
+        info "\nPopulating 'nations' database..."
+        dump = Geonames::Dump.new(:all, :dump)
+        info "\n---\nTotal #{dump.data.length} parsed."
+
+        info 'Writing to nations DB'
+        wrapper.nations dump.data
+      end
+
+      def work_spots
+        info "\nPopulating 'regions' and 'cities' database..."
+        zip = Geonames::Dump.new(Opt[:nations], :zip).data
+        dump = Geonames::Dump.new(Opt[:nations], :dump).data
+        info "\n---\nTotal #{dump.size} parsed. #{zip.size} zip codes."
+
+        info 'Join dump << zip'
+        dump = unify!(dump, zip).group_by(&:kind)
+
+        info 'Writing to DB...'
+        wrapper.batch dump
+      end
+
+      # Ugly but works?
+      def work(argv)
+        start = Time.now
+        trap_signals
         Opt.merge! parse_options(argv)
         if Opt[:locales].nil? || Opt[:locales].empty?
           Opt[:locales] = ['en']
@@ -95,7 +122,7 @@ module Geonames
         # Return Codes and Exit
         #
         if argv[0] =~ /list|codes/
-          Codes.each do |key,val|
+          Codes.each do |key, val|
             str = [val.values, key.to_s].join(' ').downcase
             if s = argv[1]
               next unless str =~ /#{s.downcase}/
@@ -119,85 +146,42 @@ module Geonames
           exit
         end
 
-        #
-        # Require georuby optionally
-        #
-        require 'geo_ruby' if Opt[:mapping] && Opt[:mapping][:geom]
-
-        #
         # Load config if we got til here
-        #
         load_config
 
-        #
         # Export Data as CSV or JSON
-        #
-        if argv[0] =~ /csv|json/
-          Geonames::Export.new(Nation.all).to_csv
-
-          #
-          # Do the magic! Import Geonames Data
-          #
-        else
-          load_adapter(Opt[:store])
-          info "Using adapter #{Opt[:store]}.."
-
-          # Nations
-          if Opt[:nations].empty? || argv[0] =~ /coun|nati/
-            info "\nPopulating 'nations' database..."
-            dump = Geonames::Dump.new(:all, :dump)
-            info "\n---\nTotal #{dump.data.length} parsed."
-
-            info 'Writing to nations DB'
-            Geonames::Models::MongoWrapper.nations dump.data, Opt[:clean]
-
-            # Regions, Cities....
-          else
-            zip = Geonames::Dump.new(Opt[:nations], :zip).data
-            dump = Geonames::Dump.new(Opt[:nations], :dump).data
-            info "\n---\nTotal #{dump.size} parsed. #{zip.size} zips."
-
-            info 'Join dump << zip'
-            dump = unify!(dump, zip).group_by(&:kind)
-
-            info 'Writing to DB...'
-            Geonames::Models::MongoWrapper.batch dump, Opt[:clean]
-            # info "Writing cities..."
-            # Geonames::Models::City.from_batch dump[:city]
-          end
-        end
+        return Geonames::Export.new(Nation.all).to_csv if argv[0] =~ /csv|json/
+
+        # Do the magic! Import Geonames Data
+        load_adapter(Opt[:store])
+        info "Using adapter #{Opt[:store]}.."
+        wrapper.clean if Opt[:clean]
+        Benchmark.measure { work_nations } unless wrapper.nations_populated?
+        Benchmark.measure { work_spots }
       end
 
       def load_adapter(name)
-        begin
-          require "geonames_local/models/#{name}"
-        rescue LoadError
-          info "Can't find adapter for #{name}"
-          stop!
-        end
+        require_relative "models/#{name}"
+      rescue LoadError => e
+        info "Can't find adapter for #{name} #{e}"
+        stop!
       end
 
       def unify!(dump, zip)
         start = Time.now
         dump.map! do |spot|
-          if other = zip.find { |d| d.code == spot.code }
-            spot.zip = other.zip
-            spot
-          else
-            spot
-          end
+          next spot unless (other = zip.find { |z| z.code == spot.code })
+          spot.zip = other.zip
+          spot
         end
         info "Done. #{(Time.now - start).to_i}s"
         dump
       end
 
       def stop!
-        puts 'Closing Geonames...'
+        info 'Closing Geonames...'
         exit
       end
-
     end # class < self
-
   end # CLI
-
 end # Geonames

data/lib/geonames_local/data/dump.rb

@@ -1,18 +1,18 @@
 module Geonames
   class Dump
+    attr_reader :data
+
     # Geonames base URL
-    URL = "http://download.geonames.org/export/"
+    URL = 'http://download.geonames.org/export/'
     # Work temporary files
-    TMP = "/tmp/geonames/"
+    TMP = '/tmp/geonames/'
 
     def initialize(target, kind)
       @kind = kind
       @data = []
-      if target.respond_to? :each
-        target.each { |n| work(n) }
-      elsif target == :all
-        nations
-      end
+
+      target.each { |n| work(n) } if target.respond_to? :each
+      nations if target == :all
     end
 
     def nations
@@ -22,7 +22,7 @@ module Geonames
       parse file
     end
 
-    def work nation
+    def work(nation)
       info "\nWorking on #{@kind} for #{nation}"
       file = get_file(nation)
       download file
@@ -30,19 +30,15 @@ module Geonames
       parse file
     end
 
-    def data
-      @data
-    end
-
     def get_file(nation)
-      nation == "nation" ? "countryInfo.txt" : "#{nation.upcase}.zip"
+      nation == 'nation' ? 'countryInfo.txt' : "#{nation.upcase}.zip"
     end
 
     def download(file)
-      Dir.mkdir(TMP) unless File.exists?(TMP)
-      Dir.mkdir(TMP + @kind.to_s) unless File.exists?(TMP + @kind.to_s)
+      Dir.mkdir(TMP) unless File.exist?(TMP)
+      Dir.mkdir(TMP + @kind.to_s) unless File.exist?(TMP + @kind.to_s)
       fname = TMP + "#{@kind}/#{file}"
-      return if File.exists?(fname)
+      return if File.exist?(fname)
       `curl #{URL}/#{@kind}/#{file} -o #{fname}`
     end
 
@@ -54,34 +50,27 @@ module Geonames
     def parse_line(l)
       return if l =~ /^#|^iso/i
       if @kind == :dump
-        if l =~ /^\D/
-          return l
-        else
-          if Opt[:level] != "all"
-            return unless l =~ /ADM\d/ # ADM2 => cities
-          end
+        return l if l =~ /^\D/
+        if Opt[:level] != 'all'
+          return unless l =~ /ADM\d/ # ADM2 => cities
         end
       end
       Spot.new(l, @kind)
     end
 
     def parse(file)
-      red = 0
       start = Time.now
-      File.open("/tmp/geonames/#{@kind}/#{file.gsub("zip", "txt")}") do |f|
+      File.open("/tmp/geonames/#{@kind}/#{file.gsub('zip', 'txt')}") do |f|
         while line = f.gets
           if record = parse_line(line)
             @data << record
-            red += 1
           end
         end
         total = Time.now - start
-        info "#{red} #{@kind} spots parsed #{total}s (#{(red / total).to_i}/s)"
+        info "#{@data.size} #{@kind} spots parsed #{total}s (#{(@data.size / total).to_i}/s)"
       end
-      rescue Errno::ENOENT => e
+    rescue Errno::ENOENT => e
       info "Failed to download #{file}, skipping. #{e}"
     end
-
   end
-
 end

data/lib/geonames_local/data/export.rb

@@ -2,20 +2,19 @@ require 'csv'
 
 module Geonames
   class Export
-
     def initialize(data)
-      info "Starting export.."
+      info 'Starting export..'
       @data = data
     end
 
     def to_csv
-      file = "export.csv"
+      file = 'export.csv'
       info "Writing #{file} (#{@data.length} objects)"
-      CSV.open("export.csv", 'w') do |csv|
+      CSV.open('export.csv', 'w') do |csv|
         csv << @data[0].export_header
         @data.each { |o| csv << o.export }
       end
-      info "Export done."
+      info 'Export done.'
     end
   end
 end

data/lib/geonames_local/data/shp.rb

@@ -4,10 +4,9 @@
 
 module Geonames
   class SHP
-
     def initialize(file)
       @file = file
-      @fname = file.split("/")[-1] rescue nil
+      @fname = file.split('/')[-1] rescue nil
       @type = Object.module_eval("::#{Opt[:type].capitalize}", __FILE__, __LINE__)
       # @ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
       @sample = nil
@@ -17,7 +16,7 @@ module Geonames
     end
 
     def shp2pg
-      info "Converting SRID"
+      info 'Converting SRID'
       `shp2pgsql -D -as 4326 #{@file} nil > /tmp/#{@fname}.dump`
     end
 
@@ -32,7 +31,7 @@ module Geonames
     end
 
     def parse
-      info "Parsing dump"
+      info 'Parsing dump'
       start = Time.now
       red = 0
       File.open("/tmp/#{@fname}.dump") do |f|
@@ -44,15 +43,15 @@ module Geonames
           end
         end
       end
-      info "#{red} parsed. #{Time.now-start}s"
+      info "#{red} parsed. #{Time.now - start}s"
     end
 
     def reduce!
-      hsh = Cache[:roads].group_by { |r| r.name }
+      hsh = Cache[:roads].group_by(&:name)
       arr = []
-      hsh.map do |key, vals|
+      hsh.map do |_key, vals|
         first = vals.delete_at(0)
-#        p vals[0].geom.geometries.concat(vals[1].geom.geometries)
+        #        p vals[0].geom.geometries.concat(vals[1].geom.geometries)
         vals.map(&:geom).each do |g|
           first.geom.geometries.concat g.geometries
         end

data/lib/geonames_local/features/road.rb

@@ -1,10 +1,10 @@
-require "rubygems"
-require "geo_ruby"
-
+#
+# Road/Street/Via/Autoban
+#
 class Road
   attr_reader :city, :region, :nation, :zone, :name, :geom, :kind, :table
 
-  def initialize(keys, vals, nation=nil, city=nil)
+  def initialize(keys, vals, nation = nil, city = nil)
     s = vals.split("\t")
     r = {}
     keys.each_with_index do |k, i|
@@ -12,7 +12,7 @@ class Road
     end
     @name = r[:name]
     @zone = r[:zone]
-    kind  = r[:kind] || @name.split(" ")[0]
+    kind  = r[:kind] || @name.split(' ')[0]
     @geom = parse_geom(r[:geom])
     @kind = parse_kind(kind)
     @city = city
@@ -22,14 +22,12 @@ class Road
 
   def parse_geom(hex)
     if hex =~ /^SRID/ # PG 8.3 support
-      hex = hex.split(";")[1]
+      hex = hex.split(';')[1]
     end
     GeoRuby::SimpleFeatures::Geometry.from_hex_ewkb(hex)
   end
 
-  def geom=(g)
-    @geom = g
-  end
+  attr_writer :geom
 
   def parse_kind(k)
     case k
@@ -40,5 +38,4 @@ class Road
       else :unknown
     end
   end
-
 end

data/lib/geonames_local/features/spot.rb

@@ -5,10 +5,10 @@ module Geonames
   class Spot
     attr_accessor :gid, :name, :ascii, :lat, :lon, :nation, :kind,
                   :code,  :pop, :tz, :geom, :region, :zip, :abbr, :id
-    alias :x :lon
-    alias :y :lat
-    alias :geoname_id :gid
-    alias :table :kind
+    alias_method :x, :lon
+    alias_method :y, :lat
+    alias_method :geoname_id, :gid
+    alias_method :table, :kind
 
     #
     # = Geonames Spot
@@ -97,7 +97,6 @@ module Geonames
     end
 
     class << self
-
       attr_accessor :collection
 
       def nearest(x, y)
@@ -110,9 +109,6 @@ module Geonames
         spot.geom = GeoRuby::SimpleFeatures::Point.from_x_y(*spot.geom)
         spot
       end
-
     end
-
   end
-
 end

data/lib/geonames_local/features/zone.rb

@@ -1,10 +1,10 @@
-require "rubygems"
-require "geo_ruby"
-
+#
+# A polygon basically.
+#
 class Zone
   attr_reader :city, :name, :geom, :kind
 
-  def initialize(keys, vals, city=nil)
+  def initialize(keys, vals, city = nil)
     s = vals.split("\t")
     r = {}
     keys.each_with_index do |k, i|
@@ -20,5 +20,4 @@ class Zone
   def parse_geom(hex)
     @geom = GeoRuby::SimpleFeatures::Geometry.from_hex_ewkb(hex)
   end
-
 end