geonames_dump 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0d54cf96d1f31ae2a2fd2a94b6d8badeb317e0df
+  data.tar.gz: b2a120c181cc803aae3172523013206e2efa3f46
+SHA512:
+  metadata.gz: 2bb9c4fddd4bc7838967891358be49dd1ef1a999275223e524a939515a58d85882a90f05fbea423a1b1abb7db9f9aa2ad79292c6e39ac217962cefde3f0bd4b2
+  data.tar.gz: de04975b021b84aea162b710124fefa8fbe06b55ca4c4c6b8429426088efa9a016651307656245633cacb26a571b28b31c4590c4691727dff96199ea60722d36

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+.DS_Store
+.*.swp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in geonames.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2012 Thomas Kienlen, La Fourmi Immo
+Including original code Copyright (c) 2009 Alex Pooley, Brown Beagle Software
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,85 @@
+# GeonamesDump
+
+GeonamesDump import geographic data from geonames project into your application, avoiding to use external service like google maps.
+It's a "gem" version of the application [brownbeagle/geonames](https://github.com/brownbeagle/geonames).
+Now you only need to include the dependency into your Gemfile and your project will include geonames.
+
+You're free to use [geocoder](https://github.com/alexreisner/geocoder) or [geokit](https://github.com/imajes/geokit) or any other geocoding solution.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'geonames_dump'
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install geonames_dump
+
+## Usage
+
+Create models and migration files
+
+    $ rails generate geonames_dump:install
+
+Import data (takes a loonnnng time!), it will download data, import countries and many features (Countries, Cities, Admin1 (first administrative subdivision), Admin2 (second level administrative subdivision))
+
+    $ rake geonames_dump:install
+
+If you need more fine grained control over the installation process you can run individual geoname rake tasks instead of the all-in-one install :
+
+    $ rake -T | grep geonames_dump
+    
+    rake geonames_dump:import:all          # Import ALL geonames data.
+    rake geonames_dump:import:many         # Import most of geonames data.
+    
+    rake geonames_dump:import:admin1       # Import admin1 codes
+    rake geonames_dump:import:admin2       # Import admin2 codes
+    rake geonames_dump:import:cities       # Import all cities, regardless of population.
+    rake geonames_dump:import:cities1000   # Import cities with population greater than 1000
+    rake geonames_dump:import:cities15000  # Import cities with population greater than 15000
+    rake geonames_dump:import:cities5000   # Import cities with population greater than 5000
+    rake geonames_dump:import:countries    # Import countries informations
+    rake geonames_dump:import:features     # Import feature data.
+    
+    rake geonames_dump:truncate:all        # Truncate all geonames data.
+    rake geonames_dump:truncate:countries  # Truncate countries informations
+    rake geonames_dump:truncate:admin1     # Truncate admin1 codes
+    rake geonames_dump:truncate:admin2     # Truncate admin2 codes
+    rake geonames_dump:truncate:cities     # Truncate cities informations
+    rake geonames_dump:truncate:features   # Truncate features informations
+
+## Geonames data usage
+
+The above commands will import geonames data in your Rails application, in other words, this will create models and fill database with place/city/country informations.
+Now to find a city for example :
+
+    GeonamesFeature.search('paris')
+
+If your request is ambiguous, like not searching Dublin in Ireland but Dublin in the USA, you may specify country :
+
+    GeonamesFeature.search('dublin, us')
+
+Models available allows to specify the type of place you want to search for :
+
+- GeonamesAdmin1, for first level of adminstrative subdivision
+- GeonamesAdmin2, for second level of adminstrative subdivision
+- GeonamesCity, for city names
+- GeonamesFeature, for generic names including all the above
+- GeonamesCountry, for country names
+
+Searching for a city like dublin may be done using :
+
+    GeonamesCity.search('dublin')
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/geonames_dump.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/geonames_dump/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Alex Pooley", "Thomas Kienlen"]
+  gem.email         = ["thomas.kienlen@lafourmi-immo.com"]
+  gem.description   = %q{GeonamesDump import geographic data from geonames project into your application, avoiding to use external service like Google Maps.}
+  gem.summary       = %q{Import data from Geonames}
+  gem.homepage      = "https://github.com/kmmndr/geonames_dump"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "geonames_dump"
+  gem.require_paths = ["lib"]
+  gem.version       = GeonamesDump::VERSION
+  gem.add_runtime_dependency 'ruby-progressbar'
+  gem.add_runtime_dependency 'activerecord-reset-pk-sequence'
+  gem.add_runtime_dependency 'rubyzip'
+end

data/lib/generators/geonames_dump/install_generator.rb

@@ -0,0 +1,32 @@
+module GeonamesDump
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      source_root File.expand_path(File.join(File.dirname(__FILE__), 'templates'))
+
+      desc <<DESC
+Description:
+    Create Geonames initial migrations files and Geonames model files in your application
+DESC
+      def copy_migrations_files
+        #template File.join('app', 'models', 'geonames_country.rb', 'config/initializers/piktur_config.rb'
+        %w(create_geonames_countries create_geonames_features).each do |file|
+          migration_template File.join('db', 'migrate', "#{file}.rb")
+        end
+      end
+
+      def copy_models_files
+        #template File.join('app', 'models', 'geonames_country.rb', 'config/initializers/piktur_config.rb'
+        %w(geonames_admin1 geonames_admin2 geonames_city geonames_country geonames_feature).each do |file|
+          copy_file File.join('app', 'models', "#{file}.rb")
+        end
+      end
+
+      def self.next_migration_number(path)
+        sleep 1 # OPTIMIZE: there might be a better way to do this ...
+        @migration_number = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i.to_s
+      end
+
+    end
+  end
+end

data/lib/generators/geonames_dump/templates/app/models/geonames_admin1.rb

@@ -0,0 +1,2 @@
+class GeonamesAdmin1 < GeonamesFeature
+end

data/lib/generators/geonames_dump/templates/app/models/geonames_admin2.rb

@@ -0,0 +1,2 @@
+class GeonamesAdmin2 < GeonamesFeature
+end

data/lib/generators/geonames_dump/templates/app/models/geonames_city.rb

@@ -0,0 +1,2 @@
+class GeonamesCity < GeonamesFeature
+end

data/lib/generators/geonames_dump/templates/app/models/geonames_country.rb

@@ -0,0 +1,25 @@
+class GeonamesCountry < ActiveRecord::Base
+  validates_uniqueness_of :geonameid
+
+  ##
+  # search by iso first, then by name if not found
+  #
+  scope :search, lambda { |q|
+    by_iso(q).count > 0 ? by_iso(q) : by_name(q)
+  }
+
+  ##
+  # search by iso code
+  #
+  scope :by_iso, lambda { |q|
+    where("iso = ? or iso3 = ?", q.upcase, q.upcase)
+  }
+
+  ##
+  # search by name
+  #
+  scope :by_name, lambda { |q|
+    where("country LIKE ?", "#{q}%")
+  }
+
+end

data/lib/generators/geonames_dump/templates/app/models/geonames_feature.rb

@@ -0,0 +1,49 @@
+class GeonamesFeature < ActiveRecord::Base
+  validates_uniqueness_of :geonameid
+  before_save :set_asciiname_first_letters
+
+  ##
+  # Search for feature, searches might include country (separated by ',')
+  #
+  scope :search, lambda { |query| 
+    virgule = query.include? ','
+
+    splited = virgule ? query.split(',') : [query]
+
+    country = virgule ? splited.last.strip : nil
+
+    queries = virgule ? splited[0...-1] : splited
+    queries = queries.join(' ').split(' ')
+
+    ret = by_name(*queries)
+
+    unless country.nil?
+      geonames_country = GeonamesCountry.search(country).first
+      ret = ret.where(:country => geonames_country.iso) unless geonames_country.nil?
+    end
+
+    ret
+  }
+
+  ##
+  # Find by names
+  #
+  scope :by_name, lambda { |*queries|
+    ret = self.scoped
+    count = queries.count
+    queries.collect.with_index do |q, idx|
+      query = idx == 0 ? "#{q}" : "%#{q}%"
+      ret = ret.where("asciiname_first_letters = ?", q[0...3].downcase)
+      ret = ret.where("name LIKE ? or asciiname LIKE ?", query, query)
+    end
+    ret
+  }
+
+  ##
+  # Set first letters of name into index column
+  #
+  def set_asciiname_first_letters
+    self.asciiname_first_letters = self.asciiname[0...3].downcase unless self.asciiname.nil?
+  end
+
+end

data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_countries.rb

@@ -0,0 +1,38 @@
+
+# http://download.geonames.org/export/dump/countryInfo.txt
+
+class CreateGeonamesCountries < ActiveRecord::Migration
+
+  def self.up
+    create_table :geonames_countries do |t|
+      t.string :iso
+      t.string :iso3
+      t.string :iso_numeric
+      t.string :fips
+      t.string :country
+      t.string :capital
+      t.integer :area # in sq. km
+      t.integer :population
+      t.string :continent
+      t.string :tld
+      t.string :currency_code
+      t.string :currency_name
+      t.string :phone
+      t.string :postal_code_format
+      t.string :postal_code_regex
+      t.string :languages
+      t.integer :geonameid
+      t.string :neighbours
+      t.string :equivalent_fips_code
+
+      t.timestamps
+    end
+
+    add_index :geonames_countries, :geonameid
+    add_index :geonames_countries, :country
+  end
+
+  def self.down
+    drop_table :geonames_countries
+  end
+end

data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_features.rb

@@ -0,0 +1,64 @@
+class CreateGeonamesFeatures < ActiveRecord::Migration
+
+  # geonameid         : integer id of record in geonames database
+  # name              : name of geographical point (utf8) varchar(200)
+  # asciiname         : name of geographical point in plain ascii characters, varchar(200)
+  # alternatenames    : alternatenames, comma separated varchar(4000) (varchar(5000) for SQL Server)
+  # latitude          : latitude in decimal degrees (wgs84)
+  # longitude         : longitude in decimal degrees (wgs84)
+  # feature class     : see http://www.geonames.org/export/codes.html, char(1)
+  # feature code      : see http://www.geonames.org/export/codes.html, varchar(10)
+  # country code      : ISO-3166 2-letter country code, 2 characters
+  # cc2               : alternate country codes, comma separated, ISO-3166 2-letter country code, 60 characters
+  # admin1 code       : fipscode (subject to change to iso code), isocode for the us and ch, see file admin1Codes.txt for display names of this code; varchar(20)
+  # admin2 code       : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80) 
+  # admin3 code       : code for third level administrative division, varchar(20)
+  # admin4 code       : code for fourth level administrative division, varchar(20)
+  # population        : bigint (4 byte int) 
+  # elevation         : in meters, integer
+  # gtopo30           : average elevation of 30'x30' (ca 900mx900m) area in meters, integer
+  # timezone          : the timezone id (see file timeZone.txt)
+  # modification date : date of last modification in yyyy-MM-dd format
+  def self.up
+    create_table :geonames_features do |t|
+      t.integer :geonameid
+      t.string :name
+      t.string :asciiname
+      t.string :alternatenames
+      t.float :latitude
+      t.float :longitude
+      t.string :feature_class
+      t.string :feature
+      t.string :country
+      t.string :cc2
+      t.string :admin1
+      t.string :admin2
+      t.string :admin3
+      t.string :admin4
+      t.integer :population
+      t.integer :elevation
+      t.integer :gtopo30
+      t.string :timezone
+      t.timestamp :modification
+      t.string :type
+      # OPTIMIZE: create another table for all names (alternate names)
+      t.string :asciiname_first_letters
+
+
+      t.timestamps
+    end
+
+    add_index :geonames_features, :geonameid
+    add_index :geonames_features, :name
+    add_index :geonames_features, :asciiname
+    add_index :geonames_features, :country
+    add_index :geonames_features, :population
+    add_index :geonames_features, :admin1
+    add_index :geonames_features, :type
+    add_index :geonames_features, :asciiname_first_letters
+  end
+
+  def self.down
+    drop_table :geonames_features
+  end
+end

data/lib/generators/geonames_dump/templates/piktur_config.rb

@@ -0,0 +1,19 @@
+Piktur.configure do |config|
+  ##config.public_abs_path = Rails.root.join(Rails.public_path).to_s 
+  #config.public_abs_path = "nullll"
+  #config.gallery_base_path = "piktur/"
+  #config.thumbs_dir = "thumbs/"
+
+  #config.thumb_extension = ".jpg"
+  #config.image_extensions = ['jpg', 'jpeg']
+  ##config.image_extensions = ['*']
+  #config.associated_extension = ".yml"
+
+  ##config.thumb_file = "%{basename}-%{width}-%{height}%{extension}"
+  #config.thumb_file = "%{basename}-size%{size}%{extension}"
+  #config.associated_file = "%{basename}%{extension}"
+  #config.watermark_file = nil
+
+  #config.thumbs_sizes = [[140, 110], [640, 480]]
+  #config.param_path = :path
+end

data/lib/geonames_dump/blocks.rb

@@ -0,0 +1,31 @@
+module GeonamesDump
+  class Blocks
+    attr_accessor :blocks
+    def initialize
+      reset
+    end
+
+    def reset
+      self.blocks = []
+    end
+
+    def add_block(&block)
+      self.blocks << block
+    end
+
+    def empty?
+      blocks.empty?
+    end
+
+    def call_and_reset
+      call
+      reset
+    end
+
+    def call
+      blocks.each do |block|
+        block.call
+      end
+    end
+  end
+end

data/lib/geonames_dump/railtie.rb

@@ -0,0 +1,11 @@
+#require 'rails'
+
+module GeonamesDump
+  class Railtie < ::Rails::Railtie #:nodoc:
+    # import rake tast into rails application
+    rake_tasks do
+      Dir[File.join(File.dirname(__FILE__),'../tasks/*.rake')].each { |f| load f }
+    end
+  end
+end
+

data/lib/geonames_dump/version.rb

@@ -0,0 +1,3 @@
+module GeonamesDump
+  VERSION = '0.0.2'
+end

data/lib/geonames_dump.rb

@@ -0,0 +1,8 @@
+require "geonames_dump/version"
+require "geonames_dump/blocks"
+require "geonames_dump/railtie" #if defined?(Rails)
+
+module GeonamesDump
+  # TODO: remove this logger
+  #ActiveRecord::Base.logger = Logger.new(STDOUT) if Rails.env.development? && !ENV['SILENT']
+end

data/lib/tasks/import.rake

@@ -0,0 +1,257 @@
+require 'net/http'
+require 'ruby-progressbar'
+require 'zip/zip'
+require 'geonames_dump'
+
+namespace :geonames_dump do
+  namespace :import do
+    CACHE_DIR = "#{Rails.root}/db/geonames_cache"
+
+    GEONAMES_FEATURES_COL_NAME = [
+        :geonameid, :name, :asciiname, :alternatenames, :latitude, :longitude, :feature_class,
+        :feature, :country, :cc2, :admin1, :admin2, :admin3, :admin4, :population, :elevation,
+        :gtopo30, :timezone, :modification
+      ]
+    GEONAMES_COUNTRIES_COL_NAME = [
+        :iso, :iso3, :iso_numeric, :fips, :country, :capital, :area, :population, :continent,
+        :tld, :currency_code, :currency_name, :phone, :postal_code_format, :postal_code_regex,
+        :languages, :geonameid, :neighbours, :equivalent_fips_code
+      ]
+    GEONAMES_ADMINS_COL_NAME = [ :code, :name, :asciiname, :geonameid ]
+
+    desc 'Prepare everything to import data'
+    task :prepare do
+      Dir::mkdir(CACHE_DIR) rescue nil
+      disable_logger
+      disable_validations if ENV['QUICK']
+    end
+
+    desc 'Import ALL geonames data.'
+    task :all => [:many, :features]
+
+    desc 'Import most of geonames data. Recommended after a clean install.'
+    task :many => [:prepare, :countries, :cities, :admin1, :admin2]
+
+    desc 'Import all cities, regardless of population.'
+    task :cities => [:prepare, :cities15000, :cities5000, :cities1000]
+
+    desc 'Import feature data. Specify Country ISO code (example : COUNTRY=FR) for just a single country. NOTE: This task can take a long time!'
+    task :features => [:prepare, :environment] do
+      download_file = ENV['COUNTRY'].present? ? ENV['COUNTRY'].upcase : 'allCountries'
+      zip_filename = download_file+'.zip'
+
+      txt_file = get_or_download("http://download.geonames.org/export/dump/#{zip_filename}")
+
+      # Import into the database.
+      File.open(txt_file) do |f|
+        #VALID_FEATURES = %w[ADMIN1]
+        #feature_attrs = VALID_FEATURES & ENV.keys
+        # TODO: add feature selection
+        insert_data(f, GEONAMES_FEATURES_COL_NAME, GeonamesCity, :title => "Features")
+      end
+    end
+
+    # geonames:import:citiesNNN where NNN is population size.
+    %w[15000 5000 1000].each do |population|
+      desc "Import cities with population greater than #{population}"
+      task "cities#{population}".to_sym => [:prepare, :environment] do
+
+        txt_file = get_or_download("http://download.geonames.org/export/dump/cities#{population}.zip")
+
+        # Import into the database.
+        File.open(txt_file) do |f|
+          insert_data(f, GEONAMES_FEATURES_COL_NAME, GeonamesCity, :title => "cities of #{population}")
+        end
+      end
+    end
+
+    desc 'Import countries informations'
+    task :countries => :environment do
+      txt_file = get_or_download('http://download.geonames.org/export/dump/countryInfo.txt')
+
+      # Import into the database.
+      File.open(txt_file) do |f|
+        insert_data(f, GEONAMES_COUNTRIES_COL_NAME, GeonamesCountry, :title => "Countries")
+      end
+    end
+
+    desc 'Import admin1 codes'
+    task :admin1 => [:prepare, :environment] do
+      txt_file = get_or_download('http://download.geonames.org/export/dump/admin1CodesASCII.txt')
+
+      # Import into the database.
+      File.open(txt_file) do |f|
+        insert_data(f, GEONAMES_ADMINS_COL_NAME, GeonamesAdmin1, :title => "Admin1 subdivisions") do |klass, attributes, col_value, idx|
+          col_value.gsub!('(general)', '')
+          col_value.strip!
+          if idx == 0
+            country, admin1 = col_value.split('.')
+            attributes[:country] = country.strip
+            attributes[:admin1] = admin1.strip rescue nil
+          else
+            attributes[GEONAMES_ADMINS_COL_NAME[idx]] = col_value
+          end
+        end
+      end
+    end
+
+    desc 'Import admin2 codes'
+    task :admin2 => [:prepare, :environment] do
+      txt_file = get_or_download('http://download.geonames.org/export/dump/admin2Codes.txt')
+
+      # Import into the database.
+      File.open(txt_file) do |f|
+        insert_data(f, GEONAMES_ADMINS_COL_NAME, GeonamesAdmin2, :title => "Admin2 subdivisions") do |klass, attributes, col_value, idx|
+          col_value.gsub!('(general)', '')
+          if idx == 0
+            country, admin1, admin2 = col_value.split('.')
+            attributes[:country] = country.strip
+            attributes[:admin1] = admin1.strip #rescue nil
+            attributes[:admin2] = admin2.strip #rescue nil
+          else
+            attributes[GEONAMES_ADMINS_COL_NAME[idx]] = col_value
+          end
+        end
+      end
+    end
+
+    private
+
+    def disable_logger
+      ActiveRecord::Base.logger = Logger.new('/dev/null')
+    end
+
+    def disable_validations
+      ActiveRecord::Base.reset_callbacks(:validate)
+    end
+
+
+    def get_or_download(url, options = {})
+      filename = File.basename(url)
+      unzip = File.extname(filename) == '.zip'
+      txt_filename = unzip ? "#{File.basename(filename, '.zip')}.txt" : filename
+      txt_file_in_cache = File.join(CACHE_DIR, options[:txt_file] || txt_filename)
+      zip_file_in_cache = File.join(CACHE_DIR, filename)
+
+      unless File::exist?(txt_file_in_cache)
+        puts 'file doesn\'t exists'
+        if unzip
+          download(url, zip_file_in_cache)
+          unzip_file(zip_file_in_cache, CACHE_DIR)
+        else
+          download(url, txt_file_in_cache)
+        end
+      else
+        puts "file already exists : #{txt_file_in_cache}"
+      end
+
+      ret = (File::exist?(txt_file_in_cache) ? txt_file_in_cache : nil)
+    end
+
+    def unzip_file(file, destination)
+      puts "unzipping #{file}"
+      Zip::ZipFile.open(file) do |zip_file|
+        zip_file.each do |f|
+          f_path = File.join(destination, f.name)
+          FileUtils.mkdir_p(File.dirname(f_path))
+          zip_file.extract(f, f_path) unless File.exist?(f_path)
+        end
+      end
+    end
+
+    def download(url, output)
+      File.open(output, "wb") do |file|
+        body = fetch(url)
+        puts "Writing #{url} to #{output}"
+        file.write(body)
+      end
+    end
+
+    def fetch(url)
+      puts "Fetching #{url}"
+      url = URI.parse(url)
+      req = Net::HTTP::Get.new(url.path)
+      res = Net::HTTP.start(url.host, url.port) {|http| http.request(req)}
+      return res.body
+    end
+
+    def insert_data(file_fd, col_names, main_klass = GeonamesFeature, options = {}, &block)
+      # Setup nice progress output.
+      file_size = file_fd.stat.size
+      title = options[:title] || 'Feature Import'
+      progress_bar = ProgressBar.create(:title => title, :total => file_size, :format => '%a |%b>%i| %p%% %t')
+
+      # create block array
+      blocks = GeonamesDump::Blocks.new
+      line_counter = 0
+
+      file_fd.each_line do |line|
+        # prepare data
+        attributes = {}
+        klass = main_klass
+
+        # skip comments
+        next if line.start_with?('#')
+
+        line_counter += 1
+
+        # read values
+        line.strip.split("\t").each_with_index do |col_value, idx|
+          col = col_names[idx]
+
+          # skip leading and trailing whitespace
+          col_value.strip!
+
+          # block may change the type of object to create
+          if block_given?
+            yield klass, attributes, col_value, idx
+          else
+            attributes[col] = col_value
+          end
+        end
+
+        # create or update object
+        #if filter?(attributes) && (block && block.call(attributes))
+        blocks.add_block do
+          if attributes.include?(:geonameid)
+            object = klass.where(geonameid: attributes[:geonameid]).first_or_initialize
+            object.update_attributes(attributes)
+            object.save if object.new_record? || object.changed?
+          else
+            klass.create(attributes)
+          end
+        end
+
+        # increase import speed by performing insert using transaction
+        if line_counter % 1000 == 0
+          ActiveRecord::Base.transaction do
+            blocks.call_and_reset
+          end
+          line_counter = 0
+        end
+
+        # move progress bar
+        progress_bar.progress = file_fd.pos
+      end
+
+      unless blocks.empty?
+        ActiveRecord::Base.transaction do
+          blocks.call_and_reset
+        end
+      end
+    end
+
+    # Return true when either:
+    #  no filter keys apply.
+    #  all applicable filter keys include the filter value.
+    def filter?(attributes)
+      return attributes.keys.all?{|key| filter_keyvalue?(key, attributes[key])}
+    end
+
+    def filter_keyvalue?(col, col_value)
+      return true unless ENV[col.to_s]
+      return ENV[col.to_s].split('|').include?(col_value.to_s)
+    end
+
+  end
+end

data/lib/tasks/install.rake

@@ -0,0 +1,4 @@
+namespace :geonames_dump do
+  desc 'Truncate and install most of geonames data (country, admin1, admin2, cities)'
+  task :install => ['truncate:all', 'import:many']
+end

data/lib/tasks/migrate.rake

@@ -0,0 +1,34 @@
+
+namespace :geonames_dump do
+  desc 'Run db:migrate to be up-to-date'
+  #task :migrate => [:environment, :copy_migrations, 'db:migrate']
+  task :migrate => [:environment, 'db:migrate']
+
+#  # Inspired by http://interblah.net/plugin-migrations
+#  desc 'Copy over the migration files and migrate'
+#  task :copy_migrations => :environment do
+#    require 'fileutils'
+#    APP_MIGRATION_DIR = "#{Rails.root}/db/migrate"
+#    PLUGIN_MIGRATION_DIR = "#{File.dirname(__FILE__)}/../../db/migrate"
+#
+#    # Find all application migrations
+#    existing_migrations = Dir[APP_MIGRATION_DIR+'/*']
+#    existing_migrations.map!{|file| File.basename(file) =~ /^[0-9]+_(.+)$/ && $1}
+#
+#    # Find all of this plugin's migrations
+#    plugin_migrations = Dir[PLUGIN_MIGRATION_DIR+'/*']
+#    plugin_migrations.map!{|file| File.basename(file)}
+#    # ... handle case when copying to plugins migrate directory.
+#    plugin_migrations.map!{|file| File.basename(file) =~ /^[0-9]+_(.+)$/ ? $1 : File.basename(file)}
+#
+#    # Compare application migrations with this plugins set of migrations, and copy new migrations.
+#    puts 'There are no new migrations to copy.' if (plugin_migrations - existing_migrations).empty?
+#    base_time = Time.now
+#    (plugin_migrations - existing_migrations).each do |new_migration|
+#      migration_timestamp = base_time.strftime('%Y%m%d%H%M%S')
+#      puts "Copying from #{PLUGIN_MIGRATION_DIR}/#{new_migration} to #{APP_MIGRATION_DIR}/#{migration_timestamp}_#{new_migration}"
+#      FileUtils::cp "#{PLUGIN_MIGRATION_DIR}/#{new_migration}", "#{APP_MIGRATION_DIR}/#{migration_timestamp}_#{new_migration}"
+#      base_time+=1
+#    end
+#  end
+end

data/lib/tasks/truncate.rake

@@ -0,0 +1,40 @@
+require 'net/http'
+require 'ruby-progressbar'
+require 'activerecord-reset-pk-sequence'
+
+namespace :geonames_dump do
+  namespace :truncate do
+
+    desc 'Truncate all geonames data.'
+    #task :all => [:countries, :cities, :admin1, :admin2]
+    task :all => [:countries, :features]
+
+    desc 'Truncate admin1 codes'
+    task :admin1 => :environment do
+      GeonamesAdmin1.delete_all #&& GeonamesAdmin1.reset_pk_sequence
+    end
+
+    desc 'Truncate admin2 codes'
+    task :admin2 => :environment do
+      GeonamesAdmin2.delete_all #&& GeonamesAdmin2.reset_pk_sequence
+    end
+
+    desc 'Truncate cities informations'
+    task :cities => :environment do
+      GeonamesCity.delete_all #&& GeonamesCity.reset_pk_sequence
+    end
+    
+    desc 'Truncate countries informations'
+    task :countries => :environment do
+      GeonamesCountry.delete_all && GeonamesCountry.reset_pk_sequence
+    end
+    
+    desc 'Truncate features informations'
+    task :features => :environment do
+      GeonamesFeature.delete_all && GeonamesFeature.reset_pk_sequence
+    end
+    
+#    private
+
+  end
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: geonames_dump
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Alex Pooley
+- Thomas Kienlen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-reset-pk-sequence
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: GeonamesDump import geographic data from geonames project into your application,
+  avoiding to use external service like Google Maps.
+email:
+- thomas.kienlen@lafourmi-immo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- geonames_dump.gemspec
+- lib/generators/geonames_dump/install_generator.rb
+- lib/generators/geonames_dump/templates/app/models/geonames_admin1.rb
+- lib/generators/geonames_dump/templates/app/models/geonames_admin2.rb
+- lib/generators/geonames_dump/templates/app/models/geonames_city.rb
+- lib/generators/geonames_dump/templates/app/models/geonames_country.rb
+- lib/generators/geonames_dump/templates/app/models/geonames_feature.rb
+- lib/generators/geonames_dump/templates/db/migrate/create_geonames_countries.rb
+- lib/generators/geonames_dump/templates/db/migrate/create_geonames_features.rb
+- lib/generators/geonames_dump/templates/piktur_config.rb
+- lib/geonames_dump.rb
+- lib/geonames_dump/blocks.rb
+- lib/geonames_dump/railtie.rb
+- lib/geonames_dump/version.rb
+- lib/tasks/import.rake
+- lib/tasks/install.rake
+- lib/tasks/migrate.rake
+- lib/tasks/truncate.rake
+homepage: https://github.com/kmmndr/geonames_dump
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Import data from Geonames
+test_files: []