geonames_rails 0.1.0

data/README.md

@@ -0,0 +1,96 @@
+Geonames-Rails
+===
+Update 2
+---
+Converted the plugin into a gem
+
+Copy this into your application's Rakefile:
+
+  Dir["#{Gem.searcher.find('geonames_rails').full_gem_path}/lib/tasks/*.rake"].each { |ext| load ext }
+ 
+And to generate the migration and models, use
+
+  rails generate geonames_rails:migration
+  rails generate geonames_rails:models
+
+Thanks to Garrett Davis for converting it into Rails 3 and John Barton for making the thing in the first place.
+
+
+Update
+---
+
+Updated the plugin for Rails 3 (v3.0.7 clean install)
+
+  * Recoded Generators
+  * Moved tasks/geonames_rails.rake to lib/tasks/geonames_rails.rake
+  * Added force_encoding to correct encoding errors in puller.rb
+
+Thanks to John Barton (joho) for saving me some time, hopefully this fork does the same.
+
+
+Getting Started
+---
+
+I need a decent plugin that can do the following things with [the geonames database](http://www.geonames.org/)
+
+  * generate a standard db migration
+  * pull down the required geonames files from the web
+  * load those geonames files in the db (assuming a schema from the migration)
+  * generate me my models for free (but leave them in models so i can hack them up later)
+
+Install the plugin by doing the following in your rails app root dir
+
+  script/plugin install git://github.com/joho/geonames-rails.git
+
+You next step is to run the generators to give you the bare bones country/city models and the db migration
+
+  script/generate geonames_migration
+  script/generate geonames_models
+
+Once you've run that migration and got your models set up how you like you're right to pull down the data straight from the geonames server and into your database. It's as easy as one little command
+
+  rake geonames_rails:pull_and_load_data
+
+Advanced Usage
+---
+
+OK, so you've had a bit of a play and decided one of a couple of things
+
+  * the default fields i've chosen suck and you want more/less of them
+  * you don't like the storage method i've chosen. maybe you want to use a document store - or something else
+
+Well, you're covered. The method for writing out the country/city data is fully pluggable. All you need is a class that implements the following two methods
+
+  class MyCustomGeonamesWriter
+    def write_country(country_mapping)
+    end
+  
+    def write_cities(country_code, city_mappings)
+    end
+  end
+
+and then you can pass an instance of that class as the second argument to the geonames loader. See the rake task for an example.
+
+---
+
+Outstanding Tasks
+---
+
+TODO
+
+  * add regions? (maybe, i'm not sure they're worth it)
+
+DONE
+
+  * rake task that will pull the latest geonames data into temp files
+  * generators for the db migration and models
+  * actually put something in the db migrations
+  * allow you to declare which fields you're using from geonames
+    - currently store the mappings between field names and column of data in classing in the Mappings module
+    - pulled out the writing of the records into a class
+    - changed the loader so you can plug in whatever writer you want 
+  * write the text to AR converter
+
+---
+
+Copyright John Barton 2009

data/lib/generators/geonames_rails/migration_generator.rb

@@ -0,0 +1,28 @@
+require 'rails/generators'
+require 'rails/generators/base'
+require 'rails/generators/migration'
+
+module GeonamesRails
+  module Generators
+    class MigrationGenerator < Rails::Generators::Base
+      source_root File.expand_path(File.join(File.dirname(__FILE__), 'migration_templates'))
+      include Rails::Generators::Migration
+      desc "add the geonames migrations"
+
+      def generate_migration
+        migration_template 'geonames_tables.rb',"db/migrate/create_geonames_tables.rb"
+      end
+
+      def self.next_migration_number(path)
+        unless @prev_migration_nr
+          @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+        else
+          @prev_migration_nr += 1
+        end
+        @prev_migration_nr.to_s
+      end
+
+    end
+    
+  end
+end

data/lib/generators/geonames_rails/migration_templates/geonames_tables.rb

@@ -0,0 +1,79 @@
+class CreateGeonamesTables < ActiveRecord::Migration
+  def self.up
+    # blah
+    
+    # # create countries
+    create_table :countries do |t|
+      # [0] iso alpha2
+      t.string :iso_code_two_letter, :null => false
+      # [1] iso alpha3
+      t.string :iso_code_three_letter, :null => false
+      # [2] iso numeric
+      t.integer :iso_number, :null => false
+      # [3] fips code
+      # [4] name
+      t.string :name, :null => false
+      # [5] capital
+      t.string :capital
+      # [6] areaInSqKm
+      # [7] population
+      t.integer :population
+      # [8] continent
+      t.string :continent
+      # [9] top level domain
+      # [10] Currency code
+      t.string :currency_code
+      # [11] Currency name
+      t.string :currency_name
+      # [12] Phone
+      # [13] Postal Code Format
+      # [14] Postal Code Regex
+      # [15] Languages
+      # [16] Geoname id
+      t.integer :geonames_id #, :null => false
+      # [17] Neighbours
+      # [18] Equivalent Fips Code
+    end
+    
+    add_index :countries, :iso_code_two_letter, :unique => true
+    add_index :countries, :geonames_id, :null => false
+
+    # create cities
+    create_table :cities do |t|
+      t.integer :country_id, :null => false
+      # [0] geonameid : integer id of record in geonames database
+      t.integer :geonames_id, :null => false
+      # [1] name : name of geographical point (utf8) varchar(200)
+      t.string :name, :null => false
+      # [2] asciiname : name of geographical point in plain ascii characters, varchar(200)
+      # [3] alternatenames : alternatenames, comma separated varchar(4000)
+      # [4] latitude : latitude in decimal degrees (wgs84)
+      t.decimal :latitude, :precision => 14, :scale => 8, :null => false
+      # [5] longitude : longitude in decimal degrees (wgs84)
+      t.decimal :longitude, :precision => 14, :scale => 8, :null => false
+      # [6] feature class : see http://www.geonames.org/export/codes.html, char(1)
+      # [7] feature code : see http://www.geonames.org/export/codes.html, varchar(10)
+      # [8] country code : ISO-3166 2-letter country code, 2 characters
+      t.string :country_iso_code_two_letters
+      # [9] cc2 : alternate country codes, comma separated, ISO-3166 2-letter country code, 60 characters
+      # [10] admin1 code : fipscode (subject to change to iso code), isocode for the us and ch, see file admin1Codes.txt for display names of this code; varchar(20)
+      # [11] admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
+      # [12] admin3 code : code for third level administrative division, varchar(20)
+      # [13] admin4 code : code for fourth level administrative division, varchar(20)
+      # [14] population : integer
+      t.integer :population
+      # [15] elevation : in meters, integer
+      # [16] gtopo30 : average elevation of 30'x30' (ca 900mx900m) area in meters, integer
+      # [17] timezone : the timezone id (see file timeZone.txt)
+      t.integer :geonames_timezone_id
+      # [18] modification date : date of last modification in yyyy-MM-dd format
+    end
+    
+    add_index :cities, :geonames_id, :unique => true
+  end
+  
+  def self.down
+    # drop all the tables
+    %w(countries cities).each { |t| drop_table t }
+  end
+end

data/lib/generators/geonames_rails/models_generator.rb

@@ -0,0 +1,18 @@
+require 'rails/generators'
+ 
+module GeonamesRails
+  module Generators
+    class ModelsGenerator < Rails::Generators::Base
+      source_root File.expand_path(File.join(File.dirname(__FILE__), 'models_templates'))
+      include Rails::Generators::Migration
+      desc "add the geonames models (city and country)"
+ 
+      def generate_models
+        %w(country city).each do |model_name|
+          copy_file "models/#{model_name}.rb", "app/models/#{model_name}.rb"
+        end
+      end
+    end
+    
+  end
+end

data/lib/generators/geonames_rails/models_templates/models/city.rb

@@ -0,0 +1,3 @@
+class City < ActiveRecord::Base
+  belongs_to :country
+end

data/lib/generators/geonames_rails/models_templates/models/country.rb

@@ -0,0 +1,3 @@
+class Country < ActiveRecord::Base
+  has_many :cities
+end

data/lib/geonames_rails.rb

@@ -0,0 +1,13 @@
+Dir["tasks/**/*.rake"].each { |ext| load ext } if defined?(Rake)
+
+require 'open-uri'
+require 'fileutils'
+
+require 'geonames_rails/loader'
+require 'geonames_rails/puller'
+require 'geonames_rails/writers/dry_run'
+require 'geonames_rails/writers/active_record'
+
+require 'generators/geonames_rails/migration_generator'
+require 'generators/geonames_rails/models_generator'
+#puts 'this code runs'

data/lib/geonames_rails/loader.rb

@@ -0,0 +1,73 @@
+require 'geonames_rails/mappings/base'
+require 'geonames_rails/mappings/city'
+require 'geonames_rails/mappings/country'
+
+module GeonamesRails
+  
+  class Loader
+    
+    def initialize(puller, writer, logger = nil)
+      @logger = logger || STDOUT
+      @puller = puller
+      @writer = writer
+    end
+    
+    def load_data
+      @puller.pull if @puller # pull geonames files down
+      
+      #load_countries
+      
+      load_cities
+      
+      @puller.cleanup if @puller # cleanup the geonames files
+    end
+    
+  protected
+    def load_countries
+      log_message "opening countries file"
+      File.open(File.join(Rails.root, 'tmp', 'countryInfo.txt'), 'r') do |f|
+        f.each_line do |line|
+          # skip comments
+          next if line.match(/^#/) || line.match(/^iso/i)
+          
+          country_mapping = Mappings::Country.new(line)
+          result = @writer.write_country(country_mapping)
+          
+          log_message result
+        end
+      end
+    end
+    
+    def load_cities
+      %w(cities1000 cities5000 cities15000).each do |city_file|
+      #%w(cities15000).each do |city_file|
+        load_cities_file(city_file)
+      end
+    end
+    
+    def load_cities_file(city_file)
+      log_message "Loading city file #{city_file}"
+      cities = []
+      File.open(File.join(Rails.root, 'tmp', "#{city_file}.txt"), 'r') do |f|
+        f.each_line { |line| cities << Mappings::City.new(line) }
+      end
+      
+      log_message "#{cities.length} cities to process"
+      
+      cities_by_country_code = cities.group_by { |city_mapping| city_mapping[:country_iso_code_two_letters] }
+      
+      cities_by_country_code.keys.each do |country_code|
+        cities = cities_by_country_code[country_code]
+        
+        result = @writer.write_cities(country_code, cities)
+        
+        log_message result
+      end
+    end
+    
+    def log_message(message)
+      @logger << message
+      @logger << "\n"
+    end
+  end
+end

data/lib/geonames_rails/mappings/base.rb

@@ -0,0 +1,16 @@
+module GeonamesRails
+  module Mappings
+    class Base < Hash
+      def initialize(line = nil)
+        if line
+          fields_from_line = line.split("\t")
+          mappings.each do |k,v|
+            self[k] = fields_from_line[v]
+          end
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/geonames_rails/mappings/city.rb

@@ -0,0 +1,49 @@
+module GeonamesRails
+  module Mappings
+    class City < Base
+    protected
+      def mappings
+        {
+          # [0] geonameid : integer id of record in geonames database
+          :geonames_id => 0,
+          # [1] name : name of geographical point (utf8) varchar(200)
+          :name => 1,
+          # [2] asciiname : name of geographical point in plain ascii characters, varchar(200)
+          :ascii_name => 2,
+          # [3] alternatenames : alternatenames, comma separated varchar(4000)
+          :alternate_name => 3,
+          # [4] latitude : latitude in decimal degrees (wgs84)
+          :latitude => 4,
+          # [5] longitude : longitude in decimal degrees (wgs84)
+          :longitude => 5,
+          # [6] feature class : see http://www.geonames.org/export/codes.html, char(1)
+          :feature_class => 6,
+          # [7] feature code : see http://www.geonames.org/export/codes.html, varchar(10)
+          :feature_code => 7,
+          # [8] country code : ISO-3166 2-letter country code, 2 characters,
+          :country_iso_code_two_letters => 8,
+          # [9] cc2 : alternate country codes, comma separated, ISO-3166 2-letter country code, 60 characters
+          :alternate_country_codes => 9,
+          # [10] admin1 code : fipscode (subject to change to iso code), isocode for the us and ch, see file admin1Codes.txt for display names of this code; varchar(20)
+          :admin_1_code => 10,
+          # [11] admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
+          :admin_2_code => 11,
+          # [12] admin3 code : code for third level administrative division, varchar(20)
+          :admin_3_code => 12,
+          # [13] admin4 code : code for fourth level administrative division, varchar(20)
+          :admin_4_code => 13,
+          # [14] population : integer
+          :population => 14,
+          # [15] elevation : in meters, integer
+          :elevation => 15,
+          # [16] gtopo30 : average elevation of 30'x30' (ca 900mx900m) area in meters, integer,
+          :average_elevation => 16,
+          # [17] timezone : the timezone id (see file timeZone.txt)
+          :geonames_timezone_id => 17,
+          # [18] modification date : date of last modification in yyyy-MM-dd format
+          :geonames_modification_date => 18
+        }
+      end
+    end
+  end
+end

data/lib/geonames_rails/mappings/country.rb

@@ -0,0 +1,49 @@
+module GeonamesRails
+  module Mappings
+    class Country < Base
+    protected
+      def mappings
+        {
+          # [0] iso alpha2
+          :iso_code_two_letter => 0,
+          # [1] iso alpha3
+          :iso_code_three_letter => 1,
+          # [2] iso numeric
+          :iso_number => 2,
+          # [3] fips code
+          :fips_code => 3,
+          # [4] name
+          :name => 4,
+          # [5] capital
+          :capital => 5,
+          # [6] areaInSqKm,
+          :area_in_square_km => 6,
+          # [7] population
+          :population => 7,
+          # [8] continent
+          :continent => 8,
+          # [9] top level domain
+          :top_level_domain => 9,
+          # [10] Currency code
+          :currency_code => 10,
+          # [11] Currency name
+          :currency_name => 11,
+          # [12] Phone
+          :phone => 12,
+          # [13] Postal Code Format
+          :postal_code_format => 13,
+          # [14] Postal Code Regex
+          :postal_code_regex => 14,
+          # [15] Languages
+          :languages => 15,
+          # [16] Geoname id
+          :geonames_id => 16,
+          # [17] Neighbours
+          :neighbours => 17,
+          # [18] Equivalent Fips Code
+          :equivalent_fips_code => 18
+        }
+      end
+    end
+  end
+end

data/lib/geonames_rails/puller.rb

@@ -0,0 +1,36 @@
+module GeonamesRails
+  class Puller
+    def pull
+      @temp_geonames_files = []
+      target_dir = File.join(Rails.root, 'tmp')
+      
+      file_names = %w(cities1000.zip cities5000.zip cities15000.zip admin1CodesASCII.txt countryInfo.txt)
+      file_names.each do |file_name|
+        url = "http://download.geonames.org/export/dump/#{file_name}"
+      
+        remote_file = open(url)
+
+        target_file_name = File.join(target_dir, file_name)
+        File.open target_file_name, 'w' do |f|
+          f.write(remote_file.read.force_encoding("UTF-8"))
+        end
+        remote_file.close
+        
+        @temp_geonames_files << target_file_name
+        
+        file_base_name, file_extension = file_name.split('.')
+        if file_extension == 'zip'
+          `unzip #{target_file_name} -d #{target_dir}`
+          @temp_geonames_files << File.join(target_dir, "#{file_base_name}.txt")
+        end
+        
+      end
+    end
+    
+    def cleanup
+      @temp_geonames_files.each do |f|
+        FileUtils.rm f
+      end
+    end
+  end
+end

data/lib/geonames_rails/writers/active_record.rb

@@ -0,0 +1,48 @@
+module GeonamesRails
+  module Writers
+    class ActiveRecord
+      
+      def write_country(country_mapping)  
+        iso_code = country_mapping[:iso_code_two_letter]
+        c = Country.find_or_initialize_by_iso_code_two_letter(iso_code)
+      
+        created_or_updating = c.new_record? ? 'Creating' : 'Updating'
+        
+        c.attributes = country_mapping.slice(:iso_code_two_letter,
+                                             :iso_code_three_letter,
+                                             :iso_number,
+                                             :name,
+                                             :capital,
+                                             :continent,
+                                             :geonames_id)
+        c.save!
+        
+        "#{created_or_updating} db record for #{iso_code}"
+      end
+      
+
+      
+      def write_cities(country_code, city_mappings)    
+        country = Country.find_by_iso_code_two_letter(country_code)
+        
+        #puts country
+        
+        city_mappings.each do |city_mapping|
+          city = City.find_or_initialize_by_geonames_id(city_mapping[:geonames_id])
+          city.country_id = country.id
+        
+          city.attributes = city_mapping.slice(:name,
+                                               :latitude,
+                                               :longitude,
+                                               :country_iso_code_two_letters,
+                                               :geonames_timezone_id)
+        
+          city.save!
+        end
+        
+        "Processed #{country.name}(#{country_code}) with #{city_mappings.length} cities"
+      end
+
+    end
+  end
+end

data/lib/geonames_rails/writers/dry_run.rb

@@ -0,0 +1,20 @@
+module GeonamesRails
+  module Writers
+    class DryRun
+      def write_country(country_mapping)
+        raise "must have a of country mapping" unless country_mapping
+        
+        "Dry run of country #{country_mapping[:name]} should have been OK"
+      end
+      
+      def write_cities(country_code, city_mappings)
+        raise "can't create cities without a country" unless country_code
+        raise "must have a set of city mappings" unless city_mappings
+        
+        raise "i'm sure there should be at least 1 city in this country" if city_mappings.empty?
+        
+        "Dry run of country #{country_code} would have written out #{city_mappings.length} cities"
+      end
+    end
+  end
+end

data/lib/tasks/geonames_rails.rake

@@ -0,0 +1,22 @@
+require 'geonames_rails'
+
+namespace :geonames_rails do
+  desc 'pull down the geonames data from the server'
+  task :pull_data => :environment do
+    GeonamesRails::Puller.new.pull
+  end
+  
+  desc 'pull geonames data, load into db, then clean up after itself'
+  task :pull_and_load_data => :environment do
+    puller = GeonamesRails::Puller.new
+    writer = ENV['DRY_RUN'] ? GeonamesRails::Writers::DryRun.new : GeonamesRails::Writers::ActiveRecord.new
+    GeonamesRails::Loader.new(puller, writer).load_data
+  end
+  
+  desc 'load the data from files you already have laying about'
+  task :load_data => :environment do
+    writer = ENV['DRY_RUN'] ? GeonamesRails::Writers::DryRun.new : GeonamesRails::Writers::ActiveRecord.new
+    GeonamesRails::Loader.new(nil, writer).load_data
+  end
+  
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification 
+name: geonames_rails
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Marius Andra
+- Garrett Davis
+- John Barton
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-02-18 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Fetch data from geonames.org and make the required models
+email: marius.andra@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/geonames_rails/writers/active_record.rb
+- lib/geonames_rails/writers/dry_run.rb
+- lib/geonames_rails/mappings/city.rb
+- lib/geonames_rails/mappings/base.rb
+- lib/geonames_rails/mappings/country.rb
+- lib/geonames_rails/puller.rb
+- lib/geonames_rails/loader.rb
+- lib/geonames_rails.rb
+- lib/generators/geonames_rails/migration_generator.rb
+- lib/generators/geonames_rails/migration_templates/geonames_tables.rb
+- lib/generators/geonames_rails/models_templates/models/city.rb
+- lib/generators/geonames_rails/models_templates/models/country.rb
+- lib/generators/geonames_rails/models_generator.rb
+- lib/tasks/geonames_rails.rake
+- README.md
+has_rdoc: true
+homepage: http://rubygems.org/gems/geonames-rails
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Geonames.org support for Rails applications
+test_files: []
+