terragona 0.1.0

data/.gitignore

@@ -0,0 +1 @@
+.idea

data/Gemfile

@@ -0,0 +1,9 @@
+source "https://rubygems.org/"
+
+gem 'pg'
+gem 'sequel', "~> 3.48.0"
+gem 'httpi', "~> 1.1.0"
+gem 'pry-debugger'
+gem 'diskcached'
+gem 'similar_text', '~> 0.0.4'
+gem 'geokit'

data/Gemfile.lock

@@ -0,0 +1,42 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.0)
+    columnize (0.9.0)
+    debugger (1.6.8)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.3.5)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.3.7)
+    diskcached (1.1.0)
+    geokit (1.9.0)
+      multi_json (>= 1.3.2)
+    httpi (1.1.1)
+      rack
+    method_source (0.8.2)
+    multi_json (1.10.1)
+    pg (0.17.1)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-debugger (0.2.3)
+      debugger (~> 1.3)
+      pry (>= 0.9.10, < 0.11.0)
+    rack (1.6.0)
+    sequel (3.48.0)
+    similar_text (0.0.4)
+    slop (3.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  diskcached
+  geokit
+  httpi (~> 1.1.0)
+  pg
+  pry-debugger
+  sequel (~> 3.48.0)
+  similar_text (~> 0.0.4)

data/README.md

@@ -0,0 +1,155 @@
+terragona
+=========
+
+Create polygons for [GeoNames](www.geonames.org) places.
+This means: Create concave polygons using geonames places and store them in a postgres/postgis database.
+See [ST_Concave_Hull](http://postgis.net/docs/ST_ConcaveHull.html).
+
+![alt tag](https://cloud.githubusercontent.com/assets/6061036/5606205/2216ff3c-9402-11e4-9123-fff91369208e.png)
+
+
+So, am I saying you can get the geometries of all places magically? Sort of... 
+As you can see, the results are not *very* accurate. But they are interesting.
+
+  
+Install
+-------
+
+`gem install terragona`
+
+or add `gem 'terragona'` to your Gemfile
+
+Usage
+-----
+
+First, create a db in postgres and install the postgis extension.
+
+You can use the GeoNames API, with the Terragona::API class, or download
+a dump, (Terragona::Dump class) and specify the dump file path in opts. The API is faster 
+but less accurate (max 1000 points per request). The dump is more accurate but much slower (please, 
+use country dumps, not the world dump: it's to big -~9 million points- and could take lots of time.). For example:
+with the API, the Italy polygon is drawn using 1000 points. With the dump, the input is ~95.000 points. 
+You can use the `max_points` option to limit this number.
+
+The slow part of the process is when points are filtered: the ones that are isolated are discarded. 
+This has to be refactored.
+
+Besides the source of the points, options `target_percent` and `max_distance_ratio` control the shape of
+the polygons. See the options. 
+
+
+With API
+```ruby
+require 'terragona'
+
+opts = {...}
+countries=[{:name=>'Argentina',:fcode=>'PCLI',:country=>'AR'},
+           {:name=>'Uruguay',:fcode=>'PCLI',:country=>'UY'}]
+
+terragona = Terragona::API.new(opts)
+terragona.create_polygons_family(countries, 'countries', 'countries_subdivisions')
+
+```
+
+With Dump, and using returned children places
+
+```ruby
+require 'terragona'
+
+opts={
+	:default_country=>'IT',
+	:target_percent=> 0.85,
+	:max_distance_ratio=>1.6,
+	:db_username=>'mydbuser',
+	:db_password=>'mydbpsswd',
+	:db_name=>'mydb',
+	:dump=>'/path/to/dump/IT.txt'}
+
+italy=[{:name=>'Italy',:fcode=>'PCLI'}]
+
+terragona=Terragona::Dump.new(opts)
+result=terragona.create_polygons_family(italy,'italy','italian_regions')
+
+italian_rest=[]
+result.each {|r|
+	italian_rest.concat(r[:children_places])
+}
+terragona.create_polygons_family(italian_rest,'province','comuni')
+```
+
+Methods
+-------
+
+```
+create_polygons(<array of places>, options)
+  
+create_polygons_family(<array of places>, <first order geometries table name>, <second order geometries table name>, options)
+```
+
+Each place in the array of places is a hash with this keys:
+
+```
+:name                
+:fcode                   GeoNames Feature Code 
+:id                      (optional)               
+:children_fcode          (optional)
+:country                 (optional)
+:field_to_compare        (optional) (:adminCode1, :adminCode2 or :adminCode3)
+:field_to_compare_value  (optional)
+```
+
+The methods create the tables, fill them with polygons and return the following hash:
+
+```
+{:children_places=>array of hashes, :points=>array of points([x,y]), :place_name=>string, :place_id=>string}
+```
+
+Options
+------
+
+```
+dump                    Only for Dump. Path to dump file.
+max_points              Only for Dump. Max number of points to consider from
+                        dump file.            
+default_country         Default country.
+geonames_username       Only for API. Geonames API username.
+cache_expiration_time   Default: 7200.
+projection              Default: EPSG 4326 (WGS84).
+target_percent          Require to draw the concave polygons. 
+                        Closer to 1: convex. Closer to 0, concave. Default: 0.8. 
+allow_holes             Can the polygons have holes? Default: false. 
+max_distance_ratio      Points distant more than this ratio times from the average 
+                        distance between points are not considered. Default: 1.6.
+minimal_polygon_points  Minimal number of points to build a polygon.
+dont_create_polygons    (boolean) Default: false.
+table                   Table where polygons are saved. This option is overriden 
+                        by args of create_polygons_family method.
+```
+
+Postgres options
+```
+db_name                The db *must* have the Postgis extension installed.
+db_username
+db_password
+db_host                Default: localhost.
+db_port                Default: 5432.
+db_max_connections     Default: 10.
+```
+
+TODO
+----
+- [x] Check of geometry type before saving
+- [x] Use dumps as input (not only API)
+- [ ] Generate multipolygon in ConcaveHull. Use some clustering algorithm.
+- [ ] Improve/replace distant points algorithm. Use some clustering algorithm.
+  
+Useful data
+-----------
+* [GeoNames Country Codes](http://www.geonames.org/countries/)
+* [GeoNames Feature Codes](http://www.geonames.org/export/codes.html)
+* [GeoNames Dumps download page](http://download.geonames.org/export/dump/)
+
+License
+-------
+
+MIT.

data/lib/terragona/base.rb

@@ -0,0 +1,58 @@
+require_relative './geonames'
+require_relative './concave_hull'
+require_relative './version'
+
+module Terragona
+  class Base
+    def initialize(options={})
+      @options=options
+      @minimal_polygon_points = options[:minimal_polygon_points] || 5
+    end
+
+    def create_polygons(names,options={})
+      opts=@options.merge(options)
+
+      concave_hull = ConcaveHull.new(opts) if not opts[:dont_create_polygons]
+
+      names.map{|n|
+        name = @geonames.search(n)
+
+        if name[:points].count < @minimal_polygon_points
+          puts "No points for #{n[:name]}"
+          next
+        end
+
+        unless opts[:dont_create_polygons]
+          if concave_hull.perform(name[:points],name[:place_name],name[:place_id])
+            puts "Polygon created for #{n[:name]}"
+          end
+        end
+        name
+      }
+    end
+
+    def create_polygons_family(names,parents_table,children_table,opts={})
+      created_names = create_polygons(names,opts.merge({:table => parents_table}))
+      children = []
+      created_names.each {|c|
+        children.concat(c[:children_places])
+      }
+      create_polygons(children,opts.merge({:table => children_table}))
+    end
+  end
+  
+  class API < Base
+    def initialize (options={})
+      super
+      @geonames = GeoNames::API.new(options) 
+    end
+  end
+  
+  class Dump < Base
+    def initialize (options={})
+      super
+      @geonames = GeoNames::Dump.new(options) 
+    end
+  end
+  
+end

data/lib/terragona/concave_hull.rb

@@ -0,0 +1,98 @@
+require 'geokit'
+require 'sequel'
+
+module Terragona
+  class ConcaveHull
+    def initialize(options = {})
+      @projection = options[:projection] || 4326
+      @table  = options[:table] || 'concave_hull'
+      @target_percent = options[:target_percent] || 0.8
+      @allow_holes = options[:allow_holes]
+      @allow_holes = false if @allow_holes.nil?
+      @max_distance_ratio = options[:max_distance_ratio] || 1.6
+
+      db_options={
+          :database=> options[:db_name],
+          :user=> options[:db_username],
+          :password=> options[:db_password],
+          :host=> options[:db_host] || 'localhost',
+          :port=> options[:db_port] || 5432,
+          :max_connections=> options[:db_max_connections] || 10
+      }
+
+      @db = Sequel.postgres(db_options)
+
+      create_table
+    end
+
+    def perform(points,tags,id)
+      filtered_points=filter_points_by_distance(points)
+      points_stringified=stringify_points(filtered_points)
+
+      query = %Q{
+      ST_ConcaveHull(
+        ST_GeomFromText(
+          'MULTIPOINT(
+            #{points_stringified}
+          )',
+          #{@projection}
+        ),
+        #{@target_percent},
+        #{@allow_holes})
+      }
+
+      create_concave_hull(query,tags,filtered_points.count,id)
+    end
+    
+    private
+    def create_table
+      @db << "DROP TABLE IF EXISTS #{@table};"
+      @db << "CREATE TABLE #{@table} (id BIGINT PRIMARY KEY, name TEXT, count INT);"
+      @db << "SELECT AddGeometryColumn('#{@table}', 'geometry',#{@projection}, 'POLYGON', 2);"
+    end
+
+    def create_concave_hull(query, tags, count, id)
+      begin
+        return unless @db["SELECT ST_GeometryType(#{query})"].first[:st_geometrytype] == 'ST_Polygon'
+        @db << "INSERT INTO #{@table} (id, name, count, geometry) VALUES (#{id},'#{clean_str tags}',#{count},#{query})"
+      rescue
+        puts "Error with #{tags}, Id: #{id}, #{count} points."
+      end
+    end
+    
+    def drop_points_table(table)
+      @db << "DROP TABLE IF EXISTS #{table};"
+    end
+
+    def filter_points_by_distance(points)     
+      random_points = points.count > 200 ? (0..200).map {|e|
+        points[rand(points.count)]
+      }.uniq : points
+            
+      distances = points.map {|p0|
+        sum = random_points.map {|pf|
+          Geokit::LatLng.new(p0[:y],p0[:x]).distance_to(Geokit::LatLng.new(pf[:y],pf[:x]))
+        }
+        {:y=>p0[:y],:x=>p0[:x],:average=>average(sum)}
+      }
+      average_distance=average(distances.map{|d| d[:average]})
+      distances.map {|d|
+        d if d[:average] <= average_distance * @max_distance_ratio
+      }.compact
+    end
+
+    def stringify_points(points)
+      points.map{|p|
+        "#{p[:x]} #{p[:y]}"
+      }.join(',')
+    end
+
+    def average(arr)
+      arr.inject{ |sum, el| sum + el }.to_f / arr.size
+    end
+
+    def clean_str(str)
+      str.gsub("'",' ')
+    end
+  end
+end

data/lib/terragona/geonames.rb

@@ -0,0 +1,186 @@
+require 'httpi'
+require 'diskcached'
+require 'similar_text'
+require 'json'
+require 'csv'
+
+module Terragona
+  module GeoNames
+    class Base
+      def initialize(args = {})
+        @default_country = args[:default_country]
+      end
+
+      def search(options)
+        country = options[:country] || @default_country
+        id = options[:id]
+        name = options[:name]
+        fcode = options[:fcode]
+
+        children_fcode = options[:children_fcode] || case fcode
+                             when 'PCLI' then 'ADM1'
+                             when 'ADM1' then 'ADM2'
+                             when 'ADM2' then 'ADM3'
+                             when 'ADM3' then 'ADM4'
+                             when 'ADM4' then 'ADM5'
+                             when 'PPLC' then 'PPLX'
+                           end
+
+        field_to_compare = options[:field_to_compare] || calculate_field_to_compare(fcode)
+        children_field_to_compare = calculate_field_to_compare(children_fcode)
+        field_to_compare_value = options[:field_to_compare_value]
+
+        if field_to_compare_value.nil?
+          fetch_geonames(name,country,nil,nil).each{|g|
+            if g[:fcode] == fcode
+              name = g[:name]
+              id = g[:geonameId]
+              field_to_compare_value = g[field_to_compare]
+              break
+            end
+          }
+        end
+
+        points = []
+        children_places = []
+                        
+        fetch_geonames(nil,country,field_to_compare,field_to_compare_value).each{|g|
+            points.push({:x=>g[:lng],:y=>g[:lat]})
+            if g[:fcode] == children_fcode and children_field_to_compare
+              child={:name=>g[:name],
+                     :id=>g[:geonameId],
+                     :fcode=>g[:fcode],
+                     :country=>g[:countryCode],
+                     :field_to_compare_value=>g[children_field_to_compare]}
+              children_places.push(child)
+            end
+        }
+
+        {:children_places=>children_places,:points=>points,:place_name=>name,:place_id=>id}
+      end
+
+      private
+
+      def calculate_field_to_compare(fcode)
+        case fcode
+          when 'PCLI' then :countryCode
+          when 'ADM1' then :adminCode1
+          when 'ADM2' then :adminCode2
+          when 'ADM3' then :adminCode3
+          else nil
+        end
+      end
+    end
+
+    class API < Base
+      URL = 'http://api.geonames.org/searchJSON'
+      def initialize(args = {})
+        super
+        @username = args[:geonames_username]
+        cache_expiration_time = args[:cache_expiration_time] || 7200
+        @cache=Diskcached.new('/tmp/cache',cache_expiration_time,true)
+      end
+
+      def fetch_geonames(name,country,admin_code_type,admin_code)
+        admin_code_str = admin_code ? "&#{admin_code_type}=#{admin_code}" : ''
+        name_str = name ? "q=#{name}&" : ''
+
+        @cache.cache("geonames_name=#{name}&country=#{country}#{admin_code_str}&full666") do
+          url = URI.escape(%Q{#{URL}?#{name_str}country=#{country}#{admin_code_str}&style=FULL&order_by=relevance&maxRows=1000&username=#{@username}})
+          request = HTTPI::Request.new(url)
+          data = HTTPI.get(request)
+          JSON.parse(data.body,:symbolize_names=>true)[:geonames]
+        end
+        
+      end
+    end
+
+    class Dump < Base
+      HEADERS = [:geonameId, 
+                 :name, 
+                 :asciiname, 
+                 :alternatenames, 
+                 :lat,
+                 :lng,
+                 :fclass,
+                 :fcode,
+                 :countryCode,
+                 :cc2,
+                 :adminCode1,
+                 :adminCode2,
+                 :adminCode3,
+                 :adminCode4,
+                 :population,
+                 :elevation,
+                 :dem,
+                 :timezone,
+                 :modificaion_date]
+                 
+      def initialize(args = {})
+        super
+        
+        if not args[:dump]
+          puts 'No dump file provided'
+          return
+        end
+        @file = File.open(args[:dump])
+        @admin_codes_cache = {:adminCode1=>{},
+                              :adminCode2=>{},
+                              :adminCode3=>{},
+                              :adminCode4=>{}}
+                              
+        @max_points = args[:max_points]                    
+      end
+      
+      def fetch_geonames(name, country, admin_code_type, admin_code)
+        if admin_code_type and 
+           @admin_codes_cache[admin_code_type] and 
+           @admin_codes_cache[admin_code_type][admin_code]
+        
+          @admin_codes_cache[admin_code_type][admin_code]
+           
+        else
+          dump_parser(name, country, admin_code_type, admin_code)
+        end
+      end
+      
+      private
+      def dump_parser(name, country, admin_code_type, admin_code)
+        @file.rewind
+        records = @max_points ? @file.first(@max_points) : @file
+        records.map {|l| 
+          begin
+            raw=CSV.parse_line(l,{:col_sep => "\t"})            
+          rescue
+            next
+          end
+
+          hash = {}
+          HEADERS.each_with_index {|h,index| hash[h] = raw[index]}
+          
+          cache_hash(hash)
+          
+          next unless (name and name.similar(hash[:name]) > 30) or
+                      (name and hash[:alternatenames] and hash[:alternatenames].include? name) or                       
+                      (admin_code_type and admin_code and hash[admin_code_type] == admin_code)
+          
+          next if (country and country != hash[:countryCode]) 
+          
+          hash  
+        }.compact
+      end
+      
+      def cache_hash(hash)
+        [:adminCode1,:adminCode2,:adminCode3,:adminCode4].each {|adm|
+          if hash[adm] and not @admin_codes_cache[adm][hash[adm]]
+            @admin_codes_cache[adm][hash[adm]]=[]
+          end
+          if hash[adm] and not @admin_codes_cache[adm][hash[adm]].include? hash
+            @admin_codes_cache[adm][hash[adm]].push(hash)
+          end
+        }
+      end
+    end
+
+  end
+end

data/lib/terragona/version.rb

@@ -0,0 +1,3 @@
+module Terragona
+  VERSION = "0.1.0"
+end

data/lib/terragona.rb

@@ -0,0 +1,2 @@
+require_relative './terragona/base'
+

data/terragona.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'terragona/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "terragona"
+  spec.version       = Terragona::VERSION
+  spec.authors       = ["Bruno Salerno"]
+  spec.email         = ["br.salerno@gmail.com"]
+  spec.description   = %q{Create polygons for geonames places}
+  spec.summary       = %q{Use API or dumps as input, draw polygons, and store them in a Postgres/Postgis db}
+  spec.homepage      = "https://github.com/BrunoSalerno/terragona"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "pg"
+  spec.add_runtime_dependency "sequel", "~> 3.48.0"
+  spec.add_runtime_dependency "httpi", "~> 1.1.0"
+  spec.add_runtime_dependency "diskcached"
+  spec.add_runtime_dependency "similar_text", "~> 0.0.4"
+  spec.add_runtime_dependency "geokit"
+  
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "pry-debugger"
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: terragona
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Bruno Salerno
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.48.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.48.0
+- !ruby/object:Gem::Dependency
+  name: httpi
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: diskcached
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: similar_text
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  name: geokit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Create polygons for geonames places
+email:
+- br.salerno@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- README.md
+- lib/terragona.rb
+- lib/terragona/base.rb
+- lib/terragona/concave_hull.rb
+- lib/terragona/geonames.rb
+- lib/terragona/version.rb
+- terragona.gemspec
+homepage: https://github.com/BrunoSalerno/terragona
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Use API or dumps as input, draw polygons, and store them in a Postgres/Postgis
+  db
+test_files: []