osm-import 0.0.0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Alexey Noskov
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/osm-import.rb

@@ -0,0 +1 @@
+require 'osm_import'

data/lib/osm_import/mapper/address.rb

@@ -0,0 +1,18 @@
+require 'osm_import/mapper'
+
+class OsmImport::Mapper::Address < OsmImport::Mapper::Base
+
+  def assigns
+    { "street" => "src.tags->'addr:street'", "housenumber" => "src.tags->'addr:housenumber'", "city" => "src.tags->'addr:city'" , "postcode" => "src.tags->'addr:postcode'" }
+  end
+
+  def fields
+    { "street" => "VARCHAR(255)", "housenumber" => 'VARCHAR(255)', "city" => "VARCHAR(255)", "postcode" => "VARCHAR(100)" }
+  end
+
+  def after_import(tt)
+    tt.conn.exec "UPDATE #{tt.name} SET city = NULL WHERE city = 'undefined'"
+    tt.conn.exec "UPDATE #{tt.name} SET city = c.name FROM raw_osm_polygon c WHERE geometry && way AND ST_Contains(way, Geometry(geometry)) AND c.place IN ('city','town','village') AND c.name IS NOT NULL AND c.name <> '' AND city IS NULL"
+  end
+
+end

data/lib/osm_import/mapper/center.rb

@@ -0,0 +1,25 @@
+require 'osm_import/mapper'
+
+class OsmImport::Mapper::Center < OsmImport::Mapper::Base
+
+  def assigns
+    {}
+  end
+
+  def fields
+    {}
+  end
+
+  def indexes
+    { name => "GIST(#{name})" }
+  end
+
+  def after_create(tt)
+    tt.add_geometry_column name, :point
+  end
+
+  def after_import(tt)
+    tt.conn.exec "UPDATE #{tt.name} SET #{name} = ST_PointOnSurface(Geometry(geometry)) WHERE #{name} IS NULL"
+  end
+
+end

data/lib/osm_import/mapper/type.rb

@@ -0,0 +1,91 @@
+require 'osm_import/mapper'
+
+class OsmImport::Mapper::Type < OsmImport::Mapper::Base
+
+  attr_reader :mappings
+
+  def initialize(*args)
+    super
+
+    @mappings = []
+    @multi = false
+  end
+
+  def assigns
+    if @multi
+      { :type => expression, :type_array => expression_multi }
+    else
+      { :type => expression }
+    end
+  end
+
+  def fields
+    if @multi
+      { :type => 'VARCHAR(100) NOT NULL', :type_array => 'VARCHAR(100)[]' }
+    else
+      { :type => 'VARCHAR(100) NOT NULL' }
+    end
+  end
+
+   def indexes
+    if @multi
+      { :type => "BTREE(type)", :type_array => "GIN(type_array)" }
+    else
+      { :type => "BTREE(type)" }
+    end
+  end
+
+  def conditions
+    [ @mappings.map{|m| map_cond(m)}.join(' OR ') ]
+  end
+
+  def expression
+    "CASE #{@mappings.map{|m| "WHEN #{map_cond(m)} THEN trim(regexp_replace(src.tags->'#{m[0]}', ',.*', ''))"}.join(' ')} END"
+  end
+
+  def expression_multi
+    "(#{@mappings.map{|m| "string_to_array(coalesce(CASE WHEN #{map_cond(m)} THEN replace(src.tags->'#{m[0]}',' ','') END,''),',')"}.join(' || ')})"
+  end
+
+  def map_cond(m)
+    m[1] ? "(src.tags->'#{m[0]}') IN ('#{m[1].join("','")}')" : "(src.tags->'#{m[0]}') IS NOT NULL"
+  end
+
+  def add_args(*args)
+    args.each do |arg|
+      if arg.is_a? Hash
+        arg.each(&method(:add_arg))
+      else
+        add_arg arg, nil
+      end
+    end
+  end
+
+  def add_arg(key, value)
+    key = to_key(key)
+
+    if key == 'multi'
+      @multi = value
+    else
+      @mappings << [key, to_keylist(value)]
+    end
+  end
+
+  private
+
+  def to_key(key)
+    raise StandardError.new("Invalid key #{key.inspect}") unless key.is_a? String or key.is_a? Symbol
+    key.to_s
+  end
+
+  def to_keylist(list)
+    if list.nil?
+      list
+    elsif list.is_a? Array
+      list.map(&method(:to_key))
+    else
+      [to_key(list)]
+    end
+  end
+
+end

data/lib/osm_import/mapper.rb

@@ -0,0 +1,48 @@
+require 'osm_import'
+
+module OsmImport::Mapper
+
+  extend OsmImport::Autoloading
+
+  def self.new(name, type)
+    self[(type || name).to_s].new(name)
+  end
+
+  class Base < Struct.new(:name)
+
+    def fields
+      { name => "VARCHAR(255)" }
+    end
+
+    def assigns
+      { name => "src.tags->'#{name}'" }
+    end
+
+    def indexes
+      {}
+    end
+
+    def after_create(*args)
+    end
+
+    def after_import(*args)
+    end
+
+  end
+
+  class String < Base
+  end
+
+  class Id < Base
+
+    def fields
+      { name => "INT8" }
+    end
+
+    def assigns
+      { name => "(src.tags->'#{name}')::INT8" }
+    end
+
+  end
+
+end

data/lib/osm_import/schema.rb

@@ -0,0 +1,51 @@
+require 'osm_import'
+
+class OsmImport::Schema
+
+  attr_reader :tables
+
+  def self.load(file)
+    new { eval File.read(file) }
+  end
+
+  def initialize(&block)
+    @tables = []
+
+    dsl.instance_eval(&block) if block
+  end
+
+  def dsl
+    @dsl ||= Dsl.new self
+  end
+
+  class Dsl < Struct.new(:schema)
+
+    def multipolygons(name, &block)
+      table :multipolygons, name, &block
+    end
+
+    def multilines(name, &block)
+      table :multilines, name, &block
+    end
+
+    def polygons(name, &block)
+      table :polygons, name, &block
+    end
+
+    def lines(name, &block)
+      table :lines, name, &block
+    end
+
+    def points(name, &block)
+      table :points, name, &block
+    end
+
+    def table(type, name, &block)
+      require 'osm_import/table'
+
+      schema.tables << OsmImport::Table.new(type, name, &block)
+    end
+
+  end
+
+end

data/lib/osm_import/table.rb

@@ -0,0 +1,56 @@
+require 'osm_import'
+
+class OsmImport::Table
+
+  attr_reader :type, :name, :type_mapper, :mappers, :after_create_callbacks, :after_import_callbacks
+
+  def initialize(type, name, &block)
+    require 'osm_import/mapper'
+
+    @type = type.to_s.gsub(/s$/,'')
+    @name = name
+    @type_mapper = OsmImport::Mapper.new(:type, :type)
+    @mappers = {}
+    @after_create_callbacks = []
+    @after_import_callbacks = []
+
+    dsl.instance_eval(&block) if block
+  end
+
+  def dsl
+    @dsl ||= Dsl.new self
+  end
+
+  class Dsl < Struct.new(:table)
+
+    def with(*args)
+      args.each do |arg|
+        if arg.is_a? Hash
+          arg.each(&method(:add_mapper))
+        else
+          add_mapper arg, nil
+        end
+      end
+    end
+
+    def map(*args)
+      table.type_mapper.add_args(*args)
+    end
+
+    def after_create(&block)
+      table.after_create_callbacks << block
+    end
+
+    def after_import(&block)
+      table.after_import_callbacks << block
+    end
+
+    private
+
+    def add_mapper(key, type)
+      table.mappers[key.to_s] = OsmImport::Mapper.new key, type
+    end
+
+  end
+
+end

data/lib/osm_import/target/pg.rb

@@ -0,0 +1,191 @@
+require 'osm_import'
+require 'pg'
+
+class OsmImport::Target::Pg < Struct.new(:options)
+
+  def projection
+    options[:projection]
+  end
+
+  def prefix
+    'osm_'
+  end
+
+  def new_prefix
+    'new_osm_'
+  end
+
+  def import(schema)
+    conn = Connection.new options
+
+    conn.exec "BEGIN TRANSACTION"
+
+    puts "Dropping old tables:"
+
+    schema.tables.each do |table|
+      conn.exec "DROP TABLE IF EXISTS #{new_prefix}#{table.name}"
+    end
+
+    puts "Preparing target tables"
+
+    tts = schema.tables.map{|table| TargetTable.new self, conn, table }
+
+    puts "Creating tables and importing data:"
+
+    tts.each do |tt|
+      tt.create! # Create table
+      tt.import! # Import data to it
+    end
+
+    puts "Deploy tables"
+    tts.each do |tt|
+      tt.deploy!
+    end
+
+    puts "Restoring geometry columns"
+    conn.exec "TRUNCATE geometry_columns"
+    conn.exec "SELECT probe_geometry_columns()"
+
+    puts "Commiting"
+    conn.exec "COMMIT"
+    conn.close
+  end
+
+  GEOTYPES = { 'point' => 'POINT', 'line' => 'LINESTRING', 'polygon' => 'POLYGON', 'multiline' => 'MULTILINESTRING', 'multipolygon' => 'MULTIPOLYGON' }
+
+  def geometry_type(type)
+    GEOTYPES[type.to_s] or raise StandardError.new("Unknown geometry type #{type.inspect}")
+  end
+
+  class Connection < Struct.new(:options)
+
+    def initialize(*args)
+      super
+      @conn = PG.connect options[:pg]
+    end
+
+    def exec(q)
+      puts "  #{q}"
+      @conn.exec q
+    end
+
+    def close
+      @conn.close
+    end
+
+  end
+
+  class TargetTable
+
+    attr_reader :target, :conn, :table, :type_mapping, :type_array_mapping, :name_mapping, :fields, :conditions, :assigns
+
+    def initialize(target, conn, table)
+      @target = target
+      @conn = conn
+      @table = table
+
+      @type_mapping = table.type_mapper.expression
+      @type_array_mapping = table.type_mapper.expression_multi
+      @name_mapping = "NULLIF(COALESCE(src.tags->'name:ru', src.tags->'name', src.tags->'int_name'), '')"
+
+      @fields = table.type_mapper.fields.merge :id => 'INT8 PRIMARY KEY',  :name => 'VARCHAR(255)', :tags => 'HSTORE', :osm_type => 'VARCHAR(10)'
+      @assigns = table.type_mapper.assigns.merge :id => osm_id_expr, :name => name_mapping, :tags => 'src.tags', :osm_type => osm_type_expr
+      @conditions = table.type_mapper.conditions
+
+      table.mappers.each do |key, mapper|
+        @fields.merge! mapper.fields
+        @assigns.merge! mapper.assigns
+      end
+    end
+
+    def osm_id_expr
+      "ABS(src.osm_id)"
+    end
+
+    def osm_type_expr(t = table.type)
+      "CASE WHEN src.osm_id < 0 THEN 'relation' ELSE '#{if t.to_s == 'point' then 'node' else 'way' end}' END"
+    end
+
+    def add_geometry_column(column, type)
+      if target.projection
+        conn.exec "SELECT AddGeometryColumn('#{name}', '#{column}', #{target.projection}, '#{target.geometry_type type}', 2)"
+      else
+        conn.exec "ALTER TABLE #{name} ADD COLUMN #{column} GEOGRAPHY(#{target.geometry_type type}, 4326)"
+      end
+    end
+
+    def name
+      "#{target.new_prefix}#{table.name}"
+    end
+
+    def create!
+      conn.exec "CREATE TABLE #{name}(#{fields.map{|k,v| "#{k} #{v}"}.join(', ')})"
+      add_geometry_column :geometry, table.type
+
+      table.type_mapper.after_create self
+
+      table.mappers.each do |key, mapper|
+        mapper.after_create self
+      end
+
+      table.after_create_callbacks.each do |cb|
+        instance_eval(&cb)
+      end
+    end
+
+    def indexes
+      @indexes ||= begin
+        res = table.type_mapper.indexes
+        res[:geometry] = "GIST(geometry)"
+
+        table.mappers.each do |key,mapper|
+          res.merge mapper.indexes
+        end
+
+        res
+      end
+    end
+
+    def import!
+      unless table.type_mapper.mappings.empty?
+        if table.type =~ /^multi/
+          geometry_expr = 'ST_Multi(way)'
+        else
+          geometry_expr = 'way'
+        end
+
+        field_keys = fields.keys.to_a
+        conn.exec "INSERT INTO #{name}(geometry, #{field_keys.join(', ')}) SELECT #{geometry_expr} AS geometry, #{field_keys.map{|k| "#{assigns[k]} AS #{k}"}.join(',')} FROM raw_osm_#{table.type.gsub('multi','')} src WHERE #{conditions.join(' AND ')}"
+      end
+
+      # Creating indexes
+
+      indexes.each do |key, desc|
+        conn.exec "CREATE INDEX #{name}_#{key}_index ON #{name} USING #{desc}"
+      end
+
+      table.type_mapper.after_import self
+
+      table.mappers.each do |key, mapper|
+        mapper.after_import self
+      end
+
+      table.after_import_callbacks.each do |cb|
+        instance_eval(&cb)
+      end
+    end
+
+    def deploy!
+      conn.exec "DROP TABLE IF EXISTS #{target.prefix}#{table.name}"
+      conn.exec "ALTER TABLE #{name} RENAME TO #{target.prefix}#{table.name}"
+      #conn.exec "ALTER INDEX #{name}_pkey RENAME TO #{target.prefix}#{table.name}_pkey"
+
+      indexes.each do |key,_|
+        conn.exec "ALTER INDEX #{name}_#{key}_index RENAME TO #{target.prefix}#{table.name}_#{key}_index"
+      end
+
+    end
+
+  end
+
+end

data/lib/osm_import/target.rb

@@ -0,0 +1,11 @@
+require 'osm_import'
+
+module OsmImport::Target
+
+  extend OsmImport::Autoloading
+
+  def self.new(name, options)
+    self[name].new(options)
+  end
+
+end

data/lib/osm_import/version.rb

@@ -0,0 +1,9 @@
+module OsmImport
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 0
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/osm_import.rb

@@ -0,0 +1,37 @@
+module OsmImport
+
+  module Autoloading
+
+    def const_missing(c)
+      require "#{underscore name}/#{underscore c}"
+      const_get c
+    end
+
+    def [](c)
+      const_get(camelize c)
+    end
+
+    private
+
+    # Based on https://github.com/intridea/omniauth/blob/v1.0.2/lib/omniauth.rb#L129-139
+    def camelize(str)
+      str.to_s.gsub(/\/(.?)/){ "::" + $1.upcase }.gsub(/(^|_)(.)/){ $2.upcase }
+    end
+
+    def underscore(str)
+      str.to_s.gsub(/::(.?)/){ "/" + $1.downcase }.gsub(/(.)([A-Z])/){ "#{$1}_#{$2.downcase}" }.downcase
+    end
+
+  end
+
+  def self.import(mapping_file, options = {})
+    require 'osm_import/schema'
+    require 'osm_import/target'
+
+    schema = Schema.load mapping_file
+    target = Target.new :pg, options
+
+    target.import(schema)
+  end
+
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification 
+name: osm-import
+version: !ruby/object:Gem::Version 
+  hash: 31
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 0
+  version: 0.0.0
+platform: ruby
+authors: 
+- Alexey Noskov
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-03-29 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: pg
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 51
+        segments: 
+        - 0
+        - 11
+        - 0
+        version: 0.11.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 2
+        - 0
+        version: "2.0"
+  type: :development
+  version_requirements: *id002
+description: Gem for importing OpenStreetMap data to PostgreSQL database
+email: 
+- alexey.noskov@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- MIT-LICENSE
+files: 
+- lib/osm-import.rb
+- lib/osm_import/mapper.rb
+- lib/osm_import/table.rb
+- lib/osm_import/mapper/type.rb
+- lib/osm_import/mapper/center.rb
+- lib/osm_import/mapper/address.rb
+- lib/osm_import/version.rb
+- lib/osm_import/target.rb
+- lib/osm_import/target/pg.rb
+- lib/osm_import/schema.rb
+- lib/osm_import.rb
+- MIT-LICENSE
+- README.rdoc
+homepage: http://github.com/alno/osm-import
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- OSM Import
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Gem for importing OpenStreetMap data to PostgreSQL database
+test_files: []
+