osm2mongo 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +23 -0
- data/.gemtest +0 -0
- data/History.txt +6 -0
- data/Manifest.txt +17 -0
- data/README.txt +62 -0
- data/Rakefile +32 -0
- data/bin/osm2mongo +3 -0
- data/lib/callbacks.rb +119 -0
- data/lib/common.rb +130 -0
- data/lib/db/abstract_interface.rb +24 -0
- data/lib/db/dbhandler.rb +40 -0
- data/lib/db/mongohandler.rb +104 -0
- data/lib/osm2mongo.rb +161 -0
- data/test/profile.rb +14 -0
- data/test/test_callbacks.rb +25 -0
- data/test/test_mongo.rb +33 -0
- data/test/test_osm2mongo.rb +8 -0
- data/test/test_preprocessor.rb +31 -0
- metadata +105 -0
data/.autotest
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'autotest/restart'
|
4
|
+
|
5
|
+
# Autotest.add_hook :initialize do |at|
|
6
|
+
# at.extra_files << "../some/external/dependency.rb"
|
7
|
+
#
|
8
|
+
# at.libs << ":../some/external"
|
9
|
+
#
|
10
|
+
# at.add_exception 'vendor'
|
11
|
+
#
|
12
|
+
# at.add_mapping(/dependency.rb/) do |f, _|
|
13
|
+
# at.files_matching(/test_.*rb$/)
|
14
|
+
# end
|
15
|
+
#
|
16
|
+
# %w(TestA TestB).each do |klass|
|
17
|
+
# at.extra_class_map[klass] = "test/test_misc.rb"
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
|
21
|
+
# Autotest.add_hook :run_command do |at|
|
22
|
+
# system "rake build"
|
23
|
+
# end
|
data/.gemtest
ADDED
File without changes
|
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
.autotest
|
2
|
+
History.txt
|
3
|
+
Manifest.txt
|
4
|
+
README.txt
|
5
|
+
Rakefile
|
6
|
+
bin/osm2mongo
|
7
|
+
lib/callbacks.rb
|
8
|
+
lib/common.rb
|
9
|
+
lib/db/abstract_interface.rb
|
10
|
+
lib/db/dbhandler.rb
|
11
|
+
lib/db/mongohandler.rb
|
12
|
+
lib/osm2mongo.rb
|
13
|
+
test/profile.rb
|
14
|
+
test/test_callbacks.rb
|
15
|
+
test/test_mongo.rb
|
16
|
+
test/test_osm2mongo.rb
|
17
|
+
test/test_preprocessor.rb
|
data/README.txt
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
= osm2mongo
|
2
|
+
|
3
|
+
* https://github.com/posseidon/osm2mongo
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
Import Openstreetmap .osm(xml) data format into MongoDB written in Ruby.
|
8
|
+
|
9
|
+
== FEATURES/PROBLEMS:
|
10
|
+
|
11
|
+
* TODO: Missing Shift class and modules under Osm2Mongo main module.
|
12
|
+
* TODO: Missing command line App.
|
13
|
+
* TODO: Finish making gem installable.
|
14
|
+
* TODO: Missing Rake task for installing dependencies and generate documentation.
|
15
|
+
|
16
|
+
== SYNOPSIS:
|
17
|
+
|
18
|
+
FIX (code sample of usage)
|
19
|
+
|
20
|
+
== REQUIREMENTS:
|
21
|
+
|
22
|
+
* mongo: ruby driver (http://rubygems.org/gems/mongo)
|
23
|
+
* nokogiri: xml parser (http://rubygems.org/gems/nokogiri)
|
24
|
+
* progressbar: just for fun and icandy :-) (http://rubygems.org/gems/progressbar)
|
25
|
+
|
26
|
+
== INSTALL:
|
27
|
+
|
28
|
+
* FIX (sudo gem install, anything else)
|
29
|
+
|
30
|
+
== DEVELOPERS:
|
31
|
+
|
32
|
+
After checking out the source, run:
|
33
|
+
|
34
|
+
$ rake newb
|
35
|
+
|
36
|
+
This task will install any missing dependencies, run the tests/specs,
|
37
|
+
and generate the RDoc.
|
38
|
+
|
39
|
+
== LICENSE:
|
40
|
+
|
41
|
+
(The MIT License)
|
42
|
+
|
43
|
+
Copyright (c) 2012 OSM2Mongo
|
44
|
+
|
45
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
46
|
+
a copy of this software and associated documentation files (the
|
47
|
+
'Software'), to deal in the Software without restriction, including
|
48
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
49
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
50
|
+
permit persons to whom the Software is furnished to do so, subject to
|
51
|
+
the following conditions:
|
52
|
+
|
53
|
+
The above copyright notice and this permission notice shall be
|
54
|
+
included in all copies or substantial portions of the Software.
|
55
|
+
|
56
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
57
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
58
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
59
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
60
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
61
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
62
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'hoe'
|
5
|
+
require 'rubygems/dependency_installer.rb'
|
6
|
+
|
7
|
+
# Hoe.plugin :compiler
|
8
|
+
# Hoe.plugin :gem_prelude_sucks
|
9
|
+
# Hoe.plugin :inline
|
10
|
+
# Hoe.plugin :racc
|
11
|
+
# Hoe.plugin :rcov
|
12
|
+
# Hoe.plugin :rubyforge
|
13
|
+
|
14
|
+
Hoe.spec 'osm2mongo' do
|
15
|
+
# HEY! If you fill these out in ~/.hoe_template/Rakefile.erb then
|
16
|
+
# you'll never have to touch them again!
|
17
|
+
# (delete this comment too, of course)
|
18
|
+
|
19
|
+
developer('Binh Nguyen Thai', 'posseidon@gmail.com')
|
20
|
+
|
21
|
+
# self.rubyforge_name = 'osm2mongo' # if different than 'osm2mongo'
|
22
|
+
end
|
23
|
+
|
24
|
+
# vim: syntax=ruby
|
25
|
+
|
26
|
+
inst = Gem::DependencyInstaller.new
|
27
|
+
begin
|
28
|
+
inst.install "progressbar"
|
29
|
+
inst.install "nokogiri"
|
30
|
+
rescue
|
31
|
+
exit(1)
|
32
|
+
end
|
data/bin/osm2mongo
ADDED
data/lib/callbacks.rb
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
require_relative 'db/mongohandler'
|
2
|
+
|
3
|
+
module Osm2Mongo
|
4
|
+
# Container: Responsible for inserting data.
|
5
|
+
#
|
6
|
+
# Purpose:
|
7
|
+
# Ability to bulk insert data by limit set at intantiation.
|
8
|
+
#
|
9
|
+
# Depends:
|
10
|
+
# - Container: Handler for MongoDB.
|
11
|
+
#
|
12
|
+
class Callbacks
|
13
|
+
NODE = "node"
|
14
|
+
WAY = "way"
|
15
|
+
RELATION = "relation"
|
16
|
+
TAG = "tag"
|
17
|
+
TEXT = "#text"
|
18
|
+
NODEREF = "nd"
|
19
|
+
MEMBER = "member"
|
20
|
+
|
21
|
+
NDS = "nodes"
|
22
|
+
TGS = "tags"
|
23
|
+
MBS = "members"
|
24
|
+
DONE = "done"
|
25
|
+
|
26
|
+
KEY = "k"
|
27
|
+
VALUE = "v"
|
28
|
+
ID = "id"
|
29
|
+
MID = "_id"
|
30
|
+
LON = "lon"
|
31
|
+
LAT = "lat"
|
32
|
+
REF = "ref"
|
33
|
+
|
34
|
+
attr_accessor :parsed
|
35
|
+
|
36
|
+
def initialize(database, collections, qlimit, parser)
|
37
|
+
@nodes = DB::Mongohandler.new(database, collections[NODE], qlimit)
|
38
|
+
@nodes.connect()
|
39
|
+
|
40
|
+
@ways = DB::Mongohandler.new(database, collections[WAY], qlimit)
|
41
|
+
@ways.use_connection(@nodes.connection)
|
42
|
+
|
43
|
+
@relations = DB::Mongohandler.new(database, collections[RELATION], qlimit)
|
44
|
+
@relations.use_connection(@nodes.connection)
|
45
|
+
|
46
|
+
parser.add_observer(self)
|
47
|
+
@parsed = false
|
48
|
+
end
|
49
|
+
|
50
|
+
def update(reader)
|
51
|
+
case reader.name
|
52
|
+
when NODE
|
53
|
+
node(reader)
|
54
|
+
when WAY
|
55
|
+
way(reader)
|
56
|
+
when RELATION
|
57
|
+
relation(reader)
|
58
|
+
when DONE
|
59
|
+
flush_all()
|
60
|
+
else
|
61
|
+
# Skip
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def flush_all
|
66
|
+
@nodes.flush()
|
67
|
+
@ways.flush()
|
68
|
+
@relations.flush()
|
69
|
+
@parsed = true
|
70
|
+
end
|
71
|
+
|
72
|
+
def node(reader)
|
73
|
+
attributes = {MID => reader.attribute(ID), LON => reader.attribute(LON), LAT => reader.attribute(LAT)}
|
74
|
+
unless reader.empty_element?
|
75
|
+
extract_children(reader, attributes)
|
76
|
+
end
|
77
|
+
@nodes.add(attributes)
|
78
|
+
end
|
79
|
+
|
80
|
+
def way(reader)
|
81
|
+
attributes = {MID => reader.attribute(ID)}
|
82
|
+
unless reader.empty_element?
|
83
|
+
extract_children(reader, attributes)
|
84
|
+
end
|
85
|
+
@ways.add(attributes)
|
86
|
+
end
|
87
|
+
|
88
|
+
def relation(reader)
|
89
|
+
attributes = {MID => reader.attribute(ID)}
|
90
|
+
unless reader.empty_element?
|
91
|
+
extract_children(reader, attributes)
|
92
|
+
end
|
93
|
+
@relations.add(attributes)
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
def extract_children(node, attributes)
|
98
|
+
nodes = []
|
99
|
+
members = []
|
100
|
+
tags = Hash.new
|
101
|
+
begin
|
102
|
+
node.read()
|
103
|
+
if (node.name == TAG )
|
104
|
+
tags[node.attribute(KEY)] = node.attribute(VALUE)
|
105
|
+
elsif (node.name == NODEREF)
|
106
|
+
nodes.push(node.attribute(REF))
|
107
|
+
elsif (node.name == MEMBER)
|
108
|
+
members.push(node.attributes)
|
109
|
+
else
|
110
|
+
# Unknown sequence
|
111
|
+
end
|
112
|
+
end while(node.name == TAG or node.name == NODEREF or node.name == MEMBER or node.name == TEXT)
|
113
|
+
|
114
|
+
attributes[TGS] = tags unless tags.empty?
|
115
|
+
attributes[NDS] = nodes unless nodes.size < 1
|
116
|
+
attributes[MBS] = members unless members.size < 1
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
data/lib/common.rb
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'net/http'
|
3
|
+
require 'progressbar'
|
4
|
+
require 'nokogiri'
|
5
|
+
require 'observer'
|
6
|
+
|
7
|
+
|
8
|
+
module Osm2Mongo
|
9
|
+
# Responsible for Http actions.
|
10
|
+
#
|
11
|
+
# Purpose:
|
12
|
+
# Preprocessing OSM data format, begins with downloading it, to importing into MongoDB.
|
13
|
+
#
|
14
|
+
# Depends:
|
15
|
+
# - Nokogiri: XML Parser
|
16
|
+
# - Progressbar: http://0xcc.net/ruby-progressbar/index.html.en
|
17
|
+
#
|
18
|
+
# This module contains:
|
19
|
+
# - Downloader method: file downloader
|
20
|
+
# - Decompressor method: unpack gzip2 osm file.
|
21
|
+
# -
|
22
|
+
#
|
23
|
+
class Common
|
24
|
+
include Observable
|
25
|
+
|
26
|
+
UNITS = %W(B KiB MiB GiB TiB).freeze
|
27
|
+
READ = "r"
|
28
|
+
ENDL = "\n"
|
29
|
+
INFO = Struct.new(:name)
|
30
|
+
DONE = "done"
|
31
|
+
|
32
|
+
|
33
|
+
#
|
34
|
+
# Download file. On success returns absolute path of downloaded file.
|
35
|
+
# ==== Attributes
|
36
|
+
# * +url+ Url address of file.
|
37
|
+
# * +destination+ Where to download it.
|
38
|
+
# ==== Examples
|
39
|
+
# result = Common::download("http://someurl.com/somedata.txt","/tmp/")
|
40
|
+
#
|
41
|
+
def download(url, destination)
|
42
|
+
uri = URI.parse(url)
|
43
|
+
http = Net::HTTP.new(uri.host)
|
44
|
+
size = http.request_head(url)['content-length'].to_f
|
45
|
+
counter = 0
|
46
|
+
file = open(destination.concat(uri.path.split("/").last), "wb")
|
47
|
+
begin
|
48
|
+
pbar = ProgressBar.new("Downloading: ", size)
|
49
|
+
http.request_get(uri.path) do |response|
|
50
|
+
response.read_body do |segment|
|
51
|
+
file.write(segment)
|
52
|
+
counter += segment.length
|
53
|
+
pbar.set(counter)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
pbar.finish
|
57
|
+
ensure
|
58
|
+
file.close()
|
59
|
+
end
|
60
|
+
return {'path' => file.path}
|
61
|
+
end
|
62
|
+
|
63
|
+
#
|
64
|
+
# Decompress a GZIP2 file to its actual path.
|
65
|
+
# ==== Attributes
|
66
|
+
# * +source+ Absolute path of archive file.
|
67
|
+
# ==== Examples
|
68
|
+
# result = Common::decompress("/tmp/hungary.osm.bz2")
|
69
|
+
#
|
70
|
+
def decompress(source)
|
71
|
+
command = Thread.new do
|
72
|
+
system("bunzip2 #{source}") # long-long programm
|
73
|
+
end
|
74
|
+
command.join
|
75
|
+
return {'path' => source.gsub!(".bz2","")}
|
76
|
+
end
|
77
|
+
|
78
|
+
#
|
79
|
+
# Push OSM XML content into Database.
|
80
|
+
# ==== Attributes
|
81
|
+
# * +osm+ absolute path to osm file.
|
82
|
+
# * +host+ MongoDB host
|
83
|
+
# * +port+ MongoDB port number
|
84
|
+
# * +db_name+ Name of database
|
85
|
+
# * +collection_name+ Collection name
|
86
|
+
# * +batch_limit+ Limit of Array for batch insert (carefull, it may vary on architecture).
|
87
|
+
# ==== Examples
|
88
|
+
# result = Common::push2mongo("/tmp/map.osm", "osm", collections, 1000)
|
89
|
+
#
|
90
|
+
def parse(osm_file)
|
91
|
+
File.open(osm_file, READ) do |osm|
|
92
|
+
reader = Nokogiri::XML::Reader(osm)
|
93
|
+
reader.read()
|
94
|
+
while reader.read()
|
95
|
+
unless reader.value?
|
96
|
+
changed()
|
97
|
+
notify_observers(reader)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
changed()
|
101
|
+
notify_observers(INFO.new(DONE))
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
#
|
107
|
+
# Print File size as readable.
|
108
|
+
# ==== Attributes
|
109
|
+
# * +size+ size in bytes
|
110
|
+
# ==== Examples
|
111
|
+
# result = Common::as_size("53344233")
|
112
|
+
#
|
113
|
+
def as_size number
|
114
|
+
if number.to_i < 1024
|
115
|
+
exponent = 0
|
116
|
+
|
117
|
+
else
|
118
|
+
max_exp = UNITS.size - 1
|
119
|
+
|
120
|
+
exponent = ( Math.log( number ) / Math.log( 1024 ) ).to_i # convert to base
|
121
|
+
exponent = max_exp if exponent > max_exp # we need this to avoid overflow for the highest unit
|
122
|
+
|
123
|
+
number /= 1024 ** exponent
|
124
|
+
end
|
125
|
+
|
126
|
+
"#{number} #{UNITS[ exponent ]}"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module DB
|
2
|
+
module AbstractInterface
|
3
|
+
|
4
|
+
class InterfaceNotImplementedError < NoMethodError
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.included(klass)
|
8
|
+
klass.send(:include, AbstractInterface::Methods)
|
9
|
+
klass.send(:extend, AbstractInterface::Methods)
|
10
|
+
end
|
11
|
+
|
12
|
+
module Methods
|
13
|
+
|
14
|
+
def api_not_implemented(klass, method_name = nil)
|
15
|
+
if method_name.nil?
|
16
|
+
caller.first.match(/in \`(.+)\'/)
|
17
|
+
method_name = $1
|
18
|
+
end
|
19
|
+
raise AbstractInterface::InterfaceNotImplementedError.new("#{klass.class.name} needs to implement '#{method_name}' for interface #{self.name}!")
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
data/lib/db/dbhandler.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
require_relative 'abstract_interface'
|
2
|
+
|
3
|
+
module DB
|
4
|
+
class Dbhandler
|
5
|
+
include AbstractInterface
|
6
|
+
|
7
|
+
#
|
8
|
+
# Abstract method
|
9
|
+
#
|
10
|
+
# Connecting to database with given:
|
11
|
+
# ==== Attributes
|
12
|
+
# * +args+ optional: host, port, database name
|
13
|
+
#
|
14
|
+
def connect(*args)
|
15
|
+
Dbhandler.api_not_implemented(self)
|
16
|
+
end
|
17
|
+
|
18
|
+
#
|
19
|
+
# Abstract method
|
20
|
+
#
|
21
|
+
# Insert data:
|
22
|
+
# ==== Attributes
|
23
|
+
# * +data+ Hash or Array
|
24
|
+
#
|
25
|
+
def insert(*args)
|
26
|
+
Dbhandler.api_not_implemented(self)
|
27
|
+
end
|
28
|
+
|
29
|
+
#
|
30
|
+
# Abstract method
|
31
|
+
#
|
32
|
+
# Insert many data at once.
|
33
|
+
# ==== Attributes
|
34
|
+
# * +data+ Array of Hashes
|
35
|
+
#
|
36
|
+
def bulk_insert(*args)
|
37
|
+
Dbhandler.api_not_implemented(self)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
require_relative 'dbhandler'
|
2
|
+
require 'mongo'
|
3
|
+
|
4
|
+
module DB
|
5
|
+
class Mongohandler < DB::Dbhandler
|
6
|
+
attr_accessor :connection, :collection, :host, :port
|
7
|
+
attr_accessor :data_array
|
8
|
+
|
9
|
+
#
|
10
|
+
# Constructor.
|
11
|
+
# ==== Attributes
|
12
|
+
# * +database_name+ database name
|
13
|
+
# * +collection_name+ collection name
|
14
|
+
# * +bulk_limit+ Array limit for bulk insert.
|
15
|
+
# * +host+ host name (default: localhost)
|
16
|
+
# * +port+ port number (default: 27017)
|
17
|
+
def initialize(database_name, collection_name, bulk_limit, host = "localhost", port = "27017")
|
18
|
+
@host = host
|
19
|
+
@port = port
|
20
|
+
@dbname = database_name
|
21
|
+
@collname = collection_name
|
22
|
+
@bulk_limit = bulk_limit
|
23
|
+
@data_array = Array.new
|
24
|
+
end
|
25
|
+
|
26
|
+
#
|
27
|
+
# Use Existing Connection
|
28
|
+
# ==== Attributes
|
29
|
+
# * +connection+ Existing connection
|
30
|
+
#
|
31
|
+
def use_connection(connection)
|
32
|
+
@connection = connection
|
33
|
+
@collection = (@connection[@dbname])[@collname]
|
34
|
+
end
|
35
|
+
|
36
|
+
#
|
37
|
+
# Connecting to Mongo database.
|
38
|
+
#
|
39
|
+
def connect
|
40
|
+
@connection = Mongo::Connection.new(@host, @port, :pool_size => 5)
|
41
|
+
@collection = (@connection[@dbname])[@collname]
|
42
|
+
end
|
43
|
+
|
44
|
+
#
|
45
|
+
# Insert data.
|
46
|
+
# ==== Attributes
|
47
|
+
# * +data+ Hash or Array
|
48
|
+
# ==== Examples
|
49
|
+
# insert({"key" => value})
|
50
|
+
#
|
51
|
+
def insert(data)
|
52
|
+
begin
|
53
|
+
@collection.insert(data)
|
54
|
+
rescue Exception => e
|
55
|
+
puts e
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
#
|
60
|
+
# Insert at once many data.
|
61
|
+
# ==== Attributes
|
62
|
+
# * +data+ Array of Hashes
|
63
|
+
# bulk_insert([{"key" => value},{"another_key" => another_value}])
|
64
|
+
#
|
65
|
+
def bulk_insert(data)
|
66
|
+
begin
|
67
|
+
@collection.insert(data)
|
68
|
+
rescue Exception => e
|
69
|
+
puts e
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
# Insert data into Data Array.
|
75
|
+
#
|
76
|
+
# Purpose:
|
77
|
+
# When limit reached DBHandler::bulk_insert will be called on data array.
|
78
|
+
#
|
79
|
+
# ==== Attributes
|
80
|
+
# * +data+ Hash or Array
|
81
|
+
#
|
82
|
+
def add(data)
|
83
|
+
if (@data_array.size >= @bulk_limit)
|
84
|
+
bulk_insert(@data_array)
|
85
|
+
@data_array.clear
|
86
|
+
@data_array.push(data)
|
87
|
+
else
|
88
|
+
@data_array.push(data)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
#
|
93
|
+
# Insert remaining data in Array and close Database connection.
|
94
|
+
#
|
95
|
+
def flush
|
96
|
+
# Push remaining data to database
|
97
|
+
bulk_insert(@data_array)
|
98
|
+
# Clear Hash Array
|
99
|
+
@data_array.clear
|
100
|
+
# Close Database connection.
|
101
|
+
@connection.close()
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
data/lib/osm2mongo.rb
ADDED
@@ -0,0 +1,161 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# == Synopsis
|
4
|
+
# This is a sample description of the application.
|
5
|
+
# Blah blah blah.
|
6
|
+
#
|
7
|
+
# == Examples
|
8
|
+
# This command does blah blah blah.
|
9
|
+
# ruby_cl_skeleton foo.txt
|
10
|
+
#
|
11
|
+
# Other examples:
|
12
|
+
# ruby_cl_skeleton -q bar.doc
|
13
|
+
# ruby_cl_skeleton --verbose foo.html
|
14
|
+
#
|
15
|
+
# == Usage
|
16
|
+
# ruby_cl_skeleton [options] source_file
|
17
|
+
#
|
18
|
+
# For help use: ruby_cl_skeleton -h
|
19
|
+
#
|
20
|
+
# == Options
|
21
|
+
# -h, --help Displays help message
|
22
|
+
# -v, --version Display the version, then exit
|
23
|
+
# -q, --quiet Output as little as possible, overrides verbose
|
24
|
+
# -V, --verbose Verbose output
|
25
|
+
# TO DO - add additional options
|
26
|
+
#
|
27
|
+
# == Author
|
28
|
+
# YourName
|
29
|
+
#
|
30
|
+
# == Copyright
|
31
|
+
# Copyright (c) 2007 YourName. Licensed under the MIT License:
|
32
|
+
# http://www.opensource.org/licenses/mit-license.php
|
33
|
+
|
34
|
+
|
35
|
+
# TO DO - replace all ruby_cl_skeleton with your app name
|
36
|
+
# TO DO - replace all YourName with your actual name
|
37
|
+
# TO DO - update Synopsis, Examples, etc
|
38
|
+
# TO DO - change license if necessary
|
39
|
+
|
40
|
+
|
41
|
+
require 'optparse'
|
42
|
+
#require 'rdoc/usage'
|
43
|
+
require 'ostruct'
|
44
|
+
require 'date'
|
45
|
+
|
46
|
+
|
47
|
+
class App
|
48
|
+
VERSION = '0.0.1'
|
49
|
+
|
50
|
+
attr_reader :options
|
51
|
+
|
52
|
+
def initialize(arguments, stdin)
|
53
|
+
@arguments = arguments
|
54
|
+
@stdin = stdin
|
55
|
+
|
56
|
+
# Set defaults
|
57
|
+
@options = OpenStruct.new
|
58
|
+
@options.verbose = false
|
59
|
+
@options.quiet = false
|
60
|
+
# TO DO - add additional defaults
|
61
|
+
end
|
62
|
+
|
63
|
+
# Parse options, check arguments, then process the command
|
64
|
+
def run
|
65
|
+
|
66
|
+
if parsed_options? && arguments_valid?
|
67
|
+
|
68
|
+
puts "Start at #{DateTime.now}\n\n" if @options.verbose
|
69
|
+
|
70
|
+
output_options if @options.verbose # [Optional]
|
71
|
+
|
72
|
+
process_arguments
|
73
|
+
process_command
|
74
|
+
|
75
|
+
puts "\nFinished at #{DateTime.now}" if @options.verbose
|
76
|
+
|
77
|
+
else
|
78
|
+
output_usage
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
|
83
|
+
protected
|
84
|
+
|
85
|
+
def parsed_options?
|
86
|
+
|
87
|
+
# Specify options
|
88
|
+
opts = OptionParser.new
|
89
|
+
opts.on('-v', '--version') { output_version ; exit 0 }
|
90
|
+
opts.on('-h', '--help') { output_help }
|
91
|
+
opts.on('-V', '--verbose') { @options.verbose = true }
|
92
|
+
opts.on('-q', '--quiet') { @options.quiet = true }
|
93
|
+
# TO DO - add additional options
|
94
|
+
|
95
|
+
opts.parse!(@arguments) rescue return false
|
96
|
+
|
97
|
+
process_options
|
98
|
+
true
|
99
|
+
end
|
100
|
+
|
101
|
+
# Performs post-parse processing on options
|
102
|
+
def process_options
|
103
|
+
@options.verbose = false if @options.quiet
|
104
|
+
end
|
105
|
+
|
106
|
+
def output_options
|
107
|
+
puts "Options:\n"
|
108
|
+
|
109
|
+
@options.marshal_dump.each do |name, val|
|
110
|
+
puts " #{name} = #{val}"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
# True if required arguments were provided
|
115
|
+
def arguments_valid?
|
116
|
+
# TO DO - implement your real logic here
|
117
|
+
true if @arguments.length == 1
|
118
|
+
end
|
119
|
+
|
120
|
+
# Setup the arguments
|
121
|
+
def process_arguments
|
122
|
+
# TO DO - place in local vars, etc
|
123
|
+
end
|
124
|
+
|
125
|
+
def output_help
|
126
|
+
output_version
|
127
|
+
#RDoc::usage() #exits app
|
128
|
+
end
|
129
|
+
|
130
|
+
def output_usage
|
131
|
+
#RDoc::usage('usage') # gets usage from comments above
|
132
|
+
end
|
133
|
+
|
134
|
+
def output_version
|
135
|
+
puts "#{File.basename(__FILE__)} version #{VERSION}"
|
136
|
+
end
|
137
|
+
|
138
|
+
def process_command
|
139
|
+
# TO DO - do whatever this app does
|
140
|
+
|
141
|
+
#process_standard_input # [Optional]
|
142
|
+
end
|
143
|
+
|
144
|
+
def process_standard_input
|
145
|
+
input = @stdin.read
|
146
|
+
# TO DO - process input
|
147
|
+
|
148
|
+
# [Optional]
|
149
|
+
#@stdin.each do |line|
|
150
|
+
# # TO DO - process each line
|
151
|
+
#end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
|
156
|
+
# TO DO - Add your Modules, Classes, etc
|
157
|
+
|
158
|
+
|
159
|
+
# Create and run the application
|
160
|
+
app = App.new(ARGV, STDIN)
|
161
|
+
app.run
|
data/test/profile.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'ruby-prof'
|
2
|
+
$LOAD_PATH << '../lib'
|
3
|
+
require 'preprocessor.rb'
|
4
|
+
|
5
|
+
# Profile the code
|
6
|
+
RubyProf.start
|
7
|
+
|
8
|
+
status = Common.parse("/tmp/map.osm")
|
9
|
+
|
10
|
+
result = RubyProf.stop
|
11
|
+
|
12
|
+
# Print a flat profile to text
|
13
|
+
printer = RubyProf::FlatPrinter.new(result)
|
14
|
+
printer.print(STDOUT)
|
@@ -0,0 +1,25 @@
|
|
1
|
+
$LOAD_PATH << '../lib'
|
2
|
+
require 'callbacks.rb'
|
3
|
+
require 'observer'
|
4
|
+
|
5
|
+
class TestCallbacks
|
6
|
+
include Observable
|
7
|
+
|
8
|
+
INFO = Struct.new(:name)
|
9
|
+
|
10
|
+
def publish
|
11
|
+
changed()
|
12
|
+
notify_observers(INFO.new("done"))
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
describe Osm2Mongo::Callbacks, "#test Observer" do
|
18
|
+
it "should open and process content of osm file" do
|
19
|
+
test = TestCallbacks.new
|
20
|
+
collections = {"node" => "nodes", "way" => "ways", "relation" => "rels"}
|
21
|
+
callback = Osm2Mongo::Callbacks.new("osm", collections, 1000, test)
|
22
|
+
test.publish()
|
23
|
+
(callback.parsed).should be(true)
|
24
|
+
end
|
25
|
+
end
|
data/test/test_mongo.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
$LOAD_PATH << '../lib/db'
|
2
|
+
require 'mongohandler.rb'
|
3
|
+
|
4
|
+
|
5
|
+
describe DB::Mongohandler, "#connect" do
|
6
|
+
it "should returns connection status active" do
|
7
|
+
mongo = DB::Mongohandler.new("local", "test", 100)
|
8
|
+
mongo.connect()
|
9
|
+
(mongo.connection.connected?).should be(true)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe DB::Mongohandler do
|
14
|
+
before do
|
15
|
+
@handler = DB::Mongohandler.new("local", "test", 100)
|
16
|
+
@handler.connect()
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should insert a single document" do
|
20
|
+
@handler.insert({"test" => "data"})
|
21
|
+
(@handler.collection.count).should be(1)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should insert at least one document" do
|
25
|
+
@handler.bulk_insert([{"test" => "data"}, {"another_test" => "another_data"}])
|
26
|
+
(@handler.collection.count).should be(2)
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
after do
|
31
|
+
@handler.collection.drop()
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
$LOAD_PATH << '../lib/'
|
2
|
+
require 'common'
|
3
|
+
require 'callbacks'
|
4
|
+
|
5
|
+
|
6
|
+
describe Osm2Mongo::Common, "#download" do
|
7
|
+
it "should returns downloaded file and location" do
|
8
|
+
#common = Osm2Mongo::Common.new()
|
9
|
+
#report = common.download("http://download.geofabrik.de/osm/europe/hungary.osm.bz2", "/tmp/")
|
10
|
+
#report['path'].should == "/tmp/hungary.osm.bz2"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe Osm2Mongo::Common, "#decompress" do
|
15
|
+
it "should returns downloaded file and location" do
|
16
|
+
#common = Osm2Mongo::Common.new()
|
17
|
+
#result = common.decompress("/tmp/hungary.osm.bz2")
|
18
|
+
#result.should_not be(nil)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe Osm2Mongo::Common, "#process map.osm" do
|
23
|
+
it "should open and process content of osm file" do
|
24
|
+
collections = {"node" => "nodes", "way" => "ways", "relation" => "rels"}
|
25
|
+
common = Osm2Mongo::Common.new()
|
26
|
+
callback = Osm2Mongo::Callbacks.new("osm", collections, 100, common)
|
27
|
+
common.parse("/tmp/map.osm")
|
28
|
+
(callback.parsed).should be(true)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
metadata
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: osm2mongo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Binh Nguyen Thai
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-06-30 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rdoc
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '3.10'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '3.10'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: hoe
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '3.0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '3.0'
|
46
|
+
description: Import Openstreetmap .osm(xml) data format into MongoDB written in Ruby.
|
47
|
+
email:
|
48
|
+
- posseidon@gmail.com
|
49
|
+
executables:
|
50
|
+
- osm2mongo
|
51
|
+
extensions: []
|
52
|
+
extra_rdoc_files:
|
53
|
+
- History.txt
|
54
|
+
- Manifest.txt
|
55
|
+
- README.txt
|
56
|
+
files:
|
57
|
+
- .autotest
|
58
|
+
- History.txt
|
59
|
+
- Manifest.txt
|
60
|
+
- README.txt
|
61
|
+
- Rakefile
|
62
|
+
- bin/osm2mongo
|
63
|
+
- lib/callbacks.rb
|
64
|
+
- lib/common.rb
|
65
|
+
- lib/db/abstract_interface.rb
|
66
|
+
- lib/db/dbhandler.rb
|
67
|
+
- lib/db/mongohandler.rb
|
68
|
+
- lib/osm2mongo.rb
|
69
|
+
- test/profile.rb
|
70
|
+
- test/test_callbacks.rb
|
71
|
+
- test/test_mongo.rb
|
72
|
+
- test/test_osm2mongo.rb
|
73
|
+
- test/test_preprocessor.rb
|
74
|
+
- .gemtest
|
75
|
+
homepage: https://github.com/posseidon/osm2mongo
|
76
|
+
licenses: []
|
77
|
+
post_install_message:
|
78
|
+
rdoc_options:
|
79
|
+
- --main
|
80
|
+
- README.txt
|
81
|
+
require_paths:
|
82
|
+
- lib
|
83
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ! '>='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ! '>='
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
requirements: []
|
96
|
+
rubyforge_project: osm2mongo
|
97
|
+
rubygems_version: 1.8.24
|
98
|
+
signing_key:
|
99
|
+
specification_version: 3
|
100
|
+
summary: Import Openstreetmap .osm(xml) data format into MongoDB written in Ruby.
|
101
|
+
test_files:
|
102
|
+
- test/test_callbacks.rb
|
103
|
+
- test/test_mongo.rb
|
104
|
+
- test/test_osm2mongo.rb
|
105
|
+
- test/test_preprocessor.rb
|