osm2mongo 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.autotest +23 -0
- data/.gemtest +0 -0
- data/History.txt +6 -0
- data/Manifest.txt +17 -0
- data/README.txt +62 -0
- data/Rakefile +32 -0
- data/bin/osm2mongo +3 -0
- data/lib/callbacks.rb +119 -0
- data/lib/common.rb +130 -0
- data/lib/db/abstract_interface.rb +24 -0
- data/lib/db/dbhandler.rb +40 -0
- data/lib/db/mongohandler.rb +104 -0
- data/lib/osm2mongo.rb +161 -0
- data/test/profile.rb +14 -0
- data/test/test_callbacks.rb +25 -0
- data/test/test_mongo.rb +33 -0
- data/test/test_osm2mongo.rb +8 -0
- data/test/test_preprocessor.rb +31 -0
- metadata +105 -0
data/.autotest
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'autotest/restart'
|
4
|
+
|
5
|
+
# Autotest.add_hook :initialize do |at|
|
6
|
+
# at.extra_files << "../some/external/dependency.rb"
|
7
|
+
#
|
8
|
+
# at.libs << ":../some/external"
|
9
|
+
#
|
10
|
+
# at.add_exception 'vendor'
|
11
|
+
#
|
12
|
+
# at.add_mapping(/dependency.rb/) do |f, _|
|
13
|
+
# at.files_matching(/test_.*rb$/)
|
14
|
+
# end
|
15
|
+
#
|
16
|
+
# %w(TestA TestB).each do |klass|
|
17
|
+
# at.extra_class_map[klass] = "test/test_misc.rb"
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
|
21
|
+
# Autotest.add_hook :run_command do |at|
|
22
|
+
# system "rake build"
|
23
|
+
# end
|
data/.gemtest
ADDED
File without changes
|
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
.autotest
|
2
|
+
History.txt
|
3
|
+
Manifest.txt
|
4
|
+
README.txt
|
5
|
+
Rakefile
|
6
|
+
bin/osm2mongo
|
7
|
+
lib/callbacks.rb
|
8
|
+
lib/common.rb
|
9
|
+
lib/db/abstract_interface.rb
|
10
|
+
lib/db/dbhandler.rb
|
11
|
+
lib/db/mongohandler.rb
|
12
|
+
lib/osm2mongo.rb
|
13
|
+
test/profile.rb
|
14
|
+
test/test_callbacks.rb
|
15
|
+
test/test_mongo.rb
|
16
|
+
test/test_osm2mongo.rb
|
17
|
+
test/test_preprocessor.rb
|
data/README.txt
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
= osm2mongo
|
2
|
+
|
3
|
+
* https://github.com/posseidon/osm2mongo
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
Import Openstreetmap .osm(xml) data format into MongoDB written in Ruby.
|
8
|
+
|
9
|
+
== FEATURES/PROBLEMS:
|
10
|
+
|
11
|
+
* TODO: Missing Shift class and modules under Osm2Mongo main module.
|
12
|
+
* TODO: Missing command line App.
|
13
|
+
* TODO: Finish making gem installable.
|
14
|
+
* TODO: Missing Rake task for installing dependencies and generate documentation.
|
15
|
+
|
16
|
+
== SYNOPSIS:
|
17
|
+
|
18
|
+
FIX (code sample of usage)
|
19
|
+
|
20
|
+
== REQUIREMENTS:
|
21
|
+
|
22
|
+
* mongo: ruby driver (http://rubygems.org/gems/mongo)
|
23
|
+
* nokogiri: xml parser (http://rubygems.org/gems/nokogiri)
|
24
|
+
* progressbar: just for fun and icandy :-) (http://rubygems.org/gems/progressbar)
|
25
|
+
|
26
|
+
== INSTALL:
|
27
|
+
|
28
|
+
* FIX (sudo gem install, anything else)
|
29
|
+
|
30
|
+
== DEVELOPERS:
|
31
|
+
|
32
|
+
After checking out the source, run:
|
33
|
+
|
34
|
+
$ rake newb
|
35
|
+
|
36
|
+
This task will install any missing dependencies, run the tests/specs,
|
37
|
+
and generate the RDoc.
|
38
|
+
|
39
|
+
== LICENSE:
|
40
|
+
|
41
|
+
(The MIT License)
|
42
|
+
|
43
|
+
Copyright (c) 2012 OSM2Mongo
|
44
|
+
|
45
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
46
|
+
a copy of this software and associated documentation files (the
|
47
|
+
'Software'), to deal in the Software without restriction, including
|
48
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
49
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
50
|
+
permit persons to whom the Software is furnished to do so, subject to
|
51
|
+
the following conditions:
|
52
|
+
|
53
|
+
The above copyright notice and this permission notice shall be
|
54
|
+
included in all copies or substantial portions of the Software.
|
55
|
+
|
56
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
57
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
58
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
59
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
60
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
61
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
62
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'hoe'
|
5
|
+
require 'rubygems/dependency_installer.rb'
|
6
|
+
|
7
|
+
# Hoe.plugin :compiler
|
8
|
+
# Hoe.plugin :gem_prelude_sucks
|
9
|
+
# Hoe.plugin :inline
|
10
|
+
# Hoe.plugin :racc
|
11
|
+
# Hoe.plugin :rcov
|
12
|
+
# Hoe.plugin :rubyforge
|
13
|
+
|
14
|
+
Hoe.spec 'osm2mongo' do
|
15
|
+
# HEY! If you fill these out in ~/.hoe_template/Rakefile.erb then
|
16
|
+
# you'll never have to touch them again!
|
17
|
+
# (delete this comment too, of course)
|
18
|
+
|
19
|
+
developer('Binh Nguyen Thai', 'posseidon@gmail.com')
|
20
|
+
|
21
|
+
# self.rubyforge_name = 'osm2mongo' # if different than 'osm2mongo'
|
22
|
+
end
|
23
|
+
|
24
|
+
# vim: syntax=ruby
|
25
|
+
|
26
|
+
inst = Gem::DependencyInstaller.new
|
27
|
+
begin
|
28
|
+
inst.install "progressbar"
|
29
|
+
inst.install "nokogiri"
|
30
|
+
rescue
|
31
|
+
exit(1)
|
32
|
+
end
|
data/bin/osm2mongo
ADDED
data/lib/callbacks.rb
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
require_relative 'db/mongohandler'
|
2
|
+
|
3
|
+
module Osm2Mongo
|
4
|
+
# Container: Responsible for inserting data.
|
5
|
+
#
|
6
|
+
# Purpose:
|
7
|
+
# Ability to bulk insert data by limit set at intantiation.
|
8
|
+
#
|
9
|
+
# Depends:
|
10
|
+
# - Container: Handler for MongoDB.
|
11
|
+
#
|
12
|
+
class Callbacks
|
13
|
+
NODE = "node"
|
14
|
+
WAY = "way"
|
15
|
+
RELATION = "relation"
|
16
|
+
TAG = "tag"
|
17
|
+
TEXT = "#text"
|
18
|
+
NODEREF = "nd"
|
19
|
+
MEMBER = "member"
|
20
|
+
|
21
|
+
NDS = "nodes"
|
22
|
+
TGS = "tags"
|
23
|
+
MBS = "members"
|
24
|
+
DONE = "done"
|
25
|
+
|
26
|
+
KEY = "k"
|
27
|
+
VALUE = "v"
|
28
|
+
ID = "id"
|
29
|
+
MID = "_id"
|
30
|
+
LON = "lon"
|
31
|
+
LAT = "lat"
|
32
|
+
REF = "ref"
|
33
|
+
|
34
|
+
attr_accessor :parsed
|
35
|
+
|
36
|
+
def initialize(database, collections, qlimit, parser)
|
37
|
+
@nodes = DB::Mongohandler.new(database, collections[NODE], qlimit)
|
38
|
+
@nodes.connect()
|
39
|
+
|
40
|
+
@ways = DB::Mongohandler.new(database, collections[WAY], qlimit)
|
41
|
+
@ways.use_connection(@nodes.connection)
|
42
|
+
|
43
|
+
@relations = DB::Mongohandler.new(database, collections[RELATION], qlimit)
|
44
|
+
@relations.use_connection(@nodes.connection)
|
45
|
+
|
46
|
+
parser.add_observer(self)
|
47
|
+
@parsed = false
|
48
|
+
end
|
49
|
+
|
50
|
+
def update(reader)
|
51
|
+
case reader.name
|
52
|
+
when NODE
|
53
|
+
node(reader)
|
54
|
+
when WAY
|
55
|
+
way(reader)
|
56
|
+
when RELATION
|
57
|
+
relation(reader)
|
58
|
+
when DONE
|
59
|
+
flush_all()
|
60
|
+
else
|
61
|
+
# Skip
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def flush_all
|
66
|
+
@nodes.flush()
|
67
|
+
@ways.flush()
|
68
|
+
@relations.flush()
|
69
|
+
@parsed = true
|
70
|
+
end
|
71
|
+
|
72
|
+
def node(reader)
|
73
|
+
attributes = {MID => reader.attribute(ID), LON => reader.attribute(LON), LAT => reader.attribute(LAT)}
|
74
|
+
unless reader.empty_element?
|
75
|
+
extract_children(reader, attributes)
|
76
|
+
end
|
77
|
+
@nodes.add(attributes)
|
78
|
+
end
|
79
|
+
|
80
|
+
def way(reader)
|
81
|
+
attributes = {MID => reader.attribute(ID)}
|
82
|
+
unless reader.empty_element?
|
83
|
+
extract_children(reader, attributes)
|
84
|
+
end
|
85
|
+
@ways.add(attributes)
|
86
|
+
end
|
87
|
+
|
88
|
+
def relation(reader)
|
89
|
+
attributes = {MID => reader.attribute(ID)}
|
90
|
+
unless reader.empty_element?
|
91
|
+
extract_children(reader, attributes)
|
92
|
+
end
|
93
|
+
@relations.add(attributes)
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
def extract_children(node, attributes)
|
98
|
+
nodes = []
|
99
|
+
members = []
|
100
|
+
tags = Hash.new
|
101
|
+
begin
|
102
|
+
node.read()
|
103
|
+
if (node.name == TAG )
|
104
|
+
tags[node.attribute(KEY)] = node.attribute(VALUE)
|
105
|
+
elsif (node.name == NODEREF)
|
106
|
+
nodes.push(node.attribute(REF))
|
107
|
+
elsif (node.name == MEMBER)
|
108
|
+
members.push(node.attributes)
|
109
|
+
else
|
110
|
+
# Unknown sequence
|
111
|
+
end
|
112
|
+
end while(node.name == TAG or node.name == NODEREF or node.name == MEMBER or node.name == TEXT)
|
113
|
+
|
114
|
+
attributes[TGS] = tags unless tags.empty?
|
115
|
+
attributes[NDS] = nodes unless nodes.size < 1
|
116
|
+
attributes[MBS] = members unless members.size < 1
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
data/lib/common.rb
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'net/http'
|
3
|
+
require 'progressbar'
|
4
|
+
require 'nokogiri'
|
5
|
+
require 'observer'
|
6
|
+
|
7
|
+
|
8
|
+
module Osm2Mongo
|
9
|
+
# Responsible for Http actions.
|
10
|
+
#
|
11
|
+
# Purpose:
|
12
|
+
# Preprocessing OSM data format, begins with downloading it, to importing into MongoDB.
|
13
|
+
#
|
14
|
+
# Depends:
|
15
|
+
# - Nokogiri: XML Parser
|
16
|
+
# - Progressbar: http://0xcc.net/ruby-progressbar/index.html.en
|
17
|
+
#
|
18
|
+
# This module contains:
|
19
|
+
# - Downloader method: file downloader
|
20
|
+
# - Decompressor method: unpack gzip2 osm file.
|
21
|
+
# -
|
22
|
+
#
|
23
|
+
class Common
|
24
|
+
include Observable
|
25
|
+
|
26
|
+
UNITS = %W(B KiB MiB GiB TiB).freeze
|
27
|
+
READ = "r"
|
28
|
+
ENDL = "\n"
|
29
|
+
INFO = Struct.new(:name)
|
30
|
+
DONE = "done"
|
31
|
+
|
32
|
+
|
33
|
+
#
|
34
|
+
# Download file. On success returns absolute path of downloaded file.
|
35
|
+
# ==== Attributes
|
36
|
+
# * +url+ Url address of file.
|
37
|
+
# * +destination+ Where to download it.
|
38
|
+
# ==== Examples
|
39
|
+
# result = Common::download("http://someurl.com/somedata.txt","/tmp/")
|
40
|
+
#
|
41
|
+
def download(url, destination)
|
42
|
+
uri = URI.parse(url)
|
43
|
+
http = Net::HTTP.new(uri.host)
|
44
|
+
size = http.request_head(url)['content-length'].to_f
|
45
|
+
counter = 0
|
46
|
+
file = open(destination.concat(uri.path.split("/").last), "wb")
|
47
|
+
begin
|
48
|
+
pbar = ProgressBar.new("Downloading: ", size)
|
49
|
+
http.request_get(uri.path) do |response|
|
50
|
+
response.read_body do |segment|
|
51
|
+
file.write(segment)
|
52
|
+
counter += segment.length
|
53
|
+
pbar.set(counter)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
pbar.finish
|
57
|
+
ensure
|
58
|
+
file.close()
|
59
|
+
end
|
60
|
+
return {'path' => file.path}
|
61
|
+
end
|
62
|
+
|
63
|
+
#
|
64
|
+
# Decompress a GZIP2 file to its actual path.
|
65
|
+
# ==== Attributes
|
66
|
+
# * +source+ Absolute path of archive file.
|
67
|
+
# ==== Examples
|
68
|
+
# result = Common::decompress("/tmp/hungary.osm.bz2")
|
69
|
+
#
|
70
|
+
def decompress(source)
|
71
|
+
command = Thread.new do
|
72
|
+
system("bunzip2 #{source}") # long-long programm
|
73
|
+
end
|
74
|
+
command.join
|
75
|
+
return {'path' => source.gsub!(".bz2","")}
|
76
|
+
end
|
77
|
+
|
78
|
+
#
|
79
|
+
# Push OSM XML content into Database.
|
80
|
+
# ==== Attributes
|
81
|
+
# * +osm+ absolute path to osm file.
|
82
|
+
# * +host+ MongoDB host
|
83
|
+
# * +port+ MongoDB port number
|
84
|
+
# * +db_name+ Name of database
|
85
|
+
# * +collection_name+ Collection name
|
86
|
+
# * +batch_limit+ Limit of Array for batch insert (carefull, it may vary on architecture).
|
87
|
+
# ==== Examples
|
88
|
+
# result = Common::push2mongo("/tmp/map.osm", "osm", collections, 1000)
|
89
|
+
#
|
90
|
+
def parse(osm_file)
|
91
|
+
File.open(osm_file, READ) do |osm|
|
92
|
+
reader = Nokogiri::XML::Reader(osm)
|
93
|
+
reader.read()
|
94
|
+
while reader.read()
|
95
|
+
unless reader.value?
|
96
|
+
changed()
|
97
|
+
notify_observers(reader)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
changed()
|
101
|
+
notify_observers(INFO.new(DONE))
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
#
|
107
|
+
# Print File size as readable.
|
108
|
+
# ==== Attributes
|
109
|
+
# * +size+ size in bytes
|
110
|
+
# ==== Examples
|
111
|
+
# result = Common::as_size("53344233")
|
112
|
+
#
|
113
|
+
def as_size number
|
114
|
+
if number.to_i < 1024
|
115
|
+
exponent = 0
|
116
|
+
|
117
|
+
else
|
118
|
+
max_exp = UNITS.size - 1
|
119
|
+
|
120
|
+
exponent = ( Math.log( number ) / Math.log( 1024 ) ).to_i # convert to base
|
121
|
+
exponent = max_exp if exponent > max_exp # we need this to avoid overflow for the highest unit
|
122
|
+
|
123
|
+
number /= 1024 ** exponent
|
124
|
+
end
|
125
|
+
|
126
|
+
"#{number} #{UNITS[ exponent ]}"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module DB
|
2
|
+
module AbstractInterface
|
3
|
+
|
4
|
+
class InterfaceNotImplementedError < NoMethodError
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.included(klass)
|
8
|
+
klass.send(:include, AbstractInterface::Methods)
|
9
|
+
klass.send(:extend, AbstractInterface::Methods)
|
10
|
+
end
|
11
|
+
|
12
|
+
module Methods
|
13
|
+
|
14
|
+
def api_not_implemented(klass, method_name = nil)
|
15
|
+
if method_name.nil?
|
16
|
+
caller.first.match(/in \`(.+)\'/)
|
17
|
+
method_name = $1
|
18
|
+
end
|
19
|
+
raise AbstractInterface::InterfaceNotImplementedError.new("#{klass.class.name} needs to implement '#{method_name}' for interface #{self.name}!")
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
data/lib/db/dbhandler.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
require_relative 'abstract_interface'
|
2
|
+
|
3
|
+
module DB
|
4
|
+
class Dbhandler
|
5
|
+
include AbstractInterface
|
6
|
+
|
7
|
+
#
|
8
|
+
# Abstract method
|
9
|
+
#
|
10
|
+
# Connecting to database with given:
|
11
|
+
# ==== Attributes
|
12
|
+
# * +args+ optional: host, port, database name
|
13
|
+
#
|
14
|
+
def connect(*args)
|
15
|
+
Dbhandler.api_not_implemented(self)
|
16
|
+
end
|
17
|
+
|
18
|
+
#
|
19
|
+
# Abstract method
|
20
|
+
#
|
21
|
+
# Insert data:
|
22
|
+
# ==== Attributes
|
23
|
+
# * +data+ Hash or Array
|
24
|
+
#
|
25
|
+
def insert(*args)
|
26
|
+
Dbhandler.api_not_implemented(self)
|
27
|
+
end
|
28
|
+
|
29
|
+
#
|
30
|
+
# Abstract method
|
31
|
+
#
|
32
|
+
# Insert many data at once.
|
33
|
+
# ==== Attributes
|
34
|
+
# * +data+ Array of Hashes
|
35
|
+
#
|
36
|
+
def bulk_insert(*args)
|
37
|
+
Dbhandler.api_not_implemented(self)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
require_relative 'dbhandler'
|
2
|
+
require 'mongo'
|
3
|
+
|
4
|
+
module DB
|
5
|
+
class Mongohandler < DB::Dbhandler
|
6
|
+
attr_accessor :connection, :collection, :host, :port
|
7
|
+
attr_accessor :data_array
|
8
|
+
|
9
|
+
#
|
10
|
+
# Constructor.
|
11
|
+
# ==== Attributes
|
12
|
+
# * +database_name+ database name
|
13
|
+
# * +collection_name+ collection name
|
14
|
+
# * +bulk_limit+ Array limit for bulk insert.
|
15
|
+
# * +host+ host name (default: localhost)
|
16
|
+
# * +port+ port number (default: 27017)
|
17
|
+
def initialize(database_name, collection_name, bulk_limit, host = "localhost", port = "27017")
|
18
|
+
@host = host
|
19
|
+
@port = port
|
20
|
+
@dbname = database_name
|
21
|
+
@collname = collection_name
|
22
|
+
@bulk_limit = bulk_limit
|
23
|
+
@data_array = Array.new
|
24
|
+
end
|
25
|
+
|
26
|
+
#
|
27
|
+
# Use Existing Connection
|
28
|
+
# ==== Attributes
|
29
|
+
# * +connection+ Existing connection
|
30
|
+
#
|
31
|
+
def use_connection(connection)
|
32
|
+
@connection = connection
|
33
|
+
@collection = (@connection[@dbname])[@collname]
|
34
|
+
end
|
35
|
+
|
36
|
+
#
|
37
|
+
# Connecting to Mongo database.
|
38
|
+
#
|
39
|
+
def connect
|
40
|
+
@connection = Mongo::Connection.new(@host, @port, :pool_size => 5)
|
41
|
+
@collection = (@connection[@dbname])[@collname]
|
42
|
+
end
|
43
|
+
|
44
|
+
#
|
45
|
+
# Insert data.
|
46
|
+
# ==== Attributes
|
47
|
+
# * +data+ Hash or Array
|
48
|
+
# ==== Examples
|
49
|
+
# insert({"key" => value})
|
50
|
+
#
|
51
|
+
def insert(data)
|
52
|
+
begin
|
53
|
+
@collection.insert(data)
|
54
|
+
rescue Exception => e
|
55
|
+
puts e
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
#
|
60
|
+
# Insert at once many data.
|
61
|
+
# ==== Attributes
|
62
|
+
# * +data+ Array of Hashes
|
63
|
+
# bulk_insert([{"key" => value},{"another_key" => another_value}])
|
64
|
+
#
|
65
|
+
def bulk_insert(data)
|
66
|
+
begin
|
67
|
+
@collection.insert(data)
|
68
|
+
rescue Exception => e
|
69
|
+
puts e
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
# Insert data into Data Array.
|
75
|
+
#
|
76
|
+
# Purpose:
|
77
|
+
# When limit reached DBHandler::bulk_insert will be called on data array.
|
78
|
+
#
|
79
|
+
# ==== Attributes
|
80
|
+
# * +data+ Hash or Array
|
81
|
+
#
|
82
|
+
def add(data)
|
83
|
+
if (@data_array.size >= @bulk_limit)
|
84
|
+
bulk_insert(@data_array)
|
85
|
+
@data_array.clear
|
86
|
+
@data_array.push(data)
|
87
|
+
else
|
88
|
+
@data_array.push(data)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
#
|
93
|
+
# Insert remaining data in Array and close Database connection.
|
94
|
+
#
|
95
|
+
def flush
|
96
|
+
# Push remaining data to database
|
97
|
+
bulk_insert(@data_array)
|
98
|
+
# Clear Hash Array
|
99
|
+
@data_array.clear
|
100
|
+
# Close Database connection.
|
101
|
+
@connection.close()
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
data/lib/osm2mongo.rb
ADDED
@@ -0,0 +1,161 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# == Synopsis
|
4
|
+
# This is a sample description of the application.
|
5
|
+
# Blah blah blah.
|
6
|
+
#
|
7
|
+
# == Examples
|
8
|
+
# This command does blah blah blah.
|
9
|
+
# ruby_cl_skeleton foo.txt
|
10
|
+
#
|
11
|
+
# Other examples:
|
12
|
+
# ruby_cl_skeleton -q bar.doc
|
13
|
+
# ruby_cl_skeleton --verbose foo.html
|
14
|
+
#
|
15
|
+
# == Usage
|
16
|
+
# ruby_cl_skeleton [options] source_file
|
17
|
+
#
|
18
|
+
# For help use: ruby_cl_skeleton -h
|
19
|
+
#
|
20
|
+
# == Options
|
21
|
+
# -h, --help Displays help message
|
22
|
+
# -v, --version Display the version, then exit
|
23
|
+
# -q, --quiet Output as little as possible, overrides verbose
|
24
|
+
# -V, --verbose Verbose output
|
25
|
+
# TO DO - add additional options
|
26
|
+
#
|
27
|
+
# == Author
|
28
|
+
# YourName
|
29
|
+
#
|
30
|
+
# == Copyright
|
31
|
+
# Copyright (c) 2007 YourName. Licensed under the MIT License:
|
32
|
+
# http://www.opensource.org/licenses/mit-license.php
|
33
|
+
|
34
|
+
|
35
|
+
# TO DO - replace all ruby_cl_skeleton with your app name
|
36
|
+
# TO DO - replace all YourName with your actual name
|
37
|
+
# TO DO - update Synopsis, Examples, etc
|
38
|
+
# TO DO - change license if necessary
|
39
|
+
|
40
|
+
|
41
|
+
require 'optparse'
|
42
|
+
#require 'rdoc/usage'
|
43
|
+
require 'ostruct'
|
44
|
+
require 'date'
|
45
|
+
|
46
|
+
|
47
|
+
class App
|
48
|
+
VERSION = '0.0.1'
|
49
|
+
|
50
|
+
attr_reader :options
|
51
|
+
|
52
|
+
def initialize(arguments, stdin)
|
53
|
+
@arguments = arguments
|
54
|
+
@stdin = stdin
|
55
|
+
|
56
|
+
# Set defaults
|
57
|
+
@options = OpenStruct.new
|
58
|
+
@options.verbose = false
|
59
|
+
@options.quiet = false
|
60
|
+
# TO DO - add additional defaults
|
61
|
+
end
|
62
|
+
|
63
|
+
# Parse options, check arguments, then process the command
|
64
|
+
def run
|
65
|
+
|
66
|
+
if parsed_options? && arguments_valid?
|
67
|
+
|
68
|
+
puts "Start at #{DateTime.now}\n\n" if @options.verbose
|
69
|
+
|
70
|
+
output_options if @options.verbose # [Optional]
|
71
|
+
|
72
|
+
process_arguments
|
73
|
+
process_command
|
74
|
+
|
75
|
+
puts "\nFinished at #{DateTime.now}" if @options.verbose
|
76
|
+
|
77
|
+
else
|
78
|
+
output_usage
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
|
83
|
+
protected
|
84
|
+
|
85
|
+
def parsed_options?
|
86
|
+
|
87
|
+
# Specify options
|
88
|
+
opts = OptionParser.new
|
89
|
+
opts.on('-v', '--version') { output_version ; exit 0 }
|
90
|
+
opts.on('-h', '--help') { output_help }
|
91
|
+
opts.on('-V', '--verbose') { @options.verbose = true }
|
92
|
+
opts.on('-q', '--quiet') { @options.quiet = true }
|
93
|
+
# TO DO - add additional options
|
94
|
+
|
95
|
+
opts.parse!(@arguments) rescue return false
|
96
|
+
|
97
|
+
process_options
|
98
|
+
true
|
99
|
+
end
|
100
|
+
|
101
|
+
# Performs post-parse processing on options
|
102
|
+
def process_options
|
103
|
+
@options.verbose = false if @options.quiet
|
104
|
+
end
|
105
|
+
|
106
|
+
def output_options
|
107
|
+
puts "Options:\n"
|
108
|
+
|
109
|
+
@options.marshal_dump.each do |name, val|
|
110
|
+
puts " #{name} = #{val}"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
# True if required arguments were provided
|
115
|
+
def arguments_valid?
|
116
|
+
# TO DO - implement your real logic here
|
117
|
+
true if @arguments.length == 1
|
118
|
+
end
|
119
|
+
|
120
|
+
# Setup the arguments
|
121
|
+
def process_arguments
|
122
|
+
# TO DO - place in local vars, etc
|
123
|
+
end
|
124
|
+
|
125
|
+
def output_help
|
126
|
+
output_version
|
127
|
+
#RDoc::usage() #exits app
|
128
|
+
end
|
129
|
+
|
130
|
+
def output_usage
|
131
|
+
#RDoc::usage('usage') # gets usage from comments above
|
132
|
+
end
|
133
|
+
|
134
|
+
def output_version
|
135
|
+
puts "#{File.basename(__FILE__)} version #{VERSION}"
|
136
|
+
end
|
137
|
+
|
138
|
+
def process_command
|
139
|
+
# TO DO - do whatever this app does
|
140
|
+
|
141
|
+
#process_standard_input # [Optional]
|
142
|
+
end
|
143
|
+
|
144
|
+
def process_standard_input
|
145
|
+
input = @stdin.read
|
146
|
+
# TO DO - process input
|
147
|
+
|
148
|
+
# [Optional]
|
149
|
+
#@stdin.each do |line|
|
150
|
+
# # TO DO - process each line
|
151
|
+
#end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
|
156
|
+
# TO DO - Add your Modules, Classes, etc
|
157
|
+
|
158
|
+
|
159
|
+
# Create and run the application
|
160
|
+
app = App.new(ARGV, STDIN)
|
161
|
+
app.run
|
data/test/profile.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'ruby-prof'
|
2
|
+
$LOAD_PATH << '../lib'
|
3
|
+
require 'preprocessor.rb'
|
4
|
+
|
5
|
+
# Profile the code
|
6
|
+
RubyProf.start
|
7
|
+
|
8
|
+
status = Common.parse("/tmp/map.osm")
|
9
|
+
|
10
|
+
result = RubyProf.stop
|
11
|
+
|
12
|
+
# Print a flat profile to text
|
13
|
+
printer = RubyProf::FlatPrinter.new(result)
|
14
|
+
printer.print(STDOUT)
|
@@ -0,0 +1,25 @@
|
|
1
|
+
$LOAD_PATH << '../lib'
|
2
|
+
require 'callbacks.rb'
|
3
|
+
require 'observer'
|
4
|
+
|
5
|
+
class TestCallbacks
|
6
|
+
include Observable
|
7
|
+
|
8
|
+
INFO = Struct.new(:name)
|
9
|
+
|
10
|
+
def publish
|
11
|
+
changed()
|
12
|
+
notify_observers(INFO.new("done"))
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
describe Osm2Mongo::Callbacks, "#test Observer" do
|
18
|
+
it "should open and process content of osm file" do
|
19
|
+
test = TestCallbacks.new
|
20
|
+
collections = {"node" => "nodes", "way" => "ways", "relation" => "rels"}
|
21
|
+
callback = Osm2Mongo::Callbacks.new("osm", collections, 1000, test)
|
22
|
+
test.publish()
|
23
|
+
(callback.parsed).should be(true)
|
24
|
+
end
|
25
|
+
end
|
data/test/test_mongo.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
$LOAD_PATH << '../lib/db'
|
2
|
+
require 'mongohandler.rb'
|
3
|
+
|
4
|
+
|
5
|
+
describe DB::Mongohandler, "#connect" do
|
6
|
+
it "should returns connection status active" do
|
7
|
+
mongo = DB::Mongohandler.new("local", "test", 100)
|
8
|
+
mongo.connect()
|
9
|
+
(mongo.connection.connected?).should be(true)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe DB::Mongohandler do
|
14
|
+
before do
|
15
|
+
@handler = DB::Mongohandler.new("local", "test", 100)
|
16
|
+
@handler.connect()
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should insert a single document" do
|
20
|
+
@handler.insert({"test" => "data"})
|
21
|
+
(@handler.collection.count).should be(1)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should insert at least one document" do
|
25
|
+
@handler.bulk_insert([{"test" => "data"}, {"another_test" => "another_data"}])
|
26
|
+
(@handler.collection.count).should be(2)
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
after do
|
31
|
+
@handler.collection.drop()
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
$LOAD_PATH << '../lib/'
|
2
|
+
require 'common'
|
3
|
+
require 'callbacks'
|
4
|
+
|
5
|
+
|
6
|
+
describe Osm2Mongo::Common, "#download" do
|
7
|
+
it "should returns downloaded file and location" do
|
8
|
+
#common = Osm2Mongo::Common.new()
|
9
|
+
#report = common.download("http://download.geofabrik.de/osm/europe/hungary.osm.bz2", "/tmp/")
|
10
|
+
#report['path'].should == "/tmp/hungary.osm.bz2"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe Osm2Mongo::Common, "#decompress" do
|
15
|
+
it "should returns downloaded file and location" do
|
16
|
+
#common = Osm2Mongo::Common.new()
|
17
|
+
#result = common.decompress("/tmp/hungary.osm.bz2")
|
18
|
+
#result.should_not be(nil)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe Osm2Mongo::Common, "#process map.osm" do
|
23
|
+
it "should open and process content of osm file" do
|
24
|
+
collections = {"node" => "nodes", "way" => "ways", "relation" => "rels"}
|
25
|
+
common = Osm2Mongo::Common.new()
|
26
|
+
callback = Osm2Mongo::Callbacks.new("osm", collections, 100, common)
|
27
|
+
common.parse("/tmp/map.osm")
|
28
|
+
(callback.parsed).should be(true)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
metadata
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: osm2mongo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Binh Nguyen Thai
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-06-30 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rdoc
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '3.10'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '3.10'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: hoe
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '3.0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '3.0'
|
46
|
+
description: Import Openstreetmap .osm(xml) data format into MongoDB written in Ruby.
|
47
|
+
email:
|
48
|
+
- posseidon@gmail.com
|
49
|
+
executables:
|
50
|
+
- osm2mongo
|
51
|
+
extensions: []
|
52
|
+
extra_rdoc_files:
|
53
|
+
- History.txt
|
54
|
+
- Manifest.txt
|
55
|
+
- README.txt
|
56
|
+
files:
|
57
|
+
- .autotest
|
58
|
+
- History.txt
|
59
|
+
- Manifest.txt
|
60
|
+
- README.txt
|
61
|
+
- Rakefile
|
62
|
+
- bin/osm2mongo
|
63
|
+
- lib/callbacks.rb
|
64
|
+
- lib/common.rb
|
65
|
+
- lib/db/abstract_interface.rb
|
66
|
+
- lib/db/dbhandler.rb
|
67
|
+
- lib/db/mongohandler.rb
|
68
|
+
- lib/osm2mongo.rb
|
69
|
+
- test/profile.rb
|
70
|
+
- test/test_callbacks.rb
|
71
|
+
- test/test_mongo.rb
|
72
|
+
- test/test_osm2mongo.rb
|
73
|
+
- test/test_preprocessor.rb
|
74
|
+
- .gemtest
|
75
|
+
homepage: https://github.com/posseidon/osm2mongo
|
76
|
+
licenses: []
|
77
|
+
post_install_message:
|
78
|
+
rdoc_options:
|
79
|
+
- --main
|
80
|
+
- README.txt
|
81
|
+
require_paths:
|
82
|
+
- lib
|
83
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ! '>='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ! '>='
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
requirements: []
|
96
|
+
rubyforge_project: osm2mongo
|
97
|
+
rubygems_version: 1.8.24
|
98
|
+
signing_key:
|
99
|
+
specification_version: 3
|
100
|
+
summary: Import Openstreetmap .osm(xml) data format into MongoDB written in Ruby.
|
101
|
+
test_files:
|
102
|
+
- test/test_callbacks.rb
|
103
|
+
- test/test_mongo.rb
|
104
|
+
- test/test_osm2mongo.rb
|
105
|
+
- test/test_preprocessor.rb
|