active_road 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +15 -0
  2. data/.rspec +1 -0
  3. data/.travis.yml +7 -2
  4. data/Gemfile +8 -2
  5. data/Guardfile +2 -6
  6. data/README.md +69 -9
  7. data/Rakefile +8 -7
  8. data/active_road.gemspec +19 -15
  9. data/app/models/active_road/access_link.rb +4 -4
  10. data/app/models/active_road/access_point.rb +5 -9
  11. data/app/models/active_road/boundary.rb +41 -0
  12. data/app/models/active_road/junction.rb +4 -18
  13. data/app/models/active_road/junction_conditionnal_cost.rb +2 -0
  14. data/app/models/active_road/junctions_physical_road.rb +5 -0
  15. data/app/models/active_road/logical_road.rb +5 -4
  16. data/app/models/active_road/osm_pbf_importer.rb +293 -0
  17. data/app/models/active_road/osm_pbf_importer_level_db.rb +784 -0
  18. data/app/models/active_road/path.rb +40 -9
  19. data/app/models/active_road/physical_road.rb +22 -27
  20. data/app/models/active_road/physical_road_conditionnal_cost.rb +3 -1
  21. data/app/models/active_road/request_conditionnal_cost_linker.rb +59 -0
  22. data/app/models/active_road/street_number.rb +60 -57
  23. data/app/models/active_road/terra_importer.rb +161 -0
  24. data/db/migrate/20120419093427_add_kind_to_physical_roads.rb +2 -2
  25. data/db/migrate/20140206091734_create_boundaries.rb +16 -0
  26. data/db/migrate/20140210132933_add_attributes_to_physical_road.rb +9 -0
  27. data/db/migrate/20140219095521_add_boundary_id_to_logical_road.rb +5 -0
  28. data/db/migrate/20140228072448_add_boundary_id_to_physical_road.rb +5 -0
  29. data/db/migrate/20140304141150_add_marker_to_physical_road.rb +5 -0
  30. data/db/migrate/20140310083550_add_tags_to_street_number.rb +5 -0
  31. data/db/migrate/20140317153437_add_index_to_conditionnal_costs.rb +6 -0
  32. data/db/migrate/20140602160047_add_physical_road_index_to_junctions_physical_road.rb +5 -0
  33. data/lib/active_road.rb +8 -2
  34. data/lib/active_road/engine.rb +4 -1
  35. data/lib/active_road/shortest_path/finder.rb +37 -46
  36. data/lib/active_road/simulation_tool.rb +73 -0
  37. data/lib/active_road/version.rb +1 -1
  38. data/lib/tasks/activeroad_tasks.rake +88 -4
  39. data/script/benchmark_import_kyotocabinet.rb +148 -0
  40. data/script/benchmark_shortest_path.rb +22 -0
  41. data/script/count_tag_in_osm_data.rb +114 -0
  42. data/script/import-tiger-numbers +3 -3
  43. data/spec/dummy/db/schema.rb +2 -1
  44. data/spec/dummy/db/structure.sql +100 -11
  45. data/spec/factories/boundary.rb +8 -0
  46. data/spec/factories/junction.rb +1 -1
  47. data/spec/factories/physical_road.rb +1 -2
  48. data/spec/fixtures/test.osm +120 -0
  49. data/spec/fixtures/test.osm.bz2 +0 -0
  50. data/spec/fixtures/test.osm.pbf +0 -0
  51. data/spec/lib/active_road/shortest_path/finder_spec.rb +143 -90
  52. data/spec/lib/active_road/shortest_path/performance_finder_spec.rb +59 -0
  53. data/spec/models/active_road/access_point_spec.rb +9 -18
  54. data/spec/models/active_road/junction_conditionnal_cost_spec.rb +4 -4
  55. data/spec/models/active_road/junction_spec.rb +34 -11
  56. data/spec/models/active_road/logical_road_spec.rb +20 -19
  57. data/spec/models/active_road/osm_pbf_importer_level_db_spec.rb +410 -0
  58. data/spec/models/active_road/path_spec.rb +1 -1
  59. data/spec/models/active_road/physical_road_conditionnal_cost_spec.rb +4 -4
  60. data/spec/models/active_road/physical_road_spec.rb +14 -3
  61. data/spec/models/active_road/request_conditionnal_cost_linker_spec.rb +65 -0
  62. data/spec/models/active_road/shared_examples/osm_pbf_importer_spec.rb +148 -0
  63. data/spec/models/active_road/street_number_spec.rb +58 -58
  64. data/spec/models/active_road/terra_importer_spec.rb +140 -0
  65. data/spec/spec_helper.rb +14 -9
  66. data/spec/support/geometry_support.rb +36 -0
  67. data/spec/support/profile.rb +19 -0
  68. data/tmp/performance/.gitignore +0 -0
  69. data/travis/before_install.sh +5 -9
  70. data/travis/before_script.sh +7 -7
  71. metadata +118 -121
  72. data/app/models/active_road/physical_road_filter.rb +0 -41
  73. data/app/models/active_road/terra_import.rb +0 -148
  74. data/spec/models/active_road/physical_road_filter_spec.rb +0 -85
  75. data/spec/models/active_road/terra_import_spec.rb +0 -113
  76. data/spec/support/georuby_ext.rb +0 -15
@@ -0,0 +1,784 @@
1
+ require 'leveldb-native'
2
+ require 'csv'
3
+
4
+ module ActiveRoad
5
+ class OsmPbfImporterLevelDb
6
+ include OsmPbfImporter
7
+
8
+ @@csv_batch_size = 100000
9
+ cattr_reader :csv_batch_size
10
+
11
+ attr_reader :ways_database_path, :nodes_database_path, :physical_roads_database_path, :junctions_database_path, :pbf_file, :split_ways
12
+
13
+ def initialize(pbf_file, split_ways = false, nodes_database_path = "/tmp/osm_pbf_nodes_leveldb", ways_database_path = "/tmp/osm_pbf_ways_leveldb")
14
+ @pbf_file = pbf_file
15
+ @split_ways = split_ways
16
+ @nodes_database_path = nodes_database_path
17
+ @ways_database_path = ways_database_path
18
+ @junctions_database_path = "/tmp/osm_pbf_junctions_leveldb"
19
+ @physical_roads_database_path = "/tmp/osm_pbf_physical_roads_leveldb"
20
+ end
21
+
22
+ def nodes_database
23
+ @nodes_database ||= LevelDBNative::DB.make nodes_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
24
+ end
25
+
26
+ def close_nodes_database
27
+ nodes_database.close!
28
+ end
29
+
30
+ def delete_nodes_database
31
+ FileUtils.remove_entry nodes_database_path if File.exists?(nodes_database_path)
32
+ end
33
+
34
+ def ways_database
35
+ @ways_database ||= LevelDBNative::DB.make ways_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
36
+ end
37
+
38
+ def close_ways_database
39
+ ways_database.close!
40
+ end
41
+
42
+ def delete_ways_database
43
+ FileUtils.remove_entry ways_database_path if File.exists?(ways_database_path)
44
+ end
45
+
46
+ def junctions_database
47
+ @junctions_database ||= LevelDBNative::DB.make junctions_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
48
+ end
49
+
50
+ def close_junctions_database
51
+ junctions_database.close!
52
+ end
53
+
54
+ def delete_junctions_database
55
+ FileUtils.remove_entry junctions_database_path if File.exists?(junctions_database_path)
56
+ end
57
+
58
+ def physical_roads_database
59
+ @physical_roads_database ||= LevelDBNative::DB.make physical_roads_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
60
+ end
61
+
62
+ def close_physical_roads_database
63
+ physical_roads_database.close!
64
+ end
65
+
66
+ def delete_physical_roads_database
67
+ FileUtils.remove_entry physical_roads_database_path if File.exists?(physical_roads_database_path)
68
+ end
69
+
70
+ def display_time(time_difference)
71
+ Time.at(time_difference.to_i).utc.strftime "%H:%M:%S"
72
+ end
73
+
74
+ def import
75
+ delete_nodes_database
76
+ delete_ways_database
77
+ delete_junctions_database
78
+ delete_physical_roads_database
79
+
80
+ leveldb_import
81
+ postgres_import
82
+
83
+ close_nodes_database
84
+ close_ways_database
85
+ close_junctions_database
86
+ close_physical_roads_database
87
+ end
88
+
89
+ def leveldb_import
90
+ # Save nodes in temporary file
91
+ backup_nodes
92
+ # Update nodes with ways in temporary file
93
+ update_nodes_with_way
94
+ # Save ways in temporary file
95
+ backup_ways
96
+ end
97
+
98
+ def postgres_import
99
+ # Save nodes in junctions
100
+ iterate_nodes
101
+
102
+ # Save relations in boundary
103
+ backup_relations_pgsql if split_ways
104
+
105
+ # Save ways in physical roads
106
+ iterate_ways
107
+
108
+ save_junctions_and_physical_roads_temporary
109
+ save_physical_road_conditionnal_costs_and_junctions
110
+
111
+ # Split and affect boundary to each way
112
+ split_way_with_boundaries if split_ways
113
+
114
+ # Save logical roads from physical roads
115
+ backup_logical_roads_pgsql if split_ways
116
+ end
117
+
118
+ def backup_nodes
119
+ Rails.logger.info "Begin to backup nodes in LevelDB nodes_database in #{nodes_database_path}"
120
+ start = Time.now
121
+ nodes_parser = ::PbfParser.new(pbf_file)
122
+ nodes_counter = 0
123
+ nodes_hash = {}
124
+
125
+ # Process the file until it finds any node
126
+ nodes_parser.next until nodes_parser.nodes.any?
127
+
128
+ until nodes_parser.nodes.empty?
129
+ nodes_database.batch do |batch|
130
+ last_node = nodes_parser.nodes.last
131
+ nodes_parser.nodes.each do |node|
132
+ nodes_counter+= 1
133
+
134
+ select_tags = selected_tags(node[:tags], @@nodes_selected_tags_keys)
135
+ batch[ node[:id].to_s ] = Marshal.dump(Node.new(node[:id].to_s, node[:lon], node[:lat], select_tags["addr:housenumber"], [], false, select_tags))
136
+ end
137
+ end
138
+ # When there's no more fileblocks to parse, #next returns false
139
+ # This avoids an infinit loop when the last fileblock still contains ways
140
+ break unless nodes_parser.next
141
+ end
142
+ Rails.logger.info "Finish to backup #{nodes_counter} nodes in LevelDB nodes_database in #{display_time(Time.now - start)} seconds"
143
+ end
144
+
145
+ def update_nodes_with_way
146
+ Rails.logger.info "Update way in nodes in LevelDB"
147
+ start = Time.now
148
+ ways_parser = ::PbfParser.new(pbf_file)
149
+ ways_counter = 0
150
+
151
+ # Process the file until it finds any way.
152
+ ways_parser.next until ways_parser.ways.any?
153
+
154
+ # Once it found at least one way, iterate to find the remaining ways.
155
+ until ways_parser.ways.empty?
156
+ nodes_readed = {}
157
+ nodes_database.batch do |batch|
158
+ ways_parser.ways.each do |way|
159
+ way_id = way[:id].to_s
160
+
161
+ if way.key?(:tags) && required_way?(@@way_for_physical_road_required_tags_keys, way[:tags])
162
+ # Don't add way to nodes if a way is a boundary
163
+ select_tags = selected_tags(way[:tags], @@way_selected_tags_keys)
164
+ node_ids = way.key?(:refs) ? way[:refs].collect(&:to_s) : []
165
+
166
+ if node_ids.present? && node_ids.size > 1
167
+ ways_counter+= 1
168
+ node_ids.each do |node_id|
169
+ if nodes_readed.has_key?(node_id)
170
+ node = nodes_readed[node_id]
171
+ else
172
+ node = Marshal.load(nodes_database[node_id])
173
+ end
174
+ node.add_way(way_id)
175
+ node.end_of_way = true if [node_ids.first, node_ids.last].include?(node_id)
176
+ nodes_readed[node_id] = node
177
+ end
178
+ end
179
+ end
180
+ end
181
+ nodes_readed.each_pair do |node_readed_id, node_readed|
182
+ batch[node_readed_id] = Marshal.dump(node_readed)
183
+ end
184
+ end
185
+
186
+ # When there's no more fileblocks to parse, #next returns false
187
+ # This avoids an infinit loop when the last fileblock still contains ways
188
+ break unless ways_parser.next
189
+ end
190
+
191
+ Rails.logger.info "Finish to update #{ways_counter} ways in nodes in LevelDB in #{display_time(Time.now - start)} seconds"
192
+ end
193
+
194
+ # def update_node_with_way(way_id, node_ids)
195
+ # # Update node data with way id
196
+ # node_ids.each do |node_id|
197
+ # node = Marshal.load(nodes_database[node_id])
198
+ # node.add_way(way_id)
199
+ # node.end_of_way = true if [node_ids.first, node_ids.last].include?(node_id)
200
+ # nodes_database[node_id] = Marshal.dump(node)
201
+ # end
202
+ # end
203
+
204
+ def backup_ways
205
+ Rails.logger.info "Begin to backup ways in LevelDB"
206
+ start = Time.now
207
+ ways_parser = ::PbfParser.new(pbf_file)
208
+ ways_counter = 0
209
+
210
+ # Process the file until it finds any way.
211
+ ways_parser.next until ways_parser.ways.any?
212
+
213
+ # Once it found at least one way, iterate to find the remaining ways.
214
+ until ways_parser.ways.empty?
215
+ ways_database.batch do |batch|
216
+ ways_parser.ways.each do |way|
217
+ way_id = way[:id].to_s
218
+
219
+ if way.key?(:tags) && required_way?(@@way_required_tags_keys, way[:tags])
220
+ select_tags = selected_tags(way[:tags], @@way_selected_tags_keys)
221
+ opt_tags = selected_tags(way[:tags], @@way_optionnal_tags_keys)
222
+ node_ids = way.key?(:refs) ? way[:refs].collect(&:to_s) : []
223
+
224
+ way = Way.new( way_id, node_ids, car?(opt_tags), bike?(opt_tags), train?(opt_tags), pedestrian?(opt_tags), select_tags["name"], select_tags["maxspeed"], select_tags["oneway"], select_tags["boundary"], select_tags["admin_level"], select_tags["addr:housenumber"], opt_tags )
225
+
226
+ ways_splitted = (way.boundary.present? || way.addr_housenumber.present?) ? [way] : split_way_with_nodes(way) # Don't split boundary and adress way
227
+
228
+ ways_splitted.each do |way_splitted|
229
+ ways_counter+= 1
230
+ batch[ way_splitted.id ] = Marshal.dump( way_splitted )
231
+ end
232
+ end
233
+ end
234
+ end
235
+
236
+ # When there's no more fileblocks to parse, #next returns false
237
+ # This avoids an infinit loop when the last fileblock still contains ways
238
+ break unless ways_parser.next
239
+ end
240
+
241
+ Rails.logger.info "Finish to backup #{ways_counter} ways in LevelDB in #{display_time(Time.now - start)} seconds"
242
+ end
243
+
244
+ def split_way_with_nodes(way)
245
+ nodes_used = []
246
+ nodes = []
247
+ # Get nodes really used and all nodes (used and for geometry need) for a way
248
+ way.nodes.each_with_index do |node_id, index|
249
+ node = Marshal.load( nodes_database[node_id.to_s] )
250
+ nodes << node
251
+ nodes_used << index if node.used?
252
+ end
253
+
254
+ ways_nodes = []
255
+ # Split way between each nodes used
256
+ if split_ways
257
+ nodes_used.each_with_index do |before_node, index|
258
+ ways_nodes << nodes.values_at(before_node..nodes_used[ index + 1]) if before_node != nodes_used.last
259
+ end
260
+ else
261
+ ways_nodes = [nodes]
262
+ end
263
+
264
+ ways_splitted = []
265
+ ways_nodes.each_with_index do |way_nodes, index|
266
+ way_tags = way.options.dup
267
+ way_tags["first_node_id"] = way_nodes.first.id
268
+ way_tags["last_node_id"] = way_nodes.last.id
269
+
270
+ # Don't add way if node_ids contains less than 2 nodes
271
+ if way_nodes.present? && way_nodes.size > 1
272
+ ways_splitted << Way.new( way.id + "-#{index}", way_nodes.collect(&:id), way.car, way.bike, way.train, way.pedestrian, way.name, way.maxspeed, way.oneway, way.boundary, way.admin_level, way.addr_housenumber, way_tags )
273
+ end
274
+ end
275
+
276
+ ways_splitted
277
+ end
278
+
279
+ def iterate_nodes
280
+ Rails.logger.debug "Begin to backup nodes in PostgreSql"
281
+
282
+ start = Time.now
283
+ nodes_counter = street_numbers_counter = 0
284
+ junctions_values = []
285
+ street_numbers_values = []
286
+ nodes_database_size = nodes_database.count
287
+
288
+ # traverse records by iterator
289
+ junction_columns = ["objectid", "geometry", "created_at", "updated_at"]
290
+ street_number_columns = ["objectid", "geometry", "number", "tags", "created_at", "updated_at"]
291
+
292
+ CSV.open("/tmp/junctions.csv", "wb:UTF-8") do |junctions_csv|
293
+ CSV.open("/tmp/street_numbers.csv", "wb:UTF-8") do |street_numbers_csv|
294
+ junctions_csv << junction_columns
295
+ street_numbers_csv << street_number_columns
296
+
297
+ nodes_database.each { |key, value|
298
+ node = Marshal.load(value)
299
+ geometry = GeoRuby::SimpleFeatures::Point.from_x_y( node.lon, node.lat, 4326) if( node.lon && node.lat )
300
+
301
+ if node.ways.present? && (node.ways.count >= 2 || node.end_of_way == true ) # Take node with at least two ways or at the end of a way
302
+ nodes_counter += 1
303
+ junctions_csv << [ node.id, geometry.as_hex_ewkb, Time.now, Time.now ]
304
+ end
305
+
306
+ if node.addr_housenumber.present?
307
+ street_numbers_counter += 1
308
+ street_numbers_csv << [ node.id, geometry.as_hex_ewkb, node.addr_housenumber, "#{node.tags.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
309
+ end
310
+
311
+ }
312
+ end
313
+ end
314
+
315
+ ActiveRoad::Junction.transaction do
316
+ ActiveRoad::Junction.pg_copy_from "/tmp/junctions.csv"
317
+ end
318
+
319
+ ActiveRoad::StreetNumber.transaction do
320
+ ActiveRoad::StreetNumber.pg_copy_from "/tmp/street_numbers.csv"
321
+ end
322
+
323
+ Rails.logger.info "Finish to backup #{nodes_counter} nodes and #{street_numbers_counter} street_numbres in PostgreSql in #{display_time(Time.now - start)} seconds"
324
+ end
325
+
326
+ def iterate_ways
327
+ Rails.logger.info "Begin to backup ways in PostgreSql"
328
+ start = Time.now
329
+
330
+ ways_counter = 0
331
+ street_numbers_counter = 0
332
+ ways_database_size = ways_database.count
333
+
334
+ # traverse records by iterator
335
+ physical_road_columns = ["objectid", "car", "bike", "train", "pedestrian", "name", "geometry", "boundary_id", "tags", "created_at", "updated_at"]
336
+ street_number_columns = ["objectid", "geometry", "number", "tags", "created_at", "updated_at"]
337
+
338
+ CSV.open("/tmp/physical_roads.csv", "wb:UTF-8") do |physical_roads_csv|
339
+ CSV.open("/tmp/street_numbers2.csv", "wb:UTF-8") do |street_numbers_csv|
340
+ physical_roads_csv << physical_road_columns
341
+ street_numbers_csv << street_number_columns
342
+
343
+ ways_database.each { |key, value|
344
+ way = Marshal.load(value)
345
+
346
+ unless way.boundary.present? # Use ways not used in relation for boundaries
347
+ nodes = []
348
+ way.nodes.each_with_index do |node_id, index|
349
+ node = Marshal.load( nodes_database[node_id.to_s] )
350
+ nodes << node
351
+ end
352
+ way_geometry = way_geometry(nodes)
353
+
354
+ if way.addr_housenumber.present? # If ways with adress
355
+ street_numbers_counter += 1
356
+ street_numbers_csv << [ way.id, way_geometry.envelope.center.as_hex_ewkb, way.addr_housenumber, "#{way.options.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
357
+ else
358
+ ways_counter += 1
359
+ way_boundary = way.boundary.present? ? way.boundary.to_i : nil
360
+ physical_roads_csv << [ way.id, way.car, way.bike, way.train, way.pedestrian, way.name, way_geometry.as_hex_ewkb, way_boundary, "#{way.options.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
361
+ end
362
+ end
363
+ }
364
+ end
365
+ end
366
+
367
+ # Save physical roads
368
+ ActiveRoad::PhysicalRoad.transaction do
369
+ ActiveRoad::PhysicalRoad.pg_copy_from "/tmp/physical_roads.csv"
370
+ end
371
+
372
+ ActiveRoad::StreetNumber.transaction do
373
+ ActiveRoad::StreetNumber.pg_copy_from "/tmp/street_numbers2.csv"
374
+ end
375
+
376
+ Rails.logger.info "Finish to backup #{ways_counter} ways and #{street_numbers_counter} street numbers in PostgreSql in #{display_time(Time.now - start)} seconds"
377
+ end
378
+
379
+ def save_junctions_and_physical_roads_temporary
380
+ Rails.logger.info "Begin to backup physical_roads and junctions in LevelDb"
381
+
382
+ start = Time.now
383
+ junctions_database.batch do |batch|
384
+ ActiveRoad::Junction.select("id,objectid").find_each do |junction|
385
+ junctions_database[junction.objectid] = junction.id.to_s
386
+ end
387
+ end
388
+
389
+ physical_roads_database.batch do |batch|
390
+ ActiveRoad::PhysicalRoad.select("id,objectid").find_each do |physical_road|
391
+ physical_roads_database[physical_road.objectid] = physical_road.id.to_s
392
+ end
393
+ end
394
+
395
+ Rails.logger.info "Finish to backup physical_roads and junctions in LevelDb in #{display_time(Time.now - start)} seconds"
396
+ end
397
+
398
+ def save_physical_road_conditionnal_costs_and_junctions
399
+ Rails.logger.info "Begin to backup ways in PostgreSql"
400
+
401
+ start = Time.now
402
+ physical_road_conditionnal_costs_counter = junctions_physical_roads_counter = 0
403
+ physical_road_conditionnal_cost_columns = ["tags", "cost", "physical_road_id"]
404
+ junction_physical_road_columns = ["physical_road_id", "junction_id"]
405
+
406
+ CSV.open("/tmp/physical_road_conditionnal_costs.csv", "wb:UTF-8") do |physical_road_conditionnal_costs_csv|
407
+ CSV.open("/tmp/junctions_physical_roads.csv", "wb:UTF-8") do |junctions_physical_roads_csv|
408
+ physical_road_conditionnal_costs_csv << physical_road_conditionnal_cost_columns
409
+ junctions_physical_roads_csv << junction_physical_road_columns
410
+
411
+ ways_database.each { |key, value|
412
+ way = Marshal.load(value)
413
+
414
+ # Save physical road conditionnal cost not for boundaries or street numbers
415
+ unless way.boundary.present? || way.addr_housenumber.present?
416
+ way_conditionnal_costs = physical_road_conditionnal_costs(way)
417
+ way_conditionnal_costs.each do |way_conditionnal_cost|
418
+ physical_road_conditionnal_costs_counter += 1
419
+ physical_road_conditionnal_costs_csv << way_conditionnal_cost + [ physical_roads_database[way.id] ]
420
+ end
421
+
422
+ way.nodes.each do |node_id|
423
+ junction_id = junctions_database[node_id]
424
+ junctions_physical_roads_counter += 1
425
+ junctions_physical_roads_csv << [ physical_roads_database[way.id], junction_id ] if junction_id.present?
426
+ end
427
+ end
428
+ }
429
+ end
430
+ end
431
+
432
+ # Save physical road conditionnal costs
433
+ ActiveRoad::PhysicalRoadConditionnalCost.transaction do
434
+ ActiveRoad::PhysicalRoadConditionnalCost.pg_copy_from "/tmp/physical_road_conditionnal_costs.csv"
435
+ end
436
+
437
+ # Save physical road and junctions link
438
+ ActiveRoad::JunctionsPhysicalRoad.transaction do
439
+ ActiveRoad::JunctionsPhysicalRoad.pg_copy_from "/tmp/junctions_physical_roads.csv"
440
+ end
441
+
442
+ Rails.logger.info "Finish to backup #{junctions_physical_roads_counter} junctions_physical_roads and #{physical_road_conditionnal_costs_counter} physical_road_conditionnal_costs in PostgreSql in #{display_time(Time.now - start)} seconds"
443
+ end
444
+
445
+ def split_way_with_boundaries
446
+ Rails.logger.info "Begin to split and affect boundaries to ways in PostgreSql"
447
+ start = Time.now
448
+
449
+ # Update physical roads entirely contains in boundaries
450
+ ActiveRoad::PhysicalRoad.connection.select_all("SELECT physical_road.id AS physical_road_id, boundary.id AS boundary_id FROM physical_roads physical_road, boundaries boundary WHERE ST_Covers( boundary.geometry, physical_road.geometry)").each_slice(@@pg_batch_size) do |group|
451
+ ActiveRoad::PhysicalRoad.transaction do
452
+ group.each do |element|
453
+ ActiveRoad::PhysicalRoad.update(element["physical_road_id"], :boundary_id => element["boundary_id"])
454
+ end
455
+ end
456
+ end
457
+
458
+ if split_ways
459
+ simple_ways = []
460
+ simple_ways_not_line_string = 0
461
+
462
+ # Fix : Produce 2 ways when way is tangent to boundary borders for each boundary
463
+ # Get geometries in boundary
464
+ sql = "SELECT b.id AS boundary_id, p.id AS physical_road_id, p.objectid AS physical_road_objectid, p.tags AS physical_road_tags, ST_AsText(p.geometry) AS physical_road_geometry,
465
+ j1.objectid AS departure_objectid, ST_AsText(j1.geometry) AS departure_geometry,
466
+ j2.objectid AS arrival_objectid, ST_AsText(j2.geometry) AS arrival_geometry,
467
+ ST_AsText( (ST_Dump(ST_Intersection( p.geometry , b.geometry))).geom ) AS intersection_geometry
468
+ FROM physical_roads p, boundaries b, junctions j1, junctions j2, junctions_physical_roads jp, junctions_physical_roads jp2
469
+ WHERE p.boundary_id IS NULL AND ST_Crosses( b.geometry, p.geometry)
470
+ AND j1.id = jp.junction_id AND p.id = jp.physical_road_id AND ST_Equals(ST_StartPoint(p.geometry), j1.geometry)
471
+ AND j2.id = jp2.junction_id AND p.id = jp2.physical_road_id AND ST_Equals(ST_EndPoint(p.geometry), j2.geometry)".gsub(/^( |\t)+/, "")
472
+ ActiveRoad::PhysicalRoad.connection.select_all( sql ).each do |result|
473
+ intersection_geometry = GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['intersection_geometry']}")
474
+
475
+ # Not take in consideration point intersection!!
476
+ if intersection_geometry.class == GeoRuby::SimpleFeatures::LineString
477
+ simple_way = SimpleWay.new(result["boundary_id"], result["physical_road_id"], result["physical_road_objectid"], result["physical_road_tags"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['physical_road_geometry']}"), result["departure_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['departure_geometry']}"), result["arrival_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['arrival_geometry']}"), intersection_geometry )
478
+ # Delete boucle line string Ex : 9938647-4
479
+ simple_ways << simple_way if simple_way.departure != simple_way.arrival
480
+ else
481
+ simple_ways_not_line_string += 1
482
+ end
483
+ end
484
+
485
+ # Get geometries not in boundaries
486
+ sql = "SELECT ST_AsText( (ST_Dump(difference_geometry)).geom ) AS difference_geometry, v.id AS physical_road_id, v.objectid AS physical_road_objectid, v.tags AS physical_road_tags, ST_AsText(v.geometry) AS physical_road_geometry,
487
+ j1.objectid AS departure_objectid, ST_AsText(j1.geometry) AS departure_geometry,
488
+ j2.objectid AS arrival_objectid, ST_AsText(j2.geometry) AS arrival_geometry
489
+ FROM
490
+ ( SELECT pr.id, pr.objectid, pr.tags, pr.geometry, pr.boundary_id, ST_Difference( pr.geometry, ST_Union( b.geometry)) as difference_geometry
491
+ FROM physical_roads pr, boundaries b
492
+ WHERE pr.boundary_id IS NULL AND ST_Crosses( b.geometry, pr.geometry)
493
+ GROUP BY pr.id, pr.geometry) v,
494
+ junctions j1, junctions j2, junctions_physical_roads jp, junctions_physical_roads jp2
495
+ WHERE j1.id = jp.junction_id AND v.id = jp.physical_road_id AND ST_Equals(ST_StartPoint(v.geometry), j1.geometry)
496
+ AND j2.id = jp2.junction_id AND v.id = jp2.physical_road_id AND ST_Equals(ST_EndPoint(v.geometry), j2.geometry)
497
+ AND NOT ST_IsEmpty(difference_geometry)".gsub(/^( |\t)+/, "")
498
+ ActiveRoad::PhysicalRoad.connection.select_all( sql ).each do |result|
499
+ difference_geometry = GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['difference_geometry']}")
500
+ if difference_geometry.class == GeoRuby::SimpleFeatures::LineString
501
+ simple_way = SimpleWay.new(nil, result["physical_road_id"], result["physical_road_objectid"], result["physical_road_tags"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['physical_road_geometry']}"), result["departure_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['departure_geometry']}"), result["arrival_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['arrival_geometry']}"), difference_geometry )
502
+ # Delete boucle line string Ex : 9938647-4
503
+ simple_ways << simple_way if simple_way.departure != simple_way.arrival
504
+ else
505
+ simple_ways_not_line_string += 1
506
+ end
507
+ end
508
+
509
+ # Prepare reordering ways
510
+ simple_ways_by_old_physical_road_id = simple_ways.group_by{|sw| sw.old_physical_road_id}
511
+
512
+ # Hack : in the code we take the first one which has an intersection point and it deletes
513
+ # dual segment tangent on the boundary borders
514
+ simple_ways_by_old_physical_road_id.each do |old_physical_road_id, ways|
515
+ ways.each do |way|
516
+ if way.departure == way.old_departure_geometry
517
+ way.departure_objectid = way.old_departure_objectid
518
+ way.previous = nil
519
+ else
520
+ way.departure_objectid = way.default_departure_objectid
521
+ way.previous = ways.detect{ |select_way| select_way.arrival == way.departure }
522
+ end
523
+
524
+ if way.arrival == way.old_arrival_geometry
525
+ way.arrival_objectid = way.old_arrival_objectid
526
+ way.next = nil
527
+ else
528
+ way.arrival_objectid = way.default_arrival_objectid
529
+ way.next = ways.detect{ |select_way| select_way.departure == way.arrival }
530
+ end
531
+ end
532
+ end
533
+
534
+ # Save new ways and junctions
535
+ #physical_roads ||= ActiveRoad::PhysicalRoad.where(:objectid => simple_ways_by_old_physical_road_id.keys).includes(:conditionnal_costs)
536
+
537
+ simple_ways_by_old_physical_road_id.each_slice(1000) { |group|
538
+ ActiveRoad::PhysicalRoad.transaction do
539
+
540
+ group.each do |old_physical_road_id, ways|
541
+ #puts ways.sort.inspect
542
+ next_way = ways.detect{ |select_way| select_way.previous == nil }
543
+ way_counter = 0
544
+ junction_counter = 0
545
+
546
+
547
+ while next_way != nil
548
+ start = Time.now
549
+
550
+ #old_physical_road = physical_roads.where(:id => old_physical_road_id)
551
+ #physical_road.conditionnal_costs = old_physical_road.conditionnal_costs
552
+
553
+ # Create departure
554
+ if next_way.previous != nil
555
+ departure = ActiveRoad::Junction.where(:objectid => "#{next_way.departure_objectid}-#{junction_counter}").first_or_create( :geometry => next_way.departure )
556
+ junction_counter += 1
557
+ else
558
+ departure = ActiveRoad::Junction.find_by_objectid(next_way.departure_objectid)
559
+ end
560
+
561
+ # Create arrival
562
+ if next_way.next != nil
563
+ arrival = ActiveRoad::Junction.where(:objectid => "#{next_way.arrival_objectid}-#{junction_counter}").first_or_create( :geometry => next_way.arrival )
564
+ else
565
+ arrival = ActiveRoad::Junction.find_by_objectid(next_way.arrival_objectid)
566
+ end
567
+
568
+ old_physical_road_tags = next_way.old_physical_road_tags_hash
569
+ old_physical_road_tags["first_node_id"] = departure.objectid
570
+ old_physical_road_tags["last_node_id"] = arrival.objectid
571
+
572
+ physical_road = ActiveRoad::PhysicalRoad.create! :objectid => "#{next_way.old_physical_road_objectid}-#{way_counter}", :boundary_id => next_way.boundary_id, :geometry => next_way.geometry, :tags => old_physical_road_tags
573
+
574
+ # Add departure and arrival to physical road
575
+ physical_road.junctions << [departure, arrival]
576
+
577
+ way_counter += 1
578
+
579
+ if way_counter > ways.size
580
+ Rails.logger.error "Infinite boucle when save physical road splitted with boundaries"
581
+ raise Exception.new "Infinite boucle when save physical road splitted with boundaries"
582
+ end
583
+
584
+ next_way = next_way.next
585
+ end
586
+
587
+ end
588
+ end
589
+ }
590
+
591
+ # Delete old ways
592
+ ActiveRoad::PhysicalRoad.destroy(simple_ways_by_old_physical_road_id.keys)
593
+ end
594
+
595
+ Rails.logger.info "Finish to split and affect boundaries to ways in PostgreSql in #{display_time(Time.now - start)} seconds"
596
+ end
597
+
598
+ class SimpleWay
599
+ include Comparable
600
+ attr_accessor :boundary_id, :old_physical_road_id, :old_physical_road_objectid, :old_physical_road_tags, :old_physical_road_geometry, :old_departure_objectid, :old_departure_geometry, :old_arrival_objectid, :old_arrival_geometry, :departure_objectid, :arrival_objectid, :geometry, :next, :previous
601
+
602
+ def initialize(boundary_id, old_physical_road_id, old_physical_road_objectid, old_physical_road_tags, old_physical_road_geometry, old_departure_objectid, old_departure_geometry, old_arrival_objectid, old_arrival_geometry, geometry)
603
+ @boundary_id = boundary_id
604
+ @old_physical_road_id = old_physical_road_id
605
+ @old_physical_road_objectid = old_physical_road_objectid
606
+ @old_physical_road_tags = old_physical_road_tags || ""
607
+ @old_physical_road_geometry = old_physical_road_geometry
608
+ @old_departure_objectid = old_departure_objectid
609
+ @old_departure_geometry = old_departure_geometry
610
+ @old_arrival_objectid = old_arrival_objectid
611
+ @old_arrival_geometry = old_arrival_geometry
612
+ @geometry = geometry
613
+ end
614
+
615
+ def old_physical_road_tags_hash
616
+ #Fix tags build from string
617
+ tags = {}.tap do |tags|
618
+ old_physical_road_tags.split(',').each do |pair|
619
+ key, value = pair.split("=>")
620
+ tags[key.gsub(/\W/, "")] = value.gsub(/\W/, "")
621
+ end
622
+ end
623
+ end
624
+
625
+ def departure
626
+ #puts "geometry class #{geometry.class}, value #{geometry.inspect}"
627
+ geometry.points.first if geometry
628
+ end
629
+
630
+ def arrival
631
+ geometry.points.last if geometry
632
+ end
633
+
634
+ def default_departure_objectid
635
+ "#{old_departure_objectid}-#{old_arrival_objectid}"
636
+ end
637
+
638
+ def default_arrival_objectid
639
+ "#{old_departure_objectid}-#{old_arrival_objectid}"
640
+ end
641
+
642
+ def <=>(another)
643
+ # puts "self : #{self.departure.inspect}, #{self.arrival.inspect}"
644
+ # puts "another : #{another.departure.inspect}, #{another.arrival.inspect}"
645
+ # puts old_physical_road_geometry.points.inspect
646
+ # puts old_physical_road_geometry.points.index(another.arrival).inspect
647
+ # puts old_physical_road_geometry.points.index(self.departure).inspect
648
+ if self.departure == another.arrival || old_physical_road_geometry.points.index(another.arrival) < old_physical_road_geometry.points.index(self.departure)
649
+ 1
650
+ elsif self.arrival == another.departure || old_physical_road_geometry.points.index(self.arrival) < old_physical_road_geometry.points.index(another.departure)
651
+ -1
652
+ else
653
+ nil
654
+ end
655
+ end
656
+
657
+ end
658
+
659
+ def way_geometry(nodes)
660
+ points = []
661
+ nodes.each do |node|
662
+ points << GeoRuby::SimpleFeatures::Point.from_x_y(node.lon, node.lat, 4326)
663
+ end
664
+
665
+ GeoRuby::SimpleFeatures::LineString.from_points(points, 4326) if points.present? && 1 < points.count
666
+ end
667
+
668
+ def find_boundary(way_geometry)
669
+ ActiveRoad::Boundary.first_contains(way_geometry)
670
+ end
671
+
672
+ def backup_relations_pgsql
673
+ Rails.logger.info "Begin to backup relations in PostgreSql"
674
+ start = Time.now
675
+ relations_parser = ::PbfParser.new(pbf_file)
676
+ boundaries_counter = 0
677
+
678
+ # traverse records by iterator
679
+ boundary_columns = ["objectid", "geometry", "name", "admin_level", "postal_code", "insee_code"]
680
+
681
+ # Process the file until it finds any relation.
682
+ relations_parser.next until relations_parser.relations.any?
683
+
684
+ # Once it found at least one relation, iterate to find the remaining relations.
685
+ CSV.open("/tmp/boundaries.csv", "wb:UTF-8") do |boundary_csv|
686
+ boundary_csv << boundary_columns
687
+
688
+ until relations_parser.relations.empty?
689
+ relations_parser.relations.each do |relation|
690
+
691
+ if relation.key?(:tags) && required_relation?(relation[:tags])
692
+ tags = selected_tags(relation[:tags], @@relation_selected_tags_keys)
693
+
694
+ # Use tags["admin_level"] == "8" because catholic boundaries exist!!
695
+ if tags["admin_level"] == "8" && tags["boundary"] == "administrative"
696
+ boundaries_counter += 1
697
+ outer_ways = {}
698
+ inner_ways = {}
699
+
700
+ begin
701
+ relation[:members][:ways].each do |member_way|
702
+ way_data = ways_database[ member_way[:id].to_s ]
703
+ way = nil
704
+ nodes = []
705
+
706
+ if way_data.present?
707
+ way = Marshal.load(way_data)
708
+ way.nodes.each do |node_id|
709
+ node = Marshal.load( nodes_database[node_id.to_s] )
710
+ nodes << node
711
+ end
712
+ else
713
+ raise StandardError, "Geometry error : impossible to find way #{member_way[:id]} for relation #{tags["name"]} with id #{relation[:id]}"
714
+ end
715
+
716
+ if member_way[:role] == "inner"
717
+ inner_ways[ member_way[:id] ] = way_geometry(nodes)
718
+ elsif member_way[:role] == "outer"
719
+ outer_ways[ member_way[:id] ] = way_geometry(nodes)
720
+ else # Fix : lot of boundaries have no tags role
721
+ outer_ways[ member_way[:id] ] = way_geometry(nodes)
722
+ end
723
+ end
724
+
725
+ boundary_polygons = extract_relation_polygon(outer_ways.values, inner_ways.values)
726
+
727
+ if boundary_polygons.present?
728
+ boundary_geometry = GeoRuby::SimpleFeatures::MultiPolygon.from_polygons( boundary_polygons ).as_hex_ewkb
729
+
730
+ boundary_csv << [ relation[:id], boundary_geometry, tags["name"], tags["admin_level"], tags["addr:postcode"], tags["ref:INSEE"] ]
731
+ end
732
+ rescue StandardError => e
733
+ Rails.logger.error "Geometry error : impossible to build polygon for relation #{tags["name"]} with id #{relation[:id]} : #{e.message}"
734
+ end
735
+ end
736
+ end
737
+ end
738
+
739
+ # When there's no more fileblocks to parse, #next returns false
740
+ # This avoids an infinit loop when the last fileblock still contains relations
741
+ break unless relations_parser.next
742
+ end
743
+ end
744
+
745
+ ActiveRoad::Boundary.transaction do
746
+ ActiveRoad::Boundary.pg_copy_from "/tmp/boundaries.csv"
747
+ end
748
+
749
+ Rails.logger.info "Finish to backup #{boundaries_counter} boundaries in PostgreSql in #{display_time(Time.now - start)} seconds"
750
+ end
751
+
752
+ def backup_logical_roads_pgsql
753
+ Rails.logger.info "Begin to backup logical roads in PostgreSql"
754
+ start = Time.now
755
+ logical_roads_counter = 0
756
+
757
+ saved_name = nil
758
+ saved_boundary = nil
759
+ saved_logical_road = nil
760
+ ActiveRoad::PhysicalRoad.where("physical_roads.name IS NOT NULL OR physical_roads.boundary_id IS NOT NULL").select("name,boundary_id,id").order(:boundary_id,:name).find_in_batches(batch_size: 2000) do |group|
761
+ ActiveRoad::LogicalRoad.transaction do
762
+ group.each do |physical_road|
763
+ not_same_name = (saved_name != physical_road.name)
764
+ not_same_boundary = (saved_boundary != physical_road.boundary_id)
765
+
766
+ saved_name = physical_road.name if not_same_name
767
+ saved_boundary = physical_road.boundary_id if not_same_boundary
768
+
769
+ if not_same_name || not_same_boundary
770
+ logical_roads_counter += 1
771
+ saved_logical_road = ActiveRoad::LogicalRoad.create(:name => saved_name, :boundary_id => saved_boundary)
772
+ end
773
+
774
+ physical_road.update_column(:logical_road_id, saved_logical_road.id) if saved_logical_road.present?
775
+ end
776
+ end
777
+ end
778
+
779
+ Rails.logger.info "Finish to backup #{logical_roads_counter} logical roads in PostgreSql in #{ display_time(Time.now - start)} seconds"
780
+ end
781
+
782
+
783
+ end
784
+ end