overture_maps 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +29 -0
- data/LICENSE.txt +21 -0
- data/README.md +416 -0
- data/app/controllers/overture_maps/application_controller.rb +28 -0
- data/app/controllers/overture_maps/attribution_controller.rb +15 -0
- data/app/controllers/overture_maps/features_controller.rb +125 -0
- data/app/controllers/overture_maps/search_controller.rb +31 -0
- data/app/controllers/overture_maps/tiles_controller.rb +85 -0
- data/config/routes.rb +13 -0
- data/exe/overture-maps-mcp +6 -0
- data/lib/generators/overture_maps/address_generator.rb +19 -0
- data/lib/generators/overture_maps/base_features_generator.rb +19 -0
- data/lib/generators/overture_maps/base_generator.rb +22 -0
- data/lib/generators/overture_maps/building_generator.rb +19 -0
- data/lib/generators/overture_maps/division_generator.rb +19 -0
- data/lib/generators/overture_maps/install_generator.rb +42 -0
- data/lib/generators/overture_maps/place_generator.rb +19 -0
- data/lib/generators/overture_maps/transportation_generator.rb +20 -0
- data/lib/generators/templates/address_migration.rb.tt +25 -0
- data/lib/generators/templates/address_model.rb.tt +4 -0
- data/lib/generators/templates/base_feature_model.rb.tt +4 -0
- data/lib/generators/templates/base_features_migration.rb.tt +27 -0
- data/lib/generators/templates/building_migration.rb.tt +24 -0
- data/lib/generators/templates/building_model.rb.tt +4 -0
- data/lib/generators/templates/category_migration.rb.tt +15 -0
- data/lib/generators/templates/category_model.rb.tt +4 -0
- data/lib/generators/templates/connector_model.rb.tt +4 -0
- data/lib/generators/templates/division_migration.rb.tt +31 -0
- data/lib/generators/templates/division_model.rb.tt +4 -0
- data/lib/generators/templates/imported_areas_migration.rb.tt +20 -0
- data/lib/generators/templates/place_migration.rb.tt +30 -0
- data/lib/generators/templates/place_model.rb.tt +4 -0
- data/lib/generators/templates/postgis_migration.rb.tt +5 -0
- data/lib/generators/templates/segment_model.rb.tt +4 -0
- data/lib/generators/templates/transportation_migration.rb.tt +38 -0
- data/lib/overture_maps/attribution.rb +66 -0
- data/lib/overture_maps/bounding_box.rb +80 -0
- data/lib/overture_maps/changelog.rb +70 -0
- data/lib/overture_maps/configuration.rb +43 -0
- data/lib/overture_maps/database.rb +74 -0
- data/lib/overture_maps/division_search.rb +44 -0
- data/lib/overture_maps/engine.rb +32 -0
- data/lib/overture_maps/geometry_parser.rb +41 -0
- data/lib/overture_maps/gers.rb +30 -0
- data/lib/overture_maps/import/downloader.rb +248 -0
- data/lib/overture_maps/import/location_based_runner.rb +176 -0
- data/lib/overture_maps/import/parquet_reader.rb +32 -0
- data/lib/overture_maps/import/record_mapper.rb +182 -0
- data/lib/overture_maps/import/runner.rb +125 -0
- data/lib/overture_maps/import.rb +27 -0
- data/lib/overture_maps/mcp_server.rb +222 -0
- data/lib/overture_maps/models/address.rb +32 -0
- data/lib/overture_maps/models/base.rb +39 -0
- data/lib/overture_maps/models/base_feature.rb +31 -0
- data/lib/overture_maps/models/building.rb +35 -0
- data/lib/overture_maps/models/category.rb +32 -0
- data/lib/overture_maps/models/connector.rb +11 -0
- data/lib/overture_maps/models/division.rb +43 -0
- data/lib/overture_maps/models/imported_area.rb +42 -0
- data/lib/overture_maps/models/place.rb +43 -0
- data/lib/overture_maps/models/segment.rb +22 -0
- data/lib/overture_maps/models.rb +25 -0
- data/lib/overture_maps/query.rb +168 -0
- data/lib/overture_maps/query_engine.rb +268 -0
- data/lib/overture_maps/releases.rb +74 -0
- data/lib/overture_maps/storage.rb +121 -0
- data/lib/overture_maps/syncer.rb +99 -0
- data/lib/overture_maps/util.rb +30 -0
- data/lib/overture_maps/version.rb +3 -0
- data/lib/overture_maps.rb +42 -0
- data/lib/tasks/overture_maps.rake +554 -0
- metadata +301 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OvertureMaps
|
|
4
|
+
# A WGS84 bounding box. The single place that parses user-supplied coordinate
|
|
5
|
+
# strings, so import and download entry points cannot drift apart.
|
|
6
|
+
class BoundingBox
|
|
7
|
+
NUMBER = /-?\d+(?:\.\d+)?/
|
|
8
|
+
# Four numbers separated by commas, whitespace, or underscores, with an
|
|
9
|
+
# optional "|display name" suffix: "47.6,-122.3,47.7,-122.2" or
|
|
10
|
+
# "47.6_-122.3_47.7_-122.2|seattle"
|
|
11
|
+
COORDS = /\A(#{NUMBER})[,\s_]+(#{NUMBER})[,\s_]+(#{NUMBER})[,\s_]+(#{NUMBER})(?:\|(.+))?\z/
|
|
12
|
+
|
|
13
|
+
attr_reader :min_lat, :min_lng, :max_lat, :max_lng, :display_name
|
|
14
|
+
|
|
15
|
+
def initialize(lat1:, lng1:, lat2:, lng2:, display_name: nil)
|
|
16
|
+
@min_lat, @max_lat = [Float(lat1), Float(lat2)].minmax
|
|
17
|
+
@min_lng, @max_lng = [Float(lng1), Float(lng2)].minmax
|
|
18
|
+
@display_name = display_name
|
|
19
|
+
|
|
20
|
+
unless (-90..90).cover?(@min_lat) && (-90..90).cover?(@max_lat) &&
|
|
21
|
+
(-180..180).cover?(@min_lng) && (-180..180).cover?(@max_lng)
|
|
22
|
+
raise ArgumentError, "coordinates out of range: #{self}"
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Returns a BoundingBox if the string looks like coordinates, else nil.
|
|
27
|
+
def self.parse(string)
|
|
28
|
+
match = COORDS.match(string.to_s.strip)
|
|
29
|
+
return nil unless match
|
|
30
|
+
|
|
31
|
+
new(lat1: match[1], lng1: match[2], lat2: match[3], lng2: match[4], display_name: match[5])
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.coordinates?(string)
|
|
35
|
+
COORDS.match?(string.to_s.strip)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Overture bbox structs come back from division searches as
|
|
39
|
+
# {"xmin" => ..., "xmax" => ..., "ymin" => ..., "ymax" => ...}
|
|
40
|
+
def self.from_overture(bbox, display_name: nil)
|
|
41
|
+
new(
|
|
42
|
+
lat1: bbox["ymin"], lng1: bbox["xmin"],
|
|
43
|
+
lat2: bbox["ymax"], lng2: bbox["xmax"],
|
|
44
|
+
display_name: display_name
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def self.around(lat:, lng:, radius_meters:)
|
|
49
|
+
lat = Float(lat)
|
|
50
|
+
lng = Float(lng)
|
|
51
|
+
lat_delta = radius_meters.to_f / 111_000
|
|
52
|
+
lng_delta = radius_meters.to_f / (111_000 * Math.cos(lat * Math::PI / 180))
|
|
53
|
+
|
|
54
|
+
new(
|
|
55
|
+
lat1: [lat - lat_delta, -90].max, lng1: [lng - lng_delta, -180].max,
|
|
56
|
+
lat2: [lat + lat_delta, 90].min, lng2: [lng + lng_delta, 180].min
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def to_s
|
|
61
|
+
"#{min_lat},#{min_lng},#{max_lat},#{max_lng}"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Filename-safe label for cache files.
|
|
65
|
+
def slug
|
|
66
|
+
return sanitized_display_name if display_name
|
|
67
|
+
|
|
68
|
+
format("%.4f_%.4f_%.4f_%.4f", min_lat, min_lng, max_lat, max_lng)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# Filename-safe and idempotent: a slug that round-trips through
|
|
74
|
+
# display_name (e.g. from ImportedArea bookkeeping) must come back
|
|
75
|
+
# unchanged, so dots and dashes survive.
|
|
76
|
+
def sanitized_display_name
|
|
77
|
+
display_name.downcase.gsub(/[^a-z0-9.\-]+/, "_").gsub(/\A_+|_+\z/, "")
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OvertureMaps
|
|
4
|
+
# Overture's per-release data changelog: parquet partitioned by theme,
|
|
5
|
+
# type, and change_type (added | removed | data_changed | unchanged),
|
|
6
|
+
# with each row carrying the feature id and its bbox. The changelog for
|
|
7
|
+
# release R describes changes relative to the release before R.
|
|
8
|
+
module Changelog
|
|
9
|
+
CHANGE_TYPES = %w[added removed data_changed unchanged].freeze
|
|
10
|
+
|
|
11
|
+
class << self
|
|
12
|
+
# IDs removed in `release` (relative to the prior release), optionally
|
|
13
|
+
# restricted to those whose bbox intersects the given area.
|
|
14
|
+
def removed_ids(theme:, type:, release:, bbox: nil)
|
|
15
|
+
sql, params = build_query(theme: theme, type: type, release: release,
|
|
16
|
+
change_type: "removed", bbox: bbox, select: "id")
|
|
17
|
+
QueryEngine.instance.query(sql, params).map { |row| row["id"] }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Per-change_type counts for a release/theme/type (for sync:status and
|
|
21
|
+
# dry runs). Returns e.g. {"added" => 123, "removed" => 4, ...}.
|
|
22
|
+
def counts(theme:, type:, release:, bbox: nil)
|
|
23
|
+
validate!(theme, type)
|
|
24
|
+
source = source_glob(theme: theme, type: type, release: Releases.validate!(release),
|
|
25
|
+
change_type: "*")
|
|
26
|
+
sql = +"SELECT change_type, count(*) AS n FROM read_parquet('#{source}', hive_partitioning=1)"
|
|
27
|
+
params = []
|
|
28
|
+
if bbox
|
|
29
|
+
sql << " WHERE bbox.xmin <= ? AND bbox.xmax >= ? AND bbox.ymin <= ? AND bbox.ymax >= ?"
|
|
30
|
+
params = bbox_params(bbox)
|
|
31
|
+
end
|
|
32
|
+
sql << " GROUP BY change_type"
|
|
33
|
+
|
|
34
|
+
QueryEngine.instance.query(sql, params)
|
|
35
|
+
.to_h { |row| [row["change_type"], Integer(row["n"])] }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def build_query(theme:, type:, release:, change_type:, bbox:, select:)
|
|
41
|
+
validate!(theme, type)
|
|
42
|
+
raise ArgumentError, "unknown change_type: #{change_type}" unless CHANGE_TYPES.include?(change_type)
|
|
43
|
+
|
|
44
|
+
source = source_glob(theme: theme, type: type, release: Releases.validate!(release),
|
|
45
|
+
change_type: change_type)
|
|
46
|
+
sql = +"SELECT #{select} FROM read_parquet('#{source}', hive_partitioning=1)"
|
|
47
|
+
params = []
|
|
48
|
+
if bbox
|
|
49
|
+
sql << " WHERE bbox.xmin <= ? AND bbox.xmax >= ? AND bbox.ymin <= ? AND bbox.ymax >= ?"
|
|
50
|
+
params = bbox_params(bbox)
|
|
51
|
+
end
|
|
52
|
+
[sql, params]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def bbox_params(bbox)
|
|
56
|
+
[bbox.max_lng, bbox.min_lng, bbox.max_lat, bbox.min_lat]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def source_glob(theme:, type:, release:, change_type:)
|
|
60
|
+
base = OvertureMaps.configuration.s3_uri.chomp("/")
|
|
61
|
+
"#{base}/changelog/#{release}/theme=#{theme}/type=#{type}/change_type=#{change_type}/*.parquet"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def validate!(theme, type)
|
|
65
|
+
types = Import::Downloader::TYPES[theme] or raise ArgumentError, "unknown theme: #{theme}"
|
|
66
|
+
raise ArgumentError, "unknown type #{type} for #{theme}" unless types.include?(type)
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OvertureMaps
|
|
4
|
+
class Configuration
|
|
5
|
+
DEFAULT_S3_HTTP_URL = "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com"
|
|
6
|
+
DEFAULT_S3_URI = "s3://overturemaps-us-west-2"
|
|
7
|
+
DEFAULT_S3_REGION = "us-west-2"
|
|
8
|
+
|
|
9
|
+
# release: Overture release string (e.g. "2026-06-17.0"). nil means "latest available".
|
|
10
|
+
# cache_dir: where downloaded parquet extracts are stored.
|
|
11
|
+
# s3_http_url/s3_uri/s3_region: data source; point at a mirror (e.g. MinIO) to avoid
|
|
12
|
+
# hitting Overture's bucket from every host.
|
|
13
|
+
# batch_size: default import batch size.
|
|
14
|
+
# timeout: HTTP open/read timeout in seconds.
|
|
15
|
+
# non_interactive: never prompt; pick the best division match and prefer fresh downloads.
|
|
16
|
+
# duckdb_cli_path: explicit path to the duckdb binary (otherwise PATH, then auto-download).
|
|
17
|
+
# logger: receives progress output; defaults to stdout logging from the rake layer.
|
|
18
|
+
# api_auth: callable invoked with the controller before every API
|
|
19
|
+
# request; render/head inside it to reject (nil = open access).
|
|
20
|
+
# api_default_limit / api_max_limit: collection endpoint page sizing.
|
|
21
|
+
# tile_feature_limit: per-tile feature cap for the MVT endpoint.
|
|
22
|
+
attr_accessor :release, :cache_dir, :s3_http_url, :s3_uri, :s3_region,
|
|
23
|
+
:batch_size, :timeout, :non_interactive, :duckdb_cli_path, :logger,
|
|
24
|
+
:api_auth, :api_default_limit, :api_max_limit, :tile_feature_limit
|
|
25
|
+
|
|
26
|
+
def initialize
|
|
27
|
+
@release = nil
|
|
28
|
+
@cache_dir = "tmp/overture"
|
|
29
|
+
@s3_http_url = DEFAULT_S3_HTTP_URL
|
|
30
|
+
@s3_uri = DEFAULT_S3_URI
|
|
31
|
+
@s3_region = DEFAULT_S3_REGION
|
|
32
|
+
@batch_size = 1000
|
|
33
|
+
@timeout = 30
|
|
34
|
+
@non_interactive = !ENV["OVERTURE_NON_INTERACTIVE"].to_s.empty?
|
|
35
|
+
@duckdb_cli_path = ENV["OVERTURE_DUCKDB_CLI"]
|
|
36
|
+
@logger = nil
|
|
37
|
+
@api_auth = nil
|
|
38
|
+
@api_default_limit = 100
|
|
39
|
+
@api_max_limit = 1000
|
|
40
|
+
@tile_feature_limit = 20_000
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OvertureMaps
|
|
4
|
+
# PostGIS helpers. Identifiers are quoted and values sanitized — nothing
|
|
5
|
+
# user-supplied is interpolated into SQL.
|
|
6
|
+
module Database
|
|
7
|
+
class << self
|
|
8
|
+
def postgis_available?
|
|
9
|
+
connection.extension_enabled?("postgis")
|
|
10
|
+
rescue StandardError
|
|
11
|
+
false
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def enable_postgis
|
|
15
|
+
connection.execute("CREATE EXTENSION IF NOT EXISTS postgis")
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def create_spatial_index(table_name, column_name = "geometry")
|
|
19
|
+
index = quote_ident("#{table_name}_#{column_name}_idx")
|
|
20
|
+
connection.execute(
|
|
21
|
+
"CREATE INDEX IF NOT EXISTS #{index} ON #{quote_ident(table_name)} " \
|
|
22
|
+
"USING GIST (#{quote_ident(column_name)})"
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def drop_spatial_index(table_name, column_name = "geometry")
|
|
27
|
+
connection.execute("DROP INDEX IF EXISTS #{quote_ident("#{table_name}_#{column_name}_idx")}")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def reindex_spatial(table_name, column_name = "geometry")
|
|
31
|
+
connection.execute("REINDEX INDEX #{quote_ident("#{table_name}_#{column_name}_idx")}")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def geometry_info(table_name, column_name = "geometry")
|
|
35
|
+
exec_sanitized(
|
|
36
|
+
"SELECT type, srid FROM geometry_columns WHERE f_table_name = ? AND f_geometry_column = ?",
|
|
37
|
+
table_name.to_s, column_name.to_s
|
|
38
|
+
).first
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def bounding_box_query(table_name, south, west, north, east, column_name = "geometry")
|
|
42
|
+
exec_sanitized(
|
|
43
|
+
"SELECT * FROM #{quote_ident(table_name)} " \
|
|
44
|
+
"WHERE #{quote_ident(column_name)} && ST_MakeEnvelope(?, ?, ?, ?, 4326)::geography",
|
|
45
|
+
Float(west), Float(south), Float(east), Float(north)
|
|
46
|
+
)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def nearest_neighbors(table_name, lat, lng, limit = 10, column_name = "geometry")
|
|
50
|
+
column = quote_ident(column_name)
|
|
51
|
+
exec_sanitized(
|
|
52
|
+
"SELECT *, ST_Distance(#{column}, ST_SetSRID(ST_MakePoint(?, ?), 4326)::geography) AS distance " \
|
|
53
|
+
"FROM #{quote_ident(table_name)} " \
|
|
54
|
+
"ORDER BY #{column} <-> ST_SetSRID(ST_MakePoint(?, ?), 4326)::geography LIMIT ?",
|
|
55
|
+
Float(lng), Float(lat), Float(lng), Float(lat), Integer(limit)
|
|
56
|
+
)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def exec_sanitized(sql, *values)
|
|
62
|
+
connection.exec_query(ActiveRecord::Base.sanitize_sql([sql, *values]))
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def connection
|
|
66
|
+
ActiveRecord::Base.connection
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def quote_ident(name)
|
|
70
|
+
connection.quote_table_name(name.to_s)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OvertureMaps
|
|
4
|
+
# Resolves location names to division areas. Prefers the locally imported
|
|
5
|
+
# overture_divisions table (fast, offline); falls back to querying the
|
|
6
|
+
# divisions theme on Overture's bucket. Both paths return the same shape:
|
|
7
|
+
# { id:, name:, subtype:, country:, region:, bbox: BoundingBox, area_km2: }
|
|
8
|
+
# ordered largest-area first.
|
|
9
|
+
module DivisionSearch
|
|
10
|
+
class << self
|
|
11
|
+
def search(query:, release: nil, limit: 20)
|
|
12
|
+
local(query: query, limit: limit) ||
|
|
13
|
+
Import::Downloader.search_divisions(query: query, release: release, limit: limit)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Returns nil (fall back to remote) when the table is missing, empty,
|
|
17
|
+
# unreachable, or has no match for the query.
|
|
18
|
+
def local(query:, limit: 20)
|
|
19
|
+
model = Models::Division
|
|
20
|
+
return nil unless model.table_exists?
|
|
21
|
+
|
|
22
|
+
rows = model.search_by_name(query).largest_first.limit(limit)
|
|
23
|
+
return nil if rows.empty?
|
|
24
|
+
|
|
25
|
+
rows.filter_map do |division|
|
|
26
|
+
bbox = division.to_bounding_box
|
|
27
|
+
next unless bbox
|
|
28
|
+
|
|
29
|
+
{
|
|
30
|
+
id: division.id,
|
|
31
|
+
name: division.name,
|
|
32
|
+
subtype: division.subtype,
|
|
33
|
+
country: division.country,
|
|
34
|
+
region: division.region,
|
|
35
|
+
bbox: bbox,
|
|
36
|
+
area_km2: Util.bbox_area_km2(bbox)
|
|
37
|
+
}
|
|
38
|
+
end.then { |results| results.empty? ? nil : results }
|
|
39
|
+
rescue StandardError
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/engine"
|
|
4
|
+
|
|
5
|
+
module OvertureMaps
|
|
6
|
+
# Mountable engine providing the read-only query API over imported data:
|
|
7
|
+
#
|
|
8
|
+
# # config/routes.rb (added by the install generator)
|
|
9
|
+
# mount OvertureMaps::Engine => "/overture"
|
|
10
|
+
#
|
|
11
|
+
# Rake tasks under lib/tasks and the controllers under app/ are picked up
|
|
12
|
+
# by Rails::Engine's conventions — no manual loading here (a second load
|
|
13
|
+
# would define every rake task twice).
|
|
14
|
+
class Engine < ::Rails::Engine
|
|
15
|
+
isolate_namespace OvertureMaps
|
|
16
|
+
|
|
17
|
+
initializer "overture_maps.configure_rgeo" do
|
|
18
|
+
ActiveSupport.on_load(:active_record) do
|
|
19
|
+
require "rgeo/active_record"
|
|
20
|
+
|
|
21
|
+
RGeo::ActiveRecord::SpatialFactoryStore.instance.tap do |store|
|
|
22
|
+
factory = RGeo::Geographic.spherical_factory(srid: 4326)
|
|
23
|
+
store.default = factory
|
|
24
|
+
%w[geometry geometry_collection line_string multi_line_string
|
|
25
|
+
multi_point multi_polygon point polygon].each do |geo_type|
|
|
26
|
+
store.register(factory, geo_type: geo_type)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rgeo"
|
|
4
|
+
require "rgeo/geo_json"
|
|
5
|
+
|
|
6
|
+
module OvertureMaps
|
|
7
|
+
# Parses the geometry encodings Overture data shows up in: WKB (binary or
|
|
8
|
+
# hex, from parquet), WKT (from DuckDB text output), and GeoJSON (hash or
|
|
9
|
+
# string). Returns RGeo features on the spherical WGS84 factory.
|
|
10
|
+
module GeometryParser
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def factory
|
|
14
|
+
@factory ||= RGeo::Geographic.spherical_factory(srid: 4326)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def parse(geom)
|
|
18
|
+
return nil if geom.nil?
|
|
19
|
+
return geom if geom.is_a?(RGeo::Feature::Instance)
|
|
20
|
+
|
|
21
|
+
case geom
|
|
22
|
+
when Hash
|
|
23
|
+
RGeo::GeoJSON.decode(geom, geo_factory: factory)
|
|
24
|
+
when String
|
|
25
|
+
parse_string(geom)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def parse_string(geom)
|
|
30
|
+
stripped = geom.strip
|
|
31
|
+
if stripped.start_with?("{")
|
|
32
|
+
RGeo::GeoJSON.decode(stripped, geo_factory: factory)
|
|
33
|
+
elsif stripped.match?(/\A[A-Za-z]/)
|
|
34
|
+
factory.parse_wkt(stripped)
|
|
35
|
+
else
|
|
36
|
+
# Binary or hex WKB; RGeo's parser auto-detects hex strings.
|
|
37
|
+
factory.parse_wkb(geom)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OvertureMaps
|
|
4
|
+
# GERS — Overture's Global Entity Reference System. IDs are dashed UUIDs
|
|
5
|
+
# (since June 2025); earlier releases used 32-char undashed hex.
|
|
6
|
+
module GERS
|
|
7
|
+
UUID_FORMAT = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
|
|
8
|
+
LEGACY_FORMAT = /\A\h{32}\z/
|
|
9
|
+
|
|
10
|
+
class << self
|
|
11
|
+
def valid_id?(id)
|
|
12
|
+
id.is_a?(String) && (id.match?(UUID_FORMAT) || id.match?(LEGACY_FORMAT))
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Looks an id up in the official registry (the unversioned source of
|
|
16
|
+
# truth for all published GERS ids). Returns a hash with keys like
|
|
17
|
+
# "version", "first_seen", "last_seen", "last_changed", "path", "bbox",
|
|
18
|
+
# or nil when unknown. Scans registry parquet on S3 — takes seconds.
|
|
19
|
+
def lookup(id)
|
|
20
|
+
raise ArgumentError, "not a GERS id: #{id.inspect}" unless valid_id?(id)
|
|
21
|
+
|
|
22
|
+
source = "#{OvertureMaps.configuration.s3_uri.chomp("/")}/registry/*.parquet"
|
|
23
|
+
rows = QueryEngine.instance.query(
|
|
24
|
+
"SELECT * FROM read_parquet('#{source}') WHERE id = ? LIMIT 1", [id]
|
|
25
|
+
)
|
|
26
|
+
rows.first
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module OvertureMaps
|
|
6
|
+
module Import
|
|
7
|
+
# Downloads Overture data: whole theme files via anonymous HTTP, and
|
|
8
|
+
# bbox-filtered extracts via DuckDB (which pushes the bbox predicate down
|
|
9
|
+
# to parquet row-group statistics, so only the relevant slice of the
|
|
10
|
+
# dataset is transferred).
|
|
11
|
+
class Downloader
|
|
12
|
+
THEMES = %w[addresses base buildings divisions places transportation].freeze
|
|
13
|
+
|
|
14
|
+
TYPES = {
|
|
15
|
+
"addresses" => %w[address],
|
|
16
|
+
"base" => %w[bathymetry infrastructure land land_cover land_use water],
|
|
17
|
+
"buildings" => %w[building building_part],
|
|
18
|
+
"divisions" => %w[division division_area division_boundary],
|
|
19
|
+
"places" => %w[place],
|
|
20
|
+
"transportation" => %w[connector segment]
|
|
21
|
+
}.freeze
|
|
22
|
+
|
|
23
|
+
# division_area subtypes worth surfacing in a name search, roughly
|
|
24
|
+
# largest to smallest.
|
|
25
|
+
DIVISION_SUBTYPES = %w[country dependency region county localadmin locality borough neighborhood].freeze
|
|
26
|
+
|
|
27
|
+
# Division areas smaller than this are sliver noise in a name search
|
|
28
|
+
# (ported from the parallel main-branch work, commit 6305670).
|
|
29
|
+
MIN_SEARCH_AREA_KM2 = 1.0
|
|
30
|
+
|
|
31
|
+
EXTRACT_FORMATS = {
|
|
32
|
+
"parquet" => "parquet",
|
|
33
|
+
"geojson" => "geojson",
|
|
34
|
+
"geojsonseq" => "geojsonseq",
|
|
35
|
+
"gpkg" => "gpkg",
|
|
36
|
+
"geopackage" => "gpkg"
|
|
37
|
+
}.freeze
|
|
38
|
+
|
|
39
|
+
attr_reader :theme, :type, :release, :output_dir
|
|
40
|
+
|
|
41
|
+
def initialize(theme:, type: nil, release: nil, output_dir: nil)
|
|
42
|
+
raise ArgumentError, "unknown theme: #{theme}" unless THEMES.include?(theme)
|
|
43
|
+
raise ArgumentError, "unknown type #{type} for #{theme}" if type && !TYPES[theme].include?(type)
|
|
44
|
+
|
|
45
|
+
@theme = theme
|
|
46
|
+
@type = type
|
|
47
|
+
@release = Releases.validate!(release || Releases.current)
|
|
48
|
+
@output_dir = output_dir || OvertureMaps.configuration.cache_dir
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def self.types_for_theme(theme)
|
|
52
|
+
TYPES[theme] || []
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def self.themes_with_types
|
|
56
|
+
TYPES
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Downloads the complete parquet files for this theme/type. These are
|
|
60
|
+
# large (buildings alone is hundreds of GB globally) — bbox extracts
|
|
61
|
+
# are almost always what you want instead.
|
|
62
|
+
def download_theme_files
|
|
63
|
+
files = list_files
|
|
64
|
+
if files.empty?
|
|
65
|
+
log "No files found for #{theme}#{type ? "/#{type}" : ""} in #{release}"
|
|
66
|
+
return 0
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
log "Found #{files.count} file(s) to download..."
|
|
70
|
+
FileUtils.mkdir_p(output_dir)
|
|
71
|
+
|
|
72
|
+
files.each do |file|
|
|
73
|
+
filename = File.basename(file[:key])
|
|
74
|
+
local_path = File.join(output_dir, filename)
|
|
75
|
+
|
|
76
|
+
result = Storage.download(file[:key], to: local_path, expected_size: file[:size])
|
|
77
|
+
if result == :skipped
|
|
78
|
+
log "Skipping #{filename} (already exists)"
|
|
79
|
+
else
|
|
80
|
+
log "Downloaded #{filename} (#{Util.format_size(file[:size])})"
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
files.count
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Writes a bbox-filtered extract for one type to a local file and
|
|
88
|
+
# returns its path (nil when the area has no data). The extract is
|
|
89
|
+
# named by theme/type/release/area, so later runs reuse it as a cache.
|
|
90
|
+
def extract_bbox(bbox, format: "parquet", output_path: nil, limit: nil)
|
|
91
|
+
target_type = type or raise ArgumentError, "extract_bbox requires a type"
|
|
92
|
+
path = output_path || extract_path(bbox, format: format)
|
|
93
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
94
|
+
|
|
95
|
+
sql, params = self.class.bbox_query(theme: theme, type: target_type, release: release,
|
|
96
|
+
bbox: bbox, limit: limit)
|
|
97
|
+
begin
|
|
98
|
+
QueryEngine.instance.copy_to(sql, params: params, output_path: path,
|
|
99
|
+
format: EXTRACT_FORMATS.fetch(format.to_s.downcase, "parquet"))
|
|
100
|
+
rescue QueryEngine::Error, StandardError => e
|
|
101
|
+
raise release_hint_error(e)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
if File.exist?(path) && File.size(path).positive?
|
|
105
|
+
path
|
|
106
|
+
else
|
|
107
|
+
FileUtils.rm_f(path)
|
|
108
|
+
nil
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Extracts every type in the theme for a bbox. Returns the paths written.
|
|
113
|
+
def extract_bbox_all_types(bbox, format: "parquet")
|
|
114
|
+
types = type ? [type] : TYPES.fetch(theme)
|
|
115
|
+
types.filter_map do |t|
|
|
116
|
+
log "Querying #{theme}/#{t} (#{release})..."
|
|
117
|
+
path = self.class.new(theme: theme, type: t, release: release, output_dir: output_dir)
|
|
118
|
+
.extract_bbox(bbox, format: format)
|
|
119
|
+
log(path ? " Saved #{File.basename(path)} (#{Util.format_size(File.size(path))})" : " No data found")
|
|
120
|
+
path
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def extract_nearby(lat:, lng:, radius_meters:, format: "parquet")
|
|
125
|
+
extract_bbox_all_types(BoundingBox.around(lat: lat, lng: lng, radius_meters: radius_meters), format: format)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# The cache path for a bbox extract of this theme/type/release.
|
|
129
|
+
def extract_path(bbox, format: "parquet")
|
|
130
|
+
ext = EXTRACT_FORMATS.fetch(format.to_s.downcase, "parquet")
|
|
131
|
+
File.join(output_dir, "#{theme}_#{type}_#{release}_#{bbox.slug}.#{ext}")
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# An existing extract for this theme/type/release/area, or nil. Matches
|
|
135
|
+
# exactly — never falls back to "some other file for the theme".
|
|
136
|
+
def cached_extract(bbox, format: "parquet")
|
|
137
|
+
path = extract_path(bbox, format: format)
|
|
138
|
+
File.exist?(path) && File.size(path).positive? ? path : nil
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def list_files
|
|
142
|
+
prefix = "release/#{release}/theme=#{theme}#{type ? "/type=#{type}" : ""}"
|
|
143
|
+
Storage.list(prefix: prefix)[:objects].select { |o| o[:key].end_with?(".parquet") }
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def self.list_types(theme:, release: nil)
|
|
147
|
+
release = Releases.validate!(release || Releases.current)
|
|
148
|
+
listing = Storage.list(prefix: "release/#{release}/theme=#{theme}/", delimiter: "/")
|
|
149
|
+
listing[:prefixes].filter_map { |p| p[/type=([^\/]+)/, 1] }.sort
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def self.list_themes(release: nil)
|
|
153
|
+
release = Releases.validate!(release || Releases.current)
|
|
154
|
+
listing = Storage.list(prefix: "release/#{release}/", delimiter: "/")
|
|
155
|
+
listing[:prefixes].filter_map { |p| p[/theme=([^\/]+)/, 1] }.sort
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Searches division areas by name. Returns hashes with :id, :name,
|
|
159
|
+
# :subtype, :country, :region, :bbox (BoundingBox) and :area_km2,
|
|
160
|
+
# largest areas first.
|
|
161
|
+
def self.search_divisions(query:, release: nil, limit: 20)
|
|
162
|
+
release = Releases.validate!(release || Releases.current)
|
|
163
|
+
source = source_glob(theme: "divisions", type: "division_area", release: release)
|
|
164
|
+
placeholders = DIVISION_SUBTYPES.map { "?" }.join(", ")
|
|
165
|
+
|
|
166
|
+
sql = <<~SQL
|
|
167
|
+
SELECT id, names.primary AS name, subtype, country, region,
|
|
168
|
+
bbox.xmin AS xmin, bbox.xmax AS xmax, bbox.ymin AS ymin, bbox.ymax AS ymax
|
|
169
|
+
FROM read_parquet('#{source}', hive_partitioning=1)
|
|
170
|
+
WHERE names.primary ILIKE ?
|
|
171
|
+
AND subtype IN (#{placeholders})
|
|
172
|
+
AND bbox.xmax > bbox.xmin AND bbox.ymax > bbox.ymin
|
|
173
|
+
ORDER BY (bbox.xmax - bbox.xmin) * (bbox.ymax - bbox.ymin) DESC
|
|
174
|
+
LIMIT #{Integer(limit)}
|
|
175
|
+
SQL
|
|
176
|
+
|
|
177
|
+
rows = QueryEngine.instance.query(sql, ["%#{query}%"] + DIVISION_SUBTYPES)
|
|
178
|
+
results = rows.map do |row|
|
|
179
|
+
bbox = BoundingBox.new(
|
|
180
|
+
lat1: row["ymin"], lng1: row["xmin"],
|
|
181
|
+
lat2: row["ymax"], lng2: row["xmax"],
|
|
182
|
+
display_name: row["name"]
|
|
183
|
+
)
|
|
184
|
+
{
|
|
185
|
+
id: row["id"],
|
|
186
|
+
name: row["name"],
|
|
187
|
+
subtype: row["subtype"],
|
|
188
|
+
country: row["country"],
|
|
189
|
+
region: row["region"],
|
|
190
|
+
bbox: bbox,
|
|
191
|
+
area_km2: Util.bbox_area_km2(bbox)
|
|
192
|
+
}
|
|
193
|
+
end
|
|
194
|
+
results.select { |r| r[:area_km2] >= MIN_SEARCH_AREA_KM2 }
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Builds the bbox-filtered SELECT for one theme/type. Intersection
|
|
198
|
+
# semantics: any feature whose bbox overlaps the query box is included,
|
|
199
|
+
# matching what "give me this area" means (the old strict-containment
|
|
200
|
+
# filter silently dropped everything touching the boundary).
|
|
201
|
+
def self.bbox_query(theme:, type:, release:, bbox:, limit: nil)
|
|
202
|
+
raise ArgumentError, "unknown theme: #{theme}" unless THEMES.include?(theme)
|
|
203
|
+
raise ArgumentError, "unknown type #{type} for #{theme}" unless TYPES[theme].include?(type)
|
|
204
|
+
|
|
205
|
+
source = source_glob(theme: theme, type: type, release: Releases.validate!(release))
|
|
206
|
+
sql = <<~SQL
|
|
207
|
+
SELECT *
|
|
208
|
+
FROM read_parquet('#{source}', hive_partitioning=1)
|
|
209
|
+
WHERE bbox.xmin <= ? AND bbox.xmax >= ?
|
|
210
|
+
AND bbox.ymin <= ? AND bbox.ymax >= ?
|
|
211
|
+
SQL
|
|
212
|
+
sql += "LIMIT #{Integer(limit)}\n" if limit
|
|
213
|
+
[sql, [bbox.max_lng, bbox.min_lng, bbox.max_lat, bbox.min_lat]]
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def self.source_glob(theme:, type:, release:)
|
|
217
|
+
"#{OvertureMaps.configuration.s3_uri.chomp("/")}/release/#{release}/theme=#{theme}/type=#{type}/*.parquet"
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
private
|
|
221
|
+
|
|
222
|
+
# Overture prunes old releases; a query against a missing one fails
|
|
223
|
+
# with an opaque DuckDB error. Add the available releases when the
|
|
224
|
+
# requested one isn't in the catalog.
|
|
225
|
+
def release_hint_error(error)
|
|
226
|
+
available = begin
|
|
227
|
+
Releases.all
|
|
228
|
+
rescue StandardError
|
|
229
|
+
nil
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
if available && !available.include?(release)
|
|
233
|
+
Import::Error.new(
|
|
234
|
+
"release #{release} is not available (available: #{available.join(", ")}); " \
|
|
235
|
+
"original error: #{error.message}"
|
|
236
|
+
)
|
|
237
|
+
else
|
|
238
|
+
error
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def log(message)
|
|
243
|
+
logger = OvertureMaps.configuration.logger
|
|
244
|
+
logger ? logger.info(message) : puts(message)
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
end
|