overture_maps 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +29 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +416 -0
  5. data/app/controllers/overture_maps/application_controller.rb +28 -0
  6. data/app/controllers/overture_maps/attribution_controller.rb +15 -0
  7. data/app/controllers/overture_maps/features_controller.rb +125 -0
  8. data/app/controllers/overture_maps/search_controller.rb +31 -0
  9. data/app/controllers/overture_maps/tiles_controller.rb +85 -0
  10. data/config/routes.rb +13 -0
  11. data/exe/overture-maps-mcp +6 -0
  12. data/lib/generators/overture_maps/address_generator.rb +19 -0
  13. data/lib/generators/overture_maps/base_features_generator.rb +19 -0
  14. data/lib/generators/overture_maps/base_generator.rb +22 -0
  15. data/lib/generators/overture_maps/building_generator.rb +19 -0
  16. data/lib/generators/overture_maps/division_generator.rb +19 -0
  17. data/lib/generators/overture_maps/install_generator.rb +42 -0
  18. data/lib/generators/overture_maps/place_generator.rb +19 -0
  19. data/lib/generators/overture_maps/transportation_generator.rb +20 -0
  20. data/lib/generators/templates/address_migration.rb.tt +25 -0
  21. data/lib/generators/templates/address_model.rb.tt +4 -0
  22. data/lib/generators/templates/base_feature_model.rb.tt +4 -0
  23. data/lib/generators/templates/base_features_migration.rb.tt +27 -0
  24. data/lib/generators/templates/building_migration.rb.tt +24 -0
  25. data/lib/generators/templates/building_model.rb.tt +4 -0
  26. data/lib/generators/templates/category_migration.rb.tt +15 -0
  27. data/lib/generators/templates/category_model.rb.tt +4 -0
  28. data/lib/generators/templates/connector_model.rb.tt +4 -0
  29. data/lib/generators/templates/division_migration.rb.tt +31 -0
  30. data/lib/generators/templates/division_model.rb.tt +4 -0
  31. data/lib/generators/templates/imported_areas_migration.rb.tt +20 -0
  32. data/lib/generators/templates/place_migration.rb.tt +30 -0
  33. data/lib/generators/templates/place_model.rb.tt +4 -0
  34. data/lib/generators/templates/postgis_migration.rb.tt +5 -0
  35. data/lib/generators/templates/segment_model.rb.tt +4 -0
  36. data/lib/generators/templates/transportation_migration.rb.tt +38 -0
  37. data/lib/overture_maps/attribution.rb +66 -0
  38. data/lib/overture_maps/bounding_box.rb +80 -0
  39. data/lib/overture_maps/changelog.rb +70 -0
  40. data/lib/overture_maps/configuration.rb +43 -0
  41. data/lib/overture_maps/database.rb +74 -0
  42. data/lib/overture_maps/division_search.rb +44 -0
  43. data/lib/overture_maps/engine.rb +32 -0
  44. data/lib/overture_maps/geometry_parser.rb +41 -0
  45. data/lib/overture_maps/gers.rb +30 -0
  46. data/lib/overture_maps/import/downloader.rb +248 -0
  47. data/lib/overture_maps/import/location_based_runner.rb +176 -0
  48. data/lib/overture_maps/import/parquet_reader.rb +32 -0
  49. data/lib/overture_maps/import/record_mapper.rb +182 -0
  50. data/lib/overture_maps/import/runner.rb +125 -0
  51. data/lib/overture_maps/import.rb +27 -0
  52. data/lib/overture_maps/mcp_server.rb +222 -0
  53. data/lib/overture_maps/models/address.rb +32 -0
  54. data/lib/overture_maps/models/base.rb +39 -0
  55. data/lib/overture_maps/models/base_feature.rb +31 -0
  56. data/lib/overture_maps/models/building.rb +35 -0
  57. data/lib/overture_maps/models/category.rb +32 -0
  58. data/lib/overture_maps/models/connector.rb +11 -0
  59. data/lib/overture_maps/models/division.rb +43 -0
  60. data/lib/overture_maps/models/imported_area.rb +42 -0
  61. data/lib/overture_maps/models/place.rb +43 -0
  62. data/lib/overture_maps/models/segment.rb +22 -0
  63. data/lib/overture_maps/models.rb +25 -0
  64. data/lib/overture_maps/query.rb +168 -0
  65. data/lib/overture_maps/query_engine.rb +268 -0
  66. data/lib/overture_maps/releases.rb +74 -0
  67. data/lib/overture_maps/storage.rb +121 -0
  68. data/lib/overture_maps/syncer.rb +99 -0
  69. data/lib/overture_maps/util.rb +30 -0
  70. data/lib/overture_maps/version.rb +3 -0
  71. data/lib/overture_maps.rb +42 -0
  72. data/lib/tasks/overture_maps.rake +554 -0
  73. metadata +301 -0
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OvertureMaps
4
+ # A WGS84 bounding box. The single place that parses user-supplied coordinate
5
+ # strings, so import and download entry points cannot drift apart.
6
+ class BoundingBox
7
+ NUMBER = /-?\d+(?:\.\d+)?/
8
+ # Four numbers separated by commas, whitespace, or underscores, with an
9
+ # optional "|display name" suffix: "47.6,-122.3,47.7,-122.2" or
10
+ # "47.6_-122.3_47.7_-122.2|seattle"
11
+ COORDS = /\A(#{NUMBER})[,\s_]+(#{NUMBER})[,\s_]+(#{NUMBER})[,\s_]+(#{NUMBER})(?:\|(.+))?\z/
12
+
13
+ attr_reader :min_lat, :min_lng, :max_lat, :max_lng, :display_name
14
+
15
+ def initialize(lat1:, lng1:, lat2:, lng2:, display_name: nil)
16
+ @min_lat, @max_lat = [Float(lat1), Float(lat2)].minmax
17
+ @min_lng, @max_lng = [Float(lng1), Float(lng2)].minmax
18
+ @display_name = display_name
19
+
20
+ unless (-90..90).cover?(@min_lat) && (-90..90).cover?(@max_lat) &&
21
+ (-180..180).cover?(@min_lng) && (-180..180).cover?(@max_lng)
22
+ raise ArgumentError, "coordinates out of range: #{self}"
23
+ end
24
+ end
25
+
26
+ # Returns a BoundingBox if the string looks like coordinates, else nil.
27
+ def self.parse(string)
28
+ match = COORDS.match(string.to_s.strip)
29
+ return nil unless match
30
+
31
+ new(lat1: match[1], lng1: match[2], lat2: match[3], lng2: match[4], display_name: match[5])
32
+ end
33
+
34
+ def self.coordinates?(string)
35
+ COORDS.match?(string.to_s.strip)
36
+ end
37
+
38
+ # Overture bbox structs come back from division searches as
39
+ # {"xmin" => ..., "xmax" => ..., "ymin" => ..., "ymax" => ...}
40
+ def self.from_overture(bbox, display_name: nil)
41
+ new(
42
+ lat1: bbox["ymin"], lng1: bbox["xmin"],
43
+ lat2: bbox["ymax"], lng2: bbox["xmax"],
44
+ display_name: display_name
45
+ )
46
+ end
47
+
48
+ def self.around(lat:, lng:, radius_meters:)
49
+ lat = Float(lat)
50
+ lng = Float(lng)
51
+ lat_delta = radius_meters.to_f / 111_000
52
+ lng_delta = radius_meters.to_f / (111_000 * Math.cos(lat * Math::PI / 180))
53
+
54
+ new(
55
+ lat1: [lat - lat_delta, -90].max, lng1: [lng - lng_delta, -180].max,
56
+ lat2: [lat + lat_delta, 90].min, lng2: [lng + lng_delta, 180].min
57
+ )
58
+ end
59
+
60
+ def to_s
61
+ "#{min_lat},#{min_lng},#{max_lat},#{max_lng}"
62
+ end
63
+
64
+ # Filename-safe label for cache files.
65
+ def slug
66
+ return sanitized_display_name if display_name
67
+
68
+ format("%.4f_%.4f_%.4f_%.4f", min_lat, min_lng, max_lat, max_lng)
69
+ end
70
+
71
+ private
72
+
73
+ # Filename-safe and idempotent: a slug that round-trips through
74
+ # display_name (e.g. from ImportedArea bookkeeping) must come back
75
+ # unchanged, so dots and dashes survive.
76
+ def sanitized_display_name
77
+ display_name.downcase.gsub(/[^a-z0-9.\-]+/, "_").gsub(/\A_+|_+\z/, "")
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OvertureMaps
4
+ # Overture's per-release data changelog: parquet partitioned by theme,
5
+ # type, and change_type (added | removed | data_changed | unchanged),
6
+ # with each row carrying the feature id and its bbox. The changelog for
7
+ # release R describes changes relative to the release before R.
8
+ module Changelog
9
+ CHANGE_TYPES = %w[added removed data_changed unchanged].freeze
10
+
11
+ class << self
12
+ # IDs removed in `release` (relative to the prior release), optionally
13
+ # restricted to those whose bbox intersects the given area.
14
+ def removed_ids(theme:, type:, release:, bbox: nil)
15
+ sql, params = build_query(theme: theme, type: type, release: release,
16
+ change_type: "removed", bbox: bbox, select: "id")
17
+ QueryEngine.instance.query(sql, params).map { |row| row["id"] }
18
+ end
19
+
20
+ # Per-change_type counts for a release/theme/type (for sync:status and
21
+ # dry runs). Returns e.g. {"added" => 123, "removed" => 4, ...}.
22
+ def counts(theme:, type:, release:, bbox: nil)
23
+ validate!(theme, type)
24
+ source = source_glob(theme: theme, type: type, release: Releases.validate!(release),
25
+ change_type: "*")
26
+ sql = +"SELECT change_type, count(*) AS n FROM read_parquet('#{source}', hive_partitioning=1)"
27
+ params = []
28
+ if bbox
29
+ sql << " WHERE bbox.xmin <= ? AND bbox.xmax >= ? AND bbox.ymin <= ? AND bbox.ymax >= ?"
30
+ params = bbox_params(bbox)
31
+ end
32
+ sql << " GROUP BY change_type"
33
+
34
+ QueryEngine.instance.query(sql, params)
35
+ .to_h { |row| [row["change_type"], Integer(row["n"])] }
36
+ end
37
+
38
+ private
39
+
40
+ def build_query(theme:, type:, release:, change_type:, bbox:, select:)
41
+ validate!(theme, type)
42
+ raise ArgumentError, "unknown change_type: #{change_type}" unless CHANGE_TYPES.include?(change_type)
43
+
44
+ source = source_glob(theme: theme, type: type, release: Releases.validate!(release),
45
+ change_type: change_type)
46
+ sql = +"SELECT #{select} FROM read_parquet('#{source}', hive_partitioning=1)"
47
+ params = []
48
+ if bbox
49
+ sql << " WHERE bbox.xmin <= ? AND bbox.xmax >= ? AND bbox.ymin <= ? AND bbox.ymax >= ?"
50
+ params = bbox_params(bbox)
51
+ end
52
+ [sql, params]
53
+ end
54
+
55
+ def bbox_params(bbox)
56
+ [bbox.max_lng, bbox.min_lng, bbox.max_lat, bbox.min_lat]
57
+ end
58
+
59
+ def source_glob(theme:, type:, release:, change_type:)
60
+ base = OvertureMaps.configuration.s3_uri.chomp("/")
61
+ "#{base}/changelog/#{release}/theme=#{theme}/type=#{type}/change_type=#{change_type}/*.parquet"
62
+ end
63
+
64
+ def validate!(theme, type)
65
+ types = Import::Downloader::TYPES[theme] or raise ArgumentError, "unknown theme: #{theme}"
66
+ raise ArgumentError, "unknown type #{type} for #{theme}" unless types.include?(type)
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OvertureMaps
4
+ class Configuration
5
+ DEFAULT_S3_HTTP_URL = "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com"
6
+ DEFAULT_S3_URI = "s3://overturemaps-us-west-2"
7
+ DEFAULT_S3_REGION = "us-west-2"
8
+
9
+ # release: Overture release string (e.g. "2026-06-17.0"). nil means "latest available".
10
+ # cache_dir: where downloaded parquet extracts are stored.
11
+ # s3_http_url/s3_uri/s3_region: data source; point at a mirror (e.g. MinIO) to avoid
12
+ # hitting Overture's bucket from every host.
13
+ # batch_size: default import batch size.
14
+ # timeout: HTTP open/read timeout in seconds.
15
+ # non_interactive: never prompt; pick the best division match and prefer fresh downloads.
16
+ # duckdb_cli_path: explicit path to the duckdb binary (otherwise PATH, then auto-download).
17
+ # logger: receives progress output; defaults to stdout logging from the rake layer.
18
+ # api_auth: callable invoked with the controller before every API
19
+ # request; render/head inside it to reject (nil = open access).
20
+ # api_default_limit / api_max_limit: collection endpoint page sizing.
21
+ # tile_feature_limit: per-tile feature cap for the MVT endpoint.
22
+ attr_accessor :release, :cache_dir, :s3_http_url, :s3_uri, :s3_region,
23
+ :batch_size, :timeout, :non_interactive, :duckdb_cli_path, :logger,
24
+ :api_auth, :api_default_limit, :api_max_limit, :tile_feature_limit
25
+
26
+ def initialize
27
+ @release = nil
28
+ @cache_dir = "tmp/overture"
29
+ @s3_http_url = DEFAULT_S3_HTTP_URL
30
+ @s3_uri = DEFAULT_S3_URI
31
+ @s3_region = DEFAULT_S3_REGION
32
+ @batch_size = 1000
33
+ @timeout = 30
34
+ @non_interactive = !ENV["OVERTURE_NON_INTERACTIVE"].to_s.empty?
35
+ @duckdb_cli_path = ENV["OVERTURE_DUCKDB_CLI"]
36
+ @logger = nil
37
+ @api_auth = nil
38
+ @api_default_limit = 100
39
+ @api_max_limit = 1000
40
+ @tile_feature_limit = 20_000
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OvertureMaps
4
+ # PostGIS helpers. Identifiers are quoted and values sanitized — nothing
5
+ # user-supplied is interpolated into SQL.
6
+ module Database
7
+ class << self
8
+ def postgis_available?
9
+ connection.extension_enabled?("postgis")
10
+ rescue StandardError
11
+ false
12
+ end
13
+
14
+ def enable_postgis
15
+ connection.execute("CREATE EXTENSION IF NOT EXISTS postgis")
16
+ end
17
+
18
+ def create_spatial_index(table_name, column_name = "geometry")
19
+ index = quote_ident("#{table_name}_#{column_name}_idx")
20
+ connection.execute(
21
+ "CREATE INDEX IF NOT EXISTS #{index} ON #{quote_ident(table_name)} " \
22
+ "USING GIST (#{quote_ident(column_name)})"
23
+ )
24
+ end
25
+
26
+ def drop_spatial_index(table_name, column_name = "geometry")
27
+ connection.execute("DROP INDEX IF EXISTS #{quote_ident("#{table_name}_#{column_name}_idx")}")
28
+ end
29
+
30
+ def reindex_spatial(table_name, column_name = "geometry")
31
+ connection.execute("REINDEX INDEX #{quote_ident("#{table_name}_#{column_name}_idx")}")
32
+ end
33
+
34
+ def geometry_info(table_name, column_name = "geometry")
35
+ exec_sanitized(
36
+ "SELECT type, srid FROM geometry_columns WHERE f_table_name = ? AND f_geometry_column = ?",
37
+ table_name.to_s, column_name.to_s
38
+ ).first
39
+ end
40
+
41
+ def bounding_box_query(table_name, south, west, north, east, column_name = "geometry")
42
+ exec_sanitized(
43
+ "SELECT * FROM #{quote_ident(table_name)} " \
44
+ "WHERE #{quote_ident(column_name)} && ST_MakeEnvelope(?, ?, ?, ?, 4326)::geography",
45
+ Float(west), Float(south), Float(east), Float(north)
46
+ )
47
+ end
48
+
49
+ def nearest_neighbors(table_name, lat, lng, limit = 10, column_name = "geometry")
50
+ column = quote_ident(column_name)
51
+ exec_sanitized(
52
+ "SELECT *, ST_Distance(#{column}, ST_SetSRID(ST_MakePoint(?, ?), 4326)::geography) AS distance " \
53
+ "FROM #{quote_ident(table_name)} " \
54
+ "ORDER BY #{column} <-> ST_SetSRID(ST_MakePoint(?, ?), 4326)::geography LIMIT ?",
55
+ Float(lng), Float(lat), Float(lng), Float(lat), Integer(limit)
56
+ )
57
+ end
58
+
59
+ private
60
+
61
+ def exec_sanitized(sql, *values)
62
+ connection.exec_query(ActiveRecord::Base.sanitize_sql([sql, *values]))
63
+ end
64
+
65
+ def connection
66
+ ActiveRecord::Base.connection
67
+ end
68
+
69
+ def quote_ident(name)
70
+ connection.quote_table_name(name.to_s)
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OvertureMaps
4
+ # Resolves location names to division areas. Prefers the locally imported
5
+ # overture_divisions table (fast, offline); falls back to querying the
6
+ # divisions theme on Overture's bucket. Both paths return the same shape:
7
+ # { id:, name:, subtype:, country:, region:, bbox: BoundingBox, area_km2: }
8
+ # ordered largest-area first.
9
+ module DivisionSearch
10
+ class << self
11
+ def search(query:, release: nil, limit: 20)
12
+ local(query: query, limit: limit) ||
13
+ Import::Downloader.search_divisions(query: query, release: release, limit: limit)
14
+ end
15
+
16
+ # Returns nil (fall back to remote) when the table is missing, empty,
17
+ # unreachable, or has no match for the query.
18
+ def local(query:, limit: 20)
19
+ model = Models::Division
20
+ return nil unless model.table_exists?
21
+
22
+ rows = model.search_by_name(query).largest_first.limit(limit)
23
+ return nil if rows.empty?
24
+
25
+ rows.filter_map do |division|
26
+ bbox = division.to_bounding_box
27
+ next unless bbox
28
+
29
+ {
30
+ id: division.id,
31
+ name: division.name,
32
+ subtype: division.subtype,
33
+ country: division.country,
34
+ region: division.region,
35
+ bbox: bbox,
36
+ area_km2: Util.bbox_area_km2(bbox)
37
+ }
38
+ end.then { |results| results.empty? ? nil : results }
39
+ rescue StandardError
40
+ nil
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/engine"
4
+
5
+ module OvertureMaps
6
+ # Mountable engine providing the read-only query API over imported data:
7
+ #
8
+ # # config/routes.rb (added by the install generator)
9
+ # mount OvertureMaps::Engine => "/overture"
10
+ #
11
+ # Rake tasks under lib/tasks and the controllers under app/ are picked up
12
+ # by Rails::Engine's conventions — no manual loading here (a second load
13
+ # would define every rake task twice).
14
+ class Engine < ::Rails::Engine
15
+ isolate_namespace OvertureMaps
16
+
17
+ initializer "overture_maps.configure_rgeo" do
18
+ ActiveSupport.on_load(:active_record) do
19
+ require "rgeo/active_record"
20
+
21
+ RGeo::ActiveRecord::SpatialFactoryStore.instance.tap do |store|
22
+ factory = RGeo::Geographic.spherical_factory(srid: 4326)
23
+ store.default = factory
24
+ %w[geometry geometry_collection line_string multi_line_string
25
+ multi_point multi_polygon point polygon].each do |geo_type|
26
+ store.register(factory, geo_type: geo_type)
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rgeo"
4
+ require "rgeo/geo_json"
5
+
6
+ module OvertureMaps
7
+ # Parses the geometry encodings Overture data shows up in: WKB (binary or
8
+ # hex, from parquet), WKT (from DuckDB text output), and GeoJSON (hash or
9
+ # string). Returns RGeo features on the spherical WGS84 factory.
10
+ module GeometryParser
11
+ module_function
12
+
13
+ def factory
14
+ @factory ||= RGeo::Geographic.spherical_factory(srid: 4326)
15
+ end
16
+
17
+ def parse(geom)
18
+ return nil if geom.nil?
19
+ return geom if geom.is_a?(RGeo::Feature::Instance)
20
+
21
+ case geom
22
+ when Hash
23
+ RGeo::GeoJSON.decode(geom, geo_factory: factory)
24
+ when String
25
+ parse_string(geom)
26
+ end
27
+ end
28
+
29
+ def parse_string(geom)
30
+ stripped = geom.strip
31
+ if stripped.start_with?("{")
32
+ RGeo::GeoJSON.decode(stripped, geo_factory: factory)
33
+ elsif stripped.match?(/\A[A-Za-z]/)
34
+ factory.parse_wkt(stripped)
35
+ else
36
+ # Binary or hex WKB; RGeo's parser auto-detects hex strings.
37
+ factory.parse_wkb(geom)
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OvertureMaps
4
+ # GERS — Overture's Global Entity Reference System. IDs are dashed UUIDs
5
+ # (since June 2025); earlier releases used 32-char undashed hex.
6
+ module GERS
7
+ UUID_FORMAT = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
8
+ LEGACY_FORMAT = /\A\h{32}\z/
9
+
10
+ class << self
11
+ def valid_id?(id)
12
+ id.is_a?(String) && (id.match?(UUID_FORMAT) || id.match?(LEGACY_FORMAT))
13
+ end
14
+
15
+ # Looks an id up in the official registry (the unversioned source of
16
+ # truth for all published GERS ids). Returns a hash with keys like
17
+ # "version", "first_seen", "last_seen", "last_changed", "path", "bbox",
18
+ # or nil when unknown. Scans registry parquet on S3 — takes seconds.
19
+ def lookup(id)
20
+ raise ArgumentError, "not a GERS id: #{id.inspect}" unless valid_id?(id)
21
+
22
+ source = "#{OvertureMaps.configuration.s3_uri.chomp("/")}/registry/*.parquet"
23
+ rows = QueryEngine.instance.query(
24
+ "SELECT * FROM read_parquet('#{source}') WHERE id = ? LIMIT 1", [id]
25
+ )
26
+ rows.first
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,248 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+
5
+ module OvertureMaps
6
+ module Import
7
+ # Downloads Overture data: whole theme files via anonymous HTTP, and
8
+ # bbox-filtered extracts via DuckDB (which pushes the bbox predicate down
9
+ # to parquet row-group statistics, so only the relevant slice of the
10
+ # dataset is transferred).
11
+ class Downloader
12
+ THEMES = %w[addresses base buildings divisions places transportation].freeze
13
+
14
+ TYPES = {
15
+ "addresses" => %w[address],
16
+ "base" => %w[bathymetry infrastructure land land_cover land_use water],
17
+ "buildings" => %w[building building_part],
18
+ "divisions" => %w[division division_area division_boundary],
19
+ "places" => %w[place],
20
+ "transportation" => %w[connector segment]
21
+ }.freeze
22
+
23
+ # division_area subtypes worth surfacing in a name search, roughly
24
+ # largest to smallest.
25
+ DIVISION_SUBTYPES = %w[country dependency region county localadmin locality borough neighborhood].freeze
26
+
27
+ # Division areas smaller than this are sliver noise in a name search
28
+ # (ported from the parallel main-branch work, commit 6305670).
29
+ MIN_SEARCH_AREA_KM2 = 1.0
30
+
31
+ EXTRACT_FORMATS = {
32
+ "parquet" => "parquet",
33
+ "geojson" => "geojson",
34
+ "geojsonseq" => "geojsonseq",
35
+ "gpkg" => "gpkg",
36
+ "geopackage" => "gpkg"
37
+ }.freeze
38
+
39
+ attr_reader :theme, :type, :release, :output_dir
40
+
41
+ def initialize(theme:, type: nil, release: nil, output_dir: nil)
42
+ raise ArgumentError, "unknown theme: #{theme}" unless THEMES.include?(theme)
43
+ raise ArgumentError, "unknown type #{type} for #{theme}" if type && !TYPES[theme].include?(type)
44
+
45
+ @theme = theme
46
+ @type = type
47
+ @release = Releases.validate!(release || Releases.current)
48
+ @output_dir = output_dir || OvertureMaps.configuration.cache_dir
49
+ end
50
+
51
+ def self.types_for_theme(theme)
52
+ TYPES[theme] || []
53
+ end
54
+
55
+ def self.themes_with_types
56
+ TYPES
57
+ end
58
+
59
+ # Downloads the complete parquet files for this theme/type. These are
60
+ # large (buildings alone is hundreds of GB globally) — bbox extracts
61
+ # are almost always what you want instead.
62
+ def download_theme_files
63
+ files = list_files
64
+ if files.empty?
65
+ log "No files found for #{theme}#{type ? "/#{type}" : ""} in #{release}"
66
+ return 0
67
+ end
68
+
69
+ log "Found #{files.count} file(s) to download..."
70
+ FileUtils.mkdir_p(output_dir)
71
+
72
+ files.each do |file|
73
+ filename = File.basename(file[:key])
74
+ local_path = File.join(output_dir, filename)
75
+
76
+ result = Storage.download(file[:key], to: local_path, expected_size: file[:size])
77
+ if result == :skipped
78
+ log "Skipping #{filename} (already exists)"
79
+ else
80
+ log "Downloaded #{filename} (#{Util.format_size(file[:size])})"
81
+ end
82
+ end
83
+
84
+ files.count
85
+ end
86
+
87
+ # Writes a bbox-filtered extract for one type to a local file and
88
+ # returns its path (nil when the area has no data). The extract is
89
+ # named by theme/type/release/area, so later runs reuse it as a cache.
90
+ def extract_bbox(bbox, format: "parquet", output_path: nil, limit: nil)
91
+ target_type = type or raise ArgumentError, "extract_bbox requires a type"
92
+ path = output_path || extract_path(bbox, format: format)
93
+ FileUtils.mkdir_p(File.dirname(path))
94
+
95
+ sql, params = self.class.bbox_query(theme: theme, type: target_type, release: release,
96
+ bbox: bbox, limit: limit)
97
+ begin
98
+ QueryEngine.instance.copy_to(sql, params: params, output_path: path,
99
+ format: EXTRACT_FORMATS.fetch(format.to_s.downcase, "parquet"))
100
+ rescue QueryEngine::Error, StandardError => e
101
+ raise release_hint_error(e)
102
+ end
103
+
104
+ if File.exist?(path) && File.size(path).positive?
105
+ path
106
+ else
107
+ FileUtils.rm_f(path)
108
+ nil
109
+ end
110
+ end
111
+
112
+ # Extracts every type in the theme for a bbox. Returns the paths written.
113
+ def extract_bbox_all_types(bbox, format: "parquet")
114
+ types = type ? [type] : TYPES.fetch(theme)
115
+ types.filter_map do |t|
116
+ log "Querying #{theme}/#{t} (#{release})..."
117
+ path = self.class.new(theme: theme, type: t, release: release, output_dir: output_dir)
118
+ .extract_bbox(bbox, format: format)
119
+ log(path ? " Saved #{File.basename(path)} (#{Util.format_size(File.size(path))})" : " No data found")
120
+ path
121
+ end
122
+ end
123
+
124
+ def extract_nearby(lat:, lng:, radius_meters:, format: "parquet")
125
+ extract_bbox_all_types(BoundingBox.around(lat: lat, lng: lng, radius_meters: radius_meters), format: format)
126
+ end
127
+
128
+ # The cache path for a bbox extract of this theme/type/release.
129
+ def extract_path(bbox, format: "parquet")
130
+ ext = EXTRACT_FORMATS.fetch(format.to_s.downcase, "parquet")
131
+ File.join(output_dir, "#{theme}_#{type}_#{release}_#{bbox.slug}.#{ext}")
132
+ end
133
+
134
+ # An existing extract for this theme/type/release/area, or nil. Matches
135
+ # exactly — never falls back to "some other file for the theme".
136
+ def cached_extract(bbox, format: "parquet")
137
+ path = extract_path(bbox, format: format)
138
+ File.exist?(path) && File.size(path).positive? ? path : nil
139
+ end
140
+
141
+ def list_files
142
+ prefix = "release/#{release}/theme=#{theme}#{type ? "/type=#{type}" : ""}"
143
+ Storage.list(prefix: prefix)[:objects].select { |o| o[:key].end_with?(".parquet") }
144
+ end
145
+
146
+ def self.list_types(theme:, release: nil)
147
+ release = Releases.validate!(release || Releases.current)
148
+ listing = Storage.list(prefix: "release/#{release}/theme=#{theme}/", delimiter: "/")
149
+ listing[:prefixes].filter_map { |p| p[/type=([^\/]+)/, 1] }.sort
150
+ end
151
+
152
+ def self.list_themes(release: nil)
153
+ release = Releases.validate!(release || Releases.current)
154
+ listing = Storage.list(prefix: "release/#{release}/", delimiter: "/")
155
+ listing[:prefixes].filter_map { |p| p[/theme=([^\/]+)/, 1] }.sort
156
+ end
157
+
158
+ # Searches division areas by name. Returns hashes with :id, :name,
159
+ # :subtype, :country, :region, :bbox (BoundingBox) and :area_km2,
160
+ # largest areas first.
161
+ def self.search_divisions(query:, release: nil, limit: 20)
162
+ release = Releases.validate!(release || Releases.current)
163
+ source = source_glob(theme: "divisions", type: "division_area", release: release)
164
+ placeholders = DIVISION_SUBTYPES.map { "?" }.join(", ")
165
+
166
+ sql = <<~SQL
167
+ SELECT id, names.primary AS name, subtype, country, region,
168
+ bbox.xmin AS xmin, bbox.xmax AS xmax, bbox.ymin AS ymin, bbox.ymax AS ymax
169
+ FROM read_parquet('#{source}', hive_partitioning=1)
170
+ WHERE names.primary ILIKE ?
171
+ AND subtype IN (#{placeholders})
172
+ AND bbox.xmax > bbox.xmin AND bbox.ymax > bbox.ymin
173
+ ORDER BY (bbox.xmax - bbox.xmin) * (bbox.ymax - bbox.ymin) DESC
174
+ LIMIT #{Integer(limit)}
175
+ SQL
176
+
177
+ rows = QueryEngine.instance.query(sql, ["%#{query}%"] + DIVISION_SUBTYPES)
178
+ results = rows.map do |row|
179
+ bbox = BoundingBox.new(
180
+ lat1: row["ymin"], lng1: row["xmin"],
181
+ lat2: row["ymax"], lng2: row["xmax"],
182
+ display_name: row["name"]
183
+ )
184
+ {
185
+ id: row["id"],
186
+ name: row["name"],
187
+ subtype: row["subtype"],
188
+ country: row["country"],
189
+ region: row["region"],
190
+ bbox: bbox,
191
+ area_km2: Util.bbox_area_km2(bbox)
192
+ }
193
+ end
194
+ results.select { |r| r[:area_km2] >= MIN_SEARCH_AREA_KM2 }
195
+ end
196
+
197
+ # Builds the bbox-filtered SELECT for one theme/type. Intersection
198
+ # semantics: any feature whose bbox overlaps the query box is included,
199
+ # matching what "give me this area" means (the old strict-containment
200
+ # filter silently dropped everything touching the boundary).
201
+ def self.bbox_query(theme:, type:, release:, bbox:, limit: nil)
202
+ raise ArgumentError, "unknown theme: #{theme}" unless THEMES.include?(theme)
203
+ raise ArgumentError, "unknown type #{type} for #{theme}" unless TYPES[theme].include?(type)
204
+
205
+ source = source_glob(theme: theme, type: type, release: Releases.validate!(release))
206
+ sql = <<~SQL
207
+ SELECT *
208
+ FROM read_parquet('#{source}', hive_partitioning=1)
209
+ WHERE bbox.xmin <= ? AND bbox.xmax >= ?
210
+ AND bbox.ymin <= ? AND bbox.ymax >= ?
211
+ SQL
212
+ sql += "LIMIT #{Integer(limit)}\n" if limit
213
+ [sql, [bbox.max_lng, bbox.min_lng, bbox.max_lat, bbox.min_lat]]
214
+ end
215
+
216
+ def self.source_glob(theme:, type:, release:)
217
+ "#{OvertureMaps.configuration.s3_uri.chomp("/")}/release/#{release}/theme=#{theme}/type=#{type}/*.parquet"
218
+ end
219
+
220
+ private
221
+
222
+ # Overture prunes old releases; a query against a missing one fails
223
+ # with an opaque DuckDB error. Add the available releases when the
224
+ # requested one isn't in the catalog.
225
+ def release_hint_error(error)
226
+ available = begin
227
+ Releases.all
228
+ rescue StandardError
229
+ nil
230
+ end
231
+
232
+ if available && !available.include?(release)
233
+ Import::Error.new(
234
+ "release #{release} is not available (available: #{available.join(", ")}); " \
235
+ "original error: #{error.message}"
236
+ )
237
+ else
238
+ error
239
+ end
240
+ end
241
+
242
+ def log(message)
243
+ logger = OvertureMaps.configuration.logger
244
+ logger ? logger.info(message) : puts(message)
245
+ end
246
+ end
247
+ end
248
+ end