curtilage 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "json"
5
+ require "time"
6
+ require "tmpdir"
7
+
8
+ module Curtilage
9
+ # Data on demand, so nothing has to be hosted: finds the regions the pins
10
+ # are in (Regions), builds any that are missing or stale into cache_dir
11
+ # (Build), and returns a Catalogue over them.
12
+ #
13
+ # With no cache_dir, everything goes in a temporary folder that cleanup
14
+ # deletes: each run downloads and builds afresh (1-5 minutes per US state).
15
+ # Give a persistent cache_dir to reuse builds until max_age_days.
16
+ class AutoData
17
+ INDEX_FILE = "geofabrik-index.json"
18
+
19
+ OPTIONS = {
20
+ cache_dir: nil, max_age_days: 31, ml_footprints: nil, duckdb: nil, extension_directory: nil,
21
+ log: $stderr, builder: nil, regions: nil
22
+ }.freeze
23
+
24
+ attr_reader :cache_dir
25
+
26
+ def initialize(**options)
27
+ @opts = OPTIONS.merge(options.compact)
28
+ @temporary = @opts[:cache_dir].nil?
29
+ @cache_dir = @opts[:cache_dir] || Dir.mktmpdir("curtilage")
30
+ FileUtils.mkdir_p(@cache_dir)
31
+ end
32
+
33
+ # A Catalogue covering the pins, building what's missing first. Pins in
34
+ # no region (open sea, or no Geofabrik shapefile) are simply not covered.
35
+ def catalogue_for(pins, margin: Lookup::DEFAULT_RADIUS_DEG)
36
+ needed = pins.flat_map { |pin| regions.around(pin, margin) }.uniq(&:path)
37
+ needed.each { |region| ensure_built(region) }
38
+ Catalogue.from_dir(@cache_dir)
39
+ end
40
+
41
+ def cleanup
42
+ FileUtils.rm_rf(@cache_dir) if @temporary
43
+ end
44
+
45
+ def regions
46
+ @regions ||= @opts[:regions] || load_regions
47
+ end
48
+
49
+ def name(region) = region.path.tr("/", "-")
50
+
51
+ private
52
+
53
+ def ensure_built(region)
54
+ with_lock(name(region)) do
55
+ next if fresh?(File.join(@cache_dir, "#{name(region)}.json"))
56
+
57
+ @opts[:log].puts "building #{region.path} into #{@cache_dir}"
58
+ build(region)
59
+ end
60
+ end
61
+
62
+ def build(region)
63
+ return @opts[:builder].call(region, @cache_dir, name(region)) if @opts[:builder]
64
+
65
+ Build.run(region.path, out_dir: @cache_dir, work_dir: File.join(@cache_dir, "work"), name: name(region),
66
+ **@opts.slice(:ml_footprints, :duckdb, :extension_directory, :log))
67
+ end
68
+
69
+ def fresh?(metadata_file)
70
+ return false unless File.file?(metadata_file)
71
+
72
+ built = Time.parse(JSON.parse(File.read(metadata_file))["built_at"].to_s)
73
+ Time.now - built < @opts[:max_age_days] * 86_400
74
+ rescue ArgumentError, JSON::ParserError
75
+ false
76
+ end
77
+
78
+ # Two jobs sharing a cache build each region once.
79
+ def with_lock(key, &)
80
+ File.open(File.join(@cache_dir, ".#{key}.lock"), File::CREAT | File::RDWR) do |lock|
81
+ lock.flock(File::LOCK_EX)
82
+ yield
83
+ end
84
+ end
85
+
86
+ def load_regions
87
+ file = File.join(@cache_dir, INDEX_FILE)
88
+ unless File.file?(file) && Time.now - File.mtime(file) < @opts[:max_age_days] * 86_400
89
+ File.write(file, Download.read(Regions::HOST, Regions::INDEX, max_bytes: Regions::MAX_INDEX_BYTES))
90
+ end
91
+ Regions.parse(File.read(file))
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,252 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+ require "fileutils"
5
+ require "json"
6
+ require "open3"
7
+ require "time"
8
+
9
+ module Curtilage
10
+ # Builds a region's features Parquet from Geofabrik's free shapefiles:
11
+ # download the newest dated extract, check the zip, unzip only the layers
12
+ # used, and run sql/build_from_shapefiles.sql with the DuckDB CLI.
13
+ # Optionally adds machine-learned building footprints (see Footprints).
14
+ #
15
+ # Network access is limited to Geofabrik (and the footprint host when asked
16
+ # for). Memory is capped (400 MB and 1 thread by default) so a build fits a
17
+ # 1 GB dyno; DuckDB spills to work_dir when a region is bigger than that.
18
+ module Build
19
+ class Error < StandardError; end
20
+
21
+ HOST = "download.geofabrik.de"
22
+ US = "north-america/us"
23
+ # Regions Geofabrik only publishes as shapefiles in parts.
24
+ PARTS = {
25
+ "#{US}/california" => %W[#{US}/california/norcal #{US}/california/socal],
26
+ }.freeze
27
+ LAYERS = %w[buildings_a traffic_a roads railways].freeze
28
+ REGION = %r{\A[a-z]+(-[a-z]+)*(/[a-z]+(-[a-z]+)*)*\z}
29
+ MAX_DOWNLOAD_BYTES = 5_000_000_000
30
+ LOOKBACK_DAYS = 8
31
+ SQL_FILE = File.expand_path("../../sql/build_from_shapefiles.sql", __dir__)
32
+
33
+ OPTIONS = {
34
+ date: nil, memory_limit: "400MB", threads: 1,
35
+ ml_footprints: nil, keep_downloads: false, name: nil, log: $stderr
36
+ }.freeze
37
+
38
+ module_function
39
+
40
+ # region: a US state ("tennessee", "new-york") or any Geofabrik region
41
+ # path ("europe/germany/bayern", "north-america/canada/ontario").
42
+ # options: see OPTIONS; ml_footprints: "microsoft" adds ML footprints;
43
+ # duckdb: and extension_directory: default to Curtilage.config.
44
+ # Returns the Parquet path, named after the region's last part unless
45
+ # name: says otherwise.
46
+ def run(region, out_dir:, work_dir:, **options)
47
+ opts = OPTIONS.merge(Curtilage.duckdb_options(options))
48
+ path = region_path(region)
49
+ check_footprints(opts[:ml_footprints])
50
+ FileUtils.mkdir_p([out_dir, work_dir])
51
+ fetched = PARTS.fetch(path, [path]).map { |part| fetch(part, opts[:date], work_dir, opts[:log]) }
52
+ dirs = fetched.map(&:first)
53
+ opts = opts.merge(temp_directory: File.join(work_dir, "duckdb_tmp"))
54
+ ml = ml_files(dirs, work_dir, opts)
55
+ out = File.join(out_dir, "#{output_name(path, opts[:name])}.parquet")
56
+ opts[:log].puts "building #{out}"
57
+ run_duckdb(sql(dirs, "#{out}.tmp", opts, ml_files: ml), opts[:duckdb])
58
+ File.rename("#{out}.tmp", out)
59
+ write_metadata(out, path, fetched.map(&:last), opts)
60
+ out
61
+ ensure
62
+ cleanup(work_dir, dirs, opts[:keep_downloads]) if dirs
63
+ end
64
+
65
+ def output_name(path, name)
66
+ name ||= File.basename(path)
67
+ raise ArgumentError, "bad output name #{name.inspect}" unless name.match?(/\A[a-z0-9-]+\z/)
68
+
69
+ name
70
+ end
71
+
72
+ def region_path(region)
73
+ region = region.to_s
74
+ raise ArgumentError, "unknown region: #{region.inspect}" unless region.match?(REGION)
75
+
76
+ region.include?("/") ? region : "#{US}/#{region}"
77
+ end
78
+
79
+ def check_footprints(source)
80
+ return if source.nil? || Footprints::SOURCES.include?(source.to_s)
81
+
82
+ raise ArgumentError, "unknown ml_footprints source #{source.inspect} (have: #{Footprints::SOURCES.join(", ")})"
83
+ end
84
+
85
+ # Downloads and unzips one Geofabrik extract; returns [folder with its
86
+ # shapefiles, extract date].
87
+ #
88
+ # Unless the whole zip is already on disk, only the layers used are
89
+ # downloaded, with range requests (ZipRange); a server or zip that can't
90
+ # do that gets the whole file instead.
91
+ def fetch(part, date, work_dir, log)
92
+ date ||= latest_date(part)
93
+ name = "#{part.tr("/", "-")}-#{date}-free.shp.zip"
94
+ zip = File.join(work_dir, name)
95
+ dir = File.join(work_dir, File.basename(name, ".shp.zip"))
96
+ return [dir, date] if !File.file?(zip) && fetch_layers(part, date, dir, log)
97
+
98
+ unless File.file?(zip)
99
+ log.puts "downloading https://#{HOST}#{url_path(part, date)}"
100
+ Download.fetch(HOST, url_path(part, date), zip, max_bytes: MAX_DOWNLOAD_BYTES)
101
+ end
102
+ check_zip(zip)
103
+ unzip_layers(zip, dir)
104
+ [dir, date]
105
+ end
106
+
107
+ def fetch_layers(part, date, dir, log)
108
+ log.puts "downloading the layers used from https://#{HOST}#{url_path(part, date)}"
109
+ patterns = LAYERS.map { |layer| "gis_osm_#{layer}_free_*" }
110
+ ZipRange.extract(HOST, url_path(part, date), patterns, dir, max_bytes: MAX_DOWNLOAD_BYTES)
111
+ true
112
+ rescue ZipRange::Unsupported => e
113
+ log.puts "range download not possible (#{e.message}); downloading the whole file"
114
+ FileUtils.rm_rf(dir)
115
+ false
116
+ end
117
+
118
+ def url_path(part, date) = "/#{part}-#{date}-free.shp.zip"
119
+
120
+ # Geofabrik publishes dated files daily; today's may not be up yet.
121
+ def latest_date(part, today: Date.today)
122
+ LOOKBACK_DAYS.times do |i|
123
+ date = (today - i).strftime("%y%m%d")
124
+ return date if Download.exists?(HOST, url_path(part, date))
125
+ end
126
+ raise Error, "Geofabrik has no dated shapefile for #{part} in the last #{LOOKBACK_DAYS} days " \
127
+ "(large regions are only offered as sub-regions)"
128
+ end
129
+
130
+ # Geofabrik publishes no checksum for shapefiles; the zip's CRCs catch a
131
+ # truncated or corrupt download.
132
+ def check_zip(zip)
133
+ _, err, status = Open3.capture3("unzip", "-tq", zip)
134
+ return if status.success?
135
+
136
+ File.delete(zip)
137
+ raise Error, "corrupt download #{File.basename(zip)} (deleted): #{err.strip[0, 200]}"
138
+ end
139
+
140
+ def unzip_layers(zip, dir)
141
+ FileUtils.mkdir_p(dir)
142
+ patterns = LAYERS.map { |layer| "gis_osm_#{layer}_free_*" }
143
+ _, err, status = Open3.capture3("unzip", "-oq", zip, *patterns, "-d", dir)
144
+ raise Error, "unzip failed: #{err.strip[0, 200]}" unless status.success?
145
+ end
146
+
147
+ # Footprint tiles covering the region's extent, or [] when not asked for.
148
+ def ml_files(dirs, work_dir, opts)
149
+ return [] unless opts[:ml_footprints]
150
+
151
+ bbox = extent(dirs, opts)
152
+ Footprints.microsoft(bbox, File.join(work_dir, "ml_footprints"), log: opts[:log]).tap do |files|
153
+ opts[:ml_bbox] = bbox
154
+ raise Error, "no ML footprint tiles cover #{bbox.inspect}" if files.empty?
155
+ end
156
+ end
157
+
158
+ # [min_lng, min_lat, max_lng, max_lat] of the region's roads.
159
+ def extent(dirs, opts)
160
+ query = "SELECT min(ST_XMin(geom)) a, min(ST_YMin(geom)) b, max(ST_XMax(geom)) c, max(ST_YMax(geom)) d " \
161
+ "FROM roads_src;"
162
+ out = run_duckdb([setup(opts), "INSTALL spatial; LOAD spatial;", layer_view("roads", dirs), query].join("\n"),
163
+ opts[:duckdb], "-json")
164
+ row = JSON.parse(out).first
165
+ %w[a b c d].map { |k| Float(row[k]) }
166
+ end
167
+
168
+ # The build query reads one view per layer; large layers come in parts
169
+ # (_1, _2, ...) and multi-part regions in several folders.
170
+ def sql(dirs, out, opts, ml_files: [])
171
+ views = LAYERS.map { |layer| layer_view(layer, dirs) }
172
+ [setup(opts), "INSTALL spatial; LOAD spatial;", *views, ml_view(ml_files, opts[:ml_bbox]),
173
+ "SET VARIABLE out_path = #{quote(out)};", File.read(SQL_FILE)].join("\n")
174
+ end
175
+
176
+ def layer_view(layer, dirs)
177
+ files = dirs.flat_map { |d| Dir.glob(File.join(d, "gis_osm_#{layer}_free_*.shp")) }.sort
178
+ raise Error, "no #{layer} shapefile in #{dirs.join(", ")}" if files.empty?
179
+
180
+ reads = files.map { |f| "SELECT * FROM ST_Read(#{quote(f)})" }.join(" UNION ALL BY NAME ")
181
+ "CREATE VIEW #{layer}_src AS #{reads};"
182
+ end
183
+
184
+ # ML footprints within the region's extent (tiles overhang it), or an
185
+ # empty view.
186
+ def ml_view(files, bbox)
187
+ return "CREATE VIEW ml_buildings_src AS SELECT NULL::GEOMETRY AS geom WHERE false;" if files.empty?
188
+
189
+ list = files.map { |f| quote(f) }.join(", ")
190
+ min_lng, min_lat, max_lng, max_lat = bbox.map { |v| Float(v) }
191
+ "CREATE VIEW ml_buildings_src AS SELECT geom FROM (SELECT ST_GeomFromGeoJSON(geometry) AS geom " \
192
+ "FROM read_json([#{list}], format = 'newline_delimited', compression = 'gzip', " \
193
+ "columns = {geometry: 'JSON'})) WHERE ST_Intersects(geom, ST_MakeEnvelope(" \
194
+ "#{min_lng}, #{min_lat}, #{max_lng}, #{max_lat}));"
195
+ end
196
+
197
+ def setup(opts)
198
+ ext = opts[:extension_directory]
199
+ [("SET extension_directory = #{quote(File.expand_path(ext))};" if ext),
200
+ "SET memory_limit = #{quote(opts[:memory_limit])};",
201
+ "SET threads = #{Integer(opts[:threads])};",
202
+ ("SET temp_directory = #{quote(opts[:temp_directory])};" if opts[:temp_directory])].compact.join("\n")
203
+ end
204
+
205
+ def run_duckdb(sql, duckdb, *)
206
+ out, err, status = Open3.capture3(duckdb, *, stdin_data: sql)
207
+ raise Error, "duckdb failed: #{err.strip[0, 500]}" unless status.success? && err.strip.empty?
208
+
209
+ out
210
+ end
211
+
212
+ # Writes <region>.json next to the Parquet file: what it holds, where it
213
+ # came from and how to attribute it. Catalogue reads these.
214
+ def write_metadata(parquet, path, dates, opts)
215
+ stats = run_duckdb([setup(opts), "SELECT feature, count(*) AS n, min(min_lng) a, min(min_lat) b, " \
216
+ "max(max_lng) c, max(max_lat) d FROM read_parquet(#{quote(parquet)}) " \
217
+ "GROUP BY feature;"].join("\n"), opts[:duckdb], "-json")
218
+ rows = JSON.parse(stats)
219
+ File.write(parquet.sub(/\.parquet\z/, ".json"), JSON.pretty_generate(metadata(parquet, path, dates, rows, opts)))
220
+ end
221
+
222
+ def metadata(parquet, path, dates, rows, opts)
223
+ source_dates = dates.map { |d| Date.strptime(d, "%y%m%d").iso8601 }
224
+ {
225
+ region: path, parquet: File.basename(parquet), format: 1,
226
+ source: "Geofabrik OpenStreetMap shapefiles", source_dates: source_dates,
227
+ built_at: Time.now.utc.iso8601, ml_footprints: opts[:ml_footprints],
228
+ bbox: [rows.map { |r| r["a"] }.min, rows.map { |r| r["b"] }.min, rows.map { |r| r["c"] }.max,
229
+ rows.map { |r| r["d"] }.max],
230
+ counts: rows.to_h { |r| [r["feature"], r["n"]] },
231
+ licence: "ODbL-1.0", attribution: attribution(opts[:ml_footprints])
232
+ }
233
+ end
234
+
235
+ def attribution(ml_footprints)
236
+ ["© OpenStreetMap contributors", ("Microsoft Global ML Building Footprints" if ml_footprints)].compact
237
+ end
238
+
239
+ def cleanup(work_dir, dirs, keep_downloads)
240
+ dirs.each { |d| FileUtils.rm_rf(d) }
241
+ FileUtils.rm_rf(File.join(work_dir, "duckdb_tmp"))
242
+ return if keep_downloads
243
+
244
+ Dir.glob(File.join(work_dir, "*.shp.zip")).each { |z| File.delete(z) }
245
+ FileUtils.rm_rf(File.join(work_dir, "ml_footprints"))
246
+ end
247
+
248
+ def quote(text)
249
+ "'#{text.to_s.gsub("'", "''")}'"
250
+ end
251
+ end
252
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Curtilage
6
+ # The built regions available to look features up in, from the
7
+ # <region>.json files Build writes next to each Parquet file. Picks the
8
+ # file(s) covering a pin, so callers don't track which state is which.
9
+ class Catalogue
10
+ Region = Struct.new(:name, :parquet, :bbox, :metadata, keyword_init: true) do
11
+ def covers?(pin, margin)
12
+ lng, lat = pin
13
+ lng.between?(bbox[0] - margin, bbox[2] + margin) && lat.between?(bbox[1] - margin, bbox[3] + margin)
14
+ end
15
+ end
16
+
17
+ attr_reader :regions
18
+
19
+ # dir: a folder of <region>.json and <region>.parquet files. base_url:
20
+ # where the Parquet files are served from instead (s3://bucket/path,
21
+ # https://...), when the metadata is local but the data isn't.
22
+ def self.from_dir(dir, base_url: nil)
23
+ regions = Dir.glob(File.join(dir, "*.json")).filter_map do |file|
24
+ meta = JSON.parse(File.read(file))
25
+ next unless meta["format"] == 1 && meta["parquet"] && meta["bbox"]&.size == 4
26
+
27
+ parquet = base_url ? "#{base_url.chomp("/")}/#{meta["parquet"]}" : File.join(dir, meta["parquet"])
28
+ Region.new(name: meta["region"], parquet: parquet, bbox: meta["bbox"].map { |v| Float(v) }, metadata: meta)
29
+ end
30
+ new(regions)
31
+ end
32
+
33
+ def initialize(regions)
34
+ @regions = regions
35
+ end
36
+
37
+ # Parquet files whose extent covers the pin (more than one near a border).
38
+ def files_for(pin, margin: Lookup::DEFAULT_RADIUS_DEG)
39
+ @regions.select { |r| r.covers?(pin, margin) }.map(&:parquet)
40
+ end
41
+
42
+ def attribution(files)
43
+ @regions.select { |r| files.include?(r.parquet) }.flat_map { |r| Array(r.metadata["attribution"]) }.uniq
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Settings shared by every call, so callers don't pass them each time.
4
+ # Defaults come from the environment:
5
+ #
6
+ # CURTILAGE_DUCKDB path to the DuckDB CLI (default: "duckdb" on PATH)
7
+ # CURTILAGE_EXTENSIONS DuckDB extension folder (default: DuckDB's own)
8
+ # CURTILAGE_CACHE_DIR where on-demand builds are kept (default: a
9
+ # temporary folder, deleted after each run)
10
+ # CURTILAGE_SOURCE geofabrik (default) or overture
11
+ #
12
+ # Curtilage.configure do |c|
13
+ # c.cache_dir = "/var/cache/curtilage"
14
+ # c.source = :overture
15
+ # end
16
+ module Curtilage
17
+ Config = Struct.new(:duckdb, :extension_directory, :cache_dir, :source, keyword_init: true)
18
+
19
+ def self.config
20
+ @config ||= Config.new(
21
+ duckdb: ENV.fetch("CURTILAGE_DUCKDB", "duckdb"),
22
+ extension_directory: ENV.fetch("CURTILAGE_EXTENSIONS", nil),
23
+ cache_dir: ENV.fetch("CURTILAGE_CACHE_DIR", nil),
24
+ source: ENV.fetch("CURTILAGE_SOURCE", "geofabrik").to_sym
25
+ )
26
+ end
27
+
28
+ def self.configure
29
+ yield config
30
+ config
31
+ end
32
+
33
+ # Back to the environment defaults (mostly for tests).
34
+ def self.reset_config!
35
+ @config = nil
36
+ end
37
+
38
+ # The DuckDB settings for Lookup, Build and Overture: explicit options win
39
+ # over the configuration.
40
+ def self.duckdb_options(options = {})
41
+ defaults = { duckdb: config.duckdb, extension_directory: config.extension_directory }.compact
42
+ defaults.merge(options.compact)
43
+ end
44
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "net/http"
4
+ require "uri"
5
+
6
+ module Curtilage
7
+ # HTTPS downloads of public map data, from an allowed host only, streamed
8
+ # to disk with a size cap: Geofabrik extracts and region index, Overture's
9
+ # file index, Microsoft's footprint tiles. (DuckDB itself reads Overture
10
+ # and bucket files.) Nothing else in the gem talks to the network.
11
+ module Download
12
+ class Error < StandardError; end
13
+
14
+ module_function
15
+
16
+ def exists?(host, path)
17
+ start(host) { |http| http.head(path).is_a?(Net::HTTPSuccess) }
18
+ end
19
+
20
+ # Writes to dest.part, then renames, so a dest file is always complete.
21
+ def fetch(host, path, dest, max_bytes:)
22
+ start(host) do |http|
23
+ http.request_get(path) do |res|
24
+ raise Error, "HTTP #{res.code} for https://#{host}#{path}" unless res.is_a?(Net::HTTPSuccess)
25
+ raise Error, "too large: https://#{host}#{path}" if res.content_length.to_i > max_bytes
26
+
27
+ stream(res, "#{dest}.part", max_bytes)
28
+ end
29
+ end
30
+ File.rename("#{dest}.part", dest)
31
+ dest
32
+ end
33
+
34
+ def read(host, path, max_bytes:)
35
+ start(host) do |http|
36
+ res = http.get(path)
37
+ raise Error, "HTTP #{res.code} for https://#{host}#{path}" unless res.is_a?(Net::HTTPSuccess)
38
+ raise Error, "too large: https://#{host}#{path}" if res.body.bytesize > max_bytes
39
+
40
+ res.body
41
+ end
42
+ end
43
+
44
+ def stream(res, dest, max_bytes)
45
+ bytes = 0
46
+ File.open(dest, "wb") do |f|
47
+ res.read_body do |chunk|
48
+ bytes += chunk.bytesize
49
+ raise Error, "download over #{max_bytes} bytes" if bytes > max_bytes
50
+
51
+ f.write(chunk)
52
+ end
53
+ end
54
+ end
55
+
56
+ def start(host, &)
57
+ Net::HTTP.start(host, 443, use_ssl: true, open_timeout: 30, read_timeout: 120, &)
58
+ end
59
+
60
+ # The path of an https URL on host, or nil for anything else.
61
+ def path_on(url, host, prefix)
62
+ uri = URI.parse(url)
63
+ return nil unless uri.scheme == "https" && uri.host == host && uri.path.start_with?(prefix)
64
+ return nil if uri.path.include?("..")
65
+
66
+ uri.path
67
+ rescue URI::InvalidURIError
68
+ nil
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,130 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Curtilage
4
+ # The OSM features around a pin, projected to metres around it.
5
+ #
6
+ # Input is a list of hashes (string or symbol keys) with "feature"
7
+ # ("building", "ml_building", "parking", "road" or "railway"), "geometry"
8
+ # (a GeoJSON geometry hash), and optionally "id", "name" and "highway"
9
+ # (the class: road or railway class, or building type). Map data is
10
+ # untrusted, so oversized inputs and broken geometry are dropped and
11
+ # counted, not raised.
12
+ class Features
13
+ MAX_FEATURES = 20_000
14
+ MAX_VERTICES = 5_000
15
+
16
+ # category: the building type from OSM (house, parking, roof, ...), or nil.
17
+ Polygon = Struct.new(:id, :kind, :name, :xy, :area, :category, keyword_init: true) do
18
+ def hull
19
+ @hull ||= Geometry.convex_hull(xy)
20
+ end
21
+
22
+ def rectangle
23
+ @rectangle ||= Geometry.minimum_rectangle(xy)
24
+ end
25
+
26
+ # [min_x, min_y, max_x, max_y], for a cheap pre-check.
27
+ def bounds
28
+ @bounds ||= Geometry.bounds(xy)
29
+ end
30
+
31
+ def near?(other, margin = 0)
32
+ Geometry.bounds_near?(bounds, other.bounds, margin)
33
+ end
34
+ end
35
+ Line = Struct.new(:id, :highway, :name, :xy, keyword_init: true)
36
+
37
+ attr_reader :pin, :buildings, :ml_buildings, :parking, :roads, :railways, :dropped
38
+
39
+ def initialize(pin, features)
40
+ @pin = pin
41
+ @buildings = []
42
+ @ml_buildings = []
43
+ @parking = []
44
+ @roads = []
45
+ @railways = []
46
+ @dropped = 0
47
+ @seen = {}
48
+ Array(features).first(MAX_FEATURES).each { |f| add(f) }
49
+ @dropped += [Array(features).size - MAX_FEATURES, 0].max
50
+ end
51
+
52
+ # A copy whose buildings are the given list (see Generator buildings:).
53
+ def with_buildings(list)
54
+ dup.tap { |copy| copy.instance_variable_set(:@buildings, list) }
55
+ end
56
+
57
+ private
58
+
59
+ def add(feature)
60
+ kind = field(feature, :feature).to_s
61
+ # A region built from several extracts repeats OSM features along the
62
+ # seam. ML footprints' ids are only unique within one file, so they're
63
+ # never treated as repeats.
64
+ id = field(feature, :id)
65
+ if id && kind != "ml_building"
66
+ return if @seen.key?([kind, id])
67
+
68
+ @seen[[kind, id]] = true
69
+ end
70
+ geometry = field(feature, :geometry)
71
+ parts = parts_of(geometry)
72
+ return @dropped += 1 if parts.nil? || parts.empty?
73
+
74
+ parts.each { |coords| add_part(feature, kind, coords) }
75
+ end
76
+
77
+ def add_part(feature, kind, coords)
78
+ return @dropped += 1 unless coords.is_a?(Array) && coords.size <= MAX_VERTICES
79
+ return @dropped += 1 unless Geometry.valid_coordinates?(coords)
80
+
81
+ xy = Geometry.project(coords, pin)
82
+ if %w[road railway].include?(kind)
83
+ return @dropped += 1 if xy.size < 2
84
+
85
+ line = Line.new(id: field(feature, :id), highway: field(feature, :highway)&.to_s,
86
+ name: field(feature, :name), xy: xy)
87
+ (kind == "road" ? @roads : @railways) << line
88
+ else
89
+ add_polygon(feature, kind, xy)
90
+ end
91
+ end
92
+
93
+ def add_polygon(feature, kind, xy)
94
+ ring = Geometry.clean(Geometry.open_ring(xy))
95
+ area = ring.size >= 3 ? Geometry.area(ring) : 0
96
+ return @dropped += 1 if area < 1 || Geometry.self_intersects?(ring)
97
+
98
+ category = field(feature, :highway).to_s
99
+ polygon = Polygon.new(id: field(feature, :id), kind: kind, name: field(feature, :name),
100
+ xy: Geometry.counter_clockwise(ring), area: area,
101
+ category: category.empty? ? nil : category)
102
+ case kind
103
+ when "building" then @buildings << polygon
104
+ when "ml_building" then @ml_buildings << polygon
105
+ when "parking" then @parking << polygon
106
+ else @dropped += 1
107
+ end
108
+ end
109
+
110
+ # Outer rings of polygons, or the lines of line strings. Holes are
111
+ # ignored: premises cover a building's courtyard too.
112
+ def parts_of(geometry)
113
+ return nil unless geometry.is_a?(Hash)
114
+
115
+ coords = field(geometry, :coordinates)
116
+ return nil unless coords.is_a?(Array)
117
+
118
+ case field(geometry, :type)
119
+ when "Polygon" then [coords.first].compact
120
+ when "MultiPolygon" then coords.filter_map { |poly| poly.is_a?(Array) ? poly.first : nil }
121
+ when "LineString" then [coords]
122
+ when "MultiLineString" then coords
123
+ end
124
+ end
125
+
126
+ def field(hash, key)
127
+ hash[key.to_s] || hash[key]
128
+ end
129
+ end
130
+ end