curtilage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +39 -0
- data/LICENSE.txt +21 -0
- data/README.md +134 -0
- data/SECURITY.md +15 -0
- data/docs/reference.md +140 -0
- data/exe/curtilage +134 -0
- data/lib/curtilage/audit.rb +88 -0
- data/lib/curtilage/auto_data.rb +94 -0
- data/lib/curtilage/build.rb +252 -0
- data/lib/curtilage/catalogue.rb +46 -0
- data/lib/curtilage/config.rb +44 -0
- data/lib/curtilage/download.rb +71 -0
- data/lib/curtilage/features.rb +130 -0
- data/lib/curtilage/footprints.rb +82 -0
- data/lib/curtilage/generator.rb +578 -0
- data/lib/curtilage/generator_lot.rb +158 -0
- data/lib/curtilage/generator_store.rb +31 -0
- data/lib/curtilage/geometry.rb +360 -0
- data/lib/curtilage/lookup.rb +110 -0
- data/lib/curtilage/overture.rb +195 -0
- data/lib/curtilage/regions.rb +85 -0
- data/lib/curtilage/validation.rb +89 -0
- data/lib/curtilage/version.rb +5 -0
- data/lib/curtilage/zip_range.rb +234 -0
- data/lib/curtilage.rb +126 -0
- data/sql/build_from_shapefiles.sql +52 -0
- metadata +77 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
require "time"
|
|
6
|
+
require "tmpdir"
|
|
7
|
+
|
|
8
|
+
module Curtilage
|
|
9
|
+
# Data on demand, so nothing has to be hosted: finds the regions the pins
|
|
10
|
+
# are in (Regions), builds any that are missing or stale into cache_dir
|
|
11
|
+
# (Build), and returns a Catalogue over them.
|
|
12
|
+
#
|
|
13
|
+
# With no cache_dir, everything goes in a temporary folder that cleanup
|
|
14
|
+
# deletes: each run downloads and builds afresh (1-5 minutes per US state).
|
|
15
|
+
# Give a persistent cache_dir to reuse builds until max_age_days.
|
|
16
|
+
class AutoData
|
|
17
|
+
INDEX_FILE = "geofabrik-index.json"
|
|
18
|
+
|
|
19
|
+
OPTIONS = {
|
|
20
|
+
cache_dir: nil, max_age_days: 31, ml_footprints: nil, duckdb: nil, extension_directory: nil,
|
|
21
|
+
log: $stderr, builder: nil, regions: nil
|
|
22
|
+
}.freeze
|
|
23
|
+
|
|
24
|
+
attr_reader :cache_dir
|
|
25
|
+
|
|
26
|
+
def initialize(**options)
|
|
27
|
+
@opts = OPTIONS.merge(options.compact)
|
|
28
|
+
@temporary = @opts[:cache_dir].nil?
|
|
29
|
+
@cache_dir = @opts[:cache_dir] || Dir.mktmpdir("curtilage")
|
|
30
|
+
FileUtils.mkdir_p(@cache_dir)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# A Catalogue covering the pins, building what's missing first. Pins in
|
|
34
|
+
# no region (open sea, or no Geofabrik shapefile) are simply not covered.
|
|
35
|
+
def catalogue_for(pins, margin: Lookup::DEFAULT_RADIUS_DEG)
|
|
36
|
+
needed = pins.flat_map { |pin| regions.around(pin, margin) }.uniq(&:path)
|
|
37
|
+
needed.each { |region| ensure_built(region) }
|
|
38
|
+
Catalogue.from_dir(@cache_dir)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def cleanup
|
|
42
|
+
FileUtils.rm_rf(@cache_dir) if @temporary
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def regions
|
|
46
|
+
@regions ||= @opts[:regions] || load_regions
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def name(region) = region.path.tr("/", "-")
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def ensure_built(region)
|
|
54
|
+
with_lock(name(region)) do
|
|
55
|
+
next if fresh?(File.join(@cache_dir, "#{name(region)}.json"))
|
|
56
|
+
|
|
57
|
+
@opts[:log].puts "building #{region.path} into #{@cache_dir}"
|
|
58
|
+
build(region)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def build(region)
|
|
63
|
+
return @opts[:builder].call(region, @cache_dir, name(region)) if @opts[:builder]
|
|
64
|
+
|
|
65
|
+
Build.run(region.path, out_dir: @cache_dir, work_dir: File.join(@cache_dir, "work"), name: name(region),
|
|
66
|
+
**@opts.slice(:ml_footprints, :duckdb, :extension_directory, :log))
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def fresh?(metadata_file)
|
|
70
|
+
return false unless File.file?(metadata_file)
|
|
71
|
+
|
|
72
|
+
built = Time.parse(JSON.parse(File.read(metadata_file))["built_at"].to_s)
|
|
73
|
+
Time.now - built < @opts[:max_age_days] * 86_400
|
|
74
|
+
rescue ArgumentError, JSON::ParserError
|
|
75
|
+
false
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Two jobs sharing a cache build each region once.
|
|
79
|
+
def with_lock(key, &)
|
|
80
|
+
File.open(File.join(@cache_dir, ".#{key}.lock"), File::CREAT | File::RDWR) do |lock|
|
|
81
|
+
lock.flock(File::LOCK_EX)
|
|
82
|
+
yield
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def load_regions
|
|
87
|
+
file = File.join(@cache_dir, INDEX_FILE)
|
|
88
|
+
unless File.file?(file) && Time.now - File.mtime(file) < @opts[:max_age_days] * 86_400
|
|
89
|
+
File.write(file, Download.read(Regions::HOST, Regions::INDEX, max_bytes: Regions::MAX_INDEX_BYTES))
|
|
90
|
+
end
|
|
91
|
+
Regions.parse(File.read(file))
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "open3"
|
|
7
|
+
require "time"
|
|
8
|
+
|
|
9
|
+
module Curtilage
|
|
10
|
+
# Builds a region's features Parquet from Geofabrik's free shapefiles:
|
|
11
|
+
# download the newest dated extract, check the zip, unzip only the layers
|
|
12
|
+
# used, and run sql/build_from_shapefiles.sql with the DuckDB CLI.
|
|
13
|
+
# Optionally adds machine-learned building footprints (see Footprints).
|
|
14
|
+
#
|
|
15
|
+
# Network access is limited to Geofabrik (and the footprint host when asked
|
|
16
|
+
# for). Memory is capped (400 MB and 1 thread by default) so a build fits a
|
|
17
|
+
# 1 GB dyno; DuckDB spills to work_dir when a region is bigger than that.
|
|
18
|
+
module Build
|
|
19
|
+
class Error < StandardError; end
|
|
20
|
+
|
|
21
|
+
HOST = "download.geofabrik.de"
|
|
22
|
+
US = "north-america/us"
|
|
23
|
+
# Regions Geofabrik only publishes as shapefiles in parts.
|
|
24
|
+
PARTS = {
|
|
25
|
+
"#{US}/california" => %W[#{US}/california/norcal #{US}/california/socal],
|
|
26
|
+
}.freeze
|
|
27
|
+
LAYERS = %w[buildings_a traffic_a roads railways].freeze
|
|
28
|
+
REGION = %r{\A[a-z]+(-[a-z]+)*(/[a-z]+(-[a-z]+)*)*\z}
|
|
29
|
+
MAX_DOWNLOAD_BYTES = 5_000_000_000
|
|
30
|
+
LOOKBACK_DAYS = 8
|
|
31
|
+
SQL_FILE = File.expand_path("../../sql/build_from_shapefiles.sql", __dir__)
|
|
32
|
+
|
|
33
|
+
OPTIONS = {
|
|
34
|
+
date: nil, memory_limit: "400MB", threads: 1,
|
|
35
|
+
ml_footprints: nil, keep_downloads: false, name: nil, log: $stderr
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
module_function
|
|
39
|
+
|
|
40
|
+
# region: a US state ("tennessee", "new-york") or any Geofabrik region
|
|
41
|
+
# path ("europe/germany/bayern", "north-america/canada/ontario").
|
|
42
|
+
# options: see OPTIONS; ml_footprints: "microsoft" adds ML footprints;
|
|
43
|
+
# duckdb: and extension_directory: default to Curtilage.config.
|
|
44
|
+
# Returns the Parquet path, named after the region's last part unless
|
|
45
|
+
# name: says otherwise.
|
|
46
|
+
def run(region, out_dir:, work_dir:, **options)
|
|
47
|
+
opts = OPTIONS.merge(Curtilage.duckdb_options(options))
|
|
48
|
+
path = region_path(region)
|
|
49
|
+
check_footprints(opts[:ml_footprints])
|
|
50
|
+
FileUtils.mkdir_p([out_dir, work_dir])
|
|
51
|
+
fetched = PARTS.fetch(path, [path]).map { |part| fetch(part, opts[:date], work_dir, opts[:log]) }
|
|
52
|
+
dirs = fetched.map(&:first)
|
|
53
|
+
opts = opts.merge(temp_directory: File.join(work_dir, "duckdb_tmp"))
|
|
54
|
+
ml = ml_files(dirs, work_dir, opts)
|
|
55
|
+
out = File.join(out_dir, "#{output_name(path, opts[:name])}.parquet")
|
|
56
|
+
opts[:log].puts "building #{out}"
|
|
57
|
+
run_duckdb(sql(dirs, "#{out}.tmp", opts, ml_files: ml), opts[:duckdb])
|
|
58
|
+
File.rename("#{out}.tmp", out)
|
|
59
|
+
write_metadata(out, path, fetched.map(&:last), opts)
|
|
60
|
+
out
|
|
61
|
+
ensure
|
|
62
|
+
cleanup(work_dir, dirs, opts[:keep_downloads]) if dirs
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def output_name(path, name)
|
|
66
|
+
name ||= File.basename(path)
|
|
67
|
+
raise ArgumentError, "bad output name #{name.inspect}" unless name.match?(/\A[a-z0-9-]+\z/)
|
|
68
|
+
|
|
69
|
+
name
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def region_path(region)
|
|
73
|
+
region = region.to_s
|
|
74
|
+
raise ArgumentError, "unknown region: #{region.inspect}" unless region.match?(REGION)
|
|
75
|
+
|
|
76
|
+
region.include?("/") ? region : "#{US}/#{region}"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def check_footprints(source)
|
|
80
|
+
return if source.nil? || Footprints::SOURCES.include?(source.to_s)
|
|
81
|
+
|
|
82
|
+
raise ArgumentError, "unknown ml_footprints source #{source.inspect} (have: #{Footprints::SOURCES.join(", ")})"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Downloads and unzips one Geofabrik extract; returns [folder with its
|
|
86
|
+
# shapefiles, extract date].
|
|
87
|
+
#
|
|
88
|
+
# Unless the whole zip is already on disk, only the layers used are
|
|
89
|
+
# downloaded, with range requests (ZipRange); a server or zip that can't
|
|
90
|
+
# do that gets the whole file instead.
|
|
91
|
+
def fetch(part, date, work_dir, log)
|
|
92
|
+
date ||= latest_date(part)
|
|
93
|
+
name = "#{part.tr("/", "-")}-#{date}-free.shp.zip"
|
|
94
|
+
zip = File.join(work_dir, name)
|
|
95
|
+
dir = File.join(work_dir, File.basename(name, ".shp.zip"))
|
|
96
|
+
return [dir, date] if !File.file?(zip) && fetch_layers(part, date, dir, log)
|
|
97
|
+
|
|
98
|
+
unless File.file?(zip)
|
|
99
|
+
log.puts "downloading https://#{HOST}#{url_path(part, date)}"
|
|
100
|
+
Download.fetch(HOST, url_path(part, date), zip, max_bytes: MAX_DOWNLOAD_BYTES)
|
|
101
|
+
end
|
|
102
|
+
check_zip(zip)
|
|
103
|
+
unzip_layers(zip, dir)
|
|
104
|
+
[dir, date]
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def fetch_layers(part, date, dir, log)
|
|
108
|
+
log.puts "downloading the layers used from https://#{HOST}#{url_path(part, date)}"
|
|
109
|
+
patterns = LAYERS.map { |layer| "gis_osm_#{layer}_free_*" }
|
|
110
|
+
ZipRange.extract(HOST, url_path(part, date), patterns, dir, max_bytes: MAX_DOWNLOAD_BYTES)
|
|
111
|
+
true
|
|
112
|
+
rescue ZipRange::Unsupported => e
|
|
113
|
+
log.puts "range download not possible (#{e.message}); downloading the whole file"
|
|
114
|
+
FileUtils.rm_rf(dir)
|
|
115
|
+
false
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def url_path(part, date) = "/#{part}-#{date}-free.shp.zip"
|
|
119
|
+
|
|
120
|
+
# Geofabrik publishes dated files daily; today's may not be up yet.
|
|
121
|
+
def latest_date(part, today: Date.today)
|
|
122
|
+
LOOKBACK_DAYS.times do |i|
|
|
123
|
+
date = (today - i).strftime("%y%m%d")
|
|
124
|
+
return date if Download.exists?(HOST, url_path(part, date))
|
|
125
|
+
end
|
|
126
|
+
raise Error, "Geofabrik has no dated shapefile for #{part} in the last #{LOOKBACK_DAYS} days " \
|
|
127
|
+
"(large regions are only offered as sub-regions)"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Geofabrik publishes no checksum for shapefiles; the zip's CRCs catch a
|
|
131
|
+
# truncated or corrupt download.
|
|
132
|
+
def check_zip(zip)
|
|
133
|
+
_, err, status = Open3.capture3("unzip", "-tq", zip)
|
|
134
|
+
return if status.success?
|
|
135
|
+
|
|
136
|
+
File.delete(zip)
|
|
137
|
+
raise Error, "corrupt download #{File.basename(zip)} (deleted): #{err.strip[0, 200]}"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def unzip_layers(zip, dir)
|
|
141
|
+
FileUtils.mkdir_p(dir)
|
|
142
|
+
patterns = LAYERS.map { |layer| "gis_osm_#{layer}_free_*" }
|
|
143
|
+
_, err, status = Open3.capture3("unzip", "-oq", zip, *patterns, "-d", dir)
|
|
144
|
+
raise Error, "unzip failed: #{err.strip[0, 200]}" unless status.success?
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Footprint tiles covering the region's extent, or [] when not asked for.
|
|
148
|
+
def ml_files(dirs, work_dir, opts)
|
|
149
|
+
return [] unless opts[:ml_footprints]
|
|
150
|
+
|
|
151
|
+
bbox = extent(dirs, opts)
|
|
152
|
+
Footprints.microsoft(bbox, File.join(work_dir, "ml_footprints"), log: opts[:log]).tap do |files|
|
|
153
|
+
opts[:ml_bbox] = bbox
|
|
154
|
+
raise Error, "no ML footprint tiles cover #{bbox.inspect}" if files.empty?
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# [min_lng, min_lat, max_lng, max_lat] of the region's roads.
|
|
159
|
+
def extent(dirs, opts)
|
|
160
|
+
query = "SELECT min(ST_XMin(geom)) a, min(ST_YMin(geom)) b, max(ST_XMax(geom)) c, max(ST_YMax(geom)) d " \
|
|
161
|
+
"FROM roads_src;"
|
|
162
|
+
out = run_duckdb([setup(opts), "INSTALL spatial; LOAD spatial;", layer_view("roads", dirs), query].join("\n"),
|
|
163
|
+
opts[:duckdb], "-json")
|
|
164
|
+
row = JSON.parse(out).first
|
|
165
|
+
%w[a b c d].map { |k| Float(row[k]) }
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# The build query reads one view per layer; large layers come in parts
|
|
169
|
+
# (_1, _2, ...) and multi-part regions in several folders.
|
|
170
|
+
def sql(dirs, out, opts, ml_files: [])
|
|
171
|
+
views = LAYERS.map { |layer| layer_view(layer, dirs) }
|
|
172
|
+
[setup(opts), "INSTALL spatial; LOAD spatial;", *views, ml_view(ml_files, opts[:ml_bbox]),
|
|
173
|
+
"SET VARIABLE out_path = #{quote(out)};", File.read(SQL_FILE)].join("\n")
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def layer_view(layer, dirs)
|
|
177
|
+
files = dirs.flat_map { |d| Dir.glob(File.join(d, "gis_osm_#{layer}_free_*.shp")) }.sort
|
|
178
|
+
raise Error, "no #{layer} shapefile in #{dirs.join(", ")}" if files.empty?
|
|
179
|
+
|
|
180
|
+
reads = files.map { |f| "SELECT * FROM ST_Read(#{quote(f)})" }.join(" UNION ALL BY NAME ")
|
|
181
|
+
"CREATE VIEW #{layer}_src AS #{reads};"
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# ML footprints within the region's extent (tiles overhang it), or an
|
|
185
|
+
# empty view.
|
|
186
|
+
def ml_view(files, bbox)
|
|
187
|
+
return "CREATE VIEW ml_buildings_src AS SELECT NULL::GEOMETRY AS geom WHERE false;" if files.empty?
|
|
188
|
+
|
|
189
|
+
list = files.map { |f| quote(f) }.join(", ")
|
|
190
|
+
min_lng, min_lat, max_lng, max_lat = bbox.map { |v| Float(v) }
|
|
191
|
+
"CREATE VIEW ml_buildings_src AS SELECT geom FROM (SELECT ST_GeomFromGeoJSON(geometry) AS geom " \
|
|
192
|
+
"FROM read_json([#{list}], format = 'newline_delimited', compression = 'gzip', " \
|
|
193
|
+
"columns = {geometry: 'JSON'})) WHERE ST_Intersects(geom, ST_MakeEnvelope(" \
|
|
194
|
+
"#{min_lng}, #{min_lat}, #{max_lng}, #{max_lat}));"
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def setup(opts)
|
|
198
|
+
ext = opts[:extension_directory]
|
|
199
|
+
[("SET extension_directory = #{quote(File.expand_path(ext))};" if ext),
|
|
200
|
+
"SET memory_limit = #{quote(opts[:memory_limit])};",
|
|
201
|
+
"SET threads = #{Integer(opts[:threads])};",
|
|
202
|
+
("SET temp_directory = #{quote(opts[:temp_directory])};" if opts[:temp_directory])].compact.join("\n")
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def run_duckdb(sql, duckdb, *)
|
|
206
|
+
out, err, status = Open3.capture3(duckdb, *, stdin_data: sql)
|
|
207
|
+
raise Error, "duckdb failed: #{err.strip[0, 500]}" unless status.success? && err.strip.empty?
|
|
208
|
+
|
|
209
|
+
out
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Writes <region>.json next to the Parquet file: what it holds, where it
|
|
213
|
+
# came from and how to attribute it. Catalogue reads these.
|
|
214
|
+
def write_metadata(parquet, path, dates, opts)
|
|
215
|
+
stats = run_duckdb([setup(opts), "SELECT feature, count(*) AS n, min(min_lng) a, min(min_lat) b, " \
|
|
216
|
+
"max(max_lng) c, max(max_lat) d FROM read_parquet(#{quote(parquet)}) " \
|
|
217
|
+
"GROUP BY feature;"].join("\n"), opts[:duckdb], "-json")
|
|
218
|
+
rows = JSON.parse(stats)
|
|
219
|
+
File.write(parquet.sub(/\.parquet\z/, ".json"), JSON.pretty_generate(metadata(parquet, path, dates, rows, opts)))
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def metadata(parquet, path, dates, rows, opts)
|
|
223
|
+
source_dates = dates.map { |d| Date.strptime(d, "%y%m%d").iso8601 }
|
|
224
|
+
{
|
|
225
|
+
region: path, parquet: File.basename(parquet), format: 1,
|
|
226
|
+
source: "Geofabrik OpenStreetMap shapefiles", source_dates: source_dates,
|
|
227
|
+
built_at: Time.now.utc.iso8601, ml_footprints: opts[:ml_footprints],
|
|
228
|
+
bbox: [rows.map { |r| r["a"] }.min, rows.map { |r| r["b"] }.min, rows.map { |r| r["c"] }.max,
|
|
229
|
+
rows.map { |r| r["d"] }.max],
|
|
230
|
+
counts: rows.to_h { |r| [r["feature"], r["n"]] },
|
|
231
|
+
licence: "ODbL-1.0", attribution: attribution(opts[:ml_footprints])
|
|
232
|
+
}
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def attribution(ml_footprints)
|
|
236
|
+
["© OpenStreetMap contributors", ("Microsoft Global ML Building Footprints" if ml_footprints)].compact
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def cleanup(work_dir, dirs, keep_downloads)
|
|
240
|
+
dirs.each { |d| FileUtils.rm_rf(d) }
|
|
241
|
+
FileUtils.rm_rf(File.join(work_dir, "duckdb_tmp"))
|
|
242
|
+
return if keep_downloads
|
|
243
|
+
|
|
244
|
+
Dir.glob(File.join(work_dir, "*.shp.zip")).each { |z| File.delete(z) }
|
|
245
|
+
FileUtils.rm_rf(File.join(work_dir, "ml_footprints"))
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def quote(text)
|
|
249
|
+
"'#{text.to_s.gsub("'", "''")}'"
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Curtilage
|
|
6
|
+
# The built regions available to look features up in, from the
|
|
7
|
+
# <region>.json files Build writes next to each Parquet file. Picks the
|
|
8
|
+
# file(s) covering a pin, so callers don't track which state is which.
|
|
9
|
+
class Catalogue
|
|
10
|
+
Region = Struct.new(:name, :parquet, :bbox, :metadata, keyword_init: true) do
|
|
11
|
+
def covers?(pin, margin)
|
|
12
|
+
lng, lat = pin
|
|
13
|
+
lng.between?(bbox[0] - margin, bbox[2] + margin) && lat.between?(bbox[1] - margin, bbox[3] + margin)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
attr_reader :regions
|
|
18
|
+
|
|
19
|
+
# dir: a folder of <region>.json and <region>.parquet files. base_url:
|
|
20
|
+
# where the Parquet files are served from instead (s3://bucket/path,
|
|
21
|
+
# https://...), when the metadata is local but the data isn't.
|
|
22
|
+
def self.from_dir(dir, base_url: nil)
|
|
23
|
+
regions = Dir.glob(File.join(dir, "*.json")).filter_map do |file|
|
|
24
|
+
meta = JSON.parse(File.read(file))
|
|
25
|
+
next unless meta["format"] == 1 && meta["parquet"] && meta["bbox"]&.size == 4
|
|
26
|
+
|
|
27
|
+
parquet = base_url ? "#{base_url.chomp("/")}/#{meta["parquet"]}" : File.join(dir, meta["parquet"])
|
|
28
|
+
Region.new(name: meta["region"], parquet: parquet, bbox: meta["bbox"].map { |v| Float(v) }, metadata: meta)
|
|
29
|
+
end
|
|
30
|
+
new(regions)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(regions)
|
|
34
|
+
@regions = regions
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Parquet files whose extent covers the pin (more than one near a border).
|
|
38
|
+
def files_for(pin, margin: Lookup::DEFAULT_RADIUS_DEG)
|
|
39
|
+
@regions.select { |r| r.covers?(pin, margin) }.map(&:parquet)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def attribution(files)
|
|
43
|
+
@regions.select { |r| files.include?(r.parquet) }.flat_map { |r| Array(r.metadata["attribution"]) }.uniq
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Settings shared by every call, so callers don't pass them each time.
|
|
4
|
+
# Defaults come from the environment:
|
|
5
|
+
#
|
|
6
|
+
# CURTILAGE_DUCKDB path to the DuckDB CLI (default: "duckdb" on PATH)
|
|
7
|
+
# CURTILAGE_EXTENSIONS DuckDB extension folder (default: DuckDB's own)
|
|
8
|
+
# CURTILAGE_CACHE_DIR where on-demand builds are kept (default: a
|
|
9
|
+
# temporary folder, deleted after each run)
|
|
10
|
+
# CURTILAGE_SOURCE geofabrik (default) or overture
|
|
11
|
+
#
|
|
12
|
+
# Curtilage.configure do |c|
|
|
13
|
+
# c.cache_dir = "/var/cache/curtilage"
|
|
14
|
+
# c.source = :overture
|
|
15
|
+
# end
|
|
16
|
+
module Curtilage
|
|
17
|
+
Config = Struct.new(:duckdb, :extension_directory, :cache_dir, :source, keyword_init: true)
|
|
18
|
+
|
|
19
|
+
def self.config
|
|
20
|
+
@config ||= Config.new(
|
|
21
|
+
duckdb: ENV.fetch("CURTILAGE_DUCKDB", "duckdb"),
|
|
22
|
+
extension_directory: ENV.fetch("CURTILAGE_EXTENSIONS", nil),
|
|
23
|
+
cache_dir: ENV.fetch("CURTILAGE_CACHE_DIR", nil),
|
|
24
|
+
source: ENV.fetch("CURTILAGE_SOURCE", "geofabrik").to_sym
|
|
25
|
+
)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def self.configure
|
|
29
|
+
yield config
|
|
30
|
+
config
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Back to the environment defaults (mostly for tests).
|
|
34
|
+
def self.reset_config!
|
|
35
|
+
@config = nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The DuckDB settings for Lookup, Build and Overture: explicit options win
|
|
39
|
+
# over the configuration.
|
|
40
|
+
def self.duckdb_options(options = {})
|
|
41
|
+
defaults = { duckdb: config.duckdb, extension_directory: config.extension_directory }.compact
|
|
42
|
+
defaults.merge(options.compact)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "uri"
|
|
5
|
+
|
|
6
|
+
module Curtilage
|
|
7
|
+
# HTTPS downloads of public map data, from an allowed host only, streamed
|
|
8
|
+
# to disk with a size cap: Geofabrik extracts and region index, Overture's
|
|
9
|
+
# file index, Microsoft's footprint tiles. (DuckDB itself reads Overture
|
|
10
|
+
# and bucket files.) Nothing else in the gem talks to the network.
|
|
11
|
+
module Download
|
|
12
|
+
class Error < StandardError; end
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def exists?(host, path)
|
|
17
|
+
start(host) { |http| http.head(path).is_a?(Net::HTTPSuccess) }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Writes to dest.part, then renames, so a dest file is always complete.
|
|
21
|
+
def fetch(host, path, dest, max_bytes:)
|
|
22
|
+
start(host) do |http|
|
|
23
|
+
http.request_get(path) do |res|
|
|
24
|
+
raise Error, "HTTP #{res.code} for https://#{host}#{path}" unless res.is_a?(Net::HTTPSuccess)
|
|
25
|
+
raise Error, "too large: https://#{host}#{path}" if res.content_length.to_i > max_bytes
|
|
26
|
+
|
|
27
|
+
stream(res, "#{dest}.part", max_bytes)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
File.rename("#{dest}.part", dest)
|
|
31
|
+
dest
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def read(host, path, max_bytes:)
|
|
35
|
+
start(host) do |http|
|
|
36
|
+
res = http.get(path)
|
|
37
|
+
raise Error, "HTTP #{res.code} for https://#{host}#{path}" unless res.is_a?(Net::HTTPSuccess)
|
|
38
|
+
raise Error, "too large: https://#{host}#{path}" if res.body.bytesize > max_bytes
|
|
39
|
+
|
|
40
|
+
res.body
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def stream(res, dest, max_bytes)
|
|
45
|
+
bytes = 0
|
|
46
|
+
File.open(dest, "wb") do |f|
|
|
47
|
+
res.read_body do |chunk|
|
|
48
|
+
bytes += chunk.bytesize
|
|
49
|
+
raise Error, "download over #{max_bytes} bytes" if bytes > max_bytes
|
|
50
|
+
|
|
51
|
+
f.write(chunk)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def start(host, &)
|
|
57
|
+
Net::HTTP.start(host, 443, use_ssl: true, open_timeout: 30, read_timeout: 120, &)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The path of an https URL on host, or nil for anything else.
|
|
61
|
+
def path_on(url, host, prefix)
|
|
62
|
+
uri = URI.parse(url)
|
|
63
|
+
return nil unless uri.scheme == "https" && uri.host == host && uri.path.start_with?(prefix)
|
|
64
|
+
return nil if uri.path.include?("..")
|
|
65
|
+
|
|
66
|
+
uri.path
|
|
67
|
+
rescue URI::InvalidURIError
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Curtilage
|
|
4
|
+
# The OSM features around a pin, projected to metres around it.
|
|
5
|
+
#
|
|
6
|
+
# Input is a list of hashes (string or symbol keys) with "feature"
|
|
7
|
+
# ("building", "ml_building", "parking", "road" or "railway"), "geometry"
|
|
8
|
+
# (a GeoJSON geometry hash), and optionally "id", "name" and "highway"
|
|
9
|
+
# (the class: road or railway class, or building type). Map data is
|
|
10
|
+
# untrusted, so oversized inputs and broken geometry are dropped and
|
|
11
|
+
# counted, not raised.
|
|
12
|
+
class Features
|
|
13
|
+
MAX_FEATURES = 20_000
|
|
14
|
+
MAX_VERTICES = 5_000
|
|
15
|
+
|
|
16
|
+
# category: the building type from OSM (house, parking, roof, ...), or nil.
|
|
17
|
+
Polygon = Struct.new(:id, :kind, :name, :xy, :area, :category, keyword_init: true) do
|
|
18
|
+
def hull
|
|
19
|
+
@hull ||= Geometry.convex_hull(xy)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def rectangle
|
|
23
|
+
@rectangle ||= Geometry.minimum_rectangle(xy)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# [min_x, min_y, max_x, max_y], for a cheap pre-check.
|
|
27
|
+
def bounds
|
|
28
|
+
@bounds ||= Geometry.bounds(xy)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def near?(other, margin = 0)
|
|
32
|
+
Geometry.bounds_near?(bounds, other.bounds, margin)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
Line = Struct.new(:id, :highway, :name, :xy, keyword_init: true)
|
|
36
|
+
|
|
37
|
+
attr_reader :pin, :buildings, :ml_buildings, :parking, :roads, :railways, :dropped
|
|
38
|
+
|
|
39
|
+
def initialize(pin, features)
|
|
40
|
+
@pin = pin
|
|
41
|
+
@buildings = []
|
|
42
|
+
@ml_buildings = []
|
|
43
|
+
@parking = []
|
|
44
|
+
@roads = []
|
|
45
|
+
@railways = []
|
|
46
|
+
@dropped = 0
|
|
47
|
+
@seen = {}
|
|
48
|
+
Array(features).first(MAX_FEATURES).each { |f| add(f) }
|
|
49
|
+
@dropped += [Array(features).size - MAX_FEATURES, 0].max
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# A copy whose buildings are the given list (see Generator buildings:).
|
|
53
|
+
def with_buildings(list)
|
|
54
|
+
dup.tap { |copy| copy.instance_variable_set(:@buildings, list) }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def add(feature)
|
|
60
|
+
kind = field(feature, :feature).to_s
|
|
61
|
+
# A region built from several extracts repeats OSM features along the
|
|
62
|
+
# seam. ML footprints' ids are only unique within one file, so they're
|
|
63
|
+
# never treated as repeats.
|
|
64
|
+
id = field(feature, :id)
|
|
65
|
+
if id && kind != "ml_building"
|
|
66
|
+
return if @seen.key?([kind, id])
|
|
67
|
+
|
|
68
|
+
@seen[[kind, id]] = true
|
|
69
|
+
end
|
|
70
|
+
geometry = field(feature, :geometry)
|
|
71
|
+
parts = parts_of(geometry)
|
|
72
|
+
return @dropped += 1 if parts.nil? || parts.empty?
|
|
73
|
+
|
|
74
|
+
parts.each { |coords| add_part(feature, kind, coords) }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def add_part(feature, kind, coords)
|
|
78
|
+
return @dropped += 1 unless coords.is_a?(Array) && coords.size <= MAX_VERTICES
|
|
79
|
+
return @dropped += 1 unless Geometry.valid_coordinates?(coords)
|
|
80
|
+
|
|
81
|
+
xy = Geometry.project(coords, pin)
|
|
82
|
+
if %w[road railway].include?(kind)
|
|
83
|
+
return @dropped += 1 if xy.size < 2
|
|
84
|
+
|
|
85
|
+
line = Line.new(id: field(feature, :id), highway: field(feature, :highway)&.to_s,
|
|
86
|
+
name: field(feature, :name), xy: xy)
|
|
87
|
+
(kind == "road" ? @roads : @railways) << line
|
|
88
|
+
else
|
|
89
|
+
add_polygon(feature, kind, xy)
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def add_polygon(feature, kind, xy)
|
|
94
|
+
ring = Geometry.clean(Geometry.open_ring(xy))
|
|
95
|
+
area = ring.size >= 3 ? Geometry.area(ring) : 0
|
|
96
|
+
return @dropped += 1 if area < 1 || Geometry.self_intersects?(ring)
|
|
97
|
+
|
|
98
|
+
category = field(feature, :highway).to_s
|
|
99
|
+
polygon = Polygon.new(id: field(feature, :id), kind: kind, name: field(feature, :name),
|
|
100
|
+
xy: Geometry.counter_clockwise(ring), area: area,
|
|
101
|
+
category: category.empty? ? nil : category)
|
|
102
|
+
case kind
|
|
103
|
+
when "building" then @buildings << polygon
|
|
104
|
+
when "ml_building" then @ml_buildings << polygon
|
|
105
|
+
when "parking" then @parking << polygon
|
|
106
|
+
else @dropped += 1
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Outer rings of polygons, or the lines of line strings. Holes are
|
|
111
|
+
# ignored: premises cover a building's courtyard too.
|
|
112
|
+
def parts_of(geometry)
|
|
113
|
+
return nil unless geometry.is_a?(Hash)
|
|
114
|
+
|
|
115
|
+
coords = field(geometry, :coordinates)
|
|
116
|
+
return nil unless coords.is_a?(Array)
|
|
117
|
+
|
|
118
|
+
case field(geometry, :type)
|
|
119
|
+
when "Polygon" then [coords.first].compact
|
|
120
|
+
when "MultiPolygon" then coords.filter_map { |poly| poly.is_a?(Array) ? poly.first : nil }
|
|
121
|
+
when "LineString" then [coords]
|
|
122
|
+
when "MultiLineString" then coords
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def field(hash, key)
|
|
127
|
+
hash[key.to_s] || hash[key]
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|