curtilage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +39 -0
- data/LICENSE.txt +21 -0
- data/README.md +134 -0
- data/SECURITY.md +15 -0
- data/docs/reference.md +140 -0
- data/exe/curtilage +134 -0
- data/lib/curtilage/audit.rb +88 -0
- data/lib/curtilage/auto_data.rb +94 -0
- data/lib/curtilage/build.rb +252 -0
- data/lib/curtilage/catalogue.rb +46 -0
- data/lib/curtilage/config.rb +44 -0
- data/lib/curtilage/download.rb +71 -0
- data/lib/curtilage/features.rb +130 -0
- data/lib/curtilage/footprints.rb +82 -0
- data/lib/curtilage/generator.rb +578 -0
- data/lib/curtilage/generator_lot.rb +158 -0
- data/lib/curtilage/generator_store.rb +31 -0
- data/lib/curtilage/geometry.rb +360 -0
- data/lib/curtilage/lookup.rb +110 -0
- data/lib/curtilage/overture.rb +195 -0
- data/lib/curtilage/regions.rb +85 -0
- data/lib/curtilage/validation.rb +89 -0
- data/lib/curtilage/version.rb +5 -0
- data/lib/curtilage/zip_range.rb +234 -0
- data/lib/curtilage.rb +126 -0
- data/sql/build_from_shapefiles.sql +52 -0
- metadata +77 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "open3"
|
|
5
|
+
require "tmpdir"
|
|
6
|
+
|
|
7
|
+
module Curtilage
|
|
8
|
+
# Features straight from Overture Maps' public GeoParquet release, read in
|
|
9
|
+
# place: nothing is downloaded up front and nothing has to be hosted.
|
|
10
|
+
# Overture's file index (a small GeoParquet listing every file's extent)
|
|
11
|
+
# narrows each pin to the one or two files covering it.
|
|
12
|
+
#
|
|
13
|
+
# Buildings Overture took from OpenStreetMap come back as "building"; its
|
|
14
|
+
# other sources (machine-learned footprints and others) as "ml_building",
|
|
15
|
+
# used only with buildings: :osm_filled or :ml. Each query tells the
|
|
16
|
+
# storage host roughly which area is being read.
|
|
17
|
+
module Overture
|
|
18
|
+
class Error < StandardError; end
|
|
19
|
+
|
|
20
|
+
STAC_HOST = "stac.overturemaps.org"
|
|
21
|
+
CATALOG = "/catalog.json"
|
|
22
|
+
MAX_INDEX_BYTES = 20_000_000
|
|
23
|
+
MAX_OUTPUT_BYTES = 200_000_000
|
|
24
|
+
ROAD_CLASSES = %w[motorway trunk primary secondary tertiary unclassified residential service].freeze
|
|
25
|
+
RAIL_CLASSES = { "standard_gauge" => "rail", "light_rail" => "light_rail", "subway" => "subway",
|
|
26
|
+
"narrow_gauge" => "narrow_gauge" }.freeze
|
|
27
|
+
COLLECTIONS = %w[building segment infrastructure].freeze
|
|
28
|
+
GROUP_CELL_DEG = 0.05
|
|
29
|
+
ATTRIBUTION = ["© OpenStreetMap contributors", "© Overture Maps Foundation"].freeze
|
|
30
|
+
|
|
31
|
+
module_function
|
|
32
|
+
|
|
33
|
+
# A list of feature lists, in pin order. options: duckdb:,
|
|
34
|
+
# extension_directory:, radius_deg:, release: (default: the latest).
|
|
35
|
+
def features_many(pins, **options)
|
|
36
|
+
options = Curtilage.duckdb_options(options)
|
|
37
|
+
radius = Float(options.fetch(:radius_deg, Lookup::DEFAULT_RADIUS_DEG))
|
|
38
|
+
boxes = pins.map { |pin| box(pin, radius) }
|
|
39
|
+
Dir.mktmpdir("curtilage-overture") do |dir|
|
|
40
|
+
index = File.join(dir, "index.parquet")
|
|
41
|
+
File.binwrite(index, Download.read(STAC_HOST, "/#{release(options)}/collections.parquet",
|
|
42
|
+
max_bytes: MAX_INDEX_BYTES))
|
|
43
|
+
files = files_for(index, boxes, options)
|
|
44
|
+
rows = run(data_sql(boxes, files, options), options)
|
|
45
|
+
group(rows, pins.size)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def release(options)
|
|
50
|
+
return options[:release] if options[:release]
|
|
51
|
+
|
|
52
|
+
latest = JSON.parse(Download.read(STAC_HOST, CATALOG, max_bytes: 1_000_000))["latest"].to_s
|
|
53
|
+
raise Error, "no latest Overture release in the catalog" unless latest.match?(/\A[\d.-]+\z/)
|
|
54
|
+
|
|
55
|
+
latest
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def box(pin, radius)
|
|
59
|
+
lng = Float(pin[0])
|
|
60
|
+
lat = Float(pin[1])
|
|
61
|
+
raise ArgumentError, "pin out of range: #{pin.inspect}" unless lng.abs <= 180 && lat.abs <= 90
|
|
62
|
+
|
|
63
|
+
[lng - radius, lat - radius, lng + radius, lat + radius]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# {[pin_index, collection] => [s3 urls]} from the local index file.
|
|
67
|
+
def files_for(index, boxes, options)
|
|
68
|
+
selects = boxes.each_with_index.map do |(x0, y0, x1, y1), i|
|
|
69
|
+
"SELECT #{i} AS pin_index, collection, assets.aws.alternate.s3.href AS href " \
|
|
70
|
+
"FROM read_parquet(#{quote(index)}) WHERE collection IN (#{COLLECTIONS.map { |c| quote(c) }.join(", ")}) " \
|
|
71
|
+
"AND bbox.xmin <= #{x1} AND bbox.xmax >= #{x0} AND bbox.ymin <= #{y1} AND bbox.ymax >= #{y0}"
|
|
72
|
+
end
|
|
73
|
+
rows = run([setup(options, remote: false), "#{selects.join(" UNION ALL ")};"].join("\n"), options)
|
|
74
|
+
rows.group_by { |r| [r["pin_index"], r["collection"]] }.transform_values do |list|
|
|
75
|
+
list.map { |r| r["href"] }.select { |h| h.to_s.start_with?("s3://overturemaps-") }
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# One scan per collection and set of files, shared by every pin those
|
|
80
|
+
# files cover: each file's metadata and row groups are read once, then
|
|
81
|
+
# rows are matched back to the pins whose boxes they fall in.
|
|
82
|
+
def data_sql(boxes, files, options)
|
|
83
|
+
scans = COLLECTIONS.flat_map do |collection|
|
|
84
|
+
groups = boxes.each_index.group_by { |i| [files[[i, collection]].to_a.sort, cell(boxes[i])] }
|
|
85
|
+
groups.filter_map do |(list, _), indexes|
|
|
86
|
+
grouped_scan(collection, list, indexes, boxes) unless list.empty?
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
return "SELECT 1 WHERE false;" if scans.empty?
|
|
90
|
+
|
|
91
|
+
[setup(options, remote: true), boxes_table(boxes), "#{scans.join(" UNION ALL ")};"].join("\n")
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def boxes_table(boxes)
|
|
95
|
+
rows = boxes.each_with_index.map { |(x0, y0, x1, y1), i| "(#{i}, #{x0}, #{y0}, #{x1}, #{y1})" }
|
|
96
|
+
"CREATE TEMP TABLE boxes (pin_index INTEGER, x0 DOUBLE, y0 DOUBLE, x1 DOUBLE, y1 DOUBLE); " \
|
|
97
|
+
"INSERT INTO boxes VALUES #{rows.join(", ")};"
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Pins are grouped within a 0.05 degree cell (about 5 km), so a group's
|
|
101
|
+
# envelope stays small.
|
|
102
|
+
def cell(box)
|
|
103
|
+
[(box[0] / GROUP_CELL_DEG).floor, (box[1] / GROUP_CELL_DEG).floor]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# The envelope is a plain range DuckDB can skip row groups with; the OR
|
|
107
|
+
# of the pins' own boxes then drops what's between them.
|
|
108
|
+
def grouped_scan(collection, files, indexes, boxes)
|
|
109
|
+
group = indexes.map { |i| boxes[i] }
|
|
110
|
+
envelope = [group.map { |b| b[0] }.min, group.map { |b| b[1] }.min, group.map { |b| b[2] }.max,
|
|
111
|
+
group.map { |b| b[3] }.max]
|
|
112
|
+
any = group.map { |b| "(#{box_filter("bbox", b)})" }.join(" OR ")
|
|
113
|
+
"SELECT p.pin_index, d.id, d.feature, d.highway, d.name, d.geometry FROM (SELECT #{columns(collection)}, " \
|
|
114
|
+
"bbox AS box FROM read_parquet([#{files.map { |f| quote(f) }.join(", ")}]) " \
|
|
115
|
+
"WHERE #{box_filter("bbox", envelope)} AND (#{any}) AND #{filter(collection)}) d " \
|
|
116
|
+
"JOIN boxes p ON p.pin_index IN (#{indexes.join(", ")}) AND d.box.xmin <= p.x1 AND d.box.xmax >= p.x0 " \
|
|
117
|
+
"AND d.box.ymin <= p.y1 AND d.box.ymax >= p.y0"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def box_filter(column, box)
|
|
121
|
+
x0, y0, x1, y1 = box
|
|
122
|
+
"#{column}.xmin <= #{x1} AND #{column}.xmax >= #{x0} AND #{column}.ymin <= #{y1} AND #{column}.ymax >= #{y0}"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Buildings Overture took from OSM are buildings; its other sources are
|
|
126
|
+
# treated as ML footprints. Roads and rail map to the build's classes.
|
|
127
|
+
def columns(collection)
|
|
128
|
+
case collection
|
|
129
|
+
when "building"
|
|
130
|
+
"id, CASE WHEN sources[1].dataset = 'OpenStreetMap' THEN 'building' ELSE 'ml_building' END AS feature, " \
|
|
131
|
+
"class AS highway, names.\"primary\" AS name, ST_AsGeoJSON(geometry)::VARCHAR AS geometry"
|
|
132
|
+
when "segment"
|
|
133
|
+
rail = RAIL_CLASSES.map { |k, v| "WHEN #{quote(k)} THEN #{quote(v)}" }.join(" ")
|
|
134
|
+
"id, CASE subtype WHEN 'road' THEN 'road' ELSE 'railway' END AS feature, " \
|
|
135
|
+
"CASE WHEN subtype = 'rail' THEN (CASE class #{rail} END) WHEN subclass = 'link' THEN class || '_link' " \
|
|
136
|
+
"ELSE class END AS highway, names.\"primary\" AS name, ST_AsGeoJSON(geometry)::VARCHAR AS geometry"
|
|
137
|
+
else
|
|
138
|
+
"id, 'parking' AS feature, NULL AS highway, names.\"primary\" AS name, " \
|
|
139
|
+
"ST_AsGeoJSON(geometry)::VARCHAR AS geometry"
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Ground-level roads and rail; parking areas (not points); buildings
|
|
144
|
+
# above ground.
|
|
145
|
+
def filter(collection)
|
|
146
|
+
case collection
|
|
147
|
+
when "building" then "NOT coalesce(is_underground, false)"
|
|
148
|
+
when "segment"
|
|
149
|
+
"((subtype = 'road' AND class IN (#{ROAD_CLASSES.map { |c| quote(c) }.join(", ")})) " \
|
|
150
|
+
"OR (subtype = 'rail' AND class IN (#{RAIL_CLASSES.keys.map { |c| quote(c) }.join(", ")}))) " \
|
|
151
|
+
"AND NOT list_has_any(flatten(list_transform(coalesce(road_flags, []) || coalesce(rail_flags, []), " \
|
|
152
|
+
"f -> f.\"values\")), ['is_tunnel', 'is_bridge'])"
|
|
153
|
+
else
|
|
154
|
+
"subtype = 'transit' AND class = 'parking' AND ST_GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def setup(options, remote:)
|
|
159
|
+
ext = options[:extension_directory]
|
|
160
|
+
[("SET extension_directory = #{quote(File.expand_path(ext))};" if ext), "INSTALL spatial; LOAD spatial;",
|
|
161
|
+
# Pins in the same files share one read of each file's metadata.
|
|
162
|
+
(if remote
|
|
163
|
+
"INSTALL httpfs; LOAD httpfs; SET s3_region = 'us-west-2'; SET parquet_metadata_cache = true; " \
|
|
164
|
+
"SET enable_http_metadata_cache = true;"
|
|
165
|
+
end)].compact.join("\n")
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Remote reads occasionally stall and DuckDB exits without a message;
|
|
169
|
+
# one retry gets past that.
|
|
170
|
+
def run(sql, options, attempts: 2)
|
|
171
|
+
out, err, status = Open3.capture3(options.fetch(:duckdb, "duckdb"), "-json", stdin_data: sql)
|
|
172
|
+
unless status.success?
|
|
173
|
+
return run(sql, options, attempts: attempts - 1) if err.strip.empty? && attempts > 1
|
|
174
|
+
|
|
175
|
+
raise Error, "duckdb failed: #{err.strip.empty? ? "no message (network?)" : err.strip[0, 500]}"
|
|
176
|
+
end
|
|
177
|
+
raise Error, "duckdb output too large" if out.bytesize > MAX_OUTPUT_BYTES
|
|
178
|
+
|
|
179
|
+
out.strip.empty? ? [] : JSON.parse(out)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def group(rows, count)
|
|
183
|
+
lists = Array.new(count) { [] }
|
|
184
|
+
rows.each do |row|
|
|
185
|
+
index = row.delete("pin_index")
|
|
186
|
+
lists[index] << row.merge("geometry" => JSON.parse(row["geometry"])) if index.is_a?(Integer)
|
|
187
|
+
end
|
|
188
|
+
lists
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def quote(text)
|
|
192
|
+
"'#{text.to_s.gsub("'", "''")}'"
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Curtilage
|
|
6
|
+
# Geofabrik's region index (outlines plus download links), used to find
|
|
7
|
+
# which region a pin is in without any prebuilt data. Only regions offered
|
|
8
|
+
# as shapefiles count; where regions nest, the smallest wins (California
|
|
9
|
+
# resolves to norcal or socal).
|
|
10
|
+
class Regions
|
|
11
|
+
HOST = "download.geofabrik.de"
|
|
12
|
+
INDEX = "/index-v1.json"
|
|
13
|
+
MAX_INDEX_BYTES = 50_000_000
|
|
14
|
+
|
|
15
|
+
Region = Struct.new(:id, :path, :bbox, :rings, :extent_area, keyword_init: true) do
|
|
16
|
+
def covers?(point)
|
|
17
|
+
lng, lat = point
|
|
18
|
+
return false unless lng.between?(bbox[0], bbox[2]) && lat.between?(bbox[1], bbox[3])
|
|
19
|
+
|
|
20
|
+
rings.any? { |ring| Geometry.contains?(ring, point) }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.download
|
|
25
|
+
parse(Download.read(HOST, INDEX, max_bytes: MAX_INDEX_BYTES))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def self.parse(json)
|
|
29
|
+
features = JSON.parse(json).fetch("features")
|
|
30
|
+
new(features.filter_map { |f| region(f) })
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.region(feature)
|
|
34
|
+
props = feature["properties"] || {}
|
|
35
|
+
path = path_from_url(props.dig("urls", "shp"))
|
|
36
|
+
rings = outer_rings(feature["geometry"])
|
|
37
|
+
return nil unless path && !rings.empty?
|
|
38
|
+
|
|
39
|
+
points = rings.flatten(1)
|
|
40
|
+
bbox = [points.map(&:first).min, points.map(&:last).min, points.map(&:first).max, points.map(&:last).max]
|
|
41
|
+
Region.new(id: props["id"], path: path, bbox: bbox, rings: rings,
|
|
42
|
+
extent_area: (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# ".../north-america/us/tennessee/tennessee-latest-free.shp.zip" ->
|
|
46
|
+
# "north-america/us/tennessee", the path Build takes.
|
|
47
|
+
def self.path_from_url(url)
|
|
48
|
+
path = Download.path_on(url.to_s, HOST, "/")
|
|
49
|
+
match = path&.match(%r{\A/((?:[a-z0-9-]+/)*?)([a-z0-9-]+)-latest-free\.shp\.zip\z})
|
|
50
|
+
return nil unless match
|
|
51
|
+
|
|
52
|
+
dirs = match[1].split("/")
|
|
53
|
+
dirs.pop if dirs.last == match[2]
|
|
54
|
+
(dirs + [match[2]]).join("/")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def self.outer_rings(geometry)
|
|
58
|
+
case geometry&.dig("type")
|
|
59
|
+
when "Polygon" then [geometry["coordinates"].first].compact
|
|
60
|
+
when "MultiPolygon" then geometry["coordinates"].filter_map(&:first)
|
|
61
|
+
else []
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
attr_reader :regions
|
|
66
|
+
|
|
67
|
+
def initialize(regions)
|
|
68
|
+
@regions = regions
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The smallest region containing the point, or nil.
|
|
72
|
+
def for(point)
|
|
73
|
+
@regions.select { |r| r.covers?(point) }.min_by(&:extent_area)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Regions holding any part of the box around the pin: one, or two near a
|
|
77
|
+
# border.
|
|
78
|
+
def around(pin, margin)
|
|
79
|
+
lng, lat = pin
|
|
80
|
+
corners = [[lng, lat], [lng - margin, lat - margin], [lng + margin, lat - margin],
|
|
81
|
+
[lng - margin, lat + margin], [lng + margin, lat + margin]]
|
|
82
|
+
corners.filter_map { |c| self.for(c) }.uniq(&:path)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Curtilage
|
|
4
|
+
# Checks a premises polygon against its site pin. Used on every generated
|
|
5
|
+
# polygon before it's offered, and usable on hand-drawn ones.
|
|
6
|
+
#
|
|
7
|
+
# Errors mean the polygon must not be used. Warnings mean a person should
|
|
8
|
+
# look before using it. Defaults come from hand-drawn premises for
|
|
9
|
+
# standalone stores (typically 2,000-15,000 m2, 4-10 vertices, pin inside).
|
|
10
|
+
module Validation
|
|
11
|
+
Result = Struct.new(:errors, :warnings, :metrics) do
|
|
12
|
+
def ok?
|
|
13
|
+
errors.empty?
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
DEFAULTS = {
|
|
18
|
+
min_area_m2: 1_000, # warn below
|
|
19
|
+
max_area_m2: 40_000, # warn above
|
|
20
|
+
max_area_error_m2: 100_000, # error above: a block, not a store's lot
|
|
21
|
+
max_vertex_distance_m: 1_000, # error above
|
|
22
|
+
max_vertices: 50, # error above
|
|
23
|
+
off_centre_m: 200, # warn above
|
|
24
|
+
swap_far_m: 10_000,
|
|
25
|
+
swap_near_m: 1_000,
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
module_function
|
|
29
|
+
|
|
30
|
+
# pin: [lng, lat]. ring: [[lng, lat], ...], closed or not.
|
|
31
|
+
def check(pin, ring, **options)
|
|
32
|
+
opts = DEFAULTS.merge(options)
|
|
33
|
+
return Result.new(["missing"], [], {}) if ring.nil? || ring.empty?
|
|
34
|
+
return Result.new(["invalid_coordinates"], [], {}) unless Geometry.valid_coordinates?(ring)
|
|
35
|
+
|
|
36
|
+
errors = Geometry.closed?(ring) ? [] : ["unclosed"]
|
|
37
|
+
open = Geometry.open_ring(ring)
|
|
38
|
+
return Result.new(errors + ["too_few_points"], [], {}) if open.uniq.size < 3
|
|
39
|
+
|
|
40
|
+
xy = Geometry.project(open, pin)
|
|
41
|
+
metrics = measure(xy).merge(points: open.size)
|
|
42
|
+
errors << "swapped" if swapped?(open, pin, xy, opts)
|
|
43
|
+
errors << "self_intersecting" if Geometry.self_intersects?(xy)
|
|
44
|
+
Result.new(errors + metric_errors(metrics, opts), metric_warnings(metrics, opts), metrics)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def metric_errors(metrics, opts)
|
|
48
|
+
errors = []
|
|
49
|
+
errors << "pin_outside" unless metrics[:pin_inside]
|
|
50
|
+
errors << "too_far_from_pin" if metrics[:max_vertex_distance_m] > opts[:max_vertex_distance_m]
|
|
51
|
+
errors << "too_many_vertices" if metrics[:points] > opts[:max_vertices]
|
|
52
|
+
errors << "too_large" if metrics[:area_m2] > opts[:max_area_error_m2]
|
|
53
|
+
errors
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def metric_warnings(metrics, opts)
|
|
57
|
+
warnings = []
|
|
58
|
+
warnings << "small" if metrics[:area_m2] < opts[:min_area_m2]
|
|
59
|
+
warnings << "large" if metrics[:area_m2] > opts[:max_area_m2]
|
|
60
|
+
warnings << "off_centre" if metrics[:centroid_offset_m] > opts[:off_centre_m]
|
|
61
|
+
warnings
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def measure(xy)
|
|
65
|
+
origin = [0.0, 0.0]
|
|
66
|
+
mbr_area, orientation = Geometry.min_rotated_rectangle(xy)
|
|
67
|
+
area = Geometry.area(xy)
|
|
68
|
+
{
|
|
69
|
+
area_m2: area.round,
|
|
70
|
+
rectangularity: mbr_area.positive? ? (area / mbr_area).round(3) : nil,
|
|
71
|
+
orientation_deg: orientation.round,
|
|
72
|
+
pin_inside: Geometry.contains?(xy, origin),
|
|
73
|
+
max_vertex_distance_m: xy.map { |x, y| Math.hypot(x, y) }.max.round,
|
|
74
|
+
centroid_offset_m: Math.hypot(*Geometry.centroid(xy)).round,
|
|
75
|
+
}
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The ring is far from the pin, but the same ring with lng/lat swapped
|
|
79
|
+
# lands on it: the classic [lat, lng] mistake.
|
|
80
|
+
def swapped?(open, pin, xy, opts)
|
|
81
|
+
return false if Math.hypot(*Geometry.centroid(xy)) < opts[:swap_far_m]
|
|
82
|
+
|
|
83
|
+
swapped = open.map(&:reverse)
|
|
84
|
+
return false unless Geometry.valid_coordinates?(swapped)
|
|
85
|
+
|
|
86
|
+
Math.hypot(*Geometry.centroid(Geometry.project(swapped, pin))) < opts[:swap_near_m]
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "zlib"
|
|
5
|
+
|
|
6
|
+
module Curtilage
|
|
7
|
+
# Extracts chosen members of a remote zip with HTTP range requests, so only
|
|
8
|
+
# the layers the build uses are downloaded (about 70% of a Geofabrik
|
|
9
|
+
# shapefile zip). Members are inflated as they stream in and checked
|
|
10
|
+
# against the zip's CRC-32s. Raises Unsupported for anything unusual
|
|
11
|
+
# (no range support, zip64, encryption, other compression), so the caller
|
|
12
|
+
# can fall back to a full download.
|
|
13
|
+
module ZipRange
|
|
14
|
+
class Unsupported < StandardError; end
|
|
15
|
+
class Corrupt < StandardError; end
|
|
16
|
+
|
|
17
|
+
Entry = Struct.new(:name, :compression, :crc, :compressed_size, :uncompressed_size, :offset, :data_end,
|
|
18
|
+
keyword_init: true)
|
|
19
|
+
|
|
20
|
+
EOCD_SIGNATURE = [0x06054b50].pack("V")
|
|
21
|
+
CENTRAL_SIGNATURE = 0x02014b50
|
|
22
|
+
LOCAL_SIGNATURE = 0x04034b50
|
|
23
|
+
TAIL_BYTES = 65_557 # end-of-central-directory record plus the longest comment
|
|
24
|
+
MAX_ENTRIES = 1_000
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
# Extracts the members whose names match any of patterns (File.fnmatch)
|
|
29
|
+
# into dir. Returns the extracted file paths.
|
|
30
|
+
def extract(host, path, patterns, dir, max_bytes:)
|
|
31
|
+
entries = entries(host, path)
|
|
32
|
+
chosen = entries.select { |e| patterns.any? { |p| File.fnmatch(p, File.basename(e.name)) } }
|
|
33
|
+
raise Unsupported, "no members match #{patterns.join(", ")}" if chosen.empty?
|
|
34
|
+
raise Unsupported, "members over #{max_bytes} bytes" if chosen.sum(&:compressed_size) > max_bytes
|
|
35
|
+
|
|
36
|
+
FileUtils.mkdir_p(dir)
|
|
37
|
+
# Each run (adjacent members) streams on its own connection, in parallel.
|
|
38
|
+
runs(chosen).map { |run| Thread.new { fetch_run(host, path, run, dir) } }.flat_map(&:value)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The zip's central directory, each entry knowing where its bytes end.
|
|
42
|
+
def entries(host, path)
|
|
43
|
+
length = content_length(host, path)
|
|
44
|
+
tail = range(host, path, [length - TAIL_BYTES, 0].max, length - 1)
|
|
45
|
+
eocd = tail.rindex(EOCD_SIGNATURE) or raise Unsupported, "no end-of-central-directory record"
|
|
46
|
+
count, cd_size, cd_offset = tail.byteslice(eocd + 10, 10).unpack("vVV")
|
|
47
|
+
if count == 0xFFFF || cd_offset == 0xFFFFFFFF || count > MAX_ENTRIES
|
|
48
|
+
raise Unsupported,
|
|
49
|
+
"zip64 or too many entries"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
list = parse_central(range(host, path, cd_offset, cd_offset + cd_size - 1), count)
|
|
53
|
+
ends = list.map(&:offset).sort + [cd_offset]
|
|
54
|
+
list.each { |e| e.data_end = ends[ends.index(e.offset) + 1] }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def parse_central(bytes, count)
|
|
58
|
+
pos = 0
|
|
59
|
+
Array.new(count) do
|
|
60
|
+
raise Corrupt, "bad central directory" unless bytes.byteslice(pos, 4).unpack1("V") == CENTRAL_SIGNATURE
|
|
61
|
+
|
|
62
|
+
flags, method, _time, _date, crc, csize, size, name_len, extra_len, comment_len =
|
|
63
|
+
bytes.byteslice(pos + 8, 26).unpack("vvvvVVVvvv")
|
|
64
|
+
offset = bytes.byteslice(pos + 42, 4).unpack1("V")
|
|
65
|
+
raise Unsupported, "encrypted member" if flags.anybits?(1)
|
|
66
|
+
raise Unsupported, "compression method #{method}" unless [0, 8].include?(method)
|
|
67
|
+
|
|
68
|
+
name = bytes.byteslice(pos + 46, name_len)
|
|
69
|
+
pos += 46 + name_len + extra_len + comment_len
|
|
70
|
+
Entry.new(name: name, compression: method, crc: crc, compressed_size: csize, uncompressed_size: size,
|
|
71
|
+
offset: offset)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Groups entries whose bytes are adjacent into one request each.
|
|
76
|
+
def runs(entries)
|
|
77
|
+
entries.sort_by(&:offset).slice_when { |a, b| a.data_end != b.offset }.to_a
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def fetch_run(host, path, run, dir)
|
|
81
|
+
reader = RunReader.new(run, dir)
|
|
82
|
+
Download.start(host) do |http|
|
|
83
|
+
request = Net::HTTP::Get.new(path)
|
|
84
|
+
request["Range"] = "bytes=#{run.first.offset}-#{run.last.data_end - 1}"
|
|
85
|
+
http.request(request) do |res|
|
|
86
|
+
raise Unsupported, "server ignored the range request" unless res.code == "206"
|
|
87
|
+
|
|
88
|
+
res.read_body { |chunk| reader << chunk }
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
reader.finish
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def content_length(host, path)
|
|
95
|
+
Download.start(host) do |http|
|
|
96
|
+
res = http.head(path)
|
|
97
|
+
raise Download::Error, "HTTP #{res.code} for https://#{host}#{path}" unless res.is_a?(Net::HTTPSuccess)
|
|
98
|
+
raise Unsupported, "no range support" unless res["Accept-Ranges"].to_s.include?("bytes")
|
|
99
|
+
|
|
100
|
+
Integer(res["Content-Length"])
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def range(host, path, first, last)
|
|
105
|
+
Download.start(host) do |http|
|
|
106
|
+
res = http.get(path, "Range" => "bytes=#{first}-#{last}")
|
|
107
|
+
raise Unsupported, "server ignored the range request" unless res.code == "206"
|
|
108
|
+
|
|
109
|
+
res.body
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Walks a run of members as bytes arrive: local header, name, data
|
|
114
|
+
# (inflated to a file), then on to the next member's offset.
|
|
115
|
+
class RunReader
|
|
116
|
+
def initialize(run, dir)
|
|
117
|
+
@members = run.dup
|
|
118
|
+
@dir = dir
|
|
119
|
+
@pos = run.first.offset
|
|
120
|
+
@buffer = +""
|
|
121
|
+
@paths = []
|
|
122
|
+
start_member
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def <<(chunk)
|
|
126
|
+
@buffer << chunk
|
|
127
|
+
step while @member && !@buffer.empty? && progress?
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def finish
|
|
131
|
+
raise Corrupt, "download ended inside #{@member.name}" if @member
|
|
132
|
+
|
|
133
|
+
@paths
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
private
|
|
137
|
+
|
|
138
|
+
def start_member
|
|
139
|
+
@member = @members.shift
|
|
140
|
+
@phase = :skip
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def progress?
|
|
144
|
+
return @buffer.bytesize >= 30 if @phase == :header
|
|
145
|
+
return @buffer.bytesize >= @name_bytes if @phase == :name
|
|
146
|
+
|
|
147
|
+
true
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def step
|
|
151
|
+
send(:"step_#{@phase}")
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def step_skip
|
|
155
|
+
take([@member.offset - @pos, @buffer.bytesize].min)
|
|
156
|
+
@phase = :header if @pos == @member.offset
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def step_header
|
|
160
|
+
header = take(30)
|
|
161
|
+
raise ZipRange::Corrupt, "bad local header" unless header.unpack1("V") == LOCAL_SIGNATURE
|
|
162
|
+
|
|
163
|
+
name_len, extra_len = header.byteslice(26, 4).unpack("vv")
|
|
164
|
+
@name_bytes = name_len + extra_len
|
|
165
|
+
@phase = :name
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def step_name
|
|
169
|
+
take(@name_bytes)
|
|
170
|
+
open_output
|
|
171
|
+
@phase = :data
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def step_data
|
|
175
|
+
bytes = take([@remaining, @buffer.bytesize].min)
|
|
176
|
+
@remaining -= bytes.bytesize
|
|
177
|
+
write(bytes)
|
|
178
|
+
finish_member if @remaining.zero?
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def open_output
|
|
182
|
+
name = File.basename(@member.name)
|
|
183
|
+
raise ZipRange::Corrupt, "bad member name" unless name.match?(/\A[\w.-]+\z/)
|
|
184
|
+
|
|
185
|
+
@path = File.join(@dir, name)
|
|
186
|
+
@file = File.open(@path, "wb")
|
|
187
|
+
@inflate = @member.compression == 8 ? Zlib::Inflate.new(-Zlib::MAX_WBITS) : nil
|
|
188
|
+
@crc = 0
|
|
189
|
+
@written = 0
|
|
190
|
+
@remaining = @member.compressed_size
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def write(bytes)
|
|
194
|
+
data = @inflate ? @inflate.inflate(bytes) : bytes
|
|
195
|
+
@written += data.bytesize
|
|
196
|
+
raise ZipRange::Corrupt, "#{@member.name} inflates past its size" if @written > @member.uncompressed_size
|
|
197
|
+
|
|
198
|
+
@crc = Zlib.crc32(data, @crc)
|
|
199
|
+
@file.write(data)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def finish_member
|
|
203
|
+
if @inflate
|
|
204
|
+
write_tail(@inflate.finish)
|
|
205
|
+
@inflate.close
|
|
206
|
+
end
|
|
207
|
+
@file.close
|
|
208
|
+
check_member
|
|
209
|
+
@paths << @path
|
|
210
|
+
start_member
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def write_tail(data)
|
|
214
|
+
@written += data.bytesize
|
|
215
|
+
@crc = Zlib.crc32(data, @crc)
|
|
216
|
+
@file.write(data)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def check_member
|
|
220
|
+
return if @crc == @member.crc && @written == @member.uncompressed_size
|
|
221
|
+
|
|
222
|
+
File.delete(@path)
|
|
223
|
+
raise ZipRange::Corrupt, "#{@member.name} failed its CRC check"
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def take(count)
|
|
227
|
+
bytes = @buffer.byteslice(0, count)
|
|
228
|
+
@buffer = @buffer.byteslice(count..) || +""
|
|
229
|
+
@pos += bytes.bytesize
|
|
230
|
+
bytes
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|