curtilage 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,195 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "open3"
5
+ require "tmpdir"
6
+
7
+ module Curtilage
8
+ # Features straight from Overture Maps' public GeoParquet release, read in
9
+ # place: nothing is downloaded up front and nothing has to be hosted.
10
+ # Overture's file index (a small GeoParquet listing every file's extent)
11
+ # narrows each pin to the one or two files covering it.
12
+ #
13
+ # Buildings Overture took from OpenStreetMap come back as "building"; its
14
+ # other sources (machine-learned footprints and others) as "ml_building",
15
+ # used only with buildings: :osm_filled or :ml. Each query tells the
16
+ # storage host roughly which area is being read.
17
+ module Overture
18
+ class Error < StandardError; end
19
+
20
+ STAC_HOST = "stac.overturemaps.org"
21
+ CATALOG = "/catalog.json"
22
+ MAX_INDEX_BYTES = 20_000_000
23
+ MAX_OUTPUT_BYTES = 200_000_000
24
+ ROAD_CLASSES = %w[motorway trunk primary secondary tertiary unclassified residential service].freeze
25
+ RAIL_CLASSES = { "standard_gauge" => "rail", "light_rail" => "light_rail", "subway" => "subway",
26
+ "narrow_gauge" => "narrow_gauge" }.freeze
27
+ COLLECTIONS = %w[building segment infrastructure].freeze
28
+ GROUP_CELL_DEG = 0.05
29
+ ATTRIBUTION = ["© OpenStreetMap contributors", "© Overture Maps Foundation"].freeze
30
+
31
+ module_function
32
+
33
+ # A list of feature lists, in pin order. options: duckdb:,
34
+ # extension_directory:, radius_deg:, release: (default: the latest).
35
+ def features_many(pins, **options)
36
+ options = Curtilage.duckdb_options(options)
37
+ radius = Float(options.fetch(:radius_deg, Lookup::DEFAULT_RADIUS_DEG))
38
+ boxes = pins.map { |pin| box(pin, radius) }
39
+ Dir.mktmpdir("curtilage-overture") do |dir|
40
+ index = File.join(dir, "index.parquet")
41
+ File.binwrite(index, Download.read(STAC_HOST, "/#{release(options)}/collections.parquet",
42
+ max_bytes: MAX_INDEX_BYTES))
43
+ files = files_for(index, boxes, options)
44
+ rows = run(data_sql(boxes, files, options), options)
45
+ group(rows, pins.size)
46
+ end
47
+ end
48
+
49
+ def release(options)
50
+ return options[:release] if options[:release]
51
+
52
+ latest = JSON.parse(Download.read(STAC_HOST, CATALOG, max_bytes: 1_000_000))["latest"].to_s
53
+ raise Error, "no latest Overture release in the catalog" unless latest.match?(/\A[\d.-]+\z/)
54
+
55
+ latest
56
+ end
57
+
58
+ def box(pin, radius)
59
+ lng = Float(pin[0])
60
+ lat = Float(pin[1])
61
+ raise ArgumentError, "pin out of range: #{pin.inspect}" unless lng.abs <= 180 && lat.abs <= 90
62
+
63
+ [lng - radius, lat - radius, lng + radius, lat + radius]
64
+ end
65
+
66
+ # {[pin_index, collection] => [s3 urls]} from the local index file.
67
+ def files_for(index, boxes, options)
68
+ selects = boxes.each_with_index.map do |(x0, y0, x1, y1), i|
69
+ "SELECT #{i} AS pin_index, collection, assets.aws.alternate.s3.href AS href " \
70
+ "FROM read_parquet(#{quote(index)}) WHERE collection IN (#{COLLECTIONS.map { |c| quote(c) }.join(", ")}) " \
71
+ "AND bbox.xmin <= #{x1} AND bbox.xmax >= #{x0} AND bbox.ymin <= #{y1} AND bbox.ymax >= #{y0}"
72
+ end
73
+ rows = run([setup(options, remote: false), "#{selects.join(" UNION ALL ")};"].join("\n"), options)
74
+ rows.group_by { |r| [r["pin_index"], r["collection"]] }.transform_values do |list|
75
+ list.map { |r| r["href"] }.select { |h| h.to_s.start_with?("s3://overturemaps-") }
76
+ end
77
+ end
78
+
79
+ # One scan per collection and set of files, shared by every pin those
80
+ # files cover: each file's metadata and row groups are read once, then
81
+ # rows are matched back to the pins whose boxes they fall in.
82
+ def data_sql(boxes, files, options)
83
+ scans = COLLECTIONS.flat_map do |collection|
84
+ groups = boxes.each_index.group_by { |i| [files[[i, collection]].to_a.sort, cell(boxes[i])] }
85
+ groups.filter_map do |(list, _), indexes|
86
+ grouped_scan(collection, list, indexes, boxes) unless list.empty?
87
+ end
88
+ end
89
+ return "SELECT 1 WHERE false;" if scans.empty?
90
+
91
+ [setup(options, remote: true), boxes_table(boxes), "#{scans.join(" UNION ALL ")};"].join("\n")
92
+ end
93
+
94
+ def boxes_table(boxes)
95
+ rows = boxes.each_with_index.map { |(x0, y0, x1, y1), i| "(#{i}, #{x0}, #{y0}, #{x1}, #{y1})" }
96
+ "CREATE TEMP TABLE boxes (pin_index INTEGER, x0 DOUBLE, y0 DOUBLE, x1 DOUBLE, y1 DOUBLE); " \
97
+ "INSERT INTO boxes VALUES #{rows.join(", ")};"
98
+ end
99
+
100
+ # Pins are grouped within a 0.05 degree cell (about 5 km), so a group's
101
+ # envelope stays small.
102
+ def cell(box)
103
+ [(box[0] / GROUP_CELL_DEG).floor, (box[1] / GROUP_CELL_DEG).floor]
104
+ end
105
+
106
+ # The envelope is a plain range DuckDB can skip row groups with; the OR
107
+ # of the pins' own boxes then drops what's between them.
108
+ def grouped_scan(collection, files, indexes, boxes)
109
+ group = indexes.map { |i| boxes[i] }
110
+ envelope = [group.map { |b| b[0] }.min, group.map { |b| b[1] }.min, group.map { |b| b[2] }.max,
111
+ group.map { |b| b[3] }.max]
112
+ any = group.map { |b| "(#{box_filter("bbox", b)})" }.join(" OR ")
113
+ "SELECT p.pin_index, d.id, d.feature, d.highway, d.name, d.geometry FROM (SELECT #{columns(collection)}, " \
114
+ "bbox AS box FROM read_parquet([#{files.map { |f| quote(f) }.join(", ")}]) " \
115
+ "WHERE #{box_filter("bbox", envelope)} AND (#{any}) AND #{filter(collection)}) d " \
116
+ "JOIN boxes p ON p.pin_index IN (#{indexes.join(", ")}) AND d.box.xmin <= p.x1 AND d.box.xmax >= p.x0 " \
117
+ "AND d.box.ymin <= p.y1 AND d.box.ymax >= p.y0"
118
+ end
119
+
120
+ def box_filter(column, box)
121
+ x0, y0, x1, y1 = box
122
+ "#{column}.xmin <= #{x1} AND #{column}.xmax >= #{x0} AND #{column}.ymin <= #{y1} AND #{column}.ymax >= #{y0}"
123
+ end
124
+
125
+ # Buildings Overture took from OSM are buildings; its other sources are
126
+ # treated as ML footprints. Roads and rail map to the build's classes.
127
+ def columns(collection)
128
+ case collection
129
+ when "building"
130
+ "id, CASE WHEN sources[1].dataset = 'OpenStreetMap' THEN 'building' ELSE 'ml_building' END AS feature, " \
131
+ "class AS highway, names.\"primary\" AS name, ST_AsGeoJSON(geometry)::VARCHAR AS geometry"
132
+ when "segment"
133
+ rail = RAIL_CLASSES.map { |k, v| "WHEN #{quote(k)} THEN #{quote(v)}" }.join(" ")
134
+ "id, CASE subtype WHEN 'road' THEN 'road' ELSE 'railway' END AS feature, " \
135
+ "CASE WHEN subtype = 'rail' THEN (CASE class #{rail} END) WHEN subclass = 'link' THEN class || '_link' " \
136
+ "ELSE class END AS highway, names.\"primary\" AS name, ST_AsGeoJSON(geometry)::VARCHAR AS geometry"
137
+ else
138
+ "id, 'parking' AS feature, NULL AS highway, names.\"primary\" AS name, " \
139
+ "ST_AsGeoJSON(geometry)::VARCHAR AS geometry"
140
+ end
141
+ end
142
+
143
+ # Ground-level roads and rail; parking areas (not points); buildings
144
+ # above ground.
145
+ def filter(collection)
146
+ case collection
147
+ when "building" then "NOT coalesce(is_underground, false)"
148
+ when "segment"
149
+ "((subtype = 'road' AND class IN (#{ROAD_CLASSES.map { |c| quote(c) }.join(", ")})) " \
150
+ "OR (subtype = 'rail' AND class IN (#{RAIL_CLASSES.keys.map { |c| quote(c) }.join(", ")}))) " \
151
+ "AND NOT list_has_any(flatten(list_transform(coalesce(road_flags, []) || coalesce(rail_flags, []), " \
152
+ "f -> f.\"values\")), ['is_tunnel', 'is_bridge'])"
153
+ else
154
+ "subtype = 'transit' AND class = 'parking' AND ST_GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
155
+ end
156
+ end
157
+
158
+ def setup(options, remote:)
159
+ ext = options[:extension_directory]
160
+ [("SET extension_directory = #{quote(File.expand_path(ext))};" if ext), "INSTALL spatial; LOAD spatial;",
161
+ # Pins in the same files share one read of each file's metadata.
162
+ (if remote
163
+ "INSTALL httpfs; LOAD httpfs; SET s3_region = 'us-west-2'; SET parquet_metadata_cache = true; " \
164
+ "SET enable_http_metadata_cache = true;"
165
+ end)].compact.join("\n")
166
+ end
167
+
168
+ # Remote reads occasionally stall and DuckDB exits without a message;
169
+ # one retry gets past that.
170
+ def run(sql, options, attempts: 2)
171
+ out, err, status = Open3.capture3(options.fetch(:duckdb, "duckdb"), "-json", stdin_data: sql)
172
+ unless status.success?
173
+ return run(sql, options, attempts: attempts - 1) if err.strip.empty? && attempts > 1
174
+
175
+ raise Error, "duckdb failed: #{err.strip.empty? ? "no message (network?)" : err.strip[0, 500]}"
176
+ end
177
+ raise Error, "duckdb output too large" if out.bytesize > MAX_OUTPUT_BYTES
178
+
179
+ out.strip.empty? ? [] : JSON.parse(out)
180
+ end
181
+
182
+ def group(rows, count)
183
+ lists = Array.new(count) { [] }
184
+ rows.each do |row|
185
+ index = row.delete("pin_index")
186
+ lists[index] << row.merge("geometry" => JSON.parse(row["geometry"])) if index.is_a?(Integer)
187
+ end
188
+ lists
189
+ end
190
+
191
+ def quote(text)
192
+ "'#{text.to_s.gsub("'", "''")}'"
193
+ end
194
+ end
195
+ end
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Curtilage
6
+ # Geofabrik's region index (outlines plus download links), used to find
7
+ # which region a pin is in without any prebuilt data. Only regions offered
8
+ # as shapefiles count; where regions nest, the smallest wins (California
9
+ # resolves to norcal or socal).
10
+ class Regions
11
+ HOST = "download.geofabrik.de"
12
+ INDEX = "/index-v1.json"
13
+ MAX_INDEX_BYTES = 50_000_000
14
+
15
+ Region = Struct.new(:id, :path, :bbox, :rings, :extent_area, keyword_init: true) do
16
+ def covers?(point)
17
+ lng, lat = point
18
+ return false unless lng.between?(bbox[0], bbox[2]) && lat.between?(bbox[1], bbox[3])
19
+
20
+ rings.any? { |ring| Geometry.contains?(ring, point) }
21
+ end
22
+ end
23
+
24
+ def self.download
25
+ parse(Download.read(HOST, INDEX, max_bytes: MAX_INDEX_BYTES))
26
+ end
27
+
28
+ def self.parse(json)
29
+ features = JSON.parse(json).fetch("features")
30
+ new(features.filter_map { |f| region(f) })
31
+ end
32
+
33
+ def self.region(feature)
34
+ props = feature["properties"] || {}
35
+ path = path_from_url(props.dig("urls", "shp"))
36
+ rings = outer_rings(feature["geometry"])
37
+ return nil unless path && !rings.empty?
38
+
39
+ points = rings.flatten(1)
40
+ bbox = [points.map(&:first).min, points.map(&:last).min, points.map(&:first).max, points.map(&:last).max]
41
+ Region.new(id: props["id"], path: path, bbox: bbox, rings: rings,
42
+ extent_area: (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]))
43
+ end
44
+
45
+ # ".../north-america/us/tennessee/tennessee-latest-free.shp.zip" ->
46
+ # "north-america/us/tennessee", the path Build takes.
47
+ def self.path_from_url(url)
48
+ path = Download.path_on(url.to_s, HOST, "/")
49
+ match = path&.match(%r{\A/((?:[a-z0-9-]+/)*?)([a-z0-9-]+)-latest-free\.shp\.zip\z})
50
+ return nil unless match
51
+
52
+ dirs = match[1].split("/")
53
+ dirs.pop if dirs.last == match[2]
54
+ (dirs + [match[2]]).join("/")
55
+ end
56
+
57
+ def self.outer_rings(geometry)
58
+ case geometry&.dig("type")
59
+ when "Polygon" then [geometry["coordinates"].first].compact
60
+ when "MultiPolygon" then geometry["coordinates"].filter_map(&:first)
61
+ else []
62
+ end
63
+ end
64
+
65
+ attr_reader :regions
66
+
67
+ def initialize(regions)
68
+ @regions = regions
69
+ end
70
+
71
+ # The smallest region containing the point, or nil.
72
+ def for(point)
73
+ @regions.select { |r| r.covers?(point) }.min_by(&:extent_area)
74
+ end
75
+
76
+ # Regions holding any part of the box around the pin: one, or two near a
77
+ # border.
78
+ def around(pin, margin)
79
+ lng, lat = pin
80
+ corners = [[lng, lat], [lng - margin, lat - margin], [lng + margin, lat - margin],
81
+ [lng - margin, lat + margin], [lng + margin, lat + margin]]
82
+ corners.filter_map { |c| self.for(c) }.uniq(&:path)
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Curtilage
4
+ # Checks a premises polygon against its site pin. Used on every generated
5
+ # polygon before it's offered, and usable on hand-drawn ones.
6
+ #
7
+ # Errors mean the polygon must not be used. Warnings mean a person should
8
+ # look before using it. Defaults come from hand-drawn premises for
9
+ # standalone stores (typically 2,000-15,000 m2, 4-10 vertices, pin inside).
10
+ module Validation
11
+ Result = Struct.new(:errors, :warnings, :metrics) do
12
+ def ok?
13
+ errors.empty?
14
+ end
15
+ end
16
+
17
+ DEFAULTS = {
18
+ min_area_m2: 1_000, # warn below
19
+ max_area_m2: 40_000, # warn above
20
+ max_area_error_m2: 100_000, # error above: a block, not a store's lot
21
+ max_vertex_distance_m: 1_000, # error above
22
+ max_vertices: 50, # error above
23
+ off_centre_m: 200, # warn above
24
+ swap_far_m: 10_000,
25
+ swap_near_m: 1_000,
26
+ }.freeze
27
+
28
+ module_function
29
+
30
+ # pin: [lng, lat]. ring: [[lng, lat], ...], closed or not.
31
+ def check(pin, ring, **options)
32
+ opts = DEFAULTS.merge(options)
33
+ return Result.new(["missing"], [], {}) if ring.nil? || ring.empty?
34
+ return Result.new(["invalid_coordinates"], [], {}) unless Geometry.valid_coordinates?(ring)
35
+
36
+ errors = Geometry.closed?(ring) ? [] : ["unclosed"]
37
+ open = Geometry.open_ring(ring)
38
+ return Result.new(errors + ["too_few_points"], [], {}) if open.uniq.size < 3
39
+
40
+ xy = Geometry.project(open, pin)
41
+ metrics = measure(xy).merge(points: open.size)
42
+ errors << "swapped" if swapped?(open, pin, xy, opts)
43
+ errors << "self_intersecting" if Geometry.self_intersects?(xy)
44
+ Result.new(errors + metric_errors(metrics, opts), metric_warnings(metrics, opts), metrics)
45
+ end
46
+
47
+ def metric_errors(metrics, opts)
48
+ errors = []
49
+ errors << "pin_outside" unless metrics[:pin_inside]
50
+ errors << "too_far_from_pin" if metrics[:max_vertex_distance_m] > opts[:max_vertex_distance_m]
51
+ errors << "too_many_vertices" if metrics[:points] > opts[:max_vertices]
52
+ errors << "too_large" if metrics[:area_m2] > opts[:max_area_error_m2]
53
+ errors
54
+ end
55
+
56
+ def metric_warnings(metrics, opts)
57
+ warnings = []
58
+ warnings << "small" if metrics[:area_m2] < opts[:min_area_m2]
59
+ warnings << "large" if metrics[:area_m2] > opts[:max_area_m2]
60
+ warnings << "off_centre" if metrics[:centroid_offset_m] > opts[:off_centre_m]
61
+ warnings
62
+ end
63
+
64
+ def measure(xy)
65
+ origin = [0.0, 0.0]
66
+ mbr_area, orientation = Geometry.min_rotated_rectangle(xy)
67
+ area = Geometry.area(xy)
68
+ {
69
+ area_m2: area.round,
70
+ rectangularity: mbr_area.positive? ? (area / mbr_area).round(3) : nil,
71
+ orientation_deg: orientation.round,
72
+ pin_inside: Geometry.contains?(xy, origin),
73
+ max_vertex_distance_m: xy.map { |x, y| Math.hypot(x, y) }.max.round,
74
+ centroid_offset_m: Math.hypot(*Geometry.centroid(xy)).round,
75
+ }
76
+ end
77
+
78
+ # The ring is far from the pin, but the same ring with lng/lat swapped
79
+ # lands on it: the classic [lat, lng] mistake.
80
+ def swapped?(open, pin, xy, opts)
81
+ return false if Math.hypot(*Geometry.centroid(xy)) < opts[:swap_far_m]
82
+
83
+ swapped = open.map(&:reverse)
84
+ return false unless Geometry.valid_coordinates?(swapped)
85
+
86
+ Math.hypot(*Geometry.centroid(Geometry.project(swapped, pin))) < opts[:swap_near_m]
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Curtilage
4
+ VERSION = "0.1.0"
5
+ end
@@ -0,0 +1,234 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "zlib"
5
+
6
+ module Curtilage
7
+ # Extracts chosen members of a remote zip with HTTP range requests, so only
8
+ # the layers the build uses are downloaded (about 70% of a Geofabrik
9
+ # shapefile zip). Members are inflated as they stream in and checked
10
+ # against the zip's CRC-32s. Raises Unsupported for anything unusual
11
+ # (no range support, zip64, encryption, other compression), so the caller
12
+ # can fall back to a full download.
13
+ module ZipRange
14
+ class Unsupported < StandardError; end
15
+ class Corrupt < StandardError; end
16
+
17
+ Entry = Struct.new(:name, :compression, :crc, :compressed_size, :uncompressed_size, :offset, :data_end,
18
+ keyword_init: true)
19
+
20
+ EOCD_SIGNATURE = [0x06054b50].pack("V")
21
+ CENTRAL_SIGNATURE = 0x02014b50
22
+ LOCAL_SIGNATURE = 0x04034b50
23
+ TAIL_BYTES = 65_557 # end-of-central-directory record plus the longest comment
24
+ MAX_ENTRIES = 1_000
25
+
26
+ module_function
27
+
28
+ # Extracts the members whose names match any of patterns (File.fnmatch)
29
+ # into dir. Returns the extracted file paths.
30
+ def extract(host, path, patterns, dir, max_bytes:)
31
+ entries = entries(host, path)
32
+ chosen = entries.select { |e| patterns.any? { |p| File.fnmatch(p, File.basename(e.name)) } }
33
+ raise Unsupported, "no members match #{patterns.join(", ")}" if chosen.empty?
34
+ raise Unsupported, "members over #{max_bytes} bytes" if chosen.sum(&:compressed_size) > max_bytes
35
+
36
+ FileUtils.mkdir_p(dir)
37
+ # Each run (adjacent members) streams on its own connection, in parallel.
38
+ runs(chosen).map { |run| Thread.new { fetch_run(host, path, run, dir) } }.flat_map(&:value)
39
+ end
40
+
41
+ # The zip's central directory, each entry knowing where its bytes end.
42
+ def entries(host, path)
43
+ length = content_length(host, path)
44
+ tail = range(host, path, [length - TAIL_BYTES, 0].max, length - 1)
45
+ eocd = tail.rindex(EOCD_SIGNATURE) or raise Unsupported, "no end-of-central-directory record"
46
+ count, cd_size, cd_offset = tail.byteslice(eocd + 10, 10).unpack("vVV")
47
+ if count == 0xFFFF || cd_offset == 0xFFFFFFFF || count > MAX_ENTRIES
48
+ raise Unsupported,
49
+ "zip64 or too many entries"
50
+ end
51
+
52
+ list = parse_central(range(host, path, cd_offset, cd_offset + cd_size - 1), count)
53
+ ends = list.map(&:offset).sort + [cd_offset]
54
+ list.each { |e| e.data_end = ends[ends.index(e.offset) + 1] }
55
+ end
56
+
57
+ def parse_central(bytes, count)
58
+ pos = 0
59
+ Array.new(count) do
60
+ raise Corrupt, "bad central directory" unless bytes.byteslice(pos, 4).unpack1("V") == CENTRAL_SIGNATURE
61
+
62
+ flags, method, _time, _date, crc, csize, size, name_len, extra_len, comment_len =
63
+ bytes.byteslice(pos + 8, 26).unpack("vvvvVVVvvv")
64
+ offset = bytes.byteslice(pos + 42, 4).unpack1("V")
65
+ raise Unsupported, "encrypted member" if flags.anybits?(1)
66
+ raise Unsupported, "compression method #{method}" unless [0, 8].include?(method)
67
+
68
+ name = bytes.byteslice(pos + 46, name_len)
69
+ pos += 46 + name_len + extra_len + comment_len
70
+ Entry.new(name: name, compression: method, crc: crc, compressed_size: csize, uncompressed_size: size,
71
+ offset: offset)
72
+ end
73
+ end
74
+
75
+ # Groups entries whose bytes are adjacent into one request each.
76
+ def runs(entries)
77
+ entries.sort_by(&:offset).slice_when { |a, b| a.data_end != b.offset }.to_a
78
+ end
79
+
80
+ def fetch_run(host, path, run, dir)
81
+ reader = RunReader.new(run, dir)
82
+ Download.start(host) do |http|
83
+ request = Net::HTTP::Get.new(path)
84
+ request["Range"] = "bytes=#{run.first.offset}-#{run.last.data_end - 1}"
85
+ http.request(request) do |res|
86
+ raise Unsupported, "server ignored the range request" unless res.code == "206"
87
+
88
+ res.read_body { |chunk| reader << chunk }
89
+ end
90
+ end
91
+ reader.finish
92
+ end
93
+
94
+ def content_length(host, path)
95
+ Download.start(host) do |http|
96
+ res = http.head(path)
97
+ raise Download::Error, "HTTP #{res.code} for https://#{host}#{path}" unless res.is_a?(Net::HTTPSuccess)
98
+ raise Unsupported, "no range support" unless res["Accept-Ranges"].to_s.include?("bytes")
99
+
100
+ Integer(res["Content-Length"])
101
+ end
102
+ end
103
+
104
+ def range(host, path, first, last)
105
+ Download.start(host) do |http|
106
+ res = http.get(path, "Range" => "bytes=#{first}-#{last}")
107
+ raise Unsupported, "server ignored the range request" unless res.code == "206"
108
+
109
+ res.body
110
+ end
111
+ end
112
+
113
+ # Walks a run of members as bytes arrive: local header, name, data
114
+ # (inflated to a file), then on to the next member's offset.
115
+ class RunReader
116
+ def initialize(run, dir)
117
+ @members = run.dup
118
+ @dir = dir
119
+ @pos = run.first.offset
120
+ @buffer = +""
121
+ @paths = []
122
+ start_member
123
+ end
124
+
125
+ def <<(chunk)
126
+ @buffer << chunk
127
+ step while @member && !@buffer.empty? && progress?
128
+ end
129
+
130
+ def finish
131
+ raise Corrupt, "download ended inside #{@member.name}" if @member
132
+
133
+ @paths
134
+ end
135
+
136
+ private
137
+
138
+ def start_member
139
+ @member = @members.shift
140
+ @phase = :skip
141
+ end
142
+
143
+ def progress?
144
+ return @buffer.bytesize >= 30 if @phase == :header
145
+ return @buffer.bytesize >= @name_bytes if @phase == :name
146
+
147
+ true
148
+ end
149
+
150
+ def step
151
+ send(:"step_#{@phase}")
152
+ end
153
+
154
+ def step_skip
155
+ take([@member.offset - @pos, @buffer.bytesize].min)
156
+ @phase = :header if @pos == @member.offset
157
+ end
158
+
159
+ def step_header
160
+ header = take(30)
161
+ raise ZipRange::Corrupt, "bad local header" unless header.unpack1("V") == LOCAL_SIGNATURE
162
+
163
+ name_len, extra_len = header.byteslice(26, 4).unpack("vv")
164
+ @name_bytes = name_len + extra_len
165
+ @phase = :name
166
+ end
167
+
168
+ def step_name
169
+ take(@name_bytes)
170
+ open_output
171
+ @phase = :data
172
+ end
173
+
174
+ def step_data
175
+ bytes = take([@remaining, @buffer.bytesize].min)
176
+ @remaining -= bytes.bytesize
177
+ write(bytes)
178
+ finish_member if @remaining.zero?
179
+ end
180
+
181
+ def open_output
182
+ name = File.basename(@member.name)
183
+ raise ZipRange::Corrupt, "bad member name" unless name.match?(/\A[\w.-]+\z/)
184
+
185
+ @path = File.join(@dir, name)
186
+ @file = File.open(@path, "wb")
187
+ @inflate = @member.compression == 8 ? Zlib::Inflate.new(-Zlib::MAX_WBITS) : nil
188
+ @crc = 0
189
+ @written = 0
190
+ @remaining = @member.compressed_size
191
+ end
192
+
193
+ def write(bytes)
194
+ data = @inflate ? @inflate.inflate(bytes) : bytes
195
+ @written += data.bytesize
196
+ raise ZipRange::Corrupt, "#{@member.name} inflates past its size" if @written > @member.uncompressed_size
197
+
198
+ @crc = Zlib.crc32(data, @crc)
199
+ @file.write(data)
200
+ end
201
+
202
+ def finish_member
203
+ if @inflate
204
+ write_tail(@inflate.finish)
205
+ @inflate.close
206
+ end
207
+ @file.close
208
+ check_member
209
+ @paths << @path
210
+ start_member
211
+ end
212
+
213
+ def write_tail(data)
214
+ @written += data.bytesize
215
+ @crc = Zlib.crc32(data, @crc)
216
+ @file.write(data)
217
+ end
218
+
219
+ def check_member
220
+ return if @crc == @member.crc && @written == @member.uncompressed_size
221
+
222
+ File.delete(@path)
223
+ raise ZipRange::Corrupt, "#{@member.name} failed its CRC check"
224
+ end
225
+
226
+ def take(count)
227
+ bytes = @buffer.byteslice(0, count)
228
+ @buffer = @buffer.byteslice(count..) || +""
229
+ @pos += bytes.bytesize
230
+ bytes
231
+ end
232
+ end
233
+ end
234
+ end