claricle 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +10 -1
- data/LICENSE +25 -0
- data/README.adoc +439 -157
- data/TODO.plan/00-overview.md +502 -0
- data/TODO.plan/01-core.md +296 -0
- data/TODO.plan/02-inspect.md +163 -0
- data/TODO.plan/03-conform.md +208 -0
- data/TODO.plan/04-convert.md +216 -0
- data/exe/claricle +1 -1
- data/lib/claricle/batch.rb +240 -0
- data/lib/claricle/cli.rb +514 -40
- data/lib/claricle/conformance_options.rb +49 -0
- data/lib/claricle/conversion_engine.rb +21 -0
- data/lib/claricle/convert.rb +21 -0
- data/lib/claricle/detector.rb +696 -0
- data/lib/claricle/emf_header.rb +33 -0
- data/lib/claricle/errors.rb +73 -0
- data/lib/claricle/fault.rb +69 -0
- data/lib/claricle/handlers/base.rb +169 -0
- data/lib/claricle/handlers/extension_points.rb +124 -0
- data/lib/claricle/handlers/metafile.rb +1011 -0
- data/lib/claricle/handlers/pdf.rb +961 -0
- data/lib/claricle/handlers/pdf_arlington/definitions.rb +54 -0
- data/lib/claricle/handlers/pdf_arlington/kind.rb +62 -0
- data/lib/claricle/handlers/pdf_arlington/predicate_gate.rb +39 -0
- data/lib/claricle/handlers/pdf_arlington/rules.rb +86 -0
- data/lib/claricle/handlers/pdf_arlington/walker.rb +131 -0
- data/lib/claricle/handlers/pdf_arlington.rb +38 -0
- data/lib/claricle/handlers/pdf_profiles.rb +66 -0
- data/lib/claricle/handlers/png.rb +1107 -0
- data/lib/claricle/handlers/postscript.rb +1357 -0
- data/lib/claricle/handlers/svg.rb +787 -0
- data/lib/claricle/image.rb +272 -0
- data/lib/claricle/lossiness/levels.rb +13 -0
- data/lib/claricle/lossiness.rb +864 -0
- data/lib/claricle/models/base.rb +460 -0
- data/lib/claricle/models/batch_item.rb +127 -0
- data/lib/claricle/models/conversion.rb +167 -0
- data/lib/claricle/models/format_capability.rb +40 -0
- data/lib/claricle/models/free_form_hash.rb +77 -0
- data/lib/claricle/models/inspection.rb +188 -0
- data/lib/claricle/models/issue.rb +23 -0
- data/lib/claricle/models/location.rb +51 -0
- data/lib/claricle/models/report.rb +54 -0
- data/lib/claricle/models/uncoerced_integer.rb +38 -0
- data/lib/claricle/postscript_fidelity/verdict.rb +37 -0
- data/lib/claricle/postscript_fidelity/view_box.rb +82 -0
- data/lib/claricle/postscript_fidelity.rb +160 -0
- data/lib/claricle/registry.rb +125 -0
- data/lib/claricle/version.rb +1 -1
- data/lib/claricle/writer.rb +468 -0
- data/lib/claricle.rb +244 -7
- metadata +201 -19
- data/claricle.gemspec +0 -42
- data/sig/claricle.rbs +0 -4
|
@@ -0,0 +1,1107 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "zlib"
|
|
4
|
+
|
|
5
|
+
require_relative "base"
|
|
6
|
+
require_relative "../models/inspection"
|
|
7
|
+
require_relative "../models/issue"
|
|
8
|
+
require_relative "../models/location"
|
|
9
|
+
require_relative "../models/report"
|
|
10
|
+
|
|
11
|
+
module Claricle
|
|
12
|
+
module Handlers
|
|
13
|
+
# A conformance issue's `code` has no upstream equivalent --
|
|
14
|
+
# png_conform's own hash carries no discriminant beyond the free-text
|
|
15
|
+
# `message` -- so one is derived from the message's fixed wording with
|
|
16
|
+
# every value-shaped detail stripped: parenthetical asides (`"(9, must
|
|
17
|
+
# be one of 0, 2, 3, 4, 6)"`), and bare numbers such as the "1.0" in a
|
|
18
|
+
# gAMA note. What is left is stable across which file produced the
|
|
19
|
+
# issue; it changes only if a png_conform release changes the wording
|
|
20
|
+
# a message ships with.
|
|
21
|
+
#
|
|
22
|
+
# A sibling module, not nested inside `Png`, for the same reason
|
|
23
|
+
# `EmfPlus` sits beside `Metafile` rather than inside it: the concern
|
|
24
|
+
# is unrelated to PNG metadata interpretation, and keeping it apart
|
|
25
|
+
# keeps both easier to read.
|
|
26
|
+
module IssueCode
|
|
27
|
+
PREFIX = "png."
|
|
28
|
+
STRIP = [/\([^)]*\)/, /\d+(\.\d+)?/].freeze
|
|
29
|
+
DEFAULT = "conform_finding"
|
|
30
|
+
|
|
31
|
+
def self.for(message)
|
|
32
|
+
slug = STRIP.reduce(message.to_s.downcase) { |text, pattern| text.gsub(pattern, " ") }
|
|
33
|
+
.gsub(/[^a-z]+/, "_")
|
|
34
|
+
.gsub(/\A_+|_+\z/, "")
|
|
35
|
+
"#{PREFIX}#{slug.empty? ? DEFAULT : slug}"
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private_constant :IssueCode
|
|
40
|
+
|
|
41
|
+
# Reads PNG metadata. It deliberately does not validate: png_conform's
|
|
42
|
+
# ValidationService would hand us a parsed ImageInfo for free, but it
|
|
43
|
+
# runs conformance checks along the way, so `inspect` would mean
|
|
44
|
+
# "conforms" for PNG and "metadata" for every other format (D17).
|
|
45
|
+
class Png < Base
|
|
46
|
+
formats :png
|
|
47
|
+
|
|
48
|
+
# IHDR is a fixed 13-byte layout in the PNG spec: two 4-byte
|
|
49
|
+
# dimensions then five single bytes. Hand-unpacking it agrees with
|
|
50
|
+
# ImageInfo on a real file, which a spec pins.
|
|
51
|
+
IHDR_LAYOUT = "NNC5"
|
|
52
|
+
IHDR_BYTES = 13
|
|
53
|
+
PHYS_LAYOUT = "NNC"
|
|
54
|
+
PHYS_BYTES = 9
|
|
55
|
+
# pHYs unit 1 is pixels per metre; unit 0 means aspect ratio only,
|
|
56
|
+
# so there is no physical resolution to report.
|
|
57
|
+
METRE_UNIT = 1
|
|
58
|
+
METRES_PER_INCH = 0.0254
|
|
59
|
+
|
|
60
|
+
# An allowlist, not "everything except IDAT". These two chunks are
|
|
61
|
+
# the only ones `gather` ever reads the payload of -- everything
|
|
62
|
+
# else is stepped over, so a PNG carrying a huge ancillary chunk
|
|
63
|
+
# never costs more than its own 8-byte header where the IO can
|
|
64
|
+
# seek. Where it cannot, `drain` reads those bytes and throws them
|
|
65
|
+
# away, so what stays bounded there is memory, not bytes read
|
|
66
|
+
# (D-bounded-read).
|
|
67
|
+
WANTED_CHUNKS = %w[IHDR pHYs].freeze
|
|
68
|
+
|
|
69
|
+
# png_conform's own vocabulary, verified against ImageInfo#color_type
|
|
70
|
+
# for all five types, so the two never disagree if a later slice does
|
|
71
|
+
# reach for the validator. Note "palette", not "indexed" -- that is
|
|
72
|
+
# the delegate's word for colour type 3.
|
|
73
|
+
COLOR_SPACES = {
|
|
74
|
+
0 => "grayscale",
|
|
75
|
+
2 => "truecolor",
|
|
76
|
+
3 => "palette",
|
|
77
|
+
4 => "grayscale+alpha",
|
|
78
|
+
6 => "truecolor+alpha"
|
|
79
|
+
}.freeze
|
|
80
|
+
|
|
81
|
+
# Lives out here rather than inside StructureScanner, its only
|
|
82
|
+
# consumer, because the `header_at` guard added alongside it pushed
|
|
83
|
+
# that class to 102 lines against Metrics/ClassLength's 100 --
|
|
84
|
+
# measured by moving this back, which fires the cop. Headroom is two
|
|
85
|
+
# lines, so item 03 should decompose the scanner rather than hoist
|
|
86
|
+
# another constant.
|
|
87
|
+
STRUCTURE_MESSAGES = {
|
|
88
|
+
duplicate_ihdr: "duplicate IHDR chunk; a PNG datastream carries exactly one",
|
|
89
|
+
missing_iend: "PNG datastream ends without an IEND chunk",
|
|
90
|
+
duplicate_iend: "a second IEND chunk follows the end of the datastream",
|
|
91
|
+
trailing_data: "unexpected bytes after the IEND chunk",
|
|
92
|
+
# The whole record span against the bytes really left. Comparing
|
|
93
|
+
# a payload length with remaining record bytes reads as
|
|
94
|
+
# "declares 13 bytes but only 17 remain", which compares two
|
|
95
|
+
# different units and tells the reader nothing.
|
|
96
|
+
chunk_overruns: "chunk needs %d bytes but only %d remain in the file",
|
|
97
|
+
header_residue: "%s left, too few for a chunk header",
|
|
98
|
+
shorter_than_signature: "file is shorter than the PNG signature",
|
|
99
|
+
ends_inside_record: "file ends inside the last chunk it declares"
|
|
100
|
+
}.freeze
|
|
101
|
+
|
|
102
|
+
# Headroom over IHDR_BYTES (13) and PHYS_BYTES (9): neither
|
|
103
|
+
# `readable` nor `dpi` ever needs more of a wanted chunk than
|
|
104
|
+
# that, so nothing here trusts a declared length past this cap --
|
|
105
|
+
# not even for IHDR or pHYs themselves (D-bounded-read).
|
|
106
|
+
MAX_CHUNK_READ = 32
|
|
107
|
+
|
|
108
|
+
# How much `drain` discards per read when the IO cannot seek.
|
|
109
|
+
# Deliberately its own number rather than MAX_CHUNK_READ: that cap
|
|
110
|
+
# bounds what is KEPT in memory, this one only bounds how many
|
|
111
|
+
# syscalls it takes to throw bytes away, and nothing it reads is
|
|
112
|
+
# retained. Measured, draining a 20 MiB unwanted chunk off a real
|
|
113
|
+
# pipe: 32 bytes took 655,360 reads and 5.29s, 16 KiB took 1,280
|
|
114
|
+
# reads and 0.031s.
|
|
115
|
+
DRAIN_BUFFER = 16_384
|
|
116
|
+
|
|
117
|
+
# png_conform's own iCCP/zTXt/iTXt validators each call
|
|
118
|
+
# `Zlib::Inflate.inflate` on attacker-controlled bytes with no output
|
|
119
|
+
# bound (measured against the installed 0.1.4 gem's
|
|
120
|
+
# `validators/ancillary/{iccp,ztxt,itxt}_validator.rb`) --
|
|
121
|
+
# `AncillaryBoundsGuard` below is what stands between that and this
|
|
122
|
+
# handler's caller. 64 MiB is comfortably past every real ICC profile
|
|
123
|
+
# measured against this guard (the largest of 24 profiles on this
|
|
124
|
+
# machine is 55,280 bytes; a synthetic, structurally valid 40 MiB
|
|
125
|
+
# profile was built and shown to pass) and a world away from the
|
|
126
|
+
# unbounded case it replaces: a 1,043,730-byte PNG carrying one iCCP
|
|
127
|
+
# chunk was measured driving `conformance_report` to a 2.19 GB peak
|
|
128
|
+
# RSS before this guard existed.
|
|
129
|
+
#
|
|
130
|
+
# Cumulative across every watched chunk in the file, not a per-chunk
|
|
131
|
+
# allowance -- png_conform's own `zTXt`/`iTXt` validators retain each
|
|
132
|
+
# decompressed chunk's text in a growing `context.store(:text_chunks,
|
|
133
|
+
# ...)` array across the WHOLE file, not one at a time. A per-chunk-only
|
|
134
|
+
# ceiling was measured letting eight 32 MiB zTXt chunks (each under it
|
|
135
|
+
# individually) through a 261,196-byte PNG that drove a 654 MB peak RSS
|
|
136
|
+
# -- the exact hazard this guard exists to stop, just split across
|
|
137
|
+
# chunks instead of concentrated in one.
|
|
138
|
+
MAX_DECOMPRESSED_BYTES = 64 * 1024 * 1024
|
|
139
|
+
|
|
140
|
+
# What `ChunkReader#gather` collects. png_conform's own chunk
|
|
141
|
+
# object is gone from this handler entirely now that its bytes
|
|
142
|
+
# come from a hand read rather than `StreamingReader#each_chunk`;
|
|
143
|
+
# only `.type` and `.data` were ever read off it, so this replaces
|
|
144
|
+
# it exactly.
|
|
145
|
+
Chunk = Data.define(:type, :data)
|
|
146
|
+
|
|
147
|
+
# Drives a chunk stream's raw IO directly, deciding purely from an
|
|
148
|
+
# 8-byte header whether to read a chunk's payload or skip it --
|
|
149
|
+
# kept apart from PNG metadata interpretation (IHDR/pHYs decoding,
|
|
150
|
+
# dpi, colour spaces) so the two concerns stay easy to read apart.
|
|
151
|
+
#
|
|
152
|
+
# png_conform 0.1.4 has no public API that exposes a chunk's
|
|
153
|
+
# length+type without first reading its data (measured against
|
|
154
|
+
# every method on `StreamingReader`), so bounding the read means
|
|
155
|
+
# reading the header by hand and deciding before the payload is
|
|
156
|
+
# ever touched.
|
|
157
|
+
class ChunkReader
|
|
158
|
+
def initialize(io)
|
|
159
|
+
@io = io
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# `into` doubles as the "already captured" record -- no separate
|
|
163
|
+
# tracking is needed since `WANTED_CHUNKS` only has two entries
|
|
164
|
+
# -- so a second chunk of a type already in `into` is skipped
|
|
165
|
+
# exactly like an unwanted one, keeping only the first.
|
|
166
|
+
def gather(into:)
|
|
167
|
+
loop do
|
|
168
|
+
length, type = read_header
|
|
169
|
+
break unless type && type != "IEND"
|
|
170
|
+
|
|
171
|
+
if wanted?(type, into)
|
|
172
|
+
break unless capture!(type, length, into: into)
|
|
173
|
+
else
|
|
174
|
+
skip(length + 4) # payload plus the CRC, neither read here
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
private
|
|
180
|
+
|
|
181
|
+
attr_reader :io
|
|
182
|
+
|
|
183
|
+
# A wanted type not already in `into` -- the second of two
|
|
184
|
+
# chunks sharing a type is treated exactly like an unwanted one.
|
|
185
|
+
def wanted?(type, into)
|
|
186
|
+
WANTED_CHUNKS.include?(type) && into.none? { |chunk| chunk.type == type }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# `nil` on a short payload or a short CRC -- either means this
|
|
190
|
+
# chunk didn't fully arrive, so `gather` stops rather than
|
|
191
|
+
# keeping a partial read.
|
|
192
|
+
def capture!(type, length, into:)
|
|
193
|
+
data = read_chunk_data(length)
|
|
194
|
+
return nil unless data && read_crc
|
|
195
|
+
|
|
196
|
+
into << Chunk.new(type: type, data: data)
|
|
197
|
+
data
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# `nil` on a short or absent header -- the same "stop, keep what
|
|
201
|
+
# `into` already holds" outcome a truncated read used to reach
|
|
202
|
+
# by raising and being rescued.
|
|
203
|
+
def read_header
|
|
204
|
+
header = io.read(8)
|
|
205
|
+
return [nil, nil] unless header && header.bytesize == 8
|
|
206
|
+
|
|
207
|
+
header.unpack("Na4")
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Reads at most MAX_CHUNK_READ bytes of a wanted chunk's
|
|
211
|
+
# payload, and accounts for the rest of `length` by skipping it
|
|
212
|
+
# -- so a chunk declaring more than the cap (an
|
|
213
|
+
# attacker-labelled "IHDR", or simply a real IHDR with trailing
|
|
214
|
+
# bytes) never leaves the stream misaligned for whatever chunk
|
|
215
|
+
# comes next. `nil` on a short read: a wanted chunk that didn't
|
|
216
|
+
# fully arrive is truncation, not something to keep.
|
|
217
|
+
def read_chunk_data(length)
|
|
218
|
+
wanted = [length, MAX_CHUNK_READ].min
|
|
219
|
+
data = io.read(wanted)
|
|
220
|
+
return nil unless data && data.bytesize == wanted
|
|
221
|
+
|
|
222
|
+
skip(length - wanted) if length > wanted
|
|
223
|
+
data
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Read, not skipped: a wanted chunk is only appended to `into`
|
|
227
|
+
# once its CRC is confirmed present, matching png_conform's own
|
|
228
|
+
# atomic chunk read, which never hands `gather` a chunk whose
|
|
229
|
+
# CRC didn't fully arrive either.
|
|
230
|
+
def read_crc
|
|
231
|
+
crc = io.read(4)
|
|
232
|
+
crc && crc.bytesize == 4
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# `seek`, unless the IO can't. A pipe and a `/dev/fd/<n>` onto
|
|
236
|
+
# one both raise `Errno::ESPIPE` -- measured off a real pipe,
|
|
237
|
+
# not assumed.
|
|
238
|
+
#
|
|
239
|
+
# Reaching that shape means naming the format:
|
|
240
|
+
# `Image.new(format: :png, path: "/dev/fd/<n>")`. It is NOT
|
|
241
|
+
# `Image.from_path`, which detects first, and detection reads
|
|
242
|
+
# the stream -- a pipe hands its bytes out once, so inspection
|
|
243
|
+
# reopens a path that has already moved past the signature.
|
|
244
|
+
# Measured on the same pipe: `from_path` reports "failed" with
|
|
245
|
+
# `png.ihdr_unreadable`, and the constructor with the format
|
|
246
|
+
# given reports "ok". Widening that is the detector's business,
|
|
247
|
+
# not this handler's.
|
|
248
|
+
#
|
|
249
|
+
# Falling back to reading and discarding in bounded increments
|
|
250
|
+
# keeps memory capped even though the byte count read off a
|
|
251
|
+
# pipe is not.
|
|
252
|
+
def skip(length)
|
|
253
|
+
io.seek(length, IO::SEEK_CUR)
|
|
254
|
+
rescue Errno::ESPIPE
|
|
255
|
+
drain(length)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def drain(length)
|
|
259
|
+
remaining = length
|
|
260
|
+
while remaining.positive?
|
|
261
|
+
requested = [remaining, DRAIN_BUFFER].min
|
|
262
|
+
chunk = io.read(requested)
|
|
263
|
+
break unless chunk
|
|
264
|
+
|
|
265
|
+
remaining -= chunk.bytesize
|
|
266
|
+
break if chunk.bytesize < requested
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Walks the raw chunk stream for `iCCP`/`zTXt`/`iTXt` and asks one
|
|
272
|
+
# question, kept running across every occurrence in the file: has
|
|
273
|
+
# decompressing them exceeded `MAX_DECOMPRESSED_BYTES` IN TOTAL. Runs
|
|
274
|
+
# BEFORE png_conform ever sees the file, so a scan that exceeds it
|
|
275
|
+
# never reaches the delegate's own unbounded `Zlib::Inflate.inflate`
|
|
276
|
+
# at all.
|
|
277
|
+
#
|
|
278
|
+
# The total is cumulative across the whole file, not reset per chunk
|
|
279
|
+
# -- see `MAX_DECOMPRESSED_BYTES`'s own comment for why a per-chunk
|
|
280
|
+
# allowance alone is not enough.
|
|
281
|
+
#
|
|
282
|
+
# Deliberately does not stop at `IEND`, unlike `ChunkReader` above --
|
|
283
|
+
# measured against the installed delegate, a chunk appended after
|
|
284
|
+
# `IEND` is still inflated and reported, so stopping here would let a
|
|
285
|
+
# bomb chunk placed after `IEND` bypass this guard while still
|
|
286
|
+
# reaching png_conform.
|
|
287
|
+
#
|
|
288
|
+
# Does not reuse `ChunkReader`'s CRC method or "nil on a short read"
|
|
289
|
+
# discipline -- both exist there to decide whether to KEEP a chunk
|
|
290
|
+
# for an inspection report, a different question. This class has its
|
|
291
|
+
# own CRC check below (`#bomb_at?`), for a different reason: whether
|
|
292
|
+
# to count a chunk's bytes toward the running total at all. A
|
|
293
|
+
# truncated or structurally malformed chunk fails that question
|
|
294
|
+
# harmlessly (see `#bomb?`) or is left for the real delegate to
|
|
295
|
+
# report exactly as it does today.
|
|
296
|
+
#
|
|
297
|
+
# A real file (never a pipe): `conformance_report`'s own delegate
|
|
298
|
+
# already requires one -- `FullLoadReader#read` rewinds its IO before
|
|
299
|
+
# reading, which raises `Errno::ESPIPE` on a pipe today, before this
|
|
300
|
+
# guard existed. `#skip` below relies on the same guarantee.
|
|
301
|
+
class AncillaryBoundsGuard
|
|
302
|
+
WATCHED = %w[iCCP zTXt iTXt].freeze
|
|
303
|
+
private_constant :WATCHED
|
|
304
|
+
|
|
305
|
+
def initialize(io)
|
|
306
|
+
@io = io
|
|
307
|
+
@total = 0
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
# `[chunk_type, byte_offset]` of the first watched chunk whose
|
|
311
|
+
# decompression pushes the file's running total past the ceiling,
|
|
312
|
+
# or `nil` if the whole file stays under it.
|
|
313
|
+
def exceeded
|
|
314
|
+
io.read(8) # the PNG signature -- alignment only, never validated here
|
|
315
|
+
loop do
|
|
316
|
+
header_offset = io.pos
|
|
317
|
+
length, type = read_header
|
|
318
|
+
break unless type
|
|
319
|
+
|
|
320
|
+
hit = process(type, length, header_offset)
|
|
321
|
+
return hit if hit
|
|
322
|
+
end
|
|
323
|
+
nil
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
private
|
|
327
|
+
|
|
328
|
+
attr_reader :io
|
|
329
|
+
|
|
330
|
+
def read_header
|
|
331
|
+
header = io.read(8)
|
|
332
|
+
return [nil, nil] unless header && header.bytesize == 8
|
|
333
|
+
|
|
334
|
+
header.unpack("Na4")
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# A watched type that exceeds the ceiling returns its own
|
|
338
|
+
# `[type, header_offset]`; anything else (unwatched, watched but
|
|
339
|
+
# safe, or watched but rejected on a check below) is skipped or
|
|
340
|
+
# already consumed and returns `nil`.
|
|
341
|
+
def process(type, length, header_offset)
|
|
342
|
+
unless WATCHED.include?(type)
|
|
343
|
+
skip(length)
|
|
344
|
+
return nil
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
[type, header_offset] if bomb_at?(type, length)
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# `false` on a short payload (truncation), a bad CRC, a chunk too
|
|
351
|
+
# malformed to locate a compressed segment in, or one the real
|
|
352
|
+
# validator would reject before ever decompressing it -- every one
|
|
353
|
+
# of those answers "not a bomb" and leaves the chunk for the real
|
|
354
|
+
# delegate to judge on its own terms.
|
|
355
|
+
#
|
|
356
|
+
# The CRC check matters for more than integrity: every installed
|
|
357
|
+
# validator calls `check_crc` FIRST and stops on failure, so a
|
|
358
|
+
# chunk with a bad CRC is never decompressed by the delegate at
|
|
359
|
+
# all -- counting it here would inflate `@total` for bytes
|
|
360
|
+
# png_conform never actually produces.
|
|
361
|
+
def bomb_at?(type, length)
|
|
362
|
+
data = io.read(length)
|
|
363
|
+
crc = io.read(4)
|
|
364
|
+
return false unless data && data.bytesize == length && crc && crc.bytesize == 4
|
|
365
|
+
return false unless crc.unpack1("N") == Zlib.crc32(type + data)
|
|
366
|
+
|
|
367
|
+
compressed = compressed_segment(type, data)
|
|
368
|
+
compressed ? bomb?(type, compressed) : false
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# Mirrors the installed validators' own field layout AND their
|
|
372
|
+
# compression-method check exactly
|
|
373
|
+
# (`validators/ancillary/{iccp,ztxt,itxt}_validator.rb`), read
|
|
374
|
+
# directly from that gem source rather than assumed -- a chunk
|
|
375
|
+
# declaring anything other than method 0 (deflate) is rejected by
|
|
376
|
+
# every validator before it ever decompresses, so counting it here
|
|
377
|
+
# would be the same overcounting the CRC check above exists to
|
|
378
|
+
# avoid. `getbyte`, not `[]` chained into `.ord`: a structurally
|
|
379
|
+
# short chunk (no compression-flag byte after its keyword) must
|
|
380
|
+
# answer "no compressed segment here", not raise.
|
|
381
|
+
def compressed_segment(type, data)
|
|
382
|
+
case type
|
|
383
|
+
when "iCCP", "zTXt"
|
|
384
|
+
null_pos = data.index("\0")
|
|
385
|
+
return nil unless null_pos && data.getbyte(null_pos + 1)&.zero? # compression method
|
|
386
|
+
|
|
387
|
+
data[(null_pos + 2)..]
|
|
388
|
+
when "iTXt"
|
|
389
|
+
itxt_compressed_segment(data)
|
|
390
|
+
end
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def itxt_compressed_segment(data)
|
|
394
|
+
first = data.index("\0")
|
|
395
|
+
return nil unless first && data.getbyte(first + 1) == 1 && data.getbyte(first + 2)&.zero? # flag, method
|
|
396
|
+
|
|
397
|
+
second = data.index("\0", first + 3)
|
|
398
|
+
return nil unless second
|
|
399
|
+
|
|
400
|
+
third = data.index("\0", second + 1)
|
|
401
|
+
third && data[(third + 1)..]
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
# Ruby stdlib's block form of `Zlib::Inflate#inflate` yields
|
|
405
|
+
# decompressed output incrementally rather than materializing all
|
|
406
|
+
# of it before returning -- a `return` inside the block (see
|
|
407
|
+
# `#accumulate` below) aborts the decompression already in
|
|
408
|
+
# progress, the same non-local-exit mechanism any `each`-style
|
|
409
|
+
# iterator supports. Measured against the real
|
|
410
|
+
# bomb's own iCCP payload: peak memory stayed proportional to the
|
|
411
|
+
# abort threshold (~39 MiB aborting at 10 MiB, ~101 MiB aborting at
|
|
412
|
+
# `MAX_DECOMPRESSED_BYTES`), not the 2.19 GB an unbounded call
|
|
413
|
+
# produces.
|
|
414
|
+
#
|
|
415
|
+
# Checked against `@total` (this chunk's own running count on top
|
|
416
|
+
# of every prior chunk's) so a single huge chunk still aborts
|
|
417
|
+
# mid-stream exactly as before, AND several sub-ceiling chunks are
|
|
418
|
+
# caught once their sum crosses it. `@total` is only committed once
|
|
419
|
+
# this chunk decompresses cleanly AND (for iTXt) its text is valid
|
|
420
|
+
# UTF-8 -- a chunk that fails either never reaches png_conform's
|
|
421
|
+
# own `store_profile_info`/`store_text_info` either (every
|
|
422
|
+
# validator calls `check_decompression` first and stops on
|
|
423
|
+
# failure; iTXt's own `check_decompression` additionally rejects
|
|
424
|
+
# invalid UTF-8 before ever storing), so it must not inflate the
|
|
425
|
+
# running total.
|
|
426
|
+
#
|
|
427
|
+
# `Zlib::Error` (malformed zlib data) answers "not a bomb" -- the
|
|
428
|
+
# real delegate already reports its own "decompression failed" for
|
|
429
|
+
# that shape, and this must not take over reporting a different
|
|
430
|
+
# problem for input that was never actually a bomb.
|
|
431
|
+
def bomb?(type, compressed)
|
|
432
|
+
this_chunk, text = decompressed_size(compressed, keep_text: type == "iTXt")
|
|
433
|
+
return true if this_chunk.nil?
|
|
434
|
+
# No `.dup`: `text` is this method's own local, built fresh by
|
|
435
|
+
# `#accumulate` and never read again after this line, so
|
|
436
|
+
# mutating its encoding in place is safe and avoids copying up
|
|
437
|
+
# to `MAX_DECOMPRESSED_BYTES` just to check it.
|
|
438
|
+
return false if type == "iTXt" && !text.force_encoding(Encoding::UTF_8).valid_encoding?
|
|
439
|
+
|
|
440
|
+
@total += this_chunk
|
|
441
|
+
false
|
|
442
|
+
rescue Zlib::Error
|
|
443
|
+
false
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
# This chunk's own full decompressed byte count and (only when
|
|
447
|
+
# `keep_text` -- iTXt only, where UTF-8 validity has to be checked
|
|
448
|
+
# against the actual decompressed text) the text itself, or `nil`
|
|
449
|
+
# the moment `@total` plus what has decompressed so far crosses the
|
|
450
|
+
# ceiling -- checked incrementally, so a single huge chunk still
|
|
451
|
+
# aborts mid-stream rather than after materializing all of it.
|
|
452
|
+
# Retaining the bytes for iTXt costs no more than the real
|
|
453
|
+
# delegate already pays: `store_text_info` also holds the whole
|
|
454
|
+
# decompressed text at once, and this is bounded by the same
|
|
455
|
+
# ceiling either way.
|
|
456
|
+
#
|
|
457
|
+
# `zstream.finished?`, not just an absence of `Zlib::Error`: the
|
|
458
|
+
# incremental block form completes without raising for a stream
|
|
459
|
+
# missing only its trailing checksum (measured -- a 40 MiB payload
|
|
460
|
+
# with its last checksum byte cut off yields the full 40 MiB here
|
|
461
|
+
# and `finished? == false`), where `Zlib::Inflate.inflate` (the
|
|
462
|
+
# module method the real validators call) raises `Zlib::BufError`
|
|
463
|
+
# on the identical bytes. Without this check that chunk's bytes
|
|
464
|
+
# would wrongly join `@total` for a chunk the delegate itself would
|
|
465
|
+
# reject and never retain -- raising here routes it through the
|
|
466
|
+
# same `rescue Zlib::Error` below every other malformed chunk
|
|
467
|
+
# already takes, rather than inventing a second "not a bomb" path.
|
|
468
|
+
def decompressed_size(compressed, keep_text:)
|
|
469
|
+
zstream = Zlib::Inflate.new
|
|
470
|
+
this_chunk, text = accumulate(zstream, compressed, keep_text)
|
|
471
|
+
return [nil, nil] if this_chunk.nil?
|
|
472
|
+
raise Zlib::BufError, "truncated zlib stream" unless zstream.finished?
|
|
473
|
+
|
|
474
|
+
[this_chunk, text]
|
|
475
|
+
ensure
|
|
476
|
+
zstream&.close unless zstream&.closed?
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
# `[nil, nil]` the moment `@total` plus what has decompressed so
|
|
480
|
+
# far crosses the ceiling -- checked incrementally, so a single
|
|
481
|
+
# huge chunk still aborts mid-stream rather than after
|
|
482
|
+
# materializing all of it.
|
|
483
|
+
def accumulate(zstream, compressed, keep_text)
|
|
484
|
+
this_chunk = 0
|
|
485
|
+
text = keep_text ? +"" : nil
|
|
486
|
+
zstream.inflate(compressed) do |piece|
|
|
487
|
+
this_chunk += piece.bytesize
|
|
488
|
+
return [nil, nil] if @total + this_chunk > MAX_DECOMPRESSED_BYTES
|
|
489
|
+
|
|
490
|
+
text << piece if text
|
|
491
|
+
end
|
|
492
|
+
[this_chunk, text]
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
def skip(length)
|
|
496
|
+
io.seek(length + 4, IO::SEEK_CUR) # payload plus the CRC, neither read here
|
|
497
|
+
end
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
# Builds the `Report` a conform operation returns, kept apart from
|
|
501
|
+
# PNG metadata interpretation for the same reason `ChunkReader`
|
|
502
|
+
# above is -- and, like it, a nested class rather than a sibling:
|
|
503
|
+
# `report` is PNG conformance specifically, where `IssueCode` above
|
|
504
|
+
# is free-text-message slugging that owes nothing to PNG at all.
|
|
505
|
+
class ConformanceMapper
|
|
506
|
+
# png_conform's own `ValidationContext#add_error` hash, keyed
|
|
507
|
+
# exactly this way -- measured against the installed 0.1.4 gem,
|
|
508
|
+
# not assumed. `chunk_type` is a String on every reachable row (a
|
|
509
|
+
# real 4-byte chunk name, or the pseudo-name "SIGNATURE" for the
|
|
510
|
+
# whole-file signature check); `offset` is nil wherever the check
|
|
511
|
+
# runs before a chunk is even located, e.g. a missing IEND.
|
|
512
|
+
ISSUE_KEYS = %i[chunk_type message severity offset].freeze
|
|
513
|
+
|
|
514
|
+
# A chunk declaring a length near the 32-bit ceiling sends the
|
|
515
|
+
# delegate's own `io.read(length)` past what a single read
|
|
516
|
+
# syscall accepts, and on a platform that rejects the length
|
|
517
|
+
# outright that fails at the OS boundary before any
|
|
518
|
+
# `ValidationContext` result exists at all -- measured,
|
|
519
|
+
# `Errno::EINVAL` from `io_fread`, on declared lengths at and
|
|
520
|
+
# above 0x80000000.
|
|
521
|
+
#
|
|
522
|
+
# WHICH platform is load-bearing, and it is why this list can
|
|
523
|
+
# look dead on a green CI run. macOS `read(2)` refuses a length
|
|
524
|
+
# above INT_MAX; Linux caps the transfer at 0x7ffff000 and
|
|
525
|
+
# returns short instead, so the delegate never raises there and
|
|
526
|
+
# reports ordinary nonconformance. Measured both ways on the same
|
|
527
|
+
# 16-byte input: `arm64-darwin25` raised `Errno::EINVAL`,
|
|
528
|
+
# `x86_64-linux` (ruby:3.3.12, CI's platform) returned 16 bytes.
|
|
529
|
+
# The spec pins the mapping below by raising the errno directly
|
|
530
|
+
# rather than by finding an input that produces it here.
|
|
531
|
+
#
|
|
532
|
+
# This is the PNG analogue of the EMF row in 03-conform.md: the
|
|
533
|
+
# delegate's *reporting style* is an
|
|
534
|
+
# exception, but the meaning is the same nonconformance a
|
|
535
|
+
# returned issue would carry, so it goes on the allowlist rather
|
|
536
|
+
# than the generic exit 4.
|
|
537
|
+
MALFORMED_INPUT = [Errno::EINVAL].freeze
|
|
538
|
+
MALFORMED_CODE = "#{IssueCode::PREFIX}chunk_length_unreadable".freeze
|
|
539
|
+
MALFORMED_MESSAGE = "a chunk declared a length the file could not supply"
|
|
540
|
+
TRUNCATED_CODE = "#{IssueCode::PREFIX}delegate_truncated".freeze
|
|
541
|
+
TRUNCATED_MESSAGE = "PNG input ended before validation could complete"
|
|
542
|
+
|
|
543
|
+
# Claricle's own finding, not a mapped png_conform message -- same
|
|
544
|
+
# treatment MALFORMED_CODE above gets, and for the same reason:
|
|
545
|
+
# there is no upstream type to synthesize one from.
|
|
546
|
+
BOUND_EXCEEDED_CODE = "#{IssueCode::PREFIX}decompression_bound_exceeded".freeze
|
|
547
|
+
|
|
548
|
+
# Never `validate_file` (03-conform.md): it hands back a
|
|
549
|
+
# `FileAnalysis` with chunk and offset already discarded, and
|
|
550
|
+
# both it and `result.validation_result.errors` report
|
|
551
|
+
# `chunk_type: nil, chunk_offset: nil` for the same input.
|
|
552
|
+
# Location survives only on the `ValidationContext` this builds
|
|
553
|
+
# by hand.
|
|
554
|
+
#
|
|
555
|
+
# All three buckets, not `all_errors` alone: a conformant file
|
|
556
|
+
# still carries `all_info` (a gAMA note, measured on a real
|
|
557
|
+
# PngSuite fixture), and reading errors only would silently drop
|
|
558
|
+
# it -- which breaks D8's tri-state, since info must never
|
|
559
|
+
# disappear from a clean file's report.
|
|
560
|
+
#
|
|
561
|
+
# `image.with_path`, not `image.with_source`: the delegate's own
|
|
562
|
+
# `FullLoadReader` takes a path or an IO and opens its own handle
|
|
563
|
+
# either way (03-conform.md's Design table names the path
|
|
564
|
+
# constructor), so nothing here needs to hand over an already-open
|
|
565
|
+
# file. `AncillaryBoundsGuard` runs first, inside that same block --
|
|
566
|
+
# one Tempfile for a content-born image, not two -- and
|
|
567
|
+
# short-circuits before `png_conform` ever opens the file if any
|
|
568
|
+
# watched chunk would decompress past its ceiling (see the class
|
|
569
|
+
# above for why).
|
|
570
|
+
def self.report(image)
|
|
571
|
+
image.with_path do |path|
|
|
572
|
+
hit = File.open(path, "rb") { |io| AncillaryBoundsGuard.new(io).exceeded }
|
|
573
|
+
|
|
574
|
+
hit ? report_for(image, [bound_exceeded_issue(*hit)]) : report_for(image, mapped_issues(path))
|
|
575
|
+
end
|
|
576
|
+
rescue *MALFORMED_INPUT
|
|
577
|
+
report_for(image, [malformed_issue])
|
|
578
|
+
rescue IOError => e
|
|
579
|
+
raise unless e.is_a?(EOFError) || e.message == "data truncated"
|
|
580
|
+
|
|
581
|
+
report_for(image, [truncated_issue])
|
|
582
|
+
end
|
|
583
|
+
|
|
584
|
+
def self.report_for(image, issues)
|
|
585
|
+
Models::Report.new(source_path: image.path, format: image.format.to_s, issues: issues)
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
def self.mapped_issues(path)
|
|
589
|
+
context = validated_context(path)
|
|
590
|
+
|
|
591
|
+
(context.all_errors + context.all_warnings + context.all_info).map { |raw| issue_from(raw) }
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
def self.validated_context(path)
|
|
595
|
+
reader = ::PngConform::Readers::FullLoadReader.new(path)
|
|
596
|
+
service = ::PngConform::Services::ValidationService.new(reader, path)
|
|
597
|
+
service.validate
|
|
598
|
+
service.context
|
|
599
|
+
ensure
|
|
600
|
+
reader&.close
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
def self.issue_from(raw)
|
|
604
|
+
chunk_type, message, severity, offset = raw.values_at(*ISSUE_KEYS)
|
|
605
|
+
message = utf8(message)
|
|
606
|
+
|
|
607
|
+
Models::Issue.new(
|
|
608
|
+
severity: severity.to_s, code: IssueCode.for(message), message: message,
|
|
609
|
+
location: location_for(chunk_name(chunk_type), offset)
|
|
610
|
+
)
|
|
611
|
+
end
|
|
612
|
+
|
|
613
|
+
def self.utf8(value)
|
|
614
|
+
value.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
# The delegate uses pseudo-names such as SIGNATURE as well as real
|
|
618
|
+
# four-letter chunk types, so preserve every name that can be
|
|
619
|
+
# represented safely instead of applying the structural scanner's
|
|
620
|
+
# narrower chunk-type grammar. Binary names still retain their byte
|
|
621
|
+
# offset while the unsafe label is omitted.
|
|
622
|
+
def self.chunk_name(value)
|
|
623
|
+
return if value.nil? || !value.valid_encoding?
|
|
624
|
+
|
|
625
|
+
value.encode(Encoding::UTF_8)
|
|
626
|
+
rescue EncodingError
|
|
627
|
+
nil
|
|
628
|
+
end
|
|
629
|
+
|
|
630
|
+
# nil rather than an all-nil Location: chunk_type is populated on
|
|
631
|
+
# every row reachable through this delegate (measured), but
|
|
632
|
+
# nothing here should invent a location for a hash shape a later
|
|
633
|
+
# png_conform release might still hand back with neither field
|
|
634
|
+
# set.
|
|
635
|
+
def self.location_for(chunk_type, offset)
|
|
636
|
+
return nil if chunk_type.nil? && offset.nil?
|
|
637
|
+
|
|
638
|
+
Models::Location.new(chunk: chunk_type, byte_offset: offset)
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
def self.malformed_issue
|
|
642
|
+
Models::Issue.new(severity: "error", code: MALFORMED_CODE, message: MALFORMED_MESSAGE)
|
|
643
|
+
end
|
|
644
|
+
|
|
645
|
+
# png_conform raises EOFError for empty input and `IOError: data
|
|
646
|
+
# truncated` for the other sub-signature lengths. Treat only those
|
|
647
|
+
# parser outcomes as nonconformance while propagating unrelated IO
|
|
648
|
+
# failures.
|
|
649
|
+
def self.truncated_issue
|
|
650
|
+
Models::Issue.new(severity: "error", code: TRUNCATED_CODE, message: TRUNCATED_MESSAGE)
|
|
651
|
+
end
|
|
652
|
+
|
|
653
|
+
# `offset` is real, not invented: `AncillaryBoundsGuard` read it off
|
|
654
|
+
# the chunk's own 8-byte header while scanning.
|
|
655
|
+
def self.bound_exceeded_issue(chunk_type, offset)
|
|
656
|
+
Models::Issue.new(
|
|
657
|
+
severity: "error", code: BOUND_EXCEEDED_CODE,
|
|
658
|
+
message: "the #{chunk_type} chunk's compressed payload decompresses past the safety bound",
|
|
659
|
+
location: Models::Location.new(chunk: chunk_type, byte_offset: offset)
|
|
660
|
+
)
|
|
661
|
+
end
|
|
662
|
+
|
|
663
|
+
private_class_method :report_for, :mapped_issues, :validated_context,
|
|
664
|
+
:issue_from, :location_for, :malformed_issue, :bound_exceeded_issue
|
|
665
|
+
end
|
|
666
|
+
|
|
667
|
+
private_constant :ConformanceMapper
|
|
668
|
+
|
|
669
|
+
# Chunk-sequence faults png_conform 0.1.4 does not correctly
|
|
670
|
+
# report (D23). Reads ONLY 8-byte chunk headers and seeks over
|
|
671
|
+
# every payload, so the bytes read are 8 x the HEADERS EXAMINED --
|
|
672
|
+
# measured 24 bytes for a 75-byte file and for a 10 MiB one. Not
|
|
673
|
+
# "independent of file size", which overstates it: a 20-byte
|
|
674
|
+
# IEND-only stream reads 8 bytes, and 100 empty chunks plus IEND read
|
|
675
|
+
# 808 (the 100 alone read 800). The
|
|
676
|
+
# guarantee is that no payload is ever read, whatever it declares.
|
|
677
|
+
#
|
|
678
|
+
# Beside `ChunkReader` rather than inside it: that one stops at
|
|
679
|
+
# IEND, keeps payloads, and supports a non-seekable pipe through
|
|
680
|
+
# `drain`. This needs `io.size` and absolute seeks, treats a short
|
|
681
|
+
# read as a fault rather than an ending, and classifies what lies
|
|
682
|
+
# past IEND.
|
|
683
|
+
class StructureScanner
|
|
684
|
+
SIGNATURE_BYTES = 8
|
|
685
|
+
HEADER_BYTES = 8
|
|
686
|
+
# Length field, type and CRC: what a chunk costs beyond its payload.
|
|
687
|
+
FRAMING_BYTES = 12
|
|
688
|
+
# REC-PNG-20031110 5.4 restricts a type code to four ASCII
|
|
689
|
+
# letters. Anything else is not a name, and handing raw bytes to
|
|
690
|
+
# `Location#chunk` is not merely untidy -- a high byte makes
|
|
691
|
+
# lutaml raise, which would take the scan down on exactly the
|
|
692
|
+
# malformed input this exists to report.
|
|
693
|
+
LETTERS = /\A[A-Za-z]{4}\z/
|
|
694
|
+
|
|
695
|
+
# Where the walk stopped, carrying the header it stopped on so
|
|
696
|
+
# the terminal issue never re-reads one.
|
|
697
|
+
Stop = Data.define(:state, :offset, :length, :type)
|
|
698
|
+
|
|
699
|
+
def initialize(io)
|
|
700
|
+
@io = io
|
|
701
|
+
@size = io.size
|
|
702
|
+
@seen_ihdr = false
|
|
703
|
+
end
|
|
704
|
+
|
|
705
|
+
# Byte-ordered, with the terminal issue last. Empty for a
|
|
706
|
+
# well-formed datastream -- never nil.
|
|
707
|
+
#
|
|
708
|
+
# Memoized because scanning twice would append twice, and the
|
|
709
|
+
# bounded-read guarantee would go with it. Both exits return the
|
|
710
|
+
# same accumulator, so they cannot drift apart into two contracts.
|
|
711
|
+
def issues
|
|
712
|
+
@issues ||= scan
|
|
713
|
+
end
|
|
714
|
+
|
|
715
|
+
private
|
|
716
|
+
|
|
717
|
+
def scan
|
|
718
|
+
found = []
|
|
719
|
+
return found << rangeless("png.chunk_truncated", :shorter_than_signature) if @size < SIGNATURE_BYTES
|
|
720
|
+
|
|
721
|
+
@found = found
|
|
722
|
+
terminate(walk)
|
|
723
|
+
found
|
|
724
|
+
end
|
|
725
|
+
|
|
726
|
+
# Ends in exactly one of three states, and the short-file guard
|
|
727
|
+
# above is what keeps that true: with `size` at least 8 and
|
|
728
|
+
# `offset` only advancing when a whole record fits,
|
|
729
|
+
# `size - offset` can never go negative.
|
|
730
|
+
def walk
|
|
731
|
+
offset = SIGNATURE_BYTES
|
|
732
|
+
until offset == @size
|
|
733
|
+
length, type = header_at(offset)
|
|
734
|
+
return Stop[:truncated, offset, length, type] unless record_fits?(offset, length)
|
|
735
|
+
|
|
736
|
+
note_duplicate(type, offset, span(length))
|
|
737
|
+
offset += span(length)
|
|
738
|
+
return Stop[:complete, offset, nil, nil] if type == "IEND"
|
|
739
|
+
end
|
|
740
|
+
Stop[:ended, offset, nil, nil]
|
|
741
|
+
end
|
|
742
|
+
|
|
743
|
+
def terminate(stop)
|
|
744
|
+
return @found << truncated(stop) if stop.state == :truncated
|
|
745
|
+
return @found << rangeless("png.chunk_truncated", :ends_inside_record) unless reached?(stop.offset)
|
|
746
|
+
|
|
747
|
+
case stop.state
|
|
748
|
+
when :ended then @found << rangeless("png.missing_iend", :missing_iend, chunk: "IEND")
|
|
749
|
+
when :complete then trailing(stop.offset)
|
|
750
|
+
end
|
|
751
|
+
end
|
|
752
|
+
|
|
753
|
+
# `record_fits?` proves a record fits inside `io.size`. It never
|
|
754
|
+
# proves those bytes EXIST: this walk reads headers and nothing
|
|
755
|
+
# else, so a stream ending INSIDE its last record walks to a clean
|
|
756
|
+
# verdict. Measured before this guard -- a 45-byte PNG cut to 41,
|
|
757
|
+
# losing only IEND's CRC, reported no issues at all: the header at
|
|
758
|
+
# offset 33 read fine and bytes 41 to 44 were never asked for.
|
|
759
|
+
#
|
|
760
|
+
# Keyed off what the WALK consumed, never off `@size`. A guard on
|
|
761
|
+
# `@size` would call an over-reporting whole file truncated, which
|
|
762
|
+
# is the same defect pointing the other way.
|
|
763
|
+
#
|
|
764
|
+
# One byte settles it, and it costs one seek on a clean file.
|
|
765
|
+
def reached?(offset)
|
|
766
|
+
return true if offset.zero?
|
|
767
|
+
|
|
768
|
+
@io.seek(offset - 1)
|
|
769
|
+
@io.read(1)&.bytesize == 1
|
|
770
|
+
end
|
|
771
|
+
|
|
772
|
+
# Only a COMPLETE repeat counts. The bounds test above returns
|
|
773
|
+
# first, so a repeated IHDR whose record is cut never arrives
|
|
774
|
+
# here -- which is the right answer, since its range would name
|
|
775
|
+
# bytes the file does not contain.
|
|
776
|
+
def note_duplicate(type, offset, span)
|
|
777
|
+
return unless type == "IHDR"
|
|
778
|
+
|
|
779
|
+
@found << issue("png.duplicate_ihdr", STRUCTURE_MESSAGES[:duplicate_ihdr], offset, span, "IHDR") if @seen_ihdr
|
|
780
|
+
@seen_ihdr = true
|
|
781
|
+
end
|
|
782
|
+
|
|
783
|
+
# The range is what is really there, not what was declared: a
|
|
784
|
+
# half-open range must never name bytes the file lacks.
|
|
785
|
+
#
|
|
786
|
+
# Stated against a TRUTHFUL `io.size`, which this scanner documents
|
|
787
|
+
# rather than enforces. Under a size that over-reports, this range and
|
|
788
|
+
# `trailing`'s both name bytes past EOF -- measured, a real file
|
|
789
|
+
# truncated to 41 bytes after `io.size` was captured yields
|
|
790
|
+
# byte_offset 56, which is 15 bytes past its end. That is the deferred
|
|
791
|
+
# gap the over-reporting examples in the spec pin as known.
|
|
792
|
+
def truncated(stop)
|
|
793
|
+
remaining = @size - stop.offset
|
|
794
|
+
message =
|
|
795
|
+
if stop.length
|
|
796
|
+
format(STRUCTURE_MESSAGES[:chunk_overruns], span(stop.length), remaining)
|
|
797
|
+
else
|
|
798
|
+
format(STRUCTURE_MESSAGES[:header_residue], bytes_phrase(remaining))
|
|
799
|
+
end
|
|
800
|
+
issue("png.chunk_truncated", message, stop.offset, remaining, letters(stop.type))
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
# Everything past IEND is outside the datastream and gets exactly
|
|
804
|
+
# one issue over the whole remainder.
|
|
805
|
+
def trailing(offset)
|
|
806
|
+
remaining = @size - offset
|
|
807
|
+
return if remaining.zero?
|
|
808
|
+
|
|
809
|
+
@found <<
|
|
810
|
+
if complete_iend_at?(offset)
|
|
811
|
+
issue("png.duplicate_iend", STRUCTURE_MESSAGES[:duplicate_iend], offset, remaining, "IEND")
|
|
812
|
+
else
|
|
813
|
+
issue("png.trailing_data", STRUCTURE_MESSAGES[:trailing_data], offset, remaining, nil)
|
|
814
|
+
end
|
|
815
|
+
end
|
|
816
|
+
|
|
817
|
+
# A type code alone is not a chunk. Calling a nine-byte remainder
|
|
818
|
+
# whose header declares 25 "a second IEND chunk" asserts a record
|
|
819
|
+
# that is not there -- the same completeness rule `note_duplicate`
|
|
820
|
+
# obeys.
|
|
821
|
+
def complete_iend_at?(offset)
|
|
822
|
+
length, type = header_at(offset)
|
|
823
|
+
type == "IEND" && record_fits?(offset, length)
|
|
824
|
+
end
|
|
825
|
+
|
|
826
|
+
# The whole record, payload and framing together.
|
|
827
|
+
def span(length) = FRAMING_BYTES + length
|
|
828
|
+
|
|
829
|
+
# Under a truthful `io.size` a residue is 1 to 7 bytes, so the
|
|
830
|
+
# singular is reachable -- the over-reporting rows in the spec do
|
|
831
|
+
# reach larger values, e.g. "1000 bytes left". Either way
|
|
832
|
+
# "1 bytes left" is copy a person reads while something is wrong.
|
|
833
|
+
def bytes_phrase(count) = "#{count} byte#{"s" unless count == 1}"
|
|
834
|
+
|
|
835
|
+
# The one completeness rule, in one place. `walk` and
|
|
836
|
+
# `complete_iend_at?` are its callers -- writing it twice, once as an
|
|
837
|
+
# overrun test and once as its inverse, is how the two drift apart.
|
|
838
|
+
# (`note_duplicate` does NOT call it; it is handed the span `walk`
|
|
839
|
+
# already computed.)
|
|
840
|
+
def record_fits?(offset, length)
|
|
841
|
+
!length.nil? && offset + span(length) <= @size
|
|
842
|
+
end
|
|
843
|
+
|
|
844
|
+
# nil when no header fits, which `walk` reads as truncation.
|
|
845
|
+
#
|
|
846
|
+
# The size test is not enough on its own, because it trusts
|
|
847
|
+
# `io.size`. A short or absent read is checked too, so the walk
|
|
848
|
+
# stays total even where that number lies -- measured, `read(8)`
|
|
849
|
+
# at EOF returns nil and `nil.unpack` raises NoMethodError. A
|
|
850
|
+
# short read is a fault here exactly as it is everywhere else in
|
|
851
|
+
# this scanner, not an ending. `ChunkReader#read_header` already
|
|
852
|
+
# takes the same care for the same reason -- cited by NAME, not by
|
|
853
|
+
# line: a same-file line range rots the moment anything above it
|
|
854
|
+
# moves, which is exactly how this comment was wrong once already.
|
|
855
|
+
def header_at(offset)
|
|
856
|
+
return nil if offset + HEADER_BYTES > @size
|
|
857
|
+
|
|
858
|
+
@io.seek(offset)
|
|
859
|
+
header = @io.read(HEADER_BYTES)
|
|
860
|
+
return nil unless header&.bytesize == HEADER_BYTES
|
|
861
|
+
|
|
862
|
+
header.unpack("Na4")
|
|
863
|
+
end
|
|
864
|
+
|
|
865
|
+
def letters(type)
|
|
866
|
+
type if type&.match?(LETTERS)
|
|
867
|
+
end
|
|
868
|
+
|
|
869
|
+
def issue(code, message, offset, length, chunk)
|
|
870
|
+
Models::Issue.new(
|
|
871
|
+
severity: "error", code: code, message: message,
|
|
872
|
+
location: Models::Location.new(byte_offset: offset, byte_length: length, chunk: chunk)
|
|
873
|
+
)
|
|
874
|
+
end
|
|
875
|
+
|
|
876
|
+
# Absent is not zero: a fault with no chunk to point at carries no
|
|
877
|
+
# range rather than an invented one at EOF.
|
|
878
|
+
def rangeless(code, key, chunk: nil)
|
|
879
|
+
Models::Issue.new(severity: "error", code: code, message: STRUCTURE_MESSAGES[key],
|
|
880
|
+
location: Models::Location.new(chunk: chunk))
|
|
881
|
+
end
|
|
882
|
+
end
|
|
883
|
+
|
|
884
|
+
# Turns a parsed IHDR (plus the rest of the chunk list, for dpi) into
|
|
885
|
+
# an Inspection model. Its own concern -- IHDR metadata, not
|
|
886
|
+
# conformance -- so it is its own nested class rather than more
|
|
887
|
+
# top-level methods on Png, matching the pattern already used for
|
|
888
|
+
# ChunkReader, AncillaryBoundsGuard, ConformanceMapper and
|
|
889
|
+
# StructureScanner above.
|
|
890
|
+
class MetadataInspector
|
|
891
|
+
# A signature-only file yields zero chunks and raises nothing, and a
|
|
892
|
+
# well-formed chunk can carry a short payload -- unpack returns nils
|
|
893
|
+
# for the missing fields rather than failing. Both are measured, so
|
|
894
|
+
# absence of an exception proves nothing and this gate is what makes
|
|
895
|
+
# "the metadata parsed" a real claim (D17).
|
|
896
|
+
def self.usable_ihdr?(ihdr)
|
|
897
|
+
!ihdr.nil? && ihdr.data.bytesize == IHDR_BYTES
|
|
898
|
+
end
|
|
899
|
+
|
|
900
|
+
def self.readable(image, ihdr, chunks)
|
|
901
|
+
width, height, depth, color_type, *encoding = ihdr.data.unpack(IHDR_LAYOUT)
|
|
902
|
+
|
|
903
|
+
Models::Inspection.new(
|
|
904
|
+
format: image.format.to_s,
|
|
905
|
+
width: width.to_f, height: height.to_f,
|
|
906
|
+
dpi: dpi(chunks), color_space: COLOR_SPACES[color_type],
|
|
907
|
+
meta: meta(depth, encoding), parse_status: "ok"
|
|
908
|
+
)
|
|
909
|
+
end
|
|
910
|
+
|
|
911
|
+
# Inspection has named slots for dimensions, dpi and colour space;
|
|
912
|
+
# what is left of IHDR is format-native, so it goes here.
|
|
913
|
+
def self.meta(depth, encoding)
|
|
914
|
+
compression, filter, interlace = encoding
|
|
915
|
+
|
|
916
|
+
{ "bit_depth" => depth, "compression" => compression,
|
|
917
|
+
"filter" => filter, "interlace" => interlace }
|
|
918
|
+
end
|
|
919
|
+
|
|
920
|
+
def self.unreadable(image)
|
|
921
|
+
Models::Inspection.new(
|
|
922
|
+
format: image.format.to_s,
|
|
923
|
+
parse_status: "failed",
|
|
924
|
+
issues: [unreadable_issue]
|
|
925
|
+
)
|
|
926
|
+
end
|
|
927
|
+
|
|
928
|
+
# One message for every way the header can be unreadable: absent,
|
|
929
|
+
# the wrong length, or never reached because the read itself failed.
|
|
930
|
+
# Naming only the length case would misdescribe the other two.
|
|
931
|
+
def self.unreadable_issue
|
|
932
|
+
Models::Issue.new(
|
|
933
|
+
severity: "error",
|
|
934
|
+
code: "png.ihdr_unreadable",
|
|
935
|
+
message: "PNG header (IHDR) could not be read"
|
|
936
|
+
)
|
|
937
|
+
end
|
|
938
|
+
|
|
939
|
+
# nil when pHYs is absent, and nil when it records an aspect ratio
|
|
940
|
+
# rather than a physical unit. Both are ordinary files, not errors.
|
|
941
|
+
#
|
|
942
|
+
# The length is checked before unpacking, because unpack skips a
|
|
943
|
+
# directive it has too few bytes for but keeps reading the rest
|
|
944
|
+
# from where it started. Measured across every length below 9:
|
|
945
|
+
# 1 to 3 give [nil, nil, 1], which passes the unit check and then
|
|
946
|
+
# multiplies nil. 5 to 7 give [value, nil, 1], which the axis check
|
|
947
|
+
# below rejects on its own. So this gate stops a crash, not a wrong
|
|
948
|
+
# number.
|
|
949
|
+
def self.dpi(chunks)
|
|
950
|
+
phys = chunks.find { |chunk| chunk.type == "pHYs" }
|
|
951
|
+
return nil unless phys && phys.data.bytesize >= PHYS_BYTES
|
|
952
|
+
|
|
953
|
+
# dpi is a single number, so a PNG with non-square pixels has no
|
|
954
|
+
# one value to report -- and reporting the x axis alone would say
|
|
955
|
+
# 72 for an image that is 72x144. Measured: X=2835 Y=5669 is
|
|
956
|
+
# exactly that case. Unequal axes give nil rather than half the
|
|
957
|
+
# truth.
|
|
958
|
+
horizontal, vertical, unit = phys.data.unpack(PHYS_LAYOUT)
|
|
959
|
+
return nil unless unit == METRE_UNIT && horizontal == vertical
|
|
960
|
+
|
|
961
|
+
horizontal * METRES_PER_INCH
|
|
962
|
+
end
|
|
963
|
+
end
|
|
964
|
+
|
|
965
|
+
private_constant :IHDR_LAYOUT, :IHDR_BYTES, :PHYS_LAYOUT, :PHYS_BYTES,
|
|
966
|
+
:METRE_UNIT, :METRES_PER_INCH, :WANTED_CHUNKS,
|
|
967
|
+
:COLOR_SPACES, :STRUCTURE_MESSAGES, :MAX_CHUNK_READ, :DRAIN_BUFFER,
|
|
968
|
+
:MAX_DECOMPRESSED_BYTES, :Chunk, :ChunkReader, :StructureScanner,
|
|
969
|
+
:AncillaryBoundsGuard, :MetadataInspector
|
|
970
|
+
|
|
971
|
+
def inspection(image)
|
|
972
|
+
chunks = read_chunks(image)
|
|
973
|
+
ihdr = chunks.find { |chunk| chunk.type == "IHDR" }
|
|
974
|
+
|
|
975
|
+
return MetadataInspector.unreadable(image) unless MetadataInspector.usable_ihdr?(ihdr)
|
|
976
|
+
|
|
977
|
+
MetadataInspector.readable(image, ihdr, chunks)
|
|
978
|
+
end
|
|
979
|
+
|
|
980
|
+
# png_conform's own mapping is documented on `ConformanceMapper.report`
|
|
981
|
+
# above. Structural issues run FIRST (D23): a cheap bounded-header
|
|
982
|
+
# pass answering whether the chunk sequence is well-formed, before
|
|
983
|
+
# the delegate's semantic opinions. Neither short-circuits nor
|
|
984
|
+
# dedupes against the other -- they were never designed to agree,
|
|
985
|
+
# and each catches shapes the other misses (a duplicate IHDR here,
|
|
986
|
+
# a bad signature only there), so both ship as-is.
|
|
987
|
+
def conformance_report(image)
|
|
988
|
+
require "png_conform"
|
|
989
|
+
|
|
990
|
+
Models::Report.new(source_path: image.path, format: image.format.to_s,
|
|
991
|
+
issues: structural_issues(image) + ConformanceMapper.report(image).issues)
|
|
992
|
+
end
|
|
993
|
+
|
|
994
|
+
private
|
|
995
|
+
|
|
996
|
+
# Lazily required (D5): the detector's `emf` is the sole eager
|
|
997
|
+
# delegate, and a gem should not pay for a parser it may never use.
|
|
998
|
+
#
|
|
999
|
+
# `StreamingReader.open`, not `FullLoadReader.new`. The full reader
|
|
1000
|
+
# is `read_until: :eof`, and using it cost three separate defects,
|
|
1001
|
+
# all measured:
|
|
1002
|
+
#
|
|
1003
|
+
# - it reads PAST `IEND`. A valid `pHYs` chunk appended after a
|
|
1004
|
+
# complete PNG was reported as that file's DPI (72.009), and two
|
|
1005
|
+
# concatenated PNGs reported `"failed"` although the first image
|
|
1006
|
+
# is perfectly readable.
|
|
1007
|
+
# - it owns the file it opens and closes it only on `#close`, which
|
|
1008
|
+
# this never called. 100 inspections held 100 descriptors open
|
|
1009
|
+
# until GC.
|
|
1010
|
+
# - it accepts only a String, so `Image.from_path(Pathname(...))`
|
|
1011
|
+
# detected and read fine and then raised `NoMethodError` on
|
|
1012
|
+
# `rewind` -- an exit-4 defect code for ordinary library input.
|
|
1013
|
+
#
|
|
1014
|
+
# The streaming reader stops at `IEND` itself, closes its own file,
|
|
1015
|
+
# and takes a Pathname.
|
|
1016
|
+
#
|
|
1017
|
+
# `chunks` is built outside the block and returned by both exits, so
|
|
1018
|
+
# a failure part-way through keeps what was already read. Dropping
|
|
1019
|
+
# it cost a real defect: a complete IHDR followed by two stray bytes
|
|
1020
|
+
# raised the truncation error and reported "PNG header (IHDR) could
|
|
1021
|
+
# not be read" for a header this had just read.
|
|
1022
|
+
#
|
|
1023
|
+
# A truncated chunk no longer raises to get that behaviour --
|
|
1024
|
+
# `ChunkReader#gather` reads its own header, payload and CRC by
|
|
1025
|
+
# hand and stops the loop on any short read, so "keep what's
|
|
1026
|
+
# already collected" is just where the loop ends, not something
|
|
1027
|
+
# caught here.
|
|
1028
|
+
def read_chunks(image)
|
|
1029
|
+
require "png_conform"
|
|
1030
|
+
|
|
1031
|
+
chunks = []
|
|
1032
|
+
image.with_path do |path|
|
|
1033
|
+
PngConform::Readers::StreamingReader.open(path) { |reader| ChunkReader.new(reader.io).gather(into: chunks) }
|
|
1034
|
+
end
|
|
1035
|
+
chunks
|
|
1036
|
+
# `gather` no longer hands a declared length to `#read` unbounded,
|
|
1037
|
+
# so `Errno::EINVAL` (what a chunk declaring 0x8000000d bytes used
|
|
1038
|
+
# to produce asking the OS for that many, on a platform whose
|
|
1039
|
+
# `read(2)` refuses a length above INT_MAX -- macOS does, Linux
|
|
1040
|
+
# does not) and the reader's own
|
|
1041
|
+
# `IOError: data truncated` cannot happen from THIS path any more
|
|
1042
|
+
# -- measured against png_conform 0.1.4 rather than assumed. This
|
|
1043
|
+
# is scoped to malformed input, not "impossible in general": an
|
|
1044
|
+
# `IOError` from an IO that closes unexpectedly mid-inspection
|
|
1045
|
+
# would still be a real defect and still propagate to exit 4.
|
|
1046
|
+
#
|
|
1047
|
+
# ENOENT is deliberately absent. A file that vanished is not
|
|
1048
|
+
# malformed input, and absorbing it reported "PNG header (IHDR)
|
|
1049
|
+
# could not be read" about a file that was no longer there. It
|
|
1050
|
+
# propagates, and the runner already maps it to exit 2 -- the code
|
|
1051
|
+
# the README gives for a missing file.
|
|
1052
|
+
#
|
|
1053
|
+
# PngConform::Error stays named although no code path in 0.1.4
|
|
1054
|
+
# raises it: it is the delegate's declared parse failure, the
|
|
1055
|
+
# gemspec admits every later 0.1.x, and a patch release that starts
|
|
1056
|
+
# raising it would otherwise turn a bad file into exit 4. ParseError
|
|
1057
|
+
# descends from it, so naming both is redundant.
|
|
1058
|
+
rescue PngConform::Error
|
|
1059
|
+
chunks
|
|
1060
|
+
end
|
|
1061
|
+
|
|
1062
|
+
# The structural pre-pass (D23), called from `conformance_report`.
|
|
1063
|
+
# Opens and closes the scanner's file independently of the
|
|
1064
|
+
# delegate.
|
|
1065
|
+
#
|
|
1066
|
+
# Refuses a non-regular path rather than handing it to
|
|
1067
|
+
# `StructureScanner`, which needs a truthful `io.size` and absolute
|
|
1068
|
+
# seeks -- neither exists for a pipe. Measured, each wrong a
|
|
1069
|
+
# different way: a named FIFO gives `io.size` 0 (fstat reports no
|
|
1070
|
+
# size for one), so a well-formed PNG came back
|
|
1071
|
+
# `shorter_than_signature`; a `/dev/fd/<n>` pipe gives `io.size` the
|
|
1072
|
+
# buffered byte count, so the walk proceeds and its first `io.seek`
|
|
1073
|
+
# raises `Errno::ESPIPE` uncaught. `InvocationError` names this a
|
|
1074
|
+
# caller problem, same as `claricle.rb`'s own path/pattern check --
|
|
1075
|
+
# unlike `ChunkReader` below, which only skips forward and drains
|
|
1076
|
+
# past `Errno::ESPIPE`, this scanner cannot read a whole unseekable
|
|
1077
|
+
# stream just to learn its size without losing the bounded-read
|
|
1078
|
+
# guarantee the class documents.
|
|
1079
|
+
#
|
|
1080
|
+
# Checked on `image.path` directly, before `with_path` ever opens
|
|
1081
|
+
# anything: nil for a content-born image, which always gets a
|
|
1082
|
+
# Tempfile from `with_path` and so never needs the check, and for a
|
|
1083
|
+
# path-born image the same string `with_path` would yield. Checking
|
|
1084
|
+
# first also means never re-opening a FIFO an earlier read in this
|
|
1085
|
+
# call (detection, `read_chunks`) already drained -- that has no
|
|
1086
|
+
# writer left, so a second open would hang rather than raise.
|
|
1087
|
+
#
|
|
1088
|
+
# `File.stat`, not `File.file?`: `file?` alone is also false for a
|
|
1089
|
+
# path that never existed, and raising InvocationError for THAT would
|
|
1090
|
+
# swallow `Errno::ENOENT` -- the exact fault `read_chunks` below
|
|
1091
|
+
# deliberately leaves unrescued so it propagates. `File.stat` raises
|
|
1092
|
+
# that same ENOENT itself for a missing path, before `ftype` is ever
|
|
1093
|
+
# reached, so a vanished path still gets the genuine error rather than
|
|
1094
|
+
# a misleading "not a regular file". `ftype`, not `file?`, because
|
|
1095
|
+
# `file?` was already the right call for THIS check (it follows a
|
|
1096
|
+
# symlink to its target, `ftype` on the path alone does not -- a
|
|
1097
|
+
# symlink to a regular file must still pass); `File.stat(p).ftype`
|
|
1098
|
+
# gets both properties from the one syscall `file?` and `exist?`
|
|
1099
|
+
# needed two of.
|
|
1100
|
+
def structural_issues(image)
|
|
1101
|
+
image.path&.then { |p| raise InvocationError, "not a regular file: #{p}" unless File.stat(p).ftype == "file" }
|
|
1102
|
+
|
|
1103
|
+
image.with_path { |path| File.open(path, "rb") { |io| StructureScanner.new(io).issues } }
|
|
1104
|
+
end
|
|
1105
|
+
end
|
|
1106
|
+
end
|
|
1107
|
+
end
|