claricle 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +10 -1
- data/LICENSE +25 -0
- data/README.adoc +439 -157
- data/TODO.plan/00-overview.md +502 -0
- data/TODO.plan/01-core.md +296 -0
- data/TODO.plan/02-inspect.md +163 -0
- data/TODO.plan/03-conform.md +208 -0
- data/TODO.plan/04-convert.md +216 -0
- data/exe/claricle +1 -1
- data/lib/claricle/batch.rb +240 -0
- data/lib/claricle/cli.rb +514 -40
- data/lib/claricle/conformance_options.rb +49 -0
- data/lib/claricle/conversion_engine.rb +21 -0
- data/lib/claricle/convert.rb +21 -0
- data/lib/claricle/detector.rb +696 -0
- data/lib/claricle/emf_header.rb +33 -0
- data/lib/claricle/errors.rb +73 -0
- data/lib/claricle/fault.rb +69 -0
- data/lib/claricle/handlers/base.rb +169 -0
- data/lib/claricle/handlers/extension_points.rb +124 -0
- data/lib/claricle/handlers/metafile.rb +1011 -0
- data/lib/claricle/handlers/pdf.rb +961 -0
- data/lib/claricle/handlers/pdf_arlington/definitions.rb +54 -0
- data/lib/claricle/handlers/pdf_arlington/kind.rb +62 -0
- data/lib/claricle/handlers/pdf_arlington/predicate_gate.rb +39 -0
- data/lib/claricle/handlers/pdf_arlington/rules.rb +86 -0
- data/lib/claricle/handlers/pdf_arlington/walker.rb +131 -0
- data/lib/claricle/handlers/pdf_arlington.rb +38 -0
- data/lib/claricle/handlers/pdf_profiles.rb +66 -0
- data/lib/claricle/handlers/png.rb +1107 -0
- data/lib/claricle/handlers/postscript.rb +1357 -0
- data/lib/claricle/handlers/svg.rb +787 -0
- data/lib/claricle/image.rb +272 -0
- data/lib/claricle/lossiness/levels.rb +13 -0
- data/lib/claricle/lossiness.rb +864 -0
- data/lib/claricle/models/base.rb +460 -0
- data/lib/claricle/models/batch_item.rb +127 -0
- data/lib/claricle/models/conversion.rb +167 -0
- data/lib/claricle/models/format_capability.rb +40 -0
- data/lib/claricle/models/free_form_hash.rb +77 -0
- data/lib/claricle/models/inspection.rb +188 -0
- data/lib/claricle/models/issue.rb +23 -0
- data/lib/claricle/models/location.rb +51 -0
- data/lib/claricle/models/report.rb +54 -0
- data/lib/claricle/models/uncoerced_integer.rb +38 -0
- data/lib/claricle/postscript_fidelity/verdict.rb +37 -0
- data/lib/claricle/postscript_fidelity/view_box.rb +82 -0
- data/lib/claricle/postscript_fidelity.rb +160 -0
- data/lib/claricle/registry.rb +125 -0
- data/lib/claricle/version.rb +1 -1
- data/lib/claricle/writer.rb +468 -0
- data/lib/claricle.rb +244 -7
- metadata +201 -19
- data/claricle.gemspec +0 -42
- data/sig/claricle.rbs +0 -4
|
@@ -0,0 +1,696 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "stringio"
|
|
4
|
+
require "rexml/parsers/pullparser"
|
|
5
|
+
require "rexml/source"
|
|
6
|
+
require "rexml/text"
|
|
7
|
+
require "rexml/xmltokens"
|
|
8
|
+
require "emf"
|
|
9
|
+
|
|
10
|
+
module Claricle
|
|
11
|
+
# The table of contents at the front of a DOS/Windows binary-preview
|
|
12
|
+
# EPS. Detection needs its magic while the PostScript handler needs the
|
|
13
|
+
# declared language-section range, so both answers live with the other
|
|
14
|
+
# format framing rather than being transcribed twice.
|
|
15
|
+
module EpsBinary
|
|
16
|
+
SIGNATURE = "\xC5\xD0\xD3\xC6".b.freeze
|
|
17
|
+
HEADER_BYTES = 30
|
|
18
|
+
POSTSCRIPT_FIELDS_OFFSET = 4
|
|
19
|
+
POSTSCRIPT_FIELDS_BYTES = 8
|
|
20
|
+
|
|
21
|
+
class << self
|
|
22
|
+
def wrapped?(bytes)
|
|
23
|
+
bytes.byteslice(0, SIGNATURE.bytesize) == SIGNATURE
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# `[offset, length]` for a complete, non-empty PostScript section,
|
|
27
|
+
# or nil when the wrapper's declared range is outside the file.
|
|
28
|
+
def postscript_range(bytes, total_bytes)
|
|
29
|
+
return nil unless wrapped?(bytes) && bytes.bytesize >= HEADER_BYTES
|
|
30
|
+
|
|
31
|
+
offset, length = bytes.byteslice(POSTSCRIPT_FIELDS_OFFSET,
|
|
32
|
+
POSTSCRIPT_FIELDS_BYTES).unpack("V2")
|
|
33
|
+
return nil if offset < HEADER_BYTES || length.zero? || offset > total_bytes
|
|
34
|
+
return nil if length > total_bytes - offset
|
|
35
|
+
|
|
36
|
+
[offset, length]
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private_constant :EpsBinary
|
|
42
|
+
|
|
43
|
+
# Attribute defaults declared in an internal DTD subset. XML 1.0
|
|
44
|
+
# requires a processor to apply them and REXML's DOM does, but the
|
|
45
|
+
# PullParser hands the declarations over raw, so this reassembles them.
|
|
46
|
+
# Its own module because the precedence rules are a self-contained
|
|
47
|
+
# concern, and Detector is about choosing between formats.
|
|
48
|
+
#
|
|
49
|
+
# Known gap, upstream: REXML raises "Bad ATTLIST declaration!" for a
|
|
50
|
+
# `#FIXED '...'` default, which XML permits. Only that combination --
|
|
51
|
+
# measured: a plain single-quoted default reads fine, and so does
|
|
52
|
+
# `#FIXED "..."`. It happens inside REXML before any event reaches us,
|
|
53
|
+
# so such a document is reported as an unknown format. It fails closed
|
|
54
|
+
# rather than misdetecting, and working around REXML's DTD parser
|
|
55
|
+
# would cost more than the gap.
|
|
56
|
+
module AttributeDefaults
|
|
57
|
+
XML_NAME = "#{REXML::XMLTokens::NAME_START_CHAR}" \
|
|
58
|
+
"#{REXML::XMLTokens::NAME_CHAR}*".freeze
|
|
59
|
+
|
|
60
|
+
# One "name TYPE [#DEFAULT] value" group of an ATTLIST body, enough
|
|
61
|
+
# to recover declaration order. Values may be absent (#REQUIRED etc).
|
|
62
|
+
DECLARATION = /
|
|
63
|
+
(?<name>#{XML_NAME})\s+
|
|
64
|
+
(?:\([^)]*\)|[A-Z]+(?:\s*\([^)]*\))?)\s*
|
|
65
|
+
(?:\#(?:REQUIRED|IMPLIED)|(?:\#FIXED\s+)?(?:"(?<dq>[^"]*)"|'(?<sq>[^']*)'))
|
|
66
|
+
/x
|
|
67
|
+
|
|
68
|
+
private_constant :XML_NAME, :DECLARATION
|
|
69
|
+
|
|
70
|
+
class << self
|
|
71
|
+
# XML accumulates ATTLIST declarations and the FIRST declaration of
|
|
72
|
+
# an attribute binds, so a later one neither replaces the element's
|
|
73
|
+
# earlier defaults nor overrides a duplicate.
|
|
74
|
+
#
|
|
75
|
+
# Duplicates inside ONE declaration need the raw source: REXML
|
|
76
|
+
# collapses them last-wins before handing over the parsed hash. The
|
|
77
|
+
# raw scan is the authority, and broader than "it only fixes the
|
|
78
|
+
# order". REXML DOES report a lone `#IMPLIED` as a nil, but it
|
|
79
|
+
# loses that tombstone to a duplicate later in the same
|
|
80
|
+
# declaration, and a tombstone has to outrank that later
|
|
81
|
+
# declaration. The scan also corrects a value REXML mis-reports,
|
|
82
|
+
# such as one separated from `#FIXED` by a tab, which arrives as
|
|
83
|
+
# `"#FIXED\t"`. REXML is the fallback, for attributes the regex
|
|
84
|
+
# could not read at all.
|
|
85
|
+
def collect(defaults, event, declarations)
|
|
86
|
+
element = event[0]
|
|
87
|
+
event[1].each { |name, value| declarations[name] = value unless declarations.key?(name) }
|
|
88
|
+
defaults[element] = declarations.merge(defaults[element] || {})
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Explicit attributes beat declared defaults, and only the final
|
|
92
|
+
# merged hash is compacted -- dropping tombstones earlier would let
|
|
93
|
+
# a later declaration resurrect an attribute XML left undeclared.
|
|
94
|
+
def for_root(event, defaults)
|
|
95
|
+
defaults.fetch(event[0], {}).merge(event[1]).compact
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# nil is a real answer here -- `#IMPLIED` means "declared, no
|
|
99
|
+
# default" -- so first-wins is decided by key?, never by the
|
|
100
|
+
# value's truthiness. RootParser also uses this exact scan to make
|
|
101
|
+
# canonical DTD-defaulted namespace prefixes visible to REXML's
|
|
102
|
+
# namespace validator before it reaches the root.
|
|
103
|
+
def declarations(raw)
|
|
104
|
+
body = raw.to_s.sub(/\A<!ATTLIST\s+\S+/, "").sub(/>\s*\z/, "")
|
|
105
|
+
body.scan(DECLARATION).each_with_object({}) do |(name, quoted, single), acc|
|
|
106
|
+
acc[name] = quoted || single unless acc.key?(name)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# The EPSF header field, scanned out of a PostScript file's first line.
|
|
113
|
+
# Its own module because the scan is a self-contained concern with three
|
|
114
|
+
# collaborating pieces, and Detector is about choosing between formats.
|
|
115
|
+
module EpsHeader
|
|
116
|
+
# Whitespace-delimited, per the DSC grammar: `EPSF` or `EPSF-<ver>`
|
|
117
|
+
# is its own header keyword. Excluding only word characters let
|
|
118
|
+
# ".EPSF", "EPSF.foo" and "(EPSF)" through. The version suffix is
|
|
119
|
+
# matched by the lookahead rather than consumed, so the longest match
|
|
120
|
+
# stays four bytes and the carry can be sized for it.
|
|
121
|
+
FIELD = /(?<!\S)EPSF(?=-|\s|\z)/
|
|
122
|
+
# One byte of left context plus the four-byte token.
|
|
123
|
+
CARRY_BYTES = 5
|
|
124
|
+
# PostScript ends a line with CR, LF or CRLF.
|
|
125
|
+
LINE_END = /[\r\n]/
|
|
126
|
+
CHUNK_BYTES = 4096
|
|
127
|
+
# A DSC header's first line is short in every real PostScript/EPS file.
|
|
128
|
+
# A first line that runs past this many windows is treated the same as
|
|
129
|
+
# one with no EPSF field at all, rather than scanned indefinitely --
|
|
130
|
+
# the same trade-off SVG_PROLOG_BYTES makes for the SVG probe, and for
|
|
131
|
+
# the same reason: worst-case cost has to be a fixed constant, not
|
|
132
|
+
# proportional to an attacker-chosen input.
|
|
133
|
+
MAX_WINDOWS = 3
|
|
134
|
+
# The most this module will ever read from a source. Public so
|
|
135
|
+
# Detector can size its own read bound to match.
|
|
136
|
+
MAX_SCAN_BYTES = MAX_WINDOWS * CHUNK_BYTES
|
|
137
|
+
|
|
138
|
+
private_constant :FIELD, :CARRY_BYTES, :LINE_END, :CHUNK_BYTES, :MAX_WINDOWS
|
|
139
|
+
|
|
140
|
+
class << self
|
|
141
|
+
# Streams the first line looking for the EPSF field. `gets` would
|
|
142
|
+
# materialise the whole line, and a newline-free %!PS file would
|
|
143
|
+
# allocate its full size -- the SVG probe never runs for
|
|
144
|
+
# PostScript, so nothing else absorbs that cost. PostScript ends a
|
|
145
|
+
# line with CR, LF or CRLF, so the scan stops at whichever comes
|
|
146
|
+
# first.
|
|
147
|
+
#
|
|
148
|
+
# The carry is sized for the *field* match, not a bare substring:
|
|
149
|
+
# the pattern inspects one byte either side of `EPSF`, so a token
|
|
150
|
+
# split across chunks needs its neighbours to travel with it.
|
|
151
|
+
def field?(source)
|
|
152
|
+
each_window(source) do |scan, final, at_end, carried|
|
|
153
|
+
return match?(scan, final: at_end, carried: carried) if final
|
|
154
|
+
return true if match?(scan, final: false, carried: carried)
|
|
155
|
+
end
|
|
156
|
+
false
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Yields the first line a window at a time, flagging the window
|
|
160
|
+
# that ends it. The carry is sized for the *field* match, not a
|
|
161
|
+
# bare substring: the pattern inspects one byte either side of
|
|
162
|
+
# `EPSF`, so a token split across chunks needs its neighbours to
|
|
163
|
+
# travel with it.
|
|
164
|
+
#
|
|
165
|
+
# `final` and `at_end` differ at the scan ceiling: reaching it
|
|
166
|
+
# stops the scan (`final`), but the byte after the ceiling was
|
|
167
|
+
# never read, so a match resting on that unseen byte has no
|
|
168
|
+
# evidence for it and `at_end` stays false. Only a genuine line
|
|
169
|
+
# ending or a genuine end of file (`at_end`) can back a match that
|
|
170
|
+
# ends exactly at the window's edge -- see `match?`.
|
|
171
|
+
def each_window(source)
|
|
172
|
+
carry = "".b
|
|
173
|
+
windows_read = 0
|
|
174
|
+
loop do
|
|
175
|
+
windows_read += 1
|
|
176
|
+
window, chunk, stop = next_window(source, carry)
|
|
177
|
+
at_end, final = window_status(stop, chunk, windows_read)
|
|
178
|
+
|
|
179
|
+
yield(stop ? window[0, stop] : window, final, at_end, carry.bytesize)
|
|
180
|
+
return if final
|
|
181
|
+
|
|
182
|
+
carry = window[-CARRY_BYTES..] || window
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# nil.to_s is "", so EOF needs no branch of its own here.
|
|
187
|
+
def next_window(source, carry)
|
|
188
|
+
chunk = source.read(CHUNK_BYTES)
|
|
189
|
+
window = carry + chunk.to_s.b
|
|
190
|
+
[window, chunk, window.index(LINE_END)]
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# `read(CHUNK_BYTES)` only ever returns fewer bytes than asked for
|
|
194
|
+
# at genuine end of stream -- a short read is as reliable an EOF
|
|
195
|
+
# signal as a nil one, and without it a file ending mid-window on
|
|
196
|
+
# its final, partial chunk was mistaken for one cut off by the
|
|
197
|
+
# ceiling instead.
|
|
198
|
+
def window_status(stop, chunk, windows_read)
|
|
199
|
+
at_end = !stop.nil? || chunk.nil? || chunk.bytesize < CHUNK_BYTES
|
|
200
|
+
[at_end, at_end || windows_read >= MAX_WINDOWS]
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# A match ending exactly at the buffer's edge had its trailing
|
|
204
|
+
# delimiter supplied by end-of-string rather than by the file, so
|
|
205
|
+
# unless this really is the end of the line it is deferred to the
|
|
206
|
+
# next window, where the following byte is known.
|
|
207
|
+
def match?(scan, final:, carried: 0)
|
|
208
|
+
offset = 0
|
|
209
|
+
|
|
210
|
+
while (found = FIELD.match(scan, offset))
|
|
211
|
+
offset = found.end(0)
|
|
212
|
+
# A match starting inside the carried bytes has already been
|
|
213
|
+
# judged, in the window where its real left-hand neighbour was
|
|
214
|
+
# visible. Reconsidering it here lets the buffer edge stand in
|
|
215
|
+
# for that neighbour, which turned "%!PS XEPSF " into an EPS.
|
|
216
|
+
#
|
|
217
|
+
# Rejecting it must not end the search, which is what an
|
|
218
|
+
# earlier version did by examining only the first match: a
|
|
219
|
+
# decoy straddling the boundary then hid a real `EPSF` later in
|
|
220
|
+
# the same window, and the file read as :ps.
|
|
221
|
+
next if found.begin(0).zero? && carried.positive?
|
|
222
|
+
|
|
223
|
+
return true if final || found.end(0) < scan.bytesize
|
|
224
|
+
|
|
225
|
+
# This one ends exactly at the edge, so its trailing delimiter
|
|
226
|
+
# came from the buffer rather than the file. Nothing after it
|
|
227
|
+
# can end sooner, so the whole window defers.
|
|
228
|
+
return false
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
false
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# The shared leaf of attribute-value resolution. Extracted because its
|
|
237
|
+
# callers want different things. `Detector.resolve` validates the result
|
|
238
|
+
# and falls back to its already-normalized input. `svg_root?` and
|
|
239
|
+
# `reserved_prefix_declared?` need the leaf raw instead: a nil from an
|
|
240
|
+
# out-of-range reference has to reach `ReservedNamespace.violated_by?` so
|
|
241
|
+
# it fails closed, and detection has to see that same failure rather than
|
|
242
|
+
# read the raw text as if it had resolved to itself. One entry point
|
|
243
|
+
# cannot serve both without hiding that.
|
|
244
|
+
module AttributeReferences
|
|
245
|
+
# The five entities XML itself defines -- measured: `unnormalize`
|
|
246
|
+
# expands exactly these (plus numeric references) and leaves every
|
|
247
|
+
# other named reference, of any case, byte-for-byte untouched, since
|
|
248
|
+
# it has no entity table beyond them. Checked against the RAW value,
|
|
249
|
+
# never a resolved one: raw "&foo;b=2" resolves to "&foo;b=2",
|
|
250
|
+
# an escaped ampersand in a query string followed by ordinary text,
|
|
251
|
+
# and once resolution has run that is indistinguishable from a
|
|
252
|
+
# genuinely unresolved "&foo;". Deliberately not "&" as the
|
|
253
|
+
# example -- the lookahead already excludes those five, so they read
|
|
254
|
+
# the same whichever value is checked and prove nothing.
|
|
255
|
+
#
|
|
256
|
+
# Use REXML's canonical XML Name production rather than maintaining a
|
|
257
|
+
# second transcription that can drift from the parser adapter below.
|
|
258
|
+
UNRESOLVED_REFERENCE = /&(?!amp;|lt;|gt;|apos;|quot;)#{REXML::XMLTokens::NAME};/
|
|
259
|
+
|
|
260
|
+
module_function
|
|
261
|
+
|
|
262
|
+
def resolve(value)
|
|
263
|
+
REXML::Text.unnormalize(value, entity_expansion_text_limit: value.bytesize)
|
|
264
|
+
rescue RangeError
|
|
265
|
+
nil
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# True for a raw value `resolve` cannot fully account for: a named
|
|
269
|
+
# reference it has no table for, left exactly as written rather than
|
|
270
|
+
# expanded. Unproven, not provably safe -- Claricle never fetches a
|
|
271
|
+
# DTD-declared entity, so a reserved namespace name smuggled in
|
|
272
|
+
# through one would sail past a check that only compared bytes.
|
|
273
|
+
#
|
|
274
|
+
# The encoding check makes the predicate total, which it stopped
|
|
275
|
+
# being when the pattern gained non-ASCII ranges: `match?` RAISES
|
|
276
|
+
# rather than answers on a subject that is neither ASCII-only nor
|
|
277
|
+
# valid UTF-8 (measured: Encoding::CompatibilityError for BINARY
|
|
278
|
+
# carrying a high byte, ArgumentError for invalid UTF-8). So a
|
|
279
|
+
# subject it cannot read reads as unresolved -- unproven, which is
|
|
280
|
+
# this module's whole meaning. Measured unreachable from `detect`
|
|
281
|
+
# today: every value REXML hands over arrives valid UTF-8, across
|
|
282
|
+
# every declared encoding and BOM tried.
|
|
283
|
+
def unresolved?(raw)
|
|
284
|
+
return true unless raw.ascii_only? ||
|
|
285
|
+
(raw.encoding == Encoding::UTF_8 && raw.valid_encoding?)
|
|
286
|
+
|
|
287
|
+
raw.match?(UNRESOLVED_REFERENCE)
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Whether a value bound to a namespace-declaration attribute name
|
|
292
|
+
# violates the two prefixes and two namespace names XML reserves for
|
|
293
|
+
# itself. Its own module because the rule is a self-contained set of
|
|
294
|
+
# constraints, unrelated to how Detector chooses between formats or
|
|
295
|
+
# resolves a value's references.
|
|
296
|
+
module ReservedNamespace
|
|
297
|
+
# XML permanently binds `xml` to this namespace; no other prefix, or
|
|
298
|
+
# the default namespace, may be bound to it either.
|
|
299
|
+
XML = "http://www.w3.org/XML/1998/namespace"
|
|
300
|
+
# Reserved for the `xmlns` prefix's own use; must never be bound to
|
|
301
|
+
# any prefix, including `xmlns` itself.
|
|
302
|
+
XMLNS = "http://www.w3.org/2000/xmlns/"
|
|
303
|
+
|
|
304
|
+
module_function
|
|
305
|
+
|
|
306
|
+
# `name` is the raw attribute name ("xmlns" or "xmlns:prefix");
|
|
307
|
+
# `bound` is its resolved value, or nil for an out-of-range
|
|
308
|
+
# reference. The caller has already refused an unresolved general
|
|
309
|
+
# entity reference before calling this -- see
|
|
310
|
+
# `AttributeReferences.unresolved?` -- so `bound` here is always
|
|
311
|
+
# either nil or fully accounted for. A lone surrogate half resolves
|
|
312
|
+
# to invalid UTF-8 without raising, so it reaches here as a string,
|
|
313
|
+
# not nil; checked before `==`, because a comparison against invalid
|
|
314
|
+
# encoding is exactly as unproven as one that raised outright.
|
|
315
|
+
def violated_by?(name, bound)
|
|
316
|
+
return true if bound.nil? || !bound.valid_encoding?
|
|
317
|
+
return true if bound == XMLNS
|
|
318
|
+
|
|
319
|
+
(name == "xmlns:xml") != (bound == XML)
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
module Detector
|
|
324
|
+
PNG_SIGNATURE = "\x89PNG\r\n\x1A\n".b.freeze
|
|
325
|
+
PDF_SIGNATURE = "%PDF-".b.freeze
|
|
326
|
+
POSTSCRIPT_SIGNATURE = "%!PS".b.freeze
|
|
327
|
+
SVG_NAMESPACE = "http://www.w3.org/2000/svg"
|
|
328
|
+
|
|
329
|
+
# The signature probes need 44 bytes: Emf::Detector reads the EMF magic
|
|
330
|
+
# as a uint32 at offset 0x28, and below that every valid EMF raises
|
|
331
|
+
# FormatError and falls through to UnknownFormat. 512 is deliberate
|
|
332
|
+
# slack over that floor -- no probe's correctness depends on it.
|
|
333
|
+
HEADER_BYTES = 512
|
|
334
|
+
# The SVG probe stops at the first start element, but REXML holds one
|
|
335
|
+
# live string for the construct it is reading -- so a file beginning
|
|
336
|
+
# "<" with no ">" cost +52MB RSS for 1MB of input, and +217MB for
|
|
337
|
+
# 16MB. Bounding the prefix makes the worst case fixed instead of
|
|
338
|
+
# proportional. A conventional XML declaration plus a full SVG 1.1
|
|
339
|
+
# DOCTYPE is about 140 bytes, so 8192 is generous; XML permits an
|
|
340
|
+
# arbitrarily long prolog, so this is a sniffing limit, not a
|
|
341
|
+
# complete one.
|
|
342
|
+
SVG_PROLOG_BYTES = 8192
|
|
343
|
+
# The most any single probe below will ever read from a source. Public
|
|
344
|
+
# so Claricle.detect can size its own bounded IO read to match --
|
|
345
|
+
# nothing downstream needs more than this, on any entry point.
|
|
346
|
+
MAX_PROBE_BYTES = [HEADER_BYTES, SVG_PROLOG_BYTES, EpsHeader::MAX_SCAN_BYTES].max
|
|
347
|
+
|
|
348
|
+
class << self
|
|
349
|
+
def detect(bytes)
|
|
350
|
+
content = ::String.new(bytes).force_encoding(Encoding::BINARY)
|
|
351
|
+
classify(content[0, HEADER_BYTES]) { StringIO.new(content) }
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
def detect_path(path)
|
|
355
|
+
File.open(path, "rb") do |io|
|
|
356
|
+
header = (io.read(HEADER_BYTES) || "").b
|
|
357
|
+
classify(header) do
|
|
358
|
+
io.rewind
|
|
359
|
+
io
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
# The root element's qname and its attributes, or nil. The qname
|
|
365
|
+
# comes back raw: a reference is not permitted in a name, so there
|
|
366
|
+
# is nothing there to resolve. Each attribute is resolved once,
|
|
367
|
+
# falling back to its normalized-but-unresolved text when
|
|
368
|
+
# resolution cannot answer -- the right default for something
|
|
369
|
+
# about to be reported as metadata, not decided on. nil covers
|
|
370
|
+
# every way the root can be unavailable: no root inside the
|
|
371
|
+
# bound, markup REXML refuses, and an encoding name it cannot
|
|
372
|
+
# use. Callers treat all three the same -- there is nothing to
|
|
373
|
+
# report about -- so they are not distinguished here. Public
|
|
374
|
+
# because Handlers::Svg needs exactly this and a second reader
|
|
375
|
+
# would have to reimplement the bound and the ATTLIST precedence
|
|
376
|
+
# -- rules this file spent four review rounds getting right --
|
|
377
|
+
# and then agree with this one. `svg?` below shares that same
|
|
378
|
+
# parsing through `root_event`, but resolves its own xmlns-bearing
|
|
379
|
+
# attributes bare, with no fallback: detection has to fail closed
|
|
380
|
+
# when a reference can't be resolved, where this method's fallback
|
|
381
|
+
# would hide that. `Detector` is itself a private constant, so
|
|
382
|
+
# nothing about the gem's documented surface changes.
|
|
383
|
+
def read_root(source)
|
|
384
|
+
found = root_event(source)
|
|
385
|
+
return nil unless found
|
|
386
|
+
|
|
387
|
+
# Resolution sits OUTSIDE the rescue below on purpose. That
|
|
388
|
+
# rescue exists for the parser's own failures -- including the
|
|
389
|
+
# bare ArgumentError REXML raises for an unusable encoding name
|
|
390
|
+
# -- and an ArgumentError or RuntimeError raised while resolving
|
|
391
|
+
# references is a different fault that must not be reported as
|
|
392
|
+
# "this file has no root".
|
|
393
|
+
[found.first, resolved(found.last)]
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
# RootSource's grammar patch for a caller needing a full-document
|
|
397
|
+
# BaseParser, not just `read_root`'s bounded one.
|
|
398
|
+
def canonical_source(content) = RootSource.for(content)
|
|
399
|
+
|
|
400
|
+
private
|
|
401
|
+
|
|
402
|
+
# The rescue wraps the parser alone, for the same reason resolution
|
|
403
|
+
# sits outside it in read_root: an ArgumentError raised while
|
|
404
|
+
# assembling attribute defaults is a fault in this file, and
|
|
405
|
+
# reporting it as "this file has no root" would hide it behind an
|
|
406
|
+
# unknown format.
|
|
407
|
+
def root_event(source)
|
|
408
|
+
parser = RootParser.new(bounded(source))
|
|
409
|
+
defaults = {}
|
|
410
|
+
while (event = next_event(parser))
|
|
411
|
+
parser.collect_defaults(defaults, event) if event.event_type == :attlistdecl
|
|
412
|
+
return [event[0], AttributeDefaults.for_root(event, defaults)] if event.start_element?
|
|
413
|
+
end
|
|
414
|
+
nil
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
# nil for end of input and for a document the parser refuses,
|
|
418
|
+
# which callers already treat the same way. The bare ArgumentError
|
|
419
|
+
# is REXML's answer to an unusable encoding name; every other
|
|
420
|
+
# ArgumentError belongs to whoever raised it.
|
|
421
|
+
def next_event(parser)
|
|
422
|
+
REXML::Parsers::PullEvent.new(parser.pull) if parser.has_next?
|
|
423
|
+
rescue REXML::ParseException, ArgumentError
|
|
424
|
+
parser.propagate_adapter_error
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
# Every root attribute, not just xmlns: the PullParser hands values
|
|
428
|
+
# back unexpanded, and `width="10"` means ten, not zero.
|
|
429
|
+
#
|
|
430
|
+
# A reference that cannot be resolved keeps its raw declaration.
|
|
431
|
+
# Two ways that happens, both measured: an out-of-range numeric
|
|
432
|
+
# reference raises RangeError, and a surrogate such as `�`
|
|
433
|
+
# resolves to INVALID UTF-8, which is worse -- it does not raise
|
|
434
|
+
# here, it detonates later, either in the dimension parse or in
|
|
435
|
+
# `to_json`. The raw form is always valid and always truthful.
|
|
436
|
+
def resolved(attributes)
|
|
437
|
+
attributes.transform_values { |value| resolve(normalized(value)) }
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
def resolve(value) = usable(AttributeReferences.resolve(value)) || value
|
|
441
|
+
|
|
442
|
+
# XML 1.0 3.3.3, and the reason it has to happen BEFORE references
|
|
443
|
+
# are expanded: in a CDATA attribute a literal tab, CR or newline
|
|
444
|
+
# is replaced by a space, while a character reference to one is
|
|
445
|
+
# kept as that character. So `id="a<LF>b"` is "a b" and
|
|
446
|
+
# `id="a
b"` is "a\nb", and both arrive here as raw text --
|
|
447
|
+
# without this they would be reported identically.
|
|
448
|
+
#
|
|
449
|
+
# CRLF is one line ending and becomes ONE space (XML 2.11 runs
|
|
450
|
+
# first). Doing it here rather than leaving it to REXML is
|
|
451
|
+
# measured, not assumed: REXML folds a literal CR to LF, but only
|
|
452
|
+
# later, inside reference resolution -- too late to be normalized
|
|
453
|
+
# -- and its own DOM never normalizes attribute values at all.
|
|
454
|
+
#
|
|
455
|
+
# The CDATA rule and only that rule. XML also trims and collapses
|
|
456
|
+
# spaces for a NON-CDATA attribute, and this deliberately does
|
|
457
|
+
# not: the type comes from the DTD, and an external subset is the
|
|
458
|
+
# normal way to declare one -- SVG's own DTD is external -- which
|
|
459
|
+
# a bounded reader can neither fetch nor should. Implementing the
|
|
460
|
+
# internal-subset half would make the value right for a locally
|
|
461
|
+
# declared NMTOKENS and wrong for the same attribute declared
|
|
462
|
+
# externally, with nothing to tell the two apart. One uniform
|
|
463
|
+
# rule is the honest answer. Measured: REXML's DOM does not
|
|
464
|
+
# collapse an internally declared NMTOKENS either.
|
|
465
|
+
def normalized(value)
|
|
466
|
+
value.gsub(/\r\n|[\t\r\n]/, " ")
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
def usable(value)
|
|
470
|
+
value if value&.valid_encoding?
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
# The block yields the source rewound to byte 0. The PNG, PDF and
|
|
474
|
+
# metafile probes match at fixed offsets and read `header`; the
|
|
475
|
+
# PostScript and SVG probes need more than a header, so they pull
|
|
476
|
+
# from the source themselves.
|
|
477
|
+
def classify(header)
|
|
478
|
+
return :png if header.start_with?(PNG_SIGNATURE)
|
|
479
|
+
return :pdf if header.start_with?(PDF_SIGNATURE)
|
|
480
|
+
return postscript_flavour(yield) if header.start_with?(POSTSCRIPT_SIGNATURE)
|
|
481
|
+
|
|
482
|
+
format = EpsBinary.wrapped?(header) ? :eps : metafile_format(header)
|
|
483
|
+
return format if format
|
|
484
|
+
return :svg if svg?(yield)
|
|
485
|
+
|
|
486
|
+
declared_or_unknown(header)
|
|
487
|
+
end
|
|
488
|
+
|
|
489
|
+
def postscript_flavour(source)
|
|
490
|
+
EpsHeader.field?(source) ? :eps : :ps
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
def metafile_format(header)
|
|
494
|
+
::Emf.detect_format(header)
|
|
495
|
+
rescue ::Emf::FormatError
|
|
496
|
+
nil
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
# An unusable encoding name in the XML declaration reaches us as a bare
|
|
500
|
+
# ArgumentError from REXML::Encoding#encoding=, not as a ParseException.
|
|
501
|
+
# It means "not SVG" like any other parse failure; letting it escape
|
|
502
|
+
# would turn a malformed file into an internal error rather than
|
|
503
|
+
# UnknownFormat. Undecodable bytes already arrive as a ParseException.
|
|
504
|
+
def svg?(source)
|
|
505
|
+
found = root_event(source)
|
|
506
|
+
return false unless found
|
|
507
|
+
|
|
508
|
+
svg_root?(*found)
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
# Only the prolog and the root start tag can matter here, and the
|
|
512
|
+
# bound is where they stop mattering: a root that begins after
|
|
513
|
+
# SVG_PROLOG_BYTES is not found at all, and the document reads as
|
|
514
|
+
# an unknown format. That is the sniffing limit SVG_PROLOG_BYTES
|
|
515
|
+
# describes, deliberately, not an accident of sizing.
|
|
516
|
+
#
|
|
517
|
+
# Takes an IO or a String, because both callers hand over
|
|
518
|
+
# whichever the image is made of: an open file for a path-born
|
|
519
|
+
# source, the bytes already in hand for a content-born one.
|
|
520
|
+
def bounded(source)
|
|
521
|
+
prefix = if source.respond_to?(:read)
|
|
522
|
+
# Nothing to normalise on this side: `read(n)` answers
|
|
523
|
+
# in ASCII-8BIT whatever the source's own encoding is.
|
|
524
|
+
# Measured for File in r, rb, r:Shift_JIS and
|
|
525
|
+
# r:ISO-8859-1, and for StringIO over a UTF-16LE string.
|
|
526
|
+
source.read(SVG_PROLOG_BYTES)
|
|
527
|
+
else
|
|
528
|
+
# byteslice, not slice: the bound is BYTES, and
|
|
529
|
+
# `source[0, n]` counts characters, so a multibyte
|
|
530
|
+
# prolog would let the handler read further than
|
|
531
|
+
# detection did and the two would disagree.
|
|
532
|
+
#
|
|
533
|
+
# `.b` because byteslice keeps the String's encoding
|
|
534
|
+
# tag. It is not what protects handler inspection --
|
|
535
|
+
# `Image.from_content` binary-tags on the way in, so
|
|
536
|
+
# a handler's bytes arrive ASCII-8BIT already. It
|
|
537
|
+
# protects whoever calls `read_root` with a String of
|
|
538
|
+
# their own: the SAME ASCII bytes tagged UTF-16LE
|
|
539
|
+
# read back zero events without it, because REXML
|
|
540
|
+
# decoded ASCII as UTF-16 and the pull loop ended, so
|
|
541
|
+
# inspection said "failed" without raising.
|
|
542
|
+
source.byteslice(0, SVG_PROLOG_BYTES).b
|
|
543
|
+
end
|
|
544
|
+
# XML defaults to UTF-8 when neither a declaration nor a BOM says
|
|
545
|
+
# otherwise. Tag, do not transcode, the already-bounded copy; REXML
|
|
546
|
+
# still detects either marker and switches encodings itself. dup also
|
|
547
|
+
# keeps a frozen read result or the empty fallback safe to retag.
|
|
548
|
+
RootSource.for((prefix || "").dup.force_encoding(Encoding::UTF_8))
|
|
549
|
+
end
|
|
550
|
+
|
|
551
|
+
def svg_root?(qname, attributes)
|
|
552
|
+
prefix, local_name = qname.include?(":") ? qname.split(":", 2) : [nil, qname]
|
|
553
|
+
return false unless local_name == "svg"
|
|
554
|
+
return false if reserved_prefix_declared?(attributes)
|
|
555
|
+
|
|
556
|
+
declared = attributes[prefix ? "xmlns:#{prefix}" : "xmlns"]
|
|
557
|
+
return false unless declared
|
|
558
|
+
|
|
559
|
+
# Normalized and resolved here, fresh from the raw attribute
|
|
560
|
+
# `root_event` handed over -- never from an ALREADY-resolved
|
|
561
|
+
# value, and never `resolve`'s raw-on-failure fallback: deciding
|
|
562
|
+
# whether this document IS an SVG needs to know a reference
|
|
563
|
+
# failed, not read its raw text as if it had resolved to itself.
|
|
564
|
+
# `reserved_prefix_declared?` below resolves this same raw text
|
|
565
|
+
# again, independently, for its own guard -- two resolutions of
|
|
566
|
+
# the SAME raw input agree and cost nothing worth threading a
|
|
567
|
+
# value through two methods to avoid. What must never happen is
|
|
568
|
+
# CASCADING, resolving one method's already-resolved output a
|
|
569
|
+
# second time: `&#x67;` means the literal text `g`, and
|
|
570
|
+
# a second pass on THAT output turns it into `g`.
|
|
571
|
+
AttributeReferences.resolve(normalized(declared)) == SVG_NAMESPACE
|
|
572
|
+
end
|
|
573
|
+
|
|
574
|
+
# A constraint on the root's own namespace declarations, explicit
|
|
575
|
+
# or DTD-defaulted, regardless of which prefix the root itself
|
|
576
|
+
# uses. Each value is resolved fresh from the raw attribute
|
|
577
|
+
# `root_event` handed over, never from an already-resolved one --
|
|
578
|
+
# delegated to ReservedNamespace once this method has picked out
|
|
579
|
+
# which merged attributes are namespace declarations at all,
|
|
580
|
+
# refused one still carrying an unresolved reference, and resolved
|
|
581
|
+
# the rest.
|
|
582
|
+
def reserved_prefix_declared?(attributes)
|
|
583
|
+
attributes.any? do |name, value|
|
|
584
|
+
next true if name == "xmlns:xmlns"
|
|
585
|
+
next false unless name == "xmlns" || name.start_with?("xmlns:")
|
|
586
|
+
next true if AttributeReferences.unresolved?(value)
|
|
587
|
+
|
|
588
|
+
ReservedNamespace.violated_by?(name, AttributeReferences.resolve(normalized(value)))
|
|
589
|
+
end
|
|
590
|
+
end
|
|
591
|
+
end
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
module Detector
|
|
595
|
+
# REXML publishes XML's canonical name grammar in XMLTokens, but its
|
|
596
|
+
# BaseParser's live root, attribute, prolog and internal-subset scans use
|
|
597
|
+
# older, narrower productions. Keep the workaround on this one bounded
|
|
598
|
+
# source rather than changing REXML's process-wide constants. Rewriting
|
|
599
|
+
# the grammar fragments, rather than enumerating current regex objects,
|
|
600
|
+
# also reaches NAME copies embedded in ATTLIST and ENTITY patterns while
|
|
601
|
+
# preserving every surrounding capture exactly.
|
|
602
|
+
class RootSource < REXML::IOSource
|
|
603
|
+
base = REXML::Parsers::BaseParser
|
|
604
|
+
tokens = REXML::XMLTokens
|
|
605
|
+
SUBSTITUTIONS = [
|
|
606
|
+
[base::NCNAME_STR.dup.freeze, tokens::NCNAME_STR.dup.freeze].freeze,
|
|
607
|
+
[base::NAME.dup.freeze, tokens::NAME.dup.freeze].freeze,
|
|
608
|
+
[base::NMTOKEN.dup.freeze, tokens::NMTOKEN.dup.freeze].freeze
|
|
609
|
+
].freeze
|
|
610
|
+
|
|
611
|
+
private_constant :SUBSTITUTIONS
|
|
612
|
+
|
|
613
|
+
attr_reader :adapter_error
|
|
614
|
+
|
|
615
|
+
def self.for(content)
|
|
616
|
+
content.ascii_only? ? StringIO.new(content) : new(content)
|
|
617
|
+
end
|
|
618
|
+
|
|
619
|
+
def initialize(content)
|
|
620
|
+
super(StringIO.new(content))
|
|
621
|
+
end
|
|
622
|
+
|
|
623
|
+
# Positional `consume` matches the REXML method BaseParser calls.
|
|
624
|
+
def match(pattern, consume = false) # rubocop:disable Style/OptionalBooleanParameter
|
|
625
|
+
pattern = canonical(pattern)
|
|
626
|
+
# The first argument was deliberately replaced, so zsuper would send
|
|
627
|
+
# the original regex and silently undo the adapter.
|
|
628
|
+
super(pattern, consume) # rubocop:disable Style/SuperArguments
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
private
|
|
632
|
+
|
|
633
|
+
def canonical(pattern)
|
|
634
|
+
return pattern unless pattern.is_a?(Regexp)
|
|
635
|
+
|
|
636
|
+
@canonical_patterns ||= {}.compare_by_identity
|
|
637
|
+
@canonical_patterns[pattern] ||= rewritten(pattern)
|
|
638
|
+
rescue StandardError => e
|
|
639
|
+
@adapter_error = e
|
|
640
|
+
raise
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
def rewritten(pattern)
|
|
644
|
+
source = SUBSTITUTIONS.reduce(pattern.source) do |candidate, (narrow, canonical)|
|
|
645
|
+
candidate.gsub(narrow) { canonical }
|
|
646
|
+
end
|
|
647
|
+
return pattern if source == pattern.source
|
|
648
|
+
|
|
649
|
+
Regexp.new(source, pattern.options)
|
|
650
|
+
end
|
|
651
|
+
end
|
|
652
|
+
|
|
653
|
+
# BaseParser validates a root prefix against namespaces it learned while
|
|
654
|
+
# parsing. Its direct String#scan of an ATTLIST body cannot be adapted by
|
|
655
|
+
# RootSource, so register canonical DTD-defaulted namespace declarations
|
|
656
|
+
# from the same scan Detector later uses to merge attribute defaults.
|
|
657
|
+
class RootParser < REXML::Parsers::BaseParser
|
|
658
|
+
def propagate_adapter_error
|
|
659
|
+
error = source.adapter_error if source.is_a?(RootSource)
|
|
660
|
+
raise error if error
|
|
661
|
+
end
|
|
662
|
+
|
|
663
|
+
# Called between pulls so Claricle-owned failures remain outside the
|
|
664
|
+
# parser-only rescue in Detector.next_event.
|
|
665
|
+
def collect_defaults(defaults, event)
|
|
666
|
+
declarations = AttributeDefaults.declarations(event[2])
|
|
667
|
+
register_declared_namespaces(declarations)
|
|
668
|
+
AttributeDefaults.collect(defaults, event, declarations)
|
|
669
|
+
end
|
|
670
|
+
|
|
671
|
+
private
|
|
672
|
+
|
|
673
|
+
def register_declared_namespaces(declarations)
|
|
674
|
+
declarations.each do |name, value|
|
|
675
|
+
next unless value && name.start_with?("xmlns:")
|
|
676
|
+
|
|
677
|
+
prefix = name.delete_prefix("xmlns:")
|
|
678
|
+
@namespaces[prefix] = value unless @namespaces.key?(prefix)
|
|
679
|
+
end
|
|
680
|
+
end
|
|
681
|
+
end
|
|
682
|
+
|
|
683
|
+
# Handlers declare their own probe (`Handlers::Base.detect`); it runs
|
|
684
|
+
# last, so a new format cannot change what a built-in one detects. The
|
|
685
|
+
# registry is absent when this file is loaded on its own, and then
|
|
686
|
+
# there are no handlers to ask.
|
|
687
|
+
def self.declared_or_unknown(header)
|
|
688
|
+
(Registry.detect(header) if defined?(Registry)) || raise(UnknownFormat, "no known image signature")
|
|
689
|
+
end
|
|
690
|
+
|
|
691
|
+
private_constant :RootSource, :RootParser
|
|
692
|
+
end
|
|
693
|
+
|
|
694
|
+
private_constant :Detector, :EpsHeader, :ReservedNamespace,
|
|
695
|
+
:AttributeDefaults, :AttributeReferences
|
|
696
|
+
end
|