@broberg/lens-daemon 0.0.0-stage → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/package.json +52 -3
- package/src/artifact-backfill.ts +239 -0
- package/src/artifact-uploads.ts +397 -0
- package/src/auth/index.ts +301 -0
- package/src/browser.ts +440 -0
- package/src/build-marker.ts +197 -0
- package/src/capture-session.ts +873 -0
- package/src/capture.ts +348 -0
- package/src/critic-dom.ts +951 -0
- package/src/critic-vision.ts +205 -0
- package/src/critic.ts +73 -0
- package/src/daemon-routes.ts +333 -0
- package/src/durability.ts +18 -0
- package/src/ensure-chromium.ts +118 -0
- package/src/export.ts +256 -0
- package/src/fill.ts +149 -0
- package/src/flow-store.ts +207 -0
- package/src/flow.ts +2953 -0
- package/src/frames.ts +195 -0
- package/src/gate.ts +123 -0
- package/src/http.ts +175 -0
- package/src/index.ts +24 -0
- package/src/manifest.ts +741 -0
- package/src/mcp.ts +562 -0
- package/src/motion-probe.ts +653 -0
- package/src/password-in-url.ts +106 -0
- package/src/pdf-text.ts +514 -0
- package/src/pixelmatch.d.ts +21 -0
- package/src/preflight.ts +76 -0
- package/src/readiness.ts +282 -0
- package/src/record-run.ts +427 -0
- package/src/record.ts +387 -0
- package/src/sandbox.ts +22 -0
- package/src/schema.ts +300 -0
- package/src/scrape.ts +116 -0
- package/src/secret-fill.ts +80 -0
- package/src/seed-storage.ts +149 -0
- package/src/service.ts +1493 -0
- package/src/storage.ts +159 -0
- package/src/store.ts +267 -0
- package/src/swallowed-errors.ts +252 -0
- package/src/validate-vision.ts +97 -0
- package/src/verify.ts +411 -0
- package/src/video.ts +211 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// F074.99 part 2 — a password in the final URL means the page was never hydrated.
|
|
2
|
+
//
|
|
3
|
+
// fd-sundhed drove /reset-password and got 12 of 12 GREEN steps while the
|
|
4
|
+
// screenshot showed both fields EMPTY. `fill` had done its job perfectly: the
|
|
5
|
+
// value went into the DOM and was read back. There was simply no application
|
|
6
|
+
// listening — Next 16 blocks cross-origin dev resources, so served as
|
|
7
|
+
// 127.0.0.1 (rather than localhost) the client bundle never arrives. The form
|
|
8
|
+
// then did a NATIVE GET:
|
|
9
|
+
//
|
|
10
|
+
// final_url = …/reset-password?password=Abcdefg1%21&confirmPassword=Abcdefg1%21
|
|
11
|
+
//
|
|
12
|
+
// A hydrated page would have submitted via JS. So the values ending up in the
|
|
13
|
+
// query string is proof, after the fact, that nothing was running — and we
|
|
14
|
+
// already return final_url, so it costs no probe, no assert and no browser work.
|
|
15
|
+
//
|
|
16
|
+
// ────────────────────────────────────────────────────────────────────────────
|
|
17
|
+
// THE RULE IS fd-sundhed's, AND IT REPLACED MINE. I proposed matching query KEYS
|
|
18
|
+
// against the names of the fields the run filled, having already rejected the
|
|
19
|
+
// cruder "there is a query string". They measured mine wrong in their own app:
|
|
20
|
+
// `q` is a search parameter in FIFTEEN places, and a GET search form typically
|
|
21
|
+
// has <input name="q">, so a manuscript that fills a search box and presses
|
|
22
|
+
// search lands on ?q=… with an EXACT name match — on a fully hydrated page doing
|
|
23
|
+
// precisely its job. A GET search form is the commonest case in the world of "a
|
|
24
|
+
// submit became a navigation carrying the field's name", and it is CORRECT.
|
|
25
|
+
// Name-matching does not escape the false positive; it walks into it.
|
|
26
|
+
//
|
|
27
|
+
// SO: only a PASSWORD value. A password in a query string is never legitimate —
|
|
28
|
+
// it lands in server logs, in browser history and in the Referer header. Both
|
|
29
|
+
// readings therefore deserve the warning:
|
|
30
|
+
//
|
|
31
|
+
// not hydrated → we caught exactly this bug
|
|
32
|
+
// hydrated → the app leaks passwords in URLs, which is WORSE than the
|
|
33
|
+
// thing we were looking for
|
|
34
|
+
//
|
|
35
|
+
// There is no third reading in which it is noise. That property is why it will
|
|
36
|
+
// still be believed in six months, and it is worth more than the coverage the
|
|
37
|
+
// broad rule would have bought.
|
|
38
|
+
//
|
|
39
|
+
// THE BROAD VERSION IS DEFERRED, NOT REJECTED: it needs a discriminator between
|
|
40
|
+
// "GET because the form meant to" and "GET because nobody caught it". Neither we
|
|
41
|
+
// nor fd-sundhed have one, and they said so rather than proposing something
|
|
42
|
+
// unmeasured.
|
|
43
|
+
|
|
44
|
+
/** A value this run typed into an input whose type is `password`.
|
|
45
|
+
*
|
|
46
|
+
* The value is held only to compare against the URL and never leaves this
|
|
47
|
+
* process: the finding names the ANCHOR and the PARAMETER, never the secret. */
|
|
48
|
+
export interface FilledPassword {
|
|
49
|
+
/** How the step addressed the field — a testid or selector, for the report. */
|
|
50
|
+
anchor: string;
|
|
51
|
+
value: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface PasswordInUrlFinding {
|
|
55
|
+
/** The query parameter the password turned up in. */
|
|
56
|
+
param: string;
|
|
57
|
+
/** The field it was typed into. */
|
|
58
|
+
anchor: string;
|
|
59
|
+
/** Ready to print. Deliberately carries no value. */
|
|
60
|
+
message: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function passwordInUrlMessage(param: string, anchor: string): string {
|
|
64
|
+
return (
|
|
65
|
+
`A PASSWORD typed into ${anchor} came back in the final URL as "?${param}=…". ` +
|
|
66
|
+
'Two readings, and both are worth acting on: the page was never hydrated, so the form ' +
|
|
67
|
+
'fell back to a native GET and nothing you clicked was handled by the app (every step ' +
|
|
68
|
+
'can still be green) — or the page WAS hydrated and this app puts passwords in URLs, ' +
|
|
69
|
+
'where they reach server logs, browser history and the Referer header. ' +
|
|
70
|
+
'Check whether the app is served on the same hostname the client bundle is allowed to load from.'
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Did a password this run typed end up in the final URL's query string?
|
|
76
|
+
*
|
|
77
|
+
* Returns the FIRST match — one finding is enough to act on, and listing every
|
|
78
|
+
* field would put the same secret's shape in the report several times.
|
|
79
|
+
*
|
|
80
|
+
* A malformed or absent url yields null rather than throwing: this runs while
|
|
81
|
+
* assembling a result that is already complete, and a crash here would destroy
|
|
82
|
+
* a report to deliver a warning.
|
|
83
|
+
*/
|
|
84
|
+
export function passwordInUrl(
|
|
85
|
+
finalUrl: string | null | undefined,
|
|
86
|
+
filled: readonly FilledPassword[],
|
|
87
|
+
): PasswordInUrlFinding | null {
|
|
88
|
+
if (!finalUrl || filled.length === 0) return null;
|
|
89
|
+
let params: URLSearchParams;
|
|
90
|
+
try {
|
|
91
|
+
params = new URL(finalUrl).searchParams;
|
|
92
|
+
} catch {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
for (const [param, got] of params.entries()) {
|
|
96
|
+
for (const f of filled) {
|
|
97
|
+
// Exact match only. A password is compared whole: a substring test would
|
|
98
|
+
// fire on a one-character password appearing inside an unrelated id, and
|
|
99
|
+
// an empty one would match everything — so empties are skipped outright.
|
|
100
|
+
if (f.value.length > 0 && got === f.value) {
|
|
101
|
+
return { param, anchor: f.anchor, message: passwordInUrlMessage(param, f.anchor) };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
package/src/pdf-text.ts
ADDED
|
@@ -0,0 +1,514 @@
|
|
|
1
|
+
// F074.63 — the measuring instrument for "is this PDF editable, or a picture?"
|
|
2
|
+
//
|
|
3
|
+
// This exists because the obvious assertion is worthless here. "It returned a
|
|
4
|
+
// PDF" is EXACTLY the green fd-sundhed's broken artifact already passes: their
|
|
5
|
+
// stitched bitmap is a perfectly valid PDF. So is its MIME type, its %PDF-
|
|
6
|
+
// header and its file size. Every cheap check passes on the very artifact this
|
|
7
|
+
// feature exists to replace. What separates the two is what Illustrator showed
|
|
8
|
+
// them when they opened it: ONE <Linked File>, ZERO text objects.
|
|
9
|
+
//
|
|
10
|
+
// THE FIRST VERSION OF THIS FILE WAS WRONG IN BOTH DIRECTIONS, and only
|
|
11
|
+
// measuring it against a real Chromium PDF found that out:
|
|
12
|
+
//
|
|
13
|
+
// · it looked for `(literal) Tj`. Chromium writes `<hex> Tj` with subset
|
|
14
|
+
// fonts, so the real text runs matched NOTHING — a textless verdict on a
|
|
15
|
+
// perfectly good vector PDF, which is the false negative that would have
|
|
16
|
+
// made this whole feature look broken.
|
|
17
|
+
// · it sliced streams on the first `endstream` it found. Binary font data
|
|
18
|
+
// contains that word, so streams were cut short and the leftover bytes
|
|
19
|
+
// matched the operator regexes by accident: it reported 3 text runs on a
|
|
20
|
+
// document, and would have reported some on a bitmap too.
|
|
21
|
+
//
|
|
22
|
+
// Both were invisible until the numbers were compared with `pdftotext` on the
|
|
23
|
+
// same file. So this version does the real work: /Length-delimited streams, the
|
|
24
|
+
// font's own /ToUnicode CMap, and actual STRINGS out the other end — which is
|
|
25
|
+
// what the acceptance criterion asks for, and what a proxy count could never be.
|
|
26
|
+
//
|
|
27
|
+
// DEPENDENCY-FREE ON PURPOSE. poppler is on the machine this was written on and
|
|
28
|
+
// gives a stronger extraction — but the CI runner has no poppler, and a gate
|
|
29
|
+
// that only passes where a system package happens to exist is a gate that goes
|
|
30
|
+
// red for a reason unrelated to the code. node's own zlib is everywhere.
|
|
31
|
+
// poppler stays the manual cross-check for one-off proof on a real document.
|
|
32
|
+
import { inflateSync } from 'node:zlib';
|
|
33
|
+
|
|
34
|
+
export interface PdfContentStats {
|
|
35
|
+
/** Text-showing operators (Tj / TJ / ' / ") actually decoded. > 0 means real
|
|
36
|
+
* text objects — the thing a designer can edit. */
|
|
37
|
+
textRuns: number;
|
|
38
|
+
/** Embedded font resources. Text runs with no font would be suspicious. */
|
|
39
|
+
fonts: number;
|
|
40
|
+
/** Image XObjects. A "picture of a document" is images >= 1 with textRuns 0. */
|
|
41
|
+
images: number;
|
|
42
|
+
/** Streams we could not decode. Reported rather than swallowed: a count of 0
|
|
43
|
+
* text runs means something different when 40 streams were unreadable. */
|
|
44
|
+
undecodableStreams: number;
|
|
45
|
+
/** F074.65 — text-showing operators whose character codes we DROPPED, because
|
|
46
|
+
* the font they select resolves to no usable /ToUnicode CMap.
|
|
47
|
+
*
|
|
48
|
+
* Deliberately NOT folded into undecodableStreams. That field counts inflate
|
|
49
|
+
* failures, and merging the two is exactly how the previous version reported
|
|
50
|
+
* `0` while silently losing an entire line of the page: the stream inflated
|
|
51
|
+
* perfectly, so nothing was wrong by its measure, and a whole headline
|
|
52
|
+
* vanished. Two different unknowns reported as one number is indistinguishable
|
|
53
|
+
* from confidence. */
|
|
54
|
+
undecodedRuns: number;
|
|
55
|
+
/** F074.65 — font programs the document draws text with, EXCLUDING the CID
|
|
56
|
+
* descendants a composite font points at (those are the same typeface counted
|
|
57
|
+
* twice, and pdffonts does not list them either). */
|
|
58
|
+
fontPrograms: PdfFontProgram[];
|
|
59
|
+
/** The extracted text itself, runs joined by a space. This is the field an
|
|
60
|
+
* assertion should use — a COUNT can be right for the wrong reason, a string
|
|
61
|
+
* the page actually rendered cannot. */
|
|
62
|
+
text: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** F074.65 — one font the PDF draws with, and whether it is a real font program
|
|
66
|
+
* or a set of drawings.
|
|
67
|
+
*
|
|
68
|
+
* A Type 3 font is not a font in the sense a designer means: it is a dictionary
|
|
69
|
+
* of little content streams, one per glyph. It prints perfectly and it opens in
|
|
70
|
+
* Illustrator as PATHS — you can nudge the shapes and you can never re-type the
|
|
71
|
+
* word. Chromium falls back to it whenever it cannot subset the source face,
|
|
72
|
+
* which on macOS means every VARIABLE font, i.e. `system-ui` and `-apple-system`
|
|
73
|
+
* — and therefore Tailwind's default `--font-sans` stack. */
|
|
74
|
+
export interface PdfFontProgram {
|
|
75
|
+
/** /BaseFont, e.g. `AAAAAA+Georgia` or `DAAAAA+.SFNS-Regular`. */
|
|
76
|
+
name: string;
|
|
77
|
+
/** /Subtype: `Type0`, `TrueType`, `Type1`, `Type3`, … */
|
|
78
|
+
subtype: string;
|
|
79
|
+
/** True for Type 3 — glyphs stored as drawings, not as an editable face. */
|
|
80
|
+
outlines: boolean;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
interface PdfStream {
|
|
84
|
+
/** The object number, when the stream belongs to a numbered object. */
|
|
85
|
+
obj: number | null;
|
|
86
|
+
/** Inflated (or raw, when not Flate) payload as latin1. */
|
|
87
|
+
body: string;
|
|
88
|
+
/** The object's dictionary text, for finding /ToUnicode and friends. */
|
|
89
|
+
dict: string;
|
|
90
|
+
decoded: boolean;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Every stream in the file, delimited by its own /Length.
|
|
94
|
+
*
|
|
95
|
+
* Length-delimited, NOT `indexOf('endstream')`: an embedded TrueType font
|
|
96
|
+
* contains those bytes, and cutting there yields a truncated stream whose
|
|
97
|
+
* leftover binary matches operator regexes by accident. That is how the first
|
|
98
|
+
* version of this file counted text runs in a document that had none. */
|
|
99
|
+
function streams(bytes: Buffer): { list: PdfStream[]; undecodable: number } {
|
|
100
|
+
const hay = bytes.toString('latin1');
|
|
101
|
+
const list: PdfStream[] = [];
|
|
102
|
+
let undecodable = 0;
|
|
103
|
+
|
|
104
|
+
const objRe = /(\d+)\s+0\s+obj\b/g;
|
|
105
|
+
const objAt: Array<{ num: number; at: number }> = [];
|
|
106
|
+
let om: RegExpExecArray | null;
|
|
107
|
+
while ((om = objRe.exec(hay)) !== null) objAt.push({ num: Number(om[1]), at: om.index });
|
|
108
|
+
|
|
109
|
+
// The lookbehind is load-bearing: `endstream` ENDS in "stream", so a bare
|
|
110
|
+
// /stream\r?\n/ matches the close of every stream as if it opened a new one.
|
|
111
|
+
// Measured before the fix — a 5-stream PDF reported 5 phantom undecodable
|
|
112
|
+
// streams, which made the "how much could I not read?" field report noise as
|
|
113
|
+
// if it were coverage.
|
|
114
|
+
const streamRe = /(?<![A-Za-z])stream\r?\n/g;
|
|
115
|
+
let sm: RegExpExecArray | null;
|
|
116
|
+
while ((sm = streamRe.exec(hay)) !== null) {
|
|
117
|
+
const start = sm.index + sm[0].length;
|
|
118
|
+
// The dictionary is whatever sits between the enclosing `N 0 obj` and here.
|
|
119
|
+
let obj: number | null = null;
|
|
120
|
+
let dictFrom = Math.max(0, sm.index - 600);
|
|
121
|
+
for (let i = objAt.length - 1; i >= 0; i -= 1) {
|
|
122
|
+
if (objAt[i]!.at < sm.index) {
|
|
123
|
+
obj = objAt[i]!.num;
|
|
124
|
+
dictFrom = objAt[i]!.at;
|
|
125
|
+
break;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const dict = hay.slice(dictFrom, sm.index);
|
|
129
|
+
const len = Number(dict.match(/\/Length\s+(\d+)/)?.[1] ?? NaN);
|
|
130
|
+
// No /Length is a stream we cannot delimit safely. Count it as undecodable
|
|
131
|
+
// rather than guessing an end — a guess here fabricates content.
|
|
132
|
+
if (!Number.isFinite(len) || len <= 0) {
|
|
133
|
+
undecodable += 1;
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
const raw = bytes.subarray(start, start + len);
|
|
137
|
+
if (/\/FlateDecode/.test(dict)) {
|
|
138
|
+
try {
|
|
139
|
+
list.push({ obj, dict, body: inflateSync(raw).toString('latin1'), decoded: true });
|
|
140
|
+
} catch {
|
|
141
|
+
undecodable += 1;
|
|
142
|
+
list.push({ obj, dict, body: '', decoded: false });
|
|
143
|
+
}
|
|
144
|
+
} else {
|
|
145
|
+
list.push({ obj, dict, body: raw.toString('latin1'), decoded: true });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return { list, undecodable };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** code → unicode, plus how many bytes one code occupies. */
|
|
152
|
+
interface CMap {
|
|
153
|
+
map: Map<number, string>;
|
|
154
|
+
codeBytes: 1 | 2;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Parse a /ToUnicode CMap: the bfchar pairs and bfrange runs. */
|
|
158
|
+
function parseCMap(body: string): CMap {
|
|
159
|
+
const map = new Map<number, string>();
|
|
160
|
+
const hexToStr = (h: string): string => {
|
|
161
|
+
const clean = h.replace(/\s+/g, '');
|
|
162
|
+
let out = '';
|
|
163
|
+
for (let i = 0; i + 3 < clean.length + 1; i += 4) {
|
|
164
|
+
const cp = parseInt(clean.slice(i, i + 4), 16);
|
|
165
|
+
if (Number.isFinite(cp)) out += String.fromCharCode(cp);
|
|
166
|
+
}
|
|
167
|
+
return out;
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
for (const block of body.match(/beginbfchar([\s\S]*?)endbfchar/g) ?? []) {
|
|
171
|
+
for (const m of block.matchAll(/<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>/g)) {
|
|
172
|
+
map.set(parseInt(m[1]!, 16), hexToStr(m[2]!));
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
for (const block of body.match(/beginbfrange([\s\S]*?)endbfrange/g) ?? []) {
|
|
176
|
+
for (const m of block.matchAll(/<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>/g)) {
|
|
177
|
+
const lo = parseInt(m[1]!, 16);
|
|
178
|
+
const hi = parseInt(m[2]!, 16);
|
|
179
|
+
const base = parseInt(m[3]!, 16);
|
|
180
|
+
// A runaway range would allocate unboundedly on a malformed file.
|
|
181
|
+
if (!(hi >= lo) || hi - lo > 65535) continue;
|
|
182
|
+
for (let c = lo; c <= hi; c += 1) map.set(c, String.fromCharCode(base + (c - lo)));
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
const cs = body.match(/begincodespacerange\s*<([0-9A-Fa-f]+)>/);
|
|
186
|
+
const codeBytes: 1 | 2 = cs && cs[1]!.length >= 4 ? 2 : 1;
|
|
187
|
+
return { map, codeBytes };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** Resource-name (`F4`) → the CMap of the font it points at.
|
|
191
|
+
*
|
|
192
|
+
* Resolved per font rather than merged across all of them: two subset fonts
|
|
193
|
+
* routinely use the SAME code for different glyphs, so one merged table would
|
|
194
|
+
* decode half the page into the wrong letters and still look like text. */
|
|
195
|
+
function fontCMaps(bytes: Buffer, all: PdfStream[]): Map<string, CMap> {
|
|
196
|
+
const hay = bytes.toString('latin1');
|
|
197
|
+
const byObj = new Map<number, PdfStream>();
|
|
198
|
+
for (const s of all) if (s.obj != null && s.decoded) byObj.set(s.obj, s);
|
|
199
|
+
|
|
200
|
+
const out = new Map<string, CMap>();
|
|
201
|
+
for (const fd of hay.matchAll(/\/Font\s*<<([\s\S]*?)>>/g)) {
|
|
202
|
+
for (const ref of fd[1]!.matchAll(/\/([A-Za-z0-9]+)\s+(\d+)\s+0\s+R/g)) {
|
|
203
|
+
const name = ref[1]!;
|
|
204
|
+
const fontObj = Number(ref[2]);
|
|
205
|
+
// BOUNDED BY THE OBJECT'S OWN `endobj`, NOT BY A MAGIC NUMBER.
|
|
206
|
+
//
|
|
207
|
+
// F074.65: this used to be `[\s\S]{0,600}?`. A Type 3 font dictionary is
|
|
208
|
+
// far larger than 600 characters because it carries /CharProcs — a named
|
|
209
|
+
// content stream for EVERY glyph — so /ToUnicode fell outside the window,
|
|
210
|
+
// the font resolved to no CMap, and every character code it selected was
|
|
211
|
+
// dropped. Measured on the committed fixture: the line "system-ui er
|
|
212
|
+
// variabel" vanished completely while undecodableStreams reported 0,
|
|
213
|
+
// because the stream inflated perfectly and only the mapping was missing.
|
|
214
|
+
const objBody = hay.match(new RegExp(`(?:^|[^0-9])${fontObj}\\s+0\\s+obj([\\s\\S]*?)endobj`));
|
|
215
|
+
const toUni = objBody?.[1]?.match(/\/ToUnicode\s+(\d+)\s+0\s+R/)?.[1];
|
|
216
|
+
if (!toUni) continue;
|
|
217
|
+
const cmapStream = byObj.get(Number(toUni));
|
|
218
|
+
if (cmapStream) out.set(name, parseCMap(cmapStream.body));
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
return out;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Decode one text-showing operand into characters.
|
|
225
|
+
*
|
|
226
|
+
* Returns what was read AND how many character codes were DROPPED. The count is
|
|
227
|
+
* the point: an unmapped code is deliberately not guessed (inventing a character
|
|
228
|
+
* would let a containment assertion pass on text the page never drew), but
|
|
229
|
+
* dropping it silently is how a whole line disappears while the report says
|
|
230
|
+
* everything was readable. */
|
|
231
|
+
function decodeOperand(
|
|
232
|
+
operand: string,
|
|
233
|
+
isHex: boolean,
|
|
234
|
+
cmap: CMap | undefined,
|
|
235
|
+
): { text: string; dropped: number } {
|
|
236
|
+
if (!isHex) {
|
|
237
|
+
// A literal string in a simple-encoded font is already readable.
|
|
238
|
+
return {
|
|
239
|
+
text: operand.replace(/\\([nrtbf()\\])/g, (_, c: string) =>
|
|
240
|
+
({ n: '\n', r: '\r', t: '\t', b: '\b', f: '\f' })[c] ?? c,
|
|
241
|
+
),
|
|
242
|
+
dropped: 0,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
const clean = operand.replace(/\s+/g, '');
|
|
246
|
+
const step = (cmap?.codeBytes ?? 2) * 2;
|
|
247
|
+
let out = '';
|
|
248
|
+
let dropped = 0;
|
|
249
|
+
for (let i = 0; i + step <= clean.length; i += step) {
|
|
250
|
+
const code = parseInt(clean.slice(i, i + step), 16);
|
|
251
|
+
if (!Number.isFinite(code)) continue;
|
|
252
|
+
// An unmapped code is dropped rather than guessed: inventing a character
|
|
253
|
+
// here would let a containment assertion pass on text the page never drew.
|
|
254
|
+
const ch = cmap?.map.get(code);
|
|
255
|
+
if (ch != null) out += ch;
|
|
256
|
+
else dropped += 1;
|
|
257
|
+
}
|
|
258
|
+
return { text: out, dropped };
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Pull the readable text out of one content stream, tracking the current font. */
|
|
262
|
+
function textFromStream(
|
|
263
|
+
body: string,
|
|
264
|
+
cmaps: Map<string, CMap>,
|
|
265
|
+
): { text: string; runs: number; undecodedRuns: number } {
|
|
266
|
+
let current: CMap | undefined;
|
|
267
|
+
let text = '';
|
|
268
|
+
let runs = 0;
|
|
269
|
+
let undecodedRuns = 0;
|
|
270
|
+
// Font selections and text-showing operators, in document order.
|
|
271
|
+
const re =
|
|
272
|
+
/\/([A-Za-z0-9]+)\s+[\d.]+\s+Tf|<([0-9A-Fa-f\s]*)>\s*(Tj|'|")|\(((?:\\.|[^)\\])*)\)\s*(Tj|'|")|\[((?:[^\]\\]|\\.)*)\]\s*TJ/g;
|
|
273
|
+
for (const m of body.matchAll(re)) {
|
|
274
|
+
if (m[1] != null) {
|
|
275
|
+
current = cmaps.get(m[1]);
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
let piece = '';
|
|
279
|
+
let dropped = 0;
|
|
280
|
+
const take = (r: { text: string; dropped: number }) => {
|
|
281
|
+
piece += r.text;
|
|
282
|
+
dropped += r.dropped;
|
|
283
|
+
};
|
|
284
|
+
if (m[2] != null) take(decodeOperand(m[2], true, current));
|
|
285
|
+
else if (m[4] != null) take(decodeOperand(m[4], false, current));
|
|
286
|
+
else if (m[6] != null) {
|
|
287
|
+
// A TJ array interleaves strings with kerning numbers; only the strings
|
|
288
|
+
// are content. The numbers are DELIBERATELY ignored.
|
|
289
|
+
//
|
|
290
|
+
// The first version turned a large negative kern into a space, on the
|
|
291
|
+
// theory that it is a word gap the page draws but never stores. Measured
|
|
292
|
+
// against fd-sundhed's real poster, that was wrong and destructive: CSS
|
|
293
|
+
// letter-spacing is emitted as exactly those kerns, so every headline came
|
|
294
|
+
// out as "f d s u n d h e d . d k" and a containment assertion on correct
|
|
295
|
+
// output FAILED. Chromium writes real space glyphs (the CMap maps them to
|
|
296
|
+
// U+0020), so the gaps do not need reconstructing — and a heuristic that
|
|
297
|
+
// invents characters can only ever make extracted text say something the
|
|
298
|
+
// page did not.
|
|
299
|
+
for (const part of m[6].matchAll(/<([0-9A-Fa-f\s]*)>|\(((?:\\.|[^)\\])*)\)/g)) {
|
|
300
|
+
if (part[1] != null) take(decodeOperand(part[1], true, current));
|
|
301
|
+
else if (part[2] != null) take(decodeOperand(part[2], false, current));
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
// F074.65 — COUNTED BEFORE THE EMPTY-PIECE SKIP, and that order is the whole
|
|
305
|
+
// fix. A run whose every code was dropped produces an empty piece, so the
|
|
306
|
+
// old `continue` returned it to the caller as if it had never existed: the
|
|
307
|
+
// page's line was gone AND nothing said so.
|
|
308
|
+
if (dropped > 0) undecodedRuns += 1;
|
|
309
|
+
if (piece.length === 0) continue;
|
|
310
|
+
runs += 1;
|
|
311
|
+
// Concatenated with NOTHING inserted between runs.
|
|
312
|
+
//
|
|
313
|
+
// Measured on fd-sundhed's real poster, which is what settled it. Joining
|
|
314
|
+
// runs with a space looked right on a plain page and destroyed every
|
|
315
|
+
// letter-spaced headline: Chromium emits one run per glyph for CSS
|
|
316
|
+
// letter-spacing, so "fdsundhed.dk" came back as "f d s u n d h e d . d k"
|
|
317
|
+
// and a containment check on CORRECT output failed. Concatenating instead
|
|
318
|
+
// loses a space at a line break ("Skaden skal" + "ikke stå" → "skalikke"),
|
|
319
|
+
// which is why pdfTextContains() compares whitespace-insensitively. The
|
|
320
|
+
// trade is deliberate: a missing space is a false NEGATIVE the caller can
|
|
321
|
+
// work around, an invented one puts characters in the text that the page
|
|
322
|
+
// never drew.
|
|
323
|
+
text += piece;
|
|
324
|
+
}
|
|
325
|
+
return { text, runs, undecodedRuns };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** F074.65 — which font PROGRAMS the document draws with.
|
|
329
|
+
*
|
|
330
|
+
* Answers the only question a graphic designer is actually asking: can I re-type
|
|
331
|
+
* this word? `/Subtype /Type3` says no — the glyphs are drawings. It is read
|
|
332
|
+
* straight out of the dictionary, so there is no heuristic and nothing to
|
|
333
|
+
* calibrate.
|
|
334
|
+
*
|
|
335
|
+
* CID DESCENDANTS ARE EXCLUDED. A composite font is a `/Type0` that points at a
|
|
336
|
+
* `/CIDFontType2` descendant: the same typeface described twice, which is why
|
|
337
|
+
* the raw `/Type /Font` count says 10 on a document `pdffonts` lists 7 rows for.
|
|
338
|
+
* Counting the descendant would inflate the "real fonts" number with duplicates
|
|
339
|
+
* and make a Type-3-heavy document look better than it is. */
|
|
340
|
+
export function fontPrograms(bytes: Buffer): PdfFontProgram[] {
|
|
341
|
+
const hay = bytes.toString('latin1');
|
|
342
|
+
const out: PdfFontProgram[] = [];
|
|
343
|
+
for (const m of hay.matchAll(/\d+\s+0\s+obj([\s\S]*?)endobj/g)) {
|
|
344
|
+
const body = m[1]!;
|
|
345
|
+
if (!/\/Type\s*\/Font\b/.test(body)) continue;
|
|
346
|
+
const subtype = body.match(/\/Subtype\s*\/([A-Za-z0-9]+)/)?.[1];
|
|
347
|
+
if (!subtype) continue;
|
|
348
|
+
// The descendant half of a composite font — the same face, counted twice.
|
|
349
|
+
if (subtype.startsWith('CIDFontType')) continue;
|
|
350
|
+
out.push({
|
|
351
|
+
name: fontName(hay, body),
|
|
352
|
+
subtype,
|
|
353
|
+
outlines: subtype === 'Type3',
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
return out;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** The font's human name, following /FontDescriptor when the dictionary has no
|
|
360
|
+
* /BaseFont of its own.
|
|
361
|
+
*
|
|
362
|
+
* A TYPE 3 FONT HAS NO /BaseFont — measured on the committed fixture, where the
|
|
363
|
+
* dictionary carries FontMatrix, Widths, CharProcs and a /FontDescriptor
|
|
364
|
+
* reference, and nothing else identifying. The name a designer needs
|
|
365
|
+
* (`DAAAAA+.SFNS-Regular`) is one hop away in that descriptor's /FontName, which
|
|
366
|
+
* is also where `pdffonts` reads it from. Without the hop the report says
|
|
367
|
+
* "(unnamed)" for exactly the fonts it exists to name, which is the whole
|
|
368
|
+
* remedy gone. */
|
|
369
|
+
function fontName(hay: string, body: string): string {
|
|
370
|
+
const direct = body.match(/\/BaseFont\s*\/([^\s/>\]]+)/)?.[1];
|
|
371
|
+
if (direct) return direct;
|
|
372
|
+
const descRef = body.match(/\/FontDescriptor\s+(\d+)\s+0\s+R/)?.[1];
|
|
373
|
+
if (descRef) {
|
|
374
|
+
const desc = hay.match(new RegExp(`(?:^|[^0-9])${descRef}\\s+0\\s+obj([\\s\\S]*?)endobj`))?.[1];
|
|
375
|
+
const named = desc?.match(/\/FontName\s*\/([^\s/>\]]+)/)?.[1];
|
|
376
|
+
if (named) return named;
|
|
377
|
+
}
|
|
378
|
+
// Reported as unnamed rather than given a plausible label: a made-up family
|
|
379
|
+
// name in the remedy sends someone looking for a font they do not have.
|
|
380
|
+
return '(unnamed)';
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/** Count and READ what the PDF actually contains. */
|
|
384
|
+
export function pdfContentStats(bytes: Buffer): PdfContentStats {
|
|
385
|
+
const { list, undecodable } = streams(bytes);
|
|
386
|
+
const cmaps = fontCMaps(bytes, list);
|
|
387
|
+
|
|
388
|
+
let textRuns = 0;
|
|
389
|
+
let undecodedRuns = 0;
|
|
390
|
+
const parts: string[] = [];
|
|
391
|
+
for (const s of list) {
|
|
392
|
+
if (!s.decoded) continue;
|
|
393
|
+
// A content stream is one that selects a font and shows text. Font programs
|
|
394
|
+
// and CMaps live in streams too and must not be scanned as content.
|
|
395
|
+
if (!/\bTf\b/.test(s.body)) continue;
|
|
396
|
+
const r = textFromStream(s.body, cmaps);
|
|
397
|
+
textRuns += r.runs;
|
|
398
|
+
undecodedRuns += r.undecodedRuns;
|
|
399
|
+
if (r.text) parts.push(r.text);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const hay = bytes.toString('latin1');
|
|
403
|
+
return {
|
|
404
|
+
textRuns,
|
|
405
|
+
fonts: (hay.match(/\/Type\s*\/Font\b/g) ?? []).length,
|
|
406
|
+
images: (hay.match(/\/Subtype\s*\/Image\b/g) ?? []).length,
|
|
407
|
+
undecodableStreams: undecodable,
|
|
408
|
+
undecodedRuns,
|
|
409
|
+
fontPrograms: fontPrograms(bytes),
|
|
410
|
+
text: parts.join(' '),
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/** The verdict fd-sundhed's designer cares about: can they edit the text?
|
|
415
|
+
*
|
|
416
|
+
* Deliberately NOT `textRuns > 0` alone. A bitmap page can carry a stray text
|
|
417
|
+
* operator (a watermark, a page number stamped over the image) and would then
|
|
418
|
+
* pass a naive check while remaining exactly as un-editable as before.
|
|
419
|
+
*
|
|
420
|
+
* F074.65 ADDS THE FOURTH CONDITION, and it is the one that was wrong in
|
|
421
|
+
* production: NO Type 3 font. A page set in a variable font (macOS `system-ui`,
|
|
422
|
+
* and therefore Tailwind's default `--font-sans`) satisfied all three original
|
|
423
|
+
* conditions — real text runs, real fonts, real extractable characters — and
|
|
424
|
+
* opened in Illustrator as paths the designer could not re-type. Every cheap
|
|
425
|
+
* check passed on exactly the artifact the tool exists to replace, one level
|
|
426
|
+
* further in than the bitmap it was written to catch.
|
|
427
|
+
*
|
|
428
|
+
* IT GOES FALSE RATHER THAN TRUE-WITH-A-WARNING. sanne proposed the warning and
|
|
429
|
+
* it is a reasonable position; this disagrees on one point. A boolean whose name
|
|
430
|
+
* answers the designer's question has to answer it — a `true` sitting beside a
|
|
431
|
+
* caveat is read as `true`, and the caveat is the half that gets skimmed. */
|
|
432
|
+
export function isVectorText(stats: PdfContentStats): boolean {
|
|
433
|
+
if (stats.fontPrograms.some((f) => f.outlines)) return false;
|
|
434
|
+
// WE SAW FONT OBJECTS AND COULD CLASSIFY NONE OF THEM.
|
|
435
|
+
//
|
|
436
|
+
// Found reviewing this card's own diff, and it is the same false-green one
|
|
437
|
+
// level further along: fontPrograms reads /Subtype out of top-level `N 0 obj`
|
|
438
|
+
// dictionaries, so a producer that puts fonts in a compressed object stream
|
|
439
|
+
// (or writes a /Subtype this does not parse) yields an EMPTY list — and an
|
|
440
|
+
// empty list has no Type 3 in it, so the outline check above passes and the
|
|
441
|
+
// verdict falls through to true. A document we could not read would be
|
|
442
|
+
// reported as safe to hand a designer.
|
|
443
|
+
//
|
|
444
|
+
// Today's only caller feeds this the bytes our own Chromium just produced, so
|
|
445
|
+
// it does not happen in practice. That is a reason it is cheap to guard, not a
|
|
446
|
+
// reason to skip it: the guard costs one line, and the assumption it rests on
|
|
447
|
+
// is invisible at the call site the day someone measures an uploaded PDF.
|
|
448
|
+
if (stats.fonts > 0 && stats.fontPrograms.length === 0) return false;
|
|
449
|
+
return stats.textRuns > 0 && stats.fonts > 0 && stats.text.trim().length > 0;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** The font families drawn as outlines, deduplicated — the ACTIONABLE half.
|
|
453
|
+
*
|
|
454
|
+
* "This PDF has outlines" tells the mockup author nothing they can do.
|
|
455
|
+
* "`.SFNS-Regular` is drawn as outlines" tells them which font to replace with a
|
|
456
|
+
* static instance, which is the whole remedy. */
|
|
457
|
+
export function outlineFontNames(stats: PdfContentStats): string[] {
|
|
458
|
+
return [...new Set(stats.fontPrograms.filter((f) => f.outlines).map((f) => f.name))].sort();
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/** Does the extracted text contain `phrase`, ignoring whitespace on both sides?
|
|
462
|
+
*
|
|
463
|
+
* This is the matcher an assertion should use, and the reason is measured, not
|
|
464
|
+
* stylistic: run boundaries in a PDF do not line up with word boundaries on the
|
|
465
|
+
* page. A line break between "Skaden skal" and "ikke stå i kø" stores no space
|
|
466
|
+
* at all, so a literal `includes()` misses a headline that is plainly there —
|
|
467
|
+
* a false negative on a correct export, which is the direction that makes a
|
|
468
|
+
* working feature look broken.
|
|
469
|
+
*
|
|
470
|
+
* What it gives up: a phrase could match across two unrelated elements that
|
|
471
|
+
* happen to abut. That is a far narrower risk than the one it removes, and it
|
|
472
|
+
* fails toward "found" only for text that genuinely IS in the document. */
|
|
473
|
+
export function pdfTextContains(source: PdfContentStats | string, phrase: string): boolean {
|
|
474
|
+
return matchPhrase(source, phrase) !== 'absent';
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/** HOW a phrase matched — the distinction `pdfTextContains` flattens.
|
|
478
|
+
*
|
|
479
|
+
* 'case-insensitive' means the page drew the words with DIFFERENT CAPITALISATION
|
|
480
|
+
* from what the caller wrote. That is a fact worth keeping, not a detail to
|
|
481
|
+
* swallow: it is almost always `text-transform` in the stylesheet, and the
|
|
482
|
+
* caller should be able to see that the page says something other than their
|
|
483
|
+
* source does. */
|
|
484
|
+
export type PhraseMatch = 'exact' | 'case-insensitive' | 'absent';
|
|
485
|
+
|
|
486
|
+
/** F074.67 — case, like whitespace, is a PAINT-TIME difference.
|
|
487
|
+
*
|
|
488
|
+
* Measured on fd-sundhed's real A3 poster: the source says "Til alle ansatte i
|
|
489
|
+
* Aalborg Kommune", `text-transform: uppercase` applies when the page is drawn,
|
|
490
|
+
* and the PDF stores "TIL ALLE ANSATTE I AALBORGKOMMUNE". A case-sensitive
|
|
491
|
+
* matcher answers a confident FALSE for a headline that is unmistakably there —
|
|
492
|
+
* and the caller goes looking for a layout bug that does not exist.
|
|
493
|
+
*
|
|
494
|
+
* THE PRECEDENT IS ALREADY HERE, which is what makes this consistent rather than
|
|
495
|
+
* a convenience: this matcher has been whitespace-insensitive since F074.63, for
|
|
496
|
+
* exactly the same reason (run boundaries do not line up with word boundaries —
|
|
497
|
+
* "Skaden skal" + "ikke stå i kø" is stored with no space at all). Case is the
|
|
498
|
+
* same category of difference the caller cannot predict from their own HTML.
|
|
499
|
+
*
|
|
500
|
+
* WHAT IS DELIBERATELY NOT WIDENED: punctuation, accents, and word-internal
|
|
501
|
+
* substrings. Those would let an assertion pass on text the page never drew,
|
|
502
|
+
* which is a worse failure than the false negative being fixed here. */
|
|
503
|
+
export function matchPhrase(source: PdfContentStats | string, phrase: string): PhraseMatch {
|
|
504
|
+
const strip = (v: string) => v.replace(/\s+/g, '');
|
|
505
|
+
const hay = typeof source === 'string' ? source : source.text;
|
|
506
|
+
const needle = strip(phrase);
|
|
507
|
+
if (needle.length === 0) return 'absent';
|
|
508
|
+
const haystack = strip(hay);
|
|
509
|
+
if (haystack.includes(needle)) return 'exact';
|
|
510
|
+
// toLowerCase, not localeCompare: this compares two strings from the SAME
|
|
511
|
+
// document pipeline, and Danish æ/ø/å round-trip through it unchanged.
|
|
512
|
+
if (haystack.toLowerCase().includes(needle.toLowerCase())) return 'case-insensitive';
|
|
513
|
+
return 'absent';
|
|
514
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// pixelmatch@6 ships no type declarations (ESM, default export only).
|
|
2
|
+
// Minimal ambient declaration for the single signature Lens uses.
|
|
3
|
+
declare module 'pixelmatch' {
|
|
4
|
+
interface PixelmatchOptions {
|
|
5
|
+
threshold?: number;
|
|
6
|
+
includeAA?: boolean;
|
|
7
|
+
alpha?: number;
|
|
8
|
+
aaColor?: [number, number, number];
|
|
9
|
+
diffColor?: [number, number, number];
|
|
10
|
+
diffColorAlt?: [number, number, number];
|
|
11
|
+
diffMask?: boolean;
|
|
12
|
+
}
|
|
13
|
+
export default function pixelmatch(
|
|
14
|
+
img1: Uint8Array,
|
|
15
|
+
img2: Uint8Array,
|
|
16
|
+
output: Uint8Array | null,
|
|
17
|
+
width: number,
|
|
18
|
+
height: number,
|
|
19
|
+
options?: PixelmatchOptions,
|
|
20
|
+
): number;
|
|
21
|
+
}
|