@acepaste/ingress 0.9.26 → 0.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli.js +65 -5
- package/dist/src/engine.d.ts +20 -1
- package/dist/src/engine.js +25 -3
- package/dist/src/image.d.ts +38 -0
- package/dist/src/image.js +45 -26
- package/dist/src/index.d.ts +4 -0
- package/dist/src/index.js +2 -0
- package/dist/src/screen.d.ts +64 -0
- package/dist/src/screen.js +120 -0
- package/dist/src/types.d.ts +5 -4
- package/dist/src/types.js +3 -3
- package/dist/src/video.d.ts +156 -0
- package/dist/src/video.js +421 -0
- package/native/ingress-vision.swift +543 -4
- package/package.json +1 -1
package/dist/src/cli.js
CHANGED
|
@@ -151,15 +151,39 @@ async function main() {
|
|
|
151
151
|
process.exitCode = 1;
|
|
152
152
|
return;
|
|
153
153
|
}
|
|
154
|
+
// ingress --video <file>: gate a video (text and codes on screen and how long they are shown, subtitle tracks, tags, sound)
|
|
155
|
+
if (flag("video")) {
|
|
156
|
+
const file = args.find((x) => !x.startsWith("--"));
|
|
157
|
+
if (!file) {
|
|
158
|
+
console.error("usage: ingress --video <file> [--json] [--captions=file.vtt] [--max-seconds=600] [--max-frames=240]");
|
|
159
|
+
process.exit(2);
|
|
160
|
+
}
|
|
161
|
+
const engine = new IngressEngine({ ...(opt("policy") ? loadPolicyFile(opt("policy")) : {}), sandbox: "off" });
|
|
162
|
+
const r = await engine.scanVideo(readFileSync(file), { url: `file:${file}`, captions: opt("captions") ? readFileSync(opt("captions"), "utf8") : undefined, maxSeconds: opt("max-seconds") ? Number(opt("max-seconds")) : undefined, maxFrames: opt("max-frames") ? Number(opt("max-frames")) : undefined, override: flag("override") });
|
|
163
|
+
if (flag("json"))
|
|
164
|
+
console.log(JSON.stringify(r, null, 2));
|
|
165
|
+
else {
|
|
166
|
+
console.log(r.envelope);
|
|
167
|
+
const c = r.coverage?.video;
|
|
168
|
+
if (c && !c.frames)
|
|
169
|
+
console.error(`\nnote: the video's frames were not read (${c.framesReason}).`);
|
|
170
|
+
else if (c)
|
|
171
|
+
console.error(`\ncoverage: ${Math.round(c.examinedSeconds ?? 0)} of ${Math.round(c.seconds ?? 0)} seconds examined; ${c.framesRead} of ${c.framesTotal} frames read for text (${c.sampling}); sound ${c.audio}${c.audioReason ? ` (${c.audioReason})` : ""}; ${c.subtitleCues} caption cue(s)`);
|
|
172
|
+
}
|
|
173
|
+
if (r.tier !== "clean")
|
|
174
|
+
process.exitCode = 1;
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
154
177
|
// ingress --image <file>: gate a picture (text in it, codes, what only a machine would read)
|
|
155
178
|
if (flag("image")) {
|
|
156
179
|
const file = args.find((x) => !x.startsWith("--"));
|
|
157
180
|
if (!file) {
|
|
158
|
-
console.error("usage: ingress --image <file> [--json] [--caption=words shown beside it]");
|
|
181
|
+
console.error("usage: ingress --image <file> [--json] [--caption=words shown beside it] [--capture=viewport [--page-host=example.com]] (--capture=viewport: the picture is a capture of a page's content area, as a browser tool's screenshot is)");
|
|
159
182
|
process.exit(2);
|
|
160
183
|
}
|
|
161
184
|
const engine = new IngressEngine({ ...(opt("policy") ? loadPolicyFile(opt("policy")) : {}), sandbox: "off" });
|
|
162
|
-
const
|
|
185
|
+
const capture = opt("capture");
|
|
186
|
+
const r = await engine.scanImage(readFileSync(file), { url: `file:${file}`, caption: opt("caption"), override: flag("override"), screen: capture === "viewport" || capture === "window" ? { capture, pageHost: opt("page-host") } : undefined });
|
|
163
187
|
if (flag("json"))
|
|
164
188
|
console.log(JSON.stringify(r, null, 2));
|
|
165
189
|
else {
|
|
@@ -548,7 +572,14 @@ async function visionCommand(a) {
|
|
|
548
572
|
}
|
|
549
573
|
const dir = mkdtempSync(join(tmpdir(), "ingress-vision-selftest-"));
|
|
550
574
|
try {
|
|
551
|
-
|
|
575
|
+
try {
|
|
576
|
+
execFileSync(visionHelperPath(), ["--make-fixtures", dir], { timeout: 180_000 });
|
|
577
|
+
}
|
|
578
|
+
catch { /* an older helper writes the pictures and not the clips; the checks below say what is missing */ }
|
|
579
|
+
if (!existsSync(join(dir, "video-plain.mp4")) || !existsSync(join(dir, "screen-drawn-window.png"))) {
|
|
580
|
+
console.error("the vision helper on this machine was built by an earlier version and cannot write the video and screen tests (run: ingress vision build)");
|
|
581
|
+
process.exit(1);
|
|
582
|
+
}
|
|
552
583
|
const engine = new IngressEngine({ sandbox: "off" }, { log: () => { } });
|
|
553
584
|
const expect = {
|
|
554
585
|
"plain.png": { refused: false }, "no-text.png": { refused: false }, "qr-link.png": { refused: false },
|
|
@@ -562,10 +593,39 @@ async function visionCommand(a) {
|
|
|
562
593
|
const ok = (r.tier !== "clean") === want.refused && (!want.kind || r.findings.some((f) => f.kind === want.kind));
|
|
563
594
|
if (!ok)
|
|
564
595
|
bad++;
|
|
565
|
-
console.log(`${ok ? "ok " : "FAIL"} ${name.padEnd(
|
|
596
|
+
console.log(`${ok ? "ok " : "FAIL"} ${name.padEnd(24)} ${r.tier}${want.kind ? ` (${want.kind})` : ""}`);
|
|
597
|
+
}
|
|
598
|
+
// a browser window drawn into a page: counts only when the picture is known to be a capture of a page's content area
|
|
599
|
+
const drawn = readFileSync(join(dir, "screen-drawn-window.png"));
|
|
600
|
+
const screens = [
|
|
601
|
+
["screen-drawn-window.png", drawn, { screen: { capture: "viewport", pageHost: "templates.example.org" } }, true, "image.drawn_browser_window"],
|
|
602
|
+
["screen-drawn-window.png", drawn, {}, false],
|
|
603
|
+
["screen-plain-form.png", readFileSync(join(dir, "screen-plain-form.png")), { screen: { capture: "viewport", pageHost: "templates.example.org" } }, false],
|
|
604
|
+
];
|
|
605
|
+
for (const [name, bytes, o, refused, kind] of screens) {
|
|
606
|
+
const r = await engine.scanImage(bytes, { url: `selftest:${name}`, ...o });
|
|
607
|
+
const ok = (r.tier !== "clean") === refused && (!kind || r.findings.some((f) => f.kind === kind));
|
|
608
|
+
if (!ok)
|
|
609
|
+
bad++;
|
|
610
|
+
console.log(`${ok ? "ok " : "FAIL"} ${name.padEnd(24)} ${r.tier}${kind ? ` (${kind})` : ""}${o?.screen ? "" : " [nothing said about how it was taken]"}`);
|
|
611
|
+
}
|
|
612
|
+
// clips: the same words are refused when shown for a fifteenth of a second, in a code, and in a cue nobody can read
|
|
613
|
+
const clips = {
|
|
614
|
+
"video-plain.mp4": { refused: false }, "video-no-text.mp4": { refused: false },
|
|
615
|
+
"video-brief-frames.mp4": { refused: true, kind: "video.brief_frame_text" }, "video-code.mp4": { refused: true, kind: "image.code_payload" },
|
|
616
|
+
"video-subtitles.mov": { refused: true, kind: "captions.unreadable_cue" },
|
|
617
|
+
};
|
|
618
|
+
for (const [name, want] of Object.entries(clips)) {
|
|
619
|
+
const r = await engine.scanVideo(readFileSync(join(dir, name)), { url: `selftest:${name}`, transcriber: false });
|
|
620
|
+
const read = r.coverage?.video?.frames === true;
|
|
621
|
+
const ok = read && (r.tier !== "clean") === want.refused && (!want.kind || r.findings.some((f) => f.kind === want.kind));
|
|
622
|
+
if (!ok)
|
|
623
|
+
bad++;
|
|
624
|
+
console.log(`${ok ? "ok " : "FAIL"} ${name.padEnd(24)} ${r.tier}${want.kind ? ` (${want.kind})` : ""}${read ? "" : ` [frames not read: ${r.coverage?.video?.framesReason}]`}`);
|
|
566
625
|
}
|
|
626
|
+
const checks = Object.keys(expect).length + screens.length + Object.keys(clips).length;
|
|
567
627
|
if (bad) {
|
|
568
|
-
console.error(`${bad} of ${
|
|
628
|
+
console.error(`${bad} of ${checks} checks failed on this machine`);
|
|
569
629
|
process.exitCode = 1;
|
|
570
630
|
}
|
|
571
631
|
}
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ import { type KeyObject } from "node:crypto";
|
|
|
14
14
|
import { acquireWithBrowser, type FetchLike } from "./acquire.js";
|
|
15
15
|
import { type VisionRunner } from "./image.js";
|
|
16
16
|
import { type AudioRunner } from "./audio.js";
|
|
17
|
+
import { type VideoRunner, type AudioExtractor } from "./video.js";
|
|
18
|
+
import type { ScreenHints } from "./screen.js";
|
|
17
19
|
import { servedWords, compareServed } from "./served.js";
|
|
18
20
|
import { runSandbox } from "./sandbox.js";
|
|
19
21
|
import { type ContentKind, type Finding, type IngressConfig, type IngressResult, type SourceInfo, type TaintReceipt, type Tier } from "./types.js";
|
|
@@ -107,7 +109,8 @@ export declare class IngressEngine {
|
|
|
107
109
|
source?: SourceInfo; /** words shown beside the image: a caption, alt text, a file name */
|
|
108
110
|
caption?: string; /** for a screenshot: the text the page's own source says is on it (see image.ts) */
|
|
109
111
|
sourceText?: string; /** how to read the pixels (default: the vision helper when built); false reads the container only */
|
|
110
|
-
vision?: VisionRunner | false;
|
|
112
|
+
vision?: VisionRunner | false; /** how the picture was taken, when known: a capture of a page's content area holds no real browser chrome (see screen.ts) */
|
|
113
|
+
screen?: ScreenHints;
|
|
111
114
|
}): Promise<IngressResult>;
|
|
112
115
|
/**
|
|
113
116
|
* Gate a sound recording: what is said in it, and what is there only for a machine (see audio.ts).
|
|
@@ -119,6 +122,22 @@ export declare class IngressEngine {
|
|
|
119
122
|
source?: SourceInfo; /** how to transcribe (default: the command in INGRESS_AUDIO_TRANSCRIBER); false reads the container only */
|
|
120
123
|
transcriber?: AudioRunner | false;
|
|
121
124
|
}): Promise<IngressResult>;
|
|
125
|
+
/**
|
|
126
|
+
* Gate a video: what is on screen and for how long, the codes in its frames, its subtitle tracks, its sound
|
|
127
|
+
* (when a transcriber is configured) and its tags (see video.ts). The delivered text is what a viewer can
|
|
128
|
+
* read on screen, the captions and the transcript. Whether the video itself goes on to the model is the
|
|
129
|
+
* caller's decision, made on the verdict. `coverage.video` says how much of it was read.
|
|
130
|
+
*/
|
|
131
|
+
scanVideo(bytes: Uint8Array, opts?: IngressOptions & {
|
|
132
|
+
url?: string;
|
|
133
|
+
source?: SourceInfo; /** how to read the frames (default: the vision helper when built); false reads nothing from the picture */
|
|
134
|
+
video?: VideoRunner | false; /** how to transcribe the sound (default: INGRESS_AUDIO_TRANSCRIBER); false skips it */
|
|
135
|
+
transcriber?: AudioRunner | false;
|
|
136
|
+
extractAudio?: AudioExtractor; /** a captions file that came with the video */
|
|
137
|
+
captions?: string;
|
|
138
|
+
maxFrames?: number;
|
|
139
|
+
maxSeconds?: number;
|
|
140
|
+
}): Promise<IngressResult>;
|
|
122
141
|
/**
|
|
123
142
|
* Gate what the model will actually receive when several sources are combined (search
|
|
124
143
|
* results, retrieved chunks, several tool results). An instruction can be split so each
|
package/dist/src/engine.js
CHANGED
|
@@ -25,8 +25,10 @@ import { adaptByKind, decodeEntities, detectKind, KINDS_WITH_MARKUP } from "./ad
|
|
|
25
25
|
import { inspectImage } from "./image.js";
|
|
26
26
|
import { detectImageFormat } from "./image-meta.js";
|
|
27
27
|
import { inspectAudio } from "./audio.js";
|
|
28
|
+
import { inspectVideo } from "./video.js";
|
|
28
29
|
const MAX_IMAGE_BYTES = 50_000_000;
|
|
29
30
|
const MAX_AUDIO_BYTES = 200_000_000;
|
|
31
|
+
const MAX_VIDEO_BYTES = 400_000_000;
|
|
30
32
|
import { servedWords, compareServed, passagesOnlyIn, looksLikeInterstitial } from "./served.js";
|
|
31
33
|
import { scanClassifier } from "./classifier.js";
|
|
32
34
|
import { runSandbox } from "./sandbox.js";
|
|
@@ -269,7 +271,7 @@ export class IngressEngine {
|
|
|
269
271
|
return this.analyze({ url, finalUrl: url, redirectChain: [], raw: text, kind: "html", externalRules: [], timings: {}, t0, override: !!opts.override, extraFindings: extra, rule, source: opts.source });
|
|
270
272
|
}
|
|
271
273
|
const ti = performance.now();
|
|
272
|
-
const insp = await inspectImage(bytes, { vision: opts.vision, caption: opts.caption, sourceText: opts.sourceText });
|
|
274
|
+
const insp = await inspectImage(bytes, { vision: opts.vision, caption: opts.caption, sourceText: opts.sourceText, screen: opts.screen });
|
|
273
275
|
return this.analyze({ url, finalUrl: url, redirectChain: [], raw: insp.report, kind: "image", externalRules: [], timings: { visibility: performance.now() - ti }, t0, override: !!opts.override, extraFindings: insp.findings, rule, source: opts.source, adapted: { segments: insp.segments, findings: [] }, imageCoverage: insp.coverage });
|
|
274
276
|
});
|
|
275
277
|
}
|
|
@@ -292,6 +294,26 @@ export class IngressEngine {
|
|
|
292
294
|
return this.analyze({ url, finalUrl: url, redirectChain: [], raw: insp.report, kind: "audio", externalRules: [], timings: { visibility: performance.now() - ti }, t0, override: !!opts.override, extraFindings: insp.findings, rule, source: opts.source, adapted: { segments: insp.segments, findings: [] }, audioCoverage: insp.coverage });
|
|
293
295
|
});
|
|
294
296
|
}
|
|
297
|
+
/**
|
|
298
|
+
* Gate a video: what is on screen and for how long, the codes in its frames, its subtitle tracks, its sound
|
|
299
|
+
* (when a transcriber is configured) and its tags (see video.ts). The delivered text is what a viewer can
|
|
300
|
+
* read on screen, the captions and the transcript. Whether the video itself goes on to the model is the
|
|
301
|
+
* caller's decision, made on the verdict. `coverage.video` says how much of it was read.
|
|
302
|
+
*/
|
|
303
|
+
async scanVideo(bytes, opts = {}) {
|
|
304
|
+
const t0 = performance.now();
|
|
305
|
+
const url = opts.url ?? "inline:video";
|
|
306
|
+
if (opts.quarantined)
|
|
307
|
+
return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 1, severity: 1, message: `source is quarantined by your organization: ${opts.quarantined}`, action: "none" }], {}, t0, "video", "hostile", true);
|
|
308
|
+
if (bytes.length > MAX_VIDEO_BYTES)
|
|
309
|
+
return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 0.5, severity: 0.5, message: `video is ${bytes.length} bytes, over the ${MAX_VIDEO_BYTES} byte limit`, action: "none" }], {}, t0, "video");
|
|
310
|
+
const rule = url.startsWith("inline:") ? undefined : this.domainRule(url);
|
|
311
|
+
return this.guarded(url, "video", t0, async () => {
|
|
312
|
+
const ti = performance.now();
|
|
313
|
+
const insp = await inspectVideo(bytes, { video: opts.video, transcriber: opts.transcriber, extractAudio: opts.extractAudio, captions: opts.captions, maxFrames: opts.maxFrames, maxSeconds: opts.maxSeconds });
|
|
314
|
+
return this.analyze({ url, finalUrl: url, redirectChain: [], raw: insp.report, kind: "video", externalRules: [], timings: { visibility: performance.now() - ti }, t0, override: !!opts.override, extraFindings: insp.findings, rule, source: opts.source, adapted: { segments: insp.segments, findings: [] }, videoCoverage: insp.coverage });
|
|
315
|
+
});
|
|
316
|
+
}
|
|
295
317
|
/**
|
|
296
318
|
* Gate what the model will actually receive when several sources are combined (search
|
|
297
319
|
* results, retrieved chunks, several tool results). An instruction can be split so each
|
|
@@ -518,7 +540,7 @@ export class IngressEngine {
|
|
|
518
540
|
// Text recognised in a picture has no characters of its own: the recogniser chose them, and on small print it
|
|
519
541
|
// mixes alphabets freely. Lookalike-letter findings there say something about the recogniser, not the author.
|
|
520
542
|
// The characters that are really in an image file are its metadata, its codes' payloads and appended text.
|
|
521
|
-
const authored = a.kind === "image" ? segments.filter((s) => /^image\/(?:metadata|code|after-end)/.test(s.path)) : a.kind === "audio" ? segments.filter((s) => s.path.startsWith("audio/metadata")) : segments;
|
|
543
|
+
const authored = a.kind === "image" ? segments.filter((s) => /^image\/(?:metadata|code|after-end)/.test(s.path)) : a.kind === "audio" ? segments.filter((s) => s.path.startsWith("audio/metadata")) : a.kind === "video" ? segments.filter((s) => /^(?:video\/(?:metadata|code)|audio\/metadata|captions|transcript)/.test(s.path)) : segments;
|
|
522
544
|
const charAll = scanCharacters(authored.map((s) => s.text).join("\n"));
|
|
523
545
|
findings.push(...charAll.findings);
|
|
524
546
|
let cleanedSegments = segments.map((s) => ({ ...s, text: scanCharacters(s.text).cleaned }));
|
|
@@ -639,7 +661,7 @@ export class IngressEngine {
|
|
|
639
661
|
timings.total = performance.now() - a.t0;
|
|
640
662
|
if (tier !== "clean")
|
|
641
663
|
this.deps.log(buildLogEntry(a.finalUrl, tier, score, findings, safePayloads, receipt, overridden));
|
|
642
|
-
return { tier, score, contentKind: a.kind, delivered, envelope, findings, receipt, timings, overridden, coverage: { code: codeScan.coverage, ...(a.imageCoverage ? { image: a.imageCoverage } : {}), ...(a.audioCoverage ? { audio: a.audioCoverage } : {}) } };
|
|
664
|
+
return { tier, score, contentKind: a.kind, delivered, envelope, findings, receipt, timings, overridden, coverage: { code: codeScan.coverage, ...(a.imageCoverage ? { image: a.imageCoverage } : {}), ...(a.audioCoverage ? { audio: a.audioCoverage } : {}), ...(a.videoCoverage ? { video: a.videoCoverage } : {}) } };
|
|
643
665
|
}
|
|
644
666
|
async fetchStylesheets(html, baseUrl) {
|
|
645
667
|
const rules = [];
|
package/dist/src/image.d.ts
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { type ImageMeta } from "./image-meta.js";
|
|
21
21
|
import { type QrVerdict } from "./qr.js";
|
|
22
|
+
import { type ScreenHints } from "./screen.js";
|
|
22
23
|
import type { Finding, TextSegment } from "./types.js";
|
|
23
24
|
export interface VisionLine {
|
|
24
25
|
text: string;
|
|
@@ -47,6 +48,14 @@ export interface VisionReport {
|
|
|
47
48
|
payload: string;
|
|
48
49
|
variant: string;
|
|
49
50
|
}>;
|
|
51
|
+
/** rectangles seen in the picture as shown (helper version 2 on): the frames of windows, dialogs and panels */
|
|
52
|
+
rects?: Array<{
|
|
53
|
+
x: number;
|
|
54
|
+
y: number;
|
|
55
|
+
w: number;
|
|
56
|
+
h: number;
|
|
57
|
+
conf?: number;
|
|
58
|
+
}>;
|
|
50
59
|
notes: string[];
|
|
51
60
|
ms?: number;
|
|
52
61
|
}
|
|
@@ -75,6 +84,19 @@ export interface ImageInspection {
|
|
|
75
84
|
/** a stable description of what was found, hashed into the receipt */
|
|
76
85
|
report: string;
|
|
77
86
|
}
|
|
87
|
+
export declare const norm: (s: string) => string;
|
|
88
|
+
export declare function trigrams(s: string): Set<string>;
|
|
89
|
+
/** share of a line's character trigrams that occur in a reference text: 1 means the reference contains the line */
|
|
90
|
+
export declare function containment(line: string, ref: Set<string>): number;
|
|
91
|
+
export declare const similar: (a: string, b: string) => number;
|
|
92
|
+
export declare const wordy: (s: string) => number;
|
|
93
|
+
export declare const enough: (lines: readonly string[]) => boolean;
|
|
94
|
+
/**
|
|
95
|
+
* A message, not scraps: at least four word-like tokens, making up most of what was read. Measured on ordinary
|
|
96
|
+
* images: a dense screenshot resized without smoothing is read as "ICD 1CU QuE Fin war: 4' 1CBOWD TOU IC0", with
|
|
97
|
+
* full confidence, by every kernel. Those are scraps of digits. Words built to be read come out as words.
|
|
98
|
+
*/
|
|
99
|
+
export declare function sentence(lines: readonly string[]): boolean;
|
|
78
100
|
/** Where the built helper is expected: INGRESS_VISION_HELPER, or <state>/bin/ingress-vision. */
|
|
79
101
|
export declare function visionHelperPath(): string;
|
|
80
102
|
/** Why the helper cannot be used, or undefined when it can. A helper in the state folder must match the hash recorded when it was built. */
|
|
@@ -106,5 +128,21 @@ export interface InspectOptions {
|
|
|
106
128
|
* an image, a canvas, an overlay. That is how a fake dialog or a fake message is painted into a real window.
|
|
107
129
|
*/
|
|
108
130
|
sourceText?: string;
|
|
131
|
+
/** how the picture was taken, when the caller knows: a capture of a page's content area holds no real browser chrome (see screen.ts) */
|
|
132
|
+
screen?: ScreenHints;
|
|
109
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* The codes read in a picture or in a video's frames: a finding and (for a code that carries words) a concealed
|
|
136
|
+
* segment for each, and one visible line saying the codes are there. `beside` is the text shown with the code,
|
|
137
|
+
* for the check that the words name one place and the code goes to another. `root` is "image" or "video".
|
|
138
|
+
*/
|
|
139
|
+
export declare function judgeCodes(barcodes: ReadonlyArray<{
|
|
140
|
+
symbology: string;
|
|
141
|
+
payload: string;
|
|
142
|
+
at?: string;
|
|
143
|
+
}>, beside: string, root: "image" | "video"): {
|
|
144
|
+
codes: ImageInspection["codes"];
|
|
145
|
+
findings: Finding[];
|
|
146
|
+
segments: TextSegment[];
|
|
147
|
+
};
|
|
110
148
|
export declare function inspectImage(bytes: Uint8Array, opts?: InspectOptions): Promise<ImageInspection>;
|
package/dist/src/image.js
CHANGED
|
@@ -28,16 +28,17 @@ import { readImageMeta } from "./image-meta.js";
|
|
|
28
28
|
import { judgeQrPayload } from "./qr.js";
|
|
29
29
|
import { hostsMentioned } from "./egress.js";
|
|
30
30
|
import { passagesOnlyIn } from "./served.js";
|
|
31
|
+
import { screenFindings } from "./screen.js";
|
|
31
32
|
// ---------- comparing readings ----------
|
|
32
|
-
const norm = (s) => s.toLowerCase().normalize("NFKD").replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
33
|
-
function trigrams(s) {
|
|
33
|
+
export const norm = (s) => s.toLowerCase().normalize("NFKD").replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
34
|
+
export function trigrams(s) {
|
|
34
35
|
const t = ` ${norm(s)} `, out = new Set();
|
|
35
36
|
for (let i = 0; i + 3 <= t.length; i++)
|
|
36
37
|
out.add(t.slice(i, i + 3));
|
|
37
38
|
return out;
|
|
38
39
|
}
|
|
39
40
|
/** share of a line's character trigrams that occur in a reference text: 1 means the reference contains the line */
|
|
40
|
-
function containment(line, ref) {
|
|
41
|
+
export function containment(line, ref) {
|
|
41
42
|
const g = trigrams(line);
|
|
42
43
|
if (!g.size)
|
|
43
44
|
return 1;
|
|
@@ -47,11 +48,11 @@ function containment(line, ref) {
|
|
|
47
48
|
hit++;
|
|
48
49
|
return hit / g.size;
|
|
49
50
|
}
|
|
50
|
-
const similar = (a, b) => { const ga = trigrams(a), gb = trigrams(b); return Math.min(containment(a, gb), containment(b, ga)); };
|
|
51
|
-
const wordy = (s) => (norm(s).match(/\p{L}{2,}/gu) ?? []).length;
|
|
51
|
+
export const similar = (a, b) => { const ga = trigrams(a), gb = trigrams(b); return Math.min(containment(a, gb), containment(b, ga)); };
|
|
52
|
+
export const wordy = (s) => (norm(s).match(/\p{L}{2,}/gu) ?? []).length;
|
|
52
53
|
const letters = (s) => (s.match(/\p{L}/gu) ?? []).length;
|
|
53
54
|
const textOf = (lines) => lines.map((l) => l.text).join("\n");
|
|
54
|
-
const enough = (lines) => { const t = lines.join(" "); return wordy(t) >= 3 && letters(t) >= 12; };
|
|
55
|
+
export const enough = (lines) => { const t = lines.join(" "); return wordy(t) >= 3 && letters(t) >= 12; };
|
|
55
56
|
/**
|
|
56
57
|
* Does a token look like a word someone wrote? Letters only, in one casing (lower, UPPER or Capitalised), and for
|
|
57
58
|
* Latin letters at least four of them with a vowel. Scripts without case or spaces count from two characters.
|
|
@@ -69,7 +70,7 @@ function wordlike(tok) {
|
|
|
69
70
|
* images: a dense screenshot resized without smoothing is read as "ICD 1CU QuE Fin war: 4' 1CBOWD TOU IC0", with
|
|
70
71
|
* full confidence, by every kernel. Those are scraps of digits. Words built to be read come out as words.
|
|
71
72
|
*/
|
|
72
|
-
function sentence(lines) {
|
|
73
|
+
export function sentence(lines) {
|
|
73
74
|
const toks = lines.join(" ").split(/\s+/).map((t) => t.replace(/^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu, "")).filter((t) => /[\p{L}\p{N}]/u.test(t));
|
|
74
75
|
const words = toks.filter(wordlike).length;
|
|
75
76
|
// a scrap: letters and digits run together, or casing no writer uses (QuE). Short words (all, the, and) are neither.
|
|
@@ -212,6 +213,37 @@ const CLICKFIX_PASTE = /\b(?:ctrl|cmd|command|⌘)\s*\+\s*v\b|\bpaste (?:it|the|
|
|
|
212
213
|
const EXT = { png: "png", jpeg: "jpg", gif: "gif", webp: "webp", bmp: "bmp", tiff: "tiff", heic: "heic", avif: "avif", ico: "ico" };
|
|
213
214
|
/** metadata fields that are routinely filled in by cameras and editors and say nothing */
|
|
214
215
|
const ROUTINE_FIELD = /^(?:exif:(?:Make|Model|Software|ImageUniqueID|PrintIM)|png:(?:Software|Creation Time|date:\w+|icc:\w+|Raw profile type \w+)|xmp:(?:xmp:CreatorTool|xmpMM:\w+|photoshop:(?:ICCProfile|ColorMode)|tiff:\w+|exif:\w+|stEvt:\w+|dc:format|pdf:Producer))$/i;
|
|
216
|
+
/**
|
|
217
|
+
* The codes read in a picture or in a video's frames: a finding and (for a code that carries words) a concealed
|
|
218
|
+
* segment for each, and one visible line saying the codes are there. `beside` is the text shown with the code,
|
|
219
|
+
* for the check that the words name one place and the code goes to another. `root` is "image" or "video".
|
|
220
|
+
*/
|
|
221
|
+
export function judgeCodes(barcodes, beside, root) {
|
|
222
|
+
const codes = [];
|
|
223
|
+
const findings = [];
|
|
224
|
+
const segments = [];
|
|
225
|
+
const add = (f) => findings.push({ pass: "channel", action: "none", ...f });
|
|
226
|
+
for (const c of barcodes.slice(0, 20)) {
|
|
227
|
+
const verdict = judgeQrPayload(c.payload);
|
|
228
|
+
codes.push({ symbology: c.symbology, payload: c.payload, verdict });
|
|
229
|
+
const what = `${c.symbology} code ${c.at ? `(on screen at ${c.at}) ` : ""}${verdict.summary}${verdict.notes.length ? ` (${verdict.notes.join("; ")})` : ""}`;
|
|
230
|
+
if (verdict.action)
|
|
231
|
+
add({ kind: "image.code_action", confidence: 0.3 + verdict.risk * 0.4, severity: 0.4 + verdict.risk * 0.4, message: `following this ${what}`, inHidden: true });
|
|
232
|
+
else
|
|
233
|
+
add({ kind: "image.code_payload", confidence: verdict.notes.length ? 0.25 + verdict.risk * 0.3 : 0.05, severity: verdict.notes.length ? 0.5 : 0.2, message: `this ${what}${verdict.url ? ". Open it only through the gate" : ""}`, inHidden: true });
|
|
234
|
+
if (verdict.text)
|
|
235
|
+
segments.push({ text: verdict.text, visibility: "concealed", path: `${root}/code:${verdict.kind}` });
|
|
236
|
+
// the words beside a code say one thing; the code says another
|
|
237
|
+
const named = hostsMentioned(beside, 20);
|
|
238
|
+
if (verdict.host && named.length && !named.some((n) => verdict.host === n || verdict.host.endsWith(`.${n}`) || n.endsWith(`.${verdict.host}`))) {
|
|
239
|
+
add({ kind: "image.caption_mismatch", confidence: 0.45, severity: 0.6, message: `the words beside the code name ${named[0]}, and the code goes to ${verdict.host}`, inHidden: true });
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
// a line for the reader, so the delivered text says a code is there without carrying its payload as an instruction
|
|
243
|
+
if (codes.length)
|
|
244
|
+
segments.push({ text: codes.map((c) => `[${c.symbology} code: ${c.verdict.summary}]`).join("\n"), visibility: "visible", path: `${root}/codes` });
|
|
245
|
+
return { codes, findings, segments };
|
|
246
|
+
}
|
|
215
247
|
export async function inspectImage(bytes, opts = {}) {
|
|
216
248
|
const meta = readImageMeta(bytes);
|
|
217
249
|
const segments = [];
|
|
@@ -320,26 +352,13 @@ export async function inspectImage(bytes, opts = {}) {
|
|
|
320
352
|
segments.push({ text: frameLines.map((f) => f.text).join("\n"), visibility: "concealed", path: "image/later-frames" });
|
|
321
353
|
add({ kind: "image.later_frame_text", confidence: 0.12, severity: 0.4, message: `${wordy(frameLines.map((f) => f.text).join(" "))} words appear only in later frames (${[...new Set(frameLines.map((f) => f.frame))].slice(0, 3).join(", ")}): a still view shows the first frame only`, inHidden: true });
|
|
322
354
|
}
|
|
355
|
+
// interface elements drawn into the picture (a browser window, a system dialog): see screen.ts
|
|
356
|
+
findings.push(...screenFindings(vision.full, vision.rects, opts.screen));
|
|
323
357
|
// codes: the payload is for the machine
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
if (verdict.action)
|
|
329
|
-
add({ kind: "image.code_action", confidence: 0.3 + verdict.risk * 0.4, severity: 0.4 + verdict.risk * 0.4, message: `following this ${what}`, inHidden: true });
|
|
330
|
-
else
|
|
331
|
-
add({ kind: "image.code_payload", confidence: verdict.notes.length ? 0.25 + verdict.risk * 0.3 : 0.05, severity: verdict.notes.length ? 0.5 : 0.2, message: `this ${what}${verdict.url ? ". Open it only through the gate" : ""}`, inHidden: true });
|
|
332
|
-
if (verdict.text)
|
|
333
|
-
segments.push({ text: verdict.text, visibility: "concealed", path: `image/code:${verdict.kind}` });
|
|
334
|
-
// the words beside a code say one thing; the code says another
|
|
335
|
-
const named = hostsMentioned(`${opts.caption ?? ""} ${shown}`, 20);
|
|
336
|
-
if (verdict.host && named.length && !named.some((n) => verdict.host === n || verdict.host.endsWith(`.${n}`) || n.endsWith(`.${verdict.host}`))) {
|
|
337
|
-
add({ kind: "image.caption_mismatch", confidence: 0.45, severity: 0.6, message: `the words beside the code name ${named[0]}, and the code goes to ${verdict.host}`, inHidden: true });
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
// a line for the reader, so the delivered text says a code is there without carrying its payload as an instruction
|
|
341
|
-
if (codes.length)
|
|
342
|
-
segments.push({ text: codes.map((c) => `[${c.symbology} code: ${c.verdict.summary}]`).join("\n"), visibility: "visible", path: "image/codes" });
|
|
358
|
+
const judged = judgeCodes(vision.barcodes, `${opts.caption ?? ""} ${shown}`, "image");
|
|
359
|
+
codes.push(...judged.codes);
|
|
360
|
+
findings.push(...judged.findings);
|
|
361
|
+
segments.push(...judged.segments);
|
|
343
362
|
}
|
|
344
363
|
else {
|
|
345
364
|
add({ kind: "image.pixels_not_inspected", confidence: 0.1, severity: 0.2, message: `the picture itself was not read: ${coverage.pixelsReason}. Text in it, codes, and the resizing, contrast and transparency checks were not run` });
|
package/dist/src/index.d.ts
CHANGED
|
@@ -41,6 +41,10 @@ export { adaptCaptions, parseCaptions, looksLikeCaptions, looksLikeTranscriptJso
|
|
|
41
41
|
export type { Cue, CaptionHints } from "./captions.js";
|
|
42
42
|
export { inspectAudio, readAudioTags, detectAudioFormat, commandRunner } from "./audio.js";
|
|
43
43
|
export type { AudioReport, AudioRunner, AudioCoverage, AudioInspection, AudioFormat } from "./audio.js";
|
|
44
|
+
export { inspectVideo, detectVideoFormat, videoHelperRunner, helperAudioExtractor, cuesToVtt, BRIEF_SECONDS } from "./video.js";
|
|
45
|
+
export type { VideoReport, VideoRunner, VideoCoverage, VideoInspection, VideoFormat, VideoOptions, AudioExtractor } from "./video.js";
|
|
46
|
+
export { drawnElements, screenFindings } from "./screen.js";
|
|
47
|
+
export type { ScreenHints, DrawnElement } from "./screen.js";
|
|
44
48
|
export { adaptNearby, parseNearby, looksLikeNearbyJson, collectNearby, parseDnsSdBrowse, parseBluetoothProfile, parseLpstatPrinters } from "./nearby.js";
|
|
45
49
|
export type { NearbyRecord, NearbyHints } from "./nearby.js";
|
|
46
50
|
export { adaptShortcut, summarizeShortcut, looksLikeShortcut, parsePlistXml } from "./shortcuts.js";
|
package/dist/src/index.js
CHANGED
|
@@ -26,6 +26,8 @@ export { judgeQrPayload } from "./qr.js";
|
|
|
26
26
|
export { adaptMessages, adaptNotifications, parseMessages, parseNotifications, looksLikeMessagesJson, looksLikeNotificationJson } from "./messages.js";
|
|
27
27
|
export { adaptCaptions, parseCaptions, looksLikeCaptions, looksLikeTranscriptJson } from "./captions.js";
|
|
28
28
|
export { inspectAudio, readAudioTags, detectAudioFormat, commandRunner } from "./audio.js";
|
|
29
|
+
export { inspectVideo, detectVideoFormat, videoHelperRunner, helperAudioExtractor, cuesToVtt, BRIEF_SECONDS } from "./video.js";
|
|
30
|
+
export { drawnElements, screenFindings } from "./screen.js";
|
|
29
31
|
export { adaptNearby, parseNearby, looksLikeNearbyJson, collectNearby, parseDnsSdBrowse, parseBluetoothProfile, parseLpstatPrinters } from "./nearby.js";
|
|
30
32
|
export { adaptShortcut, summarizeShortcut, looksLikeShortcut, parsePlistXml } from "./shortcuts.js";
|
|
31
33
|
export { ProvenanceLedger, describeMark, downloadOrigin } from "./provenance.js";
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screens: interface elements that are part of the picture rather than part of the machine.
|
|
3
|
+
*
|
|
4
|
+
* A page can draw a browser window inside itself: a title, an address bar reading some other site's address,
|
|
5
|
+
* a sign-in form underneath ("browser in the browser"). It can draw an operating-system dialog asking for a
|
|
6
|
+
* password. In a picture, a drawn window and a real one are the same pixels, so appearance alone settles
|
|
7
|
+
* nothing, with one exception that the caller has to supply:
|
|
8
|
+
*
|
|
9
|
+
* capture: "viewport" the picture is a capture of a page's content area only (what a browser-automation
|
|
10
|
+
* tool's screenshot is). Real browser chrome and real system dialogs are never in such
|
|
11
|
+
* a picture. An address bar or a system dialog seen in one was painted by the page.
|
|
12
|
+
*
|
|
13
|
+
* Without that, a picture of a whole screen or window with two address bars in it is a pop-up over a browser:
|
|
14
|
+
* usually real. That case is recorded as a note with next to no weight.
|
|
15
|
+
*
|
|
16
|
+
* Even in a viewport capture this is not proof of an attack: a help page with a screenshot of a sign-in window
|
|
17
|
+
* looks the same. So nothing here reaches "hostile" on its own, and it reaches "suspect" on its own only when
|
|
18
|
+
* the caller also says which site the page is on and the drawn address bar names a different one. With other
|
|
19
|
+
* evidence in the same picture (instructions, a paste-this lure, words the page's own source does not have)
|
|
20
|
+
* the ordinary scoring adds them up.
|
|
21
|
+
*
|
|
22
|
+
* Measured on ordinary screenshots: see docs/IMAGES.md ("Interface elements drawn into a page").
|
|
23
|
+
*/
|
|
24
|
+
import type { Finding } from "./types.js";
|
|
25
|
+
export interface ScreenLine {
|
|
26
|
+
text: string;
|
|
27
|
+
conf: number;
|
|
28
|
+
x: number;
|
|
29
|
+
y: number;
|
|
30
|
+
w: number;
|
|
31
|
+
h: number;
|
|
32
|
+
}
|
|
33
|
+
export interface ScreenRect {
|
|
34
|
+
x: number;
|
|
35
|
+
y: number;
|
|
36
|
+
w: number;
|
|
37
|
+
h: number;
|
|
38
|
+
conf?: number;
|
|
39
|
+
}
|
|
40
|
+
export interface ScreenHints {
|
|
41
|
+
/** how the picture was taken, when the caller knows */
|
|
42
|
+
capture?: "viewport" | "window" | "unknown";
|
|
43
|
+
/** the host of the page a viewport capture shows */
|
|
44
|
+
pageHost?: string;
|
|
45
|
+
}
|
|
46
|
+
export interface DrawnElement {
|
|
47
|
+
what: "browser-window" | "system-dialog";
|
|
48
|
+
/** the host the drawn address bar names */
|
|
49
|
+
host?: string;
|
|
50
|
+
/** was the element found inside a rectangle that sits wholly inside the picture? */
|
|
51
|
+
framed: boolean;
|
|
52
|
+
/** does it ask for a password or a sign-in? */
|
|
53
|
+
credentials: boolean;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Interface elements that look drawn into the picture. Pure geometry and wording; what they are worth depends
|
|
57
|
+
* on how the picture was taken (see `screenFindings`).
|
|
58
|
+
*/
|
|
59
|
+
export declare function drawnElements(lines: readonly ScreenLine[], rects?: readonly ScreenRect[]): {
|
|
60
|
+
elements: DrawnElement[];
|
|
61
|
+
topBarHost?: string;
|
|
62
|
+
};
|
|
63
|
+
/** What the drawn elements are worth, given how the picture was taken. */
|
|
64
|
+
export declare function screenFindings(lines: readonly ScreenLine[], rects: readonly ScreenRect[] | undefined, hints?: ScreenHints): Finding[];
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screens: interface elements that are part of the picture rather than part of the machine.
|
|
3
|
+
*
|
|
4
|
+
* A page can draw a browser window inside itself: a title, an address bar reading some other site's address,
|
|
5
|
+
* a sign-in form underneath ("browser in the browser"). It can draw an operating-system dialog asking for a
|
|
6
|
+
* password. In a picture, a drawn window and a real one are the same pixels, so appearance alone settles
|
|
7
|
+
* nothing, with one exception that the caller has to supply:
|
|
8
|
+
*
|
|
9
|
+
* capture: "viewport" the picture is a capture of a page's content area only (what a browser-automation
|
|
10
|
+
* tool's screenshot is). Real browser chrome and real system dialogs are never in such
|
|
11
|
+
* a picture. An address bar or a system dialog seen in one was painted by the page.
|
|
12
|
+
*
|
|
13
|
+
* Without that, a picture of a whole screen or window with two address bars in it is a pop-up over a browser:
|
|
14
|
+
* usually real. That case is recorded as a note with next to no weight.
|
|
15
|
+
*
|
|
16
|
+
* Even in a viewport capture this is not proof of an attack: a help page with a screenshot of a sign-in window
|
|
17
|
+
* looks the same. So nothing here reaches "hostile" on its own, and it reaches "suspect" on its own only when
|
|
18
|
+
* the caller also says which site the page is on and the drawn address bar names a different one. With other
|
|
19
|
+
* evidence in the same picture (instructions, a paste-this lure, words the page's own source does not have)
|
|
20
|
+
* the ordinary scoring adds them up.
|
|
21
|
+
*
|
|
22
|
+
* Measured on ordinary screenshots: see docs/IMAGES.md ("Interface elements drawn into a page").
|
|
23
|
+
*/
|
|
24
|
+
import { siteOf } from "./supplychain.js";
|
|
25
|
+
/** a line that is nothing but an address: what an address bar shows */
|
|
26
|
+
const BAR = /^(?:[^\p{L}\p{N}]{0,3}|(?:not )?secure\s*\|?\s*)?(?:https?:\/\/)?((?:[a-z0-9](?:[a-z0-9-]{0,62})\.)+[a-z]{2,24})(?::\d{2,5})?(?:[/?#]\S*)?$/iu;
|
|
27
|
+
const CREDENTIAL = /\b(?:sign[ -]?in|log[ -]?in|password|passcode|passphrase|email or phone|user ?name|verify (?:it'?s|that it'?s) you|verify your identity|one[- ]time code|verification code|2-step|two[- ]factor|continue with (?:google|apple|microsoft|facebook|github))\b/i;
|
|
28
|
+
const PASSWORD = /\b(?:password|passcode|passphrase)\b/i;
|
|
29
|
+
const SYSTEM_DIALOG = [
|
|
30
|
+
/["“][^"”]{2,40}["”] (?:would like to|wants to|is trying to) (?:access|use|control|make changes|record|receive|send|find|install|modify|add|administer)\b/i,
|
|
31
|
+
/\bdo you want to allow this app to make changes to your (?:device|computer|pc)\b/i,
|
|
32
|
+
/\b(?:macos|system (?:preferences|settings)|software update|installer|keychain(?: access)?|finder|windows security) (?:wants|is trying|needs) to\b/i,
|
|
33
|
+
/\benter (?:your|an administrator'?s?(?: name and)?|the) (?:mac |windows |login |computer )?password to (?:allow|continue|make|unlock)\b/i,
|
|
34
|
+
/\b(?:use )?touch id or enter your password\b/i,
|
|
35
|
+
/\buser account control\b/i,
|
|
36
|
+
];
|
|
37
|
+
const inside = (l, r) => { const cx = l.x + l.w / 2, cy = l.y + l.h / 2; return cx >= r.x && cx <= r.x + r.w && cy >= r.y && cy <= r.y + r.h; };
|
|
38
|
+
/** a rectangle that is an element in the picture, not the picture's own edge and not a speck */
|
|
39
|
+
const element = (r) => r.x > 0.01 && r.y > 0.01 && r.x + r.w < 0.99 && r.y + r.h < 0.99 && r.w * r.h >= 0.03 && r.w * r.h <= 0.8;
|
|
40
|
+
/**
|
|
41
|
+
* Interface elements that look drawn into the picture. Pure geometry and wording; what they are worth depends
|
|
42
|
+
* on how the picture was taken (see `screenFindings`).
|
|
43
|
+
*/
|
|
44
|
+
export function drawnElements(lines, rects = []) {
|
|
45
|
+
const out = [];
|
|
46
|
+
const good = lines.filter((l) => l.conf >= 0.4 && l.text.trim().length >= 2);
|
|
47
|
+
const frames = rects.filter(element).sort((a, b) => a.w * a.h - b.w * b.h);
|
|
48
|
+
const bars = good.map((l) => ({ l, host: BAR.exec(l.text.trim().replace(/\s+/g, ""))?.[1]?.toLowerCase() })).filter((b) => !!b.host && !b.l.text.includes("@") && b.l.text.trim().length <= 140);
|
|
49
|
+
// the bar of the window the picture was taken of, when it shows one: an address in the top strip
|
|
50
|
+
const top = bars.find((b) => b.l.y < 0.1);
|
|
51
|
+
for (const b of bars) {
|
|
52
|
+
if (b.l.y < 0.12)
|
|
53
|
+
continue;
|
|
54
|
+
// the smallest rectangle that has this address in its top quarter: the drawn window
|
|
55
|
+
const frame = frames.find((r) => inside(b.l, r) && b.l.y + b.l.h / 2 - r.y <= r.h * 0.3);
|
|
56
|
+
// what is under the bar: inside that rectangle, or failing one, in a band below the bar
|
|
57
|
+
const below = good.filter((l) => l !== b.l && l.y > b.l.y && (frame ? inside(l, frame) : l.y - b.l.y <= 0.5 && l.x + l.w >= b.l.x - 0.12 && l.x <= b.l.x + b.l.w + 0.12));
|
|
58
|
+
const asks = below.filter((l) => CREDENTIAL.test(l.text));
|
|
59
|
+
// a sign-in form, not a mention: at least two such lines, one of them short (a label or a button)
|
|
60
|
+
if (asks.length < 2 || !asks.some((l) => l.text.trim().split(/\s+/).length <= 4))
|
|
61
|
+
continue;
|
|
62
|
+
if (out.some((e) => e.what === "browser-window" && e.host === b.host))
|
|
63
|
+
continue;
|
|
64
|
+
out.push({ what: "browser-window", host: b.host, framed: !!frame, credentials: true });
|
|
65
|
+
}
|
|
66
|
+
// a system dialog's wording, with its buttons nearby
|
|
67
|
+
const groups = frames.length ? frames.map((r) => good.filter((l) => inside(l, r))).filter((g) => g.length >= 2) : [];
|
|
68
|
+
groups.push([...good]);
|
|
69
|
+
for (const [i, g] of groups.entries()) {
|
|
70
|
+
const text = g.map((l) => l.text).join(" ").replace(/\s+/g, " ");
|
|
71
|
+
if (!SYSTEM_DIALOG.some((re) => re.test(text)))
|
|
72
|
+
continue;
|
|
73
|
+
const buttons = g.filter((l) => /^(?:allow|don[’']t allow|deny|ok|cancel|yes|no|continue|use password…?|unlock|install(?: software)?|always allow)$/i.test(l.text.trim())).length;
|
|
74
|
+
if (buttons < 1)
|
|
75
|
+
continue;
|
|
76
|
+
const framed = i < groups.length - 1;
|
|
77
|
+
if (out.some((e) => e.what === "system-dialog")) {
|
|
78
|
+
if (framed)
|
|
79
|
+
out.find((e) => e.what === "system-dialog").framed = true;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
out.push({ what: "system-dialog", framed, credentials: PASSWORD.test(text) });
|
|
83
|
+
}
|
|
84
|
+
return { elements: out, topBarHost: top?.host };
|
|
85
|
+
}
|
|
86
|
+
/** What the drawn elements are worth, given how the picture was taken. */
|
|
87
|
+
export function screenFindings(lines, rects, hints = {}) {
|
|
88
|
+
const { elements, topBarHost } = drawnElements(lines, rects ?? []);
|
|
89
|
+
const findings = [];
|
|
90
|
+
const add = (f) => findings.push({ pass: "channel", action: "none", ...f });
|
|
91
|
+
const viewport = hints.capture === "viewport";
|
|
92
|
+
const pageSite = hints.pageHost ? siteOf(hints.pageHost) : undefined;
|
|
93
|
+
for (const e of elements) {
|
|
94
|
+
if (e.what === "browser-window") {
|
|
95
|
+
if (viewport) {
|
|
96
|
+
const other = !!pageSite && !!e.host && siteOf(e.host) !== pageSite;
|
|
97
|
+
add({
|
|
98
|
+
kind: "image.drawn_browser_window", confidence: other ? 0.5 : 0.3, severity: other ? 0.65 : 0.6,
|
|
99
|
+
message: `a browser window is drawn inside this page: an address bar reading ${e.host}, with a sign-in form under it${other ? `, on a page served from ${hints.pageHost}` : ""}. A capture of a page's content area never contains the browser's own address bar, so the page painted this one. Whatever is typed into it goes to the page, not to ${e.host}`,
|
|
100
|
+
location: "image/text",
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
else if (topBarHost && e.host && siteOf(topBarHost) !== siteOf(e.host)) {
|
|
104
|
+
add({
|
|
105
|
+
kind: "image.drawn_browser_window", confidence: 0.1, severity: 0.5,
|
|
106
|
+
message: `a second window with its own address bar (${e.host}) and a sign-in form sits inside a window showing ${topBarHost}. A real pop-up and one drawn by the page look the same in a picture: check where the page really is before typing into it`,
|
|
107
|
+
location: "image/text",
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
else if (viewport) {
|
|
112
|
+
add({
|
|
113
|
+
kind: "image.drawn_system_dialog", confidence: e.credentials ? 0.45 : 0.3, severity: e.credentials ? 0.65 : 0.6,
|
|
114
|
+
message: `an operating-system dialog is drawn inside this page${e.credentials ? ", asking for a password" : ""}. A capture of a page's content area never contains a real system dialog, so the page painted it. A page cannot ask for the computer's password or grant itself access`,
|
|
115
|
+
location: "image/text",
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return findings;
|
|
120
|
+
}
|