@acepaste/ingress 0.9.26 → 0.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli.js +65 -5
- package/dist/src/engine.d.ts +20 -1
- package/dist/src/engine.js +25 -3
- package/dist/src/image.d.ts +38 -0
- package/dist/src/image.js +45 -26
- package/dist/src/index.d.ts +4 -0
- package/dist/src/index.js +2 -0
- package/dist/src/screen.d.ts +64 -0
- package/dist/src/screen.js +120 -0
- package/dist/src/types.d.ts +5 -4
- package/dist/src/types.js +3 -3
- package/dist/src/video.d.ts +156 -0
- package/dist/src/video.js +421 -0
- package/native/ingress-vision.swift +543 -4
- package/package.json +1 -1
package/dist/src/types.d.ts
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
* at the same source would not see is suspect by default.
|
|
6
6
|
*/
|
|
7
7
|
import type { NetPolicy } from "./netpolicy.js";
|
|
8
|
-
export declare const ENGINE_VERSION = "0.9.
|
|
8
|
+
export declare const ENGINE_VERSION = "0.9.27";
|
|
9
9
|
/** bump whenever a detector's behavior changes; receipts carry it so verdicts are reproducible */
|
|
10
|
-
export declare const RULESET_VERSION = "2026.11.
|
|
10
|
+
export declare const RULESET_VERSION = "2026.11.20-1";
|
|
11
11
|
/**
|
|
12
12
|
* clean: delivered. suspect: refused, overridable. hostile: refused, not overridable.
|
|
13
13
|
* unreachable: nothing could be fetched or read (network error, timeout, redirect loop,
|
|
@@ -18,7 +18,7 @@ export type Tier = "clean" | "suspect" | "hostile" | "unreachable";
|
|
|
18
18
|
* What the content is. The first five are document formats; the rest are channels, each with its own adapter
|
|
19
19
|
* that turns the channel's structure into segments the passes read (see docs/ROADMAP.md, shared foundations).
|
|
20
20
|
*/
|
|
21
|
-
export type ContentKind = "html" | "markdown" | "json" | "text" | "javascript" | "log" | "email" | "calendar" | "message" | "notification" | "nearby" | "automation" | "captions" | "image" | "audio";
|
|
21
|
+
export type ContentKind = "html" | "markdown" | "json" | "text" | "javascript" | "log" | "email" | "calendar" | "message" | "notification" | "nearby" | "automation" | "captions" | "image" | "audio" | "video";
|
|
22
22
|
/**
|
|
23
23
|
* Who could have written the content. Lower is more trusted. It never makes a verdict more lenient than the
|
|
24
24
|
* default; from level 4 down, less evidence is needed to refuse.
|
|
@@ -50,7 +50,7 @@ export declare function parseScanHints(o: {
|
|
|
50
50
|
/** multiplier on the suspect threshold per trust level: strangers and broadcasts need less evidence */
|
|
51
51
|
export declare const TRUST_SUSPECT_FACTOR: Record<TrustLevel, number>;
|
|
52
52
|
export type PassName = "acquire" | "policy" | "visibility" | "character" | "script" | "supplychain" | "secrets" | "meaning" | "cloaking" | "sandbox" | "code" | "channel" | "classifier" | "plugin";
|
|
53
|
-
export type FindingKind = "policy.denied_source" | "policy.redirect_chain" | "acquire.unreachable" | "acquire.cloaking_check_incomplete" | "acquire.ai_agent_refused" | "acquire.browser_view_used" | "unicode.tag_characters" | "unicode.zero_width" | "unicode.bidi_control" | "unicode.variation_selector_smuggling" | "unicode.mixed_script_confusable" | "unicode.fullwidth_or_math_lookalike" | "unicode.combining_flood" | "unicode.encoded_payload" | "unicode.entity_smuggling" | "unicode.whitespace_channel" | "visibility.css_hidden" | "visibility.offscreen" | "visibility.attribute_only" | "visibility.html_comment" | "visibility.tiny_or_transparent" | "visibility.markdown_hidden" | "visibility.terminal_hidden" | "visibility.terminal_control" | "email.part_divergence" | "email.sender_mismatch" | "email.authentication_failed" | "email.remote_content" | "email.uninspected_part" | "calendar.alarm_action" | "calendar.description_divergence" | "message.text_divergence" | "message.sender_impersonation" | "message.scam_shape" | "message.verification_code" | "message.uninspected_part" | "notification.source_mismatch" | "nearby.authority_name" | "nearby.lookalike" | "nearby.link" | "nearby.oversized_field" | "automation.private_outside_send" | "automation.runs_code" | "automation.destination" | "automation.unreadable" | "image.clickfix" | "image.screen_not_in_source" | "captions.hidden_cue" | "captions.unreadable_cue" | "captions.not_spoken" | "captions.addressed_to_notetaker" | "captions.spoken_approval" | "audio.quiet_speech" | "audio.transcript_unstable" | "audio.metadata_text" | "audio.not_transcribed" | "image.text_only_when_resized" | "image.low_contrast_text" | "image.transparency_text" | "image.later_frame_text" | "image.metadata_text" | "image.trailing_data" | "image.active_content" | "image.code_payload" | "image.code_action" | "image.caption_mismatch" | "image.pixels_not_inspected" | "classifier.injection_likely" | "script.obfuscation" | "script.runtime_codegen" | "script.automation_detection" | "script.post_load_text_injection" | "script.exfil_beacon" | "script.binary_decoding" | "supplychain.lifecycle_script" | "supplychain.url_dependency" | "supplychain.offregistry_resolution" | "supplychain.install_instruction" | "secrets.credential_in_content" | "sandbox.text_injected_after_load" | "sandbox.webdriver_keyed_divergence" | "sandbox.computed_hidden_text" | "sandbox.blocked_egress_attempt" | "meaning.instruction_shaped" | "meaning.role_token" | "meaning.system_prompt_imitation" | "meaning.tool_call_shaped" | "meaning.exfil_markdown_image" | "meaning.exfil_link" | "meaning.adversarial_suffix" | "cloaking.agent_targeted_divergence" | "code.download_execute" | "code.encoded_execution" | "code.reverse_shell" | "code.secret_exfiltration" | "code.persistence" | `plugin.${string}`;
|
|
53
|
+
export type FindingKind = "policy.denied_source" | "policy.redirect_chain" | "acquire.unreachable" | "acquire.cloaking_check_incomplete" | "acquire.ai_agent_refused" | "acquire.browser_view_used" | "unicode.tag_characters" | "unicode.zero_width" | "unicode.bidi_control" | "unicode.variation_selector_smuggling" | "unicode.mixed_script_confusable" | "unicode.fullwidth_or_math_lookalike" | "unicode.combining_flood" | "unicode.encoded_payload" | "unicode.entity_smuggling" | "unicode.whitespace_channel" | "visibility.css_hidden" | "visibility.offscreen" | "visibility.attribute_only" | "visibility.html_comment" | "visibility.tiny_or_transparent" | "visibility.markdown_hidden" | "visibility.terminal_hidden" | "visibility.terminal_control" | "email.part_divergence" | "email.sender_mismatch" | "email.authentication_failed" | "email.remote_content" | "email.uninspected_part" | "calendar.alarm_action" | "calendar.description_divergence" | "message.text_divergence" | "message.sender_impersonation" | "message.scam_shape" | "message.verification_code" | "message.uninspected_part" | "notification.source_mismatch" | "nearby.authority_name" | "nearby.lookalike" | "nearby.link" | "nearby.oversized_field" | "automation.private_outside_send" | "automation.runs_code" | "automation.destination" | "automation.unreadable" | "image.clickfix" | "image.screen_not_in_source" | "captions.hidden_cue" | "captions.unreadable_cue" | "captions.not_spoken" | "captions.addressed_to_notetaker" | "captions.spoken_approval" | "audio.quiet_speech" | "audio.transcript_unstable" | "audio.metadata_text" | "audio.not_transcribed" | "image.text_only_when_resized" | "image.low_contrast_text" | "image.transparency_text" | "image.later_frame_text" | "image.metadata_text" | "image.trailing_data" | "image.active_content" | "image.code_payload" | "image.code_action" | "image.caption_mismatch" | "image.pixels_not_inspected" | "image.drawn_browser_window" | "image.drawn_system_dialog" | "video.brief_frame_text" | "video.metadata_text" | "video.frames_not_read" | "video.partly_read" | "classifier.injection_likely" | "script.obfuscation" | "script.runtime_codegen" | "script.automation_detection" | "script.post_load_text_injection" | "script.exfil_beacon" | "script.binary_decoding" | "supplychain.lifecycle_script" | "supplychain.url_dependency" | "supplychain.offregistry_resolution" | "supplychain.install_instruction" | "secrets.credential_in_content" | "sandbox.text_injected_after_load" | "sandbox.webdriver_keyed_divergence" | "sandbox.computed_hidden_text" | "sandbox.blocked_egress_attempt" | "meaning.instruction_shaped" | "meaning.role_token" | "meaning.system_prompt_imitation" | "meaning.tool_call_shaped" | "meaning.exfil_markdown_image" | "meaning.exfil_link" | "meaning.adversarial_suffix" | "cloaking.agent_targeted_divergence" | "code.download_execute" | "code.encoded_execution" | "code.reverse_shell" | "code.secret_exfiltration" | "code.persistence" | `plugin.${string}`;
|
|
54
54
|
export interface Finding {
|
|
55
55
|
pass: PassName;
|
|
56
56
|
kind: FindingKind;
|
|
@@ -191,6 +191,7 @@ export interface IngressResult {
|
|
|
191
191
|
code?: import("./codepass.js").CodeCoverage;
|
|
192
192
|
image?: import("./image.js").ImageCoverage;
|
|
193
193
|
audio?: import("./audio.js").AudioCoverage;
|
|
194
|
+
video?: import("./video.js").VideoCoverage;
|
|
194
195
|
};
|
|
195
196
|
}
|
|
196
197
|
export interface IngressConfig {
|
package/dist/src/types.js
CHANGED
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* Governing principle: anything the model would receive that a human looking
|
|
5
5
|
* at the same source would not see is suspect by default.
|
|
6
6
|
*/
|
|
7
|
-
export const ENGINE_VERSION = "0.9.
|
|
7
|
+
export const ENGINE_VERSION = "0.9.27";
|
|
8
8
|
/** bump whenever a detector's behavior changes; receipts carry it so verdicts are reproducible */
|
|
9
|
-
export const RULESET_VERSION = "2026.11.
|
|
10
|
-
export const CONTENT_KINDS = ["html", "markdown", "json", "text", "javascript", "log", "email", "calendar", "message", "notification", "nearby", "automation", "captions", "image", "audio"];
|
|
9
|
+
export const RULESET_VERSION = "2026.11.20-1";
|
|
10
|
+
export const CONTENT_KINDS = ["html", "markdown", "json", "text", "javascript", "log", "email", "calendar", "message", "notification", "nearby", "automation", "captions", "image", "audio", "video"];
|
|
11
11
|
/** Read a content kind and source from untyped input (an HTTP body, tool arguments). Unknown values are dropped, never guessed. */
|
|
12
12
|
export function parseScanHints(o) {
|
|
13
13
|
const kind = typeof o.kind === "string" && CONTENT_KINDS.includes(o.kind) ? o.kind : undefined;
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Video: what a viewer sees, and what is in the file only for a machine.
|
|
3
|
+
*
|
|
4
|
+
* A person watches a video once, at its own speed. A model is handed frames somebody sampled, the subtitle
|
|
5
|
+
* track as text, and a transcript of the sound. Each is a place for words the viewer never took in:
|
|
6
|
+
*
|
|
7
|
+
* on screen too briefly a frame or two of text between ordinary pictures. Nobody reads a fifteenth of a
|
|
8
|
+
* second; a model handed that frame reads all of it
|
|
9
|
+
* in a code a QR code that flashes by, or sits in a corner
|
|
10
|
+
* in the subtitle track cues styled invisible or shown too briefly to read; captions that say what was
|
|
11
|
+
* neither said nor shown (the caption adapter, captions.ts)
|
|
12
|
+
* in the sound speech too quiet to make out (audio.ts), when a transcriber is configured
|
|
13
|
+
* in the file's tags a description or comment nobody watching sees
|
|
14
|
+
*
|
|
15
|
+
* Reading frames needs the vision helper (native/ingress-vision.swift, macOS: AVFoundation and Vision). It
|
|
16
|
+
* looks at every frame as a small grey thumbnail to find where the picture changes, then reads the text in
|
|
17
|
+
* the frames that matter: every frame of a short clip; for a long one, each picture that is on screen for
|
|
18
|
+
* under half a second, the frame after each cut, and a regular grid. It reports how many frames there are and
|
|
19
|
+
* how many it read, and so does the result: a partial read is never silent.
|
|
20
|
+
*
|
|
21
|
+
* Limits, stated plainly:
|
|
22
|
+
* - Containers: what AVFoundation opens (MP4, MOV, M4V). WebM and Matroska are not read, and the result says so.
|
|
23
|
+
* - Text in a frame is read as shown. The picture checks for faint text and resizing tricks are not run on
|
|
24
|
+
* every frame.
|
|
25
|
+
* - In a long video, text that appears without changing the picture enough to register (small print over
|
|
26
|
+
* moving footage) can fall between the frames that were read.
|
|
27
|
+
* - A camera frame is a picture: send it to scanImage. There is no camera reader here.
|
|
28
|
+
* - No public set of video prompt-injection attacks was found to measure recall on; see docs/VIDEO.md.
|
|
29
|
+
*/
|
|
30
|
+
import { type VisionLine, type ImageInspection } from "./image.js";
|
|
31
|
+
import { type AudioRunner } from "./audio.js";
|
|
32
|
+
import type { Finding, TextSegment } from "./types.js";
|
|
33
|
+
export type VideoFormat = "mp4" | "mov" | "webm" | "matroska" | "avi" | "unknown";
|
|
34
|
+
export interface VideoFrame {
|
|
35
|
+
index: number;
|
|
36
|
+
t: number;
|
|
37
|
+
from: number;
|
|
38
|
+
to: number;
|
|
39
|
+
reading: number;
|
|
40
|
+
}
|
|
41
|
+
export interface VideoReport {
|
|
42
|
+
version?: number;
|
|
43
|
+
/** false when no frame could be decoded */
|
|
44
|
+
decoded?: boolean;
|
|
45
|
+
width: number;
|
|
46
|
+
height: number;
|
|
47
|
+
/** seconds */
|
|
48
|
+
duration: number;
|
|
49
|
+
fps: number;
|
|
50
|
+
examinedSeconds: number;
|
|
51
|
+
frameCount: number;
|
|
52
|
+
sampling: "every frame" | "selected" | "";
|
|
53
|
+
framesRead: number;
|
|
54
|
+
/** pictures on screen for under half a second (fast cuts and movement count), and how many of them were read */
|
|
55
|
+
shortShots: number;
|
|
56
|
+
shortShotsRead: number;
|
|
57
|
+
/** of those, pictures put in between two frames of the same scene: what hidden frames look like */
|
|
58
|
+
inserts?: number;
|
|
59
|
+
insertsRead?: number;
|
|
60
|
+
gridStep: number;
|
|
61
|
+
frames: VideoFrame[];
|
|
62
|
+
readings: Array<{
|
|
63
|
+
index: number;
|
|
64
|
+
t: number;
|
|
65
|
+
lines: VisionLine[];
|
|
66
|
+
codes: Array<{
|
|
67
|
+
symbology: string;
|
|
68
|
+
payload: string;
|
|
69
|
+
variant?: string;
|
|
70
|
+
}>;
|
|
71
|
+
}>;
|
|
72
|
+
tracks: Array<{
|
|
73
|
+
type: string;
|
|
74
|
+
codec: string;
|
|
75
|
+
language: string;
|
|
76
|
+
}>;
|
|
77
|
+
subtitles: Array<{
|
|
78
|
+
start: number;
|
|
79
|
+
end: number;
|
|
80
|
+
text: string;
|
|
81
|
+
track: number;
|
|
82
|
+
}>;
|
|
83
|
+
metadata: Array<{
|
|
84
|
+
field: string;
|
|
85
|
+
text: string;
|
|
86
|
+
}>;
|
|
87
|
+
notes: string[];
|
|
88
|
+
ms?: number;
|
|
89
|
+
}
|
|
90
|
+
/** Reads a video file. The default runs the vision helper; tests and other platforms supply their own. */
|
|
91
|
+
export type VideoRunner = (path: string, o: {
|
|
92
|
+
maxFrames: number;
|
|
93
|
+
maxSeconds: number;
|
|
94
|
+
}) => Promise<VideoReport>;
|
|
95
|
+
/** Writes the video's sound to `wavPath` as a WAV file a transcriber can read. */
|
|
96
|
+
export type AudioExtractor = (videoPath: string, wavPath: string, maxSeconds: number) => Promise<{
|
|
97
|
+
ok: boolean;
|
|
98
|
+
seconds?: number;
|
|
99
|
+
reason?: string;
|
|
100
|
+
truncated?: boolean;
|
|
101
|
+
}>;
|
|
102
|
+
export interface VideoCoverage {
|
|
103
|
+
format: string;
|
|
104
|
+
/** were the frames read (text and codes on screen, and how long each was shown)? */
|
|
105
|
+
frames: boolean;
|
|
106
|
+
/** why not, when they were not */
|
|
107
|
+
framesReason?: string;
|
|
108
|
+
seconds?: number;
|
|
109
|
+
examinedSeconds?: number;
|
|
110
|
+
framesTotal?: number;
|
|
111
|
+
framesRead?: number;
|
|
112
|
+
sampling?: string;
|
|
113
|
+
/** cues read from the file's subtitle tracks (and a captions file given beside it) */
|
|
114
|
+
subtitleCues: number;
|
|
115
|
+
audio: "transcribed" | "not transcribed" | "no audio track" | "unknown";
|
|
116
|
+
audioReason?: string;
|
|
117
|
+
/** were the captions compared with what was said and shown? */
|
|
118
|
+
captionsCompared: boolean;
|
|
119
|
+
notes: string[];
|
|
120
|
+
}
|
|
121
|
+
export interface VideoInspection {
|
|
122
|
+
format: VideoFormat;
|
|
123
|
+
video?: VideoReport;
|
|
124
|
+
codes: ImageInspection["codes"];
|
|
125
|
+
segments: TextSegment[];
|
|
126
|
+
findings: Finding[];
|
|
127
|
+
coverage: VideoCoverage;
|
|
128
|
+
/** a stable description of what was found, hashed into the receipt */
|
|
129
|
+
report: string;
|
|
130
|
+
}
|
|
131
|
+
export interface VideoOptions {
|
|
132
|
+
/** how to read the frames; `false` skips them; default: the vision helper when it is built */
|
|
133
|
+
video?: VideoRunner | false;
|
|
134
|
+
/** how to transcribe the sound (default: the command in INGRESS_AUDIO_TRANSCRIBER); `false` skips it */
|
|
135
|
+
transcriber?: AudioRunner | false;
|
|
136
|
+
/** how to get the sound out of the file (default: the vision helper) */
|
|
137
|
+
extractAudio?: AudioExtractor;
|
|
138
|
+
/** a captions file that came with the video (WebVTT or SubRip), read like a subtitle track */
|
|
139
|
+
captions?: string;
|
|
140
|
+
/** at most this many frames are read for text (default 240) */
|
|
141
|
+
maxFrames?: number;
|
|
142
|
+
/** at most this much of the video is examined (default 600 seconds) */
|
|
143
|
+
maxSeconds?: number;
|
|
144
|
+
}
|
|
145
|
+
/** Text on screen for no longer than this cannot be read by a viewer. */
|
|
146
|
+
export declare const BRIEF_SECONDS = 0.34;
|
|
147
|
+
export declare function detectVideoFormat(b: Uint8Array): VideoFormat;
|
|
148
|
+
export declare function videoHelperRunner(path?: string, timeoutMs?: number): VideoRunner;
|
|
149
|
+
export declare function helperAudioExtractor(path?: string, timeoutMs?: number): AudioExtractor;
|
|
150
|
+
/** The file's subtitle cues as a WebVTT document, so the caption adapter reads them as it reads any captions file. */
|
|
151
|
+
export declare function cuesToVtt(cues: ReadonlyArray<{
|
|
152
|
+
start: number;
|
|
153
|
+
end: number;
|
|
154
|
+
text: string;
|
|
155
|
+
}>): string;
|
|
156
|
+
export declare function inspectVideo(bytes: Uint8Array, opts?: VideoOptions): Promise<VideoInspection>;
|
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Video: what a viewer sees, and what is in the file only for a machine.
|
|
3
|
+
*
|
|
4
|
+
* A person watches a video once, at its own speed. A model is handed frames somebody sampled, the subtitle
|
|
5
|
+
* track as text, and a transcript of the sound. Each is a place for words the viewer never took in:
|
|
6
|
+
*
|
|
7
|
+
* on screen too briefly a frame or two of text between ordinary pictures. Nobody reads a fifteenth of a
|
|
8
|
+
* second; a model handed that frame reads all of it
|
|
9
|
+
* in a code a QR code that flashes by, or sits in a corner
|
|
10
|
+
* in the subtitle track cues styled invisible or shown too briefly to read; captions that say what was
|
|
11
|
+
* neither said nor shown (the caption adapter, captions.ts)
|
|
12
|
+
* in the sound speech too quiet to make out (audio.ts), when a transcriber is configured
|
|
13
|
+
* in the file's tags a description or comment nobody watching sees
|
|
14
|
+
*
|
|
15
|
+
* Reading frames needs the vision helper (native/ingress-vision.swift, macOS: AVFoundation and Vision). It
|
|
16
|
+
* looks at every frame as a small grey thumbnail to find where the picture changes, then reads the text in
|
|
17
|
+
* the frames that matter: every frame of a short clip; for a long one, each picture that is on screen for
|
|
18
|
+
* under half a second, the frame after each cut, and a regular grid. It reports how many frames there are and
|
|
19
|
+
* how many it read, and so does the result: a partial read is never silent.
|
|
20
|
+
*
|
|
21
|
+
* Limits, stated plainly:
|
|
22
|
+
* - Containers: what AVFoundation opens (MP4, MOV, M4V). WebM and Matroska are not read, and the result says so.
|
|
23
|
+
* - Text in a frame is read as shown. The picture checks for faint text and resizing tricks are not run on
|
|
24
|
+
* every frame.
|
|
25
|
+
* - In a long video, text that appears without changing the picture enough to register (small print over
|
|
26
|
+
* moving footage) can fall between the frames that were read.
|
|
27
|
+
* - A camera frame is a picture: send it to scanImage. There is no camera reader here.
|
|
28
|
+
* - No public set of video prompt-injection attacks was found to measure recall on; see docs/VIDEO.md.
|
|
29
|
+
*/
|
|
30
|
+
import { execFile } from "node:child_process";
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
32
|
+
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
33
|
+
import { tmpdir } from "node:os";
|
|
34
|
+
import { join } from "node:path";
|
|
35
|
+
import { visionHelperPath, visionHelperProblem, judgeCodes, norm, similar, wordy, enough, sentence, trigrams, containment } from "./image.js";
|
|
36
|
+
import { inspectAudio, commandRunner } from "./audio.js";
|
|
37
|
+
import { adaptCaptions } from "./captions.js";
|
|
38
|
+
/** Text on screen for no longer than this cannot be read by a viewer. */
|
|
39
|
+
export const BRIEF_SECONDS = 0.34;
|
|
40
|
+
const MAX_SCREEN_TEXT = 200_000;
|
|
41
|
+
const ascii = (b, from, to) => { let s = ""; for (let i = from; i < to && i < b.length; i++)
|
|
42
|
+
s += String.fromCharCode(b[i]); return s; };
|
|
43
|
+
export function detectVideoFormat(b) {
|
|
44
|
+
const box = ascii(b, 4, 8);
|
|
45
|
+
if (box === "ftyp")
|
|
46
|
+
return /^qt/.test(ascii(b, 8, 12)) ? "mov" : "mp4";
|
|
47
|
+
if (["moov", "mdat", "wide", "free", "skip", "pnot"].includes(box))
|
|
48
|
+
return "mov";
|
|
49
|
+
if (b.length > 4 && b[0] === 0x1a && b[1] === 0x45 && b[2] === 0xdf && b[3] === 0xa3)
|
|
50
|
+
return /webm/.test(ascii(b, 0, 64)) ? "webm" : "matroska";
|
|
51
|
+
if (ascii(b, 0, 4) === "RIFF" && ascii(b, 8, 12) === "AVI ")
|
|
52
|
+
return "avi";
|
|
53
|
+
return "unknown";
|
|
54
|
+
}
|
|
55
|
+
export function videoHelperRunner(path = visionHelperPath(), timeoutMs = 300_000) {
|
|
56
|
+
return (file, o) => new Promise((resolve, reject) => {
|
|
57
|
+
execFile(path, ["--video", file, "--max-frames", String(o.maxFrames), "--max-seconds", String(o.maxSeconds)], { timeout: timeoutMs, maxBuffer: 256 * 1024 * 1024, encoding: "utf8" }, (err, stdout, stderr) => {
|
|
58
|
+
if (err)
|
|
59
|
+
return reject(new Error(/usage: ingress-vision/.test(String(stderr)) ? "the vision helper on this machine was built before it could read video (run: ingress vision build)" : `the vision helper failed (${String(err.message).slice(0, 120)})`));
|
|
60
|
+
try {
|
|
61
|
+
const r = JSON.parse(stdout);
|
|
62
|
+
if (!Array.isArray(r.frames) || !Array.isArray(r.readings) || !Array.isArray(r.tracks))
|
|
63
|
+
throw new Error("unexpected report");
|
|
64
|
+
resolve(r);
|
|
65
|
+
}
|
|
66
|
+
catch (e) {
|
|
67
|
+
reject(new Error(`the vision helper's report could not be read (${e.message})`));
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
export function helperAudioExtractor(path = visionHelperPath(), timeoutMs = 300_000) {
|
|
73
|
+
return (video, wav, maxSeconds) => new Promise((resolve) => {
|
|
74
|
+
execFile(path, ["--extract-audio", video, wav, "--max-seconds", String(maxSeconds)], { timeout: timeoutMs, maxBuffer: 1024 * 1024, encoding: "utf8" }, (err, stdout) => {
|
|
75
|
+
if (err)
|
|
76
|
+
return resolve({ ok: false, reason: `the sound could not be taken out of the video (${String(err.message).slice(0, 120)})` });
|
|
77
|
+
try {
|
|
78
|
+
const r = JSON.parse(stdout);
|
|
79
|
+
resolve({ ok: r.ok === true, seconds: r.seconds, reason: r.reason, truncated: r.truncated });
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
resolve({ ok: false, reason: "the helper's answer about the sound could not be read" });
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
const stamp = (s) => (Number.isFinite(s) ? `${String(Math.floor(s / 60)).padStart(2, "0")}:${(s % 60).toFixed(1).padStart(4, "0")}` : "?");
|
|
88
|
+
const vttTime = (s) => { const t = Math.max(0, Number.isFinite(s) ? s : 0); const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), sec = t % 60; return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${sec.toFixed(3).padStart(6, "0")}`; };
|
|
89
|
+
/** The file's subtitle cues as a WebVTT document, so the caption adapter reads them as it reads any captions file. */
|
|
90
|
+
export function cuesToVtt(cues) {
|
|
91
|
+
return "WEBVTT\n\n" + cues.map((c) => `${vttTime(c.start)} --> ${vttTime(Math.max(c.end, c.start))}\n${c.text.replace(/-->/g, "->").replace(/\n{2,}/g, "\n").trim()}`).join("\n\n") + "\n";
|
|
92
|
+
}
|
|
93
|
+
function runsOf(frames, readings) {
|
|
94
|
+
const runs = [];
|
|
95
|
+
let open = [];
|
|
96
|
+
frames.forEach((f, p) => {
|
|
97
|
+
const lines = readings[f.reading]?.lines ?? [];
|
|
98
|
+
const next = [];
|
|
99
|
+
const byKey = new Map(open.map((r) => [r.key, r]));
|
|
100
|
+
const unmatched = [];
|
|
101
|
+
for (const l of lines) {
|
|
102
|
+
const text = l.text.trim();
|
|
103
|
+
const key = norm(text);
|
|
104
|
+
if (!key)
|
|
105
|
+
continue;
|
|
106
|
+
const r = byKey.get(key);
|
|
107
|
+
if (r && !next.includes(r)) {
|
|
108
|
+
r.end = p;
|
|
109
|
+
r.conf = Math.max(r.conf, l.conf);
|
|
110
|
+
next.push(r);
|
|
111
|
+
}
|
|
112
|
+
else
|
|
113
|
+
unmatched.push(l);
|
|
114
|
+
}
|
|
115
|
+
// the recogniser reads the same line a little differently from frame to frame
|
|
116
|
+
const rest = open.filter((r) => !next.includes(r));
|
|
117
|
+
for (const l of unmatched) {
|
|
118
|
+
const text = l.text.trim();
|
|
119
|
+
const r = rest.find((o) => !next.includes(o) && similar(o.text, text) >= 0.8);
|
|
120
|
+
if (r) {
|
|
121
|
+
r.end = p;
|
|
122
|
+
r.conf = Math.max(r.conf, l.conf);
|
|
123
|
+
next.push(r);
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
const fresh = { text, key: norm(text), conf: l.conf, start: p, end: p };
|
|
127
|
+
runs.push(fresh);
|
|
128
|
+
next.push(fresh);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
open = next;
|
|
132
|
+
});
|
|
133
|
+
return runs;
|
|
134
|
+
}
|
|
135
|
+
/** tags that players and encoders fill in and that say nothing */
|
|
136
|
+
const SAYS_SOMETHING = /descr|comment|subject|lyric|synops|keyword|©cmt|©des|ldes|\binfo|caption|summary|notes?$/i;
|
|
137
|
+
export async function inspectVideo(bytes, opts = {}) {
|
|
138
|
+
const format = detectVideoFormat(bytes);
|
|
139
|
+
const segments = [];
|
|
140
|
+
const findings = [];
|
|
141
|
+
const codes = [];
|
|
142
|
+
const coverage = { format, frames: false, subtitleCues: 0, audio: "unknown", captionsCompared: false, notes: [] };
|
|
143
|
+
const add = (f) => findings.push({ pass: "channel", action: "none", ...f });
|
|
144
|
+
const maxFrames = Math.max(8, Math.min(2000, opts.maxFrames ?? 240));
|
|
145
|
+
const maxSeconds = Math.max(1, Math.min(7200, opts.maxSeconds ?? 600));
|
|
146
|
+
let video;
|
|
147
|
+
let speech = "";
|
|
148
|
+
const dir = mkdtempSync(join(tmpdir(), "ingress-video-"));
|
|
149
|
+
try {
|
|
150
|
+
const file = join(dir, `video.${format === "mov" ? "mov" : format === "unknown" ? "bin" : format === "matroska" ? "mkv" : format}`);
|
|
151
|
+
if (format === "unknown")
|
|
152
|
+
coverage.framesReason = "not a recognised video format";
|
|
153
|
+
else if (format === "webm" || format === "matroska" || format === "avi")
|
|
154
|
+
coverage.framesReason = `${format === "avi" ? "AVI" : format === "webm" ? "WebM" : "Matroska"} is not read: the helper reads MP4, MOV and M4V`;
|
|
155
|
+
else if (opts.video === false)
|
|
156
|
+
coverage.framesReason = "reading the frames was turned off";
|
|
157
|
+
else {
|
|
158
|
+
const problem = opts.video ? undefined : visionHelperProblem();
|
|
159
|
+
if (problem)
|
|
160
|
+
coverage.framesReason = problem.replace("reading pixels", "reading video");
|
|
161
|
+
else {
|
|
162
|
+
try {
|
|
163
|
+
writeFileSync(file, bytes, { mode: 0o600 });
|
|
164
|
+
const got = await (opts.video || videoHelperRunner())(file, { maxFrames, maxSeconds });
|
|
165
|
+
coverage.notes.push(...(got.notes ?? []));
|
|
166
|
+
if (got.decoded === false)
|
|
167
|
+
coverage.framesReason = got.notes?.[0] ?? "the video could not be decoded";
|
|
168
|
+
else {
|
|
169
|
+
video = got;
|
|
170
|
+
coverage.frames = true;
|
|
171
|
+
}
|
|
172
|
+
// tags and subtitle tracks are read even when no frame could be decoded
|
|
173
|
+
if (!video && (got.metadata?.length || got.subtitles?.length || got.tracks?.length))
|
|
174
|
+
video = { ...got, frames: [], readings: [] };
|
|
175
|
+
}
|
|
176
|
+
catch (e) {
|
|
177
|
+
coverage.framesReason = e.message;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
// ----- the sound -----
|
|
182
|
+
const hasAudio = video?.tracks.some((t) => t.type === "audio");
|
|
183
|
+
if (video && !hasAudio)
|
|
184
|
+
coverage.audio = "no audio track";
|
|
185
|
+
else if (video && hasAudio) {
|
|
186
|
+
const transcriber = opts.transcriber === false ? undefined : opts.transcriber ?? commandRunner();
|
|
187
|
+
if (opts.transcriber === false) {
|
|
188
|
+
coverage.audio = "not transcribed";
|
|
189
|
+
coverage.audioReason = "transcription was turned off";
|
|
190
|
+
}
|
|
191
|
+
else if (!transcriber) {
|
|
192
|
+
coverage.audio = "not transcribed";
|
|
193
|
+
coverage.audioReason = "no transcriber is configured (set INGRESS_AUDIO_TRANSCRIBER; see tools/audio)";
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
const wav = join(dir, "sound.wav");
|
|
197
|
+
const got = await (opts.extractAudio ?? helperAudioExtractor())(file, wav, maxSeconds);
|
|
198
|
+
if (!got.ok) {
|
|
199
|
+
coverage.audio = "not transcribed";
|
|
200
|
+
coverage.audioReason = got.reason ?? "the sound could not be taken out of the video";
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
if (got.truncated)
|
|
204
|
+
coverage.notes.push(`only the first ${Math.round(maxSeconds)} seconds of sound were transcribed`);
|
|
205
|
+
const heard = await inspectAudio(readFileSync(wav), { transcriber });
|
|
206
|
+
if (heard.coverage.transcribed) {
|
|
207
|
+
coverage.audio = "transcribed";
|
|
208
|
+
speech = heard.segments.filter((s) => s.path === "audio/speech").map((s) => s.text).join("\n");
|
|
209
|
+
segments.push(...heard.segments);
|
|
210
|
+
findings.push(...heard.findings);
|
|
211
|
+
coverage.notes.push(...heard.coverage.notes);
|
|
212
|
+
}
|
|
213
|
+
else {
|
|
214
|
+
coverage.audio = "not transcribed";
|
|
215
|
+
coverage.audioReason = heard.coverage.reason ?? "the transcriber gave nothing";
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (coverage.audio === "not transcribed")
|
|
220
|
+
add({ kind: "audio.not_transcribed", confidence: 0.1, severity: 0.2, message: `the video's sound was not transcribed: ${coverage.audioReason}. What is said in it was not checked` });
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
finally {
|
|
224
|
+
try {
|
|
225
|
+
rmSync(dir, { recursive: true, force: true });
|
|
226
|
+
}
|
|
227
|
+
catch { /* temp folder */ }
|
|
228
|
+
}
|
|
229
|
+
let screenText = "";
|
|
230
|
+
if (video && coverage.frames) {
|
|
231
|
+
coverage.seconds = video.duration;
|
|
232
|
+
coverage.examinedSeconds = video.examinedSeconds;
|
|
233
|
+
coverage.framesTotal = video.frameCount;
|
|
234
|
+
coverage.framesRead = video.framesRead;
|
|
235
|
+
coverage.sampling = video.sampling;
|
|
236
|
+
const frames = [...video.frames].sort((a, b) => a.t - b.t);
|
|
237
|
+
const frameDur = video.frameCount > 0 && video.examinedSeconds > 0 ? video.examinedSeconds / video.frameCount : video.fps > 0 ? 1 / video.fps : 0.04;
|
|
238
|
+
const runs = runsOf(frames, video.readings);
|
|
239
|
+
/**
|
|
240
|
+
* The longest a run's line can have been on screen: the time between the frame read before it and the frame
|
|
241
|
+
* read after it, neither of which has it. Only frames that were read count as evidence. When every frame was
|
|
242
|
+
* read this is exact; when frames were selected, a line is called brief only if the frames right before and
|
|
243
|
+
* right after were read too (the helper reads them for every short-lived picture it chooses).
|
|
244
|
+
*/
|
|
245
|
+
const shownFor = (start, end) => {
|
|
246
|
+
const before = start > 0 ? frames[start - 1].t : -frameDur;
|
|
247
|
+
const after = end + 1 < frames.length ? frames[end + 1].t : video.examinedSeconds;
|
|
248
|
+
return Math.max(frames[end].t - frames[start].t + frameDur, after - before - frameDur);
|
|
249
|
+
};
|
|
250
|
+
const textAt = (p) => (p >= 0 && p < frames.length ? (video.readings[frames[p].reading]?.lines ?? []).map((l) => l.text).join("\n") : "");
|
|
251
|
+
// The same line read again a moment later (the recogniser missed it in a frame, or it blinks) is one showing.
|
|
252
|
+
// Same words anywhere in the video: one family. Nearly the same words within a second: the same family too.
|
|
253
|
+
const families = new Map();
|
|
254
|
+
let recent = [];
|
|
255
|
+
for (const r of [...runs].sort((a, b) => a.start - b.start)) {
|
|
256
|
+
const t = frames[r.start].t;
|
|
257
|
+
recent = recent.filter((x) => t - (frames[x.last.end].t + frameDur) <= 1);
|
|
258
|
+
let fam = families.get(r.key);
|
|
259
|
+
if (!fam)
|
|
260
|
+
fam = recent.find((x) => Math.abs(x.last.key.length - r.key.length) <= Math.max(3, r.key.length * 0.25) && similar(x.last.key, r.key) >= 0.8)?.fam;
|
|
261
|
+
if (fam)
|
|
262
|
+
fam.push(r);
|
|
263
|
+
else {
|
|
264
|
+
fam = [r];
|
|
265
|
+
families.set(r.key, fam);
|
|
266
|
+
}
|
|
267
|
+
recent.push({ fam, last: r });
|
|
268
|
+
if (recent.length > 400)
|
|
269
|
+
recent = recent.slice(-400);
|
|
270
|
+
}
|
|
271
|
+
const brief = new Set();
|
|
272
|
+
for (const fam of families.values()) {
|
|
273
|
+
const spans = [];
|
|
274
|
+
for (const r of fam) {
|
|
275
|
+
const last = spans[spans.length - 1];
|
|
276
|
+
if (last && frames[r.start].t - (frames[last.end].t + frameDur) <= 0.5)
|
|
277
|
+
last.end = Math.max(last.end, r.end);
|
|
278
|
+
else
|
|
279
|
+
spans.push({ start: r.start, end: r.end });
|
|
280
|
+
}
|
|
281
|
+
// Only lines read clearly count: on moving footage the recogniser reads a sign in the background a little
|
|
282
|
+
// differently in every frame, at middling confidence, and each reading would look like words that came and went.
|
|
283
|
+
if (spans.every((s) => shownFor(s.start, s.end) <= BRIEF_SECONDS))
|
|
284
|
+
for (const r of fam)
|
|
285
|
+
if (r.conf >= 0.8)
|
|
286
|
+
brief.add(r);
|
|
287
|
+
}
|
|
288
|
+
// Not hidden: a line that is the beginning of one that stayed (captions that build up word by word), and a
|
|
289
|
+
// line that the frames on either side also have in some reading of it.
|
|
290
|
+
const stayed = runs.filter((r) => !brief.has(r));
|
|
291
|
+
for (const r of [...brief]) {
|
|
292
|
+
const near = stayed.filter((s) => Math.abs(frames[s.start].t - frames[r.start].t) <= 3 || (frames[s.start].t <= frames[r.start].t && frames[s.end].t >= frames[r.start].t));
|
|
293
|
+
if (near.length && containment(r.text, trigrams(near.map((s) => s.text).join("\n"))) >= 0.8) {
|
|
294
|
+
brief.delete(r);
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
if (containment(r.text, trigrams(`${textAt(r.start - 1)}\n${textAt(r.end + 1)}`)) >= 0.5)
|
|
298
|
+
brief.delete(r);
|
|
299
|
+
}
|
|
300
|
+
// passages: the brief lines that appeared together. A passage is a message, not scraps (see image.ts).
|
|
301
|
+
const byStart = new Map();
|
|
302
|
+
for (const r of brief)
|
|
303
|
+
byStart.set(r.start, [...(byStart.get(r.start) ?? []), r]);
|
|
304
|
+
const hidden = [];
|
|
305
|
+
for (const [p, rs] of [...byStart.entries()].sort((a, b) => a[0] - b[0])) {
|
|
306
|
+
const texts = rs.map((r) => r.text);
|
|
307
|
+
if (!enough(texts) || !sentence(texts)) {
|
|
308
|
+
rs.forEach((r) => brief.delete(r));
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
hidden.push({ at: frames[p].t, text: texts.join("\n"), seconds: Math.max(...rs.map((r) => shownFor(r.start, r.end))) });
|
|
312
|
+
}
|
|
313
|
+
// what a viewer can read, in the order it appears
|
|
314
|
+
let total = 0, dropped = false;
|
|
315
|
+
const said = new Set();
|
|
316
|
+
const shown = [];
|
|
317
|
+
frames.forEach((f, p) => {
|
|
318
|
+
const fresh = runs.filter((r) => r.start === p && !brief.has(r) && !said.has(r.key));
|
|
319
|
+
if (!fresh.length)
|
|
320
|
+
return;
|
|
321
|
+
fresh.forEach((r) => said.add(r.key));
|
|
322
|
+
const text = fresh.map((r) => r.text).join("\n");
|
|
323
|
+
if (total + text.length > MAX_SCREEN_TEXT) {
|
|
324
|
+
dropped = true;
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
total += text.length;
|
|
328
|
+
shown.push(text);
|
|
329
|
+
segments.push({ text, visibility: "visible", path: `video/text@${stamp(f.t)}` });
|
|
330
|
+
});
|
|
331
|
+
screenText = shown.join("\n");
|
|
332
|
+
if (dropped)
|
|
333
|
+
coverage.notes.push(`more than ${MAX_SCREEN_TEXT} characters of text are shown on screen: the rest was not read`);
|
|
334
|
+
hidden.forEach((h) => segments.push({ text: h.text, visibility: "concealed", path: `video/brief-text@${stamp(h.at)}` }));
|
|
335
|
+
if (hidden.length) {
|
|
336
|
+
add({
|
|
337
|
+
kind: "video.brief_frame_text", confidence: 0.3, severity: 0.5, inHidden: true,
|
|
338
|
+
message: `${wordy(hidden.map((h) => h.text).join(" "))} words are on screen for under a third of a second (${hidden.slice(0, 3).map((h) => `at ${stamp(h.at)} for ${h.seconds.toFixed(2)} s`).join("; ")}): a viewer cannot read them, and a model handed that frame reads all of them`,
|
|
339
|
+
excerpt: hidden[0].text.replace(/\s+/g, " ").slice(0, 160),
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
// codes in the frames
|
|
343
|
+
const seenCodes = new Map();
|
|
344
|
+
for (const r of video.readings)
|
|
345
|
+
for (const c of r.codes ?? [])
|
|
346
|
+
if (!seenCodes.has(c.payload))
|
|
347
|
+
seenCodes.set(c.payload, { symbology: c.symbology, payload: c.payload, at: stamp(r.t) });
|
|
348
|
+
const judged = judgeCodes([...seenCodes.values()], screenText.slice(0, 20_000), "video");
|
|
349
|
+
codes.push(...judged.codes);
|
|
350
|
+
findings.push(...judged.findings);
|
|
351
|
+
segments.push(...judged.segments);
|
|
352
|
+
// a partial read is said out loud
|
|
353
|
+
const partial = [];
|
|
354
|
+
if (video.duration > video.examinedSeconds + 1)
|
|
355
|
+
partial.push(`only the first ${Math.round(video.examinedSeconds)} of ${Math.round(video.duration)} seconds were examined`);
|
|
356
|
+
// Pictures put in between two frames of the same scene are what hidden frames look like: any left unread is
|
|
357
|
+
// a gap. Other short-lived pictures (fast cuts, movement) are sampled, and the coverage says how many.
|
|
358
|
+
const inserts = video.inserts ?? 0, insertsRead = video.insertsRead ?? inserts;
|
|
359
|
+
if (inserts > insertsRead)
|
|
360
|
+
partial.push(`${inserts - insertsRead} of ${inserts} pictures put in between two frames of the same scene were not read`);
|
|
361
|
+
if (video.shortShots > video.shortShotsRead)
|
|
362
|
+
coverage.notes.push(`${video.shortShots} pictures are on screen for under half a second (fast cuts and movement count); ${video.shortShotsRead} of them were read`);
|
|
363
|
+
if (dropped)
|
|
364
|
+
partial.push("the text on screen was longer than is read");
|
|
365
|
+
if (partial.length)
|
|
366
|
+
add({ kind: "video.partly_read", confidence: 0.12, severity: 0.25, message: `the video was only partly read: ${partial.join("; ")}. What is in the rest was not checked` });
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
add({ kind: "video.frames_not_read", confidence: 0.1, severity: 0.2, message: `the video's frames were not read: ${coverage.framesReason}. Text and codes on screen, and how long they are shown, were not checked` });
|
|
370
|
+
}
|
|
371
|
+
if (video) {
|
|
372
|
+
// the file's tags
|
|
373
|
+
const tagged = [];
|
|
374
|
+
for (const t of video.metadata ?? []) {
|
|
375
|
+
if (!SAYS_SOMETHING.test(t.field) && wordy(t.text) < 12)
|
|
376
|
+
continue;
|
|
377
|
+
segments.push({ text: t.text, visibility: "concealed", path: `video/metadata:${t.field.slice(0, 60)}` });
|
|
378
|
+
if (wordy(t.text) >= 4)
|
|
379
|
+
tagged.push(t.field.slice(0, 40));
|
|
380
|
+
}
|
|
381
|
+
if (tagged.length)
|
|
382
|
+
add({ kind: "video.metadata_text", confidence: 0.1, severity: 0.2, message: `the file's tags hold text nobody watching sees (${tagged.slice(0, 4).join(", ")})`, inHidden: true });
|
|
383
|
+
}
|
|
384
|
+
// ----- captions: the file's subtitle tracks, and a captions file that came with it -----
|
|
385
|
+
const spokenAndShown = speech.trim().split(/\s+/).length >= 20 ? `${speech}\n${screenText}` : undefined;
|
|
386
|
+
const captionSources = [];
|
|
387
|
+
if (video?.subtitles?.length) {
|
|
388
|
+
captionSources.push(cuesToVtt(video.subtitles));
|
|
389
|
+
coverage.subtitleCues += video.subtitles.length;
|
|
390
|
+
}
|
|
391
|
+
if (opts.captions?.trim())
|
|
392
|
+
captionSources.push(opts.captions);
|
|
393
|
+
for (const [i, raw] of captionSources.entries()) {
|
|
394
|
+
const adapted = adaptCaptions(raw, { spoken: spokenAndShown });
|
|
395
|
+
if (i > 0 || !video?.subtitles?.length)
|
|
396
|
+
coverage.subtitleCues += adapted.segments.filter((s) => /\/text$/.test(s.path)).length;
|
|
397
|
+
segments.push(...adapted.segments);
|
|
398
|
+
findings.push(...adapted.findings);
|
|
399
|
+
}
|
|
400
|
+
coverage.captionsCompared = captionSources.length > 0 && !!spokenAndShown;
|
|
401
|
+
if (captionSources.length && !spokenAndShown)
|
|
402
|
+
coverage.notes.push("the captions were not compared with what is said: the sound was not transcribed");
|
|
403
|
+
// one line for the reader on what was and was not read
|
|
404
|
+
const parts = [];
|
|
405
|
+
if (coverage.frames && video)
|
|
406
|
+
parts.push(`${stamp(video.duration)} long, ${video.frameCount} frames, ${video.framesRead} read for text${video.sampling === "selected" ? " (selected; see coverage)" : ""}`);
|
|
407
|
+
else
|
|
408
|
+
parts.push(`its frames were not read (${coverage.framesReason})`);
|
|
409
|
+
if (coverage.audio === "not transcribed")
|
|
410
|
+
parts.push(`its sound was not transcribed (${coverage.audioReason})`);
|
|
411
|
+
if (coverage.audio === "transcribed")
|
|
412
|
+
parts.push("its sound was transcribed");
|
|
413
|
+
if (coverage.subtitleCues)
|
|
414
|
+
parts.push(`${coverage.subtitleCues} caption cue(s) read`);
|
|
415
|
+
segments.push({ text: `[video, ${format}: ${parts.join("; ")}]`, visibility: "visible", path: "video/note" });
|
|
416
|
+
const report = JSON.stringify({
|
|
417
|
+
sha256: createHash("sha256").update(bytes).digest("hex"), format, seconds: video?.duration, frames: video?.frameCount, framesRead: video?.framesRead, sampling: video?.sampling,
|
|
418
|
+
audio: coverage.audio, segments: segments.map((s) => ({ path: s.path, visibility: s.visibility, text: s.text })), codes: codes.map((c) => ({ symbology: c.symbology, kind: c.verdict.kind, payload: c.payload })),
|
|
419
|
+
});
|
|
420
|
+
return { format, video, codes, segments, findings, coverage, report };
|
|
421
|
+
}
|