@acepaste/ingress 0.9.6 → 0.9.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/src/adapters.d.ts +9 -0
  2. package/dist/src/adapters.js +76 -6
  3. package/dist/src/agent-hooks.d.ts +112 -0
  4. package/dist/src/agent-hooks.js +533 -0
  5. package/dist/src/approvers.d.ts +16 -0
  6. package/dist/src/approvers.js +50 -0
  7. package/dist/src/audio.d.ts +70 -0
  8. package/dist/src/audio.js +224 -0
  9. package/dist/src/audit-changes.d.ts +46 -0
  10. package/dist/src/audit-changes.js +110 -0
  11. package/dist/src/audit-local.d.ts +57 -0
  12. package/dist/src/audit-local.js +231 -0
  13. package/dist/src/audit-parsers.d.ts +1 -1
  14. package/dist/src/audit.js +48 -0
  15. package/dist/src/broker.d.ts +92 -1
  16. package/dist/src/broker.js +222 -3
  17. package/dist/src/calendar.d.ts +36 -0
  18. package/dist/src/calendar.js +227 -0
  19. package/dist/src/captions.d.ts +44 -0
  20. package/dist/src/captions.js +209 -0
  21. package/dist/src/changes.d.ts +105 -0
  22. package/dist/src/changes.js +174 -0
  23. package/dist/src/character.js +15 -1
  24. package/dist/src/classifier-model.d.ts +53 -0
  25. package/dist/src/classifier-model.js +3 -0
  26. package/dist/src/classifier.d.ts +41 -0
  27. package/dist/src/classifier.js +213 -0
  28. package/dist/src/cli.js +501 -1
  29. package/dist/src/egress.d.ts +71 -0
  30. package/dist/src/egress.js +242 -0
  31. package/dist/src/email.d.ts +42 -0
  32. package/dist/src/email.js +424 -0
  33. package/dist/src/engine.d.ts +36 -12
  34. package/dist/src/engine.js +88 -37
  35. package/dist/src/image-meta.d.ts +33 -0
  36. package/dist/src/image-meta.js +433 -0
  37. package/dist/src/image.d.ts +110 -0
  38. package/dist/src/image.js +353 -0
  39. package/dist/src/index.d.ts +50 -5
  40. package/dist/src/index.js +28 -3
  41. package/dist/src/localguard.d.ts +33 -0
  42. package/dist/src/localguard.js +61 -0
  43. package/dist/src/meaning.d.ts +2 -0
  44. package/dist/src/meaning.js +77 -22
  45. package/dist/src/messages.d.ts +79 -0
  46. package/dist/src/messages.js +431 -0
  47. package/dist/src/nearby.d.ts +52 -0
  48. package/dist/src/nearby.js +186 -0
  49. package/dist/src/netpolicy.js +50 -19
  50. package/dist/src/provenance.d.ts +56 -0
  51. package/dist/src/provenance.js +114 -0
  52. package/dist/src/qr.d.ts +25 -0
  53. package/dist/src/qr.js +113 -0
  54. package/dist/src/safeguards.d.ts +61 -0
  55. package/dist/src/safeguards.js +199 -0
  56. package/dist/src/served.d.ts +20 -0
  57. package/dist/src/served.js +53 -0
  58. package/dist/src/shell-commands.d.ts +39 -0
  59. package/dist/src/shell-commands.js +291 -0
  60. package/dist/src/shortcuts.d.ts +45 -0
  61. package/dist/src/shortcuts.js +220 -0
  62. package/dist/src/stream.d.ts +88 -0
  63. package/dist/src/stream.js +119 -0
  64. package/dist/src/terminal.d.ts +29 -0
  65. package/dist/src/terminal.js +313 -0
  66. package/dist/src/tooldefs.d.ts +84 -0
  67. package/dist/src/tooldefs.js +122 -0
  68. package/dist/src/types.d.ts +50 -6
  69. package/dist/src/types.js +15 -2
  70. package/dist/src/visibility.d.ts +23 -3
  71. package/dist/src/visibility.js +285 -78
  72. package/native/ingress-presence.swift +48 -0
  73. package/native/ingress-vision.swift +417 -0
  74. package/package.json +3 -1
@@ -21,9 +21,16 @@ import { scanSupplyChain } from "./supplychain.js";
21
21
  import { scanCode } from "./codepass.js";
22
22
  import { scanSecrets, redactSecrets, collectSecrets, scrubExcerpt } from "./secrets.js";
23
23
  import { attachEvidence } from "./evidence.js";
24
- import { adaptJavascript, adaptJson, adaptMarkdown, adaptText, decodeEntities, detectKind } from "./adapters.js";
24
+ import { adaptByKind, decodeEntities, detectKind, KINDS_WITH_MARKUP } from "./adapters.js";
25
+ import { inspectImage } from "./image.js";
26
+ import { detectImageFormat } from "./image-meta.js";
27
+ import { inspectAudio } from "./audio.js";
28
+ const MAX_IMAGE_BYTES = 50_000_000;
29
+ const MAX_AUDIO_BYTES = 200_000_000;
30
+ import { servedWords, compareServed } from "./served.js";
31
+ import { scanClassifier } from "./classifier.js";
25
32
  import { runSandbox } from "./sandbox.js";
26
- import { DEFAULT_CONFIG, ENGINE_VERSION, RULESET_VERSION, } from "./types.js";
33
+ import { DEFAULT_CONFIG, ENGINE_VERSION, RULESET_VERSION, TRUST_SUSPECT_FACTOR, } from "./types.js";
27
34
  const sha256 = (s) => createHash("sha256").update(s, "utf8").digest("hex");
28
35
  export const ASSEMBLED_LIMITS = { fragments: 32, fragmentBytes: 64_000, assembledBytes: 256_000 };
29
36
  export class IngressEngine {
@@ -177,9 +184,58 @@ export class IngressEngine {
177
184
  if (bytes > this.config.maxBodyBytes) {
178
185
  return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 0.5, severity: 0.5, message: `content is ${bytes} bytes, over the ${this.config.maxBodyBytes} byte limit`, action: "none" }], {}, t0, "text");
179
186
  }
180
- const kind = opts.isHtml === true ? "html" : opts.isHtml === false ? "text" : detectKind(opts.contentType ?? "", content, opts.url ?? "");
187
+ const kind = opts.kind ?? (opts.isHtml === true ? "html" : opts.isHtml === false ? "text" : detectKind(opts.contentType ?? "", content, opts.url ?? ""));
181
188
  const rule = url.startsWith("inline:") ? undefined : this.domainRule(url);
182
- return this.guarded(url, kind, t0, () => this.analyze({ url, finalUrl: url, redirectChain: [], raw: content, kind, externalRules: [], timings: {}, t0, override: !!opts.override, extraFindings: [], rule }));
189
+ return this.guarded(url, kind, t0, () => this.analyze({ url, finalUrl: url, redirectChain: [], raw: content, kind, externalRules: [], timings: {}, t0, override: !!opts.override, extraFindings: [], rule,
190
+ // what devices broadcast is the lowest-trust content there is, unless the caller knows better
191
+ source: kind === "nearby" && !opts.source?.trust ? { channel: "nearby devices", ...opts.source, trust: 6 } : opts.source,
192
+ hints: { spoken: opts.spoken, outsideSpeakers: opts.outsideSpeakers, knownDevices: opts.knownDevices } }));
193
+ }
194
+ /**
195
+ * Gate an image: what it says to a person, and what it says only to a machine (see image.ts).
196
+ * The delivered text is the text a person can read in the picture, with a line for each code in it.
197
+ * Whether the picture itself goes on to the model is the caller's decision, made on the verdict.
198
+ */
199
+ async scanImage(bytes, opts = {}) {
200
+ const t0 = performance.now();
201
+ const url = opts.url ?? "inline:image";
202
+ if (opts.quarantined)
203
+ return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 1, severity: 1, message: `source is quarantined by your organization: ${opts.quarantined}`, action: "none" }], {}, t0, "image", "hostile", true);
204
+ if (bytes.length > MAX_IMAGE_BYTES)
205
+ return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 0.5, severity: 0.5, message: `image is ${bytes.length} bytes, over the ${MAX_IMAGE_BYTES} byte limit`, action: "none" }], {}, t0, "image");
206
+ const rule = url.startsWith("inline:") ? undefined : this.domainRule(url);
207
+ return this.guarded(url, "image", t0, async () => {
208
+ // an SVG is markup: its text, hidden or shown, is read as a page's is
209
+ if (detectImageFormat(bytes) === "svg") {
210
+ const text = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
211
+ const extra = /<script\b|\son[a-z]+\s*=|<foreignObject\b|javascript:/i.test(text)
212
+ ? [{ pass: "channel", kind: "image.active_content", confidence: 0.5, severity: 0.6, message: "this SVG carries scripts, event handlers or embedded HTML: it is a program as well as a picture", action: "none" }]
213
+ : [];
214
+ return this.analyze({ url, finalUrl: url, redirectChain: [], raw: text, kind: "html", externalRules: [], timings: {}, t0, override: !!opts.override, extraFindings: extra, rule, source: opts.source });
215
+ }
216
+ const ti = performance.now();
217
+ const insp = await inspectImage(bytes, { vision: opts.vision, caption: opts.caption, sourceText: opts.sourceText });
218
+ return this.analyze({ url, finalUrl: url, redirectChain: [], raw: insp.report, kind: "image", externalRules: [], timings: { visibility: performance.now() - ti }, t0, override: !!opts.override, extraFindings: insp.findings, rule, source: opts.source, adapted: { segments: insp.segments, findings: [] }, imageCoverage: insp.coverage });
219
+ });
220
+ }
221
+ /**
222
+ * Gate a sound recording: what is said in it, and what is there only for a machine (see audio.ts).
223
+ * The delivered text is the transcript of what a listener hears. `captions`, when given, are compared with
224
+ * what was said.
225
+ */
226
+ async scanAudio(bytes, opts = {}) {
227
+ const t0 = performance.now();
228
+ const url = opts.url ?? "inline:audio";
229
+ if (opts.quarantined)
230
+ return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 1, severity: 1, message: `source is quarantined by your organization: ${opts.quarantined}`, action: "none" }], {}, t0, "audio", "hostile", true);
231
+ if (bytes.length > MAX_AUDIO_BYTES)
232
+ return this.refuse(url, [{ pass: "policy", kind: "policy.denied_source", confidence: 0.5, severity: 0.5, message: `recording is ${bytes.length} bytes, over the ${MAX_AUDIO_BYTES} byte limit`, action: "none" }], {}, t0, "audio");
233
+ const rule = url.startsWith("inline:") ? undefined : this.domainRule(url);
234
+ return this.guarded(url, "audio", t0, async () => {
235
+ const ti = performance.now();
236
+ const insp = await inspectAudio(bytes, { transcriber: opts.transcriber });
237
+ return this.analyze({ url, finalUrl: url, redirectChain: [], raw: insp.report, kind: "audio", externalRules: [], timings: { visibility: performance.now() - ti }, t0, override: !!opts.override, extraFindings: insp.findings, rule, source: opts.source, adapted: { segments: insp.segments, findings: [] }, audioCoverage: insp.coverage });
238
+ });
183
239
  }
184
240
  /**
185
241
  * Gate what the model will actually receive when several sources are combined (search
@@ -222,7 +278,7 @@ export class IngressEngine {
222
278
  }
223
279
  /** Cheap structural pre-check: pathological nesting makes the HTML parser super-linear. Refuse before parsing. */
224
280
  structuralAbuse(raw, kind) {
225
- if (kind !== "html")
281
+ if (!KINDS_WITH_MARKUP.has(kind))
226
282
  return null;
227
283
  // Depth is what the parser would build, not a count of open tags: <li>, <p>, <dd>, <tr> and friends close
228
284
  // implicitly, a closing tag closes everything still open inside it, and script, style and comment bodies are
@@ -296,9 +352,10 @@ export class IngressEngine {
296
352
  const abuse = this.structuralAbuse(a.raw, a.kind);
297
353
  if (abuse)
298
354
  return this.refuse(a.url, [{ pass: "policy", kind: "policy.denied_source", confidence: 0.8, severity: 0.7, message: `structural abuse: ${abuse}`, action: "none" }], timings, a.t0, a.kind);
299
- // entity decoding for non-HTML kinds (parse5 decodes for HTML)
355
+ // entity decoding for kinds without markup of their own (parse5 decodes for HTML, and for the HTML inside
356
+ // an email or an event: decoding first would turn text a message quotes, "&lt;div hidden&gt;", into a tag)
300
357
  let raw = a.raw;
301
- if (a.kind !== "html") {
358
+ if (!KINDS_WITH_MARKUP.has(a.kind) && !a.adapted) {
302
359
  const dec = decodeEntities(raw);
303
360
  if (dec.suspicious > 0) {
304
361
  findings.push({ pass: "character", kind: "unicode.entity_smuggling", confidence: Math.min(0.95, 0.6 + dec.suspicious * 0.05), severity: 0.7, message: `${dec.suspicious} HTML entities decoding to invisible or format characters`, excerpt: dec.sample, action: "neutralized" });
@@ -318,7 +375,7 @@ export class IngressEngine {
318
375
  findings.push(...v.findings);
319
376
  }
320
377
  else {
321
- const r = a.kind === "markdown" ? adaptMarkdown(raw) : a.kind === "json" ? adaptJson(raw) : a.kind === "javascript" ? adaptJavascript(raw) : adaptText(raw);
378
+ const r = a.adapted ?? adaptByKind(a.kind, raw, a.hints);
322
379
  segments = r.segments;
323
380
  findings.push(...r.findings);
324
381
  if (a.kind === "javascript")
@@ -356,7 +413,11 @@ export class IngressEngine {
356
413
  }
357
414
  // character pass
358
415
  const tch = performance.now();
359
- const charAll = scanCharacters(segments.map((s) => s.text).join("\n"));
416
+ // Text recognised in a picture has no characters of its own: the recogniser chose them, and on small print it
417
+ // mixes alphabets freely. Lookalike-letter findings there say something about the recogniser, not the author.
418
+ // The characters that are really in an image file are its metadata, its codes' payloads and appended text.
419
+ const authored = a.kind === "image" ? segments.filter((s) => /^image\/(?:metadata|code|after-end)/.test(s.path)) : a.kind === "audio" ? segments.filter((s) => s.path.startsWith("audio/metadata")) : segments;
420
+ const charAll = scanCharacters(authored.map((s) => s.text).join("\n"));
360
421
  findings.push(...charAll.findings);
361
422
  let cleanedSegments = segments.map((s) => ({ ...s, text: scanCharacters(s.text).cleaned }));
362
423
  timings.character = performance.now() - tch;
@@ -411,9 +472,19 @@ export class IngressEngine {
411
472
  passesRun.push("code");
412
473
  // meaning pass over cleaned segments and decoded payloads
413
474
  const tm = performance.now();
414
- findings.push(...scanMeaning({ segments: cleanedSegments, decodedPayloads: charAll.decodedPayloads }));
475
+ findings.push(...scanMeaning({ kind: a.kind, segments: cleanedSegments, decodedPayloads: charAll.decodedPayloads }));
415
476
  timings.meaning = performance.now() - tm;
416
477
  passesRun.push("meaning");
478
+ // learned pass: what the delivered text and each hidden passage resemble
479
+ if (this.config.classifier) {
480
+ const tcl = performance.now();
481
+ // what the model was trained on is a message's subject and body: its address lines (From, To, Date) are
482
+ // not prose, and a line of addresses is not evidence of anything
483
+ const deliveredText = cleanedSegments.filter((s) => (s.visibility === "visible" || (s.visibility === "attribute" && s.path.endsWith("@alt"))) && !/\/header:(from|to|cc|reply-to|date)$|\/sender$|\/speaker$/.test(s.path)).map((s) => s.text).join("\n");
484
+ findings.push(...scanClassifier({ visible: deliveredText, segments: cleanedSegments, kind: a.kind }));
485
+ timings.classifier = performance.now() - tcl;
486
+ passesRun.push("classifier");
487
+ }
417
488
  // plugins
418
489
  if (this.config.detectors.length) {
419
490
  const tp = performance.now();
@@ -446,7 +517,9 @@ export class IngressEngine {
446
517
  // score and tier (per-domain thresholds override)
447
518
  const score = aggregate(findings, { cloakingCompared: passesRun.includes("cloaking") });
448
519
  const hostileAt = a.rule?.hostileThreshold ?? this.config.hostileThreshold;
449
- const suspectAt = a.rule?.suspectThreshold ?? this.config.suspectThreshold;
520
+ // content from strangers, open channels and broadcasts needs less evidence to be refused; trust never loosens the default
521
+ const trust = a.source?.trust;
522
+ const suspectAt = (a.rule?.suspectThreshold ?? this.config.suspectThreshold) * (trust ? TRUST_SUSPECT_FACTOR[trust] ?? 1 : 1);
450
523
  const tier = score >= hostileAt ? "hostile" : score >= suspectAt ? "suspect" : "clean";
451
524
  // delivered text: clean passes; suspect passes only under human override; hostile never
452
525
  const overridden = tier === "suspect" && a.override;
@@ -459,12 +532,12 @@ export class IngressEngine {
459
532
  delivered = neutralizeMeaning(delivered);
460
533
  delivered = scrubExcerpt(delivered, knownSecrets);
461
534
  }
462
- const receipt = this.receipt({ url: a.url, finalUrl: a.finalUrl, redirectChain: a.redirectChain, kind: a.kind, raw: a.raw, delivered, tier, score, findings, overridden, passesRun, acepasteInvisibleCount: countHostileInvisibles(a.raw) });
535
+ const receipt = this.receipt({ url: a.url, finalUrl: a.finalUrl, redirectChain: a.redirectChain, kind: a.kind, raw: a.raw, delivered, tier, score, findings, overridden, passesRun, acepasteInvisibleCount: countHostileInvisibles(a.raw), trust });
463
536
  const envelope = buildEnvelope(a.finalUrl, tier, delivered, findings, receipt, overridden);
464
537
  timings.total = performance.now() - a.t0;
465
538
  if (tier !== "clean")
466
539
  this.deps.log(buildLogEntry(a.finalUrl, tier, score, findings, safePayloads, receipt, overridden));
467
- return { tier, score, contentKind: a.kind, delivered, envelope, findings, receipt, timings, overridden, coverage: { code: codeScan.coverage } };
540
+ return { tier, score, contentKind: a.kind, delivered, envelope, findings, receipt, timings, overridden, coverage: { code: codeScan.coverage, ...(a.imageCoverage ? { image: a.imageCoverage } : {}), ...(a.audioCoverage ? { audio: a.audioCoverage } : {}) } };
468
541
  }
469
542
  async fetchStylesheets(html, baseUrl) {
470
543
  const rules = [];
@@ -549,6 +622,7 @@ export class IngressEngine {
549
622
  tier: r.tier, score: +r.score.toFixed(4),
550
623
  overridden: r.overridden,
551
624
  passesRun: r.passesRun,
625
+ trust: r.trust, // omitted from the canonical body when the caller gave none, so older receipts verify unchanged
552
626
  findings: r.findings.map(({ pass, kind, confidence, severity, action, evidenceClass, band, artifactDigest }) => ({ pass, kind, confidence: +confidence.toFixed(3), severity, action, evidenceClass, band, artifactDigest })),
553
627
  previous: this.config.chainReceipts ? this.lastReceiptDigest : undefined,
554
628
  signatureAlg: ecdsa ? "ecdsa-p256-sha256" : this.config.signingKey ? "hmac-sha256" : undefined,
@@ -748,30 +822,7 @@ export function aggregate(findings, ctx = {}) {
748
822
  score = Math.min(1, score + 0.1);
749
823
  return +score.toFixed(4);
750
824
  }
751
- /** words of served text with markup, link targets and bare URLs removed, so Markdown and HTML of the same page compare equal */
752
- export function servedWords(text) {
753
- const t = text
754
- .replace(/!?\[([^\]]*)\]\([^)]*\)/g, " $1 ") // [label](target) and ![alt](src) -> label
755
- .replace(/<\/?[a-z][^>]*>/gi, " ") // stray tags
756
- .replace(/\b(?:https?:\/\/|www\.)\S+/gi, " ") // bare URLs
757
- .replace(/[#>*_`~|\-]{1,}/g, " "); // markdown syntax
758
- return t.toLowerCase().split(/[^\p{L}\p{N}']+/u).filter((w) => w.length > 2);
759
- }
760
- /** how much of what the agent received is absent from what the browser received (word occurrences, not distinct words) */
761
- export function compareServed(agentText, humanText) {
762
- const A = servedWords(agentText);
763
- const H = new Set(servedWords(humanText));
764
- const only = A.filter((w) => !H.has(w));
765
- let run = 0, longestAgentOnlyRun = 0;
766
- for (const w of A) {
767
- run = H.has(w) ? 0 : run + 1;
768
- if (run > longestAgentOnlyRun)
769
- longestAgentOnlyRun = run;
770
- }
771
- const sampleSeen = new Set();
772
- const sample = only.filter((w) => (sampleSeen.has(w) ? false : (sampleSeen.add(w), true))).slice(0, 30).join(" ");
773
- return { agentWords: A.length, agentOnlyWords: only.length, agentOnlyShare: A.length ? only.length / A.length : 0, longestAgentOnlyRun, sample };
774
- }
825
+ export { servedWords, compareServed };
775
826
  /**
776
827
  * Spotlighting-style envelope. Random boundary so content cannot forge the
777
828
  * closing marker. The model is told, in one line, how to treat the content.
@@ -0,0 +1,33 @@
1
+ /**
2
+ * What an image file carries besides its picture: text in metadata, data after the end of the image, extra
3
+ * frames. Pure parsing, no dependencies, any platform. The pixels themselves are read by the vision helper
4
+ * (image.ts); this module reads the container.
5
+ *
6
+ * Why it matters: a model, or the code in front of it, may be handed a file's description, comment or XMP
7
+ * packet along with the picture. A person looking at the picture is shown none of it.
8
+ */
9
+ export type ImageFormat = "png" | "jpeg" | "gif" | "webp" | "bmp" | "tiff" | "heic" | "avif" | "ico" | "svg" | "pdf" | "unknown";
10
+ export interface ImageMeta {
11
+ format: ImageFormat;
12
+ width?: number;
13
+ height?: number;
14
+ /** frames or pages, when the container says (animated PNG, GIF) */
15
+ frames?: number;
16
+ /** text found in the container: where, and what it says */
17
+ texts: Array<{
18
+ field: string;
19
+ text: string;
20
+ }>;
21
+ /** bytes after the image's own end marker */
22
+ trailingBytes: number;
23
+ /** what the trailing bytes look like, when they look like another file */
24
+ trailingLooksLike?: string;
25
+ /** an embedded preview picture (EXIF thumbnail): a second image nobody is shown */
26
+ hasThumbnail: boolean;
27
+ notes: string[];
28
+ }
29
+ export declare function detectImageFormat(b: Uint8Array): ImageFormat;
30
+ /** What a run of bytes looks like, when it looks like the start of some other kind of file. */
31
+ export declare function looksLike(b: Uint8Array): string | undefined;
32
+ /** Read an image file's container. Never throws: a file that cannot be parsed is reported as such. */
33
+ export declare function readImageMeta(bytes: Uint8Array): ImageMeta;