@acepaste/ingress 0.9.6 → 0.9.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/src/adapters.d.ts +9 -0
  2. package/dist/src/adapters.js +76 -6
  3. package/dist/src/agent-hooks.d.ts +112 -0
  4. package/dist/src/agent-hooks.js +533 -0
  5. package/dist/src/approvers.d.ts +16 -0
  6. package/dist/src/approvers.js +50 -0
  7. package/dist/src/audio.d.ts +70 -0
  8. package/dist/src/audio.js +224 -0
  9. package/dist/src/audit-changes.d.ts +46 -0
  10. package/dist/src/audit-changes.js +110 -0
  11. package/dist/src/audit-local.d.ts +57 -0
  12. package/dist/src/audit-local.js +231 -0
  13. package/dist/src/audit-parsers.d.ts +1 -1
  14. package/dist/src/audit.js +48 -0
  15. package/dist/src/broker.d.ts +92 -1
  16. package/dist/src/broker.js +222 -3
  17. package/dist/src/calendar.d.ts +36 -0
  18. package/dist/src/calendar.js +227 -0
  19. package/dist/src/captions.d.ts +44 -0
  20. package/dist/src/captions.js +209 -0
  21. package/dist/src/changes.d.ts +105 -0
  22. package/dist/src/changes.js +174 -0
  23. package/dist/src/character.js +15 -1
  24. package/dist/src/classifier-model.d.ts +53 -0
  25. package/dist/src/classifier-model.js +3 -0
  26. package/dist/src/classifier.d.ts +41 -0
  27. package/dist/src/classifier.js +213 -0
  28. package/dist/src/cli.js +501 -1
  29. package/dist/src/egress.d.ts +71 -0
  30. package/dist/src/egress.js +242 -0
  31. package/dist/src/email.d.ts +42 -0
  32. package/dist/src/email.js +424 -0
  33. package/dist/src/engine.d.ts +35 -12
  34. package/dist/src/engine.js +88 -37
  35. package/dist/src/image-meta.d.ts +33 -0
  36. package/dist/src/image-meta.js +433 -0
  37. package/dist/src/image.d.ts +104 -0
  38. package/dist/src/image.js +347 -0
  39. package/dist/src/index.d.ts +50 -5
  40. package/dist/src/index.js +28 -3
  41. package/dist/src/localguard.d.ts +33 -0
  42. package/dist/src/localguard.js +61 -0
  43. package/dist/src/meaning.d.ts +2 -0
  44. package/dist/src/meaning.js +77 -22
  45. package/dist/src/messages.d.ts +79 -0
  46. package/dist/src/messages.js +431 -0
  47. package/dist/src/nearby.d.ts +52 -0
  48. package/dist/src/nearby.js +186 -0
  49. package/dist/src/netpolicy.js +50 -19
  50. package/dist/src/provenance.d.ts +56 -0
  51. package/dist/src/provenance.js +114 -0
  52. package/dist/src/qr.d.ts +25 -0
  53. package/dist/src/qr.js +113 -0
  54. package/dist/src/safeguards.d.ts +61 -0
  55. package/dist/src/safeguards.js +199 -0
  56. package/dist/src/served.d.ts +20 -0
  57. package/dist/src/served.js +53 -0
  58. package/dist/src/shell-commands.d.ts +39 -0
  59. package/dist/src/shell-commands.js +291 -0
  60. package/dist/src/shortcuts.d.ts +45 -0
  61. package/dist/src/shortcuts.js +220 -0
  62. package/dist/src/stream.d.ts +88 -0
  63. package/dist/src/stream.js +119 -0
  64. package/dist/src/terminal.d.ts +29 -0
  65. package/dist/src/terminal.js +313 -0
  66. package/dist/src/tooldefs.d.ts +84 -0
  67. package/dist/src/tooldefs.js +122 -0
  68. package/dist/src/types.d.ts +50 -6
  69. package/dist/src/types.js +15 -2
  70. package/dist/src/visibility.d.ts +23 -3
  71. package/dist/src/visibility.js +285 -78
  72. package/native/ingress-presence.swift +48 -0
  73. package/native/ingress-vision.swift +417 -0
  74. package/package.json +3 -1
@@ -14,10 +14,20 @@
14
14
  * outside an allowlist) need approval while tainted, even on "read" tools
15
15
  * - a run that touched a quarantined source is denied side effects outright
16
16
  *
17
+ * And whatever the run has read (safeguards.ts):
18
+ * - an action that creates something which acts later (a scheduled job, an automation, a mail rule,
19
+ * a hook, an edit to an agent's own instructions or memory) always needs a person
20
+ * - an action that carries the run's marker (a canary placed in its context) is refused: the context
21
+ * itself is leaving
22
+ * - a burst of actions, or the same action over and over, is held for a person
23
+ * - a stop switch a person can set refuses everything until they clear it
24
+ *
17
25
  * Limit, stated plainly: the broker enforces only on calls routed through it. A tool the
18
26
  * agent can reach directly is outside its control.
19
27
  */
20
28
  import { createHash, randomBytes } from "node:crypto";
29
+ import { BurstCounter, DEFAULT_BURST, actionKey, carriesCanary, newCanary, persistenceShapes } from "./safeguards.js";
30
+ import { HomeCalls, carriesCode, credentialsLeaving, hostsMentioned, isForwarder, loopbackPorts, oneTimeCodes, peripheralEgress } from "./egress.js";
21
31
  export const DEFAULT_BROKER_POLICY = {
22
32
  name: "default",
23
33
  version: 2,
@@ -29,6 +39,13 @@ export const DEFAULT_BROKER_POLICY = {
29
39
  approvalTtlMs: 120_000,
30
40
  maxShownArgChars: 4000,
31
41
  onApproverUnavailable: "deny",
42
+ persistenceGuard: "always",
43
+ burst: DEFAULT_BURST,
44
+ approveWhenTainted: true,
45
+ strictAddresses: true,
46
+ callHomeLimit: 6,
47
+ massRecipients: 10,
48
+ codeTtlMs: 30 * 60_000,
32
49
  };
33
50
  /** deterministic JSON: object keys sorted at every depth, so one call has one digest */
34
51
  export function canonicalJson(v) {
@@ -190,6 +207,10 @@ export function exfilShapes(args) {
190
207
  }
191
208
  const hostAllowed = (host, allow) => allow.some((a) => { const x = a.toLowerCase().replace(/\.$/, ""); return x.startsWith(".") ? host === x.slice(1) || host.endsWith(x) : host === x; });
192
209
  const hostMatches = (host, q) => host === q || host.endsWith("." + q);
210
+ /** the source that first named this host (or a parent of it) in content the run read */
211
+ const metIn = (r, h) => r.mentioned.get(h) ?? [...r.mentioned].find(([m]) => h.endsWith("." + m))?.[1];
212
+ /** did the person name this host (or a parent of it) themselves? */
213
+ const typedBy = (r, h) => r.typed.has(h) || [...r.typed].some((t) => h.endsWith("." + t));
193
214
  export class ActionBroker {
194
215
  deps;
195
216
  policy;
@@ -198,16 +219,72 @@ export class ActionBroker {
198
219
  used = new Set();
199
220
  constructor(policy = {}, deps = {}) {
200
221
  this.deps = deps;
201
- this.policy = { ...DEFAULT_BROKER_POLICY, ...policy, readOnlyTools: policy.readOnlyTools ?? DEFAULT_BROKER_POLICY.readOnlyTools };
222
+ this.policy = { ...DEFAULT_BROKER_POLICY, ...policy, readOnlyTools: policy.readOnlyTools ?? DEFAULT_BROKER_POLICY.readOnlyTools, burst: { ...DEFAULT_BROKER_POLICY.burst, ...(policy.burst ?? {}) } };
202
223
  }
203
224
  run(runId) {
204
225
  let r = this.runs.get(runId);
205
226
  if (!r) {
206
- r = { sources: new Set(), receipts: new Set(), hosts: new Set(), quarantined: new Set() };
227
+ r = { sources: new Set(), receipts: new Set(), hosts: new Set(), quarantined: new Set(), canaries: [], burst: new BurstCounter(), codes: new Map(), mentioned: new Map(), typed: new Set(), heardPerson: false, home: new HomeCalls() };
207
228
  this.runs.set(runId, r);
208
229
  }
209
230
  return r;
210
231
  }
232
+ /** A run's state as plain data (see SerializedRun). */
233
+ exportRun(runId) {
234
+ const r = this.run(runId);
235
+ return { sources: [...r.sources].slice(-200), receipts: [...r.receipts].slice(-200), hosts: [...r.hosts].slice(-400), quarantined: [...r.quarantined], canaries: [...r.canaries], codes: [...r.codes], mentioned: [...r.mentioned].slice(-800), typed: [...r.typed].slice(-400), heardPerson: r.heardPerson, home: r.home.export(), burst: r.burst.export() };
236
+ }
237
+ /**
238
+ * What the person said in their own words (not what they pasted). A destination they named is theirs: it is
239
+ * not reported as coming from content, and repeated requests to it are not counted as calling home.
240
+ */
241
+ notePersonSaid(runId, text) {
242
+ const r = this.run(runId);
243
+ r.heardPerson = true;
244
+ for (const h of hostsMentioned(text))
245
+ if (r.typed.size < 2000)
246
+ r.typed.add(h);
247
+ }
248
+ /** Restore a run from exported state. Unknown or malformed fields are ignored, never trusted to loosen anything. */
249
+ importRun(runId, s) {
250
+ if (!s || typeof s !== "object")
251
+ return;
252
+ const r = this.run(runId);
253
+ const strs = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === "string") : []);
254
+ for (const x of strs(s.sources))
255
+ r.sources.add(x);
256
+ for (const x of strs(s.receipts))
257
+ r.receipts.add(x);
258
+ for (const x of strs(s.hosts))
259
+ r.hosts.add(x);
260
+ for (const x of strs(s.quarantined))
261
+ r.quarantined.add(x);
262
+ for (const x of strs(s.canaries))
263
+ if (!r.canaries.includes(x))
264
+ r.canaries.push(x);
265
+ for (const e of Array.isArray(s.codes) ? s.codes : [])
266
+ if (Array.isArray(e) && typeof e[0] === "string" && typeof e[1] === "number")
267
+ r.codes.set(e[0], e[1]);
268
+ for (const e of Array.isArray(s.mentioned) ? s.mentioned : [])
269
+ if (Array.isArray(e) && typeof e[0] === "string" && typeof e[1] === "string")
270
+ r.mentioned.set(e[0], e[1]);
271
+ for (const x of strs(s.typed))
272
+ r.typed.add(x);
273
+ if (s.heardPerson === true)
274
+ r.heardPerson = true;
275
+ r.home.import(s.home);
276
+ r.burst.import(s.burst);
277
+ }
278
+ /**
279
+ * A marker for this run's context. Put the returned text where the model will read it (see
280
+ * safeguards.canaryNotice). Any later action whose arguments carry it is refused: nothing the
281
+ * person asked for needs the marker to leave, so its leaving means the context is being sent out.
282
+ */
283
+ plantCanary(runId) {
284
+ const c = newCanary();
285
+ this.run(runId).canaries.push(c);
286
+ return c;
287
+ }
211
288
  quarantineHit(ref, host) {
212
289
  for (const q of this.globalQuarantine) {
213
290
  if (ref && (ref === q || ref.startsWith(q)))
@@ -232,6 +309,14 @@ export class ActionBroker {
232
309
  ingest(runId, source) {
233
310
  const r = this.run(runId);
234
311
  r.sources.add(source.ref);
312
+ if (source.text) {
313
+ const now = this.deps.now?.() ?? Date.now();
314
+ for (const c of oneTimeCodes(source.text))
315
+ r.codes.set(c, now);
316
+ for (const h of hostsMentioned(source.text))
317
+ if (!r.mentioned.has(h) && r.mentioned.size < 2000)
318
+ r.mentioned.set(h, source.ref);
319
+ }
235
320
  if (source.receiptDigest)
236
321
  r.receipts.add(source.receiptDigest);
237
322
  const host = source.host?.toLowerCase().replace(/\.$/, "");
@@ -270,6 +355,10 @@ export class ActionBroker {
270
355
  const rule = this.policy.rules.find((x) => globMatch(x.tool, action.tool));
271
356
  const readOnly = !rule && this.policy.readOnlyTools.some((g) => globMatch(g, action.tool));
272
357
  const dest = destinationsIn(action.args);
358
+ // in free text such as a shell command, "a:b" is syntax far more often than an address: only what is
359
+ // written as a network URL is held to how it parses
360
+ if (!this.policy.strictAddresses)
361
+ dest.oddities = dest.oddities.filter((o) => /international domain|\((?:https?|wss?|ftps?|sftp|ssh|git|smb|s3|gs):[\\/]{2}/i.test(o));
273
362
  const fields = walkStrings(action.args).fields;
274
363
  let decision;
275
364
  if (rule) {
@@ -294,7 +383,7 @@ export class ActionBroker {
294
383
  else if (!tainted) {
295
384
  decision = "allow";
296
385
  }
297
- else if (readOnly) {
386
+ else if (readOnly || !this.policy.approveWhenTainted) {
298
387
  decision = "allow";
299
388
  }
300
389
  else {
@@ -318,8 +407,128 @@ export class ActionBroker {
318
407
  decision = "deny";
319
408
  reasons.push(`run touched a quarantined source: ${[...r.quarantined].join(", ")}`);
320
409
  }
410
+ // ---- safeguards that apply whatever the run has read ----
411
+ const walked = walkStrings(action.args);
412
+ // something that acts later always goes to a person: one steered moment must not become a standing one
413
+ if (this.policy.persistenceGuard !== "off" && (tainted || this.policy.persistenceGuard === "always") && !readOnly) {
414
+ const p = persistenceShapes(action.tool, action.writes ?? walked.strings);
415
+ if (p.length) {
416
+ decision = strongest(decision, "require_approval");
417
+ reasons.push(...p.map((h) => `creates something that acts later: ${h.why}`));
418
+ }
419
+ }
420
+ // a burst, or the same action again and again
421
+ if (!readOnly) {
422
+ const b = r.burst.check(actionKey(action.tool, canonicalArgs), this.deps.now?.() ?? Date.now(), this.policy.burst);
423
+ if (b.length) {
424
+ decision = strongest(decision, "require_approval");
425
+ reasons.push(...b);
426
+ }
427
+ }
428
+ // ---- egress: what leaves, judged with what the run has read (egress.ts) ----
429
+ const now = this.deps.now?.() ?? Date.now();
430
+ const outside = dest.hosts.filter((h) => !hostAllowed(h, this.policy.allowHosts));
431
+ // where a destination came from: named first by content the run read, not by the person
432
+ // (a host the person named themselves is theirs, whatever content also names it)
433
+ const fromContent = outside.filter((h) => metIn(r, h) !== undefined && !typedBy(r, h));
434
+ const provenance = (h) => metIn(r, h) ?? "content this run read";
435
+ if (tainted) {
436
+ const creds = credentialsLeaving(walked.strings);
437
+ if (creds) {
438
+ decision = strongest(decision, "require_approval");
439
+ reasons.push(`the arguments carry ${creds} credential${creds === 1 ? "" : "s"}, in a run that has taken in outside content`);
440
+ }
441
+ // data going to another machine: fine toward this machine, the allowlist, or a place the person named
442
+ if (action.sends?.length) {
443
+ const urls = walked.strings.flatMap((s) => [...s.matchAll(/\b[a-z][a-z0-9+.-]{1,15}:\/\/(?:[^\s/@"']*@)?(\[[0-9a-f:.]+\]|[^\s/:?#"'<>()]+)/gi)].map((m) => m[1].toLowerCase().replace(/^\[|\]$/g, "")));
444
+ const fine = (h) => /^(?:localhost|127(?:\.\d{1,3}){3}|::1|0\.0\.0\.0)$/.test(h) || hostAllowed(h, this.policy.allowHosts) || typedBy(r, h);
445
+ if (!(urls.length && urls.every(fine))) {
446
+ decision = strongest(decision, "require_approval");
447
+ reasons.push(...action.sends.map((s) => `${s}, in a run that has taken in outside content`));
448
+ }
449
+ }
450
+ // one message to many people
451
+ const recipients = new Set(walked.strings.flatMap((s) => (s.length > 200_000 ? [] : [...s.matchAll(/[\w.+-]{1,64}@(?:[a-z0-9-]+\.)+[a-z]{2,24}/gi)].map((m) => m[0].toLowerCase())))).size;
452
+ if (recipients >= this.policy.massRecipients && !readOnly) {
453
+ decision = strongest(decision, "require_approval");
454
+ reasons.push(`the arguments name ${recipients} email addresses: one action reaching many people`);
455
+ }
456
+ const per = peripheralEgress(action.tool, walked.strings);
457
+ if (per.length) {
458
+ decision = strongest(decision, "require_approval");
459
+ reasons.push(...per.map((p) => `data would leave this machine without the network: ${p.why}`));
460
+ }
461
+ // a loopback port owned by a tunnel or proxy is the internet
462
+ for (const port of this.deps.portProgram ? loopbackPorts(walked.strings) : []) {
463
+ const prog = this.deps.portProgram(port);
464
+ if (isForwarder(prog)) {
465
+ decision = strongest(decision, "require_approval");
466
+ reasons.push(`port ${port} on this machine belongs to ${prog}, which forwards elsewhere: this is not a local destination`);
467
+ }
468
+ }
469
+ // many requests to a host the run met in outside content
470
+ for (const h of fromContent) {
471
+ const n = r.home.count(h, now);
472
+ if (n >= this.policy.callHomeLimit) {
473
+ decision = strongest(decision, "require_approval");
474
+ reasons.push(`${n} requests in ten minutes to ${h}, a host this run first met in ${provenance(h)}`);
475
+ }
476
+ }
477
+ // once a person is being asked anyway, say where each destination came from
478
+ if (decision !== "allow")
479
+ for (const h of fromContent.slice(0, 4))
480
+ reasons.push(`${h} first appeared in ${provenance(h)}${r.heardPerson ? ", not in anything you typed" : ""}`);
481
+ }
482
+ // known-bad destinations: refused whatever the run has read
483
+ if (this.deps.blocklist?.size) {
484
+ for (const h of dest.hosts) {
485
+ const hit = this.deps.blocklist.match(h);
486
+ if (hit) {
487
+ decision = "deny";
488
+ reasons.push(`${h} is on a blocklist of malware and botnet hosts (${hit})`);
489
+ }
490
+ }
491
+ }
492
+ // a verification code the run read: for the person to type, never for an agent to send
493
+ const live = [...r.codes].filter(([, at]) => now - at <= this.policy.codeTtlMs).map(([c]) => c);
494
+ const code = carriesCode(walked.strings, live);
495
+ if (code) {
496
+ // to a place that outside content named: that is the theft, and not a question. Anywhere else (a sign-in
497
+ // the person asked for) a person decides, knowing what the action carries. Short codes collide with
498
+ // ordinary numbers, so they only ever ask.
499
+ const stolen = fromContent.length > 0 && code.length >= 6;
500
+ decision = stolen ? "deny" : strongest(decision, "require_approval");
501
+ reasons.push(stolen
502
+ ? `the arguments carry a verification code this run read (ending ${code.slice(-2)}) to ${fromContent[0]}, a destination that came from ${provenance(fromContent[0])}`
503
+ : `the arguments carry a verification code this run read (ending ${code.slice(-2)}): approve only if you asked for this sign-in`);
504
+ }
505
+ // the run's marker in an outbound argument: the context itself is leaving. Not a question for approval.
506
+ if (carriesCanary(walked.strings, r.canaries)) {
507
+ decision = "deny";
508
+ reasons.push("the arguments carry this run's marker, which has no reason to leave the conversation: something is sending the conversation itself out");
509
+ }
510
+ // the stop switch outranks everything, read-only tools included
511
+ const stopped = this.deps.stopCheck?.();
512
+ if (stopped) {
513
+ decision = "deny";
514
+ reasons.push(`a person has stopped all agent actions: ${stopped}`);
515
+ }
321
516
  return { decision, reasons, digest, canonicalArgs, destinations: dest.hosts, fields };
322
517
  }
518
+ /**
519
+ * Record that an action went ahead when something other than this broker's approver let it (a harness with
520
+ * its own permission prompt): the burst and call-home counters need to know.
521
+ */
522
+ noteRan(action) {
523
+ const now = this.deps.now?.() ?? Date.now();
524
+ const canonicalArgs = canonicalJson(action.args);
525
+ const r = this.run(action.runId);
526
+ if (!this.policy.readOnlyTools.some((g) => globMatch(g, action.tool)))
527
+ r.burst.record(actionKey(action.tool, canonicalArgs), now);
528
+ for (const h of destinationsIn(action.args).hosts)
529
+ if (metIn(r, h) !== undefined)
530
+ r.home.record(h, now);
531
+ }
323
532
  /**
324
533
  * Decide, asking the approver when needed. The caller must then execute EXACTLY the
325
534
  * arguments it gets back (frozen) or not at all.
@@ -364,6 +573,16 @@ export class ActionBroker {
364
573
  c.reasons.push(outcome === "unavailable" ? "approval needed but no approver is set up or reachable, so it was refused (fails closed)" : outcome === "denied" ? "the person declined" : "the person did not answer in time");
365
574
  verdict = { decision: "require_approval", permitted, reasons: c.reasons, digest: c.digest, approval: { id: req.id, outcome }, policy: policyRef };
366
575
  }
576
+ // count what actually ran, so the burst guard sees it (read-only tools never count)
577
+ if (verdict.permitted && !this.policy.readOnlyTools.some((g) => globMatch(g, action.tool)))
578
+ this.run(action.runId).burst.record(actionKey(action.tool, c.canonicalArgs), now);
579
+ // and every request that went out, per destination, so a host being called again and again is seen
580
+ if (verdict.permitted) {
581
+ const run = this.run(action.runId);
582
+ for (const h of c.destinations)
583
+ if (metIn(run, h) !== undefined)
584
+ run.home.record(h, now);
585
+ }
367
586
  this.deps.log?.({ event: "broker.decision", at: new Date(now).toISOString(), runId: action.runId, tool: action.tool, digest: c.digest, decision: verdict.decision, permitted: verdict.permitted, reasons: verdict.reasons, approval: verdict.approval, sources: this.state(action.runId).sources, policy: policyRef });
368
587
  return { ...verdict, args: deepFreeze(JSON.parse(c.canonicalArgs)) };
369
588
  }
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Calendars and to-do items (iCalendar text, and the JSON that Google Calendar and Microsoft Graph return).
3
+ *
4
+ * Anyone can put an event on a calendar: Google Calendar adds invitations from any sender by default, and a
5
+ * subscribed calendar is a file its publisher can change at any time. An assistant asked "what's on today"
6
+ * reads every field of every event. So each field is a segment with the visibility a person would have of it
7
+ * in a calendar app: the title, time, place and description are seen; alternative descriptions, alarm text,
8
+ * attachments and extension properties are not.
9
+ *
10
+ * Beyond what the passes find in the text, three things are specific to the channel:
11
+ * - a description that says one thing in plain text and another in its HTML alternative
12
+ * - an alarm that does something (sends email, runs a program) rather than display a reminder
13
+ * - who the organizer claims to be (left to the caller's trust level: an invitation from a stranger is
14
+ * trust 4, a subscribed feed trust 5)
15
+ */
16
+ import type { Finding, TextSegment } from "./types.js";
17
+ export interface CalendarAdapterResult {
18
+ segments: TextSegment[];
19
+ findings: Finding[];
20
+ }
21
+ interface IcsProp {
22
+ name: string;
23
+ params: Record<string, string>;
24
+ value: string;
25
+ }
26
+ interface IcsComponent {
27
+ type: string;
28
+ props: IcsProp[];
29
+ children: IcsComponent[];
30
+ }
31
+ /** Parse iCalendar text into nested components. Tolerant: unknown lines are skipped, never thrown on. */
32
+ export declare function parseIcs(text: string): IcsComponent[];
33
+ /** Is this parsed JSON a calendar event, or a list of them? */
34
+ export declare function looksLikeCalendarJson(doc: unknown): boolean;
35
+ export declare function adaptCalendar(raw: string): CalendarAdapterResult;
36
+ export {};
@@ -0,0 +1,227 @@
1
+ /**
2
+ * Calendars and to-do items (iCalendar text, and the JSON that Google Calendar and Microsoft Graph return).
3
+ *
4
+ * Anyone can put an event on a calendar: Google Calendar adds invitations from any sender by default, and a
5
+ * subscribed calendar is a file its publisher can change at any time. An assistant asked "what's on today"
6
+ * reads every field of every event. So each field is a segment with the visibility a person would have of it
7
+ * in a calendar app: the title, time, place and description are seen; alternative descriptions, alarm text,
8
+ * attachments and extension properties are not.
9
+ *
10
+ * Beyond what the passes find in the text, three things are specific to the channel:
11
+ * - a description that says one thing in plain text and another in its HTML alternative
12
+ * - an alarm that does something (sends email, runs a program) rather than display a reminder
13
+ * - who the organizer claims to be (left to the caller's trust level: an invitation from a stranger is
14
+ * trust 4, a subscribed feed trust 5)
15
+ */
16
+ import { analyzeHtml } from "./visibility.js";
17
+ import { passagesOnlyIn } from "./served.js";
18
+ const unescape = (v) => v.replace(/\\n/gi, "\n").replace(/\\([,;\\])/g, "$1");
19
+ /** Parse iCalendar text into nested components. Tolerant: unknown lines are skipped, never thrown on. */
20
+ export function parseIcs(text) {
21
+ const lines = text.replace(/\r\n?/g, "\n").replace(/\n[ \t]/g, "").split("\n");
22
+ const root = { type: "ROOT", props: [], children: [] };
23
+ const stack = [root];
24
+ for (const line of lines.slice(0, 200_000)) {
25
+ if (!line.trim())
26
+ continue;
27
+ // NAME;PARAM=value;PARAM="quoted:value":VALUE (the first colon outside quotes ends the name part)
28
+ let i = 0, inQuote = false;
29
+ for (; i < line.length; i++) {
30
+ const c = line[i];
31
+ if (c === '"')
32
+ inQuote = !inQuote;
33
+ else if (c === ":" && !inQuote)
34
+ break;
35
+ }
36
+ if (i >= line.length)
37
+ continue;
38
+ const head = line.slice(0, i), value = line.slice(i + 1);
39
+ const [rawName, ...rawParams] = head.split(/;(?=(?:[^"]*"[^"]*")*[^"]*$)/);
40
+ const name = rawName.toUpperCase();
41
+ if (name === "BEGIN") {
42
+ const c = { type: value.trim().toUpperCase(), props: [], children: [] };
43
+ stack[stack.length - 1].children.push(c);
44
+ if (stack.length < 16)
45
+ stack.push(c);
46
+ continue;
47
+ }
48
+ if (name === "END") {
49
+ if (stack.length > 1 && stack[stack.length - 1].type === value.trim().toUpperCase())
50
+ stack.pop();
51
+ continue;
52
+ }
53
+ const params = {};
54
+ for (const p of rawParams) {
55
+ const eq = p.indexOf("=");
56
+ if (eq > 0)
57
+ params[p.slice(0, eq).toUpperCase()] = p.slice(eq + 1).replace(/^"|"$/g, "");
58
+ }
59
+ stack[stack.length - 1].props.push({ name, params, value });
60
+ }
61
+ return root.children;
62
+ }
63
+ const SEEN_PROPS = new Set(["SUMMARY", "DESCRIPTION", "LOCATION", "COMMENT", "CONTACT", "CATEGORIES", "RESOURCES"]);
64
+ /** properties that are structure, not prose */
65
+ const STRUCTURAL = /^(DT|DUE|COMPLETED|CREATED|LAST-MODIFIED|SEQUENCE|UID|RRULE|EXDATE|RDATE|EXRULE|TRANSP|STATUS|CLASS|PRIORITY|GEO|DURATION|TRIGGER|REPEAT|ACTION|RECURRENCE-ID|PERCENT-COMPLETE|TZ|PRODID|VERSION|CALSCALE|METHOD|FREEBUSY|X-MICROSOFT-CDO|X-APPLE-TRAVEL|X-APPLE-STRUCTURED|X-LIC-|X-GOOGLE-|X-WR-TIMEZONE)/;
66
+ function adaptComponent(c, path, segments, findings, depth = 0) {
67
+ if (depth > 8)
68
+ return;
69
+ const get = (n) => c.props.filter((p) => p.name === n);
70
+ const isAlarm = c.type === "VALARM";
71
+ let plainDescription = "";
72
+ for (const p of c.props) {
73
+ const v = unescape(p.value).trim();
74
+ if (!v)
75
+ continue;
76
+ if (p.name === "X-ALT-DESC" || (p.name === "DESCRIPTION" && /html/i.test(p.params.FMTTYPE ?? "")))
77
+ continue; // handled below
78
+ if (p.name === "ORGANIZER" || p.name === "ATTENDEE") {
79
+ // the display name is free text the sender chose
80
+ if (p.params.CN)
81
+ segments.push({ text: p.params.CN, visibility: "visible", path: `${path}/${p.name.toLowerCase()}@cn` });
82
+ continue;
83
+ }
84
+ if (p.name === "ATTACH" || p.name === "URL" || p.name === "CONFERENCE") {
85
+ // never fetched: listed so the link checks and the meaning pass see the address
86
+ if (/^https?:/i.test(v))
87
+ segments.push({ text: v, visibility: "attribute", path: `${path}/${p.name.toLowerCase()}` });
88
+ else if (p.params.ENCODING?.toUpperCase() === "BASE64" && v.length < 400_000) {
89
+ const decoded = Buffer.from(v, "base64").toString("utf8");
90
+ if (/^[\x09\x0a\x0d\x20-\x7e -￿]{8,}$/.test(decoded.slice(0, 2000)))
91
+ segments.push({ text: decoded, visibility: "attribute", path: `${path}/attach@inline` });
92
+ }
93
+ continue;
94
+ }
95
+ if (STRUCTURAL.test(p.name))
96
+ continue;
97
+ // an alarm's text is never shown in the event; in the event itself only the usual fields are
98
+ const seen = !isAlarm && SEEN_PROPS.has(p.name);
99
+ if (p.name === "DESCRIPTION" && !isAlarm)
100
+ plainDescription += `${v}\n`;
101
+ segments.push({ text: v, visibility: seen ? "visible" : "attribute", path: `${path}/${p.name.toLowerCase()}` });
102
+ }
103
+ // HTML alternative: what a rich client shows. Text its styles hide is hidden; text only the plain
104
+ // description has is what a rich client's user never sees.
105
+ for (const p of [...get("X-ALT-DESC"), ...get("DESCRIPTION").filter((x) => /html/i.test(x.params.FMTTYPE ?? ""))]) {
106
+ const html = unescape(p.value);
107
+ if (!html.trim())
108
+ continue;
109
+ const v = analyzeHtml(html);
110
+ for (const s of v.segments)
111
+ segments.push({ ...s, path: `${path}/alt-desc${s.path}` });
112
+ findings.push(...v.findings.map((f) => ({ ...f, location: `${path}/alt-desc${f.location ?? ""}` })));
113
+ if (plainDescription.trim()) {
114
+ const only = passagesOnlyIn(plainDescription, v.segments.map((s) => s.text).join(" "));
115
+ if (only.length) {
116
+ findings.push({ pass: "channel", kind: "calendar.description_divergence", confidence: 0.3, severity: 0.6, message: `the event's plain description says ${only.length} thing(s) its formatted description does not`, excerpt: only[0].slice(0, 160), location: `${path}/description`, action: "none" });
117
+ for (const [i, t] of only.entries())
118
+ segments.push({ text: t, visibility: "concealed", path: `${path}/description~plain-only[${i}]` });
119
+ }
120
+ }
121
+ }
122
+ if (isAlarm) {
123
+ const action = (get("ACTION")[0]?.value ?? "").trim().toUpperCase();
124
+ if (action === "PROCEDURE")
125
+ findings.push({ pass: "channel", kind: "calendar.alarm_action", confidence: 0.8, severity: 0.9, message: "an alarm that runs a program when it fires (ACTION:PROCEDURE)", excerpt: (get("ATTACH")[0]?.value ?? "").slice(0, 120), location: path, action: "none" });
126
+ else if (action === "EMAIL")
127
+ findings.push({ pass: "channel", kind: "calendar.alarm_action", confidence: 0.4, severity: 0.6, message: "an alarm that sends email when it fires (ACTION:EMAIL)", excerpt: get("ATTENDEE").map((a) => a.value).join(", ").slice(0, 120), location: path, action: "none" });
128
+ }
129
+ const counts = new Map();
130
+ for (const child of c.children) {
131
+ const n = counts.get(child.type) ?? 0;
132
+ counts.set(child.type, n + 1);
133
+ adaptComponent(child, `${path}/${child.type.toLowerCase()}[${n}]`, segments, findings, depth + 1);
134
+ }
135
+ }
136
+ const str = (v) => (typeof v === "string" ? v : "");
137
+ /** One event object as Google Calendar or Microsoft Graph returns it. */
138
+ function adaptEventJson(e, path, segments, findings) {
139
+ const title = str(e.summary) || str(e.subject) || str(e.title);
140
+ if (title)
141
+ segments.push({ text: title, visibility: "visible", path: `${path}/summary` });
142
+ const loc = typeof e.location === "string" ? e.location : str(e.location?.displayName);
143
+ if (loc)
144
+ segments.push({ text: loc, visibility: "visible", path: `${path}/location` });
145
+ // Google: description is a string that may hold HTML. Graph: body { contentType, content }.
146
+ const body = e.body && typeof e.body === "object" ? e.body : undefined;
147
+ const desc = str(e.description) || str(body?.content) || str(e.bodyPreview);
148
+ if (desc) {
149
+ if (/<[a-z][^>]*>/i.test(desc) || /html/i.test(str(body?.contentType))) {
150
+ const v = analyzeHtml(desc);
151
+ for (const s of v.segments)
152
+ segments.push({ ...s, path: `${path}/description${s.path}` });
153
+ findings.push(...v.findings);
154
+ }
155
+ else
156
+ segments.push({ text: desc, visibility: "visible", path: `${path}/description` });
157
+ }
158
+ const people = [e.organizer, e.creator, ...(Array.isArray(e.attendees) ? e.attendees : [])];
159
+ for (const p of people) {
160
+ if (!p || typeof p !== "object")
161
+ continue;
162
+ const name = str(p.displayName) || str(p.emailAddress?.name);
163
+ if (name)
164
+ segments.push({ text: name, visibility: "visible", path: `${path}/person@name` });
165
+ }
166
+ for (const k of ["hangoutLink", "htmlLink", "onlineMeetingUrl", "webLink"])
167
+ if (/^https?:/i.test(str(e[k])))
168
+ segments.push({ text: str(e[k]), visibility: "attribute", path: `${path}/${k}` });
169
+ for (const a of Array.isArray(e.attachments) ? e.attachments : []) {
170
+ if (!a || typeof a !== "object")
171
+ continue;
172
+ const t = [str(a.title), str(a.name), str(a.fileUrl)].filter(Boolean).join(" ");
173
+ if (t)
174
+ segments.push({ text: t, visibility: "attribute", path: `${path}/attachment` });
175
+ }
176
+ // properties a calendar app never shows
177
+ const ext = e.extendedProperties && typeof e.extendedProperties === "object" ? JSON.stringify(e.extendedProperties) : "";
178
+ if (ext.length > 2)
179
+ segments.push({ text: ext, visibility: "attribute", path: `${path}/extendedProperties` });
180
+ }
181
+ /** Is this parsed JSON a calendar event, or a list of them? */
182
+ export function looksLikeCalendarJson(doc) {
183
+ const one = (e) => !!e && typeof e === "object" && ("summary" in e || "subject" in e) && ("start" in e || "end" in e || "attendees" in e || "organizer" in e);
184
+ if (Array.isArray(doc))
185
+ return doc.length > 0 && doc.every(one);
186
+ if (doc && typeof doc === "object") {
187
+ const d = doc;
188
+ if (d.kind === "calendar#events" || (Array.isArray(d.items) && d.items.some(one)) || (Array.isArray(d.value) && d.value.some(one)))
189
+ return true;
190
+ }
191
+ return one(doc);
192
+ }
193
+ export function adaptCalendar(raw) {
194
+ const segments = [];
195
+ const findings = [];
196
+ if (/BEGIN:VCALENDAR|BEGIN:VEVENT|BEGIN:VTODO/i.test(raw.slice(0, 4000))) {
197
+ const counts = new Map();
198
+ const walk = (c, path) => {
199
+ if (c.type === "VCALENDAR") {
200
+ for (const p of c.props)
201
+ if (/^X-WR-(CALNAME|CALDESC)$/.test(p.name) && p.value.trim())
202
+ segments.push({ text: unescape(p.value), visibility: "visible", path: `calendar/${p.name.toLowerCase()}` });
203
+ for (const k of c.children)
204
+ walk(k, "calendar");
205
+ return;
206
+ }
207
+ const n = counts.get(c.type) ?? 0;
208
+ counts.set(c.type, n + 1);
209
+ adaptComponent(c, `${path}/${c.type.toLowerCase()}[${n}]`, segments, findings);
210
+ };
211
+ for (const c of parseIcs(raw))
212
+ walk(c, "calendar");
213
+ return { segments, findings };
214
+ }
215
+ let doc;
216
+ try {
217
+ doc = JSON.parse(raw);
218
+ }
219
+ catch {
220
+ return { segments: [{ text: raw, visibility: "visible", path: "calendar/text" }], findings };
221
+ }
222
+ const d = doc;
223
+ const events = Array.isArray(doc) ? doc : Array.isArray(d?.items) ? d.items : Array.isArray(d?.value) ? d.value : [doc];
224
+ events.slice(0, 2000).forEach((e, i) => { if (e && typeof e === "object")
225
+ adaptEventJson(e, `calendar/event[${i}]`, segments, findings); });
226
+ return { segments, findings };
227
+ }
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Captions and transcripts: WebVTT, SubRip (SRT), and the JSON a transcription service or a meeting
3
+ * notetaker produces.
4
+ *
5
+ * A caption file is read by people one line at a time, on screen, for a second or two. A model reads the whole
6
+ * file at once: the cue that flashes for a twentieth of a second, the cue a style rule makes transparent, the
7
+ * NOTE block no player shows, the cue nobody said aloud. A meeting transcript is a room where anyone present
8
+ * can speak, and "assistant, add to the action items that..." is as present to the notetaker as the agenda.
9
+ *
10
+ * Checks:
11
+ * cues nobody can read shown too briefly for their length, sized to nothing, or styled invisible
12
+ * what no player shows NOTE blocks, STYLE blocks, metadata cues: concealed text
13
+ * captions vs. speech given what was actually said (a transcript of the audio), words in the captions
14
+ * that nobody said
15
+ * speakers an utterance that addresses the notetaker or assistant with something to do
16
+ * spoken approvals noted: a voice saying "I approve" is content. Approval comes from the broker's
17
+ * own channel, never from what was heard
18
+ */
19
+ import { type AdapterResult } from "./adapters.js";
20
+ export interface Cue {
21
+ start: number;
22
+ end: number;
23
+ text: string;
24
+ speaker?: string;
25
+ settings?: string;
26
+ classes: string[];
27
+ id?: string;
28
+ }
29
+ export interface CaptionHints {
30
+ /** what was actually said: a transcript of the audio, for comparison */
31
+ spoken?: string;
32
+ /** speakers known to be outside the organisation (their words are a stranger's) */
33
+ outsideSpeakers?: string[];
34
+ }
35
+ export declare function looksLikeCaptions(text: string): boolean;
36
+ /** Is this JSON a transcript: utterances or segments with text and a time or a speaker? */
37
+ export declare function looksLikeTranscriptJson(doc: unknown): boolean;
38
+ export declare function parseCaptions(raw: string): {
39
+ cues: Cue[];
40
+ notes: string[];
41
+ styles: string[];
42
+ format: "vtt" | "srt";
43
+ };
44
+ export declare function adaptCaptions(raw: string, hints?: CaptionHints): AdapterResult;