onionskin-cli 0.0.3 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +198 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +102 -34
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/scripts/checker/check.py +24 -5
  9. package/scripts/checker/requirements.txt +482 -5
  10. package/scripts/requirements.txt +482 -5
  11. package/scripts/rewriter/apple/main.swift +1 -1
  12. package/scripts/rewriter/apple/onionskin-fm +0 -0
  13. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  14. package/skills/onionskin/SKILL.md +12 -9
  15. package/skills/onionskin/reference/batch.md +5 -3
  16. package/skills/onionskin/reference/formats.md +5 -1
  17. package/skills/onionskin/reference/ops.md +48 -4
  18. package/skills/onionskin/reference/troubleshooting.md +31 -0
  19. package/src/agent-detect.js +28 -2
  20. package/src/agent-prompt.js +12 -1
  21. package/src/agent-runner.js +110 -20
  22. package/src/anchor-work.js +154 -0
  23. package/src/anchor-worker.js +38 -0
  24. package/src/anchors.js +61 -11
  25. package/src/apply.js +119 -47
  26. package/src/batch.js +32 -8
  27. package/src/check-worker.js +8 -5
  28. package/src/checker.js +238 -33
  29. package/src/cli.js +248 -42
  30. package/src/constants.js +34 -1
  31. package/src/diff.js +3 -1
  32. package/src/doctor.js +12 -3
  33. package/src/docx.js +464 -87
  34. package/src/edit-hash.js +19 -0
  35. package/src/fsutil.js +14 -6
  36. package/src/history-worker.js +8 -8
  37. package/src/history.js +144 -31
  38. package/src/html-active.js +384 -0
  39. package/src/html-envelope.js +51 -0
  40. package/src/html-parser.js +8 -0
  41. package/src/html-render-worker.js +25 -0
  42. package/src/html-render.js +939 -0
  43. package/src/html-utf8.js +65 -0
  44. package/src/html-write-worker.js +14 -0
  45. package/src/html-write.js +380 -0
  46. package/src/html.js +1259 -0
  47. package/src/local.js +68 -4
  48. package/src/merge-package.js +1166 -0
  49. package/src/merge-pptx.js +98 -38
  50. package/src/merge-proof.js +1019 -0
  51. package/src/merge-worker.js +60 -0
  52. package/src/merge-xlsx.js +84 -29
  53. package/src/merge.js +286 -83
  54. package/src/nfc.js +46 -0
  55. package/src/notes.js +122 -0
  56. package/src/offload.js +419 -1
  57. package/src/open-worker.js +27 -0
  58. package/src/ops/common.js +2 -1
  59. package/src/ops/docx-table.js +5 -3
  60. package/src/ops/docx.js +58 -29
  61. package/src/ops/html.js +394 -0
  62. package/src/ops/pptx.js +82 -23
  63. package/src/ops/run.js +23 -3
  64. package/src/ops/runs.js +215 -13
  65. package/src/ops/schema.js +8 -1
  66. package/src/ops/xlsx.js +121 -10
  67. package/src/ops-worker.js +17 -0
  68. package/src/package.js +43 -4
  69. package/src/person.js +2 -1
  70. package/src/pptx.js +323 -23
  71. package/src/question.js +10 -3
  72. package/src/quick-plan.js +13 -6
  73. package/src/quick.js +1 -1
  74. package/src/redline-check.js +12 -2
  75. package/src/redline-worker.js +10 -14
  76. package/src/reply.js +46 -19
  77. package/src/retain.js +223 -0
  78. package/src/review.js +32 -18
  79. package/src/rewrite-check.js +2 -2
  80. package/src/rewriter.js +38 -8
  81. package/src/sanitize.js +16 -3
  82. package/src/server-entry.js +4 -3
  83. package/src/server.js +175 -62
  84. package/src/session.js +1151 -210
  85. package/src/setup.js +20 -2
  86. package/src/stop-gate.js +34 -0
  87. package/src/suggest-reply.js +42 -7
  88. package/src/suggest.js +14 -29
  89. package/src/text.js +161 -11
  90. package/src/update.js +517 -0
  91. package/src/voice-guide.js +15 -9
  92. package/src/voice-learn.js +15 -4
  93. package/src/voice-sample-worker.js +7 -6
  94. package/src/voice-store.js +11 -4
  95. package/src/voice-worker.js +4 -6
  96. package/src/voice.js +2 -1
  97. package/src/worker-host.js +23 -0
  98. package/src/xlsx.js +378 -97
  99. package/src/xml-tree.js +6 -3
  100. package/src/xml.js +82 -34
  101. package/src/zip-write.js +10 -5
  102. package/src/zip.js +73 -18
  103. package/vendor/parse5/LICENSE.txt +32 -0
  104. package/vendor/parse5/parse5.mjs +7822 -0
  105. package/vendor/parse5.lock.json +23 -0
  106. package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
  107. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  108. package/web/dist/index.html +2 -2
  109. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/pptx.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import { attrOf, local, readRels } from "./xlsx.js";
2
- import { walkXml } from "./xml.js";
3
- import { normText, similarity, textHash } from "./text.js";
2
+ import { readBudget, tooMany, walkXml } from "./xml.js";
3
+ import { OCCURRENCE, affords, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
4
+ import { nfc } from "./nfc.js";
4
5
  import { openZip } from "./zip.js";
5
6
  import { summarizeRuns } from "./fmt.js";
6
7
  import { untrusted } from "./sanitize.js";
@@ -13,6 +14,15 @@ const EMPTY = textHash("");
13
14
  const CTX = 40;
14
15
  const KEEP = 2000;
15
16
 
17
+ // What one reading of a deck may hold, counted over all its slides. The server reads a deck on its own thread at Send, at each delivery to
18
+ // an agent and at the end of a run, so these bound how long that keeps it from answering (outside review: three slides of 2.85 million runs
19
+ // each held it 7.2 s). Each shape costs a few hundred bytes once read. A real deck of 200 slides has a few thousand shapes and well under
20
+ // 200,000 elements.
21
+ export const MAX_SHAPES = 50_000;
22
+ export const MAX_DECK_ELEMENTS = 500_000;
23
+ export const MAX_DECK_ATTRIBUTES = 1_000_000;
24
+ export const MAX_DECK_TEXT = 8_000_000;
25
+ const deckBudget = () => readBudget({ elements: MAX_DECK_ELEMENTS, attrs: MAX_DECK_ATTRIBUTES, text: MAX_DECK_TEXT, what: "deck", split: "smaller decks" });
16
26
  const SHAPE_TAGS = new Set(["sp", "pic", "graphicFrame", "grpSp", "cxnSp"]);
17
27
  const KIND = { sp: "shape", pic: "image", graphicFrame: "table", grpSp: "group", cxnSp: "line" };
18
28
 
@@ -21,23 +31,34 @@ export function parseDeck(bytes) {
21
31
  const zip = openZip(bytes);
22
32
  const px = zip.readText("ppt/presentation.xml");
23
33
  if (px == null) throw new Error("ppt/presentation.xml missing");
24
- const rels = readRels(zip, "ppt/presentation.xml");
34
+ const budget = deckBudget(); // the presentation, its relationships and every slide of one reading share it
35
+ const rels = readRels(zip, "ppt/presentation.xml", budget);
25
36
  const slides = [];
26
37
  walkXml(px, {
27
38
  onOpen(n, a) {
28
39
  if (local(n) === "sldId") slides.push({ id: a.id ?? null, index: slides.length, part: rels.get(attrOf(a, "id")) || null });
29
40
  },
30
- });
41
+ }, budget);
42
+ // read once; a reading that failed (past a cap) fails the same way every time it is asked again, never "already read"
31
43
  let loaded = false;
44
+ let failed = null;
32
45
  const load = () => {
46
+ if (failed) throw failed;
33
47
  if (loaded) return;
34
- loaded = true;
35
- for (const s of slides) Object.assign(s, readSlide(zip, s.part));
48
+ try {
49
+ const count = { shapes: 0, budget };
50
+ const read = slides.map((s) => readSlide(zip, s.part, count));
51
+ slides.forEach((s, i) => Object.assign(s, read[i]));
52
+ loaded = true;
53
+ } catch (err) {
54
+ failed = err;
55
+ throw err;
56
+ }
36
57
  };
37
58
  return { slides, ensure: load };
38
59
  }
39
60
 
40
- function readSlide(zip, part) {
61
+ function readSlide(zip, part, count = { shapes: 0, budget: deckBudget() }) {
41
62
  const out = { shapes: [], title: "", titleHash: EMPTY };
42
63
  const xml = part ? zip.readText(part) : null;
43
64
  if (xml == null) return out;
@@ -46,10 +67,16 @@ function readSlide(zip, part) {
46
67
  let inT = false;
47
68
  const seen = new Map();
48
69
  const at = []; // open element names, to tell a text color (rPr > solidFill > srgbClr) from any other color in the slide
70
+ let mcFallbackDepth = 0;
49
71
  walkXml(xml, {
50
72
  onOpen(n, a) {
73
+ if (n === "mc:Fallback" || mcFallbackDepth > 0) {
74
+ if (n === "mc:Fallback") mcFallbackDepth++;
75
+ return;
76
+ }
51
77
  const l = local(n);
52
78
  at.push(l);
79
+ if (SHAPE_TAGS.has(l) && ++count.shapes > MAX_SHAPES) throw tooMany(count.budget, MAX_SHAPES, "shapes");
53
80
  if (SHAPE_TAGS.has(l)) stack.push({ tag: l, id: null, name: "", paras: [], isTitle: false, runs: [], run: null });
54
81
  else if (!stack.length) return;
55
82
  const top = stack[stack.length - 1];
@@ -58,7 +85,7 @@ function readSlide(zip, part) {
58
85
  else if (top.run && (l === "srgbClr" || l === "schemeClr") && at[at.length - 2] === "solidFill" && at[at.length - 3] === "rPr") top.run.p.color = String(a.val || "").toUpperCase();
59
86
  else if (top.run && l === "latin" && at[at.length - 2] === "rPr") top.run.p.font = a.typeface || "";
60
87
  if (l === "cNvPr" && top && top.id === null) {
61
- top.id = a.id != null ? Number(a.id) : null;
88
+ top.id = /^\d+$/.test(a.id ?? "") ? Number(a.id) : null;
62
89
  top.name = a.name ?? "";
63
90
  } else if (l === "ph" && top) {
64
91
  top.ph = a.type || "object"; // no type = a content placeholder
@@ -70,9 +97,14 @@ function readSlide(zip, part) {
70
97
  else if (l === "tc" && top) top.cellStart = true;
71
98
  },
72
99
  onText(t) {
100
+ if (mcFallbackDepth > 0) return;
73
101
  if (inT && para !== null) para += t;
74
102
  },
75
103
  onClose(n) {
104
+ if (mcFallbackDepth > 0) {
105
+ if (n === "mc:Fallback") mcFallbackDepth--;
106
+ return;
107
+ }
76
108
  const l = local(n);
77
109
  at.pop();
78
110
  if (l === "r" && stack.length) {
@@ -94,7 +126,7 @@ function readSlide(zip, part) {
94
126
  if (sh.isTitle && !out.title) out.title = text;
95
127
  }
96
128
  },
97
- });
129
+ }, count.budget);
98
130
  out.titleHash = textHash(out.title);
99
131
  return out;
100
132
  }
@@ -113,39 +145,278 @@ export function listShapes(deck, slideId = null) {
113
145
 
114
146
  const slideById = (deck, id) => deck.slides.find((s) => s.id === String(id)) || null;
115
147
 
148
+ // The budget (src/text.js scanBudget): resolving an anchor reads slides, the shapes of a slide and a shape's text, once per anchor. Each
149
+ // such read is spent before it runs, and an anchor whose spend fails resolves as "moved or removed" (the slide or shape alone when
150
+ // it is all that is known). Index-free scans here are over the slides of a deck or the shapes of a slide, both bounded by MAX_* at parse.
151
+ const slideCost = (deck) => deck.slides.length * 4 + 1;
152
+ const shapeCost = (slide) => slide.shapes.length * 8 + 1;
153
+
116
154
  /**
117
155
  * Engine location {slideIndex, slideId?, shapeId?, shapeName?, cell?} (+ the text the browser saw) -> durable anchor, from the written
118
156
  * bytes. `ctx` = {quote, prefix, suffix, text}. Returns null when the slide does not exist.
119
157
  */
120
158
  export function pptxDurable(deck, loc, ctx = {}) {
121
159
  deck.ensure();
122
- const slide = (loc.slideId != null && slideById(deck, loc.slideId)) || deck.slides[loc.slideIndex];
160
+ const slide = (loc.slideId != null && spend(slideCost(deck)) && slideById(deck, loc.slideId)) || deck.slides[loc.slideIndex];
123
161
  if (!slide) return null;
124
- const base = { format: "pptx", ...(loc.protected ? { protected: true } : {}), slideId: slide.id, slidePart: slide.part, slideIndex: slide.index, slideTitle: normText(slide.title).slice(0, 120), titleHash: slide.titleHash, shapeId: null, shapeName: "", ordinal: 0, kind: "slide", text: "", textHash: null, quote: "", prefix: "", suffix: "", start: null, end: null, cell: loc.cell || null };
125
- if (loc.shapeId == null) return base;
162
+ const base = { format: "pptx", ...(loc.protected ? { protected: true } : {}), slideId: slide.id, slidePart: slide.part, slideIndex: slide.index, slideTitle: titleOf(slide), titleHash: slide.titleHash, shapeId: null, shapeName: "", ordinal: 0, kind: "slide", text: "", textHash: null, quote: "", prefix: "", suffix: "", start: null, end: null, cell: loc.cell || null };
163
+ if (loc.shapeId == null || !spend(shapeCost(slide))) return base;
126
164
  const same = slide.shapes.filter((s) => s.id === loc.shapeId);
127
165
  let sh = same[0];
128
166
  if (same.length > 1) {
129
- const want = textHash(ctx.text || ctx.quote || "");
167
+ // the page's text is hashed only when the budget pays for it (null: told apart by name, then the first)
168
+ const seen = ctx.text || ctx.quote || "";
169
+ const want = affords(seen.length) && spend(weigh(seen)) ? textHash(seen) : null;
130
170
  sh = same.find((s) => s.name === loc.shapeName && s.hash === want) || same.find((s) => s.hash === want) || same.find((s) => s.name === loc.shapeName) || same[0];
131
171
  }
132
172
  if (!sh) return { ...base, shapeId: loc.shapeId, shapeName: loc.shapeName || "", kind: "shape" };
133
- const q = normText(ctx.quote || "");
134
- const raw = ctx.quote || "";
135
- const at = q ? sh.text.indexOf(raw) : -1;
173
+ // the page's quote, prefix and suffix are normalized within the budget too (each can be a whole shape's text, per comment); past it
174
+ // the anchor is the shape, without the text range
175
+ const q = normPaid(ctx.quote) ?? "";
176
+ const raw = q ? ctx.quote : "";
177
+ const prefix = (normPaid(ctx.prefix) ?? "").slice(-CTX);
178
+ const suffix = (normPaid(ctx.suffix) ?? "").slice(0, CTX);
179
+ const hintStart = Number.isInteger(loc.start) ? loc.start : null;
180
+ const ctxOpts = { prefix, suffix, nth: ctx.nth ?? ctx.occ ?? null, hintStart };
181
+ let at = -1;
182
+ let len = raw.length;
183
+ let k = null;
184
+ if (q && !ctx.shapeWide) {
185
+ const preferNorm = raw !== q || (q.includes(" ") && HAS_NON_PLAIN_WS.test(sh.text));
186
+ if (preferNorm) {
187
+ const m = findNormMatch(sh, q, ctxOpts);
188
+ if (m) [at, len, k] = [m.start, m.end - m.start, m.k];
189
+ } else {
190
+ const exact = spend(textCost(sh, raw)) ? findExactMatch(sh, raw, ctxOpts) : null;
191
+ if (exact) [at, len, k] = [exact.start, exact.end - exact.start, exact.k];
192
+ else if (exact === undefined) {
193
+ const m = findNormMatch(sh, q, ctxOpts);
194
+ if (m) [at, len, k] = [m.start, m.end - m.start, m.k];
195
+ }
196
+ }
197
+ }
136
198
  return {
137
199
  ...base,
138
200
  shapeId: sh.id, shapeName: sh.name, ordinal: sh.ordinal, kind: sh.kind,
139
- text: normText(sh.text).slice(0, KEEP), textHash: sh.hash,
201
+ text: spend(normCost(sh)) ? normShape(sh).slice(0, KEEP) : "", textHash: sh.hash,
140
202
  // a text range inside the shape: the quote and what surrounds it
141
203
  quote: ctx.shapeWide ? "" : q,
142
- prefix: normText(ctx.prefix || "").slice(-CTX),
143
- suffix: normText(ctx.suffix || "").slice(0, CTX),
204
+ prefix,
205
+ suffix,
144
206
  start: !ctx.shapeWide && at >= 0 ? at : null,
145
- end: !ctx.shapeWide && at >= 0 ? at + raw.length : null,
207
+ end: !ctx.shapeWide && at >= 0 ? at + len : null,
208
+ nth: !ctx.shapeWide && at >= 0 ? k : null,
146
209
  };
147
210
  }
148
211
 
212
+ const MARK = /\p{M}/u;
213
+ const WS_CHAR = /\s/;
214
+ export const HAS_NON_PLAIN_WS = /[\t\r\n\f\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]| /;
215
+ /**
216
+ * May a window start (or end) right before text[i]? Yes below U+0300 (no such character joins the one before it under NFC); never inside a
217
+ * surrogate pair or before a combining mark; otherwise when the character normalizes the same after the three before it as on its own
218
+ * (the longest chain of starters that compose, Hangul L V T and its like, is three: test/parse-limits.test.js checks every code point).
219
+ * Exported for src/docx.js and its test.
220
+ */
221
+ export function cuttable(text, i) {
222
+ if (i <= 0 || i >= text.length) return true;
223
+ const c = text.charCodeAt(i);
224
+ if (c < 0x300) return true;
225
+ if (c >= 0xdc00 && c <= 0xdfff) return false;
226
+ const ch = String.fromCodePoint(text.codePointAt(i));
227
+ if (MARK.test(ch)) return false;
228
+ const before = text.slice(Math.max(0, i - 3), i);
229
+ return (before + ch).normalize("NFC") === before.normalize("NFC") + ch.normalize("NFC");
230
+ }
231
+
232
+ function buildNormMap(text) {
233
+ const parts = [];
234
+ const starts = [];
235
+ const ends = [];
236
+ let seenNonWs = false;
237
+ for (let i = 0; i < text.length; ) {
238
+ if (WS_CHAR.test(text[i])) {
239
+ let j = i + 1;
240
+ while (j < text.length && WS_CHAR.test(text[j])) j++;
241
+ if (seenNonWs && j < text.length) {
242
+ parts.push(" ");
243
+ starts.push(i);
244
+ ends.push(j);
245
+ }
246
+ i = j;
247
+ continue;
248
+ }
249
+ let j = i + (text.codePointAt(i) > 0xffff ? 2 : 1);
250
+ while (j < text.length && !WS_CHAR.test(text[j]) && !cuttable(text, j)) {
251
+ j += text.codePointAt(j) > 0xffff ? 2 : 1;
252
+ }
253
+ const chunk = nfc(text.slice(i, j));
254
+ if (chunk) {
255
+ parts.push(chunk);
256
+ for (let k = 0; k < chunk.length; k++) {
257
+ starts.push(i);
258
+ ends.push(j);
259
+ }
260
+ seenNonWs = true;
261
+ }
262
+ i = j;
263
+ }
264
+ return { norm: parts.join(""), starts: Int32Array.from(starts), ends: Int32Array.from(ends) };
265
+ }
266
+
267
+ const NORM_MAP = new WeakMap();
268
+ const NORM_HITS = new WeakMap();
269
+ const NORM_OCC = new WeakMap();
270
+ const RAW_OCC = new WeakMap();
271
+ const OCC_KEEP = 16;
272
+ export function normMapFor(unit) {
273
+ let map = NORM_MAP.get(unit);
274
+ if (map === undefined) {
275
+ if (!spend(weigh(unit.text))) return null;
276
+ NORM_MAP.set(unit, (map = buildNormMap(unit.text)));
277
+ }
278
+ return map;
279
+ }
280
+ export function normOccurrences(unit, q, max = Infinity) {
281
+ const nq = normText(q);
282
+ if (!nq) return [];
283
+ if (max === Infinity) {
284
+ const cached = NORM_OCC.get(unit)?.get(nq);
285
+ if (cached) return cached;
286
+ }
287
+ const map = normMapFor(unit);
288
+ if (!map) return null;
289
+ if (!spend(searchCost(map.norm.length, nq, hitsOf(NORM_HITS, unit, map.norm, nq)))) return null;
290
+ const out = [];
291
+ for (let i = map.norm.indexOf(nq); i >= 0 && out.length < max; i = map.norm.indexOf(nq, i + nq.length)) {
292
+ if (!spend(OCCURRENCE)) return null;
293
+ out.push({ start: map.starts[i], end: map.ends[i + nq.length - 1], normStart: i, normEnd: i + nq.length });
294
+ }
295
+ if (max === Infinity) {
296
+ let m = NORM_OCC.get(unit);
297
+ if (!m) NORM_OCC.set(unit, (m = new Map()));
298
+ if (m.size >= OCC_KEEP) m.delete(m.keys().next().value);
299
+ m.set(nq, out);
300
+ }
301
+ return out;
302
+ }
303
+
304
+ function exactOccurrencesIn(sh, raw, first, second, ctxPerOcc = 0) {
305
+ let m = RAW_OCC.get(sh);
306
+ const cached = m?.get(raw);
307
+ if (cached) return cached;
308
+ if (!spend(2 * OCCURRENCE)) return null;
309
+ const occ = [first, second];
310
+ for (let i = sh.text.indexOf(raw, second + raw.length); i >= 0; i = sh.text.indexOf(raw, i + raw.length)) {
311
+ if (ctxPerOcc && !affords((occ.length + 1) * ctxPerOcc)) return null;
312
+ if (!spend(OCCURRENCE)) return null;
313
+ occ.push(i);
314
+ }
315
+ if (!m) RAW_OCC.set(sh, (m = new Map()));
316
+ if (m.size >= OCC_KEEP) m.delete(m.keys().next().value);
317
+ m.set(raw, occ);
318
+ return occ;
319
+ }
320
+
321
+ function findExactMatch(sh, raw, { prefix = "", suffix = "", nth = null, hintStart = null } = {}) {
322
+ const first = sh.text.indexOf(raw);
323
+ if (first < 0) return undefined;
324
+ const second = sh.text.indexOf(raw, first + raw.length);
325
+ if (second < 0) return { start: first, end: first + raw.length, k: 0 };
326
+ if (Number.isInteger(hintStart) && hintStart >= 0 && sh.text.slice(hintStart, hintStart + raw.length) === raw) {
327
+ let k = 0;
328
+ let found = false;
329
+ for (let i = first; i >= 0 && i <= hintStart; i = sh.text.indexOf(raw, i + raw.length)) {
330
+ if (!spend(OCCURRENCE)) {
331
+ k = null;
332
+ break;
333
+ }
334
+ if (i === hintStart) {
335
+ found = true;
336
+ break;
337
+ }
338
+ k++;
339
+ }
340
+ return { start: hintStart, end: hintStart + raw.length, k: found ? k : null };
341
+ }
342
+ const pre = normText(prefix).slice(-CTX);
343
+ const suf = normText(suffix).slice(0, CTX);
344
+ const ctxPerOcc = pre || suf ? 2 * (pre.length + suf.length + 16) : 0;
345
+ const occ = exactOccurrencesIn(sh, raw, first, second, ctxPerOcc);
346
+ if (!occ) return null;
347
+ if (ctxPerOcc && !spend(occ.length * ctxPerOcc)) return null;
348
+ const guarded = [];
349
+ for (let i = 0; i < occ.length; i++) {
350
+ const at = occ[i];
351
+ const okPre = !pre || normText(sh.text.slice(Math.max(0, at - 4 * pre.length - 16), at)).endsWith(pre);
352
+ const okSuf = !suf || normText(sh.text.slice(at + raw.length, at + raw.length + 4 * suf.length + 16)).startsWith(suf);
353
+ if (okPre && okSuf) guarded.push(i);
354
+ }
355
+ const validNth = Number.isInteger(nth) && nth >= 0 && nth < occ.length ? nth : null;
356
+ let idx;
357
+ if (guarded.length === 1) idx = guarded[0];
358
+ else if (validNth !== null && guarded.includes(validNth)) idx = validNth;
359
+ else if (Number.isInteger(hintStart) && guarded.length) {
360
+ idx = guarded.reduce((best, i) => (Math.abs(occ[i] - hintStart) < Math.abs(occ[best] - hintStart) ? i : best), guarded[0]);
361
+ } else if (guarded.length) idx = guarded[0];
362
+ else idx = validNth !== null ? validNth : 0;
363
+ return { start: occ[idx], end: occ[idx] + raw.length, k: idx };
364
+ }
365
+
366
+ function findNormMatch(sh, q, { prefix = "", suffix = "", nth = null, hintStart = null } = {}) {
367
+ const pre = normText(prefix).slice(-CTX);
368
+ const suf = normText(suffix).slice(0, CTX);
369
+ const needAll = !!(pre || suf || Number.isInteger(nth) || Number.isInteger(hintStart));
370
+ const occ = normOccurrences(sh, q, needAll ? Infinity : 1);
371
+ if (!occ?.length) return null;
372
+ if (occ.length === 1) return { ...occ[0], k: 0 };
373
+ const validNth = Number.isInteger(nth) && nth >= 0 && nth < occ.length ? nth : null;
374
+ if ((pre || suf) && !spend(occ.length * 2 * (pre.length + suf.length + 16))) return null;
375
+ const map = normMapFor(sh);
376
+ const guarded = [];
377
+ for (let i = 0; i < occ.length; i++) {
378
+ const o = occ[i];
379
+ const okPre = !pre || !map || map.norm.slice(Math.max(0, o.normStart - pre.length - 1), o.normStart).trimEnd().endsWith(pre);
380
+ const okSuf = !suf || !map || map.norm.slice(o.normEnd, o.normEnd + suf.length + 1).trimStart().startsWith(suf);
381
+ if (okPre && okSuf) guarded.push(i);
382
+ }
383
+ let idx;
384
+ if (guarded.length === 1) idx = guarded[0];
385
+ else if (validNth !== null && guarded.includes(validNth)) idx = validNth;
386
+ else if (Number.isInteger(hintStart) && guarded.length) {
387
+ idx = guarded.reduce((best, i) => (Math.abs(occ[i].start - hintStart) < Math.abs(occ[best].start - hintStart) ? i : best), guarded[0]);
388
+ } else if (guarded.length) idx = guarded[0];
389
+ else idx = validNth !== null ? validNth : 0;
390
+ return { ...occ[idx], k: idx };
391
+ }
392
+
393
+ // A shape's normalized text, kept per shape of a reading: every anchor on a shape normalized all of it again (50 comments on a shape of
394
+ // 8 million characters held the server's thread for 5.6 s while the page placed them).
395
+ const NORM = new WeakMap();
396
+ const normShape = (sh) => {
397
+ let v = NORM.get(sh);
398
+ if (v === undefined) NORM.set(sh, (v = normText(sh.text)));
399
+ return v;
400
+ };
401
+ // A slide's normalized title, kept per slide of a reading and paid from the budget the first time: every anchor on a slide normalized the
402
+ // whole title again, outside the budget. Past the budget the anchor has no title text ("", asked again by the next request); the title's
403
+ // hash, read with the slide, still identifies it.
404
+ const TITLE = new WeakMap();
405
+ const titleOf = (slide) => {
406
+ let v = TITLE.get(slide);
407
+ if (v === undefined) {
408
+ const t = normPaid(slide.title);
409
+ if (t === null) return "";
410
+ TITLE.set(slide, (v = t.slice(0, 120)));
411
+ }
412
+ return v;
413
+ };
414
+ /** what normShape(sh) is about to read: the shape's text the first time (src/text.js weigh), nothing once it is kept */
415
+ const normCost = (sh) => (NORM.has(sh) ? 0 : weightOf(sh));
416
+ /** a native substring search for `q` in a shape's text (src/text.js searchCost), priced by the shape's byte table (src/text.js hitsOf) */
417
+ const HITS = new WeakMap();
418
+ const textCost = (sh, q) => searchCost(sh.text.length, q, hitsOf(HITS, sh, sh.text, q));
419
+
149
420
  /**
150
421
  * Re-resolve against new bytes. Slide: by p:sldId id; else the part path together with the title; else the title alone when unique.
151
422
  * Shape (when the anchor has one): by id (+ ordinal); else the same name with the same text; else the same text; else the same name.
@@ -154,6 +425,7 @@ export function pptxDurable(deck, loc, ctx = {}) {
154
425
  export function resolvePptx(deck, d) {
155
426
  if (!d) return { method: "none" };
156
427
  deck.ensure();
428
+ if (!spend(slideCost(deck))) return { method: "none" };
157
429
  let slide = d.slideId != null ? slideById(deck, d.slideId) : null;
158
430
  let method = "id";
159
431
  if (!slide) {
@@ -166,6 +438,7 @@ export function resolvePptx(deck, d) {
166
438
  const out = { method, slideId: slide.id, slideIndex: slide.index, shapeId: null, shapeName: "", start: null, end: null, changed: false, text: "", titleChanged: slide.titleHash !== d.titleHash };
167
439
  if (d.shapeId == null) return out;
168
440
  const shapes = slide.shapes;
441
+ if (!spend(shapeCost(slide))) return { method: "none" };
169
442
  const byId = shapes.filter((s) => s.id === d.shapeId);
170
443
  let sh = null;
171
444
  let how = "";
@@ -183,10 +456,37 @@ export function resolvePptx(deck, d) {
183
456
  else if (name.length === 1 && d.textHash && d.textHash !== EMPTY) [sh, how] = [name[0], "name"];
184
457
  }
185
458
  if (!sh) return { method: "none" };
186
- const stext = normText(sh.text);
459
+ // The shape is found (by its id, or its name and the hash read with the slide). Its text and the quote's place in it are read only when
460
+ // the budget pays for them: past it the comment is still on its shape, without a text excerpt or a range (a shape of 7.9 million CJK
461
+ // characters costs more than the whole budget to normalize, and its comments resolved as "moved or removed" with the shape right there).
462
+ const stext = spend(normCost(sh)) ? normShape(sh) : "";
187
463
  const q = d.quote || "";
188
- const at = q ? sh.text.indexOf(q) : -1;
189
- return { ...out, method: method === "id" ? how : `${method}+${how}`, shapeId: sh.id, shapeName: sh.name, start: at >= 0 ? at : null, end: at >= 0 ? at + q.length : null, changed: sh.hash !== d.textHash, text: stext.slice(0, 200), quoteLost: !!q && at < 0 };
464
+ const searched = !!q && spend(textCost(sh, q)); // a shape-wide comment has no quote, and no search to pay for
465
+ const ctxOpts = { prefix: d.prefix ?? d.before ?? "", suffix: d.suffix ?? d.after ?? "", nth: d.nth ?? d.occ ?? null };
466
+ let at = -1;
467
+ let len = q.length;
468
+ let k = null;
469
+ if (searched) {
470
+ const preferNorm = q.includes(" ") && HAS_NON_PLAIN_WS.test(sh.text);
471
+ if (preferNorm) {
472
+ if (stext) {
473
+ const m = findNormMatch(sh, q, ctxOpts);
474
+ if (m) [at, len, k] = [m.start, m.end - m.start, m.k];
475
+ }
476
+ } else {
477
+ const exact = findExactMatch(sh, q, ctxOpts);
478
+ if (exact && !(Number.isInteger(ctxOpts.nth) && ctxOpts.nth > 0 && exact.k !== ctxOpts.nth)) {
479
+ [at, len, k] = [exact.start, exact.end - exact.start, exact.k];
480
+ } else if (exact !== null && stext) {
481
+ const m = findNormMatch(sh, q, ctxOpts);
482
+ if (m) [at, len, k] = [m.start, m.end - m.start, m.k];
483
+ else if (exact) [at, len, k] = [exact.start, exact.end - exact.start, exact.k];
484
+ } else if (exact) {
485
+ [at, len, k] = [exact.start, exact.end - exact.start, exact.k];
486
+ }
487
+ }
488
+ }
489
+ return { ...out, method: method === "id" ? how : `${method}+${how}`, shapeId: sh.id, shapeName: sh.name, start: at >= 0 ? at : null, end: at >= 0 ? at + len : null, nth: at >= 0 ? k : null, changed: sh.hash !== d.textHash, text: stext.slice(0, 200), quoteLost: searched && at < 0 };
190
490
  }
191
491
 
192
492
  /** Locator fields for the batch: the names the `apply` targets use. */
package/src/question.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { untrusted } from "./sanitize.js";
2
+ import { foldLinkText } from "./voice-guide.js";
2
3
 
3
4
  /**
4
5
  * A question the agent asks instead of declining: `{"comment":"c1","ask":"Which date?","options":["March 3","March 13"]}`.
@@ -16,7 +17,12 @@ export const QUESTION_FIX = "A question needs 2 to 4 short options (40 character
16
17
  /** The note on the card when `done` received a question it would not show. */
17
18
  export const BAD_QUESTION_NOTE = "The agent asked a question that could not be shown.";
18
19
 
19
- const LINK = /:\/\/|www\./i;
20
+ // Refuse only ://, www., or email addresses (mailto: or a@b.c), after dot-folding and invisible-char stripping.
21
+ // Bare domains without a scheme ("evil.com") are an accepted gap so ordinary text ("Finished.No", "Socket.IO", "main.py:42") always passes.
22
+ // One character before @ proves the local part is nonempty without scanning each suffix.
23
+ const LINK = /:\/\/|www\.|mailto:|[^\s@]@[^\s@.]+\.[^\s@.]/iu;
24
+ // Invisible characters left after untrusted() cleans the displayed text.
25
+ const visibleText = (s) => s.replace(/[\u00AD\u034F\u115F\u1160\u17B4\u17B5\u180E\u2800\u3164\uFE0E\uFE0F\uFFA0]/gu, "");
20
26
  const len = (s) => [...s].length; // code points, as untrusted() and the schema count
21
27
 
22
28
  /** One string: cleaned, then refused when empty, over `max` or a link. Returns {text} or {reason}. */
@@ -25,7 +31,8 @@ function cleanOne(v, max, what) {
25
31
  const text = untrusted(v, Number.MAX_SAFE_INTEGER);
26
32
  if (!text) return { reason: `${what} is empty` };
27
33
  if (len(text) > max) return { reason: `${what} is longer than ${max} characters` };
28
- if (LINK.test(text)) return { reason: `${what} contains a link` };
34
+ const folded = foldLinkText(visibleText(text));
35
+ if (LINK.test(folded)) return { reason: `${what} contains a link` };
29
36
  return { text };
30
37
  }
31
38
 
@@ -42,6 +49,6 @@ export function cleanQuestion({ ask, options } = {}) {
42
49
  if (c.reason) return refuse(c.reason);
43
50
  out.push(c.text);
44
51
  }
45
- if (new Set(out.map((o) => o.toLowerCase())).size !== out.length) return refuse("two options are the same");
52
+ if (new Set(out.map((o) => visibleText(o).toLowerCase())).size !== out.length) return refuse("two options are the same");
46
53
  return { question: { text: q.text, options: out } };
47
54
  }
package/src/quick-plan.js CHANGED
@@ -5,10 +5,15 @@
5
5
  // binds that number as `expect.count`, so a file that changed since the preview, or a tampered number, fails the count check and nothing
6
6
  // is written (the batch then goes to the agent unchanged).
7
7
  //
8
- // Ceiling: previews run the engine on the server's main thread (as the merge does); a large file pauses other requests while one runs.
9
- // The page asks 600 ms after typing stops, and answers are cached by file hash and command. Upgrade path: a worker.
8
+ // The engine's count runs in a worker thread with a time limit (src/offload.js previewOffThread), as the change itself does (src/local.js):
9
+ // a document of one paragraph of 8 million characters held the server's thread for 4 s in an outside review. A count that does not
10
+ // finish is no count: the page shows no button and the agent gets the note. The page asks 600 ms after typing stops, and answers are
11
+ // cached by file hash and command. The bytes are read whole within the readers' caps first (readChecked), so a file swapped on disk
12
+ // past a cap is refused in plain words before the engine reads it.
10
13
 
11
14
  import { checkRegularOwned, readDoc, sha256 } from "./fsutil.js";
15
+ import { readChecked } from "./session.js";
16
+ import { previewOffThread } from "./offload.js";
12
17
  import { recognize, refusalFor, wordsFor } from "./quick.js";
13
18
  import { userText } from "./sanitize.js";
14
19
 
@@ -30,8 +35,7 @@ function ridesAlong(s) {
30
35
  }
31
36
 
32
37
  async function previewOf(bytes, format, q) {
33
- const { previewAll } = await import("./ops/run.js"); // pulls in the engine: loaded on first use
34
- const p = await previewAll(bytes, format, q.op);
38
+ const p = await previewOffThread(bytes, format, q.op); // throws when the worker did not finish: no count, nothing shown or written
35
39
  const count = p.ok ? p.count : 0;
36
40
  const skipped = p.ok ? p.protected || 0 : 0;
37
41
  let why = refusalFor(q.kind, p, q.find);
@@ -56,9 +60,11 @@ export async function quickPreview(s, note) {
56
60
  } catch {
57
61
  return { match: null };
58
62
  }
59
- const key = `${sha256(bytes)}\0${s.format}\0${q.command}`;
63
+ const sha = sha256(bytes);
64
+ const key = `${sha}\0${s.format}\0${q.command}`;
60
65
  let match = cache.get(key);
61
66
  if (!match) {
67
+ readChecked(s.format, bytes, { sha }); // past a reader's cap: refused with the reason (422), the engine never sees it
62
68
  try {
63
69
  const { count, skipped, why } = await previewOf(bytes, s.format, q);
64
70
  match = { action: q.kind, command: q.command, count, protectedCount: skipped, local: !why, why: why || null, ...wordsFor(q, count, skipped) };
@@ -96,10 +102,11 @@ export const quickPlanner = {
96
102
  try {
97
103
  checkRegularOwned(s.file);
98
104
  bytes = readDoc(s.file);
105
+ if (sha256(bytes) !== batch.base_sha256) return null; // the file moved since the Send
106
+ readChecked(s.format, bytes, { sha: batch.base_sha256 }); // within the readers' caps before the engine reads it
99
107
  } catch {
100
108
  return null;
101
109
  }
102
- if (sha256(bytes) !== batch.base_sha256) return null; // the file moved since the Send
103
110
  const { count, skipped, why } = await previewOf(bytes, batch.format, q);
104
111
  if (why || count !== confirmed.count) return null; // not what the person saw
105
112
  return { reply: { summary: wordsFor(q, count, skipped).summary, replies: [], ops: [{ ...q.op, expect: { count } }] } };
package/src/quick.js CHANGED
@@ -78,7 +78,7 @@ function parseReplace(t) {
78
78
  let rest;
79
79
  if (OPEN_QUOTES.has(body[0])) {
80
80
  const q = quoted(body);
81
- if (!q || !q.rest.startsWith(" with ")) return null;
81
+ if (!q || !/^ with /i.test(q.rest)) return null;
82
82
  find = { text: q.value, quoted: true };
83
83
  rest = q.rest.slice(" with ".length);
84
84
  } else {
@@ -205,7 +205,11 @@ export function verifyTracked({ before, after, out, counts, author, date }) {
205
205
  const zb = openZip(before);
206
206
  const zo = openZip(out);
207
207
  if (zo.names.join("\0") !== za.names.join("\0")) fail("the copy's parts differ from the agent's version");
208
- for (const n of za.names) if (n !== "word/document.xml" && !zo.read(n).equals(za.read(n))) fail(`a part other than the main text differs: ${n}`);
208
+ for (const n of za.names) {
209
+ if (n === "word/document.xml") continue;
210
+ const sameRaw = zo.entries.get(n)?.method === za.entries.get(n)?.method && zo.raw(n).equals(za.raw(n));
211
+ if (!sameRaw && !zo.read(n).equals(za.read(n))) fail(`a part other than the main text differs: ${n}`);
212
+ }
209
213
  const xa = za.readText("word/document.xml");
210
214
  const xb = zb.readText("word/document.xml");
211
215
  const xo = zo.readText("word/document.xml");
@@ -221,7 +225,13 @@ export function verifyTracked({ before, after, out, counts, author, date }) {
221
225
  return n;
222
226
  };
223
227
  if (revCount(xo) !== ours.size + revCount(xa)) fail("the copy has revisions that are neither ours nor the file's own");
224
- const maxIn = Math.max(0, ...[...inputIds].filter((v) => /^\d+$/.test(v)).map(Number));
228
+ let maxIn = 0;
229
+ for (const v of inputIds) {
230
+ if (/^\d+$/.test(v)) {
231
+ const n = Number(v);
232
+ if (n > maxIn) maxIn = n;
233
+ }
234
+ }
225
235
  for (const id of ours) if (Number(id) <= maxIn) fail("a revision id is not above every id in the file");
226
236
 
227
237
  const A = parseDocxParagraphs(xa);
@@ -1,29 +1,25 @@
1
1
  import fs from "node:fs";
2
- import { parentPort, workerData } from "node:worker_threads";
3
2
  import { buildTracked } from "./redline.js";
4
3
  import { verifyTracked } from "./redline-check.js";
5
4
  import { env } from "./constants.js";
6
5
  import { openZip } from "./zip.js";
7
6
  import { replaceEntry } from "./zip-write.js";
8
7
 
9
- // One tracked-changes copy, off the server thread: build it (src/redline.js), then check it (src/redline-check.js). One message back:
8
+ // One tracked-changes copy, off the server thread (run by src/worker-host.js): build it (src/redline.js), then check it (src/redline-check.js). One message back:
10
9
  // {ok: true, bytes, counts} | {refused: [codes]} | {check_failed: why}. The copy's bytes only leave this worker after the check passed.
11
10
  // ONIONSKIN_TEST_TAMPER_EXPORT=drop-del (tests only) removes one deletion from the built copy before the check, to prove the check is
12
11
  // what stands between the builder and the disk; it can only make a copy fail, never pass.
13
- const { before, after, author, date } = workerData;
14
- let msg;
15
- try {
16
- const b = fs.readFileSync(before);
17
- const a = fs.readFileSync(after);
18
- const r = buildTracked({ before: b, after: a, author, date });
19
- if (r.refused) msg = { refused: r.refused };
20
- else {
12
+ export function work({ before, after, author, date }) {
13
+ try {
14
+ const b = fs.readFileSync(before);
15
+ const a = fs.readFileSync(after);
16
+ const r = buildTracked({ before: b, after: a, author, date });
17
+ if (r.refused) return { refused: r.refused };
21
18
  let out = r.bytes;
22
19
  if (env("TEST_TAMPER_EXPORT", "") === "drop-del") out = replaceEntry(out, "word/document.xml", openZip(out).readText("word/document.xml").replace(/<w:del [^>]*>[\s\S]*?<\/w:del>/, ""));
23
20
  const v = verifyTracked({ before: b, after: a, out, counts: r.counts, author, date });
24
- msg = v.ok ? { ok: true, bytes: out, counts: r.counts } : { check_failed: v.why };
21
+ return v.ok ? { ok: true, bytes: out, counts: r.counts } : { check_failed: v.why };
22
+ } catch (err) {
23
+ return { check_failed: `the copy could not be made: ${String(err?.message || err).slice(0, 200)}` };
25
24
  }
26
- } catch (err) {
27
- msg = { check_failed: `the copy could not be made: ${String(err?.message || err).slice(0, 200)}` };
28
25
  }
29
- parentPort.postMessage(msg);