onionskin-cli 0.0.2 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +111 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +101 -33
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/skills/onionskin/SKILL.md +9 -6
  9. package/skills/onionskin/reference/batch.md +5 -3
  10. package/skills/onionskin/reference/formats.md +5 -1
  11. package/skills/onionskin/reference/ops.md +47 -3
  12. package/skills/onionskin/reference/troubleshooting.md +31 -0
  13. package/src/agent-detect.js +28 -2
  14. package/src/agent-prompt.js +12 -1
  15. package/src/agent-runner.js +77 -19
  16. package/src/anchor-work.js +154 -0
  17. package/src/anchor-worker.js +38 -0
  18. package/src/anchors.js +61 -11
  19. package/src/apply.js +119 -47
  20. package/src/batch.js +32 -8
  21. package/src/check-worker.js +8 -5
  22. package/src/cli.js +91 -25
  23. package/src/constants.js +34 -1
  24. package/src/diff.js +3 -1
  25. package/src/docx.js +368 -58
  26. package/src/edit-hash.js +19 -0
  27. package/src/fsutil.js +1 -1
  28. package/src/history-worker.js +8 -8
  29. package/src/history.js +135 -31
  30. package/src/html-active.js +384 -0
  31. package/src/html-envelope.js +51 -0
  32. package/src/html-parser.js +8 -0
  33. package/src/html-render-worker.js +25 -0
  34. package/src/html-render.js +880 -0
  35. package/src/html-utf8.js +65 -0
  36. package/src/html-write-worker.js +14 -0
  37. package/src/html-write.js +380 -0
  38. package/src/html.js +1257 -0
  39. package/src/local.js +68 -4
  40. package/src/merge-package.js +1110 -0
  41. package/src/merge-pptx.js +98 -38
  42. package/src/merge-proof.js +1019 -0
  43. package/src/merge-worker.js +60 -0
  44. package/src/merge-xlsx.js +84 -29
  45. package/src/merge.js +286 -83
  46. package/src/nfc.js +46 -0
  47. package/src/notes.js +122 -0
  48. package/src/offload.js +400 -1
  49. package/src/open-worker.js +27 -0
  50. package/src/ops/common.js +2 -1
  51. package/src/ops/docx.js +58 -29
  52. package/src/ops/html.js +394 -0
  53. package/src/ops/pptx.js +74 -21
  54. package/src/ops/run.js +23 -3
  55. package/src/ops/runs.js +215 -13
  56. package/src/ops/schema.js +8 -1
  57. package/src/ops/xlsx.js +12 -7
  58. package/src/ops-worker.js +17 -0
  59. package/src/package.js +26 -2
  60. package/src/person.js +8 -1
  61. package/src/pptx.js +85 -21
  62. package/src/quick-plan.js +13 -6
  63. package/src/redline-worker.js +10 -14
  64. package/src/reply.js +46 -19
  65. package/src/retain.js +223 -0
  66. package/src/review.js +32 -18
  67. package/src/rewrite-check.js +2 -2
  68. package/src/sanitize.js +16 -3
  69. package/src/server.js +88 -37
  70. package/src/session.js +1143 -208
  71. package/src/stop-gate.js +34 -0
  72. package/src/suggest-reply.js +42 -7
  73. package/src/suggest.js +14 -29
  74. package/src/text.js +161 -11
  75. package/src/update.js +288 -0
  76. package/src/voice-guide.js +4 -3
  77. package/src/voice-learn.js +4 -3
  78. package/src/voice-sample-worker.js +7 -6
  79. package/src/voice-worker.js +4 -6
  80. package/src/watch.js +188 -0
  81. package/src/worker-host.js +23 -0
  82. package/src/xlsx.js +355 -94
  83. package/src/xml-tree.js +6 -3
  84. package/src/xml.js +82 -34
  85. package/src/zip-write.js +2 -1
  86. package/src/zip.js +9 -5
  87. package/vendor/parse5/LICENSE.txt +32 -0
  88. package/vendor/parse5/parse5.mjs +7822 -0
  89. package/vendor/parse5.lock.json +23 -0
  90. package/web/dist/assets/index-DkNx0LSH.css +1 -0
  91. package/web/dist/assets/{index-BUtofdtr.js → index-DytKkMdi.js} +117 -107
  92. package/web/dist/index.html +2 -2
  93. package/web/dist/assets/index-Dfizlg0i.css +0 -1
package/src/package.js CHANGED
@@ -4,6 +4,7 @@ import { openZip, ZipError } from "./zip.js";
4
4
  import { walkXml } from "./xml.js";
5
5
  import { runDeepCheck } from "./checker.js";
6
6
  import { cleanFlag } from "./sanitize.js";
7
+ import { decodeHtml, HtmlEncodingError, readHtml } from "./html.js";
7
8
 
8
9
  const MAX_TYPES_BYTES = 4 * 1024 * 1024; // [Content_Types].xml of a real document is a few KB
9
10
  // Cumulative work of the relationship scan: a hostile archive can hold thousands of individually allowed .rels parts.
@@ -14,6 +15,13 @@ const MAX_RELS_BYTES = 64 * 1024 * 1024;
14
15
  export function validatePackage(bytes, format) {
15
16
  const want = FORMATS[format];
16
17
  if (!want) throw new ZipError(`unsupported format: ${format}`);
18
+ // an HTML file is no package: its encoding is checked here (no parse; the parser lives in worker threads only, src/html-parser.js).
19
+ // Returns null; the open worker reads it whole (src/open-worker.js), a write goes through src/html-write.js
20
+ if (format === "html") {
21
+ const d = decodeHtml(bytes);
22
+ if (!d.ok) throw new HtmlEncodingError(d);
23
+ return null;
24
+ }
17
25
  const zip = openZip(bytes);
18
26
  const ct = zip.readText("[Content_Types].xml", MAX_TYPES_BYTES);
19
27
  if (ct == null) throw new ZipError("missing [Content_Types].xml");
@@ -36,11 +44,11 @@ function relTargets(zip) {
36
44
  let bytes = 0;
37
45
  for (const name of zip.names) {
38
46
  if (!name.endsWith(".rels")) continue;
39
- if (++parts > MAX_RELS_PARTS) throw new ZipError(`too many relationship parts (more than ${MAX_RELS_PARTS})`);
47
+ if (++parts > MAX_RELS_PARTS) throw new ZipError(`this file has more than ${MAX_RELS_PARTS.toLocaleString("en-US")} relationship parts, more than Onionskin can read. Save it again in Word, Excel or PowerPoint, or split it, and open that`);
40
48
  const xml = zip.readText(name);
41
49
  if (xml == null) continue;
42
50
  bytes += xml.length;
43
- if (bytes > MAX_RELS_BYTES) throw new ZipError("the relationship parts are too large in total");
51
+ if (bytes > MAX_RELS_BYTES) throw new ZipError(`the relationship parts of this file are larger than ${MAX_RELS_BYTES / 1024 / 1024} MB in all, more than Onionskin can read. Save it again in Word, Excel or PowerPoint, or split it, and open that`);
44
52
  const srcDir = path.posix.dirname(path.posix.dirname(name)); // word/_rels/x.rels -> word
45
53
  const base = srcDir === "." ? "" : srcDir;
46
54
  walkXml(xml, {
@@ -157,6 +165,21 @@ export function nodeChecks(preBytes, postBytes, format) {
157
165
  return rawNodeChecks(preBytes, postBytes, format).map(cleanFlag);
158
166
  }
159
167
 
168
+ /**
169
+ * The check of an HTML file at `done` (no package, no deeper check, PLAN-html D9): the version the agent left still reads as HTML within
170
+ * Onionskin's limits (every write through Onionskin was checked whole before it was written, src/html-write.js; a direct save was not).
171
+ * Needs the parser: runs in the check worker (src/check-worker.js).
172
+ */
173
+ function htmlChecks(postBytes) {
174
+ const skipped = { status: "skipped", reason: "the deeper checks are for Word, Excel and PowerPoint files", code: "not_for_html" };
175
+ try {
176
+ readHtml(postBytes);
177
+ return { flags: [], checks: { node: "ran", deep: skipped, notice: "Checked that the page still reads as HTML within Onionskin's limits. The deeper checks are for Word, Excel and PowerPoint files." } };
178
+ } catch (err) {
179
+ return { flags: [cleanFlag({ code: "html_unreadable", severity: "error", detail: String(err?.message || err).slice(0, 200) })], checks: { node: "ran", deep: skipped, notice: "The page the agent left cannot be read as HTML by Onionskin. Revert restores the version before the run." } };
180
+ }
181
+ }
182
+
160
183
  /** The one sentence about the deeper check. A LibreOffice check still running is "pending", never "passed". */
161
184
  export const deepNotice = (skipped = [], pending = false) =>
162
185
  `Deeper check ran${skipped.length ? `; skipped: ${skipped.map((x) => `${x.check} (${x.reason})`).join(", ")}` : ""}${pending ? "; the LibreOffice conversion check is still running (pending), the Activity list updates when it finishes" : ""}.`;
@@ -168,6 +191,7 @@ const flagKey = (f) => JSON.stringify([f.code, f.part, f.element, f.rels, f.targ
168
191
  * Returns {flags, checks}: `flags` are findings (severity info/warn/error); `checks` says what ran, so a skipped deep check is never silent.
169
192
  */
170
193
  export function checkPackages(preBytes, postBytes, format, opts = {}) {
194
+ if (format === "html") return htmlChecks(postBytes);
171
195
  const node = rawNodeChecks(preBytes, postBytes, format);
172
196
  if (node.some((f) => f.code === "invalid_package")) return { flags: node.map(cleanFlag), checks: { node: "ran", deep: { status: "skipped", reason: "the package is invalid" } } };
173
197
  const deep = runDeepCheck(preBytes, postBytes, format, { deferSoffice: !!opts.deferSoffice });
package/src/person.js CHANGED
@@ -3,7 +3,8 @@
3
3
  import { PRODUCT } from "./constants.js";
4
4
 
5
5
  const BIN = PRODUCT.bin;
6
- const DOC_EXT = /\.(docx|xlsx|pptx)$/i;
6
+ // the formats Onionskin opens, and the web files it refuses with a sentence (src/constants.js REFUSED_PAGES): `onionskin page.mhtml` says why
7
+ const DOC_EXT = /\.(docx|xlsx|pptx|html|htm|mhtml|mht|webarchive|xhtml|svg)$/i;
7
8
  export const GET_CLAUDE_CODE = "https://claude.com/product/claude-code";
8
9
 
9
10
  /**
@@ -15,6 +16,12 @@ export const AGENT_ENV = ["CLAUDECODE", "CODEX_THREAD_ID", "CODEX_TOOL_CALL_ID",
15
16
 
16
17
  export const shellQuote = (s) => `'${String(s).replace(/'/g, `'\\''`)}'`;
17
18
 
19
+ /** A path as one POSIX shell word: bare when it holds only safe characters, else in single quotes. */
20
+ export const shellWord = (s) => (/^[A-Za-z0-9_\/.\-+@%,:=]+$/.test(String(s)) ? String(s) : shellQuote(s));
21
+
22
+ /** What a person pastes in a terminal, from any folder, to reopen a file: the command with the file's absolute path. */
23
+ export const reopenCommand = (file) => `${BIN} ${shellWord(file)}`;
24
+
18
25
  /** True when an agent or a script runs this command: an agent's environment variable, or stdin or stdout that is not a terminal. */
19
26
  export function isAgentRun({ env = {}, stdinTTY = false, stdoutTTY = false } = {}) {
20
27
  if (AGENT_ENV.some((k) => env[k])) return true;
package/src/pptx.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { attrOf, local, readRels } from "./xlsx.js";
2
- import { walkXml } from "./xml.js";
3
- import { normText, similarity, textHash } from "./text.js";
2
+ import { readBudget, tooMany, walkXml } from "./xml.js";
3
+ import { affords, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
4
4
  import { openZip } from "./zip.js";
5
5
  import { summarizeRuns } from "./fmt.js";
6
6
  import { untrusted } from "./sanitize.js";
@@ -13,6 +13,15 @@ const EMPTY = textHash("");
13
13
  const CTX = 40;
14
14
  const KEEP = 2000;
15
15
 
16
+ // What one reading of a deck may hold, counted over all its slides. The server reads a deck on its own thread at Send, at each delivery to
17
+ // an agent and at the end of a run, so these bound how long that keeps it from answering (outside review: three slides of 2.85 million runs
18
+ // each held it 7.2 s). Each shape costs a few hundred bytes once read. A real deck of 200 slides has a few thousand shapes and well under
19
+ // 200,000 elements.
20
+ export const MAX_SHAPES = 50_000;
21
+ export const MAX_DECK_ELEMENTS = 500_000;
22
+ export const MAX_DECK_ATTRIBUTES = 1_000_000;
23
+ export const MAX_DECK_TEXT = 8_000_000;
24
+ const deckBudget = () => readBudget({ elements: MAX_DECK_ELEMENTS, attrs: MAX_DECK_ATTRIBUTES, text: MAX_DECK_TEXT, what: "deck", split: "smaller decks" });
16
25
  const SHAPE_TAGS = new Set(["sp", "pic", "graphicFrame", "grpSp", "cxnSp"]);
17
26
  const KIND = { sp: "shape", pic: "image", graphicFrame: "table", grpSp: "group", cxnSp: "line" };
18
27
 
@@ -21,23 +30,34 @@ export function parseDeck(bytes) {
21
30
  const zip = openZip(bytes);
22
31
  const px = zip.readText("ppt/presentation.xml");
23
32
  if (px == null) throw new Error("ppt/presentation.xml missing");
24
- const rels = readRels(zip, "ppt/presentation.xml");
33
+ const budget = deckBudget(); // the presentation, its relationships and every slide of one reading share it
34
+ const rels = readRels(zip, "ppt/presentation.xml", budget);
25
35
  const slides = [];
26
36
  walkXml(px, {
27
37
  onOpen(n, a) {
28
38
  if (local(n) === "sldId") slides.push({ id: a.id ?? null, index: slides.length, part: rels.get(attrOf(a, "id")) || null });
29
39
  },
30
- });
40
+ }, budget);
41
+ // read once; a reading that failed (past a cap) fails the same way every time it is asked again, never "already read"
31
42
  let loaded = false;
43
+ let failed = null;
32
44
  const load = () => {
45
+ if (failed) throw failed;
33
46
  if (loaded) return;
34
- loaded = true;
35
- for (const s of slides) Object.assign(s, readSlide(zip, s.part));
47
+ try {
48
+ const count = { shapes: 0, budget };
49
+ const read = slides.map((s) => readSlide(zip, s.part, count));
50
+ slides.forEach((s, i) => Object.assign(s, read[i]));
51
+ loaded = true;
52
+ } catch (err) {
53
+ failed = err;
54
+ throw err;
55
+ }
36
56
  };
37
57
  return { slides, ensure: load };
38
58
  }
39
59
 
40
- function readSlide(zip, part) {
60
+ function readSlide(zip, part, count = { shapes: 0, budget: deckBudget() }) {
41
61
  const out = { shapes: [], title: "", titleHash: EMPTY };
42
62
  const xml = part ? zip.readText(part) : null;
43
63
  if (xml == null) return out;
@@ -50,6 +70,7 @@ function readSlide(zip, part) {
50
70
  onOpen(n, a) {
51
71
  const l = local(n);
52
72
  at.push(l);
73
+ if (SHAPE_TAGS.has(l) && ++count.shapes > MAX_SHAPES) throw tooMany(count.budget, MAX_SHAPES, "shapes");
53
74
  if (SHAPE_TAGS.has(l)) stack.push({ tag: l, id: null, name: "", paras: [], isTitle: false, runs: [], run: null });
54
75
  else if (!stack.length) return;
55
76
  const top = stack[stack.length - 1];
@@ -94,7 +115,7 @@ function readSlide(zip, part) {
94
115
  if (sh.isTitle && !out.title) out.title = text;
95
116
  }
96
117
  },
97
- });
118
+ }, count.budget);
98
119
  out.titleHash = textHash(out.title);
99
120
  return out;
100
121
  }
@@ -113,39 +134,76 @@ export function listShapes(deck, slideId = null) {
113
134
 
114
135
  const slideById = (deck, id) => deck.slides.find((s) => s.id === String(id)) || null;
115
136
 
137
+ // The budget (src/text.js scanBudget): resolving an anchor reads slides, the shapes of a slide and a shape's text, once per anchor. Each
138
+ // such read is spent before it runs, and an anchor whose spend fails resolves as "moved or removed" (the slide or shape alone when
139
+ // it is all that is known). Index-free scans here are over the slides of a deck or the shapes of a slide, both bounded by MAX_* at parse.
140
+ const slideCost = (deck) => deck.slides.length * 4 + 1;
141
+ const shapeCost = (slide) => slide.shapes.length * 8 + 1;
142
+
116
143
  /**
117
144
  * Engine location {slideIndex, slideId?, shapeId?, shapeName?, cell?} (+ the text the browser saw) -> durable anchor, from the written
118
145
  * bytes. `ctx` = {quote, prefix, suffix, text}. Returns null when the slide does not exist.
119
146
  */
120
147
  export function pptxDurable(deck, loc, ctx = {}) {
121
148
  deck.ensure();
122
- const slide = (loc.slideId != null && slideById(deck, loc.slideId)) || deck.slides[loc.slideIndex];
149
+ const slide = (loc.slideId != null && spend(slideCost(deck)) && slideById(deck, loc.slideId)) || deck.slides[loc.slideIndex];
123
150
  if (!slide) return null;
124
- const base = { format: "pptx", ...(loc.protected ? { protected: true } : {}), slideId: slide.id, slidePart: slide.part, slideIndex: slide.index, slideTitle: normText(slide.title).slice(0, 120), titleHash: slide.titleHash, shapeId: null, shapeName: "", ordinal: 0, kind: "slide", text: "", textHash: null, quote: "", prefix: "", suffix: "", start: null, end: null, cell: loc.cell || null };
125
- if (loc.shapeId == null) return base;
151
+ const base = { format: "pptx", ...(loc.protected ? { protected: true } : {}), slideId: slide.id, slidePart: slide.part, slideIndex: slide.index, slideTitle: titleOf(slide), titleHash: slide.titleHash, shapeId: null, shapeName: "", ordinal: 0, kind: "slide", text: "", textHash: null, quote: "", prefix: "", suffix: "", start: null, end: null, cell: loc.cell || null };
152
+ if (loc.shapeId == null || !spend(shapeCost(slide))) return base;
126
153
  const same = slide.shapes.filter((s) => s.id === loc.shapeId);
127
154
  let sh = same[0];
128
155
  if (same.length > 1) {
129
- const want = textHash(ctx.text || ctx.quote || "");
156
+ // the page's text is hashed only when the budget pays for it (null: told apart by name, then the first)
157
+ const seen = ctx.text || ctx.quote || "";
158
+ const want = affords(seen.length) && spend(weigh(seen)) ? textHash(seen) : null;
130
159
  sh = same.find((s) => s.name === loc.shapeName && s.hash === want) || same.find((s) => s.hash === want) || same.find((s) => s.name === loc.shapeName) || same[0];
131
160
  }
132
161
  if (!sh) return { ...base, shapeId: loc.shapeId, shapeName: loc.shapeName || "", kind: "shape" };
133
- const q = normText(ctx.quote || "");
134
- const raw = ctx.quote || "";
135
- const at = q ? sh.text.indexOf(raw) : -1;
162
+ // the page's quote, prefix and suffix are normalized within the budget too (each can be a whole shape's text, per comment); past it
163
+ // the anchor is the shape, without the text range
164
+ const q = normPaid(ctx.quote) ?? "";
165
+ const raw = q ? ctx.quote : "";
166
+ const at = q && spend(textCost(sh, raw)) ? sh.text.indexOf(raw) : -1;
136
167
  return {
137
168
  ...base,
138
169
  shapeId: sh.id, shapeName: sh.name, ordinal: sh.ordinal, kind: sh.kind,
139
- text: normText(sh.text).slice(0, KEEP), textHash: sh.hash,
170
+ text: spend(normCost(sh)) ? normShape(sh).slice(0, KEEP) : "", textHash: sh.hash,
140
171
  // a text range inside the shape: the quote and what surrounds it
141
172
  quote: ctx.shapeWide ? "" : q,
142
- prefix: normText(ctx.prefix || "").slice(-CTX),
143
- suffix: normText(ctx.suffix || "").slice(0, CTX),
173
+ prefix: (normPaid(ctx.prefix) ?? "").slice(-CTX),
174
+ suffix: (normPaid(ctx.suffix) ?? "").slice(0, CTX),
144
175
  start: !ctx.shapeWide && at >= 0 ? at : null,
145
176
  end: !ctx.shapeWide && at >= 0 ? at + raw.length : null,
146
177
  };
147
178
  }
148
179
 
180
+ // A shape's normalized text, kept per shape of a reading: every anchor on a shape normalized all of it again (50 comments on a shape of
181
+ // 8 million characters held the server's thread for 5.6 s while the page placed them).
182
+ const NORM = new WeakMap();
183
+ const normShape = (sh) => {
184
+ let v = NORM.get(sh);
185
+ if (v === undefined) NORM.set(sh, (v = normText(sh.text)));
186
+ return v;
187
+ };
188
+ // A slide's normalized title, kept per slide of a reading and paid from the budget the first time: every anchor on a slide normalized the
189
+ // whole title again, outside the budget. Past the budget the anchor has no title text ("", asked again by the next request); the title's
190
+ // hash, read with the slide, still identifies it.
191
+ const TITLE = new WeakMap();
192
+ const titleOf = (slide) => {
193
+ let v = TITLE.get(slide);
194
+ if (v === undefined) {
195
+ const t = normPaid(slide.title);
196
+ if (t === null) return "";
197
+ TITLE.set(slide, (v = t.slice(0, 120)));
198
+ }
199
+ return v;
200
+ };
201
+ /** what normShape(sh) is about to read: the shape's text the first time (src/text.js weigh), nothing once it is kept */
202
+ const normCost = (sh) => (NORM.has(sh) ? 0 : weightOf(sh));
203
+ /** a native substring search for `q` in a shape's text (src/text.js searchCost), priced by the shape's byte table (src/text.js hitsOf) */
204
+ const HITS = new WeakMap();
205
+ const textCost = (sh, q) => searchCost(sh.text.length, q, hitsOf(HITS, sh, sh.text, q));
206
+
149
207
  /**
150
208
  * Re-resolve against new bytes. Slide: by p:sldId id; else the part path together with the title; else the title alone when unique.
151
209
  * Shape (when the anchor has one): by id (+ ordinal); else the same name with the same text; else the same text; else the same name.
@@ -154,6 +212,7 @@ export function pptxDurable(deck, loc, ctx = {}) {
154
212
  export function resolvePptx(deck, d) {
155
213
  if (!d) return { method: "none" };
156
214
  deck.ensure();
215
+ if (!spend(slideCost(deck))) return { method: "none" };
157
216
  let slide = d.slideId != null ? slideById(deck, d.slideId) : null;
158
217
  let method = "id";
159
218
  if (!slide) {
@@ -166,6 +225,7 @@ export function resolvePptx(deck, d) {
166
225
  const out = { method, slideId: slide.id, slideIndex: slide.index, shapeId: null, shapeName: "", start: null, end: null, changed: false, text: "", titleChanged: slide.titleHash !== d.titleHash };
167
226
  if (d.shapeId == null) return out;
168
227
  const shapes = slide.shapes;
228
+ if (!spend(shapeCost(slide))) return { method: "none" };
169
229
  const byId = shapes.filter((s) => s.id === d.shapeId);
170
230
  let sh = null;
171
231
  let how = "";
@@ -183,10 +243,14 @@ export function resolvePptx(deck, d) {
183
243
  else if (name.length === 1 && d.textHash && d.textHash !== EMPTY) [sh, how] = [name[0], "name"];
184
244
  }
185
245
  if (!sh) return { method: "none" };
186
- const stext = normText(sh.text);
246
+ // The shape is found (by its id, or its name and the hash read with the slide). Its text and the quote's place in it are read only when
247
+ // the budget pays for them: past it the comment is still on its shape, without a text excerpt or a range (a shape of 7.9 million CJK
248
+ // characters costs more than the whole budget to normalize, and its comments resolved as "moved or removed" with the shape right there).
249
+ const stext = spend(normCost(sh)) ? normShape(sh) : "";
187
250
  const q = d.quote || "";
188
- const at = q ? sh.text.indexOf(q) : -1;
189
- return { ...out, method: method === "id" ? how : `${method}+${how}`, shapeId: sh.id, shapeName: sh.name, start: at >= 0 ? at : null, end: at >= 0 ? at + q.length : null, changed: sh.hash !== d.textHash, text: stext.slice(0, 200), quoteLost: !!q && at < 0 };
251
+ const searched = !!q && spend(textCost(sh, q)); // a shape-wide comment has no quote, and no search to pay for
252
+ const at = searched ? sh.text.indexOf(q) : -1;
253
+ return { ...out, method: method === "id" ? how : `${method}+${how}`, shapeId: sh.id, shapeName: sh.name, start: at >= 0 ? at : null, end: at >= 0 ? at + q.length : null, changed: sh.hash !== d.textHash, text: stext.slice(0, 200), quoteLost: searched && at < 0 };
190
254
  }
191
255
 
192
256
  /** Locator fields for the batch: the names the `apply` targets use. */
package/src/quick-plan.js CHANGED
@@ -5,10 +5,15 @@
5
5
  // binds that number as `expect.count`, so a file that changed since the preview, or a tampered number, fails the count check and nothing
6
6
  // is written (the batch then goes to the agent unchanged).
7
7
  //
8
- // Ceiling: previews run the engine on the server's main thread (as the merge does); a large file pauses other requests while one runs.
9
- // The page asks 600 ms after typing stops, and answers are cached by file hash and command. Upgrade path: a worker.
8
+ // The engine's count runs in a worker thread with a time limit (src/offload.js previewOffThread), as the change itself does (src/local.js):
9
+ // a document of one paragraph of 8 million characters held the server's thread for 4 s in an outside review. A count that does not
10
+ // finish is no count: the page shows no button and the agent gets the note. The page asks 600 ms after typing stops, and answers are
11
+ // cached by file hash and command. The bytes are read whole within the readers' caps first (readChecked), so a file swapped on disk
12
+ // past a cap is refused in plain words before the engine reads it.
10
13
 
11
14
  import { checkRegularOwned, readDoc, sha256 } from "./fsutil.js";
15
+ import { readChecked } from "./session.js";
16
+ import { previewOffThread } from "./offload.js";
12
17
  import { recognize, refusalFor, wordsFor } from "./quick.js";
13
18
  import { userText } from "./sanitize.js";
14
19
 
@@ -30,8 +35,7 @@ function ridesAlong(s) {
30
35
  }
31
36
 
32
37
  async function previewOf(bytes, format, q) {
33
- const { previewAll } = await import("./ops/run.js"); // pulls in the engine: loaded on first use
34
- const p = await previewAll(bytes, format, q.op);
38
+ const p = await previewOffThread(bytes, format, q.op); // throws when the worker did not finish: no count, nothing shown or written
35
39
  const count = p.ok ? p.count : 0;
36
40
  const skipped = p.ok ? p.protected || 0 : 0;
37
41
  let why = refusalFor(q.kind, p, q.find);
@@ -56,9 +60,11 @@ export async function quickPreview(s, note) {
56
60
  } catch {
57
61
  return { match: null };
58
62
  }
59
- const key = `${sha256(bytes)}\0${s.format}\0${q.command}`;
63
+ const sha = sha256(bytes);
64
+ const key = `${sha}\0${s.format}\0${q.command}`;
60
65
  let match = cache.get(key);
61
66
  if (!match) {
67
+ readChecked(s.format, bytes, { sha }); // past a reader's cap: refused with the reason (422), the engine never sees it
62
68
  try {
63
69
  const { count, skipped, why } = await previewOf(bytes, s.format, q);
64
70
  match = { action: q.kind, command: q.command, count, protectedCount: skipped, local: !why, why: why || null, ...wordsFor(q, count, skipped) };
@@ -96,10 +102,11 @@ export const quickPlanner = {
96
102
  try {
97
103
  checkRegularOwned(s.file);
98
104
  bytes = readDoc(s.file);
105
+ if (sha256(bytes) !== batch.base_sha256) return null; // the file moved since the Send
106
+ readChecked(s.format, bytes, { sha: batch.base_sha256 }); // within the readers' caps before the engine reads it
99
107
  } catch {
100
108
  return null;
101
109
  }
102
- if (sha256(bytes) !== batch.base_sha256) return null; // the file moved since the Send
103
110
  const { count, skipped, why } = await previewOf(bytes, batch.format, q);
104
111
  if (why || count !== confirmed.count) return null; // not what the person saw
105
112
  return { reply: { summary: wordsFor(q, count, skipped).summary, replies: [], ops: [{ ...q.op, expect: { count } }] } };
@@ -1,29 +1,25 @@
1
1
  import fs from "node:fs";
2
- import { parentPort, workerData } from "node:worker_threads";
3
2
  import { buildTracked } from "./redline.js";
4
3
  import { verifyTracked } from "./redline-check.js";
5
4
  import { env } from "./constants.js";
6
5
  import { openZip } from "./zip.js";
7
6
  import { replaceEntry } from "./zip-write.js";
8
7
 
9
- // One tracked-changes copy, off the server thread: build it (src/redline.js), then check it (src/redline-check.js). One message back:
8
+ // One tracked-changes copy, off the server thread (run by src/worker-host.js): build it (src/redline.js), then check it (src/redline-check.js). One message back:
10
9
  // {ok: true, bytes, counts} | {refused: [codes]} | {check_failed: why}. The copy's bytes only leave this worker after the check passed.
11
10
  // ONIONSKIN_TEST_TAMPER_EXPORT=drop-del (tests only) removes one deletion from the built copy before the check, to prove the check is
12
11
  // what stands between the builder and the disk; it can only make a copy fail, never pass.
13
- const { before, after, author, date } = workerData;
14
- let msg;
15
- try {
16
- const b = fs.readFileSync(before);
17
- const a = fs.readFileSync(after);
18
- const r = buildTracked({ before: b, after: a, author, date });
19
- if (r.refused) msg = { refused: r.refused };
20
- else {
12
+ export function work({ before, after, author, date }) {
13
+ try {
14
+ const b = fs.readFileSync(before);
15
+ const a = fs.readFileSync(after);
16
+ const r = buildTracked({ before: b, after: a, author, date });
17
+ if (r.refused) return { refused: r.refused };
21
18
  let out = r.bytes;
22
19
  if (env("TEST_TAMPER_EXPORT", "") === "drop-del") out = replaceEntry(out, "word/document.xml", openZip(out).readText("word/document.xml").replace(/<w:del [^>]*>[\s\S]*?<\/w:del>/, ""));
23
20
  const v = verifyTracked({ before: b, after: a, out, counts: r.counts, author, date });
24
- msg = v.ok ? { ok: true, bytes: out, counts: r.counts } : { check_failed: v.why };
21
+ return v.ok ? { ok: true, bytes: out, counts: r.counts } : { check_failed: v.why };
22
+ } catch (err) {
23
+ return { check_failed: `the copy could not be made: ${String(err?.message || err).slice(0, 200)}` };
25
24
  }
26
- } catch (err) {
27
- msg = { check_failed: `the copy could not be made: ${String(err?.message || err).slice(0, 200)}` };
28
25
  }
29
- parentPort.postMessage(msg);
package/src/reply.js CHANGED
@@ -1,9 +1,9 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
- import { FORMATS, PRODUCT } from "./constants.js";
4
- import { ApplyError, APPLY_EXIT, applyCommand } from "./apply.js";
5
- import { parseTicket } from "./lease.js";
6
- import { isAgentHolder } from "./agent-detect.js";
3
+ import { PRODUCT, unknownFormat } from "./constants.js";
4
+ import { ApplyError, APPLY_EXIT, applyCommand, formatOfPath } from "./apply.js";
5
+ import { loadTicketRecord, parseTicket } from "./lease.js";
6
+ import { NOT_YOUR_FILE, agentMayUse, isAgentHolder } from "./agent-detect.js";
7
7
  import { QUESTION_FIX, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
8
8
  import { SUGGESTIONS_MAX, SUGGEST_FIX, cleanSuggestion } from "./review.js";
9
9
  import { VOICE_FIX, cleanVoiceDraft } from "./voice-guide.js";
@@ -186,13 +186,16 @@ export function planReplies(batch, input, { partial = false } = {}) {
186
186
  if (mode === "find") {
187
187
  if (typeof r.find !== "string" || !r.find || typeof r.replace !== "string") return refuse("bad_reply", "find needs a non-empty string and replace needs a string");
188
188
  const extras = Object.fromEntries(SHORTHAND_EXTRAS.filter((k) => k in r).map((k) => [k, r[k]]));
189
- if (format === "docx") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
189
+ if (format === "docx" || format === "html") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
190
190
  else if (format === "pptx") op = { op: "replace_text_in_shape", ...base, find: r.find, replace: r.replace, ...extras };
191
+ else if (format !== "xlsx") return unknownFormat(format, "reply find");
191
192
  else return refuse("shorthand_not_for_format", "find/replace does not apply to a .xlsx cell", "use `set` for a cell or range");
192
193
  } else if (mode === "text") {
193
194
  if (typeof r.text !== "string") return refuse("bad_reply", "text must be a string");
194
195
  if (format === "docx") op = { op: "replace_text", ...base, text: r.text };
196
+ else if (format === "html") op = { op: "set_text", ...base, text: r.text };
195
197
  else if (format === "pptx") op = { op: "set_shape_text", ...base, paragraphs: r.text.split("\n") };
198
+ else if (format !== "xlsx") return unknownFormat(format, "reply text");
196
199
  else return refuse("shorthand_not_for_format", "text does not apply to a .xlsx cell", "use `set` for a cell or range");
197
200
  } else if (mode === "set") {
198
201
  if (format !== "xlsx") return refuse("shorthand_not_for_format", `set applies to a .xlsx cell or range, not a .${format}`, "use `find`+`replace` or `text`");
@@ -264,7 +267,7 @@ const FIX = {
264
267
  skipped_protected: "Protected place (field, equation, content control or similar): it cannot be rewritten losslessly. Decline this comment with a note.",
265
268
  };
266
269
  const fixFor = (r) => FIX[r.status] || (r.status === "refused" ? `Refused by the engine${r.code ? ` (${r.code})` : ""}. Change the op as the reason says, or decline.` : undefined);
267
- const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
270
+ const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(Array.isArray(r.reasons) ? { reasons: r.reasons } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
268
271
 
269
272
  /**
270
273
  * The command. `deps` = {server: {healthy, req, json}, finish(file, results, lease) -> done's JSON, pollNext(file, timeout) -> string, readStdin(), holder?}.
@@ -279,32 +282,56 @@ export async function replyCommand(rest, deps) {
279
282
  throw new ReplyError(err.message);
280
283
  }
281
284
  const { server } = deps;
285
+ // who calls: the local runner names itself; otherwise ONIONSKIN_HOLDER (an agent Onionskin started cannot change it), else the holder the
286
+ // server saved with this ticket. It goes with every request, so the server checks the binding and the lease before it hands out anything.
287
+ const envHolder = process.env[`${PRODUCT.envPrefix}_HOLDER`];
288
+ const ticketOfArgs = parseTicket(a.lease);
289
+ const holder = deps.holder || envHolder || (ticketOfArgs ? loadTicketRecord(ticketOfArgs.batch_id)?.holder : null) || "";
290
+ const notYours = () => new ReplyError(NOT_YOUR_FILE, { exit: APPLY_EXIT.write_refused, body: { code: "not_your_file" } });
282
291
  if (!a.file) {
283
- // no file given: the ticket's batch id names the session (the token is still checked by apply and done)
292
+ // no file given: the whole ticket (batch id and token) names the session; a wrong token names none
284
293
  if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
285
- const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(parseTicket(a.lease)?.batch_id || "")}` });
294
+ const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(ticketOfArgs?.batch_id || "")}&lease_token=${encodeURIComponent(ticketOfArgs?.token || "")}&holder=${encodeURIComponent(holder)}` });
286
295
  a.file = w.status === 200 ? server.json(w.text).file || "" : "";
287
296
  if (!a.file) throw new ReplyError(`No delivered batch matches this ticket: your delivery was taken back, finished or handed on. Nothing was written. Run \`${PRODUCT.bin} poll <file>\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
288
297
  }
298
+ // an agent Onionskin started: its own input first (only its one reply file, content-free checks), then its binding to its one file,
299
+ // refused before that file is looked at, in the same words whether it exists or is open
300
+ const bg = !deps.holder && isAgentHolder(envHolder);
301
+ const ticketRequired = () => new ReplyError(`--lease <ticket> is required: it is the lease_ticket of the batch (\`${PRODUCT.bin} status <file> --ticket\` prints it again)`);
302
+ let input;
303
+ if (bg) {
304
+ if (!ticketOfArgs) throw ticketRequired();
305
+ checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticketOfArgs.batch_id);
306
+ input = await readInput(a, deps.readStdin, { quiet: true });
307
+ if (!agentMayUse(envHolder, a.file)) throw notYours();
308
+ }
289
309
  let real;
290
310
  try {
291
311
  real = fs.realpathSync(path.resolve(a.file));
292
312
  } catch {
293
313
  throw new ReplyError(`File not found: ${JSON.stringify(a.file.slice(0, 200))}`, { exit: APPLY_EXIT.error });
294
314
  }
295
- if (!FORMATS[path.extname(real).toLowerCase().slice(1)]) throw new ReplyError("Only .docx, .xlsx and .pptx files are supported");
296
- const ticket = parseTicket(a.lease);
297
- if (!ticket) throw new ReplyError(`--lease <ticket> is required: it is the lease_ticket of the batch (\`${PRODUCT.bin} status <file> --ticket\` prints it again)`);
298
- const bg = !deps.holder && isAgentHolder(process.env[`${PRODUCT.envPrefix}_HOLDER`]);
299
- if (bg) checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticket.batch_id);
300
- const input = await readInput(a, deps.readStdin, { quiet: bg });
315
+ if (!formatOfPath(real)) throw new ReplyError("Only .docx, .xlsx, .pptx, .html and .htm files are supported");
316
+ const ticket = ticketOfArgs;
317
+ if (!ticket) throw ticketRequired();
318
+ if (!bg) input = await readInput(a, deps.readStdin, { quiet: false });
301
319
 
302
- // the batch the session holds for this delivery: the only source of anchors and hashes
320
+ // authorization first: the server hands out the batch the session holds for this delivery (the only source of anchors and hashes) only
321
+ // to this ticket's holder, checked before the reply is planned or run against the file
303
322
  if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
304
- const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}` });
305
- const held = r.status === 200 ? server.json(r.text).batch : null;
306
- if (!held) throw new ReplyError(`No batch is delivered for this file, so the ticket is void. Nothing was written. Run \`${PRODUCT.bin} poll\` and wait for the user's Send.`, { exit: APPLY_EXIT.write_refused, body: { code: "no_batch" } });
307
- if (held.batch_id !== ticket.batch_id) throw new ReplyError(`The ticket belongs to batch ${ticket.batch_id}, but the delivered batch is ${held.batch_id}: your delivery was taken back, finished or handed on. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
323
+ const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}&holder=${encodeURIComponent(holder)}&batch_id=${encodeURIComponent(ticket.batch_id)}&lease_token=${encodeURIComponent(ticket.token)}` });
324
+ let refused = null;
325
+ try {
326
+ refused = r.status === 200 ? null : server.json(r.text);
327
+ } catch {
328
+ refused = {};
329
+ }
330
+ if (refused?.code === "not_your_file") throw notYours();
331
+ if (refused?.code === "no_batch") throw new ReplyError(`No batch is delivered for this file, so the ticket is void. Nothing was written. Run \`${PRODUCT.bin} poll\` and wait for the user's Send.`, { exit: APPLY_EXIT.write_refused, body: { code: "no_batch" } });
332
+ if (refused) throw new ReplyError(`This ticket is not the delivery the session holds: your delivery was taken back, finished or handed on, or the ticket is wrong. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: refused.code === "not_lease_holder" ? "not_lease_holder" : "stale_batch" } });
333
+ const held = server.json(r.text).batch;
334
+ if (!held || held.batch_id !== ticket.batch_id) throw new ReplyError(`This ticket is not the delivery the session holds. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
308
335
 
309
336
  const plan = planReplies(held, input, { partial: a.partial });
310
337
  if (plan.errors.length) throw new ReplyError("the reply cannot be applied; nothing was written and the batch is still open", { body: { code: "bad_reply", lease_kept: true, problems: plan.errors } });