onionskin-cli 0.0.3 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/CHANGELOG.md +100 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +101 -33
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/skills/onionskin/SKILL.md +9 -6
  9. package/skills/onionskin/reference/batch.md +5 -3
  10. package/skills/onionskin/reference/formats.md +5 -1
  11. package/skills/onionskin/reference/ops.md +47 -3
  12. package/skills/onionskin/reference/troubleshooting.md +31 -0
  13. package/src/agent-detect.js +28 -2
  14. package/src/agent-prompt.js +12 -1
  15. package/src/agent-runner.js +16 -5
  16. package/src/anchor-work.js +154 -0
  17. package/src/anchor-worker.js +38 -0
  18. package/src/anchors.js +61 -11
  19. package/src/apply.js +119 -47
  20. package/src/batch.js +32 -8
  21. package/src/check-worker.js +8 -5
  22. package/src/cli.js +91 -25
  23. package/src/constants.js +34 -1
  24. package/src/diff.js +3 -1
  25. package/src/docx.js +368 -58
  26. package/src/edit-hash.js +19 -0
  27. package/src/fsutil.js +1 -1
  28. package/src/history-worker.js +8 -8
  29. package/src/history.js +135 -31
  30. package/src/html-active.js +384 -0
  31. package/src/html-envelope.js +51 -0
  32. package/src/html-parser.js +8 -0
  33. package/src/html-render-worker.js +25 -0
  34. package/src/html-render.js +880 -0
  35. package/src/html-utf8.js +65 -0
  36. package/src/html-write-worker.js +14 -0
  37. package/src/html-write.js +380 -0
  38. package/src/html.js +1257 -0
  39. package/src/local.js +68 -4
  40. package/src/merge-package.js +1110 -0
  41. package/src/merge-pptx.js +98 -38
  42. package/src/merge-proof.js +1019 -0
  43. package/src/merge-worker.js +60 -0
  44. package/src/merge-xlsx.js +84 -29
  45. package/src/merge.js +286 -83
  46. package/src/nfc.js +46 -0
  47. package/src/notes.js +122 -0
  48. package/src/offload.js +400 -1
  49. package/src/open-worker.js +27 -0
  50. package/src/ops/common.js +2 -1
  51. package/src/ops/docx.js +58 -29
  52. package/src/ops/html.js +394 -0
  53. package/src/ops/pptx.js +74 -21
  54. package/src/ops/run.js +23 -3
  55. package/src/ops/runs.js +215 -13
  56. package/src/ops/schema.js +8 -1
  57. package/src/ops/xlsx.js +12 -7
  58. package/src/ops-worker.js +17 -0
  59. package/src/package.js +26 -2
  60. package/src/person.js +2 -1
  61. package/src/pptx.js +85 -21
  62. package/src/quick-plan.js +13 -6
  63. package/src/redline-worker.js +10 -14
  64. package/src/reply.js +46 -19
  65. package/src/retain.js +223 -0
  66. package/src/review.js +32 -18
  67. package/src/rewrite-check.js +2 -2
  68. package/src/sanitize.js +16 -3
  69. package/src/server.js +80 -36
  70. package/src/session.js +1138 -208
  71. package/src/stop-gate.js +34 -0
  72. package/src/suggest-reply.js +42 -7
  73. package/src/suggest.js +14 -29
  74. package/src/text.js +161 -11
  75. package/src/update.js +288 -0
  76. package/src/voice-guide.js +4 -3
  77. package/src/voice-learn.js +4 -3
  78. package/src/voice-sample-worker.js +7 -6
  79. package/src/voice-worker.js +4 -6
  80. package/src/worker-host.js +23 -0
  81. package/src/xlsx.js +355 -94
  82. package/src/xml-tree.js +6 -3
  83. package/src/xml.js +82 -34
  84. package/src/zip-write.js +2 -1
  85. package/src/zip.js +9 -5
  86. package/vendor/parse5/LICENSE.txt +32 -0
  87. package/vendor/parse5/parse5.mjs +7822 -0
  88. package/vendor/parse5.lock.json +23 -0
  89. package/web/dist/assets/index-DkNx0LSH.css +1 -0
  90. package/web/dist/assets/{index-DS-WST1E.js → index-DytKkMdi.js} +112 -108
  91. package/web/dist/index.html +2 -2
  92. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/anchors.js CHANGED
@@ -2,24 +2,59 @@ import { normText } from "./text.js";
2
2
  import { readDocxParagraphs, resolveDurable, durableFromEngine } from "./docx.js";
3
3
  import { parseDeck, pptxDurable, resolvePptx } from "./pptx.js";
4
4
  import { a1, a1Range, ch, parseWorkbook, resolveXlsx, sheetRows, xlsxDurable } from "./xlsx.js";
5
+ import { capDurable } from "./retain.js";
6
+ import { elementPath, htmlDurable, ownHash, pathString, readHtml, resolveHtml, visibleText } from "./html.js";
7
+ import { unknownFormat } from "./constants.js";
5
8
 
6
9
  // One seam per format for the three things the server does with the bytes it holds: read the units, build a durable anchor from an
7
- // engine location (at Send) and re-resolve one against newer bytes (after `done`). DOCX lives in docx.js, XLSX in xlsx.js, PPTX in pptx.js.
10
+ // engine location (at Send) and re-resolve one against newer bytes (after `done`). DOCX lives in docx.js, XLSX in xlsx.js, PPTX in pptx.js,
11
+ // HTML in html.js (whose parser exists only in a worker thread: src/html-parser.js; on the server's thread reading an HTML file throws).
8
12
 
9
13
  export function readUnits(format, bytes) {
10
14
  if (format === "docx") return { format, paragraphs: readDocxParagraphs(bytes) };
11
15
  if (format === "xlsx") return { format, wb: parseWorkbook(bytes) };
12
16
  if (format === "pptx") return { format, deck: parseDeck(bytes) };
13
- throw new Error(`unknown format ${format}`);
17
+ if (format === "html") return { format, read: readHtml(bytes) };
18
+ return unknownFormat(format, "readUnits");
14
19
  }
15
20
 
16
- /** The browser's captured anchor ({location, quote, prefix, suffix, paraText, hitKind}) -> durable anchor from the written bytes, or null. */
21
+ /** Reads the lazy parts of `units` too (every slide, every sheet), so a version past a cap is refused now, not by a later reader. */
22
+ export function readAll(units) {
23
+ if (units.format === "pptx") units.deck.ensure();
24
+ if (units.format === "xlsx") for (const sheet of units.wb.sheets) units.wb.sheetCells(sheet);
25
+ return units;
26
+ }
27
+
28
+ // The last version read, by its content hash: the server reads one version several times in a row (a Send, then the outline of the batch
29
+ // it delivers; the end of a run, then the batch it queues and the page's next resolve), so it is read once. Consumers never change the
30
+ // units (lazy caches on them are keyed by the same content). Dropped after KEEP_MS so a large file is not held in memory for nothing.
31
+ // Ceiling: one entry; two sessions read in turn read each other's versions again.
32
+ const KEEP_MS = 60_000;
33
+ let kept = null;
34
+ /** readUnits of the version whose sha256 is `sha`, reusing the last reading of it; `bytes` is the content or a function that returns it. */
35
+ export function unitsFor(format, bytes, sha) {
36
+ if (kept && kept.format === format && kept.sha === sha) return kept.units;
37
+ const units = readUnits(format, typeof bytes === "function" ? bytes() : bytes);
38
+ if (kept) clearTimeout(kept.timer);
39
+ kept = { format, sha, units, timer: setTimeout(() => (kept = null), KEEP_MS) };
40
+ kept.timer.unref?.();
41
+ return units;
42
+ }
43
+
44
+ /**
45
+ * The browser's captured anchor ({location, quote, prefix, suffix, paraText, hitKind}) -> durable anchor from the written bytes, or null.
46
+ * A page anchor whose quote was cut where it was stored (src/retain.js, `cut`) is the paragraph or shape it sits in: a range of the cut
47
+ * text would point at a shorter one. What is built is cut to the stored caps (src/retain.js capDurable).
48
+ */
17
49
  export function buildDurable(units, anchor) {
18
50
  if (!anchor?.location) return null;
19
51
  const { format } = units;
20
- if (format === "docx") return durableFromEngine(units.paragraphs, anchor.location, { quote: anchor.paraText ? "" : anchor.quote, prefix: anchor.prefix, suffix: anchor.suffix, paraText: anchor.paraText });
21
- if (format === "xlsx") return xlsxDurable(units.wb, anchor.location);
22
- return pptxDurable(units.deck, anchor.location, { quote: anchor.quote, prefix: anchor.prefix, suffix: anchor.suffix, text: anchor.quote, shapeWide: !!anchor.hitKind });
52
+ const whole = anchor.cut === true;
53
+ if (format === "docx") return capDurable(durableFromEngine(units.paragraphs, anchor.location, { quote: anchor.paraText || whole ? "" : anchor.quote, prefix: anchor.prefix, suffix: anchor.suffix, paraText: anchor.paraText }));
54
+ if (format === "xlsx") return capDurable(xlsxDurable(units.wb, anchor.location));
55
+ if (format === "pptx") return capDurable(pptxDurable(units.deck, anchor.location, { quote: anchor.quote, prefix: anchor.prefix, suffix: anchor.suffix, text: anchor.quote, shapeWide: !!anchor.hitKind || whole }));
56
+ if (format === "html") return capDurable(htmlDurable(units.read, anchor));
57
+ return unknownFormat(format, "buildDurable");
23
58
  }
24
59
 
25
60
  /** Re-resolve a stored durable anchor against new units. DOCX results keep their own shape (see docx.js). */
@@ -27,7 +62,9 @@ export function resolveDurableAny(units, durable) {
27
62
  if (!durable) return { method: "none" };
28
63
  if (units.format === "docx") return resolveDurable(units.paragraphs, durable);
29
64
  if (units.format === "xlsx") return resolveXlsx(units.wb, durable);
30
- return resolvePptx(units.deck, durable);
65
+ if (units.format === "pptx") return resolvePptx(units.deck, durable);
66
+ if (units.format === "html") return resolveHtml(units.read, durable);
67
+ return unknownFormat(units.format, "resolveDurableAny");
31
68
  }
32
69
 
33
70
  /**
@@ -144,8 +181,10 @@ export function changedRegions(format, preBytes, postBytes) {
144
181
  } else if (format === "xlsx") {
145
182
  const A = parseWorkbook(preBytes);
146
183
  const B = parseWorkbook(postBytes);
184
+ const namesB = new Map(); // sheet name -> how many sheets of B have it
185
+ for (const y of B.sheets) namesB.set(y.name, (namesB.get(y.name) || 0) + 1);
147
186
  for (const sb of B.sheets) {
148
- const sa = A.sheets.find((x) => x.name === sb.name) || A.sheets.find((x) => x.sheetId === sb.sheetId && !B.sheets.some((y) => y !== sb && y.name === x.name));
187
+ const sa = A.sheets.find((x) => x.name === sb.name) || A.sheets.find((x) => x.sheetId === sb.sheetId && !(namesB.get(x.name) > (x.name === sb.name ? 1 : 0)));
149
188
  const rowsB = sheetRows(B, sb);
150
189
  if (!sa) {
151
190
  for (const row of rowsB) push({ kind: "added", sheetId: sb.sheetId, sheetName: sb.name, range: a1Range({ r0: row.r, c0: row.cells[0].c, r1: row.r, c1: row.cells[row.cells.length - 1].c }) });
@@ -173,18 +212,29 @@ export function changedRegions(format, preBytes, postBytes) {
173
212
  const B = parseDeck(postBytes);
174
213
  A.ensure();
175
214
  B.ensure();
215
+ const slidesA = new Map();
216
+ for (const x of A.slides) if (!slidesA.has(x.id)) slidesA.set(x.id, x);
176
217
  for (const sb of B.slides) {
177
- const sa = A.slides.find((x) => x.id === sb.id);
218
+ const sa = slidesA.get(sb.id);
178
219
  if (!sa) {
179
220
  push({ kind: "added", slideId: sb.id });
180
221
  continue;
181
222
  }
223
+ const inA = new Map(); // id#ordinal -> the first such shape of the slide before, built once per slide
224
+ for (const x of sa.shapes) if (!inA.has(`${x.id}#${x.ordinal}`)) inA.set(`${x.id}#${x.ordinal}`, x);
182
225
  for (const sh of sb.shapes) {
183
- const old = sa.shapes.find((x) => x.id === sh.id && x.ordinal === sh.ordinal);
226
+ const old = inA.get(`${sh.id}#${sh.ordinal}`);
184
227
  if (!old) push({ kind: "added", slideId: sb.id, shapeId: sh.id });
185
228
  else if (old.hash !== sh.hash) push({ kind: "changed", slideId: sb.id, shapeId: sh.id });
186
229
  }
187
230
  }
188
- }
231
+ } else if (format === "html") {
232
+ // elements that hold text (and images, other leaves) by tag and own text, in order: what is new or different in the post version
233
+ const A = readHtml(preBytes);
234
+ const B = readHtml(postBytes);
235
+ const key = (u) => `${u.tag}\u0000${ownHash(u.node)}`;
236
+ const { kind } = diffSeq(A.units.map(key), B.units.map(key));
237
+ B.units.forEach((u, i) => kind[i] !== "same" && push({ kind: kind[i], nodeId: u.nodeId, path: pathString(elementPath(u.node)), text: normText(visibleText(B, u.node)).slice(0, 160) }));
238
+ } else unknownFormat(format, "changedRegions");
189
239
  return { items, truncated: items.length >= MAX_CHANGES };
190
240
  }
package/src/apply.js CHANGED
@@ -1,13 +1,15 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
- import { FORMATS, MAX_DOC_BYTES, PRODUCT, REFUSED_EXTENSIONS, env } from "./constants.js";
3
+ import { MAX_DOC_BYTES, PRODUCT, REFUSED_EXTENSIONS, env, formatOfExt } from "./constants.js";
4
4
  import { checkRegularOwned, readDoc, replaceGuarded, sha256, Snapshots, WriteConflict, writeSiblingCopy } from "./fsutil.js";
5
5
  import { hasDelivery, loadTicketRecord, parseTicket } from "./lease.js";
6
+ import { NOT_YOUR_FILE, agentMayUse, isAgentHolder } from "./agent-detect.js";
6
7
  import { validatePackage } from "./package.js";
7
8
  import { cleanFlag, hasControlChars } from "./sanitize.js";
8
9
  import crypto from "node:crypto";
9
10
  import { ensureStateDir, sessionKey } from "./paths.js";
10
11
  import { demote, summarize, runOps } from "./ops/run.js";
12
+ import { htmlWriteOffThread } from "./offload.js";
11
13
  import { validateOps } from "./ops/schema.js";
12
14
 
13
15
  export const APPLY_EXIT = { ok: 0, error: 1, invalid: 2, not_applied: 3, partial: 4, write_refused: 5 };
@@ -61,6 +63,12 @@ function readJsonFile(file, what) {
61
63
  }
62
64
  }
63
65
 
66
+ /** The format of a file by its extension: an HTML file (.html or .htm) is "html"; null when Onionskin does not edit that kind of file. */
67
+ export function formatOfPath(real) {
68
+ return formatOfExt(path.extname(real));
69
+ }
70
+ const ONLY = "Only .docx, .xlsx, .pptx, .html and .htm files are supported";
71
+
64
72
  const anchorMap = (batch) => new Map((batch?.untrusted_document_evidence?.anchors || []).map((a) => [a.id, a]));
65
73
 
66
74
  /**
@@ -83,13 +91,60 @@ function outsideChangeOf(j) {
83
91
 
84
92
  const refusal = (message) => new ApplyError(message, APPLY_EXIT.write_refused);
85
93
 
94
+ /**
95
+ * Who calls, checked before the file is read: an agent Onionskin started (ONIONSKIN_HOLDER onionskin-bg-* or onionskin-term-*) acts only on
96
+ * its own file and only as itself (no --holder of another). Returns the holder that goes with the claim: --holder, else ONIONSKIN_HOLDER,
97
+ * else the holder the server saved with the ticket.
98
+ */
99
+ function callerHolder(a, real, ticket) {
100
+ const envHolder = env("HOLDER", "");
101
+ if (!agentMayUse(envHolder, real)) throw refusal(NOT_YOUR_FILE);
102
+ if (isAgentHolder(envHolder) && a.holder && a.holder !== envHolder) throw refusal("--holder cannot name another holder for an agent Onionskin started. Nothing was written.");
103
+ return a.holder || envHolder || (ticket ? loadTicketRecord(ticket.batch_id)?.holder : "") || "";
104
+ }
105
+
106
+ /**
107
+ * The server's state for the file, asked before the file is read. With a ticket the delivered batch is handed out only to its holder, and a
108
+ * ticket the server does not accept ends the command here, with no op run against the file. Returns {up, session, batch}.
109
+ */
110
+ async function sessionState({ server, real, holder, ticket, wantBatch, report = "apply" }) {
111
+ const up = await server.healthy();
112
+ if (!up) return { up, session: null, batch: null };
113
+ const q = holder ? `&holder=${encodeURIComponent(holder)}` : "";
114
+ const r = await server.req({ method: "GET", path: `/cli/status?file=${encodeURIComponent(real)}${q}` });
115
+ const body = (t) => {
116
+ try {
117
+ return server.json(t) || {};
118
+ } catch {
119
+ return {};
120
+ }
121
+ };
122
+ if (r.status === 403 && body(r.text).code === "not_your_file") throw refusal(NOT_YOUR_FILE);
123
+ const session = r.status === 200 ? server.json(r.text) : null;
124
+ if (!session) return { up, session, batch: null };
125
+ if (ticket) {
126
+ const b = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}${q}&batch_id=${encodeURIComponent(ticket.batch_id)}&lease_token=${encodeURIComponent(ticket.token)}` });
127
+ if (b.status !== 200) {
128
+ const j = body(b.text);
129
+ if (j.code === "not_your_file") throw refusal(NOT_YOUR_FILE);
130
+ const note = j.code === "missing_lease" ? `Cannot tell which holder this ticket belongs to: set ${PRODUCT.envPrefix}_HOLDER or pass --holder. Nothing was written.` : `The server does not accept this ticket for this file (${j.code === "no_batch" ? "no batch is delivered" : "it is not the current delivery's"}). Nothing was written. Run \`${PRODUCT.bin} status <file> --ticket\` (it answers only the agent that polled this batch) or poll again.`;
131
+ // the same write_error a refused write reports, from the server's own content-free words; no op ran, so there are no results
132
+ throw new ApplyError(note, APPLY_EXIT.write_refused, { schema: `onionskin.${report}-report/1`, file: real, written: false, write_path: "none", note, write_error: { status: b.status, code: typeof j.code === "string" ? j.code : null, message: typeof j.error === "string" ? j.error.slice(0, 300) : `server answered ${b.status}` } });
133
+ }
134
+ return { up, session, batch: server.json(b.text).batch };
135
+ }
136
+ if (!wantBatch) return { up, session, batch: null };
137
+ const b = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}${q}` });
138
+ return { up, session, batch: b.status === 200 ? server.json(b.text).batch : null };
139
+ }
140
+
86
141
  /**
87
142
  * Where a write may go, and under which credentials. Fails closed: a ticket, a delivered batch or a saved delivery for the file means the
88
143
  * write goes through the live session or does not happen. Standalone writes (no review session at all) need the explicit --standalone flag
89
144
  * and are refused while a live session or a saved delivery exists for the file.
90
145
  * Returns {via: "session", claim} or {via: "standalone"}; throws ApplyError(write_refused).
91
146
  */
92
- async function resolveWriteTarget({ a, real, server, session, up, ticket }) {
147
+ async function resolveWriteTarget({ a, real, session, up, ticket, holder }) {
93
148
  const live = !!session && session.state !== "closed";
94
149
  const delivered = live && ["working", "queued"].includes(session.state);
95
150
  const saved = hasDelivery(real);
@@ -100,7 +155,6 @@ async function resolveWriteTarget({ a, real, server, session, up, ticket }) {
100
155
  if (session.state === "closed") throw refusal(`The session for this file is closed; the delivery named by --lease no longer exists. Nothing was written.`);
101
156
  if (!delivered) throw refusal(`This file has an open ${PRODUCT.name} session in state "${session.state}": no batch is delivered, so the ticket is void. Nothing was written. Run \`${PRODUCT.bin} poll\` and wait for the user's Send.`);
102
157
  if (session.delivered_batch !== ticket.batch_id) throw refusal(`The ticket belongs to batch ${ticket.batch_id}, but the delivered batch is ${session.delivered_batch}: your delivery was taken back, finished or handed on. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`);
103
- const holder = a.holder || env("HOLDER", "") || loadTicketRecord(ticket.batch_id)?.holder;
104
158
  if (!holder) throw refusal(`Cannot tell which holder this ticket belongs to: set ${PRODUCT.envPrefix}_HOLDER or pass --holder. Nothing was written.`);
105
159
  return { via: "session", claim: { holder, lease_token: ticket.token, batch_id: ticket.batch_id } };
106
160
  }
@@ -163,13 +217,24 @@ export async function applyCommand(rest, server, inline = null) {
163
217
  if (hasControlChars(real)) throw new ApplyError("This file's path contains control or bidirectional-text characters. Rename or move the file and run apply again.");
164
218
  const ext = path.extname(real).toLowerCase();
165
219
  if (REFUSED_EXTENSIONS.has(ext)) throw new ApplyError(`${ext} (macro-enabled) files are refused`);
166
- const format = ext.slice(1);
167
- if (!FORMATS[format]) throw new ApplyError("Only .docx, .xlsx and .pptx files are supported");
220
+ const format = formatOfPath(real);
221
+ if (!format) throw new ApplyError(ONLY);
168
222
 
169
223
  const doc = inline ? inline.ops : readJsonFile(a.ops, "ops file");
170
224
  const v = validateOps(doc);
171
225
  if (!v.ok) throw new ApplyError("the ops file does not match the schema", APPLY_EXIT.invalid, { schema: "onionskin.apply-report/1", status: "invalid_ops", errors: v.errors });
172
226
 
227
+ // the only credential is the ticket (--lease): not the batch JSON, not whatever the last poll of this file saved
228
+ const ticket = parseTicket(a.lease);
229
+ if (a.lease && !ticket) throw new ApplyError("--lease wants the value printed in the batch's commands (<batch id>:<token>)", APPLY_EXIT.invalid);
230
+ if (ticket && a.standalone) throw new ApplyError("--lease and --standalone contradict each other", APPLY_EXIT.invalid);
231
+ // authorization before the file is read: who calls, what the server holds for the file, and (with a ticket) that the server accepts it.
232
+ // Nothing below reads, parses or runs an op on the document until then, so a refusal carries nothing of it.
233
+ const holder = callerHolder(a, real, ticket);
234
+ const st = await sessionState({ server, real, holder, ticket, wantBatch: !inline?.batch && !a.batch });
235
+ const { up, session } = st;
236
+ const batch = inline?.batch ?? (a.batch ? readJsonFile(a.batch, "batch file") : st.batch);
237
+
173
238
  checkRegularOwned(real);
174
239
  let bytes;
175
240
  try {
@@ -178,35 +243,32 @@ export async function applyCommand(rest, server, inline = null) {
178
243
  if (err.code === "TOO_LARGE") throw new ApplyError(`document too large: ${err.message}`);
179
244
  throw err;
180
245
  }
181
- try {
182
- validatePackage(bytes, format);
183
- } catch (err) {
184
- throw new ApplyError(`Not a valid .${format}: ${err.message}`);
246
+ // an HTML file is no package: the driver reads it in the ops worker and refuses every op of a file it cannot read (src/ops/html.js)
247
+ if (format !== "html") {
248
+ try {
249
+ validatePackage(bytes, format);
250
+ } catch (err) {
251
+ throw new ApplyError(`Not a valid .${format}: ${err.message}`);
252
+ }
185
253
  }
186
- const baseSha = sha256(bytes);
187
-
188
- // where the write must go: through the server when a session tracks this file (it owns the lease, the compare-and-swap and the snapshots)
189
- let session = null;
190
- let batch = inline?.batch ?? (a.batch ? readJsonFile(a.batch, "batch file") : null);
191
- const up = await server.healthy();
192
- if (up) {
193
- const r = await server.req({ method: "GET", path: `/cli/status?file=${encodeURIComponent(real)}` });
194
- if (r.status === 200) session = server.json(r.text);
195
- if (session && !batch) {
196
- const b = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}` });
197
- if (b.status === 200) batch = server.json(b.text).batch;
254
+ // a run inside the server (src/local.js) reads the bytes within the readers' caps before the engine does, on the server's thread
255
+ if (server.checkBytes && format !== "html") {
256
+ try {
257
+ server.checkBytes(bytes, format);
258
+ } catch (err) {
259
+ throw new ApplyError(String(err?.message || err)); // nothing ran: the local runner hands the batch to the agent
198
260
  }
199
261
  }
200
- // the only credential is the ticket (--lease): not the batch JSON, not whatever the last poll of this file saved
201
- const ticket = parseTicket(a.lease);
202
- if (a.lease && !ticket) throw new ApplyError("--lease wants the value printed in the batch's commands (<batch id>:<token>)", APPLY_EXIT.invalid);
203
- if (ticket && a.standalone) throw new ApplyError("--lease and --standalone contradict each other", APPLY_EXIT.invalid);
262
+ const baseSha = sha256(bytes);
263
+ // where the write must go (no op has run yet): through the server when a session tracks this file (it owns the lease, the compare-and-swap and the snapshots)
204
264
  let target = { via: "none" };
205
- if (!a.dryRun) target = await resolveWriteTarget({ a, real, server, session, up, ticket });
265
+ if (!a.dryRun) target = await resolveWriteTarget({ a, real, session, up, ticket, holder });
206
266
  const claim = target.claim;
207
267
  const viaSession = target.via === "session";
208
268
 
209
- const { results: ran, bytes: out, warnings, blocked } = await runOps({ bytes, format, name: path.basename(real), ops: doc.ops, partial: a.partial, anchors: anchorMap(batch) });
269
+ // a run inside the server (src/local.js) has the engine work in a worker thread (server.runOps), off the server's own thread
270
+ const run = server.runOps ?? runOps;
271
+ const { results: ran, bytes: out, warnings, blocked } = await run({ bytes, format, name: path.basename(real), ops: doc.ops, partial: a.partial, anchors: anchorMap(batch) });
210
272
  const results = ran;
211
273
  const refuseWrite = (note, extra = {}) => {
212
274
  // nothing reached disk, so no op may read `applied`
@@ -245,7 +307,7 @@ export async function applyCommand(rest, server, inline = null) {
245
307
  // what a real run would do: atomic writes only when every op applies, --partial writes the ops that apply
246
308
  const writes = (r) => !!r.bytes && r.results.some((x) => x.status === "applied");
247
309
  const wouldWrite = a.partial ? writes({ bytes: out, results }) : failed === 0 && !!out;
248
- const wouldWritePartial = a.partial ? wouldWrite : failed === 0 ? wouldWrite : writes(await runOps({ bytes, format, name: path.basename(real), ops: doc.ops, partial: true, anchors: anchorMap(batch) }));
310
+ const wouldWritePartial = a.partial ? wouldWrite : failed === 0 ? wouldWrite : writes(await run({ bytes, format, name: path.basename(real), ops: doc.ops, partial: true, anchors: anchorMap(batch) }));
249
311
  report.would_write = wouldWrite;
250
312
  report.would_write_partial = wouldWritePartial;
251
313
  report.note = `dry run: nothing was written. A real run in ${report.mode} mode would ${wouldWrite ? "write the file" : "write nothing"}${a.partial ? "" : `; with --partial it would ${wouldWritePartial ? "write the ops that apply" : "write nothing"}`}.`;
@@ -312,7 +374,7 @@ export function parseCommitArgs(rest) {
312
374
  * The base the candidate was edited from is, unless --base says otherwise, the hash the server last wrote or delivered for the batch;
313
375
  * if the file moved since, the commit is refused with `conflict` and the agent starts over from a fresh copy.
314
376
  */
315
- export async function commitCommand(rest, server) {
377
+ export async function commitCommand(rest, server, { check = htmlWriteOffThread } = {}) {
316
378
  const a = parseCommitArgs(rest);
317
379
  let real;
318
380
  try {
@@ -323,35 +385,34 @@ export async function commitCommand(rest, server) {
323
385
  if (hasControlChars(real)) throw new ApplyError("This file's path contains control or bidirectional-text characters. Rename or move the file and run commit again.");
324
386
  const ext = path.extname(real).toLowerCase();
325
387
  if (REFUSED_EXTENSIONS.has(ext)) throw new ApplyError(`${ext} (macro-enabled) files are refused`);
326
- const format = ext.slice(1);
327
- if (!FORMATS[format]) throw new ApplyError("Only .docx, .xlsx and .pptx files are supported");
388
+ const format = formatOfPath(real);
389
+ if (!format) throw new ApplyError(ONLY);
328
390
  let candidatePath;
329
391
  try {
330
392
  candidatePath = fs.realpathSync(path.resolve(a.from));
331
393
  } catch {
332
394
  throw new ApplyError(`candidate file not found: ${JSON.stringify(a.from.slice(0, 200))}`, APPLY_EXIT.invalid);
333
395
  }
396
+ const ticket = parseTicket(a.lease);
397
+ if (a.lease && !ticket) throw new ApplyError("--lease wants the value printed in the batch's commands (<batch id>:<token>)", APPLY_EXIT.invalid);
398
+ if (ticket && a.standalone) throw new ApplyError("--lease and --standalone contradict each other", APPLY_EXIT.invalid);
399
+ const holder = callerHolder(a, real, ticket);
334
400
  if (candidatePath === real) throw new ApplyError("--from is the document itself: edit a COPY (cp the file to a temp path first) and commit that", APPLY_EXIT.invalid);
335
401
  const cst = fs.statSync(candidatePath);
336
402
  if (!cst.isFile() || cst.size > MAX_DOC_BYTES) throw new ApplyError(`candidate ${a.from} is not a regular file under ${MAX_DOC_BYTES / 1024 / 1024} MB`, APPLY_EXIT.invalid);
337
403
  if (a.base && !/^[0-9a-f]{64}$/.test(a.base)) throw new ApplyError("--base wants a sha256 in lower-case hex", APPLY_EXIT.invalid);
338
404
  const candidate = fs.readFileSync(candidatePath);
339
- try {
340
- validatePackage(candidate, format);
341
- } catch (err) {
342
- throw new ApplyError(`The candidate is not a valid .${format}: ${err.message}. Nothing was written.`, APPLY_EXIT.write_refused);
405
+ // an HTML candidate is checked as a whole new version (src/html-write.js): by the server for a session, below for a standalone commit
406
+ if (format !== "html") {
407
+ try {
408
+ validatePackage(candidate, format);
409
+ } catch (err) {
410
+ throw new ApplyError(`The candidate is not a valid .${format}: ${err.message}. Nothing was written.`, APPLY_EXIT.write_refused);
411
+ }
343
412
  }
344
- const ticket = parseTicket(a.lease);
345
- if (a.lease && !ticket) throw new ApplyError("--lease wants the value printed in the batch's commands (<batch id>:<token>)", APPLY_EXIT.invalid);
346
- if (ticket && a.standalone) throw new ApplyError("--lease and --standalone contradict each other", APPLY_EXIT.invalid);
347
413
 
348
- let session = null;
349
- const up = await server.healthy();
350
- if (up) {
351
- const r = await server.req({ method: "GET", path: `/cli/status?file=${encodeURIComponent(real)}` });
352
- if (r.status === 200) session = server.json(r.text);
353
- }
354
- const target = await resolveWriteTarget({ a, real, server, session, up, ticket });
414
+ const { up, session } = await sessionState({ server, real, holder, ticket, wantBatch: false, report: "commit" });
415
+ const target = await resolveWriteTarget({ a, real, session, up, ticket, holder });
355
416
  const report = { schema: "onionskin.commit-report/1", file: real, format, written: false, write_path: target.via, candidate_sha256: sha256(candidate) };
356
417
  if (target.via === "session") {
357
418
  const w = await server.req({ method: "POST", path: "/cli/commit" }, { file: real, bytes_b64: candidate.toString("base64"), ...(a.base ? { base_sha256: a.base } : {}), ...target.claim });
@@ -361,9 +422,20 @@ export async function commitCommand(rest, server) {
361
422
  }
362
423
  if (!a.base) throw new ApplyError("a standalone commit needs --base <sha256 of the file the candidate was edited from> so a newer version is never overwritten", APPLY_EXIT.invalid);
363
424
  checkRegularOwned(real);
364
- const w = standaloneWrite(real, readDoc(real), candidate, a.base, "before commit (standalone)");
425
+ const disk = readDoc(real);
426
+ let out = candidate;
427
+ if (format === "html") {
428
+ // R1, R12: the same check as the server's, in a worker, against exactly the bytes whose hash is --base (a disk that differs now and
429
+ // is put back before the write would otherwise let a change the base forbids through); the compare-and-swap of standaloneWrite runs
430
+ // after the await, against --base again. A refusal here writes no copy: the candidate was never checked.
431
+ if (sha256(disk) !== a.base) return { exit: APPLY_EXIT.write_refused, report: { ...report, note: "The file on disk is not the version --base names (someone changed it outside Onionskin); nothing was checked and nothing was written.", status: "outside_change", outside_change: { instruction: OUTSIDE_INSTRUCTION, copy_path: null, copy_name: null, copy_error: "no copy was saved: the candidate was not checked against the version on disk" } } };
432
+ const c = await check({ before: disk, after: candidate });
433
+ if (!c.ok) return { exit: APPLY_EXIT.write_refused, report: { ...report, note: `${c.reason} Nothing was written.`, write_error: { status: 422, code: c.code, message: c.reason, ...(c.reasons ? { reasons: c.reasons } : {}) } } };
434
+ out = c.bytes;
435
+ }
436
+ const w = standaloneWrite(real, disk, out, a.base, "before commit (standalone)");
365
437
  if (w.refused) return { exit: APPLY_EXIT.write_refused, report: { ...report, note: w.refused, ...(w.outside ? { status: "outside_change", outside_change: w.outside } : {}) } };
366
- return { exit: APPLY_EXIT.ok, report: { ...report, written: true, new_sha256: report.candidate_sha256, snapshot: w.snapshot, acl: w.acl, note: "standalone (explicit --standalone): written atomically; the previous version is snapshotted in the state dir" } };
438
+ return { exit: APPLY_EXIT.ok, report: { ...report, written: true, new_sha256: sha256(out), snapshot: w.snapshot, acl: w.acl, note: "standalone (explicit --standalone): written atomically; the previous version is snapshotted in the state dir" } };
367
439
  }
368
440
 
369
441
  /** The compact form of an apply or commit report (`--summary`): per-op statuses, flags, written, ok. The full report is the default. */
package/src/batch.js CHANGED
@@ -1,11 +1,13 @@
1
1
  import crypto from "node:crypto";
2
2
  import path from "node:path";
3
3
  import { fileURLToPath } from "node:url";
4
- import { PRODUCT } from "./constants.js";
4
+ import { PRODUCT, unknownFormat } from "./constants.js";
5
5
  import { untrusted, userText } from "./sanitize.js";
6
6
  import { pptxLocator } from "./pptx.js";
7
+ import { htmlLocator } from "./html.js";
7
8
  import { GUIDE_MAX, SECTIONS, SHORT_MAX } from "./voice-guide.js";
8
9
  import { xlsxLocator } from "./xlsx.js";
10
+ import { NOTE_MAX } from "./retain.js";
9
11
 
10
12
  const REFERENCE = "The skill's SKILL.md and reference/ files (batch.md, ops.md, troubleshooting.md) hold the full rules and every op.";
11
13
 
@@ -32,6 +34,13 @@ const EXAMPLES = [
32
34
  { format: "xlsx", description: "Set the cell or range an anchor names; expect.text_hash is the anchor's locator.text_hash.", op: { op: "set_cells", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, values: "<new value or =formula>" } },
33
35
  { format: "pptx", description: "Replace text inside the shape an anchor names.", op: { op: "replace_text_in_shape", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, find: "<text inside the shape>", replace: "<new text>" } },
34
36
  { format: "docx", description: "Replace text inside the paragraph an anchor names.", op: { op: "replace_text", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, find: "<text inside the paragraph>", replace: "<new text>" } },
37
+ { format: "html", description: "Replace text inside the element an anchor names (inside one text: not across a link or bold text).", op: { op: "replace_text", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, find: "<text inside the element>", replace: "<new text>" } },
38
+ { format: "html", description: "Add markup after the element an anchor names (text, formatting, lists, tables, local images and links only).", op: { op: "insert_html", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, where: "after", html: "<p>New paragraph with a <a href=\"https://example.com\">link</a>.</p>" } },
39
+ ];
40
+ // HTML (PLAN-html R1): what a write may add, and what the refusal looks like
41
+ const HTML_RULES = [
42
+ "HTML: code that runs in a browser cannot be added or changed (scripts, event handlers such as onclick, frames, forms, stylesheet links, remote images, data- attributes, URLs other than http, https, mailto, tel or relative); script and style elements cannot be edited. Such a write is refused whole with code active_content: \"The agent tried to add code that runs in a browser, so nothing was written.\"",
43
+ "HTML: replace_text works inside one text; a match across an element's edge answers spans_markup: use replace_element with the element's new markup.",
35
44
  ];
36
45
 
37
46
  /** Static, machine-readable instructions for editing the file (no document text in it). The long rules live in the skill's reference files, not in every batch. */
@@ -52,6 +61,7 @@ function howToApply(file, batchId = "batch", format = null) {
52
61
  `Or with apply (same engine as the browser): target ops by anchor id and give each an expect copied from what you read. ${REFERENCE}`,
53
62
  `If you edit with other tools, never regenerate the file from scratch and never write the document's own path: edit a COPY, then run \`${PRODUCT.bin} commit ${shellQuote(file)} --from <copy> --lease <ticket>\` (the same checked write as apply; \`done\` flags a file that changed any other way).`,
54
63
  "Anchors with protected: true cannot be edited losslessly; apply skips them (see protected_reason).",
64
+ ...(format === "html" ? HTML_RULES : []),
55
65
  ],
56
66
  examples: EXAMPLES.filter((e) => !format || e.format === format),
57
67
  };
@@ -66,7 +76,9 @@ export function replyWithOf(anchor, format) {
66
76
  if (!anchor.locator.text_hash) return "ops";
67
77
  if (format === "xlsx") return "set";
68
78
  if (format === "pptx") return anchor.unit === "shape" ? "find_replace_or_text" : "ops";
69
- return "find_replace_or_text";
79
+ if (format === "docx") return "find_replace_or_text";
80
+ if (format === "html") return anchor.unit === "text" ? "find_replace_or_text" : "ops"; // an element that holds text (replace_text, set_text); an image or a container: ops
81
+ return unknownFormat(format, "replyWithOf");
70
82
  }
71
83
 
72
84
  /** A comment the person answered: a stored question and an index that is in range of its options. */
@@ -96,21 +108,25 @@ const MAX_VALUES = { rows: 20, cols: 10 };
96
108
  const unitOfXlsx = (d) => (d.cell ? (d.r0 === d.r1 && d.c0 === d.c1 ? "cell" : "range") : /^\d+:\d+$/.test(d.range) ? "rows" : /^[A-Za-z]+:[A-Za-z]+$/.test(d.range) ? "columns" : "sheet");
97
109
 
98
110
  /** Format-specific evidence of a durable anchor: locator, quote, unit and context. All strings are document-derived and sanitized by the caller. */
99
- function evidenceOf(d, extra) {
111
+ function evidenceOf(d, extra, clean = (f) => untrusted(f, 100)) {
100
112
  if (d?.format === "xlsx") {
101
- const values = d.cell ? d.text.split("\n").slice(0, MAX_VALUES.rows).map((r) => r.split("\t").slice(0, MAX_VALUES.cols).map((v) => untrusted(v, 200))) : undefined;
113
+ // the anchor keeps its top-left cells (src/xlsx.js rangeOf); one stored by an earlier version has its whole text instead
114
+ const grid = Array.isArray(d.values) ? d.values : d.cell ? d.text.split("\n").map((r) => r.split("\t")) : null;
115
+ const values = d.cell ? grid.slice(0, MAX_VALUES.rows).map((r) => r.slice(0, MAX_VALUES.cols).map((v) => untrusted(v, 200))) : undefined;
102
116
  const c = d.ctx;
103
117
  return {
104
118
  unit: unitOfXlsx(d),
105
119
  locator: { ...xlsxLocator(d), text_hash: d.cell ? d.rangeHash : null },
106
120
  quote: d.text,
107
121
  ...(values ? { values } : {}),
108
- ...(d.cell && d.types ? { cell_types: d.types, number_formats: d.formats.map((r) => r.map((f) => untrusted(f, 100))) } : {}),
122
+ ...(d.cell && d.types ? { cell_types: d.types, number_formats: d.formats.map((r) => r.map(clean)) } : {}),
109
123
  ...(c ? { context: { row_header: untrusted(c.rowHeader, 120), column_header: untrusted(c.colHeader, 120), left: untrusted(c.left, 120), right: untrusted(c.right, 120), above: untrusted(c.up, 120), below: untrusted(c.down, 120) } } : {}),
110
124
  };
111
125
  }
112
126
  if (d?.format === "pptx") return { unit: d.kind, locator: pptxLocator(d), quote: d.quote || d.text, prefix: d.prefix, suffix: d.suffix };
113
- return null;
127
+ if (d?.format === "html") return { unit: d.kind, locator: htmlLocator(d), quote: d.quote || d.text, prefix: d.prefix, suffix: d.suffix };
128
+ if (!d || d.format === undefined || d.format === "docx") return null; // Word's evidence is built by the caller (no anchor: unresolved)
129
+ return unknownFormat(d.format, "evidenceOf");
114
130
  }
115
131
 
116
132
  /**
@@ -124,9 +140,16 @@ export function buildBatch({ file, format, baseSha, note, comments, edits, style
124
140
  const batchId = `b_${crypto.randomBytes(6).toString("hex")}`;
125
141
  const anchors = [];
126
142
  let n = 0;
143
+ // a number format is sanitized once per batch: 500 anchors on one range repeat the same 200 formats
144
+ const formats = new Map();
145
+ const cleanFormat = (f) => {
146
+ let v = formats.get(f);
147
+ if (v === undefined) formats.set(f, (v = untrusted(f, 100)));
148
+ return v;
149
+ };
127
150
  const addAnchor = (d, extra = {}) => {
128
151
  const id = `a${++n}`;
129
- const ev = evidenceOf(d, extra);
152
+ const ev = evidenceOf(d, extra, cleanFormat);
130
153
  anchors.push({
131
154
  id,
132
155
  kind: extra.kind || "paragraph",
@@ -172,7 +195,8 @@ export function buildBatch({ file, format, baseSha, note, comments, edits, style
172
195
  format,
173
196
  base_sha256: baseSha,
174
197
  note: "user_edits are changes the user already made; they are in the file; do not redo or revert them",
175
- instructions: { overall_note: userText(note), ...(style ? { style_note: style } : {}), ...(guide ? { voice_guide: guide } : {}), ...(review ? { review: { kind: review.kind, ask: review.ask } } : {}), comments: instructionComments },
198
+ // the note at NOTE_MAX: a waiting batch's note and the next one together are never cut here (longer is refused at Send, src/session.js)
199
+ instructions: { overall_note: userText(note, NOTE_MAX), ...(style ? { style_note: style } : {}), ...(guide ? { voice_guide: guide } : {}), ...(review ? { review: { kind: review.kind, ask: review.ask } } : {}), comments: instructionComments },
176
200
  untrusted_document_evidence: { file_name: untrusted(path.basename(file)), anchors, user_edits: userEdits, ...(questions.length ? { questions } : {}), ...(units ? { units, ...(unitsMore ? { units_more: unitsMore } : {}) } : {}) },
177
201
  how_to_apply: howToApply(file, batchId, format),
178
202
  agent_rules: AGENT_RULES,
@@ -1,8 +1,11 @@
1
- import { parentPort, workerData } from "node:worker_threads";
1
+ import "./html-parser.js"; // an HTML file is read again at `done` (src/package.js htmlChecks): the parser lives in worker threads only
2
2
  import { analyze } from "./analyze.js";
3
3
 
4
- try {
5
- parentPort.postMessage({ ok: true, ...analyze(Buffer.from(workerData.pre), Buffer.from(workerData.post), workerData.format, { deferSoffice: !!workerData.deferSoffice }) });
6
- } catch (err) {
7
- parentPort.postMessage({ ok: false, error: String(err?.message || err) });
4
+ /** The `done` analysis (src/offload.js analyzeOffThread), run by src/worker-host.js. One message back: {ok, flags, checks, changes} or {ok: false, error}. */
5
+ export function work(data) {
6
+ try {
7
+ return { ok: true, ...analyze(Buffer.from(data.pre), Buffer.from(data.post), data.format, { deferSoffice: !!data.deferSoffice }) };
8
+ } catch (err) {
9
+ return { ok: false, error: String(err?.message || err) };
10
+ }
8
11
  }