onionskin-cli 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +101 -33
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/skills/onionskin/SKILL.md +9 -6
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +47 -3
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +16 -5
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/cli.js +91 -25
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/docx.js +368 -58
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +1 -1
- package/src/history-worker.js +8 -8
- package/src/history.js +135 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +880 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1257 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1110 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +400 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +74 -21
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +12 -7
- package/src/ops-worker.js +17 -0
- package/src/package.js +26 -2
- package/src/person.js +2 -1
- package/src/pptx.js +85 -21
- package/src/quick-plan.js +13 -6
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/sanitize.js +16 -3
- package/src/server.js +80 -36
- package/src/session.js +1138 -208
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +288 -0
- package/src/voice-guide.js +4 -3
- package/src/voice-learn.js +4 -3
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-worker.js +4 -6
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +355 -94
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +2 -1
- package/src/zip.js +9 -5
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/index-DkNx0LSH.css +1 -0
- package/web/dist/assets/{index-DS-WST1E.js → index-DytKkMdi.js} +112 -108
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/package.js
CHANGED
|
@@ -4,6 +4,7 @@ import { openZip, ZipError } from "./zip.js";
|
|
|
4
4
|
import { walkXml } from "./xml.js";
|
|
5
5
|
import { runDeepCheck } from "./checker.js";
|
|
6
6
|
import { cleanFlag } from "./sanitize.js";
|
|
7
|
+
import { decodeHtml, HtmlEncodingError, readHtml } from "./html.js";
|
|
7
8
|
|
|
8
9
|
const MAX_TYPES_BYTES = 4 * 1024 * 1024; // [Content_Types].xml of a real document is a few KB
|
|
9
10
|
// Cumulative work of the relationship scan: a hostile archive can hold thousands of individually allowed .rels parts.
|
|
@@ -14,6 +15,13 @@ const MAX_RELS_BYTES = 64 * 1024 * 1024;
|
|
|
14
15
|
export function validatePackage(bytes, format) {
|
|
15
16
|
const want = FORMATS[format];
|
|
16
17
|
if (!want) throw new ZipError(`unsupported format: ${format}`);
|
|
18
|
+
// an HTML file is no package: its encoding is checked here (no parse; the parser lives in worker threads only, src/html-parser.js).
|
|
19
|
+
// Returns null; the open worker reads it whole (src/open-worker.js), a write goes through src/html-write.js
|
|
20
|
+
if (format === "html") {
|
|
21
|
+
const d = decodeHtml(bytes);
|
|
22
|
+
if (!d.ok) throw new HtmlEncodingError(d);
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
17
25
|
const zip = openZip(bytes);
|
|
18
26
|
const ct = zip.readText("[Content_Types].xml", MAX_TYPES_BYTES);
|
|
19
27
|
if (ct == null) throw new ZipError("missing [Content_Types].xml");
|
|
@@ -36,11 +44,11 @@ function relTargets(zip) {
|
|
|
36
44
|
let bytes = 0;
|
|
37
45
|
for (const name of zip.names) {
|
|
38
46
|
if (!name.endsWith(".rels")) continue;
|
|
39
|
-
if (++parts > MAX_RELS_PARTS) throw new ZipError(`
|
|
47
|
+
if (++parts > MAX_RELS_PARTS) throw new ZipError(`this file has more than ${MAX_RELS_PARTS.toLocaleString("en-US")} relationship parts, more than Onionskin can read. Save it again in Word, Excel or PowerPoint, or split it, and open that`);
|
|
40
48
|
const xml = zip.readText(name);
|
|
41
49
|
if (xml == null) continue;
|
|
42
50
|
bytes += xml.length;
|
|
43
|
-
if (bytes > MAX_RELS_BYTES) throw new ZipError(
|
|
51
|
+
if (bytes > MAX_RELS_BYTES) throw new ZipError(`the relationship parts of this file are larger than ${MAX_RELS_BYTES / 1024 / 1024} MB in all, more than Onionskin can read. Save it again in Word, Excel or PowerPoint, or split it, and open that`);
|
|
44
52
|
const srcDir = path.posix.dirname(path.posix.dirname(name)); // word/_rels/x.rels -> word
|
|
45
53
|
const base = srcDir === "." ? "" : srcDir;
|
|
46
54
|
walkXml(xml, {
|
|
@@ -157,6 +165,21 @@ export function nodeChecks(preBytes, postBytes, format) {
|
|
|
157
165
|
return rawNodeChecks(preBytes, postBytes, format).map(cleanFlag);
|
|
158
166
|
}
|
|
159
167
|
|
|
168
|
+
/**
|
|
169
|
+
* The check of an HTML file at `done` (no package, no deeper check, PLAN-html D9): the version the agent left still reads as HTML within
|
|
170
|
+
* Onionskin's limits (every write through Onionskin was checked whole before it was written, src/html-write.js; a direct save was not).
|
|
171
|
+
* Needs the parser: runs in the check worker (src/check-worker.js).
|
|
172
|
+
*/
|
|
173
|
+
function htmlChecks(postBytes) {
|
|
174
|
+
const skipped = { status: "skipped", reason: "the deeper checks are for Word, Excel and PowerPoint files", code: "not_for_html" };
|
|
175
|
+
try {
|
|
176
|
+
readHtml(postBytes);
|
|
177
|
+
return { flags: [], checks: { node: "ran", deep: skipped, notice: "Checked that the page still reads as HTML within Onionskin's limits. The deeper checks are for Word, Excel and PowerPoint files." } };
|
|
178
|
+
} catch (err) {
|
|
179
|
+
return { flags: [cleanFlag({ code: "html_unreadable", severity: "error", detail: String(err?.message || err).slice(0, 200) })], checks: { node: "ran", deep: skipped, notice: "The page the agent left cannot be read as HTML by Onionskin. Revert restores the version before the run." } };
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
160
183
|
/** The one sentence about the deeper check. A LibreOffice check still running is "pending", never "passed". */
|
|
161
184
|
export const deepNotice = (skipped = [], pending = false) =>
|
|
162
185
|
`Deeper check ran${skipped.length ? `; skipped: ${skipped.map((x) => `${x.check} (${x.reason})`).join(", ")}` : ""}${pending ? "; the LibreOffice conversion check is still running (pending), the Activity list updates when it finishes" : ""}.`;
|
|
@@ -168,6 +191,7 @@ const flagKey = (f) => JSON.stringify([f.code, f.part, f.element, f.rels, f.targ
|
|
|
168
191
|
* Returns {flags, checks}: `flags` are findings (severity info/warn/error); `checks` says what ran, so a skipped deep check is never silent.
|
|
169
192
|
*/
|
|
170
193
|
export function checkPackages(preBytes, postBytes, format, opts = {}) {
|
|
194
|
+
if (format === "html") return htmlChecks(postBytes);
|
|
171
195
|
const node = rawNodeChecks(preBytes, postBytes, format);
|
|
172
196
|
if (node.some((f) => f.code === "invalid_package")) return { flags: node.map(cleanFlag), checks: { node: "ran", deep: { status: "skipped", reason: "the package is invalid" } } };
|
|
173
197
|
const deep = runDeepCheck(preBytes, postBytes, format, { deferSoffice: !!opts.deferSoffice });
|
package/src/person.js
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
import { PRODUCT } from "./constants.js";
|
|
4
4
|
|
|
5
5
|
const BIN = PRODUCT.bin;
|
|
6
|
-
|
|
6
|
+
// the formats Onionskin opens, and the web files it refuses with a sentence (src/constants.js REFUSED_PAGES): `onionskin page.mhtml` says why
|
|
7
|
+
const DOC_EXT = /\.(docx|xlsx|pptx|html|htm|mhtml|mht|webarchive|xhtml|svg)$/i;
|
|
7
8
|
export const GET_CLAUDE_CODE = "https://claude.com/product/claude-code";
|
|
8
9
|
|
|
9
10
|
/**
|
package/src/pptx.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { attrOf, local, readRels } from "./xlsx.js";
|
|
2
|
-
import { walkXml } from "./xml.js";
|
|
3
|
-
import { normText, similarity, textHash } from "./text.js";
|
|
2
|
+
import { readBudget, tooMany, walkXml } from "./xml.js";
|
|
3
|
+
import { affords, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
|
|
4
4
|
import { openZip } from "./zip.js";
|
|
5
5
|
import { summarizeRuns } from "./fmt.js";
|
|
6
6
|
import { untrusted } from "./sanitize.js";
|
|
@@ -13,6 +13,15 @@ const EMPTY = textHash("");
|
|
|
13
13
|
const CTX = 40;
|
|
14
14
|
const KEEP = 2000;
|
|
15
15
|
|
|
16
|
+
// What one reading of a deck may hold, counted over all its slides. The server reads a deck on its own thread at Send, at each delivery to
|
|
17
|
+
// an agent and at the end of a run, so these bound how long that keeps it from answering (outside review: three slides of 2.85 million runs
|
|
18
|
+
// each held it 7.2 s). Each shape costs a few hundred bytes once read. A real deck of 200 slides has a few thousand shapes and well under
|
|
19
|
+
// 200,000 elements.
|
|
20
|
+
export const MAX_SHAPES = 50_000;
|
|
21
|
+
export const MAX_DECK_ELEMENTS = 500_000;
|
|
22
|
+
export const MAX_DECK_ATTRIBUTES = 1_000_000;
|
|
23
|
+
export const MAX_DECK_TEXT = 8_000_000;
|
|
24
|
+
const deckBudget = () => readBudget({ elements: MAX_DECK_ELEMENTS, attrs: MAX_DECK_ATTRIBUTES, text: MAX_DECK_TEXT, what: "deck", split: "smaller decks" });
|
|
16
25
|
const SHAPE_TAGS = new Set(["sp", "pic", "graphicFrame", "grpSp", "cxnSp"]);
|
|
17
26
|
const KIND = { sp: "shape", pic: "image", graphicFrame: "table", grpSp: "group", cxnSp: "line" };
|
|
18
27
|
|
|
@@ -21,23 +30,34 @@ export function parseDeck(bytes) {
|
|
|
21
30
|
const zip = openZip(bytes);
|
|
22
31
|
const px = zip.readText("ppt/presentation.xml");
|
|
23
32
|
if (px == null) throw new Error("ppt/presentation.xml missing");
|
|
24
|
-
const
|
|
33
|
+
const budget = deckBudget(); // the presentation, its relationships and every slide of one reading share it
|
|
34
|
+
const rels = readRels(zip, "ppt/presentation.xml", budget);
|
|
25
35
|
const slides = [];
|
|
26
36
|
walkXml(px, {
|
|
27
37
|
onOpen(n, a) {
|
|
28
38
|
if (local(n) === "sldId") slides.push({ id: a.id ?? null, index: slides.length, part: rels.get(attrOf(a, "id")) || null });
|
|
29
39
|
},
|
|
30
|
-
});
|
|
40
|
+
}, budget);
|
|
41
|
+
// read once; a reading that failed (past a cap) fails the same way every time it is asked again, never "already read"
|
|
31
42
|
let loaded = false;
|
|
43
|
+
let failed = null;
|
|
32
44
|
const load = () => {
|
|
45
|
+
if (failed) throw failed;
|
|
33
46
|
if (loaded) return;
|
|
34
|
-
|
|
35
|
-
|
|
47
|
+
try {
|
|
48
|
+
const count = { shapes: 0, budget };
|
|
49
|
+
const read = slides.map((s) => readSlide(zip, s.part, count));
|
|
50
|
+
slides.forEach((s, i) => Object.assign(s, read[i]));
|
|
51
|
+
loaded = true;
|
|
52
|
+
} catch (err) {
|
|
53
|
+
failed = err;
|
|
54
|
+
throw err;
|
|
55
|
+
}
|
|
36
56
|
};
|
|
37
57
|
return { slides, ensure: load };
|
|
38
58
|
}
|
|
39
59
|
|
|
40
|
-
function readSlide(zip, part) {
|
|
60
|
+
function readSlide(zip, part, count = { shapes: 0, budget: deckBudget() }) {
|
|
41
61
|
const out = { shapes: [], title: "", titleHash: EMPTY };
|
|
42
62
|
const xml = part ? zip.readText(part) : null;
|
|
43
63
|
if (xml == null) return out;
|
|
@@ -50,6 +70,7 @@ function readSlide(zip, part) {
|
|
|
50
70
|
onOpen(n, a) {
|
|
51
71
|
const l = local(n);
|
|
52
72
|
at.push(l);
|
|
73
|
+
if (SHAPE_TAGS.has(l) && ++count.shapes > MAX_SHAPES) throw tooMany(count.budget, MAX_SHAPES, "shapes");
|
|
53
74
|
if (SHAPE_TAGS.has(l)) stack.push({ tag: l, id: null, name: "", paras: [], isTitle: false, runs: [], run: null });
|
|
54
75
|
else if (!stack.length) return;
|
|
55
76
|
const top = stack[stack.length - 1];
|
|
@@ -94,7 +115,7 @@ function readSlide(zip, part) {
|
|
|
94
115
|
if (sh.isTitle && !out.title) out.title = text;
|
|
95
116
|
}
|
|
96
117
|
},
|
|
97
|
-
});
|
|
118
|
+
}, count.budget);
|
|
98
119
|
out.titleHash = textHash(out.title);
|
|
99
120
|
return out;
|
|
100
121
|
}
|
|
@@ -113,39 +134,76 @@ export function listShapes(deck, slideId = null) {
|
|
|
113
134
|
|
|
114
135
|
const slideById = (deck, id) => deck.slides.find((s) => s.id === String(id)) || null;
|
|
115
136
|
|
|
137
|
+
// The budget (src/text.js scanBudget): resolving an anchor reads slides, the shapes of a slide and a shape's text, once per anchor. Each
|
|
138
|
+
// such read is spent before it runs, and an anchor whose spend fails resolves as "moved or removed" (the slide or shape alone when
|
|
139
|
+
// it is all that is known). Index-free scans here are over the slides of a deck or the shapes of a slide, both bounded by MAX_* at parse.
|
|
140
|
+
const slideCost = (deck) => deck.slides.length * 4 + 1;
|
|
141
|
+
const shapeCost = (slide) => slide.shapes.length * 8 + 1;
|
|
142
|
+
|
|
116
143
|
/**
|
|
117
144
|
* Engine location {slideIndex, slideId?, shapeId?, shapeName?, cell?} (+ the text the browser saw) -> durable anchor, from the written
|
|
118
145
|
* bytes. `ctx` = {quote, prefix, suffix, text}. Returns null when the slide does not exist.
|
|
119
146
|
*/
|
|
120
147
|
export function pptxDurable(deck, loc, ctx = {}) {
|
|
121
148
|
deck.ensure();
|
|
122
|
-
const slide = (loc.slideId != null && slideById(deck, loc.slideId)) || deck.slides[loc.slideIndex];
|
|
149
|
+
const slide = (loc.slideId != null && spend(slideCost(deck)) && slideById(deck, loc.slideId)) || deck.slides[loc.slideIndex];
|
|
123
150
|
if (!slide) return null;
|
|
124
|
-
const base = { format: "pptx", ...(loc.protected ? { protected: true } : {}), slideId: slide.id, slidePart: slide.part, slideIndex: slide.index, slideTitle:
|
|
125
|
-
if (loc.shapeId == null) return base;
|
|
151
|
+
const base = { format: "pptx", ...(loc.protected ? { protected: true } : {}), slideId: slide.id, slidePart: slide.part, slideIndex: slide.index, slideTitle: titleOf(slide), titleHash: slide.titleHash, shapeId: null, shapeName: "", ordinal: 0, kind: "slide", text: "", textHash: null, quote: "", prefix: "", suffix: "", start: null, end: null, cell: loc.cell || null };
|
|
152
|
+
if (loc.shapeId == null || !spend(shapeCost(slide))) return base;
|
|
126
153
|
const same = slide.shapes.filter((s) => s.id === loc.shapeId);
|
|
127
154
|
let sh = same[0];
|
|
128
155
|
if (same.length > 1) {
|
|
129
|
-
|
|
156
|
+
// the page's text is hashed only when the budget pays for it (null: told apart by name, then the first)
|
|
157
|
+
const seen = ctx.text || ctx.quote || "";
|
|
158
|
+
const want = affords(seen.length) && spend(weigh(seen)) ? textHash(seen) : null;
|
|
130
159
|
sh = same.find((s) => s.name === loc.shapeName && s.hash === want) || same.find((s) => s.hash === want) || same.find((s) => s.name === loc.shapeName) || same[0];
|
|
131
160
|
}
|
|
132
161
|
if (!sh) return { ...base, shapeId: loc.shapeId, shapeName: loc.shapeName || "", kind: "shape" };
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
const
|
|
162
|
+
// the page's quote, prefix and suffix are normalized within the budget too (each can be a whole shape's text, per comment); past it
|
|
163
|
+
// the anchor is the shape, without the text range
|
|
164
|
+
const q = normPaid(ctx.quote) ?? "";
|
|
165
|
+
const raw = q ? ctx.quote : "";
|
|
166
|
+
const at = q && spend(textCost(sh, raw)) ? sh.text.indexOf(raw) : -1;
|
|
136
167
|
return {
|
|
137
168
|
...base,
|
|
138
169
|
shapeId: sh.id, shapeName: sh.name, ordinal: sh.ordinal, kind: sh.kind,
|
|
139
|
-
text:
|
|
170
|
+
text: spend(normCost(sh)) ? normShape(sh).slice(0, KEEP) : "", textHash: sh.hash,
|
|
140
171
|
// a text range inside the shape: the quote and what surrounds it
|
|
141
172
|
quote: ctx.shapeWide ? "" : q,
|
|
142
|
-
prefix:
|
|
143
|
-
suffix:
|
|
173
|
+
prefix: (normPaid(ctx.prefix) ?? "").slice(-CTX),
|
|
174
|
+
suffix: (normPaid(ctx.suffix) ?? "").slice(0, CTX),
|
|
144
175
|
start: !ctx.shapeWide && at >= 0 ? at : null,
|
|
145
176
|
end: !ctx.shapeWide && at >= 0 ? at + raw.length : null,
|
|
146
177
|
};
|
|
147
178
|
}
|
|
148
179
|
|
|
180
|
+
// A shape's normalized text, kept per shape of a reading: every anchor on a shape normalized all of it again (50 comments on a shape of
|
|
181
|
+
// 8 million characters held the server's thread for 5.6 s while the page placed them).
|
|
182
|
+
const NORM = new WeakMap();
|
|
183
|
+
const normShape = (sh) => {
|
|
184
|
+
let v = NORM.get(sh);
|
|
185
|
+
if (v === undefined) NORM.set(sh, (v = normText(sh.text)));
|
|
186
|
+
return v;
|
|
187
|
+
};
|
|
188
|
+
// A slide's normalized title, kept per slide of a reading and paid from the budget the first time: every anchor on a slide normalized the
|
|
189
|
+
// whole title again, outside the budget. Past the budget the anchor has no title text ("", asked again by the next request); the title's
|
|
190
|
+
// hash, read with the slide, still identifies it.
|
|
191
|
+
const TITLE = new WeakMap();
|
|
192
|
+
const titleOf = (slide) => {
|
|
193
|
+
let v = TITLE.get(slide);
|
|
194
|
+
if (v === undefined) {
|
|
195
|
+
const t = normPaid(slide.title);
|
|
196
|
+
if (t === null) return "";
|
|
197
|
+
TITLE.set(slide, (v = t.slice(0, 120)));
|
|
198
|
+
}
|
|
199
|
+
return v;
|
|
200
|
+
};
|
|
201
|
+
/** what normShape(sh) is about to read: the shape's text the first time (src/text.js weigh), nothing once it is kept */
|
|
202
|
+
const normCost = (sh) => (NORM.has(sh) ? 0 : weightOf(sh));
|
|
203
|
+
/** a native substring search for `q` in a shape's text (src/text.js searchCost), priced by the shape's byte table (src/text.js hitsOf) */
|
|
204
|
+
const HITS = new WeakMap();
|
|
205
|
+
const textCost = (sh, q) => searchCost(sh.text.length, q, hitsOf(HITS, sh, sh.text, q));
|
|
206
|
+
|
|
149
207
|
/**
|
|
150
208
|
* Re-resolve against new bytes. Slide: by p:sldId id; else the part path together with the title; else the title alone when unique.
|
|
151
209
|
* Shape (when the anchor has one): by id (+ ordinal); else the same name with the same text; else the same text; else the same name.
|
|
@@ -154,6 +212,7 @@ export function pptxDurable(deck, loc, ctx = {}) {
|
|
|
154
212
|
export function resolvePptx(deck, d) {
|
|
155
213
|
if (!d) return { method: "none" };
|
|
156
214
|
deck.ensure();
|
|
215
|
+
if (!spend(slideCost(deck))) return { method: "none" };
|
|
157
216
|
let slide = d.slideId != null ? slideById(deck, d.slideId) : null;
|
|
158
217
|
let method = "id";
|
|
159
218
|
if (!slide) {
|
|
@@ -166,6 +225,7 @@ export function resolvePptx(deck, d) {
|
|
|
166
225
|
const out = { method, slideId: slide.id, slideIndex: slide.index, shapeId: null, shapeName: "", start: null, end: null, changed: false, text: "", titleChanged: slide.titleHash !== d.titleHash };
|
|
167
226
|
if (d.shapeId == null) return out;
|
|
168
227
|
const shapes = slide.shapes;
|
|
228
|
+
if (!spend(shapeCost(slide))) return { method: "none" };
|
|
169
229
|
const byId = shapes.filter((s) => s.id === d.shapeId);
|
|
170
230
|
let sh = null;
|
|
171
231
|
let how = "";
|
|
@@ -183,10 +243,14 @@ export function resolvePptx(deck, d) {
|
|
|
183
243
|
else if (name.length === 1 && d.textHash && d.textHash !== EMPTY) [sh, how] = [name[0], "name"];
|
|
184
244
|
}
|
|
185
245
|
if (!sh) return { method: "none" };
|
|
186
|
-
|
|
246
|
+
// The shape is found (by its id, or its name and the hash read with the slide). Its text and the quote's place in it are read only when
|
|
247
|
+
// the budget pays for them: past it the comment is still on its shape, without a text excerpt or a range (a shape of 7.9 million CJK
|
|
248
|
+
// characters costs more than the whole budget to normalize, and its comments resolved as "moved or removed" with the shape right there).
|
|
249
|
+
const stext = spend(normCost(sh)) ? normShape(sh) : "";
|
|
187
250
|
const q = d.quote || "";
|
|
188
|
-
const
|
|
189
|
-
|
|
251
|
+
const searched = !!q && spend(textCost(sh, q)); // a shape-wide comment has no quote, and no search to pay for
|
|
252
|
+
const at = searched ? sh.text.indexOf(q) : -1;
|
|
253
|
+
return { ...out, method: method === "id" ? how : `${method}+${how}`, shapeId: sh.id, shapeName: sh.name, start: at >= 0 ? at : null, end: at >= 0 ? at + q.length : null, changed: sh.hash !== d.textHash, text: stext.slice(0, 200), quoteLost: searched && at < 0 };
|
|
190
254
|
}
|
|
191
255
|
|
|
192
256
|
/** Locator fields for the batch: the names the `apply` targets use. */
|
package/src/quick-plan.js
CHANGED
|
@@ -5,10 +5,15 @@
|
|
|
5
5
|
// binds that number as `expect.count`, so a file that changed since the preview, or a tampered number, fails the count check and nothing
|
|
6
6
|
// is written (the batch then goes to the agent unchanged).
|
|
7
7
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
8
|
+
// The engine's count runs in a worker thread with a time limit (src/offload.js previewOffThread), as the change itself does (src/local.js):
|
|
9
|
+
// a document of one paragraph of 8 million characters held the server's thread for 4 s in an outside review. A count that does not
|
|
10
|
+
// finish is no count: the page shows no button and the agent gets the note. The page asks 600 ms after typing stops, and answers are
|
|
11
|
+
// cached by file hash and command. The bytes are read whole within the readers' caps first (readChecked), so a file swapped on disk
|
|
12
|
+
// past a cap is refused in plain words before the engine reads it.
|
|
10
13
|
|
|
11
14
|
import { checkRegularOwned, readDoc, sha256 } from "./fsutil.js";
|
|
15
|
+
import { readChecked } from "./session.js";
|
|
16
|
+
import { previewOffThread } from "./offload.js";
|
|
12
17
|
import { recognize, refusalFor, wordsFor } from "./quick.js";
|
|
13
18
|
import { userText } from "./sanitize.js";
|
|
14
19
|
|
|
@@ -30,8 +35,7 @@ function ridesAlong(s) {
|
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
async function previewOf(bytes, format, q) {
|
|
33
|
-
const
|
|
34
|
-
const p = await previewAll(bytes, format, q.op);
|
|
38
|
+
const p = await previewOffThread(bytes, format, q.op); // throws when the worker did not finish: no count, nothing shown or written
|
|
35
39
|
const count = p.ok ? p.count : 0;
|
|
36
40
|
const skipped = p.ok ? p.protected || 0 : 0;
|
|
37
41
|
let why = refusalFor(q.kind, p, q.find);
|
|
@@ -56,9 +60,11 @@ export async function quickPreview(s, note) {
|
|
|
56
60
|
} catch {
|
|
57
61
|
return { match: null };
|
|
58
62
|
}
|
|
59
|
-
const
|
|
63
|
+
const sha = sha256(bytes);
|
|
64
|
+
const key = `${sha}\0${s.format}\0${q.command}`;
|
|
60
65
|
let match = cache.get(key);
|
|
61
66
|
if (!match) {
|
|
67
|
+
readChecked(s.format, bytes, { sha }); // past a reader's cap: refused with the reason (422), the engine never sees it
|
|
62
68
|
try {
|
|
63
69
|
const { count, skipped, why } = await previewOf(bytes, s.format, q);
|
|
64
70
|
match = { action: q.kind, command: q.command, count, protectedCount: skipped, local: !why, why: why || null, ...wordsFor(q, count, skipped) };
|
|
@@ -96,10 +102,11 @@ export const quickPlanner = {
|
|
|
96
102
|
try {
|
|
97
103
|
checkRegularOwned(s.file);
|
|
98
104
|
bytes = readDoc(s.file);
|
|
105
|
+
if (sha256(bytes) !== batch.base_sha256) return null; // the file moved since the Send
|
|
106
|
+
readChecked(s.format, bytes, { sha: batch.base_sha256 }); // within the readers' caps before the engine reads it
|
|
99
107
|
} catch {
|
|
100
108
|
return null;
|
|
101
109
|
}
|
|
102
|
-
if (sha256(bytes) !== batch.base_sha256) return null; // the file moved since the Send
|
|
103
110
|
const { count, skipped, why } = await previewOf(bytes, batch.format, q);
|
|
104
111
|
if (why || count !== confirmed.count) return null; // not what the person saw
|
|
105
112
|
return { reply: { summary: wordsFor(q, count, skipped).summary, replies: [], ops: [{ ...q.op, expect: { count } }] } };
|
package/src/redline-worker.js
CHANGED
|
@@ -1,29 +1,25 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
-
import { parentPort, workerData } from "node:worker_threads";
|
|
3
2
|
import { buildTracked } from "./redline.js";
|
|
4
3
|
import { verifyTracked } from "./redline-check.js";
|
|
5
4
|
import { env } from "./constants.js";
|
|
6
5
|
import { openZip } from "./zip.js";
|
|
7
6
|
import { replaceEntry } from "./zip-write.js";
|
|
8
7
|
|
|
9
|
-
// One tracked-changes copy, off the server thread: build it (src/redline.js), then check it (src/redline-check.js). One message back:
|
|
8
|
+
// One tracked-changes copy, off the server thread (run by src/worker-host.js): build it (src/redline.js), then check it (src/redline-check.js). One message back:
|
|
10
9
|
// {ok: true, bytes, counts} | {refused: [codes]} | {check_failed: why}. The copy's bytes only leave this worker after the check passed.
|
|
11
10
|
// ONIONSKIN_TEST_TAMPER_EXPORT=drop-del (tests only) removes one deletion from the built copy before the check, to prove the check is
|
|
12
11
|
// what stands between the builder and the disk; it can only make a copy fail, never pass.
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
if (r.refused) msg = { refused: r.refused };
|
|
20
|
-
else {
|
|
12
|
+
export function work({ before, after, author, date }) {
|
|
13
|
+
try {
|
|
14
|
+
const b = fs.readFileSync(before);
|
|
15
|
+
const a = fs.readFileSync(after);
|
|
16
|
+
const r = buildTracked({ before: b, after: a, author, date });
|
|
17
|
+
if (r.refused) return { refused: r.refused };
|
|
21
18
|
let out = r.bytes;
|
|
22
19
|
if (env("TEST_TAMPER_EXPORT", "") === "drop-del") out = replaceEntry(out, "word/document.xml", openZip(out).readText("word/document.xml").replace(/<w:del [^>]*>[\s\S]*?<\/w:del>/, ""));
|
|
23
20
|
const v = verifyTracked({ before: b, after: a, out, counts: r.counts, author, date });
|
|
24
|
-
|
|
21
|
+
return v.ok ? { ok: true, bytes: out, counts: r.counts } : { check_failed: v.why };
|
|
22
|
+
} catch (err) {
|
|
23
|
+
return { check_failed: `the copy could not be made: ${String(err?.message || err).slice(0, 200)}` };
|
|
25
24
|
}
|
|
26
|
-
} catch (err) {
|
|
27
|
-
msg = { check_failed: `the copy could not be made: ${String(err?.message || err).slice(0, 200)}` };
|
|
28
25
|
}
|
|
29
|
-
parentPort.postMessage(msg);
|
package/src/reply.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import {
|
|
4
|
-
import { ApplyError, APPLY_EXIT, applyCommand } from "./apply.js";
|
|
5
|
-
import { parseTicket } from "./lease.js";
|
|
6
|
-
import { isAgentHolder } from "./agent-detect.js";
|
|
3
|
+
import { PRODUCT, unknownFormat } from "./constants.js";
|
|
4
|
+
import { ApplyError, APPLY_EXIT, applyCommand, formatOfPath } from "./apply.js";
|
|
5
|
+
import { loadTicketRecord, parseTicket } from "./lease.js";
|
|
6
|
+
import { NOT_YOUR_FILE, agentMayUse, isAgentHolder } from "./agent-detect.js";
|
|
7
7
|
import { QUESTION_FIX, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
|
|
8
8
|
import { SUGGESTIONS_MAX, SUGGEST_FIX, cleanSuggestion } from "./review.js";
|
|
9
9
|
import { VOICE_FIX, cleanVoiceDraft } from "./voice-guide.js";
|
|
@@ -186,13 +186,16 @@ export function planReplies(batch, input, { partial = false } = {}) {
|
|
|
186
186
|
if (mode === "find") {
|
|
187
187
|
if (typeof r.find !== "string" || !r.find || typeof r.replace !== "string") return refuse("bad_reply", "find needs a non-empty string and replace needs a string");
|
|
188
188
|
const extras = Object.fromEntries(SHORTHAND_EXTRAS.filter((k) => k in r).map((k) => [k, r[k]]));
|
|
189
|
-
if (format === "docx") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
|
|
189
|
+
if (format === "docx" || format === "html") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
|
|
190
190
|
else if (format === "pptx") op = { op: "replace_text_in_shape", ...base, find: r.find, replace: r.replace, ...extras };
|
|
191
|
+
else if (format !== "xlsx") return unknownFormat(format, "reply find");
|
|
191
192
|
else return refuse("shorthand_not_for_format", "find/replace does not apply to a .xlsx cell", "use `set` for a cell or range");
|
|
192
193
|
} else if (mode === "text") {
|
|
193
194
|
if (typeof r.text !== "string") return refuse("bad_reply", "text must be a string");
|
|
194
195
|
if (format === "docx") op = { op: "replace_text", ...base, text: r.text };
|
|
196
|
+
else if (format === "html") op = { op: "set_text", ...base, text: r.text };
|
|
195
197
|
else if (format === "pptx") op = { op: "set_shape_text", ...base, paragraphs: r.text.split("\n") };
|
|
198
|
+
else if (format !== "xlsx") return unknownFormat(format, "reply text");
|
|
196
199
|
else return refuse("shorthand_not_for_format", "text does not apply to a .xlsx cell", "use `set` for a cell or range");
|
|
197
200
|
} else if (mode === "set") {
|
|
198
201
|
if (format !== "xlsx") return refuse("shorthand_not_for_format", `set applies to a .xlsx cell or range, not a .${format}`, "use `find`+`replace` or `text`");
|
|
@@ -264,7 +267,7 @@ const FIX = {
|
|
|
264
267
|
skipped_protected: "Protected place (field, equation, content control or similar): it cannot be rewritten losslessly. Decline this comment with a note.",
|
|
265
268
|
};
|
|
266
269
|
const fixFor = (r) => FIX[r.status] || (r.status === "refused" ? `Refused by the engine${r.code ? ` (${r.code})` : ""}. Change the op as the reason says, or decline.` : undefined);
|
|
267
|
-
const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
|
|
270
|
+
const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(Array.isArray(r.reasons) ? { reasons: r.reasons } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
|
|
268
271
|
|
|
269
272
|
/**
|
|
270
273
|
* The command. `deps` = {server: {healthy, req, json}, finish(file, results, lease) -> done's JSON, pollNext(file, timeout) -> string, readStdin(), holder?}.
|
|
@@ -279,32 +282,56 @@ export async function replyCommand(rest, deps) {
|
|
|
279
282
|
throw new ReplyError(err.message);
|
|
280
283
|
}
|
|
281
284
|
const { server } = deps;
|
|
285
|
+
// who calls: the local runner names itself; otherwise ONIONSKIN_HOLDER (an agent Onionskin started cannot change it), else the holder the
|
|
286
|
+
// server saved with this ticket. It goes with every request, so the server checks the binding and the lease before it hands out anything.
|
|
287
|
+
const envHolder = process.env[`${PRODUCT.envPrefix}_HOLDER`];
|
|
288
|
+
const ticketOfArgs = parseTicket(a.lease);
|
|
289
|
+
const holder = deps.holder || envHolder || (ticketOfArgs ? loadTicketRecord(ticketOfArgs.batch_id)?.holder : null) || "";
|
|
290
|
+
const notYours = () => new ReplyError(NOT_YOUR_FILE, { exit: APPLY_EXIT.write_refused, body: { code: "not_your_file" } });
|
|
282
291
|
if (!a.file) {
|
|
283
|
-
// no file given: the ticket
|
|
292
|
+
// no file given: the whole ticket (batch id and token) names the session; a wrong token names none
|
|
284
293
|
if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
|
|
285
|
-
const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(
|
|
294
|
+
const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(ticketOfArgs?.batch_id || "")}&lease_token=${encodeURIComponent(ticketOfArgs?.token || "")}&holder=${encodeURIComponent(holder)}` });
|
|
286
295
|
a.file = w.status === 200 ? server.json(w.text).file || "" : "";
|
|
287
296
|
if (!a.file) throw new ReplyError(`No delivered batch matches this ticket: your delivery was taken back, finished or handed on. Nothing was written. Run \`${PRODUCT.bin} poll <file>\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
|
|
288
297
|
}
|
|
298
|
+
// an agent Onionskin started: its own input first (only its one reply file, content-free checks), then its binding to its one file,
|
|
299
|
+
// refused before that file is looked at, in the same words whether it exists or is open
|
|
300
|
+
const bg = !deps.holder && isAgentHolder(envHolder);
|
|
301
|
+
const ticketRequired = () => new ReplyError(`--lease <ticket> is required: it is the lease_ticket of the batch (\`${PRODUCT.bin} status <file> --ticket\` prints it again)`);
|
|
302
|
+
let input;
|
|
303
|
+
if (bg) {
|
|
304
|
+
if (!ticketOfArgs) throw ticketRequired();
|
|
305
|
+
checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticketOfArgs.batch_id);
|
|
306
|
+
input = await readInput(a, deps.readStdin, { quiet: true });
|
|
307
|
+
if (!agentMayUse(envHolder, a.file)) throw notYours();
|
|
308
|
+
}
|
|
289
309
|
let real;
|
|
290
310
|
try {
|
|
291
311
|
real = fs.realpathSync(path.resolve(a.file));
|
|
292
312
|
} catch {
|
|
293
313
|
throw new ReplyError(`File not found: ${JSON.stringify(a.file.slice(0, 200))}`, { exit: APPLY_EXIT.error });
|
|
294
314
|
}
|
|
295
|
-
if (!
|
|
296
|
-
const ticket =
|
|
297
|
-
if (!ticket) throw
|
|
298
|
-
|
|
299
|
-
if (bg) checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticket.batch_id);
|
|
300
|
-
const input = await readInput(a, deps.readStdin, { quiet: bg });
|
|
315
|
+
if (!formatOfPath(real)) throw new ReplyError("Only .docx, .xlsx, .pptx, .html and .htm files are supported");
|
|
316
|
+
const ticket = ticketOfArgs;
|
|
317
|
+
if (!ticket) throw ticketRequired();
|
|
318
|
+
if (!bg) input = await readInput(a, deps.readStdin, { quiet: false });
|
|
301
319
|
|
|
302
|
-
// the batch the session holds for this delivery
|
|
320
|
+
// authorization first: the server hands out the batch the session holds for this delivery (the only source of anchors and hashes) only
|
|
321
|
+
// to this ticket's holder, checked before the reply is planned or run against the file
|
|
303
322
|
if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
|
|
304
|
-
const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}` });
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
323
|
+
const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}&holder=${encodeURIComponent(holder)}&batch_id=${encodeURIComponent(ticket.batch_id)}&lease_token=${encodeURIComponent(ticket.token)}` });
|
|
324
|
+
let refused = null;
|
|
325
|
+
try {
|
|
326
|
+
refused = r.status === 200 ? null : server.json(r.text);
|
|
327
|
+
} catch {
|
|
328
|
+
refused = {};
|
|
329
|
+
}
|
|
330
|
+
if (refused?.code === "not_your_file") throw notYours();
|
|
331
|
+
if (refused?.code === "no_batch") throw new ReplyError(`No batch is delivered for this file, so the ticket is void. Nothing was written. Run \`${PRODUCT.bin} poll\` and wait for the user's Send.`, { exit: APPLY_EXIT.write_refused, body: { code: "no_batch" } });
|
|
332
|
+
if (refused) throw new ReplyError(`This ticket is not the delivery the session holds: your delivery was taken back, finished or handed on, or the ticket is wrong. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: refused.code === "not_lease_holder" ? "not_lease_holder" : "stale_batch" } });
|
|
333
|
+
const held = server.json(r.text).batch;
|
|
334
|
+
if (!held || held.batch_id !== ticket.batch_id) throw new ReplyError(`This ticket is not the delivery the session holds. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
|
|
308
335
|
|
|
309
336
|
const plan = planReplies(held, input, { partial: a.partial });
|
|
310
337
|
if (plan.errors.length) throw new ReplyError("the reply cannot be applied; nothing was written and the batch is still open", { body: { code: "bad_reply", lease_kept: true, problems: plan.errors } });
|