onionskin-cli 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +136 -0
  2. package/INSTALL.md +17 -0
  3. package/LICENSE +21 -0
  4. package/NOTICE +46 -0
  5. package/README.md +88 -2
  6. package/docs/contracts/batch.schema.json +839 -0
  7. package/docs/contracts/history.schema.json +126 -0
  8. package/docs/contracts/ops.schema.json +541 -0
  9. package/package.json +54 -4
  10. package/scripts/checker/check.py +411 -0
  11. package/scripts/checker/requirements.txt +5 -0
  12. package/scripts/install-check.mjs +9 -0
  13. package/scripts/requirements.txt +5 -0
  14. package/scripts/rewriter/apple/main.swift +135 -0
  15. package/scripts/rewriter/apple/onionskin-fm +0 -0
  16. package/scripts/rewriter/apple/onionskin-fm.json +6 -0
  17. package/skills/onionskin/SKILL.md +117 -0
  18. package/skills/onionskin/reference/batch.md +94 -0
  19. package/skills/onionskin/reference/formats.md +15 -0
  20. package/skills/onionskin/reference/ops.md +253 -0
  21. package/skills/onionskin/reference/troubleshooting.md +73 -0
  22. package/skills/onionskin/reference/voice-guide.md +86 -0
  23. package/skills/onionskin/voice.example.md +51 -0
  24. package/src/agent-detect.js +149 -0
  25. package/src/agent-events.js +159 -0
  26. package/src/agent-prompt.js +27 -0
  27. package/src/agent-runner.js +593 -0
  28. package/src/analyze.js +17 -0
  29. package/src/anchors.js +190 -0
  30. package/src/apply.js +387 -0
  31. package/src/auth.js +130 -0
  32. package/src/batch.js +322 -0
  33. package/src/check-worker.js +8 -0
  34. package/src/checker.js +293 -0
  35. package/src/cli.js +516 -0
  36. package/src/constants.js +20 -0
  37. package/src/diff.js +267 -0
  38. package/src/doctor.js +247 -0
  39. package/src/docx.js +438 -0
  40. package/src/fmt.js +30 -0
  41. package/src/fsutil.js +348 -0
  42. package/src/history-worker.js +16 -0
  43. package/src/history.js +576 -0
  44. package/src/lease.js +79 -0
  45. package/src/local.js +242 -0
  46. package/src/merge-pptx.js +198 -0
  47. package/src/merge-xlsx.js +453 -0
  48. package/src/merge.js +410 -0
  49. package/src/offload.js +34 -0
  50. package/src/ops/common.js +48 -0
  51. package/src/ops/docx-table.js +323 -0
  52. package/src/ops/docx.js +503 -0
  53. package/src/ops/engine.js +41 -0
  54. package/src/ops/pptx.js +336 -0
  55. package/src/ops/run.js +115 -0
  56. package/src/ops/runs.js +127 -0
  57. package/src/ops/schema.js +109 -0
  58. package/src/ops/xlsx.js +277 -0
  59. package/src/package.js +189 -0
  60. package/src/paths.js +64 -0
  61. package/src/person.js +66 -0
  62. package/src/pptx.js +194 -0
  63. package/src/question.js +47 -0
  64. package/src/quick-plan.js +107 -0
  65. package/src/quick.js +164 -0
  66. package/src/redline-check.js +251 -0
  67. package/src/redline-worker.js +29 -0
  68. package/src/redline.js +438 -0
  69. package/src/reply.js +410 -0
  70. package/src/review.js +274 -0
  71. package/src/rewrite-check.js +1014 -0
  72. package/src/rewriter-cleanup.js +105 -0
  73. package/src/rewriter-setup.js +104 -0
  74. package/src/rewriter.js +396 -0
  75. package/src/sanitize.js +42 -0
  76. package/src/sentence-span.js +25 -0
  77. package/src/server-entry.js +13 -0
  78. package/src/server.js +576 -0
  79. package/src/session.js +1910 -0
  80. package/src/settings.js +161 -0
  81. package/src/setup.js +160 -0
  82. package/src/stamp-words.js +10 -0
  83. package/src/stamps.js +59 -0
  84. package/src/style.js +110 -0
  85. package/src/suggest-reply.js +84 -0
  86. package/src/suggest.js +261 -0
  87. package/src/text.js +31 -0
  88. package/src/tracked-words.js +43 -0
  89. package/src/voice-guide.js +182 -0
  90. package/src/voice-learn.js +314 -0
  91. package/src/voice-sample-worker.js +95 -0
  92. package/src/voice-samples.js +329 -0
  93. package/src/voice-store.js +305 -0
  94. package/src/voice-worker.js +28 -0
  95. package/src/voice.js +297 -0
  96. package/src/xlsx.js +454 -0
  97. package/src/xml-tree.js +118 -0
  98. package/src/xml.js +127 -0
  99. package/src/zip-write.js +74 -0
  100. package/src/zip.js +88 -0
  101. package/vendor/engine/DocViewer-BZLXUcxJ.js +15137 -0
  102. package/vendor/engine/THIRD_PARTY_LICENSES.txt +79 -0
  103. package/vendor/engine/collab-BVynXMqc.js +2591 -0
  104. package/vendor/engine/collab.js +2 -0
  105. package/vendor/engine/core.js +4 -0
  106. package/vendor/engine/default-worker-DogNT09A.js +35625 -0
  107. package/vendor/engine/embed.js +107 -0
  108. package/vendor/engine/index.js +5 -0
  109. package/vendor/engine/office-engine.embed.js +184 -0
  110. package/vendor/engine/office-engine.worker.js +36030 -0
  111. package/vendor/engine/xlsx-ops-SYAv6V0f.js +939 -0
  112. package/vendor/engine-node/LICENSE +21 -0
  113. package/vendor/engine-node/engine-node.mjs +46618 -0
  114. package/vendor/engine.lock.json +26 -0
  115. package/web/dist/assets/index-CFISATkn.js +197 -0
  116. package/web/dist/assets/index-spXE4oTz.css +1 -0
  117. package/web/dist/index.html +14 -0
@@ -0,0 +1,314 @@
1
+ // Learning at Send, stage 1 (code only), and the browser routes for it. The pure rules are in src/voice.js, the store in
2
+ // src/voice-store.js, the DOCX step in src/voice-worker.js; this module only joins them to the session and the server.
3
+ //
4
+ // What reaches the store from a Send: the five booleans per edit pair and the untrusted examples (from the worker, over three stored
5
+ // versions), and the trait ids that fixed English phrases in the person's own comments point at. Never a comment's text, a file name or a
6
+ // path. Everything here fails quietly: learning never stops a Send, and a step that fails learns nothing.
7
+ import { Worker } from "node:worker_threads";
8
+ import { buildVoiceBatch, printedSize } from "./batch.js";
9
+ import { env } from "./constants.js";
10
+ import { checkRegularOwned, readDoc, sha256 } from "./fsutil.js";
11
+ import { userText } from "./sanitize.js";
12
+ import { HttpError } from "./session.js"; // a cycle (session.js imports voiceJob from here): both only use each other inside functions
13
+ import { readStamps } from "./stamps.js";
14
+ import { BUILT_IN_STAMP_WORDS, STAMP_LANGUAGES } from "./stamp-words.js";
15
+ import { TRAITS, readStyle, styleNote, writeStyle } from "./style.js";
16
+ import { parseGuide } from "./voice-guide.js";
17
+ import { budget, pickRequests, pickRewrites, voiceBatchMax, voiceShares } from "./voice-samples.js";
18
+ import { VoiceError, appendRecords, discardDraft, forget, readDraft, readEvidence, readGuide, setAnswer, setLearning, writeGuide } from "./voice-store.js";
19
+ import { commentHits, suggestions } from "./voice.js";
20
+
21
+ const WORKER_TIMEOUT_MS = () => Number(env("VOICE_TIMEOUT_MS", 60_000));
22
+ const TRAIT_IDS = TRAITS.map((t) => t.id);
23
+
24
+ // The built-in stamps' words live in src/stamp-words.js (the server's copy of web/src/stamps.js); re-exported for test/voice-learn.test.js.
25
+ export { BUILT_IN_STAMP_WORDS, STAMP_LANGUAGES };
26
+
27
+ /**
28
+ * Every text a stamp sends: the built-in words (each language for Translate) and the person's own stamps. A comment whose words are
29
+ * exactly one of these was a tap, not typing, so its phrases are not counted ("Shorter" tapped four times is not four requests).
30
+ * Ceiling: a stamp's words the person then edited before sending, or typed out by hand, count as typing.
31
+ */
32
+ function stampTexts() {
33
+ const out = new Set();
34
+ for (const w of BUILT_IN_STAMP_WORDS) for (const l of w.includes("{language}") ? STAMP_LANGUAGES : [""]) out.add(w.replace("{language}", l));
35
+ for (const s of readStamps()) out.add(s.words);
36
+ return new Set([...out].map((w) => userText(w).trim()));
37
+ }
38
+
39
+ /**
40
+ * What one Send gives learning, or null for nothing. `sent` are the comments that left draft in this Send (each counted once, when it
41
+ * first goes out); `last` is the newest history update before this Send's own. The three versions are taken only when the file is a
42
+ * Word document, this Send wrote the person's edits, the update before it is an agent run that ended on exactly the bytes this Send
43
+ * started from, and all three versions are still stored: then pre (before the agent), post (what the agent wrote) and mine (what the
44
+ * person sent) are linked through the agent's output. A Send after a Revert, an outside change or another Send of the person's has no
45
+ * agent run right before it, so it gives no pair. Neither does a run done on this Mac (history kind `local`, src/local.js): its words are
46
+ * the person's own, put in by a closed pattern, so edits on top of it say nothing about how a model writes. A quick fix's
47
+ * "Use this" is the exception (`local` with `model: true`): the local writing model wrote that sentence, so edits on top of it are
48
+ * rewrites of model text, the same reasoning as for an agent run. Nothing at all while learning is off.
49
+ * Returns {key, versions: {pre, post, mine} (snapshot blob paths) | null, hits: [trait ids]}.
50
+ */
51
+ export function voiceJob(s, { last = null, diskSha = null, writtenSha = null, sent = [] } = {}) {
52
+ if (!readEvidence().learning) return null;
53
+ const stamps = stampTexts();
54
+ // a re-sent answered question carries the words already counted when it first went out
55
+ const typed = sent.filter((c) => typeof c.text === "string" && !c.answer && !stamps.has(c.text.trim()));
56
+ const hits = typed.flatMap((c) => commentHits(c.text));
57
+ let versions = null;
58
+ if (s.format === "docx" && diskSha && writtenSha && writtenSha !== diskSha && (last?.kind === "agent" || (last?.kind === "local" && last.model === true)) && last.after === diskSha && last.before !== diskSha) {
59
+ const shas = [last.before, diskSha, writtenSha];
60
+ if (shas.every((h) => s.snapshots.has(h))) versions = { pre: s.snapshots.blob(shas[0]), post: s.snapshots.blob(shas[1]), mine: s.snapshots.blob(shas[2]) };
61
+ }
62
+ if (!versions && !hits.length) return null;
63
+ return { key: s.key, versions, hits };
64
+ }
65
+
66
+ const running = new Set(); // live workers, ended by stopVoice
67
+ let epoch = 0; // moved on by Forget and the learning switch: a job that started before either stores nothing
68
+
69
+ /** Runs src/voice-worker.js over three blob paths. Resolves {pairs, examples}, or null when it fails, crashes or times out (then it is ended). */
70
+ function analyze(versions) {
71
+ return new Promise((resolve) => {
72
+ let worker;
73
+ let settled = false;
74
+ let timer = null;
75
+ const finish = (r) => {
76
+ if (settled) return;
77
+ settled = true;
78
+ clearTimeout(timer);
79
+ running.delete(worker);
80
+ worker?.terminate();
81
+ resolve(r);
82
+ };
83
+ try {
84
+ worker = new Worker(new URL("./voice-worker.js", import.meta.url), { workerData: versions });
85
+ } catch {
86
+ return finish(null);
87
+ }
88
+ running.add(worker);
89
+ timer = setTimeout(() => finish(null), WORKER_TIMEOUT_MS());
90
+ worker.once("message", (m) => finish(m?.ok ? { pairs: m.pairs, examples: m.examples } : null));
91
+ worker.once("error", () => finish(null));
92
+ worker.once("exit", () => finish(null));
93
+ });
94
+ }
95
+
96
+ /** The server's `voice` listener: stores the comment hits at once, then the worker's pairs and examples. Never throws. */
97
+ export async function learnFromSend(job) {
98
+ try {
99
+ const started = epoch;
100
+ if (job.hits?.length) appendRecords({ hits: job.hits });
101
+ if (!job.versions) return;
102
+ const r = await analyze(job.versions);
103
+ if (r && epoch === started) appendRecords({ pairs: r.pairs, examples: r.examples });
104
+ } catch {} // learning is never worth a failed Send or a crashed server
105
+ }
106
+
107
+ /** Ends every running learning worker (the server is stopping). */
108
+ export function stopVoice() {
109
+ for (const w of [...running]) w.terminate();
110
+ running.clear();
111
+ }
112
+
113
+ // ---- browser routes (cookie, launch token, Origin: checked by the server before these run) ------------------------------------
114
+
115
+ export const VOICE_POSTS = ["/api/voice/answer", "/api/voice/guide", "/api/voice/forget", "/api/voice/learning", "/api/voice/draft/discard"];
116
+ /** The body cap of POST /api/voice/guide: a 7,000-character guide in a script of up to 4 UTF-8 bytes a character, plus the envelope. */
117
+ export const GUIDE_BODY_MAX = 32 * 1024;
118
+ const ANSWER_ACTIONS = ["on", "later", "never"];
119
+
120
+ /**
121
+ * GET /api/voice. `suggestions` are fixed trait ids and whole numbers only (the page writes every sentence from its own templates);
122
+ * `examples` are untrusted document-derived text, shown as text and exported only when ticked; `guide` and `style` are the person's own.
123
+ */
124
+ export function voiceState() {
125
+ const ev = readEvidence();
126
+ const style = readStyle();
127
+ const guide = readGuide();
128
+ const draft = readDraft(); // re-checked on every read; agent text, shown as plain text and never saved without the person
129
+ return {
130
+ learning: ev.learning,
131
+ suggestions: suggestions(ev, style),
132
+ examples: ev.examples.map(({ id, agent, mine }) => ({ id, agent, mine })),
133
+ examples_untrusted: true,
134
+ examples_count: ev.examples.length,
135
+ guide: { text: guide.text, on: guide.on, send: guide.send || "full", structured: parseGuide(guide.text).structured },
136
+ draft: draft ? { text: draft.text, at: draft.at, batch_id: draft.batch_id } : null,
137
+ draft_untrusted: true,
138
+ style,
139
+ };
140
+ }
141
+
142
+ /** Only these fields, nothing else: a page that sends any other field (text for a suggestion, say) gets a 400 and nothing is written. */
143
+ function only(body, keys) {
144
+ if (!body || typeof body !== "object" || Array.isArray(body)) throw new VoiceError("the request must be an object");
145
+ const extra = Object.keys(body).filter((k) => !keys.includes(k));
146
+ if (extra.length) throw new VoiceError(keys.length ? `only ${keys.join(" and ")} can be sent` : "nothing can be sent with this request");
147
+ }
148
+
149
+ /** POST /api/voice/*. Strict: a wrong shape is a 400 (VoiceError or StyleError) and writes nothing. Returns voiceState(). */
150
+ export function voicePost(p, body) {
151
+ if (p === "/api/voice/answer") {
152
+ only(body, ["trait", "action"]);
153
+ const { trait, action } = body;
154
+ if (typeof trait !== "string" || !TRAIT_IDS.includes(trait)) throw new VoiceError(`trait must be one of: ${TRAIT_IDS.join(", ")}`);
155
+ if (typeof action !== "string" || !ANSWER_ACTIONS.includes(action)) throw new VoiceError(`action must be one of: ${ANSWER_ACTIONS.join(", ")}`);
156
+ if (action === "on") {
157
+ // the sentence that goes to the agent is the trait's constant in src/style.js; the person's own words stay as they are
158
+ const cur = readStyle();
159
+ writeStyle({ traits: [...cur.traits, trait], words: cur.words });
160
+ } else setAnswer(trait, action);
161
+ } else if (p === "/api/voice/guide") {
162
+ only(body, ["text", "on", "send"]);
163
+ writeGuide({ text: body.text, on: body.on, send: body.send }); // more than 7,000 characters is a 400, never a cut; no send is the full guide
164
+ } else if (p === "/api/voice/forget") {
165
+ only(body, []);
166
+ epoch++;
167
+ forget();
168
+ } else if (p === "/api/voice/learning") {
169
+ only(body, ["on"]);
170
+ if (typeof body.on !== "boolean") throw new VoiceError("on must be true or false");
171
+ epoch++;
172
+ setLearning(body.on);
173
+ } else if (p === "/api/voice/draft/discard") {
174
+ // `at` names the draft that was on screen: a newer one that arrived meanwhile was never read, and stays
175
+ only(body, ["at"]);
176
+ if (!Number.isSafeInteger(body.at) || body.at < 0) throw new VoiceError("at must be the time of the draft on screen");
177
+ if (readDraft()?.at === body.at) discardDraft();
178
+ } else throw new VoiceError("unknown request");
179
+ return voiceState();
180
+ }
181
+
182
+ // ---- "Build my voice guide" (stage 2): the request ------------------------------------------------------------------------------
183
+
184
+ export const MAX_ADDED = 5; // added sources in all: files plus pasted texts
185
+ export const ADDED_FILE_MAX = 20 * 1024 * 1024;
186
+ export const PASTE_MAX = 20_000;
187
+ const FILE_FORMATS = ["docx", "pptx", "text"];
188
+ const BUILD_FIELDS = ["open_file", "edits", "comments", "improve", "texts", "files"];
189
+ const AGENT_PAIRS_MAX = 20; // the newest agent runs whose two versions are read to leave their text out
190
+ /** The body cap of POST /api/voice/build (instead of MAX_BODY): five 20 MB files in base64, plus 1 MB. */
191
+ export const VOICE_BUILD_MAX_BODY = Math.ceil((MAX_ADDED * ADDED_FILE_MAX * 4) / 3) + 1024 * 1024;
192
+ const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/;
193
+ const SAMPLE_TIMEOUT_MS = () => Number(env("VOICE_SAMPLE_TIMEOUT_MS", 60_000));
194
+
195
+ const NO_SAMPLE = "Choose at least one sample of your writing.";
196
+ const UNREADABLE = "That file could not be read. Choose a Word, PowerPoint or text file.";
197
+
198
+ const httpError = (status, message, code) => new HttpError(status, message, { code });
199
+
200
+ /** One added file {format, bytesB64}: exact fields, a known format, strict base64 (Buffer.from never fails on bad input), 1 byte to 20 MB. */
201
+ function addedFile(f) {
202
+ if (!f || typeof f !== "object" || Array.isArray(f)) throw new VoiceError(UNREADABLE);
203
+ if (Object.keys(f).some((k) => k !== "format" && k !== "bytesB64")) throw new VoiceError("a file is only {format, bytesB64}");
204
+ if (!FILE_FORMATS.includes(f.format) || typeof f.bytesB64 !== "string") throw new VoiceError(UNREADABLE);
205
+ if (f.bytesB64.length > Math.ceil((ADDED_FILE_MAX * 4) / 3) + 4) throw new VoiceError("Each file can be up to 20 MB.");
206
+ if (!f.bytesB64 || f.bytesB64.length % 4 !== 0 || !BASE64.test(f.bytesB64)) throw new VoiceError(UNREADABLE);
207
+ const bytes = Buffer.from(f.bytesB64, "base64");
208
+ if (bytes.length > ADDED_FILE_MAX) throw new VoiceError("Each file can be up to 20 MB.");
209
+ return { format: f.format, bytes };
210
+ }
211
+
212
+ /** The person's typed comments in this file (stamp taps left out: they are not typing), oldest first. */
213
+ function typedComments(s) {
214
+ const stamps = stampTexts();
215
+ return (s.comments || []).filter((c) => typeof c.text === "string" && c.text.trim() && !stamps.has(c.text.trim())).map((c) => c.text);
216
+ }
217
+
218
+ /** The stored version pairs of the agent and local runs on this file, newest last, for leaving what they wrote out of the open file's sample. */
219
+ function agentPairs(s) {
220
+ const runs = (s.history || []).filter((u) => (u.kind === "agent" || u.kind === "local") && s.snapshots.has(u.before) && s.snapshots.has(u.after));
221
+ return runs.slice(-AGENT_PAIRS_MAX).map((u) => ({ before: s.snapshots.blob(u.before), after: s.snapshots.blob(u.after) }));
222
+ }
223
+
224
+ /** Runs src/voice-sample-worker.js. Resolves {sources} or {error: code}; a crash or a timeout (60 s) is "unreadable" and the worker is ended. */
225
+ function sampleInWorker(job) {
226
+ return new Promise((resolve) => {
227
+ let worker;
228
+ let settled = false;
229
+ let timer = null;
230
+ const finish = (r) => {
231
+ if (settled) return;
232
+ settled = true;
233
+ clearTimeout(timer);
234
+ running.delete(worker);
235
+ worker?.terminate();
236
+ resolve(r);
237
+ };
238
+ try {
239
+ worker = new Worker(new URL("./voice-sample-worker.js", import.meta.url), { workerData: job });
240
+ } catch {
241
+ return finish({ error: "unreadable" });
242
+ }
243
+ running.add(worker);
244
+ timer = setTimeout(() => finish({ error: "unreadable" }), SAMPLE_TIMEOUT_MS());
245
+ worker.once("message", (m) => finish(m?.ok && Array.isArray(m.sources) ? { sources: m.sources } : { error: m?.code || "unreadable" }));
246
+ worker.once("error", () => finish({ error: "unreadable" }));
247
+ worker.once("exit", () => finish({ error: "unreadable" }));
248
+ });
249
+ }
250
+
251
+ /**
252
+ * POST /api/voice/build?s=<key> {open_file, edits, comments, improve, texts, files}: the person pressed "Ask the agent". Every field is
253
+ * required and checked exactly; anything else is a 400 and nothing is sent. Only what they ticked or added goes in, and only as untrusted
254
+ * evidence: the open file's saved version (`open_file`, not for Excel), their rewrites of the agent's text (`edits`), their typed comments
255
+ * (`comments`), up to 5 added files (Word, PowerPoint or UTF-8 text, at most 20 MB each, parsed in a worker with a 60 s limit) and pasted
256
+ * texts (at most 20,000 characters each). `improve` sends the guide they saved. Nothing is stored apart from the batch. The batch is
257
+ * trimmed to fit what `poll` prints (src/voice-samples.js budget) and handed to Sessions.requestVoice. Returns the session.
258
+ */
259
+ export async function buildVoiceRequest(sessions, key, body) {
260
+ only(body, BUILD_FIELDS);
261
+ for (const k of ["open_file", "edits", "comments", "improve"]) if (typeof body[k] !== "boolean") throw new VoiceError(`${k} must be true or false`);
262
+ if (!Array.isArray(body.texts) || !Array.isArray(body.files)) throw new VoiceError("texts and files must be lists");
263
+ if (body.texts.length + body.files.length > MAX_ADDED) throw new VoiceError("You can add up to 5 in all.");
264
+ for (const t of body.texts) {
265
+ if (typeof t !== "string" || !t.trim()) throw new VoiceError(NO_SAMPLE);
266
+ if ([...t].length > PASTE_MAX) throw new VoiceError("Pasted text can be up to 20,000 characters.");
267
+ }
268
+ const files = body.files.map(addedFile);
269
+ const s = sessions.get(key);
270
+ if (s.closed) throw httpError(409, "session closed", "closed");
271
+ if (s.pending || s.delivered) throw httpError(409, "Wait for the agent to finish, then try again.", "busy");
272
+ if (body.open_file && s.format === "xlsx") throw new VoiceError("Excel files are not used as writing samples.");
273
+ const examples = body.edits ? readEvidence().examples : [];
274
+ if (!body.open_file && !files.length && !body.texts.length && !examples.length) throw new VoiceError(NO_SAMPLE);
275
+ const saved = readGuide();
276
+ if (body.improve && !saved.text) throw new VoiceError("There is no saved guide to improve.");
277
+
278
+ let open = null;
279
+ if (body.open_file) {
280
+ let bytes;
281
+ try {
282
+ checkRegularOwned(s.file);
283
+ bytes = readDoc(s.file);
284
+ } catch {
285
+ throw httpError(409, "This file could not be read. Reload it, then try again.", "gone");
286
+ }
287
+ if (sha256(bytes) !== s.baseSha) throw httpError(409, "The file changed outside Onionskin. Reload it, then try again.", "changed_since");
288
+ open = { format: s.format, bytes, agentPairs: agentPairs(s) };
289
+ }
290
+ const baseSha = s.baseSha;
291
+ const style = styleNote();
292
+ const currentGuide = body.improve ? saved.text : "";
293
+ const batch = buildVoiceBatch({ format: s.format, baseSha, style, currentGuide });
294
+ const max = voiceBatchMax();
295
+ const sources = (open ? 1 : 0) + files.length + body.texts.length;
296
+ const shares = voiceShares({ fixed: printedSize(batch), sources, rewrites: body.edits, requests: body.comments, max });
297
+ let sampled = { sources: [] };
298
+ if (sources) {
299
+ sampled = await sampleInWorker({ open, files, texts: body.texts, shares: shares.source });
300
+ if (sampled.error) throw new VoiceError(UNREADABLE);
301
+ }
302
+ const evidence = {
303
+ sources: sampled.sources,
304
+ rewrites: body.edits ? pickRewrites(examples, shares.rewrites) : [],
305
+ my_requests: body.comments ? pickRequests(typedComments(s), shares.requests) : [],
306
+ };
307
+ const withEvidence = (e) => ({ ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, voice_evidence: e } });
308
+ const fitted = budget(evidence, (e) => printedSize(withEvidence(e)), max);
309
+ if (!fitted) throw new VoiceError("This request is too large to send.");
310
+ // requestVoice checks again: the worker took time, and a Send, a review or an outside change may have come in meanwhile
311
+ if (s.baseSha !== baseSha) throw httpError(409, "The file changed. Try again.", "changed_since");
312
+ sessions.requestVoice(key, withEvidence(fitted));
313
+ return s;
314
+ }
@@ -0,0 +1,95 @@
1
+ import fs from "node:fs";
2
+ import { parentPort, workerData } from "node:worker_threads";
3
+ import { readDocxParagraphs } from "./docx.js";
4
+ import { validatePackage } from "./package.js";
5
+ import { parseDeck } from "./pptx.js";
6
+ import { agentTexts, sampleDocx, samplePptx, sampleText } from "./voice-samples.js";
7
+
8
+ /**
9
+ * The parse step of "Build my voice guide": bytes in, samples out (src/voice-samples.js). Pure over its input apart from reading the
10
+ * stored version blobs named in `open.agentPairs`. Every Office file is checked as a package first (validatePackage: the main part's
11
+ * content type, openZip's size and expansion limits) and a macro-enabled one is refused. Excel files are not used as writing samples.
12
+ *
13
+ * Input: {open?: {format, bytes, agentPairs: [{before, after}] (snapshot blob paths)}, files: [{format, bytes}], texts: [string], shares}.
14
+ * `format` is "docx", "pptx" or "text" (a .txt or .md file, UTF-8); `shares` is one number for every source or a list, one per source in
15
+ * the order open, files, texts. Returns {sources: [{format, total, sampled, paragraphs}]} in that order. Throws a SampleError naming the
16
+ * source (its index in that order) and a fixed code: "format", "macro" or "unreadable". No file name or path is ever in the output.
17
+ */
18
+ export class SampleError extends Error {
19
+ constructor(index, code) {
20
+ super(`source ${index}: ${code}`);
21
+ this.index = index;
22
+ this.code = code;
23
+ }
24
+ }
25
+
26
+ const MACRO_PART = /(^|\/)vbaProject\.bin$/i;
27
+
28
+ function office(bytes, format, index) {
29
+ let zip;
30
+ try {
31
+ zip = validatePackage(bytes, format);
32
+ } catch (err) {
33
+ throw new SampleError(index, /macro/i.test(String(err?.message)) ? "macro" : "unreadable");
34
+ }
35
+ if (zip.names.some((n) => MACRO_PART.test(n))) throw new SampleError(index, "macro");
36
+ }
37
+
38
+ const textsOf = (format, bytes) => {
39
+ if (format === "docx") return readDocxParagraphs(bytes).map((p) => p.text);
40
+ const deck = parseDeck(bytes);
41
+ deck.ensure();
42
+ return deck.slides.flatMap((s) => [s.title, ...(s.shapes || []).map((sh) => sh.text)]);
43
+ };
44
+
45
+ // a Buffer posted to a worker arrives as a plain Uint8Array: the parsers want a Buffer
46
+ const asBuffer = (b) => (Buffer.isBuffer(b) ? b : b instanceof Uint8Array ? Buffer.from(b.buffer, b.byteOffset, b.byteLength) : b);
47
+
48
+ function sampleOne({ format, bytes: raw }, share, index, exclude = null) {
49
+ const bytes = asBuffer(raw);
50
+ try {
51
+ if (format === "docx") {
52
+ office(bytes, format, index);
53
+ return { format, ...sampleDocx(readDocxParagraphs(bytes, { detail: true }), share, { exclude }) };
54
+ }
55
+ if (format === "pptx") {
56
+ office(bytes, format, index);
57
+ return { format, ...samplePptx(parseDeck(bytes), share, { exclude }) };
58
+ }
59
+ if (format === "text") return { format, ...sampleText(new TextDecoder("utf-8", { fatal: true }).decode(bytes), share) };
60
+ } catch (err) {
61
+ throw err instanceof SampleError ? err : new SampleError(index, "unreadable");
62
+ }
63
+ throw new SampleError(index, "format");
64
+ }
65
+
66
+ export function sampleSources({ open = null, files = [], texts = [], shares = 0 } = {}) {
67
+ const shareOf = (i) => (Array.isArray(shares) ? shares[i] : shares);
68
+ const sources = [];
69
+ if (open) {
70
+ // a version pair that cannot be read (pruned, broken) just adds nothing to what is left out
71
+ const pairs = [];
72
+ for (const p of open.agentPairs || []) {
73
+ try {
74
+ pairs.push({ before: textsOf(open.format, fs.readFileSync(p.before)), after: textsOf(open.format, fs.readFileSync(p.after)) });
75
+ } catch {}
76
+ }
77
+ sources.push(sampleOne(open, shareOf(0), 0, agentTexts(pairs)));
78
+ }
79
+ for (const f of files) sources.push(sampleOne(f, shareOf(sources.length), sources.length));
80
+ for (const t of texts) {
81
+ if (typeof t !== "string") throw new SampleError(sources.length, "format");
82
+ sources.push({ format: "text", ...sampleText(t, shareOf(sources.length)) });
83
+ }
84
+ return { sources };
85
+ }
86
+
87
+ // Run in a worker thread (the same pattern as voice-worker.js), with a timeout in the caller. One message back: {ok: true, sources} or
88
+ // {ok: false, index, code}. Anything else that goes wrong is {ok: false, index: -1, code: "unreadable"}.
89
+ if (parentPort) {
90
+ try {
91
+ parentPort.postMessage({ ok: true, ...sampleSources(workerData) });
92
+ } catch (err) {
93
+ parentPort.postMessage(err instanceof SampleError ? { ok: false, index: err.index, code: err.code } : { ok: false, index: -1, code: "unreadable" });
94
+ }
95
+ }