onionskin-cli 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +136 -0
  2. package/INSTALL.md +17 -0
  3. package/LICENSE +21 -0
  4. package/NOTICE +46 -0
  5. package/README.md +88 -2
  6. package/docs/contracts/batch.schema.json +839 -0
  7. package/docs/contracts/history.schema.json +126 -0
  8. package/docs/contracts/ops.schema.json +541 -0
  9. package/package.json +54 -4
  10. package/scripts/checker/check.py +411 -0
  11. package/scripts/checker/requirements.txt +5 -0
  12. package/scripts/install-check.mjs +9 -0
  13. package/scripts/requirements.txt +5 -0
  14. package/scripts/rewriter/apple/main.swift +135 -0
  15. package/scripts/rewriter/apple/onionskin-fm +0 -0
  16. package/scripts/rewriter/apple/onionskin-fm.json +6 -0
  17. package/skills/onionskin/SKILL.md +117 -0
  18. package/skills/onionskin/reference/batch.md +94 -0
  19. package/skills/onionskin/reference/formats.md +15 -0
  20. package/skills/onionskin/reference/ops.md +253 -0
  21. package/skills/onionskin/reference/troubleshooting.md +73 -0
  22. package/skills/onionskin/reference/voice-guide.md +86 -0
  23. package/skills/onionskin/voice.example.md +51 -0
  24. package/src/agent-detect.js +149 -0
  25. package/src/agent-events.js +159 -0
  26. package/src/agent-prompt.js +27 -0
  27. package/src/agent-runner.js +593 -0
  28. package/src/analyze.js +17 -0
  29. package/src/anchors.js +190 -0
  30. package/src/apply.js +387 -0
  31. package/src/auth.js +130 -0
  32. package/src/batch.js +322 -0
  33. package/src/check-worker.js +8 -0
  34. package/src/checker.js +293 -0
  35. package/src/cli.js +516 -0
  36. package/src/constants.js +20 -0
  37. package/src/diff.js +267 -0
  38. package/src/doctor.js +247 -0
  39. package/src/docx.js +438 -0
  40. package/src/fmt.js +30 -0
  41. package/src/fsutil.js +348 -0
  42. package/src/history-worker.js +16 -0
  43. package/src/history.js +576 -0
  44. package/src/lease.js +79 -0
  45. package/src/local.js +242 -0
  46. package/src/merge-pptx.js +198 -0
  47. package/src/merge-xlsx.js +453 -0
  48. package/src/merge.js +410 -0
  49. package/src/offload.js +34 -0
  50. package/src/ops/common.js +48 -0
  51. package/src/ops/docx-table.js +323 -0
  52. package/src/ops/docx.js +503 -0
  53. package/src/ops/engine.js +41 -0
  54. package/src/ops/pptx.js +336 -0
  55. package/src/ops/run.js +115 -0
  56. package/src/ops/runs.js +127 -0
  57. package/src/ops/schema.js +109 -0
  58. package/src/ops/xlsx.js +277 -0
  59. package/src/package.js +189 -0
  60. package/src/paths.js +64 -0
  61. package/src/person.js +66 -0
  62. package/src/pptx.js +194 -0
  63. package/src/question.js +47 -0
  64. package/src/quick-plan.js +107 -0
  65. package/src/quick.js +164 -0
  66. package/src/redline-check.js +251 -0
  67. package/src/redline-worker.js +29 -0
  68. package/src/redline.js +438 -0
  69. package/src/reply.js +410 -0
  70. package/src/review.js +274 -0
  71. package/src/rewrite-check.js +1014 -0
  72. package/src/rewriter-cleanup.js +105 -0
  73. package/src/rewriter-setup.js +104 -0
  74. package/src/rewriter.js +396 -0
  75. package/src/sanitize.js +42 -0
  76. package/src/sentence-span.js +25 -0
  77. package/src/server-entry.js +13 -0
  78. package/src/server.js +576 -0
  79. package/src/session.js +1910 -0
  80. package/src/settings.js +161 -0
  81. package/src/setup.js +160 -0
  82. package/src/stamp-words.js +10 -0
  83. package/src/stamps.js +59 -0
  84. package/src/style.js +110 -0
  85. package/src/suggest-reply.js +84 -0
  86. package/src/suggest.js +261 -0
  87. package/src/text.js +31 -0
  88. package/src/tracked-words.js +43 -0
  89. package/src/voice-guide.js +182 -0
  90. package/src/voice-learn.js +314 -0
  91. package/src/voice-sample-worker.js +95 -0
  92. package/src/voice-samples.js +329 -0
  93. package/src/voice-store.js +305 -0
  94. package/src/voice-worker.js +28 -0
  95. package/src/voice.js +297 -0
  96. package/src/xlsx.js +454 -0
  97. package/src/xml-tree.js +118 -0
  98. package/src/xml.js +127 -0
  99. package/src/zip-write.js +74 -0
  100. package/src/zip.js +88 -0
  101. package/vendor/engine/DocViewer-BZLXUcxJ.js +15137 -0
  102. package/vendor/engine/THIRD_PARTY_LICENSES.txt +79 -0
  103. package/vendor/engine/collab-BVynXMqc.js +2591 -0
  104. package/vendor/engine/collab.js +2 -0
  105. package/vendor/engine/core.js +4 -0
  106. package/vendor/engine/default-worker-DogNT09A.js +35625 -0
  107. package/vendor/engine/embed.js +107 -0
  108. package/vendor/engine/index.js +5 -0
  109. package/vendor/engine/office-engine.embed.js +184 -0
  110. package/vendor/engine/office-engine.worker.js +36030 -0
  111. package/vendor/engine/xlsx-ops-SYAv6V0f.js +939 -0
  112. package/vendor/engine-node/LICENSE +21 -0
  113. package/vendor/engine-node/engine-node.mjs +46618 -0
  114. package/vendor/engine.lock.json +26 -0
  115. package/web/dist/assets/index-CFISATkn.js +197 -0
  116. package/web/dist/assets/index-spXE4oTz.css +1 -0
  117. package/web/dist/index.html +14 -0
@@ -0,0 +1,329 @@
1
+ // Samples of the person's writing for "Build my voice guide" (stage 2). Pure: nothing here reads or writes a file.
2
+ //
3
+ // The agent gets a sample of each writing source the person picked, never a whole document: a long file is cut down by a fixed,
4
+ // deterministic rule (the same input always gives the same sample), paragraphs an agent wrote are left out of the open file, and the
5
+ // whole printed batch is kept under a budget so it fits in an agent's shell output. Every text that leaves here went through
6
+ // `untrusted()` (no control or bidi characters, one line, capped) and carries no file name or path: it is quoted data for the agent, never
7
+ // an instruction, and the person reads whatever the agent makes of it before anything is saved.
8
+ import { env } from "./constants.js";
9
+ import { untrusted } from "./sanitize.js";
10
+ import { normText, similarity, textHash } from "./text.js";
11
+
12
+ /** The printed batch (`onionskin poll` pretty-prints it) must stay at or under this many characters. Assumed shell output limit: about 30,000. */
13
+ export const VOICE_BATCH_MAX_CHARS = 28_000;
14
+ export const voiceBatchMax = () => Number(env("VOICE_BATCH_MAX_CHARS", VOICE_BATCH_MAX_CHARS));
15
+ export const PARA_MAX = 1200; // characters per sampled paragraph
16
+ export const SLIDE_TEXT_MAX = 600; // characters per slide title or shape text
17
+ export const REQUEST_MAX = 300; // characters per typed comment
18
+ export const MAX_REWRITES = 20;
19
+ export const MAX_REQUESTS = 30;
20
+ export const REWRITES_SHARE = 6000;
21
+ export const REQUESTS_SHARE = 2000;
22
+ export const SOURCE_MIN_SHARE = 2000;
23
+ export const AGENT_TEXTS_MAX = 500; // agent paragraphs compared by similarity (the hash check has no cap)
24
+ export const AGENT_SIMILAR = 0.5;
25
+ const OPEN_N = 3;
26
+ const CLOSE_N = 2;
27
+ const HEADINGS_N = 8;
28
+ const LIST_N = 6;
29
+ const MIN_WORDS = 4; // a body paragraph shorter than this says little about how someone writes
30
+ // What one sampled paragraph costs in the pretty-printed batch beyond its JSON string: indentation at its depth, the keys, the commas.
31
+ // An estimate for the shares; budget() measures the real thing.
32
+ const ITEM_OVERHEAD = 80;
33
+ const OBJ = new RegExp("\\uFFFC", "g"); // an inline picture's placeholder in DOCX text
34
+ const HEADING_STYLE = /heading|title/i;
35
+ const SKIP_PLACEHOLDERS = new Set(["sldNum", "dt", "ftr", "hdr"]);
36
+
37
+ const segmenter = new Intl.Segmenter("und", { granularity: "word" });
38
+ /** Words in any script (a CJK paragraph is many words, not one). */
39
+ export function wordCount(s) {
40
+ let n = 0;
41
+ for (const seg of segmenter.segment(s)) if (seg.isWordLike) n++;
42
+ return n;
43
+ }
44
+
45
+ /** Every index of 0..n-1 once, spread evenly: first, last, middle, then the quarters, and so on. Deterministic. */
46
+ export function spreadOrder(n) {
47
+ const out = [];
48
+ if (n <= 0) return out;
49
+ const seen = new Uint8Array(n);
50
+ const add = (i) => {
51
+ if (!seen[i]) (seen[i] = 1), out.push(i);
52
+ };
53
+ add(0);
54
+ for (let parts = 1; out.length < n; parts *= 2) for (let j = 0; j <= parts; j++) add(Math.round((j * (n - 1)) / parts));
55
+ return out;
56
+ }
57
+
58
+ /** `n` of `list`, evenly spaced, in order (all of it when n >= its length). */
59
+ export function evenPick(list, n) {
60
+ if (n >= list.length) return [...list];
61
+ if (n <= 0) return [];
62
+ if (n === 1) return [list[0]];
63
+ const out = [];
64
+ for (let j = 0; j < n; j++) {
65
+ const x = list[Math.round((j * (list.length - 1)) / (n - 1))];
66
+ if (out[out.length - 1] !== x) out.push(x);
67
+ }
68
+ return out;
69
+ }
70
+
71
+ const cost = (t) => JSON.stringify(t).length + ITEM_OVERHEAD;
72
+
73
+ /**
74
+ * The fixed rule over a source's candidate units [{k: "h"|"li"|"p", t}] (already filtered, deduplicated, agent text left out): the first
75
+ * 3 body paragraphs ("open"), the last 2 ("close"), up to 8 headings and up to 6 list items (evenly spaced), then body paragraphs spread
76
+ * evenly, each taken only while it fits in `share` characters. Output in document order, with the count of candidates and of samples.
77
+ */
78
+ function pick(units, share) {
79
+ const items = units.map((u, i) => ({ i, k: u.k, t: untrusted(u.t, u.max ?? PARA_MAX) })).filter((u) => u.t);
80
+ const body = items.filter((u) => u.k === "p").map((u) => u.i);
81
+ const open = body.slice(0, OPEN_N);
82
+ const close = body.slice(Math.max(OPEN_N, body.length - CLOSE_N));
83
+ const rest = body.slice(OPEN_N, Math.max(OPEN_N, body.length - CLOSE_N));
84
+ const label = new Map([...open.map((i) => [i, "open"]), ...close.map((i) => [i, "close"])]);
85
+ const order = [
86
+ ...open,
87
+ ...close,
88
+ ...evenPick(items.filter((u) => u.k === "h").map((u) => u.i), HEADINGS_N),
89
+ ...evenPick(items.filter((u) => u.k === "li").map((u) => u.i), LIST_N),
90
+ ...spreadOrder(rest.length).map((j) => rest[j]),
91
+ ];
92
+ const byIndex = new Map(items.map((u) => [u.i, u]));
93
+ const chosen = new Set();
94
+ let used = 0;
95
+ for (const i of order) {
96
+ const c = cost(byIndex.get(i).t);
97
+ if (used + c > share) continue;
98
+ used += c;
99
+ chosen.add(i);
100
+ }
101
+ const paragraphs = items.filter((u) => chosen.has(u.i)).map((u) => ({ k: label.get(u.i) || u.k, t: u.t }));
102
+ return { total: items.length, sampled: paragraphs.length, paragraphs };
103
+ }
104
+
105
+ /** Drops empty, duplicate (by textHash) and agent-written units, in that order, keeping the first of each text. */
106
+ function distinct(units, exclude) {
107
+ const seen = new Set();
108
+ const out = [];
109
+ for (const u of units) {
110
+ if (!normText(u.t)) continue;
111
+ const h = textHash(u.t);
112
+ if (seen.has(h)) continue;
113
+ seen.add(h);
114
+ if (exclude && isAgentText(u.t, exclude)) continue;
115
+ out.push(u);
116
+ }
117
+ return out;
118
+ }
119
+
120
+ /**
121
+ * A sample of a Word document. `paras` from readDocxParagraphs(bytes, {detail: true}) (src/docx.js). Left out: protected paragraphs,
122
+ * paragraphs in tables (and nested tables), empty ones, body paragraphs under 4 words, repeats, and, with `exclude` (from agentTexts),
123
+ * paragraphs an agent wrote. A heading is a paragraph whose style name (or id) says heading or title; a list item has a list.
124
+ */
125
+ export function sampleDocx(paras, share, { exclude = null } = {}) {
126
+ const units = [];
127
+ for (const p of Array.isArray(paras) ? paras : []) {
128
+ if (!p || p.protected || !Array.isArray(p.path) || p.path.length !== 1 || typeof p.text !== "string") continue;
129
+ const t = p.text.replace(OBJ, " ");
130
+ const k = HEADING_STYLE.test(p.styleName || p.style || "") ? "h" : p.list ? "li" : "p";
131
+ if (k === "p" && wordCount(normText(t)) < MIN_WORDS) continue;
132
+ units.push({ k, t });
133
+ }
134
+ return pick(distinct(units, exclude), share);
135
+ }
136
+
137
+ /**
138
+ * A sample of a PowerPoint deck from parseDeck(bytes) (src/pptx.js): slides spread evenly, each slide's title and the text of its other
139
+ * text shapes (no tables, slide numbers, dates, headers or footers), at most 600 characters each, while they fit in `share`. With
140
+ * `exclude`, text an agent wrote is left out.
141
+ */
142
+ export function samplePptx(deck, share, { exclude = null } = {}) {
143
+ deck.ensure();
144
+ const slides = deck.slides.map((s) => {
145
+ const units = [];
146
+ if (s.title) units.push({ k: "h", t: s.title, max: SLIDE_TEXT_MAX });
147
+ for (const sh of s.shapes || []) {
148
+ if (sh.isTitle || sh.kind !== "shape" || SKIP_PLACEHOLDERS.has(sh.placeholder)) continue;
149
+ units.push({ k: "p", t: sh.text, max: SLIDE_TEXT_MAX });
150
+ }
151
+ return units;
152
+ });
153
+ // repeats and agent text are dropped across the whole deck, then whole slides are taken in an even spread
154
+ const seen = new Set();
155
+ const kept = slides.map((units) =>
156
+ units.filter((u) => {
157
+ if (!normText(u.t)) return false;
158
+ const h = textHash(u.t);
159
+ if (seen.has(h)) return false;
160
+ seen.add(h);
161
+ return !(exclude && isAgentText(u.t, exclude));
162
+ })
163
+ );
164
+ const flat = kept.flatMap((units, s) => units.map((u) => ({ ...u, s, t: untrusted(u.t, SLIDE_TEXT_MAX) }))).filter((u) => u.t);
165
+ const chosen = new Set();
166
+ let used = 0;
167
+ for (const s of spreadOrder(kept.length)) {
168
+ for (const u of flat.filter((x) => x.s === s)) {
169
+ const c = cost(u.t);
170
+ if (used + c > share) continue;
171
+ used += c;
172
+ chosen.add(u);
173
+ }
174
+ }
175
+ const paragraphs = flat.filter((u) => chosen.has(u)).map((u) => ({ k: u.k, t: u.t }));
176
+ return { total: flat.length, sampled: paragraphs.length, paragraphs };
177
+ }
178
+
179
+ const LIST_LINE = /^\s*(?:[-*+•]|\d{1,3}[.)])\s+(?=\S)/u;
180
+ const MD_HEADING = /^#{1,6}\s+(?=\S)/u;
181
+ /**
182
+ * A sample of pasted text or a .txt or .md file: split on blank lines; a one-line block starting with "#" is a heading, a block whose
183
+ * every line starts with a list marker gives one list item per line; everything else is a body paragraph. Then the same rule as Word.
184
+ */
185
+ export function sampleText(str, share) {
186
+ const units = [];
187
+ for (const block of String(str ?? "").replace(/\r\n?/g, "\n").split(/\n[ \t]*\n/)) {
188
+ const lines = block.split("\n").filter((l) => l.trim());
189
+ if (!lines.length) continue;
190
+ if (lines.length === 1 && MD_HEADING.test(lines[0])) units.push({ k: "h", t: lines[0].replace(MD_HEADING, "") });
191
+ else if (lines.every((l) => LIST_LINE.test(l))) for (const l of lines) units.push({ k: "li", t: l.replace(LIST_LINE, "") });
192
+ else if (wordCount(normText(lines.join(" "))) >= MIN_WORDS) units.push({ k: "p", t: lines.join(" ") });
193
+ }
194
+ return pick(distinct(units, null), share);
195
+ }
196
+
197
+ /**
198
+ * What agents wrote in a file, from its stored version pairs: for each agent or local run whose two versions are both still stored,
199
+ * `before` and `after` are the paragraph (or shape) texts of the two versions. A paragraph of `after` whose hash is not in `before` was
200
+ * written by the run. Returns {hashes, texts}: every such hash, and the newest 500 texts for the similarity check.
201
+ * Ceiling: text an agent wrote in a run whose versions were pruned, or text pasted in from elsewhere, is not known here.
202
+ */
203
+ export function agentTexts(pairs) {
204
+ const hashes = new Set();
205
+ const texts = [];
206
+ for (const pair of Array.isArray(pairs) ? pairs : []) {
207
+ if (!pair || !Array.isArray(pair.before) || !Array.isArray(pair.after)) continue;
208
+ const before = new Set(pair.before.map((t) => textHash(t)));
209
+ for (const t of pair.after) {
210
+ if (typeof t !== "string" || !normText(t)) continue;
211
+ const h = textHash(t);
212
+ if (before.has(h) || hashes.has(h)) continue;
213
+ hashes.add(h);
214
+ texts.push(t);
215
+ }
216
+ }
217
+ const kept = texts.slice(-AGENT_TEXTS_MAX);
218
+ return { hashes, texts: kept, lens: kept.map((t) => normText(t).toLowerCase().length) };
219
+ }
220
+
221
+ /**
222
+ * True when a paragraph (text or {text}) is one an agent wrote: its hash is an agent paragraph's, or its similarity to one is at least
223
+ * 0.5 (the person's light edit of agent text is still agent text; a heavy rewrite is theirs). A length check skips pairs whose bigram
224
+ * counts alone keep them under 0.5, so the 500 x 600 worst case stays cheap.
225
+ */
226
+ export function isAgentText(p, set) {
227
+ const t = typeof p === "string" ? p : p?.text;
228
+ if (typeof t !== "string" || !set) return false;
229
+ if (set.hashes?.has(textHash(t))) return true;
230
+ const la = normText(t).toLowerCase().length - 1;
231
+ const texts = set.texts || [];
232
+ for (let i = 0; i < texts.length; i++) {
233
+ const lb = (set.lens?.[i] ?? normText(texts[i]).toLowerCase().length) - 1;
234
+ if (la > 0 && lb > 0 && 4 * Math.min(la, lb) < la + lb) continue; // Dice <= 2*min/(la+lb) < 0.5
235
+ if (similarity(t, texts[i]) >= AGENT_SIMILAR) return true;
236
+ }
237
+ return false;
238
+ }
239
+
240
+ /**
241
+ * The person's rewrites of agent text for the agent: the newest of the stored examples ({agent, mine}, oldest first, from
242
+ * src/voice-store.js) up to 20 and `share` characters, oldest first, as {ai, mine}.
243
+ */
244
+ export function pickRewrites(examples, share = REWRITES_SHARE) {
245
+ const out = [];
246
+ let used = 0;
247
+ for (const e of [...(Array.isArray(examples) ? examples : [])].reverse()) {
248
+ if (out.length >= MAX_REWRITES) break;
249
+ if (!e || typeof e.agent !== "string" || typeof e.mine !== "string") continue;
250
+ const r = { ai: untrusted(e.agent, PARA_MAX), mine: untrusted(e.mine, PARA_MAX) };
251
+ if (!r.ai || !r.mine) continue;
252
+ const c = JSON.stringify(r).length + ITEM_OVERHEAD;
253
+ if (used + c > share) continue;
254
+ used += c;
255
+ out.unshift(r);
256
+ }
257
+ return out;
258
+ }
259
+
260
+ /** The person's typed comments (newest last; stamp taps already left out by the caller): the newest 30 that fit `share`, in order, each at most 300 characters. */
261
+ export function pickRequests(texts, share = REQUESTS_SHARE) {
262
+ const seen = new Set();
263
+ const out = [];
264
+ let used = 0;
265
+ for (const raw of [...(Array.isArray(texts) ? texts : [])].reverse()) {
266
+ if (out.length >= MAX_REQUESTS) break;
267
+ const t = untrusted(raw, REQUEST_MAX);
268
+ if (!t || seen.has(t)) continue;
269
+ seen.add(t);
270
+ const c = cost(t);
271
+ if (used + c > share) continue;
272
+ used += c;
273
+ out.unshift(t);
274
+ }
275
+ return out;
276
+ }
277
+
278
+ /**
279
+ * How the room left in the batch is shared: `fixed` is the printed size of the batch with no evidence, `sources` the number of writing
280
+ * sources. Rewrites get up to 6,000 and typed comments up to 2,000 when chosen; the rest is split evenly across the sources, at least 2,000
281
+ * each (budget() then trims what does not fit).
282
+ */
283
+ export function voiceShares({ fixed, sources, rewrites = false, requests = false, max = voiceBatchMax() }) {
284
+ const room = Math.max(0, max - fixed);
285
+ const rw = rewrites ? Math.min(REWRITES_SHARE, room) : 0;
286
+ const rq = requests ? Math.min(REQUESTS_SHARE, Math.max(0, room - rw)) : 0;
287
+ const source = sources > 0 ? Math.max(SOURCE_MIN_SHARE, Math.floor((room - rw - rq) / sources)) : 0;
288
+ return { rewrites: rw, requests: rq, source };
289
+ }
290
+
291
+ // the order paragraphs are given up in when the batch is too big: body first, then list items, headings, closers, openers
292
+ const DROP_ORDER = ["p", "li", "h", "close", "open"];
293
+
294
+ /**
295
+ * Trims `evidence` ({sources: [{paragraphs, sampled, ...}], rewrites, my_requests}) until `measure(evidence)` (the printed size of the
296
+ * whole batch it goes into) is at most `max`. Deterministic: from the source whose paragraphs print largest, evenly spaced paragraphs of
297
+ * the first kind in DROP_ORDER that any source still has; when no source has any left, the oldest rewrite, then the oldest comment.
298
+ * Returns a trimmed copy, or null when even an empty evidence does not fit (then nothing is sent). Never returns an over-budget result.
299
+ */
300
+ export function budget(evidence, measure, max = voiceBatchMax()) {
301
+ const e = {
302
+ ...evidence,
303
+ sources: (evidence.sources || []).map((s) => ({ ...s, paragraphs: [...(s.paragraphs || [])] })),
304
+ rewrites: [...(evidence.rewrites || [])],
305
+ my_requests: [...(evidence.my_requests || [])],
306
+ };
307
+ for (;;) {
308
+ const size = measure(e);
309
+ if (size <= max) return e;
310
+ const over = size - max;
311
+ const kind = DROP_ORDER.find((k) => e.sources.some((s) => s.paragraphs.some((p) => p.k === k)));
312
+ if (kind) {
313
+ let best = null;
314
+ let bestSize = -1;
315
+ for (const s of e.sources) {
316
+ if (!s.paragraphs.some((p) => p.k === kind)) continue;
317
+ const n = JSON.stringify(s.paragraphs).length;
318
+ if (n > bestSize) (best = s), (bestSize = n);
319
+ }
320
+ const of = best.paragraphs.filter((p) => p.k === kind);
321
+ const each = of.reduce((n, p) => n + cost(p.t), 0) / of.length;
322
+ const drop = new Set(evenPick(of, Math.min(of.length, Math.max(1, Math.ceil(over / each)))));
323
+ best.paragraphs = best.paragraphs.filter((p) => !drop.has(p));
324
+ best.sampled = best.paragraphs.length;
325
+ } else if (e.rewrites.length) e.rewrites.shift();
326
+ else if (e.my_requests.length) e.my_requests.shift();
327
+ else return null;
328
+ }
329
+ }
@@ -0,0 +1,305 @@
1
+ // Learning evidence and the saved voice guide, per machine, in the state dir (all 0600, written with writeStateFile).
2
+ // voice-evidence.json: {v:1, learning, pairs:[{at, short, sentences, exclaim, restored, fmt}] (last 30, booleans only),
3
+ // hits:[{at, trait}] (last 40, comment-phrase trait ids only), examples:[{id, at, agent, mine}] (last 30, untrusted, each side at most
4
+ // 400 characters), answers:{<trait>: {later: <pairs kept at the time>} | {never: true}}}. It never holds a file name or a comment.
5
+ // voice-guide.json: {v:1, text, on, send?: "short"}: the person's own saved guide, at most 7,000 characters. `send: "short"` sends only
6
+ // its "In short" section (src/voice-guide.js); no `send` means the full guide.
7
+ // voice-draft.json: {v:1, at, batch_id, text}: the agent's latest draft of a guide, waiting for the person's review. Agent text: it is
8
+ // checked by cleanVoiceDraft on the way in and again on every read, and nothing copies it into the guide (only the person's Save does).
9
+ // Reads are strict: anything that does not fit the shape is dropped record by record. A missing evidence file reads as empty with
10
+ // learning on (the default); a file that is there but cannot be trusted as a whole (oversized, a link, not a regular file, someone else's,
11
+ // unreadable, not JSON, the wrong version, no boolean switch) reads as empty with learning OFF, so a switch the person turned off never
12
+ // fails open and nothing new is recorded until they turn it on again. Every write goes through the validation below, so a bad call
13
+ // changes nothing.
14
+ // Guard on voice-guide.json (the same pattern as style.json in src/style.js): this module keeps the exact file text it last wrote or first
15
+ // read, and re-reads the file on every read. A file changed by anything else while the server runs is put back from that copy and
16
+ // counted (guideChangedOutside). Ceiling: the copy lives in memory, so a change made while no server runs (an agent with a shell can stop
17
+ // it) is trusted by the next server's first read.
18
+ import crypto from "node:crypto";
19
+ import fs from "node:fs";
20
+ import path from "node:path";
21
+ import { fileURLToPath } from "node:url";
22
+ import { checkRegularOwned } from "./fsutil.js";
23
+ import { ensureStateDir, stateDir, writeStateFile } from "./paths.js";
24
+ import { userText } from "./sanitize.js";
25
+ import { SettingsError } from "./settings.js";
26
+ import { TRAITS } from "./style.js";
27
+ import { COMMENT_TRAITS, SIGNALS, exampleSide } from "./voice.js";
28
+ import { GUIDE_MAX, cleanVoiceDraft, shortForm } from "./voice-guide.js";
29
+
30
+ export { GUIDE_MAX };
31
+ export const MAX_PAIRS = 30;
32
+ export const MAX_HITS = 40;
33
+ export const MAX_EXAMPLES = 30;
34
+ export const SENDS = ["full", "short"];
35
+ const MAX_FILE_BYTES = 256 * 1024; // the largest valid evidence file is far below this; a bigger file is not ours
36
+
37
+ const TRAIT_IDS = TRAITS.map((t) => t.id);
38
+ const ANSWERS = ["later", "never"];
39
+ const ID = /^[0-9a-f]{12}$/;
40
+
41
+ export class VoiceError extends SettingsError {
42
+ constructor(message) {
43
+ super(400, message);
44
+ }
45
+ }
46
+
47
+ const evidenceFile = () => path.join(stateDir(), "voice-evidence.json");
48
+ const guideFile = () => path.join(stateDir(), "voice-guide.json");
49
+ const draftFile = () => path.join(stateDir(), "voice-draft.json");
50
+ const emptyEvidence = () => ({ v: 1, learning: true, pairs: [], hits: [], examples: [], answers: {} });
51
+
52
+ /**
53
+ * Parsed version-1 JSON object from a plain file of ours; "missing" when nothing is there (lstat ENOENT only: a dangling link is there);
54
+ * null for anything else (link, not a regular file, someone else's, too big, unreadable, not an object, another version).
55
+ */
56
+ function readObject(file) {
57
+ try {
58
+ if (checkRegularOwned(file).size > MAX_FILE_BYTES) return null;
59
+ const raw = JSON.parse(fs.readFileSync(file, "utf8"));
60
+ return raw && typeof raw === "object" && !Array.isArray(raw) && raw.v === 1 ? raw : null;
61
+ } catch (err) {
62
+ return err?.code === "ENOENT" ? "missing" : null;
63
+ }
64
+ }
65
+ const at = (x) => Number.isSafeInteger(x) && x >= 0;
66
+ const plain = (x) => x && typeof x === "object" && !Array.isArray(x);
67
+
68
+ function cleanPair(p) {
69
+ if (!plain(p) || !at(p.at) || SIGNALS.some((s) => typeof p[s] !== "boolean")) return null;
70
+ return { at: p.at, ...Object.fromEntries(SIGNALS.map((s) => [s, p[s]])) };
71
+ }
72
+ const cleanHit = (h) => (plain(h) && at(h.at) && typeof h.trait === "string" && COMMENT_TRAITS.includes(h.trait) ? { at: h.at, trait: h.trait } : null);
73
+ const exampleId = (agent, mine) => crypto.createHash("sha256").update(`${agent}\0${mine}`).digest("hex").slice(0, 12);
74
+ function cleanExample(e) {
75
+ if (!plain(e) || !at(e.at) || typeof e.agent !== "string" || typeof e.mine !== "string") return null;
76
+ // the same side rule as exampleOf: empty, too long, an invisible character or a link is refused, never cleaned or cut
77
+ const agent = exampleSide(e.agent);
78
+ const mine = exampleSide(e.mine);
79
+ if (agent == null || mine == null) return null;
80
+ return { id: exampleId(agent, mine), at: e.at, agent, mine };
81
+ }
82
+ function cleanAnswers(a) {
83
+ const out = {};
84
+ if (!plain(a)) return out;
85
+ for (const trait of TRAIT_IDS) {
86
+ if (!Object.hasOwn(a, trait) || !plain(a[trait])) continue;
87
+ if (a[trait].never === true) out[trait] = { never: true };
88
+ else if (Number.isSafeInteger(a[trait].later) && a[trait].later >= 0) out[trait] = { later: a[trait].later };
89
+ }
90
+ return out;
91
+ }
92
+ const list = (x, clean, max) => (Array.isArray(x) ? x.map(clean).filter(Boolean).slice(-max) : []);
93
+
94
+ /** The stored evidence, validated. No file reads as empty with learning on; a broken file reads as empty with learning off (fail closed). */
95
+ export function readEvidence() {
96
+ const raw = readObject(evidenceFile());
97
+ if (raw === "missing") return emptyEvidence();
98
+ if (!raw || typeof raw.learning !== "boolean") return { ...emptyEvidence(), learning: false };
99
+ const examples = list(raw.examples, cleanExample, Infinity);
100
+ const unique = examples.filter((e, i) => examples.findLastIndex((x) => x.id === e.id) === i).slice(-MAX_EXAMPLES);
101
+ return {
102
+ v: 1,
103
+ learning: raw.learning,
104
+ pairs: list(raw.pairs, cleanPair, MAX_PAIRS),
105
+ hits: list(raw.hits, cleanHit, MAX_HITS),
106
+ examples: unique,
107
+ answers: cleanAnswers(raw.answers),
108
+ };
109
+ }
110
+
111
+ function save(ev) {
112
+ ensureStateDir();
113
+ writeStateFile(evidenceFile(), JSON.stringify(ev, null, 2) + "\n");
114
+ return ev;
115
+ }
116
+
117
+ /**
118
+ * Appends one Send's records and returns the stored evidence. `pairs` are the worker's signal objects, `hits` trait ids from commentHits,
119
+ * `examples` the worker's {agent, mine}; `now` stamps them. Anything that does not validate is dropped, not stored. Does nothing (and writes
120
+ * nothing) while learning is off. The newest records are kept, older ones fall off the front.
121
+ */
122
+ export function appendRecords({ pairs = [], hits = [], examples = [] } = {}, now = Date.now()) {
123
+ const ev = readEvidence();
124
+ if (!ev.learning) return ev;
125
+ const add = (items, clean) => (Array.isArray(items) ? items.map((x) => clean(plain(x) ? { ...x, at: now } : x)).filter(Boolean) : []);
126
+ const newHits = add((Array.isArray(hits) ? hits : []).map((trait) => ({ trait })), cleanHit);
127
+ const next = {
128
+ ...ev,
129
+ pairs: [...ev.pairs, ...add(pairs, cleanPair)].slice(-MAX_PAIRS),
130
+ hits: [...ev.hits, ...newHits].slice(-MAX_HITS),
131
+ examples: [...ev.examples, ...add(examples, cleanExample)],
132
+ };
133
+ next.examples = next.examples.filter((e, i) => next.examples.findLastIndex((x) => x.id === e.id) === i).slice(-MAX_EXAMPLES);
134
+ return save(next);
135
+ }
136
+
137
+ /** "Not now" ({later: pairs kept now}) or "Don't suggest this" ({never}) for one trait. An unknown trait or action is a 400, nothing written. */
138
+ export function setAnswer(trait, action) {
139
+ if (typeof trait !== "string" || !TRAIT_IDS.includes(trait)) throw new VoiceError(`trait must be one of: ${TRAIT_IDS.join(", ")}`);
140
+ if (typeof action !== "string" || !ANSWERS.includes(action)) throw new VoiceError(`action must be one of: ${ANSWERS.join(", ")}`);
141
+ const ev = readEvidence();
142
+ return save({ ...ev, answers: { ...ev.answers, [trait]: action === "never" ? { never: true } : { later: ev.pairs.length } } });
143
+ }
144
+
145
+ /** Forget the kept edits, comment hits and examples. The learning switch stays; "never" answers stay (a choice, not evidence); "later" goes (it counted kept pairs). */
146
+ export function forget() {
147
+ const ev = readEvidence();
148
+ const answers = Object.fromEntries(Object.entries(ev.answers).filter(([, a]) => a.never === true));
149
+ return save({ ...emptyEvidence(), learning: ev.learning, answers });
150
+ }
151
+
152
+ /** The "Learn from my edits" switch. Turning it off keeps what is stored (Forget empties it) and stops new records. */
153
+ export function setLearning(on) {
154
+ if (typeof on !== "boolean") throw new VoiceError("learning must be true or false");
155
+ return save({ ...readEvidence(), learning: on });
156
+ }
157
+
158
+ // Characters the person cannot see in the guide box (tag characters, variation selectors, zero-width spaces, U+206A to U+206F, CGJ, soft
159
+ // hyphen, U+180E, format characters such as U+FFF9): stripped from the guide, which goes to the agent. A lone zero-width non-joiner or
160
+ // joiner (U+200C, U+200D) stays: Persian and several Indic scripts need it to spell a word, and emoji sequences use it; two or more in a
161
+ // row, which spell nothing, go. Private-use and unassigned code points stay (they show as a box, and a newer emoji would otherwise vanish).
162
+ // Cost: an emoji written with U+FE0F can show in its plain text style.
163
+ const GUIDE_HIDDEN = /[\p{Default_Ignorable_Code_Point}\p{Cf}]/gu;
164
+ const JOINERS = /[\u200C\u200D]{2,}/g;
165
+ const guideText = (value) =>
166
+ userText(value, Infinity)
167
+ .replace(GUIDE_HIDDEN, (c) => (c === "\u200C" || c === "\u200D" ? c : ""))
168
+ .replace(JOINERS, "");
169
+
170
+ const NO_GUIDE = () => ({ text: "", on: false });
171
+
172
+ /** What a guide file's text says: {text, on} plus `send: "short"` when chosen and possible. Broken, the wrong shape or over the cap is no guide. */
173
+ function parseGuideFile(raw) {
174
+ let obj = null;
175
+ try {
176
+ obj = raw == null ? null : JSON.parse(raw);
177
+ } catch {}
178
+ if (!plain(obj) || obj.v !== 1 || typeof obj.text !== "string" || typeof obj.on !== "boolean") return NO_GUIDE();
179
+ const text = guideText(obj.text);
180
+ if ([...text].length > GUIDE_MAX) return NO_GUIDE();
181
+ // a short form the text no longer holds (a hand-edited file) sends the full guide, which the person also approved
182
+ return obj.send === "short" && shortForm(text) ? { text, on: obj.on, send: "short" } : { text, on: obj.on };
183
+ }
184
+
185
+ // The guide as a file beside the skill, read the way a .env is: skills/onionskin/voice.md in this app's folder (never in git, never in the
186
+ // zip; voice.example.md beside it is the sample). It is the guide, switched on, while nothing is saved in the state folder; a guide saved
187
+ // from the page is stored there and wins from then on. ONIONSKIN_VOICE_FILE names another file ("" turns this off). A custom state folder
188
+ // (ONIONSKIN_STATE_DIR, as every test uses) reads no file unless ONIONSKIN_VOICE_FILE is set, so an isolated state stays isolated.
189
+ const VOICE_FILE = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "skills", "onionskin", "voice.md");
190
+ function fileGuide() {
191
+ const named = process.env.ONIONSKIN_VOICE_FILE;
192
+ if (named === "" || (named === undefined && process.env.ONIONSKIN_STATE_DIR)) return NO_GUIDE();
193
+ try {
194
+ const file = named ?? VOICE_FILE;
195
+ if (checkRegularOwned(file).size > MAX_FILE_BYTES) return NO_GUIDE();
196
+ const text = guideText(fs.readFileSync(file, "utf8")).trim();
197
+ return text && [...text].length <= GUIDE_MAX ? { text, on: true } : NO_GUIDE();
198
+ } catch {
199
+ return NO_GUIDE(); // no file is the usual case
200
+ }
201
+ }
202
+
203
+ const NOT_OURS = Symbol("not ours"); // a link, not a regular file, someone else's, too big or unreadable: never read, never written through
204
+ /** The guide file's exact text; null when there is none (lstat ENOENT only: a dangling link is there); NOT_OURS for anything else. */
205
+ function rawGuideFile() {
206
+ try {
207
+ if (checkRegularOwned(guideFile()).size > MAX_FILE_BYTES) return NOT_OURS;
208
+ return fs.readFileSync(guideFile(), "utf8");
209
+ } catch (err) {
210
+ return err?.code === "ENOENT" ? null : NOT_OURS;
211
+ }
212
+ }
213
+
214
+ let known = null; // {raw, guide}: the guide file text this server last wrote or first read (null = no file), and what it parses to
215
+ let outside = 0; // outside changes found and put back since this server started
216
+
217
+ /**
218
+ * Re-reads the guide file. The first read is trusted; after that, any other text (a missing file or a link included) is put back from
219
+ * the copy: the copy is written again (writeStateFile renames over a link, never through it), or the file removed when there was none.
220
+ * A first read that found a file that is not ours reads as no guide and leaves that file alone.
221
+ */
222
+ function checkGuide() {
223
+ const raw = rawGuideFile();
224
+ if (!known) known = { raw, guide: raw === NOT_OURS ? NO_GUIDE() : raw === null ? fileGuide() : parseGuideFile(raw) };
225
+ else if (raw !== known.raw) {
226
+ outside++;
227
+ try {
228
+ if (known.raw === null) fs.rmSync(guideFile(), { force: true });
229
+ else if (known.raw !== NOT_OURS) writeStateFile(guideFile(), known.raw);
230
+ } catch {} // the copy is still what counts, even when the file cannot be put back
231
+ }
232
+ return { ...known.guide };
233
+ }
234
+
235
+ /** The saved guide, through the guard: {text, on} and `send: "short"` when the person chose the short form. No guide reads as {text: "", on: false}. */
236
+ export const readGuide = () => checkGuide();
237
+
238
+ /**
239
+ * Checks the guide file now (putting it back if it changed) and returns how many outside changes this server has found and undone since
240
+ * it started. It never resets: a caller keeps the number from when a batch was delivered and compares it at done.
241
+ */
242
+ export function guideChangedOutside() {
243
+ checkGuide();
244
+ return outside;
245
+ }
246
+
247
+ /**
248
+ * Strict: exactly what the box showed, control, bidi and invisible characters stripped. More than 7,000 characters (counted after
249
+ * stripping) is a 400 and nothing is written, never a cut. `send` is "full" (the default) or "short"; "short" needs an "In short" section
250
+ * of at most 700 characters, or it is a 400.
251
+ */
252
+ export function writeGuide({ text, on, send = "full" } = {}) {
253
+ checkGuide(); // an outside change is counted and undone even when this write is refused
254
+ if (typeof text !== "string") throw new VoiceError("the guide must be text");
255
+ if (typeof on !== "boolean") throw new VoiceError("on must be true or false");
256
+ if (typeof send !== "string" || !SENDS.includes(send)) throw new VoiceError(`send must be one of: ${SENDS.join(", ")}`);
257
+ const clean = guideText(text).trim();
258
+ if ([...clean].length > GUIDE_MAX) throw new VoiceError(`the guide can be at most ${GUIDE_MAX} characters`);
259
+ if (send === "short" && !shortForm(clean)) throw new VoiceError('the short form needs an "In short" section of at most 700 characters');
260
+ ensureStateDir();
261
+ const raw = JSON.stringify({ v: 1, text: clean, on, ...(send === "short" ? { send } : {}) }, null, 2) + "\n";
262
+ writeStateFile(guideFile(), raw);
263
+ known = { raw, guide: parseGuideFile(raw) }; // only after the write succeeded
264
+ return { ...known.guide };
265
+ }
266
+
267
+ /** The guide text for `instructions.voice_guide`: "" unless a guide is saved and switched on; only "In short" when the person chose the short form. */
268
+ export const voiceGuide = () => {
269
+ const g = readGuide();
270
+ if (!g.on) return "";
271
+ return g.send === "short" ? shortForm(g.text) : g.text;
272
+ };
273
+
274
+ // ---- the agent's draft ---------------------------------------------------------------------------------------------------------
275
+
276
+ const BATCH_ID = /^b_[0-9a-f]{12}$/;
277
+
278
+ /**
279
+ * The draft waiting for review: {at, batch_id, text, sections}, or null. Re-checked on every read with cleanVoiceDraft: a file that is
280
+ * missing, not ours, the wrong shape, or holds a draft that would be refused now reads as no draft.
281
+ */
282
+ export function readDraft() {
283
+ const raw = readObject(draftFile());
284
+ if (!plain(raw) || !at(raw.at) || typeof raw.batch_id !== "string" || !BATCH_ID.test(raw.batch_id) || typeof raw.text !== "string") return null;
285
+ const d = cleanVoiceDraft(raw.text);
286
+ return d.code ? null : { at: raw.at, batch_id: raw.batch_id, text: d.text, sections: d.sections };
287
+ }
288
+
289
+ /**
290
+ * Stores an agent's draft in place of any earlier one. Strict: a batch id that is not one, more than 32 KB, or a draft that
291
+ * cleanVoiceDraft refuses throws a VoiceError carrying `code` and writes nothing. Returns what readDraft will return.
292
+ */
293
+ export function writeDraft({ batch_id, text } = {}, now = Date.now()) {
294
+ if (typeof batch_id !== "string" || !BATCH_ID.test(batch_id)) throw new VoiceError("batch_id must be a batch id");
295
+ const d = cleanVoiceDraft(text); // not text is bad_structure, over 32 KB too_long, before anything else runs
296
+ if (d.code) throw Object.assign(new VoiceError(d.reason), { code: d.code });
297
+ ensureStateDir();
298
+ writeStateFile(draftFile(), JSON.stringify({ v: 1, at: now, batch_id, text: d.text }, null, 2) + "\n");
299
+ return { at: now, batch_id, text: d.text, sections: d.sections };
300
+ }
301
+
302
+ /** "Discard draft": removes the draft file (a link is removed as a link, its target untouched). Nothing there is fine. */
303
+ export function discardDraft() {
304
+ fs.rmSync(draftFile(), { force: true });
305
+ }
@@ -0,0 +1,28 @@
1
+ import fs from "node:fs";
2
+ import { parentPort, workerData } from "node:worker_threads";
3
+ import { readDocxParagraphs } from "./docx.js";
4
+ import { documentRuns, editPairs, exampleOf, pairSignals } from "./voice.js";
5
+
6
+ /**
7
+ * The learning step for one Send: three DOCX versions linked through the agent's output (pre, post, mine) in, the per-pair signals and the
8
+ * export examples out. Pure over bytes. {pairs: [{short, sentences, exclaim, restored, fmt}], examples: [{agent, mine}]}; nothing else
9
+ * from the document leaves it (the examples are untrusted document-derived text, already sanitized and capped by exampleOf).
10
+ */
11
+ export function analyzeVoice(preBytes, postBytes, mineBytes) {
12
+ const pre = readDocxParagraphs(preBytes);
13
+ const post = readDocxParagraphs(postBytes);
14
+ const found = editPairs(pre, post, readDocxParagraphs(mineBytes));
15
+ const runs = documentRuns([...pre, ...post]); // an example's "mine" side may not be text either version already holds
16
+ return { pairs: found.map(pairSignals), examples: found.map((p) => exampleOf(p, runs)).filter(Boolean) };
17
+ }
18
+
19
+ // Run in a worker thread (same pattern as history-worker.js): workerData = {pre, post, mine}, three snapshot blob paths. One message back,
20
+ // {ok: true, pairs, examples} or {ok: false, error}. A version that cannot be read or parsed fails the whole job and learns nothing.
21
+ if (parentPort) {
22
+ try {
23
+ const { pre, post, mine } = workerData;
24
+ parentPort.postMessage({ ok: true, ...analyzeVoice(fs.readFileSync(pre), fs.readFileSync(post), fs.readFileSync(mine)) });
25
+ } catch (err) {
26
+ parentPort.postMessage({ ok: false, error: String(err?.message || err) });
27
+ }
28
+ }