onionskin-cli 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -0
- package/INSTALL.md +17 -0
- package/LICENSE +21 -0
- package/NOTICE +46 -0
- package/README.md +88 -2
- package/docs/contracts/batch.schema.json +839 -0
- package/docs/contracts/history.schema.json +126 -0
- package/docs/contracts/ops.schema.json +541 -0
- package/package.json +54 -4
- package/scripts/checker/check.py +411 -0
- package/scripts/checker/requirements.txt +5 -0
- package/scripts/install-check.mjs +9 -0
- package/scripts/requirements.txt +5 -0
- package/scripts/rewriter/apple/main.swift +135 -0
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +6 -0
- package/skills/onionskin/SKILL.md +117 -0
- package/skills/onionskin/reference/batch.md +94 -0
- package/skills/onionskin/reference/formats.md +15 -0
- package/skills/onionskin/reference/ops.md +253 -0
- package/skills/onionskin/reference/troubleshooting.md +73 -0
- package/skills/onionskin/reference/voice-guide.md +86 -0
- package/skills/onionskin/voice.example.md +51 -0
- package/src/agent-detect.js +149 -0
- package/src/agent-events.js +159 -0
- package/src/agent-prompt.js +27 -0
- package/src/agent-runner.js +593 -0
- package/src/analyze.js +17 -0
- package/src/anchors.js +190 -0
- package/src/apply.js +387 -0
- package/src/auth.js +130 -0
- package/src/batch.js +322 -0
- package/src/check-worker.js +8 -0
- package/src/checker.js +293 -0
- package/src/cli.js +516 -0
- package/src/constants.js +20 -0
- package/src/diff.js +267 -0
- package/src/doctor.js +247 -0
- package/src/docx.js +438 -0
- package/src/fmt.js +30 -0
- package/src/fsutil.js +348 -0
- package/src/history-worker.js +16 -0
- package/src/history.js +576 -0
- package/src/lease.js +79 -0
- package/src/local.js +242 -0
- package/src/merge-pptx.js +198 -0
- package/src/merge-xlsx.js +453 -0
- package/src/merge.js +410 -0
- package/src/offload.js +34 -0
- package/src/ops/common.js +48 -0
- package/src/ops/docx-table.js +323 -0
- package/src/ops/docx.js +503 -0
- package/src/ops/engine.js +41 -0
- package/src/ops/pptx.js +336 -0
- package/src/ops/run.js +115 -0
- package/src/ops/runs.js +127 -0
- package/src/ops/schema.js +109 -0
- package/src/ops/xlsx.js +277 -0
- package/src/package.js +189 -0
- package/src/paths.js +64 -0
- package/src/person.js +66 -0
- package/src/pptx.js +194 -0
- package/src/question.js +47 -0
- package/src/quick-plan.js +107 -0
- package/src/quick.js +164 -0
- package/src/redline-check.js +251 -0
- package/src/redline-worker.js +29 -0
- package/src/redline.js +438 -0
- package/src/reply.js +410 -0
- package/src/review.js +274 -0
- package/src/rewrite-check.js +1014 -0
- package/src/rewriter-cleanup.js +105 -0
- package/src/rewriter-setup.js +104 -0
- package/src/rewriter.js +396 -0
- package/src/sanitize.js +42 -0
- package/src/sentence-span.js +25 -0
- package/src/server-entry.js +13 -0
- package/src/server.js +576 -0
- package/src/session.js +1910 -0
- package/src/settings.js +161 -0
- package/src/setup.js +160 -0
- package/src/stamp-words.js +10 -0
- package/src/stamps.js +59 -0
- package/src/style.js +110 -0
- package/src/suggest-reply.js +84 -0
- package/src/suggest.js +261 -0
- package/src/text.js +31 -0
- package/src/tracked-words.js +43 -0
- package/src/voice-guide.js +182 -0
- package/src/voice-learn.js +314 -0
- package/src/voice-sample-worker.js +95 -0
- package/src/voice-samples.js +329 -0
- package/src/voice-store.js +305 -0
- package/src/voice-worker.js +28 -0
- package/src/voice.js +297 -0
- package/src/xlsx.js +454 -0
- package/src/xml-tree.js +118 -0
- package/src/xml.js +127 -0
- package/src/zip-write.js +74 -0
- package/src/zip.js +88 -0
- package/vendor/engine/DocViewer-BZLXUcxJ.js +15137 -0
- package/vendor/engine/THIRD_PARTY_LICENSES.txt +79 -0
- package/vendor/engine/collab-BVynXMqc.js +2591 -0
- package/vendor/engine/collab.js +2 -0
- package/vendor/engine/core.js +4 -0
- package/vendor/engine/default-worker-DogNT09A.js +35625 -0
- package/vendor/engine/embed.js +107 -0
- package/vendor/engine/index.js +5 -0
- package/vendor/engine/office-engine.embed.js +184 -0
- package/vendor/engine/office-engine.worker.js +36030 -0
- package/vendor/engine/xlsx-ops-SYAv6V0f.js +939 -0
- package/vendor/engine-node/LICENSE +21 -0
- package/vendor/engine-node/engine-node.mjs +46618 -0
- package/vendor/engine.lock.json +26 -0
- package/web/dist/assets/index-CFISATkn.js +197 -0
- package/web/dist/assets/index-spXE4oTz.css +1 -0
- package/web/dist/index.html +14 -0
package/src/merge.js
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
import { fmtNote } from "./fmt.js";
|
|
2
|
+
import { docxFmt, lcsPairs, matchParagraphs } from "./history.js";
|
|
3
|
+
import { readDocxParagraphs } from "./docx.js";
|
|
4
|
+
import { normText } from "./text.js";
|
|
5
|
+
|
|
6
|
+
// What the user did while the agent worked, carried onto the agent's version. Three versions of one DOCX: `base` (what the page
|
|
7
|
+
// loaded: the file as sent), `mine` (base plus the user's edits, saved by the page) and `theirs` (the file on disk after the agent).
|
|
8
|
+
// A paragraph only the user changed becomes an op on `theirs`, run through the same engine, expect check and package check as an
|
|
9
|
+
// agent's `apply`. A paragraph both sides changed is never guessed at: the agent's text stays and the user's text is handed back
|
|
10
|
+
// (`unplaced`), so nothing typed is lost without a word.
|
|
11
|
+
// Formatting the user changed without changing the text (run properties by character, and the paragraph's style, alignment and
|
|
12
|
+
// list) is set on the agent's paragraph when the agent left that formatting alone; on a paragraph the agent rewrote only the
|
|
13
|
+
// paragraph's own style, alignment and list are set. A body paragraph the user moved is moved in the agent's version when the
|
|
14
|
+
// agent left it alone and the paragraph next to its new place is still there. Each of these is proven on the result before it is
|
|
15
|
+
// kept: if one did not come out as planned, the merge runs again without any of them and says so.
|
|
16
|
+
// Ceilings: pictures and anything outside paragraphs are not carried. Not carried and reported: formatting of a kind the engine
|
|
17
|
+
// cannot set (character style, theme color, shading, spacing, indent), moves inside tables, a moved paragraph that ends a section or
|
|
18
|
+
// holds one end of a comment or bookmark, and formatting changed on a paragraph the user also moved (the agent's paragraph is moved
|
|
19
|
+
// as it is; the report names what the user changed inside it when that is certain, and asks to check the formatting whenever the user's copy
|
|
20
|
+
// reads differently, which every cut and paste in the editor does). Not carried and not reported: run formatting changed in a
|
|
21
|
+
// paragraph whose text the user also changed, the formatting of a paragraph the user added (it takes its neighbor's), and a move
|
|
22
|
+
// of a paragraph the user also edited (its text is carried in the old place). A paragraph the user split keeps its run formatting
|
|
23
|
+
// only in the part before the first changed character.
|
|
24
|
+
|
|
25
|
+
const CAP = 2000;
|
|
26
|
+
const cut = (t) => String(t ?? "").slice(0, CAP);
|
|
27
|
+
const label = (p) => {
|
|
28
|
+
const t = normText(p.text);
|
|
29
|
+
return t ? `"${t.split(" ").slice(0, 8).join(" ")}${t.split(" ").length > 8 ? "..." : ""}"` : "an empty paragraph";
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The smallest `find` / `replace` that turns `before` into `after` and occurs exactly once in `before` (so run formatting around it
|
|
34
|
+
* is kept), or null when there is none within the op's limits (then the whole paragraph text is set).
|
|
35
|
+
*/
|
|
36
|
+
export function minimalReplace(before, after) {
|
|
37
|
+
let p = 0;
|
|
38
|
+
const max = Math.min(before.length, after.length);
|
|
39
|
+
while (p < max && before[p] === after[p]) p++;
|
|
40
|
+
let s = 0;
|
|
41
|
+
while (s < max - p && before[before.length - 1 - s] === after[after.length - 1 - s]) s++;
|
|
42
|
+
let a = p;
|
|
43
|
+
let b = before.length - s; // [a, b) of `before` is replaced
|
|
44
|
+
const count = (needle) => {
|
|
45
|
+
let n = 0;
|
|
46
|
+
for (let i = before.indexOf(needle); i >= 0; i = before.indexOf(needle, i + 1)) n++;
|
|
47
|
+
return n;
|
|
48
|
+
};
|
|
49
|
+
// widen (left first, then right) until the find text is not empty and occurs once
|
|
50
|
+
while ((b === a || count(before.slice(a, b)) !== 1) && (a > 0 || b < before.length)) {
|
|
51
|
+
if (a > 0) a--;
|
|
52
|
+
else b++;
|
|
53
|
+
if (b - a > CAP) return null;
|
|
54
|
+
}
|
|
55
|
+
if (b === a) return null; // an empty paragraph has nothing to find
|
|
56
|
+
const find = before.slice(a, b);
|
|
57
|
+
const replace = before.slice(a, p) + after.slice(p, after.length - s) + before.slice(before.length - s, b);
|
|
58
|
+
return find.length <= CAP && replace.length <= 20000 ? { find, replace } : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const RUN_WORDS = { b: "bold", i: "italic", u: "underline", strike: "strikethrough", color: "color", sz: "size", font: "font", hl: "highlight", va: "superscript or subscript" };
|
|
62
|
+
const ALIGN = { "": "", left: "left", center: "center", right: "right", both: "justify" }; // w:jc as written -> what the engine sets
|
|
63
|
+
const list = (words) => (words.length > 1 ? `${words.slice(0, -1).join(", ")} and ${words[words.length - 1]}` : words[0] || "");
|
|
64
|
+
const listSig = (p) => (p.list ? `${p.list.kind === "other" ? `other ${p.list.numId}` : p.list.kind} ${p.list.ilvl}` : "");
|
|
65
|
+
const sameList = (a, b) => (a.list?.numId || "") === (b.list?.numId || "") && (a.list?.ilvl || 0) === (b.list?.ilvl || 0);
|
|
66
|
+
const styleKey = (v) => normText(v || "normal").toLowerCase();
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Run formatting of `from` -> `to`, two readings of the same text (`spans` of readDocxParagraphs with detail), by character:
|
|
70
|
+
* {ranges: [{start, end, set}], other}. `set` holds the properties that differ, with the value in `to` (null = not set there): the
|
|
71
|
+
* patch the engine applies. `other` says a property that can only be compared differs somewhere; such a stretch gets no patch.
|
|
72
|
+
* Ranges are maximal, so equal differences give equal results. null when a side is unreadable or the lengths differ.
|
|
73
|
+
*/
|
|
74
|
+
export function runPatches(from, to) {
|
|
75
|
+
const total = (spans) => spans.reduce((n, x) => n + x.n, 0);
|
|
76
|
+
if (!from || !to || total(from) !== total(to)) return null;
|
|
77
|
+
const ranges = [];
|
|
78
|
+
let other = false;
|
|
79
|
+
let x = 0;
|
|
80
|
+
let y = 0;
|
|
81
|
+
let xe = from[0]?.n ?? 0;
|
|
82
|
+
let ye = to[0]?.n ?? 0;
|
|
83
|
+
for (let at = 0, n = total(from); at < n; ) {
|
|
84
|
+
const end = Math.min(xe, ye);
|
|
85
|
+
if (from[x].rest !== to[y].rest) other = true;
|
|
86
|
+
else {
|
|
87
|
+
const set = {};
|
|
88
|
+
for (const k of Object.keys(RUN_WORDS)) if (from[x].set[k] !== to[y].set[k]) set[k] = to[y].set[k] ?? null;
|
|
89
|
+
const key = JSON.stringify(set);
|
|
90
|
+
const last = ranges[ranges.length - 1];
|
|
91
|
+
if (last && last.end === at && last.key === key) last.end = end;
|
|
92
|
+
else if (key !== "{}") ranges.push({ start: at, end, set, key });
|
|
93
|
+
}
|
|
94
|
+
at = end;
|
|
95
|
+
if (at === xe && ++x < from.length) xe += from[x].n;
|
|
96
|
+
if (at === ye && ++y < to.length) ye += to[y].n;
|
|
97
|
+
}
|
|
98
|
+
return { ranges: ranges.map(({ key, ...r }) => r), other };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Run properties whose value changes inside `to` at a place where `from` has one look on both sides: a difference between
|
|
103
|
+
* stretches of one paragraph that nothing but a change made there explains (a paste writes the look out on every run, but
|
|
104
|
+
* evenly). `from` and `to` are two readings of the same text; [] when one is unreadable.
|
|
105
|
+
*/
|
|
106
|
+
function newBreaks(from, to) {
|
|
107
|
+
const keys = new Set();
|
|
108
|
+
const total = (spans) => spans.reduce((n, x) => n + x.n, 0);
|
|
109
|
+
if (!from || !to || total(from) !== total(to)) return [];
|
|
110
|
+
let x = 0;
|
|
111
|
+
let xe = from[0]?.n ?? 0;
|
|
112
|
+
let at = 0;
|
|
113
|
+
for (let y = 0; y < to.length - 1; y++) {
|
|
114
|
+
at += to[y].n; // the place where to[y] ends and to[y + 1] starts
|
|
115
|
+
while (x < from.length - 1 && xe < at) xe += from[++x].n;
|
|
116
|
+
const next = xe === at && x < from.length - 1 ? from[x + 1] : from[x];
|
|
117
|
+
if (from[x].rest !== next.rest) continue;
|
|
118
|
+
for (const k of Object.keys(RUN_WORDS)) if (to[y].set[k] !== to[y + 1].set[k] && from[x].set[k] === next.set[k]) keys.add(k);
|
|
119
|
+
}
|
|
120
|
+
return Object.keys(RUN_WORDS).filter((k) => keys.has(k));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** The paragraph's own formatting, `from` -> `to`: {set} is what the engine can set (style, align, list, ilvl), {no} names what it cannot. */
|
|
124
|
+
function paraPatch(from, to) {
|
|
125
|
+
const set = {};
|
|
126
|
+
const no = [];
|
|
127
|
+
if ((from.style || "") !== (to.style || "")) set.style = to.style ? to.styleName || to.style : null; // null: back to the default style
|
|
128
|
+
if (from.align !== to.align) to.align in ALIGN ? (set.align = ALIGN[to.align]) : no.push("this kind of alignment");
|
|
129
|
+
if (listSig(from) !== listSig(to)) {
|
|
130
|
+
if (!to.list) set.list = "none";
|
|
131
|
+
else if (to.list.kind === "other") no.push("this kind of list");
|
|
132
|
+
else if (from.list?.kind === to.list.kind) set.ilvl = to.list.ilvl;
|
|
133
|
+
else (set.list = to.list.kind), (set.ilvl = to.list.ilvl);
|
|
134
|
+
}
|
|
135
|
+
if (from.ppr !== to.ppr) no.push("spacing, indent and other paragraph settings");
|
|
136
|
+
return { set, no };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Plans the merge. Arguments are paragraph lists (readDocxParagraphs with detail). Returns
|
|
141
|
+
* {items: [{kind, label, text, t, op, ...}], unplaced: [{label, text, why}]}: every item is tried on `theirs`. `t` is the index in
|
|
142
|
+
* `theirs` of the paragraph the op names (for a new paragraph: its anchor); a moved paragraph also has `to`, its new neighbor, and
|
|
143
|
+
* a `note` (with `check`, see `lost`) when its formatting in `mine` is not the original's.
|
|
144
|
+
*/
|
|
145
|
+
export function planDocxMerge(base, mine, theirs) {
|
|
146
|
+
const { mapA: baseToMine, mapB: mineToBase, moved } = matchParagraphs(base, mine);
|
|
147
|
+
// a base paragraph the agent left alone, by position: the same text in the same order
|
|
148
|
+
const { mapA: baseToTheirs } = lcsPairs(base.map((p) => p.hash), theirs.map((p) => p.hash));
|
|
149
|
+
const agent = matchParagraphs(base, theirs); // where the agent's other paragraphs came from: an edited one, one it moved
|
|
150
|
+
const at = (i) => (theirs[baseToTheirs[i]]?.path ? baseToTheirs[i] : -1);
|
|
151
|
+
const locator = (n) => ({ body_path: theirs[n].path, text_hash: theirs[n].hash });
|
|
152
|
+
const theirsHas = new Set(theirs.map((p) => p.hash));
|
|
153
|
+
const items = [];
|
|
154
|
+
const unplaced = [];
|
|
155
|
+
const no = (p, why, text = p.text) => unplaced.push({ label: label(p), text: cut(text), why });
|
|
156
|
+
|
|
157
|
+
// The agent's version of a base paragraph whose text it changed, only when there is no doubt which one it is: the same unique
|
|
158
|
+
// paragraph id, or the one paragraph between the same two untouched neighbors.
|
|
159
|
+
const ids = (L) => L.reduce((m, p) => (p.paraId ? m.set(p.paraId, (m.get(p.paraId) || 0) + 1) : m), new Map());
|
|
160
|
+
let idCount = null;
|
|
161
|
+
const rewritten = (i) => {
|
|
162
|
+
const n = agent.mapA[i];
|
|
163
|
+
if (n < 0 || baseToTheirs[i] >= 0 || agent.moved.has(n) || !theirs[n].path) return -1;
|
|
164
|
+
idCount ||= [ids(base), ids(theirs)];
|
|
165
|
+
const id = base[i].paraId;
|
|
166
|
+
if (id && theirs[n].paraId === id && idCount[0].get(id) === 1 && idCount[1].get(id) === 1) return n;
|
|
167
|
+
const before = i === 0 ? n === 0 : baseToTheirs[i - 1] >= 0 && baseToTheirs[i - 1] === n - 1;
|
|
168
|
+
const after = i === base.length - 1 ? n === theirs.length - 1 : baseToTheirs[i + 1] >= 0 && baseToTheirs[i + 1] === n + 1;
|
|
169
|
+
return before && after ? n : -1;
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Formatting the user changed on a paragraph (`b` in base, `m` in mine). `typed`: the user changed its text too, so only the
|
|
174
|
+
* paragraph's own formatting can be told apart from the typing.
|
|
175
|
+
*/
|
|
176
|
+
const format = (b, m, i, typed) => {
|
|
177
|
+
const para = paraPatch(b, m);
|
|
178
|
+
const exact = !typed && b.text === m.text ? runPatches(b.spans, m.spans) : null;
|
|
179
|
+
const words = exact ? [...new Set(exact.ranges.flatMap((r) => Object.keys(r.set)))].map((k) => RUN_WORDS[k]) : [];
|
|
180
|
+
// the texts differ in white space only, or a run could not be read: the summary History uses still says that something changed
|
|
181
|
+
const vague = !typed && !exact ? fmtNote({ props: b.fmt }, { props: m.fmt }) : null;
|
|
182
|
+
const cannot = [...para.no, ...(exact?.other ? ["character styles and text effects"] : [])];
|
|
183
|
+
if (!Object.keys(para.set).length && !words.length && !vague && !cannot.length) return;
|
|
184
|
+
const kept = at(i);
|
|
185
|
+
const n = kept >= 0 ? kept : typed ? -1 : rewritten(i);
|
|
186
|
+
const mineWords = [...("style" in para.set ? ["style"] : []), ...("align" in para.set ? ["alignment"] : []), ...("list" in para.set || "ilvl" in para.set ? ["list"] : []), ...words];
|
|
187
|
+
if (n < 0) {
|
|
188
|
+
if (typed) return; // the text was handed back with its own reason
|
|
189
|
+
return no(m, `You changed formatting here${mineWords.length ? ` (${list(mineWords)})` : ""}, but the agent changed, moved or removed this paragraph, so it was not carried over.`, "");
|
|
190
|
+
}
|
|
191
|
+
const a = theirs[n];
|
|
192
|
+
const carry = {};
|
|
193
|
+
const done = [];
|
|
194
|
+
const clash = [];
|
|
195
|
+
const notes = [];
|
|
196
|
+
if ("style" in para.set) (a.style || "") === (b.style || "") ? ((carry.style = para.set.style), done.push("style")) : clash.push("style");
|
|
197
|
+
if ("align" in para.set) a.align === b.align ? ((carry.align = para.set.align), done.push("alignment")) : clash.push("alignment");
|
|
198
|
+
if ("list" in para.set || "ilvl" in para.set) {
|
|
199
|
+
if (sameList(a, b)) (Object.assign(carry, "list" in para.set ? { list: para.set.list } : {}, "ilvl" in para.set ? { ilvl: para.set.ilvl } : {}), done.push("list"));
|
|
200
|
+
else clash.push("list");
|
|
201
|
+
}
|
|
202
|
+
if (clash.length) notes.push(`The agent changed the ${list(clash)} here too, so yours was not carried over.`);
|
|
203
|
+
let runs = [];
|
|
204
|
+
if (words.length) {
|
|
205
|
+
const same = kept >= 0 && a.text === b.text ? runPatches(b.spans, a.spans) : null;
|
|
206
|
+
if (kept < 0) notes.push(`The agent rewrote this paragraph, so the ${list(words)} you changed inside it was not carried over.`);
|
|
207
|
+
else if (same && !same.ranges.length && !same.other) ((runs = exact.ranges), done.push(...words));
|
|
208
|
+
else notes.push(`The agent changed formatting inside this paragraph too, so the ${list(words)} you changed was not carried over.`);
|
|
209
|
+
} else if (vague) notes.push(`Formatting inside this paragraph is different in your version (${vague}). It could not be carried over.`);
|
|
210
|
+
if (cannot.length) notes.push(`Changes to ${list(cannot)} are not carried over.`);
|
|
211
|
+
if (!done.length) return no(m, notes.join(" "), "");
|
|
212
|
+
items.push({ kind: "format", label: label(m), text: "", hash: null, t: n, words: done, note: notes.join(" "), op: { op: "format_paragraph", target: locator(n), expect: { text_hash: a.hash }, ...(Object.keys(carry).length ? { para: carry } : {}), ...(runs.length ? { runs } : {}) } });
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* What to say about the formatting of a paragraph the user moved (`b` in base, `m` in mine) when the move is carried: {note,
|
|
217
|
+
* check}, or {} when the copy reads as the original does. Certain (named): a look that differs between stretches of the copy
|
|
218
|
+
* where the original has one. Anything else that differs may be the paste or may be the user's (a heading picked, a list, or the
|
|
219
|
+
* whole paragraph made bold): `check` marks the line as a request to look, not a loss. The named line says the same about the
|
|
220
|
+
* rest of the paragraph's formatting, since a named word can hide a heading picked before the cut.
|
|
221
|
+
*/
|
|
222
|
+
const lost = (b, m) => {
|
|
223
|
+
const para = paraPatch(b, m);
|
|
224
|
+
const exact = b.text === m.text ? runPatches(b.spans, m.spans) : null;
|
|
225
|
+
const sure = newBreaks(b.spans, m.spans).map((k) => RUN_WORDS[k]);
|
|
226
|
+
if (sure.length) return { note: `The ${list(sure)} you changed inside this paragraph was not: the paragraph has the formatting of the agent's version.` };
|
|
227
|
+
if (!Object.keys(para.set).length && !para.no.length && exact && !exact.ranges.length && !exact.other) return {};
|
|
228
|
+
return { note: "This paragraph has the formatting of the agent's version: if you changed its formatting too, set it again.", check: true };
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
// anchors for new paragraphs: the nearest paragraph before (else after) that the agent left alone. The user may have edited it
|
|
232
|
+
// (a split leaves the first half as an edit of the original): every op is bound to the agent's file before any op runs.
|
|
233
|
+
const still = (j) => mineToBase[j] >= 0 && !moved.has(j); // a paragraph of the base that is where it was
|
|
234
|
+
const anchorOf = (j) => (still(j) ? at(mineToBase[j]) : -1);
|
|
235
|
+
const inserts = []; // {where: "after"|"before", to: index in theirs, m, move?: index in theirs of the paragraph to move}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* A paragraph the user moved: the agent's paragraph is moved as it is, placed by the paragraph right before it in the user's
|
|
239
|
+
* version (else the one right after), which the agent must have left alone. The formatting of the user's copy is never set: a
|
|
240
|
+
* paragraph that was cut and pasted comes back without its style, alignment and list and with its look written out on every run
|
|
241
|
+
* (seen in the real editor), so setting it would take the heading or the bullet off the agent's paragraph. It is read for the
|
|
242
|
+
* report only (`lost`).
|
|
243
|
+
*/
|
|
244
|
+
const move = (m, j, i) => {
|
|
245
|
+
const n = baseToTheirs[i];
|
|
246
|
+
const more = lost(base[i], m);
|
|
247
|
+
const t = theirs[n];
|
|
248
|
+
const say = (why) => void no(m, why, "");
|
|
249
|
+
if (!t) return say(agent.moved.has(agent.mapA[i]) ? "You moved this paragraph and so did the agent. It is where the agent put it." : "You moved this paragraph, but the agent changed or removed it, so the move was not carried over.");
|
|
250
|
+
if (t.path?.length !== 1 || m.path?.length !== 1) return say("You moved this paragraph in or out of a table. Moves that involve tables are not carried over.");
|
|
251
|
+
if (t.sect) return say("You moved this paragraph, but it ends a section of the document, so the move was not carried over.");
|
|
252
|
+
if (t.loose) return say("You moved this paragraph, but a comment or bookmark starts or ends inside it, so the move was not carried over.");
|
|
253
|
+
const body = (k) => (k >= 0 && k < mine.length && anchorOf(k) >= 0 && theirs[anchorOf(k)].path.length === 1 ? anchorOf(k) : -1);
|
|
254
|
+
let k = j - 1;
|
|
255
|
+
while (k >= 0 && !still(k)) k--;
|
|
256
|
+
if (body(k) >= 0) return void inserts.push({ where: "after", to: body(k), m, move: n, ...more });
|
|
257
|
+
k = j + 1;
|
|
258
|
+
while (k < mine.length && !still(k)) k++;
|
|
259
|
+
if (body(k) >= 0) return void inserts.push({ where: "before", to: body(k), m, move: n, ...more });
|
|
260
|
+
say("You moved this paragraph, but the agent changed the paragraphs next to its new place, so the move was not carried over.");
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
mine.forEach((m, j) => {
|
|
264
|
+
const i = mineToBase[j];
|
|
265
|
+
if (i >= 0) {
|
|
266
|
+
const b = base[i];
|
|
267
|
+
if (moved.has(j)) return move(m, j, i); // the same text in another place
|
|
268
|
+
if (b.hash === m.hash) return normText(m.text) ? format(b, m, i, false) : undefined;
|
|
269
|
+
const t = at(i);
|
|
270
|
+
if (t < 0 && theirsHas.has(m.hash)) return; // the same text is already there (an earlier pass carried it)
|
|
271
|
+
if (t < 0) return no(m, "The agent changed this paragraph too. Its version is in the document; this is yours.");
|
|
272
|
+
const r = minimalReplace(b.text, m.text);
|
|
273
|
+
const op = { op: "replace_text", target: locator(t), expect: { text_hash: b.hash }, ...(r ? r : { text: m.text }) };
|
|
274
|
+
items.push({ kind: "changed", label: label(m), text: cut(m.text), hash: m.hash, t, op });
|
|
275
|
+
return format(b, m, i, true);
|
|
276
|
+
}
|
|
277
|
+
if (!normText(m.text)) return; // an empty line that was added: nothing to lose
|
|
278
|
+
let k = j - 1;
|
|
279
|
+
while (k >= 0 && anchorOf(k) < 0) k--;
|
|
280
|
+
if (k >= 0) return inserts.push({ where: "after", to: anchorOf(k), m });
|
|
281
|
+
k = j + 1;
|
|
282
|
+
while (k < mine.length && anchorOf(k) < 0) k++;
|
|
283
|
+
if (k < mine.length) return inserts.push({ where: "before", to: anchorOf(k), m });
|
|
284
|
+
no(m, "There was no unchanged paragraph near this new one to place it by.");
|
|
285
|
+
});
|
|
286
|
+
// several new or moved paragraphs after one anchor: each lands right after the anchor, so the last goes first
|
|
287
|
+
for (const x of inserts.filter((x) => x.where === "after").reverse().concat(inserts.filter((x) => x.where === "before"))) {
|
|
288
|
+
const to = locator(x.to);
|
|
289
|
+
if (x.move === undefined) items.push({ kind: "added", label: label(x.m), text: cut(x.m.text), hash: x.m.hash, t: x.to, where: x.where, op: { op: x.where === "after" ? "insert_paragraph_after" : "insert_paragraph_before", target: to, expect: { text_hash: to.text_hash }, text: x.m.text } });
|
|
290
|
+
else items.push({ kind: "moved", label: label(x.m), text: "", hash: x.m.hash, t: x.move, to: x.to, where: x.where, ...(x.note ? { note: x.note, check: x.check } : {}), op: { op: "move_paragraph", target: locator(x.move), expect: { text_hash: x.m.hash }, to, where: x.where } });
|
|
291
|
+
}
|
|
292
|
+
base.forEach((b, i) => {
|
|
293
|
+
if (baseToMine[i] >= 0 || !normText(b.text)) return;
|
|
294
|
+
const t = at(i);
|
|
295
|
+
if (t >= 0) items.push({ kind: "removed", label: label(b), text: "", hash: null, t, op: { op: "delete_paragraph", target: locator(t), expect: { text_hash: b.hash } } });
|
|
296
|
+
else no(b, "You deleted this paragraph, but the agent changed it, so it was kept.", "");
|
|
297
|
+
});
|
|
298
|
+
// replacements first, then formatting, then new and moved paragraphs, then deletions (an anchor is never a paragraph this plan deletes)
|
|
299
|
+
const order = { changed: 0, format: 1, added: 2, moved: 2, removed: 3 };
|
|
300
|
+
items.sort((x, y) => order[x.kind] - order[y.kind]);
|
|
301
|
+
return { items, unplaced };
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const SOFT = new Set(["format", "moved"]); // carried only when the result proves them; the merge can always do without
|
|
305
|
+
const PROTECTED = "a protected paragraph (it holds a field, a content control or something else that cannot be rewritten safely)";
|
|
306
|
+
/** Why one item is not in the document, in words for the user. `r` is the op's result when it ran. */
|
|
307
|
+
function reason(it, r) {
|
|
308
|
+
const prot = r?.status === "skipped_protected";
|
|
309
|
+
if (it.kind === "moved") return prot ? `You moved this paragraph, but it or the paragraph next to its new place is ${PROTECTED}, so the move was not carried over.` : "You moved this paragraph, but the move could not be made on the agent's version.";
|
|
310
|
+
if (it.kind === "format") return `${prot ? `This is ${PROTECTED}, so the ${list(it.words)} you changed here was not carried over.` : `The ${list(it.words)} you changed here could not be set on the agent's version.`}${it.note ? ` ${it.note}` : ""}`;
|
|
311
|
+
return "This edit could not be placed on the agent's version.";
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** The order of paragraphs (as text hashes, with the index in `theirs` each came from) the applied ops must give. */
|
|
315
|
+
function expected(theirs, items, results) {
|
|
316
|
+
const seq = theirs.map((p, n) => ({ n, hash: p.hash }));
|
|
317
|
+
const find = (n) => seq.findIndex((x) => x.n === n);
|
|
318
|
+
items.forEach((it, k) => {
|
|
319
|
+
if (results[k]?.status !== "applied") return;
|
|
320
|
+
const i = find(it.t);
|
|
321
|
+
if (i < 0) return;
|
|
322
|
+
if (it.kind === "changed") seq[i] = { n: it.t, hash: it.hash };
|
|
323
|
+
else if (it.kind === "removed") seq.splice(i, 1);
|
|
324
|
+
else if (it.kind === "added") seq.splice(it.where === "after" ? i + 1 : i, 0, { n: -1, hash: it.hash });
|
|
325
|
+
else if (it.kind === "moved") {
|
|
326
|
+
const [x] = seq.splice(i, 1);
|
|
327
|
+
const to = find(it.to);
|
|
328
|
+
seq.splice(it.where === "after" ? to + 1 : to, 0, x);
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
return seq;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Is the paragraph `o` of the result the agent's paragraph `t` with exactly the formatting this item set, and nothing else different? */
|
|
335
|
+
function formatted(it, t, o) {
|
|
336
|
+
const para = it.op.para || {};
|
|
337
|
+
if ("style" in para ? styleKey(o.styleName || o.style) !== styleKey(para.style) : (o.style || "") !== (t.style || "")) return false;
|
|
338
|
+
if ("align" in para ? ALIGN[o.align] !== para.align : o.align !== t.align) return false;
|
|
339
|
+
if ("list" in para ? (para.list === "none" ? !!o.list : o.list?.kind !== para.list || o.list.ilvl !== (para.ilvl ?? 0)) : "ilvl" in para ? o.list?.numId !== t.list?.numId || o.list?.ilvl !== para.ilvl : !sameList(o, t)) return false;
|
|
340
|
+
if (o.ppr !== t.ppr) return false;
|
|
341
|
+
if (o.text !== t.text) return !it.op.runs; // the user's typing went into this paragraph as well: only its own properties were set
|
|
342
|
+
const got = runPatches(t.spans, o.spans);
|
|
343
|
+
return !!got && !got.other && JSON.stringify(got.ranges) === JSON.stringify(it.op.runs || []);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Runs the plan on the agent's bytes. `runOps` is injected (src/ops/run.js) so this file stays free of the engine. Returns
|
|
348
|
+
* {bytes: Buffer|null, merged, unplaced}. Whatever did not apply, or is not found in the result afterwards, is in `unplaced`;
|
|
349
|
+
* so is a line for an edit that was carried only in part. `check: true` on a line: the edit is in the document and the line asks
|
|
350
|
+
* the user to look at something that may or may not be theirs.
|
|
351
|
+
*/
|
|
352
|
+
export async function mergeDocx({ base, mine, theirs, name, runOps }) {
|
|
353
|
+
const read = (bytes) => readDocxParagraphs(bytes, { detail: true });
|
|
354
|
+
const agents = read(theirs);
|
|
355
|
+
const plan = planDocxMerge(read(base), read(mine), agents);
|
|
356
|
+
const unplaced = [...plan.unplaced];
|
|
357
|
+
if (!plan.items.length) return { bytes: null, merged: 0, unplaced };
|
|
358
|
+
let r = await attempt(plan.items, { theirs, agents, name, runOps, read });
|
|
359
|
+
if (r.redo) {
|
|
360
|
+
// a move or a formatting change did not come out as planned: nothing unproven is written, the rest is carried without them
|
|
361
|
+
for (const it of plan.items.filter((x) => SOFT.has(x.kind))) unplaced.push({ label: it.label, text: "", why: reason(it) });
|
|
362
|
+
const rest = plan.items.filter((x) => !SOFT.has(x.kind));
|
|
363
|
+
r = rest.length ? await attempt(rest, { theirs, agents, name, runOps, read }) : { bytes: null, merged: 0, unplaced: [] };
|
|
364
|
+
}
|
|
365
|
+
return { bytes: r.bytes, merged: r.merged, unplaced: [...unplaced, ...r.unplaced] };
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/** One run of `items` on the agent's bytes, checked on the result. {redo: true} asks for a run without moves and formatting. */
|
|
369
|
+
async function attempt(items, { theirs, agents, name, runOps, read }) {
|
|
370
|
+
const unplaced = [];
|
|
371
|
+
const soft = items.some((x) => SOFT.has(x.kind));
|
|
372
|
+
const { results, bytes, blocked } = await runOps({ bytes: theirs, format: "docx", name, ops: items.map((x) => x.op), partial: true });
|
|
373
|
+
const fail = (it, why) => unplaced.push({ label: it.label, text: it.text, why });
|
|
374
|
+
if (!bytes) {
|
|
375
|
+
if (blocked && soft) return { redo: true };
|
|
376
|
+
for (const [n, it] of items.entries()) fail(it, blocked ? "Carrying your edits over would have damaged the file, so none were applied." : reason(it, results[n]));
|
|
377
|
+
return { bytes: null, merged: 0, unplaced };
|
|
378
|
+
}
|
|
379
|
+
// the proof is the result, not the report. First: nothing of the agent's version is gone except what an op named.
|
|
380
|
+
const out = read(bytes);
|
|
381
|
+
const count = (list) => list.reduce((m, h) => m.set(h, (m.get(h) || 0) + 1), new Map());
|
|
382
|
+
const named = count(items.filter((x, n) => (x.kind === "changed" || x.kind === "removed") && results[n]?.status === "applied").map((x) => x.op.expect.text_hash));
|
|
383
|
+
const after = count(out.map((p) => p.hash));
|
|
384
|
+
for (const [h, n] of count(agents.map((p) => p.hash))) {
|
|
385
|
+
if ((after.get(h) || 0) + (named.get(h) || 0) < n) {
|
|
386
|
+
if (soft) return { redo: true };
|
|
387
|
+
for (const it of items) fail(it, "Carrying your edits over would have changed the agent's text, so none were applied.");
|
|
388
|
+
return { bytes: null, merged: 0, unplaced };
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
// then, for moves and formatting: every paragraph is where the applied ops put it, and each formatted one differs from the
|
|
392
|
+
// agent's by exactly what was set
|
|
393
|
+
if (items.some((x, n) => SOFT.has(x.kind) && results[n]?.status === "applied")) {
|
|
394
|
+
const seq = expected(agents, items, results);
|
|
395
|
+
if (seq.length !== out.length || seq.some((x, k) => x.hash !== out[k].hash)) return { redo: true };
|
|
396
|
+
const where = new Map(seq.map((x, k) => [x.n, k]));
|
|
397
|
+
if (items.some((it, n) => it.kind === "format" && results[n]?.status === "applied" && !formatted(it, agents[it.t], out[where.get(it.t)]))) return { redo: true };
|
|
398
|
+
}
|
|
399
|
+
// then: every carried text is in the merged file
|
|
400
|
+
const have = new Set(out.map((p) => p.hash));
|
|
401
|
+
let merged = 0;
|
|
402
|
+
items.forEach((it, n) => {
|
|
403
|
+
if (results[n]?.status !== "applied") return fail(it, reason(it, results[n]));
|
|
404
|
+
if (it.hash && !have.has(it.hash)) return fail(it, "This edit did not land as typed.");
|
|
405
|
+
merged++;
|
|
406
|
+
if (it.note && it.kind === "moved") unplaced.push({ label: it.label, text: "", why: `Your move was carried over. ${it.note}`, ...(it.check ? { check: true } : {}) });
|
|
407
|
+
else if (it.note) fail(it, `Your ${list(it.words)} change was carried over. ${it.note}`);
|
|
408
|
+
});
|
|
409
|
+
return { bytes, merged, unplaced };
|
|
410
|
+
}
|
package/src/offload.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Worker } from "node:worker_threads";
|
|
2
|
+
import { env } from "./constants.js";
|
|
3
|
+
|
|
4
|
+
const WORKER_TIMEOUT_MS = () => Number(env("CHECK_TIMEOUT_MS", 180_000)) + 60_000;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Runs the `done` analysis in a worker thread: parsing a large package and the deeper check (a blocking spawn) must not stop the server from
|
|
8
|
+
* answering status, poll, the browser's events and other sessions. Resolves to {postSha, batchId, flags, checks, changes}; a worker that fails,
|
|
9
|
+
* crashes or exceeds the timeout resolves to a stated failure (never to "clean") and is terminated.
|
|
10
|
+
* `inputs` = {pre, post, format, postSha, batchId, deferSoffice}.
|
|
11
|
+
*/
|
|
12
|
+
export function analyzeOffThread(inputs) {
|
|
13
|
+
const { pre, post, format, postSha, batchId, deferSoffice } = inputs;
|
|
14
|
+
const failed = (why) => ({
|
|
15
|
+
postSha, batchId, changes: null,
|
|
16
|
+
flags: [{ code: "checks_failed", severity: "warn", detail: why }],
|
|
17
|
+
checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The package checks did not finish: ${why}. The file was not verified.` },
|
|
18
|
+
});
|
|
19
|
+
return new Promise((resolve) => {
|
|
20
|
+
let settled = false;
|
|
21
|
+
const worker = new Worker(new URL("./check-worker.js", import.meta.url), { workerData: { pre, post, format, deferSoffice } });
|
|
22
|
+
const finish = (r) => {
|
|
23
|
+
if (settled) return;
|
|
24
|
+
settled = true;
|
|
25
|
+
clearTimeout(timer);
|
|
26
|
+
worker.terminate();
|
|
27
|
+
resolve(r);
|
|
28
|
+
};
|
|
29
|
+
const timer = setTimeout(() => finish(failed(`timed out after ${Math.round(WORKER_TIMEOUT_MS() / 1000)}s`)), WORKER_TIMEOUT_MS());
|
|
30
|
+
worker.once("message", (m) => finish(m.ok ? { postSha, batchId, flags: m.flags, checks: m.checks, changes: m.changes } : failed(m.error)));
|
|
31
|
+
worker.once("error", (err) => finish(failed(String(err?.message || err))));
|
|
32
|
+
worker.once("exit", (code) => finish(failed(`the check worker exited (${code})`)));
|
|
33
|
+
});
|
|
34
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { exactHash, normText, textHash } from "../text.js";
|
|
2
|
+
import { untrusted } from "../sanitize.js";
|
|
3
|
+
|
|
4
|
+
export { normText, textHash };
|
|
5
|
+
|
|
6
|
+
/** Result helpers. Statuses: applied, skipped_protected, refused, anchor_not_found, expect_mismatch (and not_applied, set by the runner). */
|
|
7
|
+
export const fail = (status, reason, extra = {}) => ({ ok: false, status, reason, ...extra });
|
|
8
|
+
export const refused = (code, reason) => fail("refused", reason, { code });
|
|
9
|
+
|
|
10
|
+
/** Code of an engine error: the engine attaches `code` to its refusals; anything else is a plain failure. */
|
|
11
|
+
export const engineFailure = (err) => ({ ...refused(typeof err?.code === "string" ? err.code : "engine_error", String(err?.message || err).split("\n")[0].slice(0, 300)), dirty: true }); // dirty: the engine may be half-way through the edit
|
|
12
|
+
|
|
13
|
+
export const excerpt = (text) => untrusted(text, 200);
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* `expect` against the target's text. mode "contains": quote must occur in the text (DOCX, PPTX). mode "exact": the text must equal it character for character after NFC (XLSX regions).
|
|
17
|
+
* Returns null when it holds, else an expect_mismatch failure carrying a short excerpt of what is really there.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
export function checkExpect(expect, text, mode = "contains") {
|
|
21
|
+
const exact = mode === "exact";
|
|
22
|
+
const hash = exact ? exactHash(text) : textHash(text);
|
|
23
|
+
if ("quote" in expect) {
|
|
24
|
+
const ok = exact ? text.normalize("NFC") === expect.quote.normalize("NFC") : normText(text).includes(normText(expect.quote));
|
|
25
|
+
if (!ok) return fail("expect_mismatch", "the target no longer says what `expect.quote` says", { actual_excerpt: excerpt(text) });
|
|
26
|
+
}
|
|
27
|
+
if ("text_hash" in expect && expect.text_hash !== hash) return fail("expect_mismatch", "the target's text hash differs from `expect.text_hash`", { actual_excerpt: excerpt(text), actual_text_hash: hash });
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export const colLetters = (n) => {
|
|
32
|
+
let s = "";
|
|
33
|
+
for (n += 1; n; n = Math.floor((n - 1) / 26)) s = String.fromCharCode(65 + ((n - 1) % 26)) + s;
|
|
34
|
+
return s;
|
|
35
|
+
};
|
|
36
|
+
export const colIndex = (letters) => [...letters.toUpperCase()].reduce((n, ch) => n * 26 + ch.charCodeAt(0) - 64, 0) - 1;
|
|
37
|
+
|
|
38
|
+
/** "B2" or "B2:D9" -> {r0,c0,r1,c1} (0-based, inclusive), normalized so r0<=r1 and c0<=c1 */
|
|
39
|
+
export function parseA1(a1) {
|
|
40
|
+
const m = /^([A-Za-z]{1,3})([0-9]{1,7})(?::([A-Za-z]{1,3})([0-9]{1,7}))?$/.exec(a1);
|
|
41
|
+
if (!m) return null;
|
|
42
|
+
const r0 = Number(m[2]) - 1;
|
|
43
|
+
const c0 = colIndex(m[1]);
|
|
44
|
+
const r1 = m[4] ? Number(m[4]) - 1 : r0;
|
|
45
|
+
const c1 = m[3] ? colIndex(m[3]) : c0;
|
|
46
|
+
if (r0 < 0 || r1 < 0) return null;
|
|
47
|
+
return { r0: Math.min(r0, r1), c0: Math.min(c0, c1), r1: Math.max(r0, r1), c1: Math.max(c0, c1) };
|
|
48
|
+
}
|