@adeu/core 1.23.0 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +185 -53
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +185 -53
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/diff.ts +23 -8
- package/src/engine.ts +132 -39
- package/src/repro_qa_report_v6.test.ts +486 -0
- package/src/sanitize/core.ts +57 -1
- package/src/sanitize/report.ts +5 -3
- package/src/sanitize/sanitize.test.ts +5 -4
- package/src/sanitize/transforms.ts +94 -16
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Repro tests for the 2026-07-18 black-box QA and UX assessment
|
|
3
|
+
* (adeu 1.23.0+8102b64) — Node parity with
|
|
4
|
+
* python/tests/test_repro_qa_report_v6.py.
|
|
5
|
+
*
|
|
6
|
+
* Finding index:
|
|
7
|
+
* C1 finalize_document reports "Result: CLEAN" while docProps/custom.xml
|
|
8
|
+
* (custom document properties) survives in the SAVED package. The Node
|
|
9
|
+
* engine additionally leaked ALL removed parts (including customXml/*)
|
|
10
|
+
* through pkg.unzipped: save() zips every original member, so parts
|
|
11
|
+
* dropped from pkg.parts still shipped with their original bytes.
|
|
12
|
+
* dc:identifier / dc:language / cp:version were silently retained.
|
|
13
|
+
* H1 a table-row modification followed by an insert_row anchored on the
|
|
14
|
+
* modified row (the shape `generate_structured_edits` emits, replayed
|
|
15
|
+
* without private pins — the MCP process_document_batch shape) either
|
|
16
|
+
* fails or inserts the row at the wrong position: the clean-view
|
|
17
|
+
* fallback resolves a clean-view offset, but the row lookup ran
|
|
18
|
+
* against the raw mapper.
|
|
19
|
+
* H2 a context-wrapped paragraph insertion (target "two.\n\nFinal" →
|
|
20
|
+
* new "two.\n\nAdded.\n\nFinal") is word-diffed after context
|
|
21
|
+
* trimming; dmp cross-matches punctuation between context and inserted
|
|
22
|
+
* text, stranding characters and gluing paragraphs after accept-all.
|
|
23
|
+
*
|
|
24
|
+
* (C2 — extract overwriting its own input — is a Python-CLI-only surface;
|
|
25
|
+
* the Node packages expose no file-writing extract command.)
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { describe, it, expect } from "vitest";
|
|
29
|
+
import { strFromU8, strToU8, unzipSync, zipSync } from "fflate";
|
|
30
|
+
import {
|
|
31
|
+
createTestDocument,
|
|
32
|
+
addParagraph,
|
|
33
|
+
addTable,
|
|
34
|
+
setCellText,
|
|
35
|
+
} from "./test-utils.js";
|
|
36
|
+
import { DocumentObject } from "./docx/bridge.js";
|
|
37
|
+
import { extractTextFromBuffer, _extractTextFromDoc } from "./ingest.js";
|
|
38
|
+
import { generate_structured_edits } from "./diff.js";
|
|
39
|
+
import { RedlineEngine } from "./engine.js";
|
|
40
|
+
import { finalize_document } from "./sanitize/core.js";
|
|
41
|
+
|
|
42
|
+
const CUSTOM_PROPS_XML =
|
|
43
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
|
|
44
|
+
'<Properties xmlns="http://schemas.openxmlformats.org/officeDocument/2006/custom-properties" ' +
|
|
45
|
+
'xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">' +
|
|
46
|
+
'<property fmtid="{D5CDD505-2E9C-101B-9397-08002B2CF9AE}" pid="2" name="ClientSecret">' +
|
|
47
|
+
"<vt:lpwstr>TOP-SECRET-ORCHID</vt:lpwstr></property>" +
|
|
48
|
+
'<property fmtid="{D5CDD505-2E9C-101B-9397-08002B2CF9AE}" pid="3" name="MatterNumber">' +
|
|
49
|
+
"<vt:lpwstr>MAT-998877</vt:lpwstr></property>" +
|
|
50
|
+
"</Properties>";
|
|
51
|
+
|
|
52
|
+
const CUSTOM_XML_ITEM =
|
|
53
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
|
|
54
|
+
"<clientData><name>Kontoso Legal Oy</name><id>SENTINEL-HETU-131052-308T</id></clientData>";
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Injects secret-bearing metadata (custom properties part, customXml part,
|
|
58
|
+
* core identifier/language/version) into a saved DOCX buffer at ZIP level —
|
|
59
|
+
* the same fixture shape the QA report used.
|
|
60
|
+
*/
|
|
61
|
+
function injectMetadata(buffer: Buffer): Buffer {
|
|
62
|
+
const unzipped = unzipSync(new Uint8Array(buffer));
|
|
63
|
+
|
|
64
|
+
let ct = strFromU8(unzipped["[Content_Types].xml"]);
|
|
65
|
+
ct = ct.replace(
|
|
66
|
+
"</Types>",
|
|
67
|
+
'<Override PartName="/docProps/custom.xml" ContentType="application/vnd.openxmlformats-officedocument.custom-properties+xml"/>' +
|
|
68
|
+
'<Override PartName="/customXml/item1.xml" ContentType="application/xml"/>' +
|
|
69
|
+
"</Types>",
|
|
70
|
+
);
|
|
71
|
+
unzipped["[Content_Types].xml"] = strToU8(ct);
|
|
72
|
+
|
|
73
|
+
let rels = strFromU8(unzipped["_rels/.rels"]);
|
|
74
|
+
rels = rels.replace(
|
|
75
|
+
"</Relationships>",
|
|
76
|
+
'<Relationship Id="rIdCustomProps" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/custom-properties" Target="docProps/custom.xml"/>' +
|
|
77
|
+
"</Relationships>",
|
|
78
|
+
);
|
|
79
|
+
unzipped["_rels/.rels"] = strToU8(rels);
|
|
80
|
+
|
|
81
|
+
let core = strFromU8(unzipped["docProps/core.xml"]);
|
|
82
|
+
core = core
|
|
83
|
+
.replace("<dc:creator></dc:creator>", "<dc:creator>Alice Example</dc:creator>")
|
|
84
|
+
.replace("</cp:coreProperties>", "<dc:identifier>CLIENT-12345</dc:identifier><dc:language>fi-FI</dc:language><cp:version>9.9-internal</cp:version></cp:coreProperties>");
|
|
85
|
+
unzipped["docProps/core.xml"] = strToU8(core);
|
|
86
|
+
|
|
87
|
+
unzipped["docProps/custom.xml"] = strToU8(CUSTOM_PROPS_XML);
|
|
88
|
+
unzipped["customXml/item1.xml"] = strToU8(CUSTOM_XML_ITEM);
|
|
89
|
+
|
|
90
|
+
return Buffer.from(zipSync(unzipped));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function buildMetadataDoc(): Promise<DocumentObject> {
|
|
94
|
+
const doc = await createTestDocument();
|
|
95
|
+
addParagraph(doc, "Body content that is perfectly fine to share.");
|
|
96
|
+
const buf = injectMetadata(await doc.save());
|
|
97
|
+
return DocumentObject.load(buf);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function packageText(buffer: Buffer): string {
|
|
101
|
+
const unzipped = unzipSync(new Uint8Array(buffer));
|
|
102
|
+
return Object.values(unzipped)
|
|
103
|
+
.map((data) => strFromU8(data))
|
|
104
|
+
.join("\n");
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// ---------------------------------------------------------------------------
|
|
108
|
+
// C1 — sanitize must remove custom properties / customXml from SAVED bytes
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
|
|
111
|
+
describe("QA v6 C1: custom document properties & retained core fields", () => {
|
|
112
|
+
it("removes docProps/custom.xml from the saved package", async () => {
|
|
113
|
+
const doc = await buildMetadataDoc();
|
|
114
|
+
const { outBuffer, reportText } = await finalize_document(doc, {
|
|
115
|
+
filename: "metadata.docx",
|
|
116
|
+
sanitize_mode: "full",
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
expect(outBuffer).toBeDefined();
|
|
120
|
+
const names = Object.keys(unzipSync(new Uint8Array(outBuffer!)));
|
|
121
|
+
expect(names).not.toContain("docProps/custom.xml");
|
|
122
|
+
const pkgText = packageText(outBuffer!);
|
|
123
|
+
expect(pkgText).not.toContain("TOP-SECRET-ORCHID");
|
|
124
|
+
expect(pkgText).not.toContain("MAT-998877");
|
|
125
|
+
expect(reportText.toLowerCase()).toContain("custom");
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("removes customXml parts from the saved package (pkg.unzipped leak)", async () => {
|
|
129
|
+
const doc = await buildMetadataDoc();
|
|
130
|
+
const { outBuffer } = await finalize_document(doc, {
|
|
131
|
+
filename: "metadata.docx",
|
|
132
|
+
sanitize_mode: "full",
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const names = Object.keys(unzipSync(new Uint8Array(outBuffer!)));
|
|
136
|
+
expect(names.filter((n) => n.startsWith("customXml/"))).toEqual([]);
|
|
137
|
+
expect(packageText(outBuffer!)).not.toContain("SENTINEL-HETU-131052-308T");
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it("scrubs dc:identifier, dc:language and cp:version from core.xml", async () => {
|
|
141
|
+
const doc = await buildMetadataDoc();
|
|
142
|
+
const { outBuffer, reportText } = await finalize_document(doc, {
|
|
143
|
+
filename: "metadata.docx",
|
|
144
|
+
sanitize_mode: "full",
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
const unzipped = unzipSync(new Uint8Array(outBuffer!));
|
|
148
|
+
const core = strFromU8(unzipped["docProps/core.xml"]);
|
|
149
|
+
expect(core).not.toContain("CLIENT-12345");
|
|
150
|
+
expect(core).not.toContain("fi-FI");
|
|
151
|
+
expect(core).not.toContain("9.9-internal");
|
|
152
|
+
// Scrubbed fields must be enumerated in the report, not silent.
|
|
153
|
+
expect(reportText).toContain("CLIENT-12345");
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("keep-markup mode also removes custom properties", async () => {
|
|
157
|
+
const doc = await buildMetadataDoc();
|
|
158
|
+
const { outBuffer } = await finalize_document(doc, {
|
|
159
|
+
filename: "metadata.docx",
|
|
160
|
+
sanitize_mode: "keep-markup",
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
const names = Object.keys(unzipSync(new Uint8Array(outBuffer!)));
|
|
164
|
+
expect(names).not.toContain("docProps/custom.xml");
|
|
165
|
+
expect(packageText(outBuffer!)).not.toContain("TOP-SECRET-ORCHID");
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("sanitized output still loads and keeps body content", async () => {
|
|
169
|
+
const doc = await buildMetadataDoc();
|
|
170
|
+
const { outBuffer } = await finalize_document(doc, {
|
|
171
|
+
filename: "metadata.docx",
|
|
172
|
+
sanitize_mode: "full",
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
const text = await extractTextFromBuffer(outBuffer!, true);
|
|
176
|
+
expect(text).toContain("perfectly fine to share");
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
// ---------------------------------------------------------------------------
|
|
181
|
+
// H1 — row ops replayed without pins (the MCP batch shape)
|
|
182
|
+
// ---------------------------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
const ORIG_ROWS = [
|
|
185
|
+
["Seats", "5", "€100"],
|
|
186
|
+
["Support", "1", "€500"],
|
|
187
|
+
];
|
|
188
|
+
const MOD_ROWS = [
|
|
189
|
+
["Seats", "10", "€125"],
|
|
190
|
+
["Support", "1", "Included"],
|
|
191
|
+
["Storage", "100 GB", "€50"],
|
|
192
|
+
];
|
|
193
|
+
|
|
194
|
+
async function buildTableDoc(rows: string[][]): Promise<DocumentObject> {
|
|
195
|
+
const doc = await createTestDocument();
|
|
196
|
+
addParagraph(doc, "Pricing schedule below.");
|
|
197
|
+
const tbl = addTable(doc, rows.length, rows[0].length);
|
|
198
|
+
for (let r = 0; r < rows.length; r++) {
|
|
199
|
+
for (let c = 0; c < rows[r].length; c++) setCellText(tbl, r, c, rows[r][c]);
|
|
200
|
+
}
|
|
201
|
+
addParagraph(doc, "Terms follow the table.");
|
|
202
|
+
return doc;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function extractWithStructure(doc: DocumentObject): { text: string; structure: any } {
|
|
206
|
+
return (_extractTextFromDoc as any)(doc, true, false, false, true);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Simulates the JSON round trip: private positional pins do not survive. */
|
|
210
|
+
function stripPins(edits: any[]): any[] {
|
|
211
|
+
return edits.map((e) => {
|
|
212
|
+
const clone = { ...e };
|
|
213
|
+
delete clone._match_start_index;
|
|
214
|
+
delete clone._resolved_start_idx;
|
|
215
|
+
return clone;
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
describe("QA v6 H1: diff row ops must replay without pins", () => {
|
|
220
|
+
it("row modify + insert_row anchored on the modified row lands correctly", async () => {
|
|
221
|
+
const orig = await buildTableDoc(ORIG_ROWS);
|
|
222
|
+
const mod = await buildTableDoc(MOD_ROWS);
|
|
223
|
+
const origBuf = await orig.save();
|
|
224
|
+
const modBuf = await mod.save();
|
|
225
|
+
|
|
226
|
+
const o = extractWithStructure(await DocumentObject.load(origBuf));
|
|
227
|
+
const m = extractWithStructure(await DocumentObject.load(modBuf));
|
|
228
|
+
const { edits, warnings } = generate_structured_edits(o.text, o.structure, m.text, m.structure);
|
|
229
|
+
expect(warnings).toEqual([]);
|
|
230
|
+
|
|
231
|
+
const workDoc = await DocumentObject.load(origBuf);
|
|
232
|
+
const engine = new RedlineEngine(workDoc, "QA");
|
|
233
|
+
const stats = engine.process_batch(stripPins(edits));
|
|
234
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
235
|
+
|
|
236
|
+
engine.accept_all_revisions(true);
|
|
237
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
238
|
+
const wantText = await extractTextFromBuffer(modBuf, true);
|
|
239
|
+
expect(finalText).toBe(wantText);
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
it("inserted row lands below its modified anchor, not at a stale offset", async () => {
|
|
243
|
+
const orig = await buildTableDoc(ORIG_ROWS);
|
|
244
|
+
const mod = await buildTableDoc(MOD_ROWS);
|
|
245
|
+
const origBuf = await orig.save();
|
|
246
|
+
|
|
247
|
+
const o = extractWithStructure(await DocumentObject.load(origBuf));
|
|
248
|
+
const m = extractWithStructure(mod);
|
|
249
|
+
const { edits } = generate_structured_edits(o.text, o.structure, m.text, m.structure);
|
|
250
|
+
|
|
251
|
+
const workDoc = await DocumentObject.load(origBuf);
|
|
252
|
+
const engine = new RedlineEngine(workDoc, "QA");
|
|
253
|
+
engine.process_batch(stripPins(edits));
|
|
254
|
+
engine.accept_all_revisions(true);
|
|
255
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
256
|
+
|
|
257
|
+
const support = finalText.indexOf("Support | 1 | Included");
|
|
258
|
+
const storage = finalText.indexOf("Storage | 100 GB | €50");
|
|
259
|
+
expect(support, finalText).toBeGreaterThanOrEqual(0);
|
|
260
|
+
expect(storage, finalText).toBeGreaterThanOrEqual(0);
|
|
261
|
+
expect(support, `wrong row order:\n${finalText}`).toBeLessThan(storage);
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
it("delete_row + modify composition replays without pins", async () => {
|
|
265
|
+
const orig = await buildTableDoc([
|
|
266
|
+
["Alpha", "1", "a"],
|
|
267
|
+
["Beta", "2", "b"],
|
|
268
|
+
["Gamma", "3", "c"],
|
|
269
|
+
]);
|
|
270
|
+
const mod = await buildTableDoc([
|
|
271
|
+
["Alpha", "9", "z"],
|
|
272
|
+
["Gamma", "3", "c"],
|
|
273
|
+
]);
|
|
274
|
+
const origBuf = await orig.save();
|
|
275
|
+
const modBuf = await mod.save();
|
|
276
|
+
|
|
277
|
+
const o = extractWithStructure(await DocumentObject.load(origBuf));
|
|
278
|
+
const m = extractWithStructure(await DocumentObject.load(modBuf));
|
|
279
|
+
const { edits } = generate_structured_edits(o.text, o.structure, m.text, m.structure);
|
|
280
|
+
|
|
281
|
+
const workDoc = await DocumentObject.load(origBuf);
|
|
282
|
+
const engine = new RedlineEngine(workDoc, "QA");
|
|
283
|
+
const stats = engine.process_batch(stripPins(edits));
|
|
284
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
285
|
+
|
|
286
|
+
engine.accept_all_revisions(true);
|
|
287
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
288
|
+
const wantText = await extractTextFromBuffer(modBuf, true);
|
|
289
|
+
expect(finalText).toBe(wantText);
|
|
290
|
+
});
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
// ---------------------------------------------------------------------------
|
|
294
|
+
// H2 — paragraph boundaries through context-wrapped insertions
|
|
295
|
+
// ---------------------------------------------------------------------------
|
|
296
|
+
|
|
297
|
+
async function buildParagraphDoc(): Promise<DocumentObject> {
|
|
298
|
+
const doc = await createTestDocument();
|
|
299
|
+
addParagraph(doc, "Intro paragraph one.");
|
|
300
|
+
addParagraph(doc, "Middle paragraph two.");
|
|
301
|
+
addParagraph(doc, "Final sentinel: END-OF-DOCUMENT-9f2c.");
|
|
302
|
+
return doc;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
describe("QA v6 H2: paragraph boundary preservation", () => {
|
|
306
|
+
it("context-wrapped paragraph insertion keeps its separators", async () => {
|
|
307
|
+
const doc = await buildParagraphDoc();
|
|
308
|
+
const buf = await doc.save();
|
|
309
|
+
|
|
310
|
+
const workDoc = await DocumentObject.load(buf);
|
|
311
|
+
const engine = new RedlineEngine(workDoc, "QA");
|
|
312
|
+
// The self-contained JSON shape the Python CLI emits — and the shape an
|
|
313
|
+
// LLM naturally writes into process_document_batch.
|
|
314
|
+
const stats = engine.process_batch([
|
|
315
|
+
{
|
|
316
|
+
type: "modify",
|
|
317
|
+
target_text: "two.\n\nFinal",
|
|
318
|
+
new_text: "two.\n\nAdditional sentence inserted before the final marker.\n\nFinal",
|
|
319
|
+
} as any,
|
|
320
|
+
]);
|
|
321
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
322
|
+
|
|
323
|
+
engine.accept_all_revisions(true);
|
|
324
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
325
|
+
expect(finalText).not.toContain("marker.Final");
|
|
326
|
+
expect(finalText).not.toContain("two..");
|
|
327
|
+
expect(finalText).toBe(
|
|
328
|
+
"Intro paragraph one.\n\nMiddle paragraph two.\n\n" +
|
|
329
|
+
"Additional sentence inserted before the final marker.\n\n" +
|
|
330
|
+
"Final sentinel: END-OF-DOCUMENT-9f2c.",
|
|
331
|
+
);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
it("docx-to-docx structured diff of an inserted paragraph round-trips (pinned, in-process)", async () => {
|
|
335
|
+
// Node's structured edits are an in-process shape: pure insertions ride
|
|
336
|
+
// the private pins (the Python CLI additionally rewrites them via
|
|
337
|
+
// make_edits_self_contained for its JSON output). The invariant here is
|
|
338
|
+
// apply(A, diff(A,B)) + accept-all == B with the paragraph boundary kept.
|
|
339
|
+
const orig = await buildParagraphDoc();
|
|
340
|
+
const origBuf = await orig.save();
|
|
341
|
+
|
|
342
|
+
const mod = await createTestDocument();
|
|
343
|
+
addParagraph(mod, "Intro paragraph one.");
|
|
344
|
+
addParagraph(mod, "Middle paragraph two.");
|
|
345
|
+
addParagraph(mod, "Additional sentence inserted before the final marker.");
|
|
346
|
+
addParagraph(mod, "Final sentinel: END-OF-DOCUMENT-9f2c.");
|
|
347
|
+
const modBuf = await mod.save();
|
|
348
|
+
|
|
349
|
+
const o = extractWithStructure(await DocumentObject.load(origBuf));
|
|
350
|
+
const m = extractWithStructure(await DocumentObject.load(modBuf));
|
|
351
|
+
const { edits } = generate_structured_edits(o.text, o.structure, m.text, m.structure);
|
|
352
|
+
|
|
353
|
+
const workDoc = await DocumentObject.load(origBuf);
|
|
354
|
+
const engine = new RedlineEngine(workDoc, "QA");
|
|
355
|
+
const stats = engine.process_batch(edits as any);
|
|
356
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
357
|
+
|
|
358
|
+
engine.accept_all_revisions(true);
|
|
359
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
360
|
+
const wantText = await extractTextFromBuffer(modBuf, true);
|
|
361
|
+
expect(finalText).not.toContain("marker.Final");
|
|
362
|
+
expect(finalText).toBe(wantText);
|
|
363
|
+
});
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
// ---------------------------------------------------------------------------
|
|
367
|
+
// Hypothesis-found regressions (2026-07-18 fuzz hunt, ported from Python)
|
|
368
|
+
// ---------------------------------------------------------------------------
|
|
369
|
+
|
|
370
|
+
describe("QA v6 fuzz-found regressions", () => {
|
|
371
|
+
it("paragraph split keeps the suffix in the new paragraph", async () => {
|
|
372
|
+
const doc = await createTestDocument();
|
|
373
|
+
addParagraph(doc, "alpha beta.");
|
|
374
|
+
const buf = await doc.save();
|
|
375
|
+
|
|
376
|
+
const workDoc = await DocumentObject.load(buf);
|
|
377
|
+
const engine = new RedlineEngine(workDoc, "Fuzz");
|
|
378
|
+
const stats = engine.process_batch([
|
|
379
|
+
{ type: "modify", target_text: "alpha beta.", new_text: "alpha.\n\nbeta." } as any,
|
|
380
|
+
]);
|
|
381
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
382
|
+
|
|
383
|
+
engine.accept_all_revisions(true);
|
|
384
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
385
|
+
expect(finalText).toBe("alpha.\n\nbeta.");
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
it("three-way paragraph split with changed content", async () => {
|
|
389
|
+
const doc = await createTestDocument();
|
|
390
|
+
addParagraph(doc, "p q.");
|
|
391
|
+
const buf = await doc.save();
|
|
392
|
+
|
|
393
|
+
const workDoc = await DocumentObject.load(buf);
|
|
394
|
+
const engine = new RedlineEngine(workDoc, "Fuzz");
|
|
395
|
+
const stats = engine.process_batch([
|
|
396
|
+
{ type: "modify", target_text: "p q.", new_text: "p.\n\nq.\n\nr." } as any,
|
|
397
|
+
]);
|
|
398
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
399
|
+
|
|
400
|
+
engine.accept_all_revisions(true);
|
|
401
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
402
|
+
expect(finalText).toBe("p.\n\nq.\n\nr.");
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
it("plain paragraph starting with a number stays plain (no phantom list)", async () => {
|
|
406
|
+
const doc = await createTestDocument();
|
|
407
|
+
addParagraph(doc, "Intro.");
|
|
408
|
+
const buf = await doc.save();
|
|
409
|
+
|
|
410
|
+
const workDoc = await DocumentObject.load(buf);
|
|
411
|
+
const engine = new RedlineEngine(workDoc, "Fuzz");
|
|
412
|
+
const stats = engine.process_batch([
|
|
413
|
+
{ type: "modify", target_text: "Intro.", new_text: "Intro.\n\n2024. Year in review." } as any,
|
|
414
|
+
]);
|
|
415
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
416
|
+
|
|
417
|
+
engine.accept_all_revisions(true);
|
|
418
|
+
const outBuf = await workDoc.save();
|
|
419
|
+
const finalText = await extractTextFromBuffer(outBuf, true);
|
|
420
|
+
expect(finalText).toBe("Intro.\n\n2024. Year in review.");
|
|
421
|
+
const { strFromU8, unzipSync } = await import("fflate");
|
|
422
|
+
const xml = strFromU8(unzipSync(new Uint8Array(outBuf))["word/document.xml"]);
|
|
423
|
+
expect(xml).not.toContain("ListNumber");
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
it("the projection's own '1. ' marker still creates a list item", async () => {
|
|
427
|
+
const doc = await createTestDocument();
|
|
428
|
+
addParagraph(doc, "Intro.");
|
|
429
|
+
const buf = await doc.save();
|
|
430
|
+
|
|
431
|
+
const workDoc = await DocumentObject.load(buf);
|
|
432
|
+
const engine = new RedlineEngine(workDoc, "Fuzz");
|
|
433
|
+
const stats = engine.process_batch([
|
|
434
|
+
{ type: "modify", target_text: "Intro.", new_text: "Intro.\n\n1. first item" } as any,
|
|
435
|
+
]);
|
|
436
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
437
|
+
|
|
438
|
+
const outBuf = await workDoc.save();
|
|
439
|
+
const { strFromU8, unzipSync } = await import("fflate");
|
|
440
|
+
const xml = strFromU8(unzipSync(new Uint8Array(outBuf))["word/document.xml"]);
|
|
441
|
+
expect(xml).toContain("ListNumber");
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
it("cell edit on repetitive content lands in the right cell", async () => {
|
|
445
|
+
const orig = await buildTableDoc([["0", "0", "0"]]);
|
|
446
|
+
const mod = await buildTableDoc([["0", "0 0", "0"]]);
|
|
447
|
+
const origBuf = await orig.save();
|
|
448
|
+
const modBuf = await mod.save();
|
|
449
|
+
|
|
450
|
+
const o = extractWithStructure(await DocumentObject.load(origBuf));
|
|
451
|
+
const m = extractWithStructure(await DocumentObject.load(modBuf));
|
|
452
|
+
const { edits, warnings } = generate_structured_edits(o.text, o.structure, m.text, m.structure);
|
|
453
|
+
expect(warnings).toEqual([]);
|
|
454
|
+
|
|
455
|
+
const workDoc = await DocumentObject.load(origBuf);
|
|
456
|
+
const engine = new RedlineEngine(workDoc, "Fuzz");
|
|
457
|
+
const stats = engine.process_batch(stripPins(edits));
|
|
458
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
459
|
+
|
|
460
|
+
engine.accept_all_revisions(true);
|
|
461
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
462
|
+
const wantText = await extractTextFromBuffer(modBuf, true);
|
|
463
|
+
expect(finalText).toBe(wantText);
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
it("multi-cell row modification replays without pins", async () => {
|
|
467
|
+
const orig = await buildTableDoc([["0 0", "0 0", "00"]]);
|
|
468
|
+
const mod = await buildTableDoc([["0 0", "0", "0"]]);
|
|
469
|
+
const origBuf = await orig.save();
|
|
470
|
+
const modBuf = await mod.save();
|
|
471
|
+
|
|
472
|
+
const o = extractWithStructure(await DocumentObject.load(origBuf));
|
|
473
|
+
const m = extractWithStructure(await DocumentObject.load(modBuf));
|
|
474
|
+
const { edits } = generate_structured_edits(o.text, o.structure, m.text, m.structure);
|
|
475
|
+
|
|
476
|
+
const workDoc = await DocumentObject.load(origBuf);
|
|
477
|
+
const engine = new RedlineEngine(workDoc, "Fuzz");
|
|
478
|
+
const stats = engine.process_batch(stripPins(edits));
|
|
479
|
+
expect(stats.edits_skipped, JSON.stringify(stats.skipped_details)).toBe(0);
|
|
480
|
+
|
|
481
|
+
engine.accept_all_revisions(true);
|
|
482
|
+
const finalText = await extractTextFromBuffer(await workDoc.save(), true);
|
|
483
|
+
const wantText = await extractTextFromBuffer(modBuf, true);
|
|
484
|
+
expect(finalText).toBe(wantText);
|
|
485
|
+
});
|
|
486
|
+
});
|
package/src/sanitize/core.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { strFromU8, unzipSync } from 'fflate';
|
|
1
2
|
import { DocumentObject } from '../docx/bridge.js';
|
|
2
3
|
import { SanitizeReport } from './report.js';
|
|
3
4
|
import * as transforms from './transforms.js';
|
|
4
|
-
import { findAllDescendants } from '../docx/dom.js';
|
|
5
|
+
import { findAllDescendants, parseXml } from '../docx/dom.js';
|
|
5
6
|
|
|
6
7
|
export interface FinalizeOptions {
|
|
7
8
|
filename: string;
|
|
@@ -67,6 +68,7 @@ export async function finalize_document(doc: DocumentObject, options: FinalizeOp
|
|
|
67
68
|
report.add_transform_lines(transforms.scrub_doc_properties(doc));
|
|
68
69
|
report.add_transform_lines(transforms.scrub_timestamps(doc));
|
|
69
70
|
report.add_transform_lines(transforms.strip_custom_xml(doc));
|
|
71
|
+
report.add_transform_lines(transforms.strip_custom_properties(doc));
|
|
70
72
|
report.add_transform_lines(transforms.strip_image_alt_text(doc));
|
|
71
73
|
|
|
72
74
|
const warnings = transforms.audit_hyperlinks(doc);
|
|
@@ -130,5 +132,59 @@ export async function finalize_document(doc: DocumentObject, options: FinalizeOp
|
|
|
130
132
|
if (report.warnings.length > 0) report.status = 'clean_with_warnings';
|
|
131
133
|
|
|
132
134
|
const outBuffer = await doc.save();
|
|
135
|
+
verifySanitizedPackage(outBuffer);
|
|
133
136
|
return { reportText: report.render(), outBuffer };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Core-property elements the pipeline claims to scrub; the post-sanitize
|
|
140
|
+
// verification re-checks each one in the SAVED bytes.
|
|
141
|
+
const VERIFIED_CORE_FIELDS: Array<[string, string]> = [
|
|
142
|
+
['creator', 'author (dc:creator)'],
|
|
143
|
+
['lastModifiedBy', 'last modified by (cp:lastModifiedBy)'],
|
|
144
|
+
['identifier', 'identifier (dc:identifier)'],
|
|
145
|
+
['description', 'description (dc:description)'],
|
|
146
|
+
['keywords', 'keywords (cp:keywords)'],
|
|
147
|
+
['category', 'category (cp:category)'],
|
|
148
|
+
['subject', 'subject (dc:subject)'],
|
|
149
|
+
['contentStatus', 'content status (cp:contentStatus)'],
|
|
150
|
+
['language', 'language (dc:language)'],
|
|
151
|
+
['version', 'version (cp:version)'],
|
|
152
|
+
];
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Post-sanitize package scan: re-open the SAVED bytes —
|
|
156
|
+
* bypassing every in-memory caching layer — and verify the claims the report
|
|
157
|
+
* is about to make. A "Result: CLEAN" verdict over a package that still
|
|
158
|
+
* carries custom properties or an identifier is worse than no sanitizer.
|
|
159
|
+
*/
|
|
160
|
+
export function verifySanitizedPackage(outBuffer: Buffer): void {
|
|
161
|
+
const unzipped = unzipSync(new Uint8Array(outBuffer));
|
|
162
|
+
const names = Object.keys(unzipped);
|
|
163
|
+
const problems: string[] = [];
|
|
164
|
+
|
|
165
|
+
if (names.includes('docProps/custom.xml')) {
|
|
166
|
+
problems.push('docProps/custom.xml (custom document properties) is still in the package');
|
|
167
|
+
}
|
|
168
|
+
if (names.some(n => n.startsWith('customXml/'))) {
|
|
169
|
+
problems.push('customXml/* parts are still in the package');
|
|
170
|
+
}
|
|
171
|
+
if (names.includes('docProps/core.xml')) {
|
|
172
|
+
const core = parseXml(strFromU8(unzipped['docProps/core.xml']));
|
|
173
|
+
for (const [local, label] of VERIFIED_CORE_FIELDS) {
|
|
174
|
+
for (const el of transforms.findDescendantsByLocalName(core.documentElement! as unknown as Element, local)) {
|
|
175
|
+
if ((el.textContent || '').trim()) {
|
|
176
|
+
problems.push(`core property ${label} still contains a value`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (problems.length > 0) {
|
|
183
|
+
throw new Error(
|
|
184
|
+
'Sanitize integrity check failed — the saved package still contains metadata this run ' +
|
|
185
|
+
'claims to remove:\n - ' +
|
|
186
|
+
problems.join('\n - ') +
|
|
187
|
+
'\nRefusing to report a clean document.',
|
|
188
|
+
);
|
|
189
|
+
}
|
|
134
190
|
}
|
package/src/sanitize/report.ts
CHANGED
|
@@ -38,9 +38,11 @@ export class SanitizeReport {
|
|
|
38
38
|
this.removed_comment_lines.push(line);
|
|
39
39
|
}
|
|
40
40
|
} else if (
|
|
41
|
-
lower.includes("author") || lower.includes("template") || lower.includes("company") ||
|
|
42
|
-
lower.includes("manager") || lower.includes("metadata") || lower.includes("timestamp") ||
|
|
43
|
-
lower.includes("custom xml") || lower.includes("
|
|
41
|
+
lower.includes("author") || lower.includes("template") || lower.includes("company") ||
|
|
42
|
+
lower.includes("manager") || lower.includes("metadata") || lower.includes("timestamp") ||
|
|
43
|
+
lower.includes("custom xml") || lower.includes("custom propert") || lower.includes("identifier") ||
|
|
44
|
+
lower.includes("language") || lower.includes("version") ||
|
|
45
|
+
lower.includes("last modified by") || lower.includes("revision count") || lower.includes("last printed")
|
|
44
46
|
) {
|
|
45
47
|
this.metadata_lines.push(line);
|
|
46
48
|
} else if (lower.includes("hyperlink") || lower.includes("warning")) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect, vi } from "vitest";
|
|
2
|
+
import { zipSync } from "fflate";
|
|
2
3
|
import { DOMParser } from "@xmldom/xmldom";
|
|
3
4
|
import { readFileSync } from "node:fs";
|
|
4
5
|
import { resolve, dirname } from "node:path";
|
|
@@ -186,7 +187,7 @@ describe("Finalize Document (Core)", () => {
|
|
|
186
187
|
doc.pkg.parts.push(settingsPart);
|
|
187
188
|
|
|
188
189
|
// Mock the doc.save buffer return
|
|
189
|
-
doc.save = vi.fn().mockResolvedValue(Buffer.from(
|
|
190
|
+
doc.save = vi.fn().mockResolvedValue(Buffer.from(zipSync({})));
|
|
190
191
|
|
|
191
192
|
const res = await finalize_document(doc, {
|
|
192
193
|
filename: "test.docx",
|
|
@@ -231,7 +232,7 @@ describe("Finalize Document (Core)", () => {
|
|
|
231
232
|
</w:p>
|
|
232
233
|
`);
|
|
233
234
|
|
|
234
|
-
doc.save = vi.fn().mockResolvedValue(Buffer.from(
|
|
235
|
+
doc.save = vi.fn().mockResolvedValue(Buffer.from(zipSync({})));
|
|
235
236
|
|
|
236
237
|
await finalize_document(doc, {
|
|
237
238
|
filename: "test.docx",
|
|
@@ -258,7 +259,7 @@ describe("Finalize Document (Core)", () => {
|
|
|
258
259
|
"http://schemas.microsoft.com/office/word/2023/wordml/word16du",
|
|
259
260
|
);
|
|
260
261
|
|
|
261
|
-
doc.save = vi.fn().mockResolvedValue(Buffer.from(
|
|
262
|
+
doc.save = vi.fn().mockResolvedValue(Buffer.from(zipSync({})));
|
|
262
263
|
|
|
263
264
|
await finalize_document(doc, {
|
|
264
265
|
filename: "test.docx",
|
|
@@ -285,7 +286,7 @@ describe("Finalize Document (Core)", () => {
|
|
|
285
286
|
expect(original_xml).toContain("w:commentReference");
|
|
286
287
|
|
|
287
288
|
// Mock the doc.save buffer return
|
|
288
|
-
doc.save = vi.fn().mockResolvedValue(Buffer.from(
|
|
289
|
+
doc.save = vi.fn().mockResolvedValue(Buffer.from(zipSync({})));
|
|
289
290
|
|
|
290
291
|
// 2. Act: Finalize the document in full sanitize mode with accept_all: true
|
|
291
292
|
await finalize_document(doc, {
|