@ansonlai/docx-redline-js 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +36 -10
- package/README.md +83 -6
- package/adapters/xml-adapter.js +73 -10
- package/core/list-targeting.js +3 -0
- package/core/paragraph-targeting.js +33 -7
- package/core/redline-validation.js +22 -0
- package/core/types.js +122 -27
- package/core/xml-query.js +3 -1
- package/dist/docx-redline-js.esm.js +1124 -545
- package/dist/docx-redline-js.esm.js.map +4 -4
- package/dist/docx-redline-js.esm.min.js +79 -78
- package/dist/docx-redline-js.esm.min.js.map +4 -4
- package/docs/TESTING.md +687 -0
- package/docs/VALIDATION.md +81 -2
- package/docs/WORD-MANUAL-REVIEW.md +138 -0
- package/docs/plans/2026-08-30-reliability-testing-improvements.md +488 -0
- package/docs/plans/2026-09-01-performance-and-complexity-reduction.md +210 -0
- package/docs/plans/{2026-03-01-release-0.1.4-design.md → completed/2026-03-01-release-0.1.4-design.md} +2 -0
- package/docs/plans/{2026-03-01-release-0.1.4.md → completed/2026-03-01-release-0.1.4.md} +5 -3
- package/docs/plans/{2026-05-31-architectural changes.md → completed/2026-05-31-architectural changes.md } +2 -0
- package/docs/plans/completed/2026-08-02-reliability-improvements.md +1155 -0
- package/docs/test-comparison-dashboard.html +95 -0
- package/docs/validation-reports/2026-08-30-phase-1-word-visual-preflight.md +22 -0
- package/docs/validation-reports/2026-08-30-phase-2-word-visual-preflight.md +24 -0
- package/docs/validation-reports/2026-08-30-phase-3-coverage.md +73 -0
- package/docs/validation-reports/2026-09-02-multilevel-bullets-visual-review.md +82 -0
- package/docs/validation-reports/2026-09-02-multimodal-visual-samples.md +114 -0
- package/docs/validation-reports/2026-09-02-visual-failures-preflight.md +79 -0
- package/engine/format-extraction.js +1 -1
- package/engine/formatting-removal.js +84 -99
- package/engine/oxml-engine.js +176 -83
- package/engine/reconstruction-mapper.js +276 -79
- package/engine/reconstruction-mode.js +20 -6
- package/engine/reconstruction-writer.js +117 -72
- package/engine/run-builders.js +3 -3
- package/engine/surgical-mode.js +3 -2
- package/engine/table-mode.js +27 -16
- package/index.d.ts +95 -3
- package/index.js +14 -13
- package/orchestration/list-structural-fallback.js +16 -39
- package/package.json +22 -4
- package/pipeline/diff-engine.js +174 -55
- package/pipeline/ingestion-export.js +39 -24
- package/pipeline/ingestion-paragraph.js +7 -5
- package/pipeline/list-generation.js +27 -18
- package/pipeline/patching.js +2 -3
- package/pipeline/pipeline.js +65 -36
- package/pipeline/serialization.js +13 -5
- package/scripts/build-test-dashboard.mjs +43 -0
- package/scripts/check-types.mjs +16 -24
- package/scripts/export-validation-fixtures.mjs +191 -45
- package/scripts/fetch-superdoc-corpus.mjs +61 -0
- package/scripts/generate-test-dashboard.mjs +199 -0
- package/scripts/inspect-visual-evidence.mjs +271 -0
- package/scripts/lib/minimal-zip.mjs +199 -18
- package/scripts/lib/word-coverage-catalogue.mjs +207 -0
- package/scripts/lib/word-coverage-metadata.mjs +93 -0
- package/scripts/lib/zip-reader.mjs +64 -0
- package/scripts/package-superdoc-word-fixtures.ps1 +64 -0
- package/scripts/prepare-corpus-word-visual-review.mjs +84 -0
- package/scripts/prepare-superdoc-word-corpus.mjs +284 -0
- package/scripts/prepare-word-review.mjs +77 -0
- package/scripts/prepare-word-visual-review.mjs +90 -0
- package/scripts/render-agenda-multilevel.mjs +70 -0
- package/scripts/render-case22.mjs +73 -0
- package/scripts/render-case40.ps1 +35 -0
- package/scripts/render-multilevel-bullet-images.py +58 -0
- package/scripts/render-multilevel-bullet-visual.ps1 +32 -0
- package/scripts/render-multilevel-cases.mjs +80 -0
- package/scripts/report-coverage-gaps.mjs +103 -0
- package/scripts/report-word-coverage.mjs +71 -0
- package/scripts/sample-multimodal-visual-check.mjs +221 -0
- package/scripts/test-multilevel-bullet-visual.mjs +187 -0
- package/scripts/word-com-corpus-suite.ps1 +43 -0
- package/scripts/word-com-corpus-visual-suite.ps1 +116 -0
- package/scripts/word-com-differential.ps1 +158 -16
- package/scripts/word-com-suite.ps1 +19 -0
- package/scripts/word-com-visual-suite.ps1 +132 -0
- package/services/comment-engine.js +51 -46
- package/services/comment-locator.js +0 -1
- package/services/comment-package.js +11 -10
- package/services/numbering-service.js +1 -1
- package/services/revision-comment-management.js +31 -10
- package/services/standalone-docx-plumbing.js +45 -34
- package/services/standalone-operation-runner.js +315 -75
- package/services/table-reconciliation.js +23 -11
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
* text no-op but the target is marker-prefixed plain text (for example `1. X`).
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import { createSerializer, parseOoxmlSafe } from '../adapters/xml-adapter.js';
|
|
7
7
|
import { getXmlParseError } from '../core/xml-query.js';
|
|
8
|
+
import { createWordElement } from '../core/word-xml.js';
|
|
8
9
|
import {
|
|
9
10
|
getDocumentParagraphNodes,
|
|
10
11
|
normalizeWhitespaceForTargeting
|
|
@@ -217,10 +218,7 @@ export function enforceListBindingOnParagraphNodes(nodes, options = {}) {
|
|
|
217
218
|
|
|
218
219
|
let pPr = getDirectWordChild(paragraph, 'pPr');
|
|
219
220
|
if (!pPr) {
|
|
220
|
-
pPr = ownerDoc
|
|
221
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
222
|
-
'w:pPr'
|
|
223
|
-
);
|
|
221
|
+
pPr = createWordElement(ownerDoc, 'w:pPr');
|
|
224
222
|
paragraph.insertBefore(pPr, paragraph.firstChild);
|
|
225
223
|
}
|
|
226
224
|
|
|
@@ -236,29 +234,20 @@ export function enforceListBindingOnParagraphNodes(nodes, options = {}) {
|
|
|
236
234
|
|
|
237
235
|
let numPr = getDirectWordChild(pPr, 'numPr');
|
|
238
236
|
if (!numPr) {
|
|
239
|
-
numPr = ownerDoc
|
|
240
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
241
|
-
'w:numPr'
|
|
242
|
-
);
|
|
237
|
+
numPr = createWordElement(ownerDoc, 'w:numPr');
|
|
243
238
|
pPr.appendChild(numPr);
|
|
244
239
|
}
|
|
245
240
|
|
|
246
241
|
let ilvlEl = getDirectWordChild(numPr, 'ilvl');
|
|
247
242
|
if (!ilvlEl) {
|
|
248
|
-
ilvlEl = ownerDoc
|
|
249
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
250
|
-
'w:ilvl'
|
|
251
|
-
);
|
|
243
|
+
ilvlEl = createWordElement(ownerDoc, 'w:ilvl');
|
|
252
244
|
numPr.appendChild(ilvlEl);
|
|
253
245
|
}
|
|
254
246
|
ilvlEl.setAttribute('w:val', String(ilvl));
|
|
255
247
|
|
|
256
248
|
let numIdEl = getDirectWordChild(numPr, 'numId');
|
|
257
249
|
if (!numIdEl) {
|
|
258
|
-
numIdEl = ownerDoc
|
|
259
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
260
|
-
'w:numId'
|
|
261
|
-
);
|
|
250
|
+
numIdEl = createWordElement(ownerDoc, 'w:numId');
|
|
262
251
|
numPr.appendChild(numIdEl);
|
|
263
252
|
}
|
|
264
253
|
numIdEl.setAttribute('w:val', String(numId));
|
|
@@ -269,8 +258,8 @@ export function enforceListBindingOnParagraphNodes(nodes, options = {}) {
|
|
|
269
258
|
}
|
|
270
259
|
|
|
271
260
|
function getFirstParagraphFromOxml(oxml) {
|
|
272
|
-
const
|
|
273
|
-
|
|
261
|
+
const doc = parseOoxmlSafe(oxml, 'application/xml').doc;
|
|
262
|
+
if (!doc) return null;
|
|
274
263
|
const parseError = getXmlParseError(doc);
|
|
275
264
|
if (parseError) return null;
|
|
276
265
|
const paragraphs = getDocumentParagraphNodes(doc);
|
|
@@ -301,8 +290,8 @@ function setElementVal(element, value) {
|
|
|
301
290
|
}
|
|
302
291
|
|
|
303
292
|
function extractFirstParagraphNumIdFromOxml(oxml) {
|
|
304
|
-
const
|
|
305
|
-
|
|
293
|
+
const doc = parseOoxmlSafe(oxml, 'application/xml').doc;
|
|
294
|
+
if (!doc) return null;
|
|
306
295
|
const parseError = getXmlParseError(doc);
|
|
307
296
|
if (parseError) return null;
|
|
308
297
|
|
|
@@ -321,9 +310,9 @@ function extractFirstParagraphNumIdFromOxml(oxml) {
|
|
|
321
310
|
function applyStartOverrideToNumberingXml(numberingXml, targetNumId, startAt, options = {}) {
|
|
322
311
|
if (!numberingXml || !targetNumId || !Number.isInteger(startAt) || startAt < 1) return numberingXml;
|
|
323
312
|
const setAbstractStartOverride = options.setAbstractStartOverride !== false;
|
|
324
|
-
const parser = createParser();
|
|
325
313
|
const serializer = createSerializer();
|
|
326
|
-
const numberingDoc =
|
|
314
|
+
const numberingDoc = parseOoxmlSafe(numberingXml, 'application/xml').doc;
|
|
315
|
+
if (!numberingDoc) return numberingXml;
|
|
327
316
|
const parseError = getXmlParseError(numberingDoc);
|
|
328
317
|
if (parseError) return numberingXml;
|
|
329
318
|
|
|
@@ -343,20 +332,14 @@ function applyStartOverrideToNumberingXml(numberingXml, targetNumId, startAt, op
|
|
|
343
332
|
return ilvl === 0;
|
|
344
333
|
}) || null;
|
|
345
334
|
if (!lvlOverride) {
|
|
346
|
-
lvlOverride = numberingDoc
|
|
347
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
348
|
-
'w:lvlOverride'
|
|
349
|
-
);
|
|
335
|
+
lvlOverride = createWordElement(numberingDoc, 'w:lvlOverride');
|
|
350
336
|
lvlOverride.setAttribute('w:ilvl', '0');
|
|
351
337
|
target.appendChild(lvlOverride);
|
|
352
338
|
}
|
|
353
339
|
|
|
354
340
|
let startOverride = Array.from(lvlOverride.getElementsByTagNameNS('*', 'startOverride'))[0] || null;
|
|
355
341
|
if (!startOverride) {
|
|
356
|
-
startOverride = numberingDoc
|
|
357
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
358
|
-
'w:startOverride'
|
|
359
|
-
);
|
|
342
|
+
startOverride = createWordElement(numberingDoc, 'w:startOverride');
|
|
360
343
|
lvlOverride.appendChild(startOverride);
|
|
361
344
|
}
|
|
362
345
|
setElementVal(startOverride, startAt);
|
|
@@ -377,20 +360,14 @@ function applyStartOverrideToNumberingXml(numberingXml, targetNumId, startAt, op
|
|
|
377
360
|
return ilvl === 0;
|
|
378
361
|
}) || null;
|
|
379
362
|
if (!lvl) {
|
|
380
|
-
lvl = numberingDoc
|
|
381
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
382
|
-
'w:lvl'
|
|
383
|
-
);
|
|
363
|
+
lvl = createWordElement(numberingDoc, 'w:lvl');
|
|
384
364
|
lvl.setAttribute('w:ilvl', '0');
|
|
385
365
|
abstractNum.appendChild(lvl);
|
|
386
366
|
}
|
|
387
367
|
|
|
388
368
|
let startNode = Array.from(lvl.getElementsByTagNameNS('*', 'start'))[0] || null;
|
|
389
369
|
if (!startNode) {
|
|
390
|
-
startNode = numberingDoc
|
|
391
|
-
'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
|
|
392
|
-
'w:start'
|
|
393
|
-
);
|
|
370
|
+
startNode = createWordElement(numberingDoc, 'w:start');
|
|
394
371
|
lvl.insertBefore(startNode, lvl.firstChild);
|
|
395
372
|
}
|
|
396
373
|
setElementVal(startNode, startAt);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ansonlai/docx-redline-js",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Host-independent OOXML reconciliation engine for .docx manipulation with track changes",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -52,13 +52,31 @@
|
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@xmldom/xmldom": "^0.9.0",
|
|
55
|
-
"
|
|
55
|
+
"c8": "^12.0.0",
|
|
56
|
+
"docx-preview": "^0.4.0",
|
|
57
|
+
"esbuild": "^0.28.1",
|
|
58
|
+
"eslint": "^10.9.1",
|
|
59
|
+
"jszip": "^3.10.1",
|
|
60
|
+
"typescript": "^7.0.2"
|
|
56
61
|
},
|
|
57
62
|
"scripts": {
|
|
58
63
|
"build": "node scripts/build.mjs",
|
|
59
|
-
"check:types": "node scripts/check-types.mjs",
|
|
64
|
+
"check:types": "tsc -p tsconfig.types.json && node scripts/check-types.mjs",
|
|
65
|
+
"lint": "eslint index.js adapters core engine pipeline services orchestration scripts tests",
|
|
66
|
+
"test:coverage": "c8 --reporter=text --reporter=json --reporter=json-summary --reports-dir=coverage node scripts/run-tests.mjs",
|
|
67
|
+
"coverage:gaps": "node scripts/report-coverage-gaps.mjs",
|
|
60
68
|
"smoke:word": "powershell -File scripts/word-com-smoke.ps1",
|
|
61
69
|
"smoke:word:diff": "powershell -File scripts/word-com-differential.ps1",
|
|
70
|
+
"test:word": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/word-com-suite.ps1",
|
|
71
|
+
"test:word:visual": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/word-com-visual-suite.ps1",
|
|
72
|
+
"test:corpus:word": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/word-com-corpus-suite.ps1",
|
|
73
|
+
"test:corpus:word:visual": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/word-com-corpus-visual-suite.ps1",
|
|
74
|
+
"test:visual:inspect": "node scripts/inspect-visual-evidence.mjs",
|
|
75
|
+
"test:visual:sample": "node scripts/sample-multimodal-visual-check.mjs",
|
|
76
|
+
"corpus:fetch:superdoc": "node scripts/fetch-superdoc-corpus.mjs",
|
|
77
|
+
"report:word:coverage": "node scripts/report-word-coverage.mjs",
|
|
78
|
+
"report:test:dashboard": "node scripts/build-test-dashboard.mjs",
|
|
79
|
+
"review:word:prepare": "node scripts/prepare-word-review.mjs",
|
|
62
80
|
"test": "node scripts/run-tests.mjs",
|
|
63
81
|
"test:isolation": "node tests/no_word_api_index_check.mjs && node tests/core_dependency_graph_check.mjs",
|
|
64
82
|
"prepublishOnly": "npm run test:isolation && npm run build"
|
|
@@ -81,4 +99,4 @@
|
|
|
81
99
|
"engines": {
|
|
82
100
|
"node": ">=20.0.0"
|
|
83
101
|
}
|
|
84
|
-
}
|
|
102
|
+
}
|
package/pipeline/diff-engine.js
CHANGED
|
@@ -7,7 +7,54 @@
|
|
|
7
7
|
import { diff_match_patch } from 'diff-match-patch';
|
|
8
8
|
import { DiffOp } from '../core/types.js';
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
export const TOKEN_CODE_POINT_BASE = 0x10000;
|
|
11
|
+
export const MAX_DIFF_TOKENS = 0x40000;
|
|
12
|
+
|
|
13
|
+
// diff-match-patch's JavaScript port operates on UTF-16 code units, not
|
|
14
|
+
// Unicode code points. Keep its private encoding in the BMP while skipping
|
|
15
|
+
// surrogates; larger token sets take the deterministic token-array fallback.
|
|
16
|
+
const BMP_FIRST_CODE = 1;
|
|
17
|
+
const BMP_BEFORE_SURROGATES = 0xD800 - BMP_FIRST_CODE;
|
|
18
|
+
const BMP_AFTER_SURROGATES = 0x10000 - 0xE000;
|
|
19
|
+
const DMP_SAFE_TOKEN_LIMIT = BMP_BEFORE_SURROGATES + BMP_AFTER_SURROGATES;
|
|
20
|
+
|
|
21
|
+
export class DiffTokenLimitError extends Error {
|
|
22
|
+
constructor(limit = MAX_DIFF_TOKENS) {
|
|
23
|
+
super(`Word diff exceeds the safe limit of ${limit} unique tokens.`);
|
|
24
|
+
this.name = 'DiffTokenLimitError';
|
|
25
|
+
this.code = 'DIFF_TOKEN_LIMIT';
|
|
26
|
+
this.limit = limit;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function isDiffTokenLimitError(error) {
|
|
31
|
+
return error?.code === 'DIFF_TOKEN_LIMIT';
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function createDiffEngine(options = {}) {
|
|
35
|
+
const timeout = options.diffTimeoutSeconds ?? 0;
|
|
36
|
+
if (!Number.isFinite(timeout) || timeout < 0) {
|
|
37
|
+
throw new TypeError('diffTimeoutSeconds must be a finite non-negative number.');
|
|
38
|
+
}
|
|
39
|
+
const engine = new diff_match_patch();
|
|
40
|
+
engine.Diff_Timeout = timeout;
|
|
41
|
+
return engine;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function tokenize(text) {
|
|
45
|
+
const tokens = [];
|
|
46
|
+
const leading = text.match(/^\s+/);
|
|
47
|
+
if (leading) tokens.push(leading[0]);
|
|
48
|
+
|
|
49
|
+
const regex = /(\S+)(\s*)/g;
|
|
50
|
+
regex.lastIndex = leading?.[0].length || 0;
|
|
51
|
+
let match;
|
|
52
|
+
while ((match = regex.exec(text)) !== null) {
|
|
53
|
+
if (match[1]) tokens.push(match[1]);
|
|
54
|
+
if (match[2]) tokens.push(match[2]);
|
|
55
|
+
}
|
|
56
|
+
return tokens;
|
|
57
|
+
}
|
|
11
58
|
|
|
12
59
|
/**
|
|
13
60
|
* Converts text into word tokens represented as unique characters.
|
|
@@ -15,47 +62,47 @@ const DMP = new diff_match_patch();
|
|
|
15
62
|
*
|
|
16
63
|
* @param {string} text1 - First text to tokenize
|
|
17
64
|
* @param {string} text2 - Second text to tokenize
|
|
18
|
-
* @
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
};
|
|
58
|
-
}
|
|
65
|
+
* @param {{ maxTokens?: number }} [options={}] - Internal/test capacity override
|
|
66
|
+
* @returns {{ chars1: string, chars2: string, wordArray: string[], tokenIds1: number[], tokenIds2: number[] }}
|
|
67
|
+
*/
|
|
68
|
+
export function wordsToChars(text1, text2, options = {}) {
|
|
69
|
+
const wordArray = [];
|
|
70
|
+
const wordHash = new Map();
|
|
71
|
+
const maxTokens = options.maxTokens ?? MAX_DIFF_TOKENS;
|
|
72
|
+
if (!Number.isInteger(maxTokens) || maxTokens < 1 || maxTokens > MAX_DIFF_TOKENS) {
|
|
73
|
+
throw new RangeError(`maxTokens must be an integer from 1 to ${MAX_DIFF_TOKENS}.`);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function mapTokensToChars(tokens) {
|
|
77
|
+
let chars = '';
|
|
78
|
+
const tokenIds = [];
|
|
79
|
+
for (const token of tokens) {
|
|
80
|
+
let tokenId = wordHash.get(token);
|
|
81
|
+
if (tokenId === undefined) {
|
|
82
|
+
if (wordArray.length >= maxTokens) throw new DiffTokenLimitError(maxTokens);
|
|
83
|
+
tokenId = wordArray.length;
|
|
84
|
+
wordArray.push(token);
|
|
85
|
+
wordHash.set(token, tokenId);
|
|
86
|
+
}
|
|
87
|
+
tokenIds.push(tokenId);
|
|
88
|
+
chars += String.fromCodePoint(TOKEN_CODE_POINT_BASE + tokenId);
|
|
89
|
+
}
|
|
90
|
+
return { chars, tokenIds };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const tokens1 = tokenize(text1);
|
|
94
|
+
const tokens2 = tokenize(text2);
|
|
95
|
+
const encoded1 = mapTokensToChars(tokens1);
|
|
96
|
+
const encoded2 = mapTokensToChars(tokens2);
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
chars1: encoded1.chars,
|
|
100
|
+
chars2: encoded2.chars,
|
|
101
|
+
wordArray,
|
|
102
|
+
tokenIds1: encoded1.tokenIds,
|
|
103
|
+
tokenIds2: encoded2.tokenIds
|
|
104
|
+
};
|
|
105
|
+
}
|
|
59
106
|
|
|
60
107
|
/**
|
|
61
108
|
* Converts character-encoded diffs back to actual word diffs.
|
|
@@ -65,20 +112,87 @@ export function wordsToChars(text1, text2) {
|
|
|
65
112
|
* @returns {Array} DMP-style diff array with actual words
|
|
66
113
|
*/
|
|
67
114
|
export function charsToWords(diffs, wordArray) {
|
|
68
|
-
const
|
|
115
|
+
const originalChars = diffs.filter(([op]) => op !== 1).map(([, chars]) => chars).join('');
|
|
116
|
+
const modifiedChars = diffs.filter(([op]) => op !== -1).map(([, chars]) => chars).join('');
|
|
117
|
+
|
|
118
|
+
const decode = chars => {
|
|
119
|
+
const tokenIds = [];
|
|
120
|
+
for (let i = 0; i < chars.length;) {
|
|
121
|
+
const codePoint = chars.codePointAt(i);
|
|
122
|
+
const tokenId = codePoint - TOKEN_CODE_POINT_BASE;
|
|
123
|
+
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= wordArray.length) {
|
|
124
|
+
throw new RangeError(`Diff token code point U+${codePoint.toString(16).toUpperCase()} has no mapping.`);
|
|
125
|
+
}
|
|
126
|
+
tokenIds.push(tokenId);
|
|
127
|
+
i += codePoint > 0xFFFF ? 2 : 1;
|
|
128
|
+
}
|
|
129
|
+
return tokenIds;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
// Reconstruct both encoded sides before decoding. A UTF-16-based diff
|
|
133
|
+
// engine may split a surrogate pair across adjacent diff tuples even
|
|
134
|
+
// though the complete original/modified streams remain valid.
|
|
135
|
+
return deterministicLargeTokenDiff(decode(originalChars), decode(modifiedChars), wordArray);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function tokenIdToBmpChar(tokenId) {
|
|
139
|
+
const code = tokenId < BMP_BEFORE_SURROGATES
|
|
140
|
+
? BMP_FIRST_CODE + tokenId
|
|
141
|
+
: 0xE000 + (tokenId - BMP_BEFORE_SURROGATES);
|
|
142
|
+
return String.fromCharCode(code);
|
|
143
|
+
}
|
|
69
144
|
|
|
70
|
-
|
|
145
|
+
function bmpCharToTokenId(charCode) {
|
|
146
|
+
if (charCode >= BMP_FIRST_CODE && charCode < 0xD800) return charCode - BMP_FIRST_CODE;
|
|
147
|
+
if (charCode >= 0xE000 && charCode <= 0xFFFF) return BMP_BEFORE_SURROGATES + charCode - 0xE000;
|
|
148
|
+
throw new RangeError(`BMP diff token U+${charCode.toString(16).toUpperCase()} has no mapping.`);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function encodeTokenIdsForDmp(tokenIds) {
|
|
152
|
+
let chars = '';
|
|
153
|
+
for (const tokenId of tokenIds) chars += tokenIdToBmpChar(tokenId);
|
|
154
|
+
return chars;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function decodeBmpDiffs(diffs, wordArray) {
|
|
158
|
+
return diffs.map(([op, chars]) => {
|
|
71
159
|
const parts = [];
|
|
72
|
-
for (let
|
|
73
|
-
const
|
|
74
|
-
if (
|
|
75
|
-
|
|
160
|
+
for (let index = 0; index < chars.length; index++) {
|
|
161
|
+
const tokenId = bmpCharToTokenId(chars.charCodeAt(index));
|
|
162
|
+
if (tokenId >= wordArray.length) {
|
|
163
|
+
throw new RangeError(`BMP diff token ${tokenId} has no mapping.`);
|
|
76
164
|
}
|
|
165
|
+
parts.push(wordArray[tokenId]);
|
|
77
166
|
}
|
|
78
|
-
|
|
167
|
+
return [op, parts.join('')];
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function deterministicLargeTokenDiff(tokenIds1, tokenIds2, wordArray) {
|
|
172
|
+
let prefixLength = 0;
|
|
173
|
+
const sharedLength = Math.min(tokenIds1.length, tokenIds2.length);
|
|
174
|
+
while (prefixLength < sharedLength && tokenIds1[prefixLength] === tokenIds2[prefixLength]) {
|
|
175
|
+
prefixLength++;
|
|
79
176
|
}
|
|
80
177
|
|
|
81
|
-
|
|
178
|
+
let suffixLength = 0;
|
|
179
|
+
while (
|
|
180
|
+
suffixLength < sharedLength - prefixLength
|
|
181
|
+
&& tokenIds1[tokenIds1.length - 1 - suffixLength] === tokenIds2[tokenIds2.length - 1 - suffixLength]
|
|
182
|
+
) {
|
|
183
|
+
suffixLength++;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const joinTokens = ids => ids.map(id => wordArray[id]).join('');
|
|
187
|
+
const diffs = [];
|
|
188
|
+
if (prefixLength) diffs.push([0, joinTokens(tokenIds1.slice(0, prefixLength))]);
|
|
189
|
+
|
|
190
|
+
const deleted = tokenIds1.slice(prefixLength, tokenIds1.length - suffixLength);
|
|
191
|
+
const inserted = tokenIds2.slice(prefixLength, tokenIds2.length - suffixLength);
|
|
192
|
+
if (deleted.length) diffs.push([-1, joinTokens(deleted)]);
|
|
193
|
+
if (inserted.length) diffs.push([1, joinTokens(inserted)]);
|
|
194
|
+
if (suffixLength) diffs.push([0, joinTokens(tokenIds1.slice(tokenIds1.length - suffixLength))]);
|
|
195
|
+
return diffs;
|
|
82
196
|
}
|
|
83
197
|
|
|
84
198
|
/**
|
|
@@ -86,7 +200,7 @@ export function charsToWords(diffs, wordArray) {
|
|
|
86
200
|
*
|
|
87
201
|
* @param {string} originalText - Original text
|
|
88
202
|
* @param {string} newText - New text
|
|
89
|
-
* @param {{ cleanupSemantic?: boolean }} [options={}] - Diff options
|
|
203
|
+
* @param {{ cleanupSemantic?: boolean, diffTimeoutSeconds?: number, maxTokens?: number }} [options={}] - Diff options
|
|
90
204
|
* @returns {Array<[number, string]>}
|
|
91
205
|
*/
|
|
92
206
|
export function computeWordDiffs(originalText, newText, options = {}) {
|
|
@@ -104,13 +218,18 @@ export function computeWordDiffs(originalText, newText, options = {}) {
|
|
|
104
218
|
|
|
105
219
|
const { cleanupSemantic = true } = options;
|
|
106
220
|
|
|
107
|
-
const {
|
|
108
|
-
|
|
221
|
+
const { wordArray, tokenIds1, tokenIds2 } = wordsToChars(originalText, newText, options);
|
|
222
|
+
if (wordArray.length > DMP_SAFE_TOKEN_LIMIT) {
|
|
223
|
+
return deterministicLargeTokenDiff(tokenIds1, tokenIds2, wordArray);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const dmp = createDiffEngine(options);
|
|
227
|
+
const charDiffs = dmp.diff_main(encodeTokenIdsForDmp(tokenIds1), encodeTokenIdsForDmp(tokenIds2));
|
|
109
228
|
if (cleanupSemantic) {
|
|
110
|
-
|
|
229
|
+
dmp.diff_cleanupSemantic(charDiffs);
|
|
111
230
|
}
|
|
112
231
|
|
|
113
|
-
return
|
|
232
|
+
return decodeBmpDiffs(charDiffs, wordArray);
|
|
114
233
|
}
|
|
115
234
|
|
|
116
235
|
/**
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* - Run-level bold/italic formatting
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { parseOoxmlSafe } from '../adapters/xml-adapter.js';
|
|
12
12
|
import { NS_W } from '../core/types.js';
|
|
13
13
|
import { getXmlParseError } from '../core/xml-query.js';
|
|
14
14
|
|
|
@@ -19,21 +19,9 @@ function hasParserError(doc) {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
function parseWordOoxml(ooxml) {
|
|
22
|
-
const
|
|
23
|
-
if (
|
|
24
|
-
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
try {
|
|
28
|
-
const parser = createParser();
|
|
29
|
-
const doc = parser.parseFromString(source, 'application/xml');
|
|
30
|
-
if (hasParserError(doc)) {
|
|
31
|
-
return null;
|
|
32
|
-
}
|
|
33
|
-
return doc;
|
|
34
|
-
} catch {
|
|
35
|
-
return null;
|
|
36
|
-
}
|
|
22
|
+
const parsed = parseOoxmlSafe(ooxml, 'application/xml');
|
|
23
|
+
if (parsed.error || hasParserError(parsed.doc)) return { ...parsed, doc: null };
|
|
24
|
+
return parsed;
|
|
37
25
|
}
|
|
38
26
|
|
|
39
27
|
function getWordParagraphs(doc) {
|
|
@@ -223,17 +211,31 @@ function paragraphToMarkdown(paragraph) {
|
|
|
223
211
|
* @returns {string}
|
|
224
212
|
*/
|
|
225
213
|
export function ingestWordOoxmlToPlainText(ooxml) {
|
|
226
|
-
|
|
227
|
-
|
|
214
|
+
return ingestWordOoxmlToPlainTextResult(ooxml).text;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Result-returning plain-text ingestion for callers that need parse failures
|
|
219
|
+
* distinguished from legitimately empty documents.
|
|
220
|
+
*
|
|
221
|
+
* @param {unknown} ooxml
|
|
222
|
+
* @returns {{ text: string, status: 'ok'|'error', error?: {code:string,message:string}, warnings?: string[] }}
|
|
223
|
+
*/
|
|
224
|
+
export function ingestWordOoxmlToPlainTextResult(ooxml) {
|
|
225
|
+
const parsed = parseWordOoxml(ooxml);
|
|
226
|
+
if (!parsed.doc) {
|
|
227
|
+
return { text: '', status: 'error', error: parsed.error, warnings: parsed.warnings };
|
|
228
|
+
}
|
|
229
|
+
const doc = parsed.doc;
|
|
228
230
|
|
|
229
231
|
const paragraphs = getWordParagraphs(doc);
|
|
230
232
|
if (paragraphs.length === 0) {
|
|
231
233
|
const fallback = normalizeInlineWhitespace(doc.documentElement?.textContent || '');
|
|
232
|
-
return fallback;
|
|
234
|
+
return { text: fallback, status: 'ok', warnings: parsed.warnings };
|
|
233
235
|
}
|
|
234
236
|
|
|
235
237
|
const lines = paragraphs.map(paragraphToPlainText);
|
|
236
|
-
return lines.join('\n\n').trim();
|
|
238
|
+
return { text: lines.join('\n\n').trim(), status: 'ok', warnings: parsed.warnings };
|
|
237
239
|
}
|
|
238
240
|
|
|
239
241
|
/**
|
|
@@ -243,14 +245,27 @@ export function ingestWordOoxmlToPlainText(ooxml) {
|
|
|
243
245
|
* @returns {string}
|
|
244
246
|
*/
|
|
245
247
|
export function ingestWordOoxmlToMarkdown(ooxml) {
|
|
246
|
-
|
|
247
|
-
|
|
248
|
+
return ingestWordOoxmlToMarkdownResult(ooxml).text;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Result-returning markdown ingestion counterpart.
|
|
253
|
+
*
|
|
254
|
+
* @param {unknown} ooxml
|
|
255
|
+
* @returns {{ text: string, status: 'ok'|'error', error?: {code:string,message:string}, warnings?: string[] }}
|
|
256
|
+
*/
|
|
257
|
+
export function ingestWordOoxmlToMarkdownResult(ooxml) {
|
|
258
|
+
const parsed = parseWordOoxml(ooxml);
|
|
259
|
+
if (!parsed.doc) {
|
|
260
|
+
return { text: '', status: 'error', error: parsed.error, warnings: parsed.warnings };
|
|
261
|
+
}
|
|
262
|
+
const doc = parsed.doc;
|
|
248
263
|
|
|
249
264
|
const paragraphs = getWordParagraphs(doc);
|
|
250
265
|
if (paragraphs.length === 0) {
|
|
251
|
-
return '';
|
|
266
|
+
return { text: '', status: 'ok', warnings: parsed.warnings };
|
|
252
267
|
}
|
|
253
268
|
|
|
254
269
|
const lines = paragraphs.map(paragraphToMarkdown);
|
|
255
|
-
return lines.join('\n\n').trim();
|
|
270
|
+
return { text: lines.join('\n\n').trim(), status: 'ok', warnings: parsed.warnings };
|
|
256
271
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import { NS_W, RunKind, ContainerKind } from '../core/types.js';
|
|
8
8
|
import { appendParagraphBoundary, advanceOffsetForParagraphBoundary } from '../core/paragraph-offset-policy.js';
|
|
9
|
-
import {
|
|
9
|
+
import { parseOoxmlSafe, serializeXml } from '../adapters/xml-adapter.js';
|
|
10
10
|
import { warn, error as logError } from '../adapters/logger.js';
|
|
11
11
|
import {
|
|
12
12
|
getElementsByTagNS,
|
|
@@ -35,10 +35,12 @@ export function ingestOoxml(ooxmlString, options = {}) {
|
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
try {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
const parsed = preParsedDoc ? { doc: preParsedDoc, error: null } : parseOoxmlSafe(ooxmlString, 'application/xml');
|
|
39
|
+
const doc = parsed.doc;
|
|
40
|
+
if (parsed.error || !doc) {
|
|
41
|
+
logError('OOXML parse error:', parsed.error?.message);
|
|
42
|
+
return { runModel, acceptedText, pPr: null, error: parsed.error };
|
|
43
|
+
}
|
|
42
44
|
|
|
43
45
|
const parseError = getXmlParseError(doc);
|
|
44
46
|
if (parseError) {
|
|
@@ -30,9 +30,10 @@ export async function executeListGeneration(options) {
|
|
|
30
30
|
originalRunModel = [],
|
|
31
31
|
originalText = '',
|
|
32
32
|
generateRedlines = true,
|
|
33
|
-
author = 'AI',
|
|
34
|
-
font = null,
|
|
35
|
-
|
|
33
|
+
author = 'AI',
|
|
34
|
+
font = null,
|
|
35
|
+
revisionIdAllocator = null,
|
|
36
|
+
numberingService
|
|
36
37
|
} = options;
|
|
37
38
|
|
|
38
39
|
const normalizedListText = normalizeCompositeListMarkers(cleanText);
|
|
@@ -72,13 +73,18 @@ export async function executeListGeneration(options) {
|
|
|
72
73
|
if (tableData.headers.length > 0 || tableData.rows.length > 0) {
|
|
73
74
|
if (generateRedlines && results.length === 0 && deletionRuns.length > 0) {
|
|
74
75
|
results.push(serializeToOoxml(deletionRuns, null, [], {
|
|
75
|
-
author,
|
|
76
|
-
generateRedlines,
|
|
77
|
-
font
|
|
78
|
-
|
|
76
|
+
author,
|
|
77
|
+
generateRedlines,
|
|
78
|
+
font,
|
|
79
|
+
revisionIdAllocator
|
|
80
|
+
}));
|
|
79
81
|
}
|
|
80
82
|
|
|
81
|
-
results.push(generateTableOoxml(tableData, {
|
|
83
|
+
results.push(generateTableOoxml(tableData, {
|
|
84
|
+
generateRedlines,
|
|
85
|
+
author,
|
|
86
|
+
revisionIdAllocator
|
|
87
|
+
}));
|
|
82
88
|
i = tableBlock.endIndex;
|
|
83
89
|
continue;
|
|
84
90
|
}
|
|
@@ -92,9 +98,10 @@ export async function executeListGeneration(options) {
|
|
|
92
98
|
numberingContext,
|
|
93
99
|
numberingService,
|
|
94
100
|
generateRedlines,
|
|
95
|
-
author,
|
|
96
|
-
font,
|
|
97
|
-
|
|
101
|
+
author,
|
|
102
|
+
font,
|
|
103
|
+
revisionIdAllocator,
|
|
104
|
+
deletionRuns
|
|
98
105
|
);
|
|
99
106
|
results.push(entry.ooxml);
|
|
100
107
|
}
|
|
@@ -226,9 +233,10 @@ function buildListEntry(
|
|
|
226
233
|
numberingContext,
|
|
227
234
|
numberingService,
|
|
228
235
|
generateRedlines,
|
|
229
|
-
author,
|
|
230
|
-
font,
|
|
231
|
-
|
|
236
|
+
author,
|
|
237
|
+
font,
|
|
238
|
+
revisionIdAllocator,
|
|
239
|
+
deletionRuns
|
|
232
240
|
) {
|
|
233
241
|
let pPrXml = '';
|
|
234
242
|
let segmentText = '';
|
|
@@ -272,9 +280,10 @@ function buildListEntry(
|
|
|
272
280
|
|
|
273
281
|
return {
|
|
274
282
|
ooxml: serializeToOoxml(runModel, pPrXml, formatHints, {
|
|
275
|
-
author,
|
|
276
|
-
generateRedlines,
|
|
277
|
-
font
|
|
278
|
-
|
|
283
|
+
author,
|
|
284
|
+
generateRedlines,
|
|
285
|
+
font,
|
|
286
|
+
revisionIdAllocator
|
|
287
|
+
})
|
|
279
288
|
};
|
|
280
289
|
}
|