@ansonlai/docx-redline-js 0.2.0 → 0.4.0

package/scripts/lib/word-coverage-catalogue.mjs

@@ -0,0 +1,207 @@
+import { readFileSync } from 'fs';
+
+import { WORD_TASK_CASES } from '../../tests/fixtures/word-task-cases.mjs';
+import {
+    COVERAGE_ORACLES,
+    COVERAGE_STRUCTURES,
+    COVERAGE_TASKS,
+    sortCoverageMetadata,
+    validateCoverageMetadata
+} from './word-coverage-metadata.mjs';
+
+const corpusCatalogue = JSON.parse(readFileSync(
+    new URL('../../tests/corpus/superdoc-word-scenarios.json', import.meta.url),
+    'utf8'
+));
+const corpusManifest = JSON.parse(readFileSync(
+    new URL('../../tests/corpus/superdoc-english-legal-administrative.json', import.meta.url),
+    'utf8'
+));
+const corpusCoverage = JSON.parse(readFileSync(
+    new URL('../../tests/corpus/superdoc-word-coverage.json', import.meta.url),
+    'utf8'
+));
+const priorities = JSON.parse(readFileSync(
+    new URL('../../tests/fixtures/coverage-matrix-priorities.json', import.meta.url),
+    'utf8'
+));
+
+const manifestById = new Map(corpusManifest.documents.map(document => [document.id, document]));
+
+export function validateScenarioIdentities(cases) {
+    const identities = new Set();
+    for (const item of cases) {
+        if (!item.identity || identities.has(item.identity)) {
+            throw new Error(`Duplicate or missing scenario identity: ${item.identity || '<missing>'}`);
+        }
+        identities.add(item.identity);
+    }
+}
+
+function elementCount(xml, localName) {
+    return (xml.match(new RegExp(`<w:${localName}(?:\\s|/|>)`, 'g')) || []).length;
+}
+
+function assertSyntheticClaim(testCase, structure) {
+    const xml = testCase.sourceDocumentXml || '';
+    const rules = {
+        'plain-paragraph': () => !testCase.sourceText?.includes('\n') && !testCase.original.includes('\n'),
+        'multi-paragraph': () => elementCount(xml, 'p') > 1 || testCase.sourceText?.includes('\n')
+            || testCase.original.includes('\n') || testCase.modified.includes('\n'),
+        'formatted-runs': () => /<(?:w:)?(?:b|i|u)(?:\s|\/|>)/.test(xml) || /(?:\*\*|\*|\+\+)/.test(testCase.modified),
+        list: () => elementCount(xml, 'numPr') > 0,
+        table: () => elementCount(xml, 'tbl') > 0,
+        bookmark: () => elementCount(xml, 'bookmarkStart') > 0 && elementCount(xml, 'bookmarkEnd') > 0,
+        hyperlink: () => elementCount(xml, 'hyperlink') > 0,
+        'content-control': () => elementCount(xml, 'sdt') > 0,
+        'tab-break': () => elementCount(xml, 'tab') > 0 || elementCount(xml, 'br') > 0,
+        field: () => elementCount(xml, 'fldChar') > 0,
+        comment: () => elementCount(xml, 'commentRangeStart') > 0 && Boolean(testCase.packageParts?.commentsXml),
+        note: () => (elementCount(xml, 'footnoteReference') > 0 && Boolean(testCase.packageParts?.footnotesXml))
+            || (elementCount(xml, 'endnoteReference') > 0 && Boolean(testCase.packageParts?.endnotesXml)),
+        'header-footer': () => (elementCount(xml, 'headerReference') > 0 && Boolean(testCase.packageParts?.headers))
+            || (elementCount(xml, 'footerReference') > 0 && Boolean(testCase.packageParts?.footers)),
+        'section-boundary': () => elementCount(xml, 'sectPr') > 0,
+        'prior-revisions': () => elementCount(xml, 'ins') > 0 || elementCount(xml, 'del') > 0
+    };
+    if (!rules[structure]()) {
+        throw new Error(`synthetic:${testCase.name}: ${structure} is not supported by the fixture`);
+    }
+}
+
+function syntheticTask(testCase) {
+    if (testCase.operation?.type === 'list-change') return 'list-change';
+    if (testCase.operation?.type === 'table-reconciliation') return 'table-reconciliation';
+    if (testCase.expectAtomicRollback) return 'mixed-batch';
+    if (testCase.expectNoOp) return 'accept-reject';
+    if (testCase.task.startsWith('apply-')) return 'format';
+    if (testCase.task.includes('deletion') || testCase.task.startsWith('delete-')) return 'delete';
+    if (testCase.task.includes('insertion') || testCase.task.startsWith('insert-')
+        || testCase.task === 'preserve-dollar-delimiters') return 'insert';
+    return 'replace';
+}
+
+function corpusStructures(scenario) {
+    const labels = new Set([scenario.shape, ...scenario.coverage]);
+    const structures = new Set();
+    if (scenario.shape === 'list') structures.add('list');
+    if (scenario.shape === 'table-form') structures.add('table');
+    if (scenario.shape === 'body-paragraph' || scenario.shape === 'administrative-layout' || scenario.shape === 'legal-apparatus') {
+        structures.add('plain-paragraph');
+    }
+    const mappings = [
+        [['table', 'tables', 'table-cell'], 'table'],
+        [['numbering', 'numbered-list'], 'list'],
+        [['bookmark', 'bookmarks', 'bookmark-adjacency'], 'bookmark'],
+        [['header', 'headers', 'footer', 'footers'], 'header-footer'],
+        [['footnotes-part'], 'note'],
+        [['tabs'], 'tab-break'],
+        [['field-adjacency'], 'field'],
+        [['multi-section', 'section-properties'], 'section-boundary'],
+        [['format-only', 'formatting', 'formatted-runs'], 'formatted-runs']
+    ];
+    for (const [sourceLabels, structure] of mappings) {
+        if (sourceLabels.some(label => labels.has(label))) structures.add(structure);
+    }
+    return [...structures];
+}
+
+function corpusScenarioKey(scenario) {
+    return scenario.key || scenario.id;
+}
+
+function corpusSourceId(scenario) {
+    return scenario.sourceId || scenario.id;
+}
+
+function corpusOperations(scenario) {
+    return scenario.operations || [scenario.operation];
+}
+
+export function loadCoverageCatalogue() {
+    const synthetic = WORD_TASK_CASES.map(testCase => {
+        const identity = `synthetic:${testCase.name}`;
+        const metadata = sortCoverageMetadata(validateCoverageMetadata(testCase.coverageMetadata, identity));
+        if (metadata.task !== syntheticTask(testCase)) {
+            throw new Error(`${identity}: task ${metadata.task} is unsupported by fixture task ${testCase.task}`);
+        }
+        for (const structure of metadata.structures) assertSyntheticClaim(testCase, structure);
+        return { identity, lane: 'synthetic', category: testCase.category, detail: testCase.task, metadata };
+    });
+
+    const corpus = corpusCatalogue.scenarios.map(scenario => {
+        const scenarioKey = corpusScenarioKey(scenario);
+        const sourceId = corpusSourceId(scenario);
+        const identity = `superdoc:${scenarioKey}`;
+        const source = manifestById.get(sourceId);
+        if (!source) throw new Error(`${identity}: source is absent from the pinned corpus manifest`);
+        const operations = corpusOperations(scenario);
+        const supportedOperationTypes = new Set(['replace', 'format', 'list-change', 'table-reconciliation', 'insert', 'delete']);
+        for (const operation of operations) {
+            if (!supportedOperationTypes.has(operation?.type)) {
+                throw new Error(`${identity}: unsupported corpus operation type ${operation?.type}`);
+            }
+        }
+        const declaredStructures = corpusCoverage.structuresByScenario[scenarioKey];
+        const supportedStructures = corpusStructures(scenario).sort();
+        if (!declaredStructures || JSON.stringify([...declaredStructures].sort()) !== JSON.stringify(supportedStructures)) {
+            throw new Error(`${identity}: declared structures do not match reviewed fixture labels`);
+        }
+        const declaredTask = corpusCoverage.tasksByScenario?.[scenarioKey]
+            || (operations.length > 1 ? 'mixed-batch' : (operations[0]?.type || 'replace'));
+        const metadata = sortCoverageMetadata(validateCoverageMetadata({
+            ...corpusCoverage.defaults,
+            task: declaredTask,
+            structures: declaredStructures
+        }, identity));
+        const expectedTask = operations.length > 1 ? 'mixed-batch' : (operations[0]?.type || 'replace');
+        if (metadata.task !== expectedTask) {
+            throw new Error(`${identity}: task ${metadata.task} does not match operation type ${expectedTask}`);
+        }
+        return { identity, lane: 'superdoc', category: source.type, detail: scenario.shape, metadata };
+    });
+    const cases = [...synthetic, ...corpus];
+    if (Object.keys(corpusCoverage.structuresByScenario).length !== corpus.length) {
+        throw new Error('SuperDoc coverage metadata contains a missing or unknown scenario identity');
+    }
+    validateScenarioIdentities(cases);
+    return { cases, priorities };
+}
+
+export function buildCoverageMatrix(cases) {
+    const cells = new Map();
+    for (const task of COVERAGE_TASKS) {
+        for (const structure of COVERAGE_STRUCTURES) cells.set(`${task}/${structure}`, []);
+    }
+    for (const testCase of cases) {
+        for (const structure of testCase.metadata.structures) {
+            cells.get(`${testCase.metadata.task}/${structure}`).push(testCase.identity);
+        }
+    }
+    return cells;
+}
+
+export function validateCoveragePriorities(cases, priorityConfig = priorities) {
+    const cells = buildCoverageMatrix(cases);
+    const dispositions = new Map(priorityConfig.emptyCellDispositions.map(item => [`${item.task}/${item.structure}`, item]));
+    for (const item of [...priorityConfig.highPriorityCells, ...priorityConfig.emptyCellDispositions]) {
+        if (!COVERAGE_TASKS.includes(item.task) || !COVERAGE_STRUCTURES.includes(item.structure)) {
+            throw new Error(`Unknown priority cell: ${item.task}/${item.structure}`);
+        }
+    }
+    for (const cell of priorityConfig.highPriorityCells) {
+        const key = `${cell.task}/${cell.structure}`;
+        if (cells.get(key).length === 0 && !dispositions.has(key)) {
+            throw new Error(`High-priority empty cell lacks a disposition: ${key}`);
+        }
+    }
+    for (const [key, disposition] of dispositions) {
+        if (!['planned', 'excluded'].includes(disposition.status) || !disposition.reason || !disposition.dependency) {
+            throw new Error(`${key}: disposition requires status, reason, and dependency`);
+        }
+        if (cells.get(key).length > 0) throw new Error(`${key}: obsolete empty-cell disposition is still present`);
+    }
+    return cells;
+}
+
+export { COVERAGE_ORACLES, COVERAGE_STRUCTURES, COVERAGE_TASKS };

package/scripts/lib/word-coverage-metadata.mjs

@@ -0,0 +1,93 @@
+export const COVERAGE_TASKS = Object.freeze([
+    'replace',
+    'insert',
+    'delete',
+    'format',
+    'comment',
+    'accept-reject',
+    'list-change',
+    'table-reconciliation',
+    'mixed-batch'
+]);
+
+export const COVERAGE_STRUCTURES = Object.freeze([
+    'plain-paragraph',
+    'multi-paragraph',
+    'formatted-runs',
+    'list',
+    'table',
+    'bookmark',
+    'hyperlink',
+    'content-control',
+    'tab-break',
+    'field',
+    'comment',
+    'note',
+    'header-footer',
+    'section-boundary',
+    'prior-revisions'
+]);
+
+export const COVERAGE_ORACLES = Object.freeze([
+    'js-exact-round-trip',
+    'runtime-validator',
+    'xsd',
+    'libreoffice',
+    'synthetic-word',
+    'real-document-word',
+    'ai-word-visual-preflight',
+    'human-word-visual-review'
+]);
+
+export const MANUAL_REVIEW_STATES = Object.freeze([
+    'missing',
+    'current',
+    'stale'
+]);
+
+const vocabularySets = {
+    task: new Set(COVERAGE_TASKS),
+    structures: new Set(COVERAGE_STRUCTURES),
+    oracles: new Set(COVERAGE_ORACLES)
+};
+
+export function validateCoverageMetadata(metadata, identity) {
+    if (!metadata || typeof metadata !== 'object') {
+        throw new Error(`${identity}: coverage metadata is missing`);
+    }
+    if (!vocabularySets.task.has(metadata.task)) {
+        throw new Error(`${identity}: unknown coverage task ${JSON.stringify(metadata.task)}`);
+    }
+    for (const field of ['structures', 'oracles']) {
+        if (!Array.isArray(metadata[field]) || metadata[field].length === 0) {
+            throw new Error(`${identity}: ${field} must be a non-empty array`);
+        }
+        if (new Set(metadata[field]).size !== metadata[field].length) {
+            throw new Error(`${identity}: ${field} contains duplicates`);
+        }
+        for (const label of metadata[field]) {
+            if (!vocabularySets[field].has(label)) {
+                throw new Error(`${identity}: unknown ${field} label ${JSON.stringify(label)}`);
+            }
+        }
+    }
+    if (!MANUAL_REVIEW_STATES.includes(metadata.manualReview?.status)) {
+        throw new Error(`${identity}: manualReview.status must be missing, current, or stale`);
+    }
+    if (metadata.manualReview.status !== 'missing' && !metadata.manualReview.reviewedAt) {
+        throw new Error(`${identity}: reviewedAt is required for a ${metadata.manualReview.status} review`);
+    }
+    return metadata;
+}
+
+export function sortCoverageMetadata(metadata) {
+    return {
+        ...metadata,
+        structures: [...metadata.structures].sort(
+            (a, b) => COVERAGE_STRUCTURES.indexOf(a) - COVERAGE_STRUCTURES.indexOf(b)
+        ),
+        oracles: [...metadata.oracles].sort(
+            (a, b) => COVERAGE_ORACLES.indexOf(a) - COVERAGE_ORACLES.indexOf(b)
+        )
+    };
+}

package/scripts/lib/zip-reader.mjs

@@ -0,0 +1,64 @@
+/**
+ * Small read-only ZIP helper for validation tooling.
+ *
+ * This intentionally stays outside the runtime package path. It supports the
+ * stored and deflated entries used by DOCX packages without adding a ZIP
+ * dependency to the library.
+ */
+
+import { inflateRawSync } from 'zlib';
+
+const EOCD_SIGNATURE = 0x06054b50;
+const CENTRAL_DIR_SIGNATURE = 0x02014b50;
+const LOCAL_FILE_SIGNATURE = 0x04034b50;
+
+function findEndOfCentralDirectory(zipBuffer) {
+    const minOffset = Math.max(0, zipBuffer.length - 0xffff - 22);
+    for (let offset = zipBuffer.length - 22; offset >= minOffset; offset -= 1) {
+        if (zipBuffer.readUInt32LE(offset) === EOCD_SIGNATURE) return offset;
+    }
+    throw new Error('Invalid zip: End of central directory not found');
+}
+
+function readEntryData(zipBuffer, localHeaderOffset, compressedSize, compressionMethod) {
+    if (zipBuffer.readUInt32LE(localHeaderOffset) !== LOCAL_FILE_SIGNATURE) {
+        throw new Error('Invalid zip: local file header signature mismatch');
+    }
+
+    const fileNameLength = zipBuffer.readUInt16LE(localHeaderOffset + 26);
+    const extraLength = zipBuffer.readUInt16LE(localHeaderOffset + 28);
+    const dataStart = localHeaderOffset + 30 + fileNameLength + extraLength;
+    const compressed = zipBuffer.subarray(dataStart, dataStart + compressedSize);
+
+    if (compressionMethod === 0) return Buffer.from(compressed);
+    if (compressionMethod === 8) return inflateRawSync(compressed);
+    throw new Error(`Unsupported compression method: ${compressionMethod}`);
+}
+
+/** @returns {Map<string, Buffer>} */
+export function unzipEntries(zipBuffer) {
+    const eocdOffset = findEndOfCentralDirectory(zipBuffer);
+    const totalEntries = zipBuffer.readUInt16LE(eocdOffset + 10);
+    const centralDirOffset = zipBuffer.readUInt32LE(eocdOffset + 16);
+    const entries = new Map();
+    let offset = centralDirOffset;
+
+    for (let index = 0; index < totalEntries; index += 1) {
+        if (zipBuffer.readUInt32LE(offset) !== CENTRAL_DIR_SIGNATURE) {
+            throw new Error('Invalid zip: central directory signature mismatch');
+        }
+
+        const compressionMethod = zipBuffer.readUInt16LE(offset + 10);
+        const compressedSize = zipBuffer.readUInt32LE(offset + 20);
+        const fileNameLength = zipBuffer.readUInt16LE(offset + 28);
+        const extraLength = zipBuffer.readUInt16LE(offset + 30);
+        const commentLength = zipBuffer.readUInt16LE(offset + 32);
+        const localHeaderOffset = zipBuffer.readUInt32LE(offset + 42);
+        const name = zipBuffer.toString('utf8', offset + 46, offset + 46 + fileNameLength);
+
+        entries.set(name, readEntryData(zipBuffer, localHeaderOffset, compressedSize, compressionMethod));
+        offset += 46 + fileNameLength + extraLength + commentLength;
+    }
+
+    return entries;
+}

package/scripts/package-superdoc-word-fixtures.ps1

@@ -0,0 +1,64 @@
+param(
+    [string]$FixturesDir = "tmp/superdoc-word-fixtures",
+    [string]$SourcesDir = "tmp/superdoc-corpus"
+)
+
+$ErrorActionPreference = 'Stop'
+Add-Type -AssemblyName System.IO.Compression
+Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+function Get-UntouchedPartHashes([string]$path, [string]$changedPart) {
+    $hashes = @{}
+    $zip = [IO.Compression.ZipFile]::OpenRead($path)
+    try {
+        foreach ($entry in $zip.Entries) {
+            if ($entry.FullName -eq $changedPart) { continue }
+            $stream = $entry.Open()
+            $sha = [Security.Cryptography.SHA256]::Create()
+            try {
+                $hashes[$entry.FullName] = [BitConverter]::ToString($sha.ComputeHash($stream)).Replace('-', '').ToLowerInvariant()
+            }
+            finally {
+                $sha.Dispose()
+                $stream.Dispose()
+            }
+        }
+    }
+    finally { $zip.Dispose() }
+    return $hashes
+}
+
+$resolvedFixtures = (Resolve-Path -LiteralPath $FixturesDir).Path
+$resolvedSources = (Resolve-Path -LiteralPath $SourcesDir).Path
+$suite = Get-Content -LiteralPath (Join-Path $resolvedFixtures 'suite.json') -Raw -Encoding UTF8 | ConvertFrom-Json
+
+foreach ($case in $suite.cases) {
+    $sourcePath = Join-Path $resolvedSources "$($case.sourceId).docx"
+    $outputPath = Join-Path $resolvedFixtures "$($case.name).docx"
+    $xmlPath = Join-Path $resolvedFixtures "$($case.name).document.xml"
+    $revisionPart = if ($case.revisionPart) { [string]$case.revisionPart } else { 'word/document.xml' }
+    $before = Get-UntouchedPartHashes $sourcePath $revisionPart
+
+    Copy-Item -LiteralPath $sourcePath -Destination $outputPath -Force
+    $zip = [IO.Compression.ZipFile]::Open($outputPath, [IO.Compression.ZipArchiveMode]::Update)
+    try {
+        $oldEntry = $zip.GetEntry($revisionPart)
+        if ($null -eq $oldEntry) { throw "Missing $revisionPart in $sourcePath" }
+        $oldEntry.Delete()
+        $newEntry = $zip.CreateEntry($revisionPart, [IO.Compression.CompressionLevel]::Optimal)
+        $stream = $newEntry.Open()
+        try {
+            $bytes = [Text.UTF8Encoding]::new($false).GetBytes((Get-Content -LiteralPath $xmlPath -Raw -Encoding UTF8))
+            $stream.Write($bytes, 0, $bytes.Length)
+        }
+        finally { $stream.Dispose() }
+    }
+    finally { $zip.Dispose() }
+
+    $after = Get-UntouchedPartHashes $outputPath $revisionPart
+    if ($before.Count -ne $after.Count) { throw "$($case.name): package part count changed" }
+    foreach ($name in $before.Keys) {
+        if ($after[$name] -ne $before[$name]) { throw "$($case.name): untouched package part changed: $name" }
+    }
+    Write-Output "Packaged $($case.name) (verified $($before.Count) untouched parts)"
+}

package/scripts/prepare-corpus-word-visual-review.mjs

@@ -0,0 +1,84 @@
+import { mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { dirname, join, resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+const manifest = JSON.parse(readFileSync(
+    new URL('../tests/corpus/superdoc-english-legal-administrative.json', import.meta.url),
+    'utf8'
+));
+const catalogue = JSON.parse(readFileSync(
+    new URL('../tests/corpus/superdoc-word-scenarios.json', import.meta.url),
+    'utf8'
+));
+const sourceById = new Map(manifest.documents.map(item => [item.id, item]));
+const VIEW_NAMES = Object.freeze(['allMarkup', 'acceptAll', 'rejectAll']);
+
+export function selectCorpusVisualReviewCases(requestedKeys = []) {
+    const requested = new Set(requestedKeys);
+    const cases = catalogue.scenarios.map((scenario, index) => {
+        const scenarioKey = scenario.key || scenario.id;
+        const sourceId = scenario.sourceId || scenario.id;
+        const source = sourceById.get(sourceId);
+        const caseToken = scenario.key || sourceId.slice(0, 12);
+        return {
+            scenarioKey,
+            name: `${String(index + 1).padStart(2, '0')}-${source.type}-${caseToken}`,
+            category: source.type,
+            shape: scenario.shape,
+            operationCount: (scenario.operations || [scenario.operation]).length
+        };
+    });
+    if (requested.size > 0) {
+        const known = new Set(cases.map(item => item.scenarioKey));
+        const unknown = [...requested].filter(key => !known.has(key));
+        if (unknown.length) throw new Error(`Unknown corpus visual-review case(s): ${unknown.join(', ')}`);
+        return cases.filter(item => requested.has(item.scenarioKey));
+    }
+    return cases.filter(item => !/^[a-f0-9]{64}$/.test(item.scenarioKey));
+}
+
+export function buildCorpusVisualReviewManifest(requestedKeys = []) {
+    const selected = selectCorpusVisualReviewCases(requestedKeys);
+    return {
+        schemaVersion: 1,
+        reviewType: 'render-only-corpus-visual-preflight',
+        certification: {
+            status: 'pending',
+            reviewer: null,
+            reviewedAt: null,
+            note: 'Rendering proves that Word produced inspectable evidence; it does not certify visual correctness.'
+        },
+        word: { version: null, build: null },
+        cases: selected.map(item => ({
+            identity: `superdoc:${item.scenarioKey}`,
+            ...item,
+            renderStatus: 'pending',
+            views: Object.fromEntries(VIEW_NAMES.map(view => [view, {
+                status: 'pending',
+                pdf: `${item.name}-${view}.pdf`,
+                pages: null,
+                bytes: null
+            }]))
+        }))
+    };
+}
+
+function parseArgs(argv) {
+    const requestedKeys = argv.filter(value => value.startsWith('--case=')).map(value => value.slice(7));
+    const outputArgument = argv.find(value => value.startsWith('--output='));
+    return {
+        requestedKeys,
+        outputPath: outputArgument
+            ? resolve(process.cwd(), outputArgument.slice(9))
+            : join(process.cwd(), 'tmp', 'superdoc-word-visual-review', 'manifest.json')
+    };
+}
+
+const isCli = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isCli) {
+    const { requestedKeys, outputPath } = parseArgs(process.argv.slice(2));
+    const review = buildCorpusVisualReviewManifest(requestedKeys);
+    mkdirSync(dirname(outputPath), { recursive: true });
+    writeFileSync(outputPath, `${JSON.stringify(review, null, 2)}\n`, 'utf8');
+    console.log(`Prepared ${review.cases.length} new SuperDoc scenarios for Word visual rendering: ${outputPath}`);
+}