@stll/folio-core 0.32.2 → 0.33.1

package/dist/compare/compare.js

@@ -0,0 +1,412 @@
+import { FolioDocxReviewer } from "../ai-edits/headless.js";
+import "../document-operations.js";
+import { pairFolioDocumentStories } from "../document-stories.js";
+import { planStoryCompare } from "./plan.js";
+import { withFixedPackageDates } from "./reproducible-package.js";
+import { CompareDocxApplyError, CompareDocxFinalParagraphMarkError, CompareDocxOperationLimitError, CompareDocxParseError, CompareDocxRoundTripError, CompareDocxSerializeError, InvalidCompareDocxOptionsError } from "./types.js";
+import { classifyProjectionMismatch, deletedFinalParagraphMarks, projectSupportedInlineFormatting } from "./verification.js";
+import { Result, panic } from "better-result";
+//#region src/compare/compare.ts
+/**
+* Deterministic `.docx` compare: two packages in, one redlined package plus a
+* JSON change list out.
+*
+* ## Determinism contract
+*
+* `compareDocx(base, target, options)` is a pure function of its three
+* arguments. It reads no clock and no randomness: revision dates come from
+* `options.timestamp`, and revision ids from a seed derived from the base
+* document's own highest existing revision id, so two runs over the same
+* inputs produce byte-identical buffers and deeply equal change lists.
+*
+* ## Round-trip contract
+*
+* Accepting every tracked change in the result yields the target's content;
+* rejecting every one yields the base's. Both directions are checked before
+* the call returns, and the verdict travels with the result as
+* `verification`. Anything the comparison does not cover is reported in
+* `unsupported`, or fails the call, rather than being silently dropped.
+*
+* An unproven redline is refused by default. `onUnverified: "emit"` returns it
+* anyway, with every invariant that did not hold named: a caller that would
+* rather show its best attempt and say what is missing can, and one that wants
+* a redline it can stand behind still gets nothing else.
+*
+* ## Stages
+*
+* The call is four named steps: {@link parseComparison},
+* {@link planComparison}, {@link applyComparison}, {@link serializeComparison}.
+* `compareDocx` is their composition and nothing else, so the benchmark can
+* time the stages separately without keeping a second copy of the pipeline
+* that would drift from this one.
+*
+* @packageDocumentation
+*/
+/**
+* Cap on operations one comparison generates. Both inputs are untrusted
+* documents, and each operation costs a document walk plus revision ids.
+*/
+const MAX_COMPARE_OPERATIONS = 1e4;
+const parseSide = async (buffer, side, author) => await Result.tryPromise({
+	try: async () => await FolioDocxReviewer.fromBuffer(buffer, { author }),
+	catch: (cause) => new CompareDocxParseError({
+		message: `The ${side} document could not be parsed.`,
+		side,
+		cause
+	})
+});
+/** Read before either side is resolved, so it describes the package as it arrived. */
+const existingRevisionsOf = (reviewer) => {
+	let highest = 0;
+	let present = false;
+	for (const { handle } of reviewer.listStories()) {
+		const story = reviewer.readReviewedStory({
+			story: handle,
+			view: "current-markup"
+		});
+		for (const change of story?.changes ?? []) {
+			highest = Math.max(highest, change.id);
+			present = true;
+		}
+	}
+	return {
+		idSeed: highest + 1,
+		present
+	};
+};
+/**
+* One story's text-and-structure projection: every block's text tagged with
+* the table cell it sits in. The tag is what makes the self-check below see a
+* paragraph that landed beside a table instead of inside it.
+*/
+const projectBlocks = (blocks) => {
+	return blocks.map(({ text, table, styleId, listLevel }) => {
+		return `${table ? `t${String(table.tableIndex)}r${String(table.rowIndex)}c${String(table.cellIndex)}g${String(table.gridColumnIndex)}x${String(table.columnSpan)}y${String(table.rowSpan)}p${String(table.paragraphIndex)}` : "body"}|${styleId ?? ""}|${listLevel ?? ""}|${text}`;
+	});
+};
+/** Verify formatting only where the plan claims a text-equal formatting change. */
+const formattingRoundTripFailure = ({ invariant, story, changes, actualBlocks, expectedBlocks, expectedBlockId }) => {
+	const expectedIndexById = new Map(expectedBlocks.map(({ id }, index) => [id, index]));
+	const checkedExpectedIds = /* @__PURE__ */ new Set();
+	for (const change of changes) {
+		if (change.kind !== "format") continue;
+		const expectedId = expectedBlockId(change);
+		if (checkedExpectedIds.has(expectedId)) continue;
+		checkedExpectedIds.add(expectedId);
+		const expectedIndex = expectedIndexById.get(expectedId) ?? -1;
+		const expected = expectedBlocks.at(expectedIndex);
+		const actual = actualBlocks.at(expectedIndex);
+		if (expectedIndex === -1 || !actual || !expected) return {
+			invariant,
+			cause: "inline-formatting",
+			story,
+			detail: "a text-equal aligned block could not be projected for formatting verification"
+		};
+		if (projectSupportedInlineFormatting(actual) !== projectSupportedInlineFormatting(expected)) return {
+			invariant,
+			cause: "inline-formatting",
+			story,
+			detail: "supported inline formatting differs in a text-equal aligned block"
+		};
+	}
+	return null;
+};
+const numberingKey = ({ numId, level }) => `${String(numId)}:${String(level)}`;
+const sameNumbering = (left, right) => left.format === right.format && left.levelText === right.levelText && left.start === right.start;
+const compareNumbering = (base, target) => {
+	const baseLevels = new Map(base.readNumberingDefinitions().map((level) => [numberingKey(level), level]));
+	const targetLevels = new Map(target.readNumberingDefinitions().map((level) => [numberingKey(level), level]));
+	const changes = [];
+	for (const [key, before] of baseLevels) {
+		const after = targetLevels.get(key) ?? null;
+		if (after === null || !sameNumbering(before, after)) changes.push({
+			kind: "numbering",
+			numId: before.numId,
+			level: before.level,
+			before,
+			after
+		});
+	}
+	for (const [key, after] of targetLevels) if (!baseLevels.has(key)) changes.push({
+		kind: "numbering",
+		numId: after.numId,
+		level: after.level,
+		before: null,
+		after
+	});
+	return changes.toSorted((left, right) => left.numId - right.numId || left.level - right.level);
+};
+/** Stage 1: both packages to editor models, paired story by story. */
+const parseComparison = async (base, target, options) => {
+	const packageDate = new Date(options.timestamp);
+	if (Number.isNaN(packageDate.getTime())) return Result.err(new InvalidCompareDocxOptionsError({
+		message: "timestamp must be a date the package can be stamped with.",
+		option: "timestamp",
+		receivedValue: options.timestamp
+	}));
+	const baseParse = await parseSide(base, "base", options.author);
+	if (baseParse.isErr()) return Result.err(baseParse.error);
+	const targetParse = await parseSide(target, "target", options.author);
+	if (targetParse.isErr()) return Result.err(targetParse.error);
+	const reviewer = baseParse.value;
+	const targetReviewer = targetParse.value;
+	const existing = existingRevisionsOf(reviewer);
+	const pairs = [];
+	const unsupported = [];
+	for (const { baseStory, revisedStory: targetStory } of pairFolioDocumentStories(reviewer.listStories().map(({ handle }) => handle), targetReviewer.listStories().map(({ handle }) => handle))) {
+		if (!baseStory) {
+			unsupported.push({
+				reason: "story-missing-in-base",
+				baseStory: null,
+				targetStory
+			});
+			continue;
+		}
+		if (!targetStory) {
+			unsupported.push({
+				reason: "story-missing-in-target",
+				baseStory,
+				targetStory: null
+			});
+			continue;
+		}
+		reviewer.resolveReviewedStory({
+			story: baseStory,
+			view: "final"
+		});
+		targetReviewer.resolveReviewedStory({
+			story: targetStory,
+			view: "final"
+		});
+		const baseSnapshot = reviewer.snapshotStory(baseStory);
+		const targetSnapshot = targetReviewer.snapshotStory(targetStory);
+		if (!baseSnapshot || !targetSnapshot) {
+			unsupported.push({
+				reason: "story-not-editable",
+				baseStory,
+				targetStory
+			});
+			continue;
+		}
+		pairs.push({
+			baseStory,
+			targetStory,
+			baseSnapshot,
+			targetSnapshot
+		});
+	}
+	return Result.ok({
+		granularity: options.granularity ?? "word",
+		baseBuffer: base,
+		baseCarriedRevisions: existing.present,
+		reviewer,
+		targetReviewer,
+		revisionStamp: {
+			date: options.timestamp,
+			idSeed: existing.idSeed
+		},
+		packageDate,
+		pairs,
+		numberingChanges: compareNumbering(reviewer, targetReviewer),
+		unsupported
+	});
+};
+/**
+* Stage 2: align every paired story and derive its operations. Pure — no
+* parsing, no serialization, no clock.
+*/
+const planComparison = ({ pairs }) => {
+	const planned = [];
+	for (const pair of pairs) {
+		const plan = planStoryCompare({
+			story: pair.baseStory,
+			baseSnapshot: pair.baseSnapshot,
+			targetSnapshot: pair.targetSnapshot,
+			maxOperations: MAX_COMPARE_OPERATIONS
+		});
+		if (plan === null) return Result.err(new CompareDocxOperationLimitError({
+			message: "The comparison needs more operations than the engine generates.",
+			limit: MAX_COMPARE_OPERATIONS
+		}));
+		planned.push({
+			pair,
+			plan
+		});
+	}
+	return Result.ok(planned);
+};
+/**
+* Stage 3: write the planned operations into the base document as tracked
+* changes, then check the work rather than trust it. Both directions of the
+* round trip are checked, structure included: accepting the story's generated
+* revisions must reproduce the target, and rejecting them must reproduce the
+* base it was compared from. A difference the operation vocabulary cannot
+* express would otherwise leave a redline that reads plausibly and is wrong.
+*
+* The check reports rather than throws. {@link compareDocx} decides what to do
+* with an unverified result, because "give me your best attempt and tell me
+* what you could not represent" and "give me nothing unless you can prove it"
+* are both legitimate asks and only the caller knows which one it is making.
+*/
+const applyComparison = ({ reviewer, revisionStamp, granularity, numberingChanges }, planned) => {
+	const changes = [...numberingChanges];
+	const failures = [];
+	let idSeed = revisionStamp.idSeed;
+	for (const { pair, plan } of planned) {
+		changes.push(...plan.changes);
+		if (plan.operations.length === 0) continue;
+		const baseBeforeBlocks = reviewer.readReviewedStory({
+			story: pair.baseStory,
+			view: "final"
+		})?.snapshot.blocks ?? [];
+		const baseBefore = projectBlocks(baseBeforeBlocks);
+		const { skipped, nextRevisionId } = reviewer.applyDocumentOperationsToStory({
+			story: pair.baseStory,
+			snapshot: pair.baseSnapshot,
+			revisionStamp: {
+				date: revisionStamp.date,
+				idSeed
+			},
+			wordDiff: { granularity },
+			batch: {
+				version: 1,
+				mode: "tracked-changes",
+				operations: plan.operations
+			}
+		});
+		if (nextRevisionId === void 0) panic("The applier did not report where it left the revision-id counter", { story: pair.baseStory });
+		idSeed = nextRevisionId;
+		if (skipped.length > 0) return Result.err(new CompareDocxApplyError({
+			message: "Some derived operations were refused, so the result would not match the target.",
+			skipped
+		}));
+		const acceptedStory = reviewer.readReviewedStory({
+			story: pair.baseStory,
+			view: "final"
+		});
+		const acceptFailure = classifyProjectionMismatch({
+			invariant: "accept-reproduces-target",
+			story: pair.baseStory,
+			actual: projectBlocks(acceptedStory?.snapshot.blocks ?? []),
+			expected: projectBlocks(pair.targetSnapshot.blocks)
+		});
+		if (acceptFailure) failures.push(acceptFailure);
+		else {
+			const formattingFailure = formattingRoundTripFailure({
+				invariant: "accept-reproduces-target",
+				story: pair.baseStory,
+				changes: plan.changes,
+				actualBlocks: acceptedStory?.snapshot.blocks ?? [],
+				expectedBlocks: pair.targetSnapshot.blocks,
+				expectedBlockId: ({ targetBlockId }) => targetBlockId
+			});
+			if (formattingFailure) failures.push(formattingFailure);
+		}
+		const rejectedStory = reviewer.readReviewedStory({
+			story: pair.baseStory,
+			view: "original"
+		});
+		const rejectFailure = classifyProjectionMismatch({
+			invariant: "reject-reproduces-base",
+			story: pair.baseStory,
+			actual: projectBlocks(rejectedStory?.snapshot.blocks ?? []),
+			expected: baseBefore
+		});
+		if (rejectFailure) failures.push(rejectFailure);
+		else {
+			const formattingFailure = formattingRoundTripFailure({
+				invariant: "reject-reproduces-base",
+				story: pair.baseStory,
+				changes: plan.changes,
+				actualBlocks: rejectedStory?.snapshot.blocks ?? [],
+				expectedBlocks: pair.baseSnapshot.blocks,
+				expectedBlockId: ({ baseBlockId }) => baseBlockId
+			});
+			if (formattingFailure) failures.push(formattingFailure);
+		}
+	}
+	return Result.ok({
+		changes,
+		verification: failures.length === 0 ? { status: "verified" } : {
+			status: "unverified",
+			failures
+		}
+	});
+};
+/**
+* Stage 4: the result package, with every ZIP entry date pinned.
+*
+* A comparison that found nothing returns the base bytes as they arrived. A
+* change is only ever reported alongside the operations that realize it, so no
+* operations means no changes, and re-serializing then rewrites a document
+* nobody edited: on a 2,200-block pair that was a second of work to reproduce
+* the input.
+*
+* Unless the base carried revisions of its own. Then the compared base was its
+* accepted view, the arriving bytes are a different document, and handing them
+* back would make rejecting the result land before the previous reviewer's
+* edits rather than after them.
+*
+* The short-circuit lives here rather than in {@link compareDocx} so that
+* every caller of the stages sees the same decision. Putting it in the
+* composition let the benchmark's own composition disagree with the shipped
+* one within a single run.
+*/
+const serializeComparison = async ({ baseBuffer, baseCarriedRevisions, reviewer, packageDate }, planned) => {
+	if (!baseCarriedRevisions && planned.every(({ plan }) => plan.operations.length === 0)) return Result.ok(baseBuffer);
+	const deletions = deletedFinalParagraphMarks(reviewer.toDocument());
+	const [firstDeletion] = deletions;
+	if (firstDeletion !== void 0) return Result.err(new CompareDocxFinalParagraphMarkError({
+		message: `A container's final paragraph mark carries a ${firstDeletion.kind}, which no consumer can resolve: ${firstDeletion.container} paragraph ${String(firstDeletion.paragraphIndex)}.`,
+		deletions
+	}));
+	return await Result.tryPromise({
+		try: async () => await withFixedPackageDates(await reviewer.toBuffer(), packageDate),
+		catch: (cause) => new CompareDocxSerializeError({
+			message: "The compared document could not be serialized.",
+			cause
+		})
+	});
+};
+/**
+* Compare `base` against `target` and return `base` carrying the tracked
+* changes that turn it into `target`, alongside the change list describing
+* them.
+*
+* The result is verified by default: a redline whose round trip cannot be
+* proven is refused rather than returned, because one that reads plausibly and
+* is wrong is worse than none. `onUnverified: "emit"` asks for the opposite
+* trade — the best redline available, plus the typed list of what could not be
+* represented — for a caller that would rather show something and say what is
+* missing.
+*/
+const compareDocx = async (base, target, options) => {
+	const parsed = await parseComparison(base, target, options);
+	if (parsed.isErr()) return Result.err(parsed.error);
+	const planned = planComparison(parsed.value);
+	if (planned.isErr()) return Result.err(planned.error);
+	const applied = applyComparison(parsed.value, planned.value);
+	if (applied.isErr()) return Result.err(applied.error);
+	const { changes, verification } = applied.value;
+	if (verification.status === "unverified" && (options.onUnverified ?? "refuse") === "refuse") {
+		const [firstFailure] = verification.failures;
+		if (firstFailure === void 0) panic("An unverified comparison reported no failing invariant");
+		return Result.err(new CompareDocxRoundTripError({
+			message: `The generated tracked changes do not satisfy ${firstFailure.invariant}: ${firstFailure.detail}`,
+			story: firstFailure.story,
+			invariant: firstFailure.invariant,
+			cause: firstFailure.cause,
+			failures: verification.failures
+		}));
+	}
+	const serialized = await serializeComparison(parsed.value, planned.value);
+	if (serialized.isErr()) return Result.err(serialized.error);
+	return Result.ok({
+		buffer: serialized.value,
+		changes,
+		verification,
+		unsupported: parsed.value.unsupported
+	});
+};
+//#endregion
+export { MAX_COMPARE_OPERATIONS, applyComparison, compareDocx, parseComparison, planComparison, serializeComparison };

package/dist/compare/formatting.d.ts

@@ -0,0 +1,29 @@
+import { FolioAIBlock, FolioAIInlineFormatting } from "../ai-edits/types.js";
+//#region src/compare/formatting.d.ts
+/** One run of characters whose supported inline formatting differs. */
+type InlineFormattingSegment = {
+  /** Zero-based UTF-16 offset into the block's visible text. */
+  startOffset: number;
+  endOffset: number;
+  /** Only the properties that differ, set to the target document's value. */
+  formatting: FolioAIInlineFormatting;
+};
+type InlineFormattingSegmentsOptions = {
+  baseBlock: FolioAIBlock;
+  targetBlock: FolioAIBlock;
+  /** Refuse (return `null`) rather than build more segments than this. */
+  maxSegments: number;
+};
+/**
+ * Segments where `targetBlock`'s bold / italic / underline / strike differs from
+ * `baseBlock`'s, for two blocks that carry the same text. Returns `null` only
+ * when the diff would exceed `maxSegments`.
+ *
+ * A block whose runs cannot be aligned to its text reports no segments rather
+ * than guessing: attributing formatting to the wrong characters is worse than
+ * missing a formatting-only change, and the caller has no offset it could
+ * trust instead.
+ */
+declare const inlineFormattingSegments: ({ baseBlock, targetBlock, maxSegments }: InlineFormattingSegmentsOptions) => InlineFormattingSegment[] | null;
+//#endregion
+export { InlineFormattingSegment, inlineFormattingSegments };

package/dist/compare/formatting.js

@@ -0,0 +1,84 @@
+//#region src/compare/formatting.ts
+const changedSupportedFormatting = (base, target) => ({
+	...Boolean(base.bold) !== Boolean(target.bold) && { bold: Boolean(target.bold) },
+	...Boolean(base.italic) !== Boolean(target.italic) && { italic: Boolean(target.italic) },
+	...Boolean(base.underline) !== Boolean(target.underline) && { underline: Boolean(target.underline) },
+	...Boolean(base.strike) !== Boolean(target.strike) && { strike: Boolean(target.strike) }
+});
+const sameInlineFormatting = (left, right) => left.bold === right.bold && left.italic === right.italic && left.underline === right.underline && left.strike === right.strike;
+const hasInlineFormatting = (formatting) => formatting.bold !== void 0 || formatting.italic !== void 0 || formatting.underline !== void 0 || formatting.strike !== void 0;
+/**
+* A block's runs, or `null` when they cannot describe the block's text.
+* Non-text inline content (a field, an image) leaves the concatenated run text
+* shorter than the block text; attributing formatting by offset would then
+* point at the wrong characters, so the caller must back off instead.
+*/
+const previewRunsForBlock = (block) => {
+	const runs = block.previewRuns ?? [{ text: block.text }];
+	return runs.map(({ text }) => text).join("") === block.text ? runs : null;
+};
+/**
+* Segments where `targetBlock`'s bold / italic / underline / strike differs from
+* `baseBlock`'s, for two blocks that carry the same text. Returns `null` only
+* when the diff would exceed `maxSegments`.
+*
+* A block whose runs cannot be aligned to its text reports no segments rather
+* than guessing: attributing formatting to the wrong characters is worse than
+* missing a formatting-only change, and the caller has no offset it could
+* trust instead.
+*/
+const inlineFormattingSegments = ({ baseBlock, targetBlock, maxSegments }) => {
+	const baseRuns = previewRunsForBlock(baseBlock);
+	const targetRuns = previewRunsForBlock(targetBlock);
+	if (!baseRuns || !targetRuns || baseBlock.text.length === 0) return [];
+	const segments = [];
+	let baseRunIndex = 0;
+	let targetRunIndex = 0;
+	let baseRunOffset = 0;
+	let targetRunOffset = 0;
+	let textOffset = 0;
+	while (baseRunIndex < baseRuns.length && targetRunIndex < targetRuns.length) {
+		const baseRun = baseRuns[baseRunIndex];
+		const targetRun = targetRuns[targetRunIndex];
+		if (!baseRun || !targetRun) break;
+		const length = Math.min(baseRun.text.length - baseRunOffset, targetRun.text.length - targetRunOffset);
+		if (length <= 0) {
+			if (baseRunOffset >= baseRun.text.length) {
+				baseRunIndex++;
+				baseRunOffset = 0;
+			}
+			if (targetRunOffset >= targetRun.text.length) {
+				targetRunIndex++;
+				targetRunOffset = 0;
+			}
+			continue;
+		}
+		const formatting = changedSupportedFormatting(baseRun, targetRun);
+		if (hasInlineFormatting(formatting)) {
+			const previous = segments.at(-1);
+			if (previous && previous.endOffset === textOffset && sameInlineFormatting(previous.formatting, formatting)) previous.endOffset += length;
+			else {
+				if (segments.length >= maxSegments) return null;
+				segments.push({
+					startOffset: textOffset,
+					endOffset: textOffset + length,
+					formatting
+				});
+			}
+		}
+		textOffset += length;
+		baseRunOffset += length;
+		targetRunOffset += length;
+		if (baseRunOffset >= baseRun.text.length) {
+			baseRunIndex++;
+			baseRunOffset = 0;
+		}
+		if (targetRunOffset >= targetRun.text.length) {
+			targetRunIndex++;
+			targetRunOffset = 0;
+		}
+	}
+	return segments;
+};
+//#endregion
+export { inlineFormattingSegments };

package/dist/compare/plan.d.ts

@@ -0,0 +1,22 @@
+import { FolioAIEditOperation, FolioAIEditSnapshot } from "../ai-edits/types.js";
+import { FolioDocumentStoryHandle } from "../ai-edits/headless.js";
+import { CompareChange } from "./types.js";
+//#region src/compare/plan.d.ts
+type CompareStoryPlan = {
+  changes: CompareChange[];
+  operations: FolioAIEditOperation[];
+};
+type PlanStoryCompareOptions = {
+  story: FolioDocumentStoryHandle;
+  baseSnapshot: FolioAIEditSnapshot;
+  targetSnapshot: FolioAIEditSnapshot;
+  /** Cap on generated operations; the caller turns `null` into its own error. */
+  maxOperations: number;
+};
+/**
+ * Plan one story's comparison, or `null` when it needs more operations than
+ * `maxOperations`.
+ */
+declare const planStoryCompare: ({ story, baseSnapshot, targetSnapshot, maxOperations }: PlanStoryCompareOptions) => CompareStoryPlan | null;
+//#endregion
+export { CompareStoryPlan, PlanStoryCompareOptions, planStoryCompare };