@stll/folio-core 0.59.0 → 0.61.0

package/dist/docx/paragraphPropertySource.js

@@ -5,7 +5,6 @@ import { parseParagraphProperties } from "./paragraphProperties.js";
 import { visitDocxParagraphs } from "./paragraphTraversal.js";
 import { parseXmlDocument } from "./xmlParser.js";
 import { TaggedError, panic } from "better-result";
-import { PluginKey } from "prosemirror-state";
 //#region src/docx/paragraphPropertySource.ts
 const paragraphPropertySourceEmissionFingerprint = Symbol("paragraphPropertySourceEmissionFingerprint");
 const paragraphPropertySourceIndentationBaseline = Symbol("paragraphPropertySourceIndentationBaseline");
@@ -793,7 +792,7 @@ const createProseParagraphWithPropertySource = (nodeType, sourceParagraph, optio
 	linkProseParagraphPropertySource(paragraph, sourceParagraph);
 	return paragraph;
 };
-const paragraphPropertySourceTransfersKey = new PluginKey("paragraphPropertySourceTransfers");
+const paragraphPropertySourceTransfersKey = "stll.folio.meta.paragraphPropertySourceTransfers";
 /** Join adjacent paragraphs when revision semantics deliberately select the right owner. */
 const joinProseParagraphsWithRightPropertySource = ({ attrs, pos, transaction }) => {
 	const $pos = transaction.doc.resolve(pos);

package/dist/docx/parseContext.d.ts

@@ -1,8 +1,9 @@
-import { ParseWarning, ParseWarningCode } from "@stll/docx-core/model";
+import { ImportedIdCollisionKind, ParseWarning, ParseWarningCode } from "@stll/docx-core/model";
 //#region src/docx/parseContext.d.ts
 /** What a call site states; the collector supplies the rest. */
 type ParseWarningReport = {
   code: ParseWarningCode;
+  identityKind?: ImportedIdCollisionKind;
   /** The element as written, prefix included. */
   element?: string;
   /** The best position this part can name, e.g. `style "Heading1"`. */

package/dist/docx/parseContext.js

@@ -30,6 +30,7 @@ const createParseWarningCollector = (part = PACKAGE_PART) => {
 		retainedByCode.set(report.code, kept + 1);
 		retained.push({
 			code: report.code,
+			...report.identityKind === void 0 ? {} : { identityKind: report.identityKind },
 			location: {
 				...location,
 				...report.element === void 0 ? {} : { element: report.element },

package/dist/docx/parseWarningMessage.js

@@ -22,6 +22,7 @@ const PARSE_WARNING_MESSAGES = {
 	[PARSE_WARNING_CODES.documentPartMissing]: () => "No document.xml found in DOCX",
 	[PARSE_WARNING_CODES.documentModelIssue]: (warning) => warning.detail ?? "The document model reported an issue.",
 	[PARSE_WARNING_CODES.duplicateCommentId]: (warning) => `Dropped ${plural(warning.count, "comment")} repeating a w:id another comment already defines.`,
+	[PARSE_WARNING_CODES.duplicateImportedId]: (warning) => `Repeated imported identity${quoted(warning.value)}${where(warning)}: ${warning.detail ?? "multiple definitions share an identity."}`,
 	[PARSE_WARNING_CODES.missingCommentId]: (warning) => `Dropped ${plural(warning.count, "comment")} with no readable w:id.`,
 	[PARSE_WARNING_CODES.duplicateNoteId]: (warning) => `Dropped ${plural(warning.count, "note")} repeating a w:id another note already defines.`,
 	[PARSE_WARNING_CODES.danglingCommentReference]: (warning) => `Removed ${plural(warning.count, "dangling comment reference marker")} whose comments.xml entries are missing.`,

package/dist/docx/parser.js

@@ -1,7 +1,9 @@
 import { toArrayBuffer } from "../utils/docxInput.js";
 import { loadFontsWithMapping } from "../utils/fontLoader.js";
+import { decodeText, encodeUtf8 } from "../utils/textEncoding.js";
 import { MAX_PACKAGE_TIFF_PIXELS, convertTiffToPngDataUrl, isTiffMimeType } from "../utils/tiffConverter.js";
 import { ALT_CHUNK_PLAIN_TEXT_LIMIT_BYTES, getAltChunkRelationshipId, isAltChunkMarkup } from "./altChunk.js";
+import { normalizeBookmarkIdsInXmlParts } from "./bookmarkIdNormalization.js";
 import { DUPLICATE_COMMENT_ID_WARNING, normalizeCommentIds } from "./commentIdNormalization.js";
 import { parseComments } from "./commentParser.js";
 import { DANGLING_COMMENT_REFERENCE_WARNING, UNBALANCED_COMMENT_RANGE_WARNING, normalizeCommentReferences } from "./commentReferenceNormalization.js";
@@ -16,6 +18,7 @@ import { parseEndnotes, parseFootnotes } from "./footnoteParser.js";
 import { parseFooter, parseHeader } from "./headerFooterParser.js";
 import { DANGLING_FOOTER_REFERENCE_WARNING, DANGLING_HEADER_REFERENCE_WARNING, assignHeaderFooterRoles, normalizeHeaderFooterReferences } from "./headerFooterReferenceNormalization.js";
 import { assignHeaderFooterVerbatimXml, canReplayHeaderFooterVerbatim, captureDocumentSourceBaseline, captureHeaderFooterPackageBaselines, refreshHeaderFooterVerbatimFingerprint } from "./headerFooterVerbatim.js";
+import { diagnoseImportedIdCollisions } from "./importedIdCollisions.js";
 import { extractMetafileRaster, isMetafileMimeType } from "./metafileRaster.js";
 import { renderEmfSvg } from "./metafileSvg.js";
 import { DocxModelValidationError, formatDocumentModelIssues, validateFolioDocumentModel } from "./modelValidation.js";
@@ -34,7 +37,7 @@ import { parseSettings } from "./settingsParser.js";
 import { parseStylesPackage } from "./styleParser.js";
 import { applyThemeFontLang, parseTheme } from "./themeParser.js";
 import { UNBALANCED_MOVE_RANGE_WARNING, normalizeTrackedMoveRanges } from "./trackedMoveRangeNormalization.js";
-import { getEntryUncompressedSize, getMediaMimeType, mediaToDataUrl, unzipDocx } from "./unzip.js";
+import { getEntryUncompressedSize, getMediaMimeType, mediaToDataUrl, replaceRawDocxXmlParts, unzipDocx } from "./unzip.js";
 import { getAttribute, getChildElements, getLocalName, parseXml, parseXmlDocument } from "./xmlParser.js";
 import { FOLIO_XML_RESOURCE_LIMITS } from "./xmlResourceLimits.js";
 import { TaggedError, panic } from "better-result";
@@ -120,7 +123,7 @@ const extractAltChunkPlainText = async (xml, context) => {
 	if (bytes.byteLength > context.budget.remainingBytes) return;
 	context.budget.remainingBytes -= bytes.byteLength;
 	try {
-		return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+		return decodeText(bytes, { fatal: true });
 	} catch {
 		return;
 	}
@@ -192,6 +195,8 @@ async function parseDocxWithPreviewBudget(input, options, previewBudget) {
 			password,
 			extractAllXml: false
 		}));
+		diagnoseImportedIdCollisions(raw.allXml, parseContext);
+		await replaceRawDocxXmlParts(raw, normalizeBookmarkIdsInXmlParts(raw.allXml));
 		const repairedDocumentTree = await normalizeRawDocxNumericIds(raw);
 		const paragraphPropertySourceDigest = sha256Hex(raw.originalBuffer);
 		if (raw.wasEncrypted) parseContext.warn({ code: PARSE_WARNING_CODES.packageDecrypted });
@@ -556,7 +561,7 @@ async function buildMediaMap(raw, rels) {
 		}
 		const emfSvg = isReferenced && mimeType === EMF_MIME_TYPE && remainingEmfPreviewBytes > 0 ? renderEmfSvg(data) : null;
 		if (emfSvg) {
-			const svgBytes = new TextEncoder().encode(emfSvg);
+			const svgBytes = encodeUtf8(emfSvg);
 			if (svgBytes.byteLength <= remainingEmfPreviewBytes) {
 				remainingEmfPreviewBytes -= svgBytes.byteLength;
 				const mediaFile = {
@@ -585,8 +590,8 @@ async function buildMediaMap(raw, rels) {
 		if (normalizedPath !== path) media.set(normalizedPath, mediaFile);
 	}
 	for (const [path, xml] of raw.allXml.entries()) {
-		if (!/^word\/diagrams\/[^/]+\.xml$/iu.test(path) || new TextEncoder().encode(xml).byteLength > FOLIO_XML_RESOURCE_LIMITS.maxBytes) continue;
-		const data = new TextEncoder().encode(xml).buffer;
+		if (!/^word\/diagrams\/[^/]+\.xml$/iu.test(path) || encodeUtf8(xml).byteLength > FOLIO_XML_RESOURCE_LIMITS.maxBytes) continue;
+		const data = encodeUtf8(xml).buffer;
 		const file = {
 			path,
 			filename: path.split("/").pop() ?? path,

package/dist/docx/preservedRunContent.js

@@ -1,3 +1,4 @@
+import { sliceText } from "../utils/sliceText.js";
 import { captureVerbatimXml } from "./verbatimCapture.js";
 import { findChildrenByLocalName, getChildElements, getLocalName, getTextContent } from "./xmlParser.js";
 //#region src/docx/preservedRunContent.ts
@@ -31,7 +32,11 @@ const preservedRunChildText = (element) => {
 	if (sources === void 0) return "";
 	const parts = [];
 	for (const localName of sources) for (const source of findChildrenByLocalName(element, localName)) collectTextContent(source, parts);
-	return parts.join("").slice(0, MAX_PRESERVED_TEXT_LENGTH);
+	return sliceText({
+		text: parts.join(""),
+		units: "codeUnits",
+		end: MAX_PRESERVED_TEXT_LENGTH
+	});
 };
 /**
 * Capture one unmodelled run child.
@@ -90,7 +95,11 @@ const preserveInlineChild = (element) => {
 	return {
 		type: "preservedInline",
 		xml: captureVerbatimXml(element),
-		text: parts.join("").slice(0, MAX_PRESERVED_TEXT_LENGTH)
+		text: sliceText({
+			text: parts.join(""),
+			units: "codeUnits",
+			end: MAX_PRESERVED_TEXT_LENGTH
+		})
 	};
 };
 //#endregion

package/dist/docx/relsParser.js

@@ -86,7 +86,7 @@ function parseRelationships(relsXml) {
 		};
 		if (targetMode === "External") relationship.targetMode = "External";
 		else if (targetMode === "Internal") relationship.targetMode = "Internal";
-		map.set(id, relationship);
+		if (!map.has(id)) map.set(id, relationship);
 	}
 	return map;
 }

package/dist/docx/removeHeaderFooterParts.js

@@ -1,3 +1,4 @@
+import { decodeText, encodeUtf8 } from "../utils/textEncoding.js";
 import { isUnsafePackagePath } from "./packageParts.js";
 import { parseRelationships, resolveRelativePath } from "./relsParser.js";
 import { captureVerbatimXml } from "./verbatimCapture.js";
@@ -18,7 +19,7 @@ const removeResolvedHeaderFooterParts = async ({ document, zip, removedReference
 	if (removedReferences.length === 0) return;
 	const relsFile = zip.file(DOCUMENT_RELS_PATH);
 	if (!relsFile) return panic("A resolved header/footer part has no document relationships");
-	const relsXml = await relsFile.async("text");
+	const relsXml = await relsFile.async("uint8array").then((bytes) => decodeText(bytes, { ignoreBOM: true }));
 	const relationships = parseRelationships(relsXml);
 	const removedIds = /* @__PURE__ */ new Set();
 	const candidates = /* @__PURE__ */ new Set();
@@ -39,7 +40,7 @@ const removeResolvedHeaderFooterParts = async ({ document, zip, removedReference
 	}
 	for (const file of Object.values(zip.files)) {
 		if (file.dir || !file.name.endsWith(".rels") || file.name === DOCUMENT_RELS_PATH) continue;
-		const otherRelationships = parseRelationships(await file.async("text"));
+		const otherRelationships = parseRelationships(await file.async("uint8array").then((bytes) => decodeText(bytes, { ignoreBOM: true })));
 		for (const relationship of otherRelationships.values()) if (relationship.targetMode !== "External") candidates.delete(resolveRelativePath(file.name, relationship.target));
 	}
 	const retiredMedia = /* @__PURE__ */ new Set();
@@ -48,7 +49,7 @@ const removeResolvedHeaderFooterParts = async ({ document, zip, removedReference
 		const relsPath = `${path.slice(0, slash + 1)}_rels/${path.slice(slash + 1)}.rels`;
 		const partRels = zip.file(relsPath);
 		if (!partRels) continue;
-		for (const relationship of parseRelationships(await partRels.async("text")).values()) {
+		for (const relationship of parseRelationships(await partRels.async("uint8array").then((bytes) => decodeText(bytes, { ignoreBOM: true }))).values()) {
 			if (!officeRelationshipHasType(relationship.type, "image") || relationship.targetMode === "External") continue;
 			const mediaPath = resolveRelativePath(relsPath, relationship.target);
 			if (!isUnsafePackagePath(mediaPath) && mediaPath.startsWith("word/media/")) retiredMedia.add(mediaPath);
@@ -58,7 +59,7 @@ const removeResolvedHeaderFooterParts = async ({ document, zip, removedReference
 	writeZipPart({
 		zip,
 		path: DOCUMENT_RELS_PATH,
-		data: withoutChildren(relsXml, (child) => getNamespaceUri(child) === OOXML_NS.pr && getLocalName(child.name) === "Relationship" && removedIds.has(getAttribute(child, null, "Id") ?? "")),
+		data: encodeUtf8(withoutChildren(relsXml, (child) => getNamespaceUri(child) === OOXML_NS.pr && getLocalName(child.name) === "Relationship" && removedIds.has(getAttribute(child, null, "Id") ?? ""))),
 		options: {
 			compression: "DEFLATE",
 			compressionOptions
@@ -75,7 +76,7 @@ const removeResolvedHeaderFooterParts = async ({ document, zip, removedReference
 	}
 	for (const file of Object.values(zip.files)) {
 		if (file.dir || !file.name.endsWith(".rels")) continue;
-		for (const relationship of parseRelationships(await file.async("text")).values()) if (relationship.targetMode !== "External") retiredMedia.delete(resolveRelativePath(file.name, relationship.target));
+		for (const relationship of parseRelationships(await file.async("uint8array").then((bytes) => decodeText(bytes, { ignoreBOM: true }))).values()) if (relationship.targetMode !== "External") retiredMedia.delete(resolveRelativePath(file.name, relationship.target));
 	}
 	for (const path of retiredMedia) {
 		zip.remove(path);
@@ -83,11 +84,11 @@ const removeResolvedHeaderFooterParts = async ({ document, zip, removedReference
 	}
 	const contentTypes = zip.file("[Content_Types].xml");
 	if (!contentTypes) return panic("The package has no content types");
-	const contentTypesXml = await contentTypes.async("text");
+	const contentTypesXml = await contentTypes.async("uint8array").then((bytes) => decodeText(bytes, { ignoreBOM: true }));
 	writeZipPart({
 		zip,
 		path: "[Content_Types].xml",
-		data: withoutChildren(contentTypesXml, (child) => getNamespaceUri(child) === OOXML_NS.ct && getLocalName(child.name) === "Override" && candidates.has((getAttribute(child, null, "PartName") ?? "").replace(/^\//u, ""))),
+		data: encodeUtf8(withoutChildren(contentTypesXml, (child) => getNamespaceUri(child) === OOXML_NS.ct && getLocalName(child.name) === "Override" && candidates.has((getAttribute(child, null, "PartName") ?? "").replace(/^\//u, "")))),
 		options: {
 			compression: "DEFLATE",
 			compressionOptions

package/dist/docx/replyToComment.js

@@ -1,5 +1,6 @@
 import { allocateCommentId, seedCommentIdAbove } from "../prosemirror/commentIdAllocator.js";
 import { generateHexId } from "../utils/hexId.js";
+import { commentTextParagraph } from "./commentTextParagraph.js";
 //#region src/docx/replyToComment.ts
 /**
 * Create a threaded reply to an existing comment.
@@ -41,19 +42,8 @@ const createReply = (comments, parentCommentId, input) => {
 	for (const { id } of comments) seedCommentIdAbove(id);
 	const used = usedParaIds(comments);
 	if (!rootLastParagraph.paraId) rootLastParagraph.paraId = freshParaId(used);
-	const replyParagraph = {
-		type: "paragraph",
-		formatting: {},
-		paraId: freshParaId(used),
-		content: [{
-			type: "run",
-			formatting: {},
-			content: [{
-				type: "text",
-				text: input.text
-			}]
-		}]
-	};
+	const replyParagraph = commentTextParagraph(input.text);
+	replyParagraph.paraId = freshParaId(used);
 	return {
 		id: allocateCommentId(),
 		author: input.author,