@stll/folio-cli 0.5.1 → 0.5.2

package/dist/bin.js

@@ -1,17 +1,23 @@
 #!/usr/bin/env node
 import { runFolioCli } from "./cli.js";
+import { panic } from "better-result";
+import { createTextReader, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 //#region src/bin.ts
 const readStdin = async () => {
+	const reader = createTextReader({ ignoreBOM: true });
 	const chunks = [];
-	for await (const chunk of process.stdin) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
-	return Buffer.concat(chunks).toString("utf8");
+	for await (const chunk of process.stdin) {
+		if (!(chunk instanceof Uint8Array)) panic("Standard input must provide byte chunks");
+		chunks.push(reader.read(chunk));
+	}
+	return chunks.join("") + reader.finish();
 };
 process.exitCode = await runFolioCli(process.argv.slice(2), {
 	stdout: (text) => {
-		process.stdout.write(text);
+		process.stdout.write(encodeUtf8(text));
 	},
 	stderr: (text) => {
-		process.stderr.write(text);
+		process.stderr.write(encodeUtf8(text));
 	},
 	readStdin,
 	isTTY: process.stdout.isTTY,

package/dist/editor-lease.js

@@ -3,6 +3,7 @@ import { fileVersionOf } from "./file-system.js";
 import { DEFAULT_LEASE_OWNER, acquireLease, publicLockHolder, readLease } from "./lock.js";
 import { readSidecarFile, writeNewSidecarFile } from "./sidecar.js";
 import { Result } from "better-result";
+import { decodeText, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 import { watch } from "node:fs";
 import { readFile, readdir, rename, rm } from "node:fs/promises";
 import path from "node:path";
@@ -103,7 +104,7 @@ const pendingFlushRequests = async (documentPath, now = /* @__PURE__ */ new Date
 		const requestPath = path.join(directory, name);
 		const bytes = await readSidecarFile(requestPath);
 		if (bytes.isErr()) continue;
-		const request = parseRequest(new TextDecoder().decode(bytes.value));
+		const request = parseRequest(decodeText(bytes.value));
 		if (request === null) continue;
 		if (Date.parse(request.deadline) > now.getTime()) pending.push(request);
 		else await rm(requestPath, { force: true });
@@ -113,7 +114,7 @@ const pendingFlushRequests = async (documentPath, now = /* @__PURE__ */ new Date
 const placeRequest = async (documentPath, request) => {
 	const target = flushRequestPathFor(documentPath, request.id);
 	const temporary = path.join(path.dirname(documentPath), `.${path.basename(documentPath)}.folio-flush.${request.id}.tmp`);
-	const written = await writeNewSidecarFile(temporary, new TextEncoder().encode(`${JSON.stringify(request)}\n`));
+	const written = await writeNewSidecarFile(temporary, encodeUtf8(`${JSON.stringify(request)}\n`));
 	if (written.isErr()) return Result.err(written.error);
 	if ((await Result.tryPromise(() => rename(temporary, target))).isOk()) return Result.ok();
 	await rm(temporary, { force: true });

package/dist/execute-read.js

@@ -4,6 +4,7 @@ import { MAX_READ_BLOCKS } from "./registry.js";
 import { Result, panic } from "better-result";
 import { executeFolioToolCallUntyped } from "@stll/folio-agents/execute";
 import { FOLIO_AGENT_TOOL_NAMES } from "@stll/folio-agents/types";
+import { base64ToBytes, bytesToBase64Url, decodeText, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 import { createReviewerBridge } from "@stll/folio-agents/bridges/reviewer";
 import { compareDocxVersions, formatVersionDiffForLLM } from "@stll/folio-agents/compare";
 //#region src/execute-read.ts
@@ -36,12 +37,12 @@ const labelBlocks = (blocks, sources, cells = /* @__PURE__ */ new Map()) => bloc
 		...cell !== void 0 && { tableCell: cell }
 	};
 });
-const encodeReadCursor = ({ fileVersion, afterBlockId }) => Buffer.from(JSON.stringify({
+const encodeReadCursor = ({ fileVersion, afterBlockId }) => bytesToBase64Url(encodeUtf8(JSON.stringify({
 	v: fileVersion,
 	a: afterBlockId
-}), "utf8").toString("base64url");
+})));
 const decodeReadCursor = (cursor) => {
-	const parsed = Result.try(() => JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")));
+	const parsed = Result.try(() => JSON.parse(decodeText(base64ToBytes(cursor), { ignoreBOM: true })));
 	if (parsed.isErr() || !isRecord(parsed.value)) return null;
 	const { v, a } = parsed.value;
 	return typeof v === "string" && typeof a === "string" ? {
@@ -54,7 +55,7 @@ const invalidInput = (message, hint) => cliError({
 	message,
 	hint
 });
-const byteLength = (value) => Buffer.byteLength(JSON.stringify(value) ?? "", "utf8");
+const byteLength = (value) => encodeUtf8(JSON.stringify(value) ?? "").byteLength;
 /** Room kept in a paged response for its envelope fields and `nextCursor`. */
 const PAGE_ENVELOPE_RESERVE_BYTES = 512;
 /**

package/dist/flags.js

@@ -1,5 +1,6 @@
 import { FOLIO_CLI_ERROR_CODES, cliError } from "./errors.js";
 import { Result, panic } from "better-result";
+import { decodeText, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 import { readFile } from "node:fs/promises";
 //#region src/flags.ts
 /**
@@ -82,13 +83,13 @@ const readInputSource = async ({ source, readStdin }) => {
 	else if (source.startsWith("@")) {
 		const inputPath = source.slice(1);
 		const read = await Result.tryPromise({
-			try: () => readFile(inputPath, "utf8"),
+			try: () => readFile(inputPath).then((bytes) => decodeText(bytes, { ignoreBOM: true })),
 			catch: () => usage(`Cannot read --input file ${inputPath}.`)
 		});
 		if (read.isErr()) return Result.err(read.error);
 		text = read.value;
 	}
-	if (Buffer.byteLength(text, "utf8") > 16777216) return Result.err(usage(`--input is over the ${MAX_INPUT_BYTES}-byte limit.`));
+	if (encodeUtf8(text).byteLength > 16777216) return Result.err(usage(`--input is over the ${MAX_INPUT_BYTES}-byte limit.`));
 	const parsed = Result.try(() => JSON.parse(text));
 	if (parsed.isErr()) return Result.err(usage("--input is not valid JSON.", "Pass inline JSON, @file.json, or - for stdin."));
 	const value = parsed.value;

package/dist/journal.js

@@ -3,6 +3,7 @@ import { errnoCode, fileVersionOf, sameFile } from "./file-system.js";
 import { checkPackageIntegrity } from "./package-parts.js";
 import { SIDECAR_DIRECTORY, ensureSidecarParent, inspectPath, openSidecarFile, readSidecarFile, syncDirectory } from "./sidecar.js";
 import { Result } from "better-result";
+import { decodeText, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 import { constants } from "node:fs";
 import { readdir, rename, rm } from "node:fs/promises";
 import path from "node:path";
@@ -42,7 +43,7 @@ const appendJournal = async (journalPath, entry) => {
 	const { handle } = opened.value;
 	const written = await Result.tryPromise(async () => {
 		try {
-			await handle.writeFile(`${JSON.stringify(entry)}\n`);
+			await handle.writeFile(encodeUtf8(`${JSON.stringify(entry)}\n`));
 			await handle.sync();
 		} finally {
 			await handle.close();
@@ -90,7 +91,7 @@ const findCommit = async (journalPath, txId) => {
 	const bytes = await readSidecarFile(journalPath);
 	if (bytes.isErr()) return Result.err(bytes.error);
 	let found;
-	for (const line of new TextDecoder().decode(bytes.value).split("\n")) {
+	for (const line of decodeText(bytes.value).split("\n")) {
 		if (!line.includes(txId)) continue;
 		if (isDiscardRecord(line, txId)) return Result.ok(void 0);
 		const commit = parseCommit(line);
@@ -121,7 +122,7 @@ const latestCommitFor = async (documentPath, journalPath) => {
 	if (parent.isErr() || parent.value.type !== "directory") return null;
 	const bytes = await readSidecarFile(journal);
 	if (bytes.isErr()) return null;
-	const lines = new TextDecoder().decode(bytes.value).split("\n");
+	const lines = decodeText(bytes.value).split("\n");
 	const discarded = /* @__PURE__ */ new Set();
 	for (let index = lines.length - 1; index >= 0; index--) {
 		const parsed = Result.try(() => JSON.parse(lines[index] ?? ""));

package/dist/lock.js

@@ -2,6 +2,7 @@ import { FOLIO_CLI_ERROR_CODES, cliError } from "./errors.js";
 import { errnoCode } from "./file-system.js";
 import { inspectPath, readSidecarFile, writeNewSidecarFile } from "./sidecar.js";
 import { Result } from "better-result";
+import { decodeText, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 import { link, rename, rm } from "node:fs/promises";
 import path from "node:path";
 import { randomUUID } from "node:crypto";
@@ -90,7 +91,7 @@ const readLease = async (documentPath) => {
 	if (entry.value.type === "missing") return { type: "free" };
 	const modifiedMs = entry.value.type === "file" ? entry.value.modifiedMs : Date.now();
 	const bytes = await readSidecarFile(lockPath);
-	const holder = bytes.isOk() ? parseHolder(new TextDecoder().decode(bytes.value)) : null;
+	const holder = bytes.isOk() ? parseHolder(decodeText(bytes.value)) : null;
 	return holder === null ? {
 		type: "unreadable",
 		modifiedMs
@@ -132,7 +133,7 @@ const placeLock = async (lockPath, contents, mode) => {
 		await rm(temporary, { force: true });
 	}
 };
-const encodeHolder = (holder) => new TextEncoder().encode(`${JSON.stringify(holder)}\n`);
+const encodeHolder = (holder) => encodeUtf8(`${JSON.stringify(holder)}\n`);
 /** How long a swap lock may stand before another process breaks it (its holder crashed). */
 const SWAP_STALE_MS = 10 * 1e3;
 /** How long to wait for another process's swap before refusing. */
@@ -152,7 +153,7 @@ const lockSwapPathFor = (documentPath) => `${lockPathFor(documentPath)}.swap`;
 */
 const underSwapLock = async (documentPath, change) => {
 	const swapPath = lockSwapPathFor(documentPath);
-	const marker = new TextEncoder().encode(`${process.pid}\n`);
+	const marker = encodeUtf8(`${process.pid}\n`);
 	const deadline = Date.now() + SWAP_WAIT_MS;
 	for (;;) {
 		const taken = await placeLock(swapPath, marker, "create");

package/dist/main-document-part.js

@@ -1,5 +1,6 @@
 import { FOLIO_CLI_ERROR_CODES, cliError } from "./errors.js";
 import { Result } from "better-result";
+import { createTextReader } from "@stll/folio-core/utils/textEncoding";
 import path from "node:path";
 import { XMLParser, XMLValidator } from "fast-xml-parser";
 import JSZip from "jszip";
@@ -121,23 +122,23 @@ const childrenNamed = (element, namespace, localName) => element.children.filter
 const partKey = (name) => name.toLowerCase();
 /** Inflate at most `limit` bytes of an entry; `truncated` when it holds more. */
 const inflatePrefix = (entry, limit) => new Promise((resolve, reject) => {
-	const decoder = new TextDecoder();
+	const decoder = createTextReader();
 	let text = "";
 	let size = 0;
 	const stream = entry.internalStream("uint8array");
 	stream.on("data", (chunk) => {
 		const room = limit - size;
 		size += chunk.length;
-		text += decoder.decode(chunk.subarray(0, Math.max(0, room)), { stream: true });
+		text += decoder.read(chunk.subarray(0, Math.max(0, room)));
 		if (size > limit) {
 			stream.pause();
 			resolve({
-				text: text + decoder.decode(),
+				text: text + decoder.finishPrefix(),
 				truncated: true
 			});
 		}
 	}).on("error", reject).on("end", () => resolve({
-		text: text + decoder.decode(),
+		text: text + decoder.finish(),
 		truncated: false
 	})).resume();
 });

package/dist/mcp-kit/schema.js

@@ -1,3 +1,4 @@
+import { encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 //#region src/mcp-kit/schema.ts
 const ANNOTATION_KEYWORDS = /* @__PURE__ */ new Set([
 	"$comment",
@@ -92,6 +93,6 @@ const hoistRepeatedSchemas = (schema, { minBytes = 200 } = {}) => {
 	} : schema;
 };
 /** The UTF-8 size of what `tools/list` sends for these tools. */
-const advertisedBytes = (tools) => new TextEncoder().encode(JSON.stringify(tools)).length;
+const advertisedBytes = (tools) => encodeUtf8(JSON.stringify(tools)).length;
 //#endregion
 export { advertisedBytes, compactSchema, hoistRepeatedSchemas };

package/dist/mcp-kit/surface.js

@@ -1,6 +1,8 @@
 import { KIT_ERROR_CODES, closestNames, didYouMean, failure, success, toCallResult, validationError } from "./envelope.js";
 import { readToolInput } from "./input.js";
 import { compactSchema } from "./schema.js";
+import { Result } from "better-result";
+import { base64ToBytes, bytesToBase64Url, decodeText, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 //#region src/mcp-kit/surface.ts
 /**
 * A tool surface with lazy discovery. The frequent tools are listed directly
@@ -102,9 +104,11 @@ const compareIds = (a, b) => {
 	if (a === b) return 0;
 	return a < b ? -1 : 1;
 };
-const encodeCursor = (id) => Buffer.from(id, "utf8").toString("base64url");
+const encodeCursor = (id) => bytesToBase64Url(encodeUtf8(id));
 const decodeCursor = (cursor) => {
-	const decoded = Buffer.from(cursor, "base64url").toString("utf8");
+	const parsed = Result.try(() => decodeText(base64ToBytes(cursor), { ignoreBOM: true }));
+	if (parsed.isErr()) return null;
+	const decoded = parsed.value;
 	return decoded.length > 0 && encodeCursor(decoded) === cursor ? decoded : null;
 };
 const hasJsonType = (value, type) => {

package/dist/output.js

@@ -1,3 +1,4 @@
+import { sliceText } from "@stll/folio-core/utils/sliceText";
 //#region src/output.ts
 const OUTPUT_FORMATS = ["json", "text"];
 const isOutputFormat = (value) => OUTPUT_FORMATS.some((format) => format === value);
@@ -43,7 +44,11 @@ const renderResult = (tool, result) => {
 	if (tool === "compare_documents" && isRecord(result) && typeof result["text"] === "string") return [result["text"]];
 	return null;
 };
-const shortVersion = (value) => text(value).slice(0, 12);
+const shortVersion = (value) => sliceText({
+	text: text(value),
+	units: "codeUnits",
+	end: 12
+});
 /** One line per committed or replayed transaction. */
 const renderReceipt = (receipt) => {
 	const parts = Array.isArray(receipt["changedParts"]) ? receipt["changedParts"].length : 0;

package/dist/package-parts.js

@@ -1,6 +1,7 @@
 import { FOLIO_CLI_ERROR_CODES, cliError } from "./errors.js";
 import { checkWordprocessingPackage } from "./main-document-part.js";
 import { Result } from "better-result";
+import { decodeText } from "@stll/folio-core/utils/textEncoding";
 import path from "node:path";
 import { FolioDocxReviewer } from "@stll/folio-core/server";
 import { XMLParser, XMLValidator } from "fast-xml-parser";
@@ -73,7 +74,7 @@ const nextRevisionIdSeed = async (bytes) => {
 	let highest = 0;
 	for (const [part, entry] of partEntries(zip.value)) {
 		if (!part.endsWith(".xml")) continue;
-		const xml = await entry.async("string");
+		const xml = await entry.async("uint8array").then((partBytes) => decodeText(partBytes, { ignoreBOM: true }));
 		for (const prefix of prefixesBoundTo(xml, WORDPROCESSINGML_NAMESPACES)) {
 			const pattern = new RegExp(`\\s${escapeRegExp(prefix)}:id\\s*=\\s*"(\\d+)"`, "gu");
 			for (const [, id] of xml.matchAll(pattern)) highest = Math.max(highest, Number(id));
@@ -123,7 +124,7 @@ const integrityError = (message) => cliError({
 const checkChangedPart = async ({ zip, parts, part }) => {
 	const entry = parts.get(part);
 	if (entry === void 0 || !(part.endsWith(".xml") || part.endsWith(".rels"))) return null;
-	const xml = await entry.async("string");
+	const xml = await entry.async("uint8array").then((partBytes) => decodeText(partBytes, { ignoreBOM: true }));
 	if (XMLValidator.validate(xml) !== true) return `${part} is not well-formed XML.`;
 	if (part.endsWith(".rels")) {
 		const lowerCaseParts = new Set([...parts.keys()].map((name) => name.toLowerCase()));
@@ -136,7 +137,7 @@ const checkChangedPart = async ({ zip, parts, part }) => {
 	}
 	const prefixes = prefixesBoundTo(xml, RELATIONSHIP_NAMESPACES);
 	if (prefixes.size === 0) return null;
-	const relsXml = await zip.file(relationshipsPartFor(part))?.async("string");
+	const relsXml = await zip.file(relationshipsPartFor(part))?.async("uint8array").then((partBytes) => decodeText(partBytes, { ignoreBOM: true }));
 	const ids = new Set(parseRelationships(relsXml ?? "").map(({ id }) => id));
 	for (const prefix of prefixes) {
 		const pattern = new RegExp(`\\s${escapeRegExp(prefix)}:[A-Za-z]+\\s*=\\s*"([^"]*)"`, "gu");

package/dist/package.js

@@ -1,4 +1,4 @@
 //#region package.json
-var version = "0.5.1";
+var version = "0.5.2";
 //#endregion
 export { version };

package/dist/preview-commands.js

@@ -6,6 +6,7 @@ import { resolveTransactionDate } from "./provenance.js";
 import { buildDisplayList, checkPages, displayListHtml, renderPdf, renderPng } from "./render.js";
 import { startPreviewServer } from "./serve.js";
 import { Result } from "better-result";
+import { encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 import { parseArgs } from "node:util";
 import path from "node:path";
 //#region src/preview-commands.ts
@@ -140,7 +141,7 @@ const render = async ({ values, positionals }) => {
 			const list = await buildDisplayList(file.value);
 			const selected = list.isOk() ? checkPages(list.value, pages) : list;
 			output = selected.isOk() ? Result.ok({
-				bytes: new TextEncoder().encode(displayListHtml(selected.value, path.basename(file.value.path))),
+				bytes: encodeUtf8(displayListHtml(selected.value, path.basename(file.value.path))),
 				pageCount: selected.value.pages.length
 			}) : Result.err(selected.error);
 			break;

package/dist/serve.js

@@ -4,6 +4,7 @@ import { latestCommitFor } from "./journal.js";
 import { buildDisplayList, displayListHtml } from "./render.js";
 import { SIDECAR_DIRECTORY } from "./sidecar.js";
 import { Result } from "better-result";
+import { bytesToBase64, bytesToBase64Url, encodeUtf8 } from "@stll/folio-core/utils/textEncoding";
 import { watch } from "node:fs";
 import { stat } from "node:fs/promises";
 import path from "node:path";
@@ -85,7 +86,7 @@ const startPreviewServer = async ({ documentPath, port }) => {
 	if (initial.isErr()) return Result.err(initial.error);
 	const realPath = initial.value.path;
 	const title = path.basename(realPath);
-	const prefix = `/${randomBytes(24).toString("base64url")}/`;
+	const prefix = `/${bytesToBase64Url(randomBytes(24))}/`;
 	let stamp = null;
 	let rendered = null;
 	let rendering = null;
@@ -129,7 +130,7 @@ const startPreviewServer = async ({ documentPath, port }) => {
 		rendered = result;
 		stamp = next;
 		if (changed) {
-			const message = `data: ${JSON.stringify({ fileVersion: result.fileVersion })}\n\n`;
+			const message = encodeUtf8(`data: ${JSON.stringify({ fileVersion: result.fileVersion })}\n\n`);
 			for (const listener of listeners) listener.write(message);
 		}
 		return result;
@@ -144,7 +145,7 @@ const startPreviewServer = async ({ documentPath, port }) => {
 			...SECURITY_HEADERS,
 			...headers
 		});
-		response.end(body);
+		response.end(encodeUtf8(body));
 	};
 	const handle = async (request, response) => {
 		if (!allowedHosts().has(request.headers.host ?? "")) {
@@ -165,7 +166,7 @@ const startPreviewServer = async ({ documentPath, port }) => {
 		}
 		const route = url.pathname.slice(prefix.length);
 		if (route === "") {
-			const nonce = randomBytes(16).toString("base64");
+			const nonce = bytesToBase64(randomBytes(16));
 			const page = await current();
 			respond(response, 200, shellHtml(title, nonce, page.type === "page" ? page.pageWidthPx : null), {
 				"Content-Type": "text/html; charset=utf-8",
@@ -203,7 +204,7 @@ const startPreviewServer = async ({ documentPath, port }) => {
 			listeners.add(response);
 			request.on("close", () => listeners.delete(response));
 			const page = await current();
-			response.write(`data: ${JSON.stringify({ fileVersion: page.fileVersion })}\n\n`);
+			response.write(encodeUtf8(`data: ${JSON.stringify({ fileVersion: page.fileVersion })}\n\n`));
 			return;
 		}
 		respond(response, 404, "not found", { "Content-Type": "text/plain" });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stll/folio-cli",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "Command line and stdio MCP server for reading, reviewing, and redlining .docx files with folio: tracked-change edits, comments, and comparisons written atomically with version preconditions.",
   "keywords": [
     "cli",
@@ -39,15 +39,18 @@
   "exports": {
     ".": {
       "types": "./dist/cli.d.ts",
-      "import": "./dist/cli.js"
+      "import": "./dist/cli.js",
+      "default": "./dist/cli.js"
     },
     "./editor-lease": {
       "types": "./dist/editor-lease.d.ts",
-      "import": "./dist/editor-lease.js"
+      "import": "./dist/editor-lease.js",
+      "default": "./dist/editor-lease.js"
     },
     "./save": {
       "types": "./dist/save.d.ts",
-      "import": "./dist/save.js"
+      "import": "./dist/save.js",
+      "default": "./dist/save.js"
     }
   },
   "publishConfig": {
@@ -71,8 +74,9 @@
     "@fontsource/tinos": "^5.2.7",
     "@modelcontextprotocol/server": "2.0.0",
     "@stll/agent-input": "0.1.0",
-    "@stll/folio-agents": "^0.16.1",
-    "@stll/folio-core": "^0.60.0",
+    "@stll/folio-agents": "^0.16.2",
+    "@stll/folio-core": "^0.61.0",
+    "@stll/utf8-boundaries": "0.1.0",
     "better-result": "3.0.1",
     "fast-xml-parser": "^5.10.1",
     "jszip": "3.10.2"
@@ -82,7 +86,21 @@
     "playwright-core": "1.61.1"
   },
   "peerDependencies": {
-    "playwright-core": ">=1.50.0"
+    "playwright-core": ">=1.50.0",
+    "prosemirror-commands": "^1.7.1",
+    "prosemirror-dropcursor": "^1.8.2",
+    "prosemirror-gapcursor": "^1.4.1",
+    "prosemirror-history": "^1.5.0",
+    "prosemirror-inputrules": "^1.5.0",
+    "prosemirror-keymap": "^1.2.3",
+    "prosemirror-model": "^1.25.9",
+    "prosemirror-state": "^1.4.4",
+    "prosemirror-tables": "^1.8.5",
+    "prosemirror-transform": "^1.12.0",
+    "prosemirror-view": "^1.42.6",
+    "y-prosemirror": "^1.3.7",
+    "y-protocols": "^1.0.6",
+    "yjs": "^13.6.31"
   },
   "peerDependenciesMeta": {
     "playwright-core": {