@remit/ui 0.0.120 → 0.0.121

package/src/components/rich-text-spellcheck-worker.test.ts

@@ -0,0 +1,253 @@
+/**
+ * The worker against real dictionaries. What runs here is the code that ships
+ * inside the worker, over the Hunspell build in `build/hunspell/` and the
+ * upstream `.aff`/`.dic` pairs the image stages — so a language that passes
+ * here is a language a writer is actually checked in.
+ *
+ * The engine and the dictionaries are fetched at runtime, so the fetch is where
+ * this stands in: file URLs into `build/` and `node_modules/`, byte for byte
+ * what the server would hand a browser.
+ */
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { before, describe, it } from "node:test";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import type {
+	SpellWorkerRequest,
+	SpellWorkerResponse,
+} from "./rich-text-spellcheck.js";
+
+const require = createRequire(import.meta.url);
+const engineDir = fileURLToPath(
+	new URL("../../../../build/hunspell/", import.meta.url),
+);
+const BASE = pathToFileURL(engineDir).href;
+
+const dictionaryDir = (tag: string): string =>
+	dirname(require.resolve(`dictionary-${tag.toLowerCase()}`));
+
+/** The one path that answers 503, so the download failure has a producer. */
+let refuse: string | null = null;
+
+/**
+ * The image serves these files brotli-compressed, so `content-length` is the
+ * compressed length while what arrives is the decompressed file. A quarter
+ * stands in for that ratio: anything reading the header instead of the build's
+ * own figure opens the notice at a size it then walks straight past.
+ */
+const asOnTheWire = (bytes: Buffer): string =>
+	String(Math.max(1, Math.round(bytes.byteLength / 4)));
+
+const served = async (url: string): Promise<Response> => {
+	if (url === refuse) return new Response("no", { status: 503 });
+	const dictionary = /dictionaries\/([^/]+)\/(index\.(?:aff|dic))$/.exec(url);
+	const path = dictionary
+		? join(dictionaryDir(dictionary[1]), dictionary[2])
+		: fileURLToPath(url);
+	const bytes = await readFile(path);
+	return new Response(bytes, {
+		headers: { "content-length": asOnTheWire(bytes) },
+	});
+};
+
+/** What the build weighs a language at: the engine and its two files. */
+const weightOf = async (tag: string): Promise<number> => {
+	const directory = dictionaryDir(tag);
+	const files = await Promise.all([
+		readFile(join(engineDir, "hunspell.wasm")),
+		readFile(join(directory, "index.aff")),
+		readFile(join(directory, "index.dic")),
+	]);
+	return files.reduce((total, bytes) => total + bytes.byteLength, 0);
+};
+
+let post: (message: SpellWorkerRequest) => void;
+const heard: SpellWorkerResponse[] = [];
+const waiting = new Map<string, (message: SpellWorkerResponse) => void>();
+
+const nextMessage = <T extends SpellWorkerResponse["type"]>(
+	type: T,
+): Promise<Extract<SpellWorkerResponse, { type: T }>> =>
+	new Promise((resolve) => {
+		waiting.set(type, (message) =>
+			resolve(message as Extract<SpellWorkerResponse, { type: T }>),
+		);
+	});
+
+const openOn = async (
+	tag: string,
+): Promise<Extract<SpellWorkerResponse, { type: "ready" | "failed" }>> => {
+	const ready = nextMessage("ready");
+	const failed = nextMessage("failed");
+	post({
+		type: "open",
+		language: tag,
+		base: BASE,
+		bytesExpected: await weightOf(tag),
+	});
+	return Promise.race([ready, failed]);
+};
+
+const checked = async (
+	tag: string,
+	text: string,
+): Promise<readonly string[]> => {
+	const answer = nextMessage("checked");
+	post({
+		type: "check",
+		requestId: "1",
+		language: tag,
+		revision: 1,
+		spans: [{ spanId: "a", text }],
+	});
+	const { findings } = await answer;
+	return findings.map((finding) => text.slice(finding.start, finding.end));
+};
+
+const suggested = async (tag: string, word: string): Promise<string[]> => {
+	const answer = nextMessage("suggested");
+	post({ type: "suggest", requestId: "1", language: tag, word });
+	return [...(await answer).suggestions];
+};
+
+before(async () => {
+	Object.defineProperty(globalThis, "fetch", {
+		value: (input: RequestInfo | URL) => served(String(input)),
+		configurable: true,
+	});
+	Object.defineProperty(globalThis, "postMessage", {
+		value: (message: SpellWorkerResponse) => {
+			heard.push(message);
+			const settle = waiting.get(message.type);
+			if (!settle) return;
+			waiting.delete(message.type);
+			settle(message);
+		},
+		configurable: true,
+	});
+	Object.defineProperty(globalThis, "addEventListener", {
+		value: (
+			_type: string,
+			listener: (event: { data: SpellWorkerRequest }) => void,
+		) => {
+			post = (message) => listener({ data: message });
+		},
+		configurable: true,
+	});
+	await import("./rich-text-spellcheck-worker.js");
+});
+
+describe("English, against SCOWL", () => {
+	it("marks what is misspelt and leaves the rest alone", async () => {
+		assert.equal((await openOn("en")).type, "ready");
+		assert.deepEqual(await checked("en", "Ths report is redy today"), [
+			"Ths",
+			"redy",
+		]);
+		assert.deepEqual(
+			await checked("en", "The meeting notes are attached."),
+			[],
+			"a sentence of real words carries no marks",
+		);
+	});
+
+	it("offers the correction a writer meant", async () => {
+		assert.ok(
+			(await suggested("en", "recieve")).includes("receive"),
+			"the engine suggests, and it knows the word",
+		);
+		assert.ok((await suggested("en", "meetign")).includes("meeting"));
+	});
+
+	it("reports the bytes as they arrive", () => {
+		const progress = heard.filter((message) => message.type === "opening");
+		assert.ok(progress.length > 0, "a download that says nothing looks broken");
+		const last = progress.at(-1);
+		assert.ok(last && last.bytesLoaded > 0 && last.bytesTotal > 0);
+	});
+});
+
+describe("British English, against the same source", () => {
+	it("takes the spellings the other English does not", async () => {
+		assert.equal((await openOn("en-GB")).type, "ready");
+		assert.deepEqual(
+			await checked("en-GB", "I will organise the colour of the centre."),
+			[],
+		);
+		assert.deepEqual(
+			await checked("en-GB", "The kolour of the sentre."),
+			["kolour", "sentre"],
+			"a real British misspelling is still a misspelling",
+		);
+	});
+});
+
+describe("Dutch, against OpenTaal", () => {
+	it("knows the words the stub never did", async () => {
+		assert.equal((await openOn("nl")).type, "ready");
+		assert.deepEqual(
+			await checked(
+				"nl",
+				"De vergadering van vanmiddag gaat over de begroting.",
+			),
+			[],
+		);
+	});
+
+	it("marks a Dutch misspelling and corrects it in Dutch", async () => {
+		assert.deepEqual(
+			await checked("nl", "De vergaderingg gaat over de begrooting."),
+			["vergaderingg", "begrooting"],
+		);
+		assert.ok((await suggested("nl", "vergaderingg")).includes("vergadering"));
+	});
+});
+
+describe("what the composer is told the download costs", () => {
+	it("names the size the build weighed, not the length on the wire", async () => {
+		const from = heard.length;
+		assert.equal((await openOn("nl")).type, "ready");
+		const progress = heard
+			.slice(from)
+			.filter((message) => message.type === "opening");
+		const weight = await weightOf("nl");
+
+		assert.ok(progress.length > 1, "a download reported once is not reported");
+		assert.deepEqual(
+			[...new Set(progress.map((message) => message.bytesTotal))],
+			[weight],
+			"a figure that is rewritten mid-download was never the size of anything",
+		);
+		assert.equal(
+			progress.at(-1)?.bytesLoaded,
+			weight,
+			"and what arrives ends on it rather than past it",
+		);
+	});
+});
+
+describe("when the dictionary does not arrive", () => {
+	it("says which file and what it answered", async () => {
+		refuse = `${BASE}dictionaries/nl/index.dic`;
+		const answer = await openOn("nl");
+		refuse = null;
+
+		assert.equal(answer.type, "failed");
+		assert.equal(answer.type === "failed" && answer.reason, "download");
+		assert.match(
+			answer.type === "failed" ? answer.detail : "",
+			/index\.dic answered 503/,
+			"the status is what a report link has to carry",
+		);
+	});
+
+	it("marks nothing rather than everything once it has failed", async () => {
+		assert.deepEqual(
+			await checked("nl", "De vergaderingg gaat over de begrooting."),
+			[],
+			"the browser's own checker is the one on screen now",
+		);
+	});
+});

package/src/components/rich-text-spellcheck-worker.ts

@@ -1,8 +1,12 @@
 /**
  * The checking side, off the main thread. It answers with the revision it was
  * asked at and never touches the document, which is what lets the editor throw
- * a late answer away. The word list it consults is a stub; the engine that
- * replaces it changes nothing above this file.
+ * a late answer away.
+ *
+ * One worker holds one language's engine. Everything before `ready` is a
+ * download, reported as it goes, and every way it can end badly is a message
+ * the composer can put words to — never silence, because the browser's own
+ * checker is still on and the writer must not be left with neither.
  */
 
 import type {
@@ -11,10 +15,11 @@ import type {
 	SpellWorkerRequest,
 	SpellWorkerResponse,
 } from "./rich-text-spellcheck.js";
+import type { SpellEngine } from "./rich-text-spellcheck-engine.js";
+import { openEngine } from "./rich-text-spellcheck-engine.js";
 import {
-	dictionaryFor,
 	findMisspellings,
-	suggestionsFor,
+	SUGGESTION_LIMIT,
 } from "./rich-text-spellcheck-words.js";
 
 interface WorkerScope {
@@ -25,11 +30,13 @@ interface WorkerScope {
 	): void;
 }
 
-const spell = (request: CheckRequest): Finding[] => {
-	const words = dictionaryFor(request.language);
-	if (!words) return [];
-	return request.spans.flatMap((span) =>
-		findMisspellings(span.text, words).map((range) => ({
+const scope = globalThis as unknown as WorkerScope;
+
+let engine: SpellEngine | null = null;
+
+const spell = (request: CheckRequest, checker: SpellEngine): Finding[] =>
+	request.spans.flatMap((span) =>
+		findMisspellings(span.text, (word) => checker.spell(word)).map((range) => ({
 			spanId: span.spanId,
 			start: range.start,
 			end: range.end,
@@ -37,22 +44,63 @@ const spell = (request: CheckRequest): Finding[] => {
 			suggestions: [],
 		})),
 	);
-};
 
-const scope = globalThis as unknown as WorkerScope;
+const open = async (
+	language: string,
+	base: string,
+	bytesExpected: number,
+): Promise<void> => {
+	// Whatever was answering before is not answering for this language, and a
+	// failed open must not leave the previous dictionary marking the new text.
+	engine?.close();
+	engine = null;
+	const result = await openEngine({
+		base,
+		tag: language,
+		bytesExpected,
+		onProgress: ({ bytesLoaded, bytesTotal }) =>
+			scope.postMessage({
+				type: "opening",
+				language,
+				bytesLoaded,
+				bytesTotal,
+			}),
+	});
+	if (!result.ok) {
+		scope.postMessage({
+			type: "failed",
+			language,
+			reason: result.reason,
+			detail: result.detail,
+		});
+		return;
+	}
+	engine = result.engine;
+	scope.postMessage({ type: "ready", language });
+};
 
 scope.addEventListener("message", ({ data }) => {
 	if (data.type === "open") {
-		scope.postMessage({ type: "ready", language: data.language });
+		open(data.language, data.base, data.bytesExpected).catch(
+			(error: unknown) => {
+				scope.postMessage({
+					type: "failed",
+					language: data.language,
+					reason: "engine",
+					detail: error instanceof Error ? error.message : String(error),
+				});
+			},
+		);
 		return;
 	}
 	if (data.type === "suggest") {
-		const words = dictionaryFor(data.language);
 		scope.postMessage({
 			type: "suggested",
 			requestId: data.requestId,
 			word: data.word,
-			suggestions: words ? suggestionsFor(data.word, words) : [],
+			suggestions: engine
+				? engine.suggest(data.word).slice(0, SUGGESTION_LIMIT)
+				: [],
 		});
 		return;
 	}
@@ -60,6 +108,6 @@ scope.addEventListener("message", ({ data }) => {
 		type: "checked",
 		requestId: data.requestId,
 		revision: data.revision,
-		findings: spell(data),
+		findings: engine ? spell(data, engine) : [],
 	});
 });

package/src/components/rich-text-spellcheck.test.ts

@@ -22,12 +22,19 @@ import type {
 	ProviderStatus,
 	SpellcheckOptions,
 	SpellProvider,
+	SpellWorkerRequest,
+	SpellWorkerResponse,
 } from "./rich-text-spellcheck.js";
 import {
-	dictionaryFor,
-	findMisspellings,
-	suggestionsFor,
-} from "./rich-text-spellcheck-words.js";
+	stubKnows,
+	stubSuggestionsFor,
+} from "./rich-text-spellcheck-double.js";
+import {
+	CHECK_DEADLINE_MS,
+	openSpellProvider,
+	type SpellWorkerPort,
+} from "./rich-text-spellcheck-provider.js";
+import { findMisspellings } from "./rich-text-spellcheck-words.js";
 
 const SENTENCE = "Ths report is redy today";
 const IDLE_MS = 400;
@@ -77,19 +84,20 @@ const stubSpellcheck = (
 	tune: {
 		revisionOf?: (request: CheckRequest, nth: number) => number;
 		hold?: boolean;
+		/** Opens on a dictionary still arriving, the way a real one does. */
+		downloading?: boolean;
 	} = {},
 ): Stub => {
 	const asked: CheckRequest[] = [];
 	const held: (() => void)[] = [];
 	const listeners = new Set<(status: ProviderStatus) => void>();
 	let closed = 0;
-	const words = dictionaryFor("en") ?? new Set<string>();
 
 	const answer = (request: CheckRequest, nth: number): CheckResponse => ({
 		requestId: request.requestId,
 		revision: tune.revisionOf?.(request, nth) ?? request.revision,
 		findings: request.spans.flatMap((span) =>
-			findMisspellings(span.text, words).map(
+			findMisspellings(span.text, stubKnows).map(
 				(range): Finding => ({
 					spanId: span.spanId,
 					start: range.start,
@@ -105,7 +113,16 @@ const stubSpellcheck = (
 		language: "en",
 		onStatus: (listener) => {
 			listeners.add(listener);
-			listener({ state: "ready", language: "en" });
+			listener(
+				tune.downloading
+					? {
+							state: "opening",
+							language: "en",
+							bytesLoaded: 0,
+							bytesTotal: 167_936,
+						}
+					: { state: "ready", language: "en" },
+			);
 			return () => {
 				listeners.delete(listener);
 			};
@@ -122,7 +139,7 @@ const stubSpellcheck = (
 			Promise.resolve({
 				requestId: request.requestId,
 				word: request.word,
-				suggestions: suggestionsFor(request.word, words),
+				suggestions: stubSuggestionsFor(request.word),
 			}),
 		close: () => {
 			closed += 1;
@@ -532,6 +549,100 @@ describe("spellcheck marks", () => {
 		assert.equal(seen.at(-1)?.state, "failed", "the caller is told why");
 	});
 
+	it("checks the document the download was holding up", async () => {
+		const spellcheck = stubSpellcheck({ downloading: true });
+		await mount({
+			initialHtml: `<p>${SENTENCE}</p>`,
+			lang: "en",
+			spellcheck,
+		});
+		await settle();
+		assert.equal(
+			marks(),
+			undefined,
+			"nothing of ours is on screen while the dictionary is on its way",
+		);
+		assert.equal(editable().getAttribute("spellcheck"), "true");
+
+		// A real dictionary arrives in twenty of these before it is ready.
+		await act(async () => {
+			for (const bytesLoaded of [65_536, 131_072, 167_936]) {
+				spellcheck.push({
+					state: "opening",
+					language: "en",
+					bytesLoaded,
+					bytesTotal: 167_936,
+				});
+			}
+		});
+		await act(async () => {
+			spellcheck.push({ state: "ready", language: "en" });
+		});
+		await settle();
+
+		assert.deepEqual(
+			offsets(),
+			[
+				[0, 3],
+				[14, 18],
+			],
+			"the leaves that were waiting on the dictionary are the first pass",
+		);
+	});
+
+	it("hands checking back when the engine takes a pass and freezes", async () => {
+		const seen: ProviderStatus[] = [];
+		const posted: SpellWorkerRequest[] = [];
+		let deliver: ((message: SpellWorkerResponse) => void) | undefined;
+		// A worker that opens, says it is ready, and then answers nothing at all —
+		// a wedged WebAssembly instance, which no `error` event ever announces.
+		const dead: SpellWorkerPort = {
+			post: (message) => posted.push(message),
+			listen: (listener) => {
+				deliver = listener;
+			},
+			fail: () => {},
+			terminate: () => {},
+		};
+
+		await mount({
+			initialHtml: `<p>${SENTENCE}</p>`,
+			lang: "en",
+			spellcheck: {
+				provider: async (language: string) =>
+					openSpellProvider(language, "/spellcheck/", dead),
+				onStatus: (status: ProviderStatus) => seen.push(status),
+			},
+		});
+		await act(async () => {
+			deliver?.({ type: "ready", language: "en" });
+		});
+		await settle();
+		assert.ok(
+			posted.some((message) => message.type === "check"),
+			"the pass went out",
+		);
+		assert.equal(
+			editable().getAttribute("spellcheck"),
+			"false",
+			"ours is the checker on screen while it is answering",
+		);
+
+		await act(async () => {
+			await new Promise((resolve) =>
+				setTimeout(resolve, CHECK_DEADLINE_MS + IDLE_MS),
+			);
+		});
+
+		assert.equal(seen.at(-1)?.state, "failed", "silence is named, not endured");
+		assert.equal(
+			editable().getAttribute("spellcheck"),
+			"true",
+			"the browser checks again rather than nobody checking",
+		);
+		assert.equal(marks(), undefined);
+	});
+
 	it("says when a language has no dictionary", async () => {
 		const seen: ProviderStatus[] = [];
 		await mount({
@@ -544,7 +655,10 @@ describe("spellcheck marks", () => {
 		});
 		await settle();
 
-		assert.deepEqual(seen, [{ state: "unavailable", language: "de" }]);
+		assert.deepEqual(seen, [
+			{ state: "opening", language: "de", bytesLoaded: 0, bytesTotal: 0 },
+			{ state: "unavailable", language: "de" },
+		]);
 		assert.equal(editable().getAttribute("spellcheck"), "true");
 	});
 

package/src/components/rich-text-spellcheck.ts

@@ -52,7 +52,19 @@ export interface SuggestResponse {
 }
 
 export type ProviderStatus =
-	| { readonly state: "opening"; readonly language: LanguageTag }
+	/**
+	 * Carries the download rather than a verdict about it: a composer that has to
+	 * say "still fetching Dutch (2.4 MB)" needs the two numbers, and where the
+	 * line between quick and slow falls belongs to whatever renders it, not here.
+	 * The total is what the build weighed the files at, so it is right from the
+	 * first report and never moves under the writer.
+	 */
+	| {
+			readonly state: "opening";
+			readonly language: LanguageTag;
+			readonly bytesLoaded: number;
+			readonly bytesTotal: number;
+	  }
 	| { readonly state: "ready"; readonly language: LanguageTag }
 	/** Nothing here checks this language, and the browser is welcome to. */
 	| { readonly state: "unavailable"; readonly language: LanguageTag }
@@ -85,11 +97,28 @@ export interface SpellcheckOptions {
 }
 
 export type SpellWorkerRequest =
-	| { readonly type: "open"; readonly language: LanguageTag }
+	/**
+	 * `base` is where the engine and the dictionaries are served from, decided by
+	 * the build and passed in rather than read inside the worker — so what the
+	 * worker fetches is visible at the seam and can be pointed elsewhere.
+	 */
+	| {
+			readonly type: "open";
+			readonly language: LanguageTag;
+			readonly base: string;
+			/** What the build weighed these files at, for the download notice. */
+			readonly bytesExpected: number;
+	  }
 	| ({ readonly type: "check" } & CheckRequest)
 	| ({ readonly type: "suggest" } & SuggestRequest);
 
 export type SpellWorkerResponse =
+	| {
+			readonly type: "opening";
+			readonly language: LanguageTag;
+			readonly bytesLoaded: number;
+			readonly bytesTotal: number;
+	  }
 	| { readonly type: "ready"; readonly language: LanguageTag }
 	| {
 			readonly type: "failed";

package/src/rich-text.ts

@@ -55,9 +55,9 @@ export {
 	type SpellWorkerPort,
 } from "./components/rich-text-spellcheck-provider.js";
 export {
+	findMisspellings,
 	normaliseWord,
 	SUGGESTION_LIMIT,
-	suggestionsFor,
 } from "./components/rich-text-spellcheck-words.js";
 export {
 	type ComposeCaret,