harper.js 0.13.0

package/package.json

@@ -0,0 +1,24 @@
+{
+	"name": "harper.js",
+	"version": "0.13.0",
+	"type": "module",
+	"scripts": {
+		"dev": "vite",
+		"build": "tsc && vite build",
+		"test": "vitest run"
+	},
+	"dependencies": {
+		"wasm": "link:../../harper-wasm/pkg"
+	},
+	"devDependencies": {
+		"@vitest/browser": "^2.1.8",
+		"playwright": "^1.49.1",
+		"typescript": "~5.6.2",
+		"vite": "^5.1.8",
+		"vite-plugin-dts": "^4.3.0",
+		"vite-plugin-virtual": "^0.3.0",
+		"vitest": "^2.1.8"
+	},
+	"main": "dist/harper.js",
+	"types": "dist/harper.d.ts"
+}

package/src/Linter.test.ts

@@ -0,0 +1,135 @@
+import { expect, test } from 'vitest';
+import WorkerLinter from './WorkerLinter';
+import LocalLinter from './LocalLinter';
+
+const linters = {
+	WorkerLinter: WorkerLinter,
+	LocalLinter: LocalLinter
+};
+
+for (const [linterName, Linter] of Object.entries(linters)) {
+	test(`${linterName} detects repeated words`, async () => {
+		const linter = new Linter();
+
+		const lints = await linter.lint('The the problem is...');
+
+		expect(lints.length).toBe(1);
+	});
+
+	test(`${linterName} detects repeated words with multiple synchronous requests`, async () => {
+		const linter = new Linter();
+
+		const promises = [
+			linter.lint('The problem is that that...'),
+			linter.lint('The problem is...'),
+			linter.lint('The the problem is...')
+		];
+
+		const results = [];
+
+		for (const promise of promises) {
+			results.push(await promise);
+		}
+
+		expect(results[0].length).toBe(1);
+		expect(results[0][0].suggestions().length).toBe(1);
+		expect(results[1].length).toBe(0);
+		expect(results[2].length).toBe(1);
+	});
+
+	test(`${linterName} detects repeated words with concurrent requests`, async () => {
+		const linter = new Linter();
+
+		const promises = [
+			linter.lint('The problem is that that...'),
+			linter.lint('The problem is...'),
+			linter.lint('The the problem is...')
+		];
+
+		const results = await Promise.all(promises);
+
+		expect(results[0].length).toBe(1);
+		expect(results[0][0].suggestions().length).toBe(1);
+		expect(results[1].length).toBe(0);
+		expect(results[2].length).toBe(1);
+	});
+
+	test(`${linterName} detects lorem ipsum paragraph as not english`, async () => {
+		const linter = new Linter();
+
+		const result = await linter.isLikelyEnglish(
+			'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'
+		);
+
+		expect(result).toBeTypeOf('boolean');
+		expect(result).toBe(false);
+	});
+
+	test(`${linterName} can run setup without issues`, async () => {
+		const linter = new Linter();
+
+		await linter.setup();
+	});
+
+	test(`${linterName} contains configuration option for repetition`, async () => {
+		const linter = new Linter();
+
+		const lintConfig = await linter.getLintConfig();
+		expect(lintConfig).toHaveProperty('repeated_words');
+	});
+
+	test(`${linterName} can both get and set its configuration`, async () => {
+		const linter = new Linter();
+
+		let lintConfig = await linter.getLintConfig();
+
+		for (const key of Object.keys(lintConfig)) {
+			lintConfig[key] = true;
+		}
+
+		await linter.setLintConfig(lintConfig);
+		lintConfig = await linter.getLintConfig();
+
+		for (const key of Object.keys(lintConfig)) {
+			expect(lintConfig[key]).toBe(true);
+		}
+	});
+
+	test(`${linterName} can make things title case`, async () => {
+		const linter = new Linter();
+
+		const titleCase = await linter.toTitleCase('THIS IS A TEST FOR MAKING TITLES');
+
+		expect(titleCase).toBe('This Is a Test for Making Titles');
+	});
+}
+
+test('Linters have the same config format', async () => {
+	const configs = [];
+
+	for (const Linter of Object.values(linters)) {
+		const linter = new Linter();
+
+		configs.push(await linter.getLintConfig());
+	}
+
+	for (const config of configs) {
+		expect(config).toEqual(configs[0]);
+		expect(config).toBeTypeOf('object');
+	}
+});
+
+test('Linters have the same JSON config format', async () => {
+	const configs = [];
+
+	for (const Linter of Object.values(linters)) {
+		const linter = new Linter();
+
+		configs.push(await linter.getLintConfigAsJSON());
+	}
+
+	for (const config of configs) {
+		expect(config).toEqual(configs[0]);
+		expect(config).toBeTypeOf('string');
+	}
+});

package/src/Linter.ts

@@ -0,0 +1,35 @@
+import type { Lint, Span, Suggestion } from 'wasm';
+import { LintConfig } from './main';
+
+/** A interface for an object that can perform linting actions. */
+export default interface Linter {
+	/** Complete any setup that is necessary before linting. This may include downloading and compiling the WebAssembly binary.
+	 * This setup will complete when needed regardless of whether you call this function.
+	 * This function exists to allow you to do this work when it is of least impact to the user experiences (i.e. while you're loading something else). */
+	setup(): Promise<void>;
+	/** Lint the provided text. */
+	lint(text: string): Promise<Lint[]>;
+	/** Apply a suggestion to the given text, returning the transformed result. */
+	applySuggestion(text: string, suggestion: Suggestion, span: Span): Promise<string>;
+	/** Determine if the provided text is likely to be intended to be English.
+	 * The algorithm can be described as "proof of concept" and as such does not work terribly well.*/
+	isLikelyEnglish(text: string): Promise<boolean>;
+	/** Determine which parts of a given string are intended to be English, returning those bits.
+	 * The algorithm can be described as "proof of concept" and as such does not work terribly well.*/
+	isolateEnglish(text: string): Promise<string>;
+
+	/** Get the linter's current configuration. */
+	getLintConfig(): Promise<LintConfig>;
+
+	/** Set the linter's current configuration. */
+	setLintConfig(config: LintConfig): Promise<void>;
+
+	/** Get the linter's current configuration as JSON. */
+	getLintConfigAsJSON(): Promise<string>;
+
+	/** Set the linter's current configuration from JSON. */
+	setLintConfigWithJSON(config: string): Promise<void>;
+
+	/** Convert a string to Chicago-style title case. */
+	toTitleCase(text: string): Promise<string>;
+}

package/src/LocalLinter.ts

@@ -0,0 +1,77 @@
+import type { Lint, Span, Suggestion, Linter as WasmLinter } from 'wasm';
+import Linter from './Linter';
+import loadWasm from './loadWasm';
+import { LintConfig } from './main';
+
+/** A Linter that runs in the current JavaScript context (meaning it is allowed to block the event loop). */
+export default class LocalLinter implements Linter {
+	private inner: WasmLinter | undefined;
+
+	/** Initialize the WebAssembly and construct the inner Linter. */
+	private async initialize(): Promise<void> {
+		if (!this.inner) {
+			const wasm = await loadWasm();
+			wasm.setup();
+			this.inner = wasm.Linter.new();
+		}
+	}
+
+	async setup(): Promise<void> {
+		await this.initialize();
+		this.inner!.lint('');
+	}
+
+	async lint(text: string): Promise<Lint[]> {
+		await this.initialize();
+		let lints = this.inner!.lint(text);
+
+		// We only want to show fixable errors.
+		lints = lints.filter((lint) => lint.suggestion_count() > 0);
+
+		return lints;
+	}
+
+	async applySuggestion(text: string, suggestion: Suggestion, span: Span): Promise<string> {
+		const wasm = await loadWasm();
+		return wasm.apply_suggestion(text, span, suggestion);
+	}
+
+	async isLikelyEnglish(text: string): Promise<boolean> {
+		await this.initialize();
+		return this.inner!.is_likely_english(text);
+	}
+
+	async isolateEnglish(text: string): Promise<string> {
+		await this.initialize();
+		return this.inner!.isolate_english(text);
+	}
+
+	async getLintConfig(): Promise<LintConfig> {
+		await this.initialize();
+
+		return this.inner!.get_lint_config_as_object();
+	}
+
+	async setLintConfig(config: LintConfig): Promise<void> {
+		await this.initialize();
+
+		this.inner!.set_lint_config_from_object(config);
+	}
+
+	async getLintConfigAsJSON(): Promise<string> {
+		await this.initialize();
+
+		return this.inner!.get_lint_config_as_json();
+	}
+
+	async setLintConfigWithJSON(config: string): Promise<void> {
+		await this.initialize();
+
+		this.inner!.set_lint_config_from_json(config);
+	}
+
+	async toTitleCase(text: string): Promise<string> {
+		const wasm = await loadWasm();
+		return wasm.to_title_case(text);
+	}
+}

package/src/WorkerLinter/communication.test.ts

@@ -0,0 +1,63 @@
+import { expect, test } from 'vitest';
+import { deserializeArg, serializeArg } from './communication';
+import { Span } from 'wasm';
+import LocalLinter from '../LocalLinter';
+
+test('works with strings', async () => {
+	const start = 'This is a string';
+
+	const end = await deserializeArg(structuredClone(await serializeArg(start)));
+
+	expect(end).toBe(start);
+	expect(typeof end).toBe(typeof start);
+});
+
+test('works with false booleans', async () => {
+	const start = false;
+
+	const end = await deserializeArg(structuredClone(await serializeArg(start)));
+
+	expect(end).toBe(start);
+	expect(typeof end).toBe(typeof start);
+});
+
+test('works with true booleans', async () => {
+	const start = true;
+
+	const end = await deserializeArg(structuredClone(await serializeArg(start)));
+
+	expect(end).toBe(start);
+	expect(typeof end).toBe(typeof start);
+});
+
+test('works with numbers', async () => {
+	const start = 123;
+
+	const end = await deserializeArg(structuredClone(await serializeArg(start)));
+
+	expect(end).toBe(start);
+	expect(typeof end).toBe(typeof start);
+});
+
+test('works with Spans', async () => {
+	const start = Span.new(123, 321);
+
+	const end = await deserializeArg(structuredClone(await serializeArg(start)));
+
+	expect(end.start).toBe(start.start);
+	expect(end.len()).toBe(start.len());
+	expect(typeof end).toBe(typeof start);
+});
+
+test('works with Lints', async () => {
+	const linter = new LocalLinter();
+	const lints = await linter.lint('This is an test.');
+	const start = lints[0];
+
+	expect(start).not.toBeNull();
+
+	const end = await deserializeArg(structuredClone(await serializeArg(start)));
+
+	expect(end.message()).toBe(start.message());
+	expect(end.lint_kind()).toBe(start.lint_kind());
+});

package/src/WorkerLinter/communication.ts

@@ -0,0 +1,111 @@
+/** This module aims to define the communication protocol between the main thread and the worker.
+ * Note that most of the complication here comes from the fact that we can't serialize function calls or referenced WebAssembly memory.*/
+
+import loadWasm from '../loadWasm';
+
+export type Type =
+	| 'string'
+	| 'number'
+	| 'boolean'
+	| 'Suggestion'
+	| 'Lint'
+	| 'Span'
+	| 'Array'
+	| 'undefined';
+
+/** Serializable argument to a procedure to be run on the web worker. */
+export type RequestArg = {
+	json: string;
+	type: Type;
+};
+
+export async function serialize(req: DeserializedRequest): Promise<SerializedRequest> {
+	return {
+		procName: req.procName,
+		args: await Promise.all(req.args.map(serializeArg))
+	};
+}
+
+export async function serializeArg(arg: any): Promise<RequestArg> {
+	const { Lint, Span, Suggestion } = await loadWasm();
+
+	if (Array.isArray(arg)) {
+		return { json: JSON.stringify(await Promise.all(arg.map(serializeArg))), type: 'Array' };
+	}
+
+	switch (typeof arg) {
+		case 'string':
+		case 'number':
+		case 'boolean':
+		case 'undefined':
+			// @ts-expect-error see the `Type` type.
+			return { json: JSON.stringify(arg), type: typeof arg };
+	}
+
+	if (arg.to_json != undefined) {
+		const json = arg.to_json();
+		let type: Type | undefined = undefined;
+
+		if (arg instanceof Lint) {
+			type = 'Lint';
+		} else if (arg instanceof Suggestion) {
+			type = 'Suggestion';
+		} else if (arg instanceof Span) {
+			type = 'Span';
+		}
+
+		if (type == undefined) {
+			throw new Error('Unhandled case');
+		}
+
+		return { json, type };
+	}
+
+	throw new Error('Unhandled case');
+}
+
+export async function deserializeArg(requestArg: RequestArg): Promise<any> {
+	const { Lint, Span, Suggestion } = await loadWasm();
+
+	switch (requestArg.type) {
+		case 'undefined':
+			return undefined;
+		case 'boolean':
+		case 'number':
+		case 'string':
+			return JSON.parse(requestArg.json);
+		case 'Suggestion':
+			return Suggestion.from_json(requestArg.json);
+		case 'Lint':
+			return Lint.from_json(requestArg.json);
+		case 'Span':
+			return Span.from_json(requestArg.json);
+		case 'Array':
+			return await Promise.all(JSON.parse(requestArg.json).map(deserializeArg));
+		default:
+			throw new Error(`Unhandled case: ${requestArg.type}`);
+	}
+}
+
+/** An object that is sent to the web worker to request work to be done. */
+export type SerializedRequest = {
+	/** The procedure to be executed. */
+	procName: string;
+	/** The arguments to the procedure */
+	args: RequestArg[];
+};
+
+/** An object that is received by the web worker to request work to be done. */
+export type DeserializedRequest = {
+	/** The procedure to be executed. */
+	procName: string;
+	/** The arguments to the procedure */
+	args: any[];
+};
+
+export async function deserialize(request: SerializedRequest): Promise<DeserializedRequest> {
+	return {
+		procName: request.procName,
+		args: await Promise.all(request.args.map(deserializeArg))
+	};
+}

package/src/WorkerLinter/index.ts

@@ -0,0 +1,131 @@
+import { DeserializedRequest, deserializeArg, serialize } from './communication';
+import type { Lint, Suggestion, Span } from 'wasm';
+import Linter from '../Linter';
+import Worker from './worker.js?worker&inline';
+import { getWasmUri } from '../loadWasm';
+import { LintConfig } from '../main';
+
+/** The data necessary to complete a request once the worker has responded. */
+type RequestItem = {
+	resolve: (item: unknown) => void;
+	reject: (item: unknown) => void;
+	request: DeserializedRequest;
+};
+
+/** A Linter that spins up a dedicated web worker to do processing on a separate thread.
+ * Main benefit: this Linter will not block the event loop for large documents.
+ *
+ * NOTE: This class will not work properly in Node. In that case, just use `LocalLinter`.
+ * Also requires top-level await to work. */
+export default class WorkerLinter implements Linter {
+	private worker;
+	private requestQueue: RequestItem[];
+	private working = true;
+
+	constructor() {
+		this.worker = new Worker();
+		this.requestQueue = [];
+
+		// Fires when the worker sends 'ready'.
+		this.worker.onmessage = () => {
+			this.setupMainEventListeners();
+
+			this.worker.postMessage(getWasmUri());
+
+			this.working = false;
+			this.submitRemainingRequests();
+		};
+	}
+
+	private setupMainEventListeners() {
+		this.worker.onmessage = (e: MessageEvent) => {
+			const { resolve } = this.requestQueue.shift()!;
+			deserializeArg(e.data).then((v) => {
+				resolve(v);
+
+				this.working = false;
+
+				this.submitRemainingRequests();
+			});
+		};
+
+		this.worker.onmessageerror = (e: MessageEvent) => {
+			const { reject } = this.requestQueue.shift()!;
+			reject(e.data);
+			this.working = false;
+
+			this.submitRemainingRequests();
+		};
+	}
+
+	setup(): Promise<void> {
+		return this.rpc('setup', []);
+	}
+
+	lint(text: string): Promise<Lint[]> {
+		return this.rpc('lint', [text]);
+	}
+
+	applySuggestion(text: string, suggestion: Suggestion, span: Span): Promise<string> {
+		return this.rpc('applySuggestion', [text, suggestion, span]);
+	}
+
+	isLikelyEnglish(text: string): Promise<boolean> {
+		return this.rpc('isLikelyEnglish', [text]);
+	}
+
+	isolateEnglish(text: string): Promise<string> {
+		return this.rpc('isolateEnglish', [text]);
+	}
+
+	async getLintConfig(): Promise<LintConfig> {
+		return JSON.parse(await this.getLintConfigAsJSON());
+	}
+
+	setLintConfig(config: LintConfig): Promise<void> {
+		return this.setLintConfigWithJSON(JSON.stringify(config));
+	}
+
+	getLintConfigAsJSON(): Promise<string> {
+		return this.rpc('getLintConfigAsJSON', []);
+	}
+
+	setLintConfigWithJSON(config: string): Promise<void> {
+		return this.rpc('setLintConfigWithJSON', [config]);
+	}
+
+	toTitleCase(text: string): Promise<string> {
+		return this.rpc('toTitleCase', [text]);
+	}
+
+	/** Run a procedure on the remote worker. */
+	private async rpc(procName: string, args: any[]): Promise<any> {
+		const promise = new Promise((resolve, reject) => {
+			this.requestQueue.push({
+				resolve,
+				reject,
+				request: { procName, args }
+			});
+
+			this.submitRemainingRequests();
+		});
+
+		return promise;
+	}
+
+	private async submitRemainingRequests() {
+		if (this.working) {
+			return;
+		}
+
+		this.working = true;
+
+		if (this.requestQueue.length > 0) {
+			const { request } = this.requestQueue[0];
+
+			this.worker.postMessage(await serialize(request));
+		} else {
+			this.working = false;
+		}
+	}
+}

package/src/WorkerLinter/worker.js

@@ -0,0 +1,24 @@
+import { setWasmUri } from '../loadWasm';
+import LocalLinter from '../LocalLinter';
+import { deserialize, serializeArg } from './communication';
+
+const linter = new LocalLinter();
+
+/** @param {SerializedRequest} v  */
+async function processRequest(v) {
+	const { procName, args } = await deserialize(v);
+
+	let res = await linter[procName](...args);
+	postMessage(await serializeArg(res));
+}
+
+self.onmessage = function (e) {
+	setWasmUri(e.data);
+
+	self.onmessage = function (e) {
+		processRequest(e.data);
+	};
+};
+
+// Notify the main thread that we are ready
+postMessage('ready');

package/src/loadWasm.ts

@@ -0,0 +1,26 @@
+// @ts-expect-error because this virtual module hasn't been added to a `d.ts` file.
+import wasmUri from 'virtual:wasm';
+
+let curWasmUri = wasmUri;
+
+/** Get the currently set data URI for the WebAssembly module.
+ * I'm not a huge of the singleton, but we're swapping out same data, just from a different source, so the state doesn't meaningfully change. */
+export function getWasmUri(): string {
+	return curWasmUri;
+}
+
+/** Set the data URI for the WebAssembly module. */
+export function setWasmUri(uri: string) {
+	curWasmUri = uri;
+}
+
+/** Load the WebAssembly manually and dynamically, making sure to setup infrastructure.
+ * You can use an optional data URL for the WebAssembly file if the module is being loaded from a Web Worker.
+ * */
+export default async function loadWasm() {
+	const wasm = await import('wasm');
+	// @ts-ignore
+	await wasm.default({ module_or_path: getWasmUri() });
+
+	return wasm;
+}

package/src/main.test.ts

@@ -0,0 +1,8 @@
+import { expect, test } from 'vitest';
+import { SuggestionKind as WasmSuggestionKind } from 'wasm';
+import { SuggestionKind } from './main';
+
+test('Wasm and JS SuggestionKinds agree', async () => {
+	expect(SuggestionKind.Remove).toBe(WasmSuggestionKind.Remove);
+	expect(SuggestionKind.Replace).toBe(WasmSuggestionKind.Replace);
+});

package/src/main.ts

@@ -0,0 +1,14 @@
+import type { Lint, Span, Suggestion } from 'wasm';
+import Linter from './Linter';
+import LocalLinter from './LocalLinter';
+import WorkerLinter from './WorkerLinter';
+
+export { LocalLinter, WorkerLinter };
+export type { Linter, Lint, Span, Suggestion };
+
+export enum SuggestionKind {
+	Replace = 0,
+	Remove = 1
+}
+
+export type LintConfig = Record<string, boolean | undefined>;

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+	"compilerOptions": {
+		"target": "ES2020",
+		"useDefineForClassFields": true,
+		"module": "ESNext",
+		"lib": ["ES2020", "DOM", "DOM.Iterable"],
+		"skipLibCheck": true,
+		/* Bundler mode */
+		"moduleResolution": "bundler",
+		"allowImportingTsExtensions": true,
+		"isolatedModules": true,
+		"moduleDetection": "force",
+		"noEmit": true,
+		/* Linting */
+		"strict": true,
+		"noUnusedLocals": true,
+		"noUnusedParameters": true,
+		"noFallthroughCasesInSwitch": true,
+		"types": ["vite/client"]
+	},
+	"include": ["src"]
+}