@live-codes/ocaml 0.0.0-stage → 0.1.1

package/bin/copy-assets.mjs

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+// Copies the assets an implementation needs into a directory you serve.
+//
+// A page cannot read a file out of node_modules, so the runtime assets have to
+// be published by whatever serves the page. This writes the set for one
+// implementation at a time, so using the compiler does not mean serving the
+// interpreter's assets or the other way round.
+import { cp, mkdir, readdir } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ASSETS = fileURLToPath(new URL('../assets/', import.meta.url));
+
+const USAGE = `Copy the OCaml assets that ship with this package into a directory you serve.
+
+  ocaml-copy-assets [directory] [--compiler|--interpreter|--all]
+
+  directory      where to write them (default: ./ocaml)
+  --compiler     the compiler: the two compiler programs and the stdlib (default)
+  --interpreter  the toplevel: its runtime, interfaces and libraries
+  --all          both, each in its own subdirectory
+  --print-path   print the packaged assets directory and exit
+  --help         print this
+
+The default is --compiler, so a compiler user serves nothing they do not need.
+`;
+
+const args = process.argv.slice(2);
+if (args.includes('--help') || args.includes('-h')) {
+	console.log(USAGE);
+	process.exit(0);
+}
+if (args.includes('--print-path')) {
+	console.log(ASSETS);
+	process.exit(0);
+}
+
+const which = args.includes('--all')
+	? ['compiler', 'interpreter']
+	: args.includes('--interpreter')
+		? ['interpreter']
+		: ['compiler'];
+
+const root = resolve(args.find((arg) => !arg.startsWith('-')) ?? 'ocaml');
+
+// One implementation is written straight into the target directory so that
+// `baseUrl` is simply that directory; both are nested, because each has its own
+// `stdlib/` and they would collide.
+const targetFor = (implementation) =>
+	which.length === 1 ? root : resolve(root, implementation);
+
+for (const implementation of which) {
+	const source = resolve(ASSETS, implementation);
+	const target = targetFor(implementation);
+	await mkdir(target, { recursive: true });
+	for (const entry of await readdir(source, { withFileTypes: true })) {
+		await cp(resolve(source, entry.name), resolve(target, entry.name), {
+			recursive: true,
+			force: true
+		});
+	}
+	console.log(`${implementation} assets copied to ${target}`);
+}
+
+console.log('');
+console.log(`Serve ${which.length === 1 ? 'that directory' : 'those directories'} and pass the URL as baseUrl,`);
+console.log('for example:');
+for (const implementation of which) {
+	console.log(
+		`  ${implementation}: ` +
+			"new URL('" +
+			(which.length === 1 ? '/ocaml/' : `/${implementation}/`) +
+			"', location.href)"
+	);
+}

package/package.json

@@ -1,6 +1,43 @@
 {
   "name": "@live-codes/ocaml",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Run OCaml in the browser with no server: a compiler that emits JavaScript, and a toplevel that runs OCaml directly",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/live-codes/browser-ocaml.git",
+    "directory": "packages/ocaml"
+  },
+  "homepage": "https://github.com/live-codes/browser-ocaml#readme",
+  "bugs": "https://github.com/live-codes/browser-ocaml/issues",
+  "type": "module",
+  "license": "LGPL-2.1-or-later WITH OCaml-LGPL-linking-exception",
+  "exports": {
+    ".": "./src/index.js",
+    "./compiler": "./src/compiler/index.js",
+    "./interpreter": "./src/interpreter/index.js"
+  },
+  "bin": {
+    "ocaml-copy-assets": "./bin/copy-assets.mjs"
+  },
+  "files": [
+    "src",
+    "bin",
+    "assets",
+    "README.md",
+    "LICENSE",
+    "THIRD-PARTY-NOTICES.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "keywords": [
+    "ocaml",
+    "compiler",
+    "toplevel",
+    "playground",
+    "browser"
+  ]
+}

package/src/compiler/compile.js

@@ -0,0 +1,195 @@
+// Compile OCaml source to JavaScript.
+//
+// Two compiler programs, each in its own worker: the compiler-libs driver turns
+// source into bytecode, js_of_ocaml turns bytecode into JavaScript. A worker per
+// step because a js_of_ocaml program runs its main on load and returns -- it
+// cannot be reused.
+//
+// Everything crosses between the page and the compilers through the virtual
+// filesystem; nothing is written to disk. The result travels as hex, because a
+// raw bytecode stream would not survive being put on a text channel.
+
+const lines = (text) => text.split('\n').filter((line) => line.trim() !== '');
+
+// Two kinds of upstream noise reach the worker's stderr, and neither says anything
+// about the program being compiled:
+//
+//  - js_of_ocaml is a command-line program and calls `exit` after writing its
+//    output -- and in a browser-target build `exit` is not implemented, so it
+//    raises. The output has already arrived by then.
+//  - js_of_ocaml's runtime tries to `importScripts` its own chunk files, which
+//    cannot work once the worker is created from a data URL -- the trick that lets
+//    the compiler run from a CDN, since a worker script must otherwise be
+//    same-origin. It falls back to the files it was handed and the bundle is
+//    correct. A genuinely fatal load leaves no output at all, and that is reported
+//    separately by `compile`.
+const PHANTOM =
+	/Function 'exit' not implemented|Failed to execute 'importScripts'|^Uncaught NetworkError:/;
+const diagnosticsOf = (text) => lines(text).filter((line) => !PHANTOM.test(line));
+
+// Byte-for-byte, and deliberately not TextDecoder: in browsers "latin1" and
+// "iso-8859-1" are both aliases for windows-1252, which remaps bytes 0x80-0x9F.
+// That is enough to make a .cmi unreadable -- "Corrupted compiled interface" --
+// so bytes are copied one at a time, chunked because a 1.6 MB .cma would
+// overflow the argument list.
+function bytesToString(bytes) {
+	let s = '';
+	const CHUNK = 0x8000;
+	for (let i = 0; i < bytes.length; i += CHUNK) {
+		s += String.fromCharCode.apply(null, bytes.subarray(i, i + CHUNK));
+	}
+	return s;
+}
+
+function hexToBytes(hex) {
+	const bytes = new Uint8Array(hex.length / 2);
+	for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.substr(i * 2, 2), 16);
+	return bytes;
+}
+
+// A worker script has to be same-origin with whatever creates it: `new Worker()`
+// refuses a cross-origin URL, and CORS does not help -- it is the script fetch the
+// browser will not make. So the worker is created from a small data URL whose only
+// job is to `importScripts` the harness, which *is* allowed across origins and then
+// loads everything else itself.
+//
+// A data URL rather than a blob because the compiler may itself be running inside a
+// worker, where `URL.createObjectURL` is not dependable. The URL is interpolated
+// into JavaScript, so it is quoted with `JSON.stringify` and base64-encoded: the
+// base64 form needs no escaping of its own.
+const toDataUrl = (content, type = 'text/javascript') =>
+	`data:${type};charset=UTF-8;base64,` + btoa(content);
+
+const workerDataUrlFor = (script) =>
+	toDataUrl(`importScripts(${JSON.stringify(String(script))});`);
+
+async function runInWorker(baseUrl, script, files, env, argv, fs = []) {
+	const worker = new Worker(workerDataUrlFor(new URL('worker.js', baseUrl).href));
+	return new Promise((resolve, reject) => {
+		worker.onmessage = (event) => {
+			worker.terminate();
+			resolve(event.data);
+		};
+		worker.onerror = (event) => {
+			worker.terminate();
+			reject(new Error(event.message || 'the compiler worker failed to load'));
+		};
+		worker.postMessage({ script, files, env, argv, fs });
+	});
+}
+
+// The compiler finds parts of its own runtime through a filesystem -- `Fs` --
+// while it emits a program. Outside a browser those files are on disk beside it;
+// inside one there is no filesystem, and the program it emits is then missing
+// pieces of its own runtime, silently: a chunk that provides classes or helpers
+// rather than primitives leaves no dummy behind. So they are shipped and
+// registered, under both the plain and the `+`-prefixed name the compiler may
+// ask for. Loaded once per page, not once per compile.
+let runtimeChunks;
+
+async function loadRuntime(baseUrl) {
+	if (runtimeChunks) return runtimeChunks;
+	const manifest = await (await fetch(new URL('runtime/manifest.txt', baseUrl))).text();
+	runtimeChunks = (
+		await Promise.all(
+			lines(manifest).map(async (name) => {
+				const content = bytesToString(
+					new Uint8Array(await (await fetch(new URL(`runtime/${name}`, baseUrl))).arrayBuffer())
+				);
+				return [
+					{ name, content },
+					{ name: `+${name}`, content }
+				];
+			})
+		)
+	).flat();
+	return runtimeChunks;
+}
+
+// The standard library the driver reads, as one `js_of_ocaml build-fs` artifact:
+// a script that registers every file in the pseudo filesystem, through
+// `jsoo_create_file` when a runtime is already up and by queueing on
+// `jsoo_fs_tmp` when not. The worker mounts it before the compiler program, and
+// the program's runtime replays the queue as it starts.
+//
+// It used to be 71 files, listed in a manifest and fetched one at a time on
+// every compile. One artifact is one request, and no JavaScript has to assemble
+// it. The files register at `/stdlib`, which is why `OCAMLLIB` below is
+// absolute rather than the bare name it used to be.
+const STDLIB = 'stdlib.js';
+const STDLIB_DIR = '/stdlib';
+
+/**
+ * Compile OCaml source to JavaScript.
+ *
+ * @param {string} source
+ * @param {object} options
+ * @param {string|URL} options.baseUrl - where the compiler assets are served from.
+ * @param {string} [options.driver] - the source-to-bytecode program in that directory.
+ * @param {string} [options.jsoo] - the bytecode-to-JavaScript program.
+ * @param {string} [options.stdlib] - the filesystem artifact carrying the standard library.
+ * @returns {Promise<{js: string|null, diagnostics: string[], error?: string}>}
+ */
+export async function compile(source, { baseUrl, driver = 'compile_stdout.browser.js', jsoo = 'jsoo.js', stdlib = STDLIB }) {
+	if (typeof source !== 'string') throw new TypeError('compile: source must be a string');
+	const runtimeFiles = await loadRuntime(baseUrl);
+	const files = [{ name: 'main.ml', content: source }, ...runtimeFiles];
+
+	// Source -> bytecode, as hex.
+	const front = await runInWorker(
+		baseUrl,
+		new URL(driver, baseUrl).href,
+		files,
+		{ OCAMLLIB: STDLIB_DIR },
+		['node', 'compile_stdout', 'main.ml'],
+		[new URL(stdlib, baseUrl).href]
+	);
+	const diagnostics = diagnosticsOf(front.err.join('\n'));
+	// The driver prints its hex with no newlines anywhere, so its chunks are
+	// pieces of one line and go back together without separators.
+	const hex = front.out.join('').trim();
+	if (!/^[0-9a-f]+$/.test(hex)) {
+		return { js: null, diagnostics, error: 'compilation failed' };
+	}
+
+	// Bytecode -> JavaScript, on stdout.
+	//
+	// `--target-env=browser` because that is where the result runs.
+	//
+	// `--pretty` is for the reader -- the emitted JavaScript is what a Compiled
+	// Code pane shows -- and it also keeps the output line-oriented, which the
+	// joining below depends on.
+	const back = await runInWorker(
+		baseUrl,
+		new URL(jsoo, baseUrl).href,
+		[{ name: 'main.byte', content: bytesToString(hexToBytes(hex)) }, ...runtimeFiles],
+		{},
+		['node', 'js_of_ocaml', 'main.byte', '-o', '-', '--target-env=browser', '--pretty']
+	);
+	const backDiagnostics = diagnosticsOf(back.err.join('\n'));
+	// js_of_ocaml's stdout is line-buffered, so it hands over one console call
+	// per line: the chunks have to be rejoined with newlines, not concatenated.
+	// Concatenated, the whole program lands on a single line and ends up inside
+	// the header comment it begins with -- it then runs nothing, prints nothing
+	// and raises nothing. That is a miserable failure to debug: 64 KB of
+	// plausible-looking, commented-out code.
+	const js = back.out.join('\n');
+	// A bundle that arrives as one line would be swallowed by the header comment
+	// it starts with, and would then run nothing at all. Fail loudly rather than
+	// hand back a program that quietly does nothing.
+	if (!js.includes('\n')) {
+		return {
+			js: null,
+			diagnostics: [...diagnostics, ...backDiagnostics],
+			error: 'the compiler produced a single-line bundle, which would not run'
+		};
+	}
+	if (js.trim() === '') {
+		return {
+			js: null,
+			diagnostics: [...diagnostics, ...backDiagnostics],
+			error: 'js_of_ocaml produced no output'
+		};
+	}
+	return { js, diagnostics: [...diagnostics, ...backDiagnostics] };
+}

package/src/compiler/index.js

@@ -0,0 +1,46 @@
+// The compiler entry point.
+//
+//   import { createCompiler } from '@live-codes/ocaml';
+//
+//   const compiler = await createCompiler({ baseUrl });
+//   const { js, diagnostics } = await compiler.compile(source);
+//
+// A page cannot read files out of node_modules, so the compiler assets have to
+// be published by whatever serves the page -- `ocaml-copy-assets` writes a copy
+// into a directory you serve, and `baseUrl` points at it. Creating a compiler
+// checks the assets are actually there, so a bad baseUrl fails here rather than
+// at the first compile.
+import { compile as compileSource } from './compile.js';
+
+export async function createCompiler({ baseUrl } = {}) {
+	if (!baseUrl) {
+		throw new Error(
+			'baseUrl is required: serve the compiler assets and pass their URL. ' +
+				'Run `ocaml-copy-assets --compiler <dir>` to write them, then ' +
+				"`createCompiler({ baseUrl: new URL('/ocaml-compiler/', location.href) })`."
+		);
+	}
+	const base = new URL(baseUrl, globalThis.location?.href ?? 'http://localhost/');
+
+	// The standard library is one 5 MB artifact, so it is not fetched just to
+	// prove it is reachable -- a HEAD is enough to catch a wrong baseUrl, and
+	// the worker's `importScripts` would fail much less legibly later.
+	const stdlib = new URL('stdlib.js', base);
+	const response = await fetch(stdlib, { method: 'HEAD' }).catch(() => null);
+	if (!response || !response.ok) {
+		throw new Error(
+			`No compiler assets at ${base}: ${stdlib} is not there. ` +
+				'Run `ocaml-copy-assets --compiler <dir>` and point baseUrl at the directory you serve.'
+		);
+	}
+
+	return {
+		banner: 'ocaml compiler',
+		/** Compile a program. Returns `{ js, diagnostics, error? }`. */
+		compile(source) {
+			return compileSource(source, { baseUrl: base });
+		}
+	};
+}
+
+export { compileSource as compile };

package/src/compiler/worker.js

@@ -0,0 +1,46 @@
+// The compiler worker: one compiler per worker.
+//
+// A js_of_ocaml program runs its main on load and returns, so a worker cannot
+// run two of them -- each compile needs a fresh one, created and terminated.
+//
+// Everything the compiler needs arrives through the virtual filesystem and the
+// environment; its output is collected by overriding console, because with the
+// browser filesystem fd 1 and fd 2 are wired to console.log and console.error
+// (runtime/js/fs_fake.js). A worker has no other way in: the output never
+// reaches a file the page can read.
+self.onmessage = (event) => {
+	const { script, files, env, argv, fs = [] } = event.data;
+	const out = [];
+	const err = [];
+	const realLog = console.log;
+	const realError = console.error;
+
+	console.log = (...args) => out.push(args.join(' '));
+	console.error = (...args) => err.push(args.join(' '));
+
+	try {
+		globalThis.jsoo_fs_tmp = files;
+		globalThis.jsoo_env = env;
+		// Browsers have no `process`; js_of_ocaml reads `process?.argv` for argv,
+		// so a fake one is enough. Note Sys.argv.(0) is argv[1].
+		globalThis.process = { argv };
+		// Filesystem artifacts first. Nothing has defined `jsoo_create_file` yet,
+		// so they queue their files on `jsoo_fs_tmp` -- and the program's runtime
+		// replays that queue at start-up. Loading them after the program would
+		// take the other branch and register the files directly, so the order is
+		// not load-bearing, but this is the cheaper one.
+		for (const url of fs) importScripts(url);
+		importScripts(script);
+		// The chunks go back as they arrived, not joined: the caller knows how to
+		// put them together, and joining here would lose the newlines between
+		// them. js_of_ocaml's stdout is line-buffered, so it hands over one call
+		// per line, and a program that arrives as a single line is a program
+		// swallowed by its own header comment.
+		self.postMessage({ out, err });
+	} catch (error) {
+		self.postMessage({ out, err: [...err, String((error && error.message) || error)] });
+	} finally {
+		console.log = realLog;
+		console.error = realError;
+	}
+};

package/src/index.js

@@ -0,0 +1,12 @@
+// @live-codes/ocaml
+//
+// Two implementations of OCaml in the browser, each behind its own entry point
+// so that using one costs nothing for the other:
+//
+//   @live-codes/ocaml              this file -- the compiler
+//   @live-codes/ocaml/compiler     OCaml source -> JavaScript
+//   @live-codes/ocaml/interpreter  a toplevel that runs OCaml directly
+//
+// The compiler is the default because that is the model the other LiveCodes
+// languages use: the source becomes JavaScript, which the page then runs.
+export * from './compiler/index.js';

package/src/interpreter/api.js

@@ -0,0 +1,182 @@
+// One implementation behind the package entry point.
+//
+// There are two ways to run it, and the caller's situation decides which:
+//
+//  - If the runtime is already loaded in this context (a host that called
+//    `importScripts('worker.js')`, as LiveCodes' compiler worker would), it is
+//    used in place. A host that is already a worker should not start a nested
+//    one just to hold a runtime it could call directly.
+//  - Otherwise a worker is started for you, for standalone use where nobody
+//    else owns the message loop.
+//
+// `baseUrl` is what makes the first case work: the runtime cannot discover
+// where its assets are once it has been imported into someone else's worker,
+// so it is told rather than left to infer.
+
+const shape = (raw, runMs) => {
+	const failed = raw.exitCode === null;
+	const lines = raw.diagnostics ? raw.diagnostics.split('\n').filter(Boolean) : [];
+	return {
+		stdout: raw.stdout,
+		stderr: raw.stderr,
+		output: raw.output,
+		diagnostics: lines,
+		// Empty on success, so `errors.length` is a reliable failure test. Warnings
+		// from a successful run are in `diagnostics`, not here.
+		errors: failed ? lines : [],
+		exitCode: failed ? null : Number(raw.exitCode),
+		outcome: raw.outcome,
+		// A runtime failure (a missing asset, a failed start) rather than the
+		// program's diagnostics. Null when nothing went wrong at that level.
+		error: raw.error ?? null,
+		runMs
+	};
+};
+
+const absolute = (baseUrl) =>
+	new URL(baseUrl, globalThis.location?.href ?? 'http://localhost/').href;
+
+export function createApi() {
+	/**
+	 * Start an OCaml runtime.
+	 *
+	 * @param {object} options
+	 * @param {string|URL} options.baseUrl - where the runtime assets are served from. Required:
+	 *   the assets ship in this package but a page cannot read them off disk, so they have to be
+	 *   published somewhere fetchable (see `ocaml-copy-assets`).
+	 * @param {object} [options.runtime] - an already-loaded `self.ocamlWasm` to use in place,
+	 *   instead of starting a worker. Defaults to one already present in this context.
+	 * @param {string[]} [options.args] - default program argv. `Sys.argv.(0)` is `"main.ml"`.
+	 * @param {{name: string, contents: string}[]} [options.files] - default files, written into the
+	 *   runtime's filesystem so `open_in "name"` finds them.
+	 */
+	async function createCompiler(options = {}) {
+		const { baseUrl, args: defaultArgs = [], files: defaultFiles = [] } = options;
+		if (!baseUrl) {
+			throw new Error(
+				'baseUrl is required: serve the runtime assets and pass their URL. ' +
+					'Run `ocaml-copy-assets <dir>` to write them, then ' +
+					"`createCompiler({ baseUrl: new URL('/ocaml/', location.href) })`."
+			);
+		}
+
+		const resolvedBase = absolute(baseUrl);
+		let disposed = false;
+
+		const runtime = options.runtime ?? globalThis.ocamlWasm;
+		if (runtime) {
+			runtime.setBaseUrl(resolvedBase);
+			runtime.init();
+			return {
+				banner: 'in-thread',
+				/** Run a program in this context. Synchronous underneath; the Promise is for parity. */
+				run(code, input = '', runOptions = {}) {
+					if (disposed) return Promise.reject(new Error('This compiler has been disposed.'));
+					if (typeof code !== 'string') {
+						return Promise.reject(
+							new Error('run() needs the program source as its first argument.')
+						);
+					}
+					const started = performance.now();
+					// `run` does not throw: a failure comes back as `error`.
+					const raw = runtime.run({
+						code,
+						input,
+						args: runOptions.args ?? defaultArgs,
+						files: runOptions.files ?? defaultFiles
+					});
+					return Promise.resolve(shape(raw, performance.now() - started));
+				},
+				dispose() {
+					disposed = true;
+				}
+			};
+		}
+
+		const workerUrl = new URL('worker-entry.js', resolvedBase);
+		const worker = new Worker(workerUrl);
+
+		// `ready` is posted once the toplevel has initialised, so a bad baseUrl fails here rather
+		// than at the first run.
+		const banner = await new Promise((resolve, reject) => {
+			const onMessage = (event) => {
+				const message = event.data;
+				if (message && message[0] === 'ready') {
+					worker.removeEventListener('message', onMessage);
+					resolve(message[1]);
+				}
+			};
+			worker.addEventListener('message', onMessage);
+			worker.addEventListener('error', (event) => {
+				reject(
+					new Error(
+						`Failed to load the OCaml runtime from ${workerUrl}: ${event.message ?? 'unknown error'}`
+					)
+				);
+			});
+		});
+
+		// One toplevel, one environment: runs share it, so they are serialised rather than
+		// interleaved. A queued run still resolves - it just waits.
+		let queue = Promise.resolve();
+
+		/**
+		 * Run a program.
+		 *
+		 * @param {string} code - OCaml source.
+		 * @param {string} [input] - stdin, handed to the program once and then closed.
+		 * @param {object} [runOptions] - per-run overrides: `args`, `files`.
+		 * @returns {Promise<{stdout: string, stderr: string, output: string, diagnostics: string[],
+		 *   errors: string[], exitCode: number|null, outcome: string, error: string|null,
+		 *   runMs: number}>}
+		 */
+		function run(code, input = '', runOptions = {}) {
+			if (disposed) return Promise.reject(new Error('This compiler has been disposed.'));
+			if (typeof code !== 'string') {
+				return Promise.reject(new Error('run() needs the program source as its first argument.'));
+			}
+
+			const started = performance.now();
+			const done = queue.then(
+				() =>
+					new Promise((resolve) => {
+						const onMessage = (event) => {
+							const message = event.data;
+							if (!message || message[0] !== 'result') return;
+							worker.removeEventListener('message', onMessage);
+							const [, exitCode, stdout, stderr, outcome, output, diagnostics] = message;
+							resolve(
+								shape(
+									{ exitCode, stdout, stderr, outcome, output, diagnostics, error: null },
+									performance.now() - started
+								)
+							);
+						};
+						worker.addEventListener('message', onMessage);
+						worker.postMessage({
+							code,
+							input,
+							args: runOptions.args ?? defaultArgs,
+							files: runOptions.files ?? defaultFiles
+						});
+					})
+			);
+			queue = done.then(
+				() => undefined,
+				() => undefined
+			);
+			return done;
+		}
+
+		/** Stop the worker. Further runs reject. */
+		function dispose() {
+			if (disposed) return;
+			disposed = true;
+			worker.terminate();
+		}
+
+		return { banner, run, dispose };
+	}
+
+	return { createCompiler };
+}

package/src/interpreter/index.js

@@ -0,0 +1,5 @@
+import { createApi } from './api.js';
+
+const api = createApi();
+
+export const createCompiler = api.createCompiler;