@live-codes/ocaml 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-codes/ocaml",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Run OCaml in the browser with no server: a compiler that emits JavaScript, and a toplevel that runs OCaml directly",
   "repository": {
     "type": "git",

package/src/compiler/compile.js

@@ -11,11 +11,24 @@
 
 const lines = (text) => text.split('\n').filter((line) => line.trim() !== '');
 
-// js_of_ocaml is a command-line program and calls `exit` after writing its
-// output -- and in a browser-target build `exit` is not implemented, so it
-// raises. The output has already arrived by then, so this is noise rather than
-// a diagnostic. Filter it, since the CLI is not ours to change.
-const PHANTOM = /Function 'exit' not implemented/;
+// Three kinds of upstream noise reach the worker's stderr, and none says anything
+// about the program being compiled:
+//
+//  - js_of_ocaml is a command-line program and calls `exit` after writing its
+//    output -- and in a browser-target build `exit` is not implemented, so it
+//    raises. The output has already arrived by then.
+//  - js_of_ocaml's runtime tries to `importScripts` its own chunk files, which
+//    cannot work once the worker is created from a data URL -- the trick that lets
+//    the compiler run from a CDN, since a worker script must otherwise be
+//    same-origin. It falls back to the files it was handed and the bundle is
+//    correct. A genuinely fatal load leaves no output at all, and that is reported
+//    separately by `compile`.
+//  - js_of_ocaml warns when it cannot represent an integer exactly, and does so
+//    about its own generated runtime rather than about the program. It is filtered
+//    on request; the cost is that a program relying on such an integer will not see
+//    it either.
+const PHANTOM =
+	/Function 'exit' not implemented|Failed to execute 'importScripts'|^Uncaught NetworkError:|Warning \[integer-overflow\]/;
 const diagnosticsOf = (text) => lines(text).filter((line) => !PHANTOM.test(line));
 
 // Byte-for-byte, and deliberately not TextDecoder: in browsers "latin1" and
@@ -38,9 +51,25 @@ function hexToBytes(hex) {
 	return bytes;
 }
 
-function runInWorker(baseUrl, script, files, env, argv, fs = []) {
+// A worker script has to be same-origin with whatever creates it: `new Worker()`
+// refuses a cross-origin URL, and CORS does not help -- it is the script fetch the
+// browser will not make. So the worker is created from a small data URL whose only
+// job is to `importScripts` the harness, which *is* allowed across origins and then
+// loads everything else itself.
+//
+// A data URL rather than a blob because the compiler may itself be running inside a
+// worker, where `URL.createObjectURL` is not dependable. The URL is interpolated
+// into JavaScript, so it is quoted with `JSON.stringify` and base64-encoded: the
+// base64 form needs no escaping of its own.
+const toDataUrl = (content, type = 'text/javascript') =>
+	`data:${type};charset=UTF-8;base64,` + btoa(content);
+
+const workerDataUrlFor = (script) =>
+	toDataUrl(`importScripts(${JSON.stringify(String(script))});`);
+
+async function runInWorker(baseUrl, script, files, env, argv, fs = [], libs) {
+	const worker = new Worker(workerDataUrlFor(new URL('worker.js', baseUrl).href));
 	return new Promise((resolve, reject) => {
-		const worker = new Worker(new URL('worker.js', baseUrl));
 		worker.onmessage = (event) => {
 			worker.terminate();
 			resolve(event.data);
@@ -49,7 +78,7 @@ function runInWorker(baseUrl, script, files, env, argv, fs = []) {
 			worker.terminate();
 			reject(new Error(event.message || 'the compiler worker failed to load'));
 		};
-		worker.postMessage({ script, files, env, argv, fs });
+		worker.postMessage({ script, files, env, argv, fs, libs });
 	});
 }
 
@@ -94,8 +123,92 @@ async function loadRuntime(baseUrl) {
 const STDLIB = 'stdlib.js';
 const STDLIB_DIR = '/stdlib';
 
+// The libraries a program names are fetched on demand rather than shipped up front.
+// `libs/index.txt` says which files a module needs, and a compile that could not find
+// a module is retried with those files added. Only what a program uses is fetched, and
+// once fetched it stays, so the set converges as a session goes on.
+const required = new Set();
+let libraryIndex;
+
+const loadLibraryIndex = async (baseUrl) => {
+	const byUnit = new Map();
+	try {
+		const text = await (await fetch(new URL('libs/index.txt', baseUrl))).text();
+		for (const line of lines(text)) {
+			const [unit, files] = line.split('\t');
+			if (unit && files) byUnit.set(unit.toLowerCase(), files.split(' ').filter(Boolean));
+		}
+		// The stub libraries are small and every link may need them, so they are always
+		// fetched rather than being part of a group.
+		const stubs = await (await fetch(new URL('libs/stubs.txt', baseUrl))).text();
+		for (const name of lines(stubs)) required.add(name.trim());
+	} catch {
+		// No index, so nothing can be fetched on demand: a program gets the standard
+		// library and whatever else it can link without one.
+	}
+	return byUnit;
+};
+
+// The two ways the compiler says a library is missing: a module it cannot find at all,
+// and a unit with nothing to link against.
+const missingUnits = (diagnostics) => {
+	const text = diagnostics.join('\n');
+	const names = new Set();
+	for (const [, name] of text.matchAll(/Unbound module ([A-Za-z0-9_']+)/g)) names.add(name);
+	for (const [, block] of text.matchAll(
+		/No implementation provided for the following modules:([\s\S]*?)(?:\n\s*\n|$)/g
+	)) {
+		for (const [, name] of block.matchAll(/([A-Za-z0-9_']+) referenced from/g)) names.add(name);
+	}
+	return [...names];
+};
+
+const filesFor = (byUnit, names) => {
+	const files = new Set();
+	const known = [...byUnit.keys()];
+	for (const name of names) {
+		const key = name.toLowerCase();
+		const exact = byUnit.get(key);
+		if (exact) {
+			for (const file of exact) files.add(file);
+			continue;
+		}
+		// A submodule is named without its package: Dom_html is Js_of_ocaml's.
+		for (const unit of known.filter((u) => u.endsWith('__' + key))) {
+			for (const file of byUnit.get(unit)) files.add(file);
+		}
+	}
+	return [...files];
+};
+
+// Byte by byte, not through TextDecoder: these are binary files, and decoding them as
+// UTF-8 would corrupt every byte above 127 into something else entirely.
+const toByteString = (bytes) => {
+	const chunks = [];
+	for (let i = 0; i < bytes.length; i += 0x8000) {
+		chunks.push(String.fromCharCode(...bytes.subarray(i, i + 0x8000)));
+	}
+	return chunks.join('');
+};
+
+// The library files are fetched here, in the page, rather than in the worker: a worker
+// created from a data URL has an opaque origin, so a fetch from inside it to the server
+// that served the page is refused. The page is same-origin, so it can.
+const fetchLibraries = async (baseUrl, names) => {
+	const base = new URL('libs/', baseUrl);
+	return Promise.all(
+		names.map(async (name) => {
+			const bytes = new Uint8Array(await (await fetch(new URL(name, base))).arrayBuffer());
+			// The stub libraries are looked for at /static, from the stub directory of
+			// the build; everything else is read from the compiler's library directory.
+			const path = (name.startsWith('dll') ? '/static/' : '/stdlib/') + name;
+			return { path, content: toByteString(bytes) };
+		})
+	);
+};
+
 /**
- * Compile OCaml source to JavaScript.
+ * Compile OCaml source to JavaScript, fetching whatever libraries it turns out to need.
  *
  * @param {string} source
  * @param {object} options
@@ -105,19 +218,58 @@ const STDLIB_DIR = '/stdlib';
  * @param {string} [options.stdlib] - the filesystem artifact carrying the standard library.
  * @returns {Promise<{js: string|null, diagnostics: string[], error?: string}>}
  */
-export async function compile(source, { baseUrl, driver = 'compile_stdout.browser.js', jsoo = 'jsoo.js', stdlib = STDLIB }) {
+export async function compile(source, options) {
+	libraryIndex ??= await loadLibraryIndex(options.baseUrl);
+	let libs = [...required];
+	for (let attempt = 0; attempt < 6; attempt++) {
+		const result = await compileOnce(source, { ...options, libs });
+		const added = filesFor(libraryIndex, missingUnits(result.diagnostics)).filter(
+			(name) => !required.has(name)
+		);
+		if (added.length === 0) return result;
+		for (const name of added) required.add(name);
+		libs = [...required];
+	}
+	// Six rounds and still missing something: run once more, so the caller sees the
+	// compiler's own error rather than a guess about it.
+	return compileOnce(source, { ...options, libs });
+}
+
+// zarith's primitives are C stubs, and `zarith_stubs_js` implements them in JavaScript as a
+// js_of_ocaml runtime chunk -- `//Provides:` and `//Requires:` lines that have to be a *link
+// input* to js_of_ocaml, not text pasted into the output. Only as an input does js_of_ocaml
+// honour them and put each definition where its dependencies already exist; concatenated,
+// they land outside its primitive table and the program fails at load with
+// "f.ml_z_init is not a function".
+const RUNTIME_CHUNK = 'zarith_stubs_js.runtime.js';
+const CHUNK_AS_INPUT = 'zarith_stubs_js.js';
+let runtimeChunk;
+
+const loadRuntimeChunk = async (baseUrl) => {
+	runtimeChunk ??= await (await fetch(new URL('libs/' + RUNTIME_CHUNK, baseUrl))).text();
+	return runtimeChunk;
+};
+
+async function compileOnce(
+	source,
+	{ baseUrl, driver = 'compile_stdout.browser.js', jsoo = 'jsoo.js', stdlib = STDLIB, libs = [] }
+) {
 	if (typeof source !== 'string') throw new TypeError('compile: source must be a string');
 	const runtimeFiles = await loadRuntime(baseUrl);
 	const files = [{ name: 'main.ml', content: source }, ...runtimeFiles];
+	const libraryFiles = await fetchLibraries(baseUrl, libs);
 
-	// Source -> bytecode, as hex.
+	// Source -> bytecode, as hex. The library files go over with the source, one request
+	// each, from the browser's cache after the first compile. The bytecode ->
+	// JavaScript step needs none of them.
 	const front = await runInWorker(
 		baseUrl,
 		new URL(driver, baseUrl).href,
 		files,
 		{ OCAMLLIB: STDLIB_DIR },
 		['node', 'compile_stdout', 'main.ml'],
-		[new URL(stdlib, baseUrl).href]
+		[new URL(stdlib, baseUrl).href],
+		libraryFiles
 	);
 	const diagnostics = diagnosticsOf(front.err.join('\n'));
 	// The driver prints its hex with no newlines anywhere, so its chunks are
@@ -134,12 +286,31 @@ export async function compile(source, { baseUrl, driver = 'compile_stdout.browse
 	// `--pretty` is for the reader -- the emitted JavaScript is what a Compiled
 	// Code pane shows -- and it also keeps the output line-oriented, which the
 	// joining below depends on.
+	const stubs = required.has('zarith.cma') ? await loadRuntimeChunk(baseUrl) : null;
 	const back = await runInWorker(
 		baseUrl,
 		new URL(jsoo, baseUrl).href,
-		[{ name: 'main.byte', content: bytesToString(hexToBytes(hex)) }, ...runtimeFiles],
+		[
+			{ name: 'main.byte', content: bytesToString(hexToBytes(hex)) },
+			...runtimeFiles,
+			// The chunk is mounted as a file and named on the command line, which is what
+			// makes js_of_ocaml treat it as a link input rather than something to paste in.
+			...(stubs ? [{ name: CHUNK_AS_INPUT, content: stubs }] : [])
+		],
 		{},
-		['node', 'js_of_ocaml', 'main.byte', '-o', '-', '--target-env=browser', '--pretty']
+		[
+			'node',
+			'js_of_ocaml',
+			// The chunk comes before the bytecode: js_of_ocaml decides how to read its
+			// positional inputs from the first one, so a .js has to lead or the bytecode
+			// gets parsed as JavaScript.
+			...(stubs ? [CHUNK_AS_INPUT] : []),
+			'main.byte',
+			'-o',
+			'-',
+			'--target-env=browser',
+			'--pretty'
+		]
 	);
 	const backDiagnostics = diagnosticsOf(back.err.join('\n'));
 	// js_of_ocaml's stdout is line-buffered, so it hands over one console call

package/src/compiler/worker.js

@@ -8,8 +8,42 @@
 // browser filesystem fd 1 and fd 2 are wired to console.log and console.error
 // (runtime/js/fs_fake.js). A worker has no other way in: the output never
 // reaches a file the page can read.
-self.onmessage = (event) => {
-	const { script, files, env, argv, fs = [] } = event.data;
+// Bytes to a bytestring, one character per byte, chunked so a multi-megabyte archive
+// does not overflow the argument list. Deliberately not TextDecoder: in browsers
+// "latin1" and "iso-8859-1" are both aliases for windows-1252, which remaps
+// 0x80-0x9F, and that is enough to make a .cmi or a .cma unreadable.
+const bytesToString = (bytes) => {
+	let s = '';
+	const CHUNK = 0x8000;
+	for (let i = 0; i < bytes.length; i += CHUNK) {
+		s += String.fromCharCode.apply(null, bytes.subarray(i, i + CHUNK));
+	}
+	return s;
+};
+
+// The library files this compile needs. The *page* fetches them, not this worker: a
+// worker created from a data URL has an opaque origin, so a fetch from here to the
+// server that served the page is refused. They arrive with the path each belongs at,
+// which is the same shape a filesystem artifact uses.
+const mountLibraries = (files) => {
+	const archives = [];
+	for (const { path, content } of files) {
+		globalThis.jsoo_fs_tmp.push({ name: path, content });
+		if (path.endsWith('.cma')) archives.push(path.slice(path.lastIndexOf('/') + 1));
+	}
+	// What the linker links, and the order matters: a library has to come before the
+	// archive that references it. stdlib first, because that is where a program's
+	// runtime starts, then js_of_ocaml's runtime, then the rest.
+	const rank = (name) => (name === 'stdlib.cma' ? 0 : name === 'jsoo_runtime.cma' ? 1 : 2);
+	archives.sort((a, b) => rank(a) - rank(b) || (a < b ? -1 : a > b ? 1 : 0));
+	globalThis.jsoo_fs_tmp.push({
+		name: '/stdlib/cmas.txt',
+		content: ['stdlib.cma', ...archives].join('\n') + '\n',
+	});
+};
+
+self.onmessage = async (event) => {
+	const { script, files, env, argv, fs = [], libs = [] } = event.data;
 	const out = [];
 	const err = [];
 	const realLog = console.log;
@@ -30,6 +64,8 @@ self.onmessage = (event) => {
 		// take the other branch and register the files directly, so the order is
 		// not load-bearing, but this is the cheaper one.
 		for (const url of fs) importScripts(url);
+		// Then the libraries, which have to be in place before the program starts.
+		if (libs.length > 0) mountLibraries(libs);
 		importScripts(script);
 		// The chunks go back as they arrived, not joined: the caller knows how to
 		// put them together, and joining here would lose the newlines between