@live-codes/odin-wasm 0.0.0-stage → 0.2.0

package/src/engine.js

@@ -0,0 +1,220 @@
+// The compile → run engine. It has no opinion about where the compiler module
+// and the standard library came from; `assets.js` reads them, this runs them.
+//
+// The compiler is a WASI command module, so every compile is a fresh instance
+// over a fresh in-memory file system: the standard library, the user's file,
+// and an `out/` to write into. The module it emits then runs on one of two
+// hosts, depending on the target it was built for.
+
+import { WasiFileSystem, WasiExit, createWasi, runWasi } from './wasi-host.js';
+import { createJsHost, JsExit } from './host-js.js';
+
+// Where the compiler looks for base/, core/ and vendor/.
+const ODIN_ROOT = '/odin';
+const PROGRAM = '/out/main.wasm';
+
+/** The targets the compiler can emit, and the page can run. */
+export const TARGETS = ['wasi_wasm32', 'js_wasm32'];
+
+const encoder = new TextEncoder();
+
+const capture = () => {
+  const stdoutParts = [];
+  const stderrParts = [];
+  const order = [];
+  return {
+    stdout: (text) => {
+      stdoutParts.push(text);
+      order.push(text);
+    },
+    stderr: (text) => {
+      stderrParts.push(text);
+      order.push(text);
+    },
+    stdoutText: () => stdoutParts.join(''),
+    stderrText: () => stderrParts.join(''),
+    outputText: () => order.join(''),
+  };
+};
+
+/** The lines a caller reads as diagnostics, with the empty ones dropped. */
+const diagnosticLines = (text) =>
+  text
+    .split('\n')
+    .map((line) => line.trimEnd())
+    .filter((line) => line.trim() !== '');
+
+/** Why a compile produced no module, as the lines a caller reads. */
+const compileFailure = (compiled) =>
+  compiled.crash
+    ? [compiled.crash]
+    : compiled.diagnostics.length > 0
+      ? compiled.diagnostics
+      : [`The compiler produced no module (exit code ${compiled.exitCode}).`];
+
+/**
+ * @param {object} assets
+ * @param {WebAssembly.Module} assets.compilerModule - the Odin compiler
+ * @param {[string, Uint8Array][]} assets.libraryEntries - the packed standard library
+ */
+export function createEngine({ compilerModule, libraryEntries }) {
+  /** A fresh file system with the standard library under /odin. */
+  const libraryFs = () => {
+    const fs = new WasiFileSystem();
+    for (const [path, data] of libraryEntries) fs.writeFile(`${ODIN_ROOT}/${path}`, data);
+    // The compiler registers its collections (base, core, vendor) from
+    // directories and requires all three to exist; the pack ships no vendor
+    // packages, so the directory is empty.
+    fs.mkdirAll(`${ODIN_ROOT}/vendor`);
+    return fs;
+  };
+
+  const compile = ({ code, target, compileArgs, fileName }) => {
+    const fs = libraryFs();
+    fs.writeFile(`/src/${fileName}`, encoder.encode(code));
+    fs.mkdirAll('/out');
+
+    const compilerOut = [];
+    const compilerErr = [];
+    const args = [
+      'odin',
+      'build',
+      '/src',
+      `-out:${PROGRAM}`,
+      '-backend:wasm',
+      `-target:${target}`,
+      ...compileArgs,
+    ];
+
+    let exitCode = 1;
+    let crash = '';
+    try {
+      exitCode = runWasi(compilerModule, createWasi(fs, {
+        args,
+        env: [`ODIN_ROOT=${ODIN_ROOT}`],
+        stdout: (text) => compilerOut.push(text),
+        stderr: (text) => compilerErr.push(text),
+      }));
+    } catch (error) {
+      crash = error?.message ?? String(error);
+    }
+
+    return {
+      exitCode,
+      artifact: exitCode === 0 ? fs.readFile(PROGRAM) : null,
+      diagnostics: diagnosticLines(compilerOut.join('') + compilerErr.join('')),
+      crash,
+    };
+  };
+
+  /**
+   * Compile only, returning the module for the caller to run.
+   *
+   * A page that needs the DOM has to instantiate the program on the main thread — a worker has no
+   * `document`, and Odin's JS runtime is written for the page — so the caller takes the module
+   * instead of an output string.
+   *
+   * @returns {{ok: boolean, wasm: Uint8Array|null, errors: string[], compileMs: number}}
+   */
+  const compileOnly = ({ code, target, compileArgs = [], fileName = 'main.odin' }) => {
+    const started = now();
+    const compiled = compile({ code, target, compileArgs: [...compileArgs], fileName });
+    const compileMs = Math.round(now() - started);
+    if (compiled.artifact === null || compiled.artifact.length === 0) {
+      return { ok: false, wasm: null, errors: compileFailure(compiled), compileMs };
+    }
+    return { ok: true, wasm: compiled.artifact, errors: [], compileMs };
+  };
+
+  const runWasiTarget = (artifact, { input, args }) => {
+    const captured = capture();
+    let exitCode = 0;
+    try {
+      exitCode = runWasi(new WebAssembly.Module(artifact.slice()), createWasi(new WasiFileSystem(), {
+        args: ['main.wasm', ...args],
+        stdin: input,
+        stdout: captured.stdout,
+        stderr: captured.stderr,
+      }));
+    } catch (error) {
+      if (error instanceof WasiExit) exitCode = error.code;
+      else {
+        captured.stderr(`The program trapped: ${error?.message ?? error}\n`);
+        exitCode = 1;
+      }
+    }
+    return { exitCode, ...capturedText(captured) };
+  };
+
+  const runJsTarget = async (artifact, { input, args }) => {
+    const captured = capture();
+    const host = createJsHost({ input, args: ['main.wasm', ...args], stdout: captured.stdout, stderr: captured.stderr });
+    let exitCode = 0;
+    try {
+      const { instance } = await WebAssembly.instantiate(artifact.slice(), host.imports);
+      host.setMemory(instance.exports.memory);
+      if (instance.exports._start) instance.exports._start();
+      if (instance.exports._end) instance.exports._end();
+    } catch (error) {
+      if (error instanceof JsExit) exitCode = error.code;
+      else {
+        captured.stderr(`The program trapped: ${error?.message ?? error}\n`);
+        exitCode = 1;
+      }
+    }
+    return { exitCode, ...capturedText(captured) };
+  };
+
+  /**
+   * Compile and run one program.
+   *
+   * @param {object} params
+   * @param {string} params.code
+   * @param {string|Uint8Array} params.input
+   * @param {string} params.target
+   * @param {string[]} params.args
+   * @param {string[]} params.compileArgs
+   * @param {string} params.fileName
+   */
+  const compileAndRun = async ({ code, input = '', target, args = [], compileArgs = [], fileName = 'main.odin' }) => {
+    const compileStarted = now();
+    const compiled = compile({ code, target, compileArgs: [...compileArgs], fileName });
+    const compileMs = Math.round(now() - compileStarted);
+
+    if (compiled.artifact === null || compiled.artifact.length === 0) {
+      return { stdout: '', stderr: '', output: '', errors: compileFailure(compiled), exitCode: null, compileMs, runMs: null };
+    }
+
+    const runStarted = now();
+    let ran;
+    try {
+      ran = target === 'js_wasm32'
+        ? await runJsTarget(compiled.artifact, { input, args })
+        : runWasiTarget(compiled.artifact, { input, args });
+    } catch (error) {
+      ran = { exitCode: 1, stdout: '', stderr: `The program could not run: ${error?.message ?? error}\n`, output: '', };
+      ran.output = ran.stderr;
+    }
+    const runMs = Math.round(now() - runStarted);
+
+    return {
+      stdout: ran.stdout,
+      stderr: ran.stderr,
+      output: ran.output,
+      errors: [],
+      exitCode: ran.exitCode,
+      compileMs,
+      runMs,
+    };
+  };
+
+  return { compileAndRun, compileOnly };
+}
+
+const capturedText = (captured) => ({
+  stdout: captured.stdoutText(),
+  stderr: captured.stderrText(),
+  output: captured.outputText(),
+});
+
+const now = () => (globalThis.performance?.now?.() ?? Date.now());

package/src/host-js.js

@@ -0,0 +1,112 @@
+// The host a `js_wasm32` program runs on, outside a page.
+//
+// A js_wasm32 module imports `odin_env` (the console, stdin, `evaluate`, the
+// math intrinsics), `odin_dom` (elements and events), `webgl` and `webgl2`.
+// Only `odin_env` can do anything without a DOM, so the other three are stubs
+// that accept any call and return 0 — a program that reaches for the DOM runs
+// and does nothing rather than failing to instantiate. A page that wants the
+// DOM and the canvas supplies its own host; see the repository's page, which
+// uses Odin's own `odin.js`.
+
+/** Thrown by `trap`/`abort`, so an exit is not confused with a real crash. */
+export class JsExit extends Error {
+  constructor(code) {
+    super(`The program exited (code ${code}).`);
+    this.name = 'JsExit';
+    this.code = code;
+  }
+}
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+// Any import the host does not define resolves to a no-op returning 0, so a
+// module written against a different set of host functions still instantiates.
+const withFallback = (target) =>
+  new Proxy(target, {
+    get: (object, property) =>
+      typeof property === 'string' && !(property in object) ? () => 0 : object[property],
+  });
+
+/**
+ * @param {object} options
+ * @param {string|Uint8Array} [options.input] - the program's stdin, handed out once
+ * @param {string[]} [options.args] - argv, including argv[0]
+ * @param {(text: string) => void} [options.stdout]
+ * @param {(text: string) => void} [options.stderr]
+ */
+export function createJsHost({ input = '', args = [], stdout = () => {}, stderr = () => {} } = {}) {
+  const inputBytes = typeof input === 'string' ? encoder.encode(input) : new Uint8Array(input);
+  let inputOffset = 0;
+
+  let memory = null;
+  const bytes = () => new Uint8Array(memory.buffer);
+  const readString = (ptr, len) => decoder.decode(bytes().subarray(ptr, ptr + len));
+
+  // Any call to a DOM or WebGL import is accepted and returns 0.
+  const inertNamespace = new Proxy(
+    {},
+    { get: (_target, property) => (typeof property === 'string' ? () => 0 : undefined) }
+  );
+
+  const odin_env = {
+    write(fd, ptr, len) {
+      const text = readString(ptr, len);
+      (fd === 2 ? stderr : stdout)(text);
+    },
+    // Standard input, for the standard library's js streams — see the
+    // repository's patches/core/os/file_js.odin, which is in the shipped pack.
+    read(fd, ptr, len) {
+      if (fd !== 0) return 0;
+      const count = Math.max(0, Math.min(len, inputBytes.length - inputOffset));
+      if (count > 0) {
+        bytes().set(inputBytes.subarray(inputOffset, inputOffset + count), ptr);
+        inputOffset += count;
+      }
+      return count;
+    },
+    trap() {
+      throw new JsExit(1);
+    },
+    abort() {
+      throw new JsExit(1);
+    },
+    alert() {},
+    // The one interop call that works without a DOM: it evaluates in the global
+    // scope, so a page that embeds this can draw through it.
+    evaluate(ptr, len) {
+      const source = readString(ptr, len);
+      (0, globalThis.eval)(source);
+    },
+    open() {},
+    time_now: () => BigInt(Date.now()),
+    tick_now: () => globalThis.performance?.now?.() ?? Date.now(),
+    time_sleep() {},
+    sqrt: Math.sqrt,
+    sin: Math.sin,
+    cos: Math.cos,
+    pow: Math.pow,
+    fmuladd: (x, y, z) => x * y + z,
+    ln: Math.log,
+    exp: Math.exp,
+    ldexp: (x, exponent) => x * 2 ** exponent,
+    rand_bytes(ptr, len) {
+      bytes().set(globalThis.crypto.getRandomValues(new Uint8Array(len)), ptr);
+    },
+  };
+
+  return {
+    imports: {
+      env: withFallback({}),
+      odin_env: withFallback(odin_env),
+      odin_dom: inertNamespace,
+      webgl: inertNamespace,
+      webgl2: inertNamespace,
+    },
+    /** The module exports its own memory, so it is set after instantiation. */
+    setMemory(mem) {
+      memory = mem;
+    },
+    args,
+  };
+}

package/src/index.js

@@ -0,0 +1,9 @@
+// The entry every environment gets unless something more specific matches, so
+// it has to work without a filesystem: no `node:fs` here, and `baseUrl` is
+// required because a browser cannot read a file that lives inside an npm package.
+import { createApi } from './api.js';
+
+const api = createApi({ packaged: null });
+
+export const createCompiler = api.createCompiler;
+export const { TARGETS } = api;

package/src/index.node.js

@@ -0,0 +1,9 @@
+// Node, where the assets that ship in this package can be read off disk, so
+// `baseUrl` becomes optional and `createCompiler()` is enough on its own.
+import { createApi } from './api.js';
+import { packagedAssets } from './packaged.node.js';
+
+const api = createApi({ packaged: packagedAssets });
+
+export const createCompiler = api.createCompiler;
+export const { TARGETS } = api;

package/src/pack.js

@@ -0,0 +1,40 @@
+// The packed standard library: Odin's `base/` and `core/`, in the container the
+// repository's packer writes.
+//
+// The compiler reads its standard library from files, so a page must give it
+// one. A flat, self-describing container is smaller than a tar (no per-entry
+// headers) and inflates with the one decompressor a browser and Node both have.
+//
+// Format (before gzip):
+//   "ODINPOC1"          8 bytes
+//   count               u32 little-endian
+//   count × { pathLen:u32, path:utf8, dataLen:u32, data:bytes }
+
+export const PACK_MAGIC = 'ODINPOC1';
+
+/**
+ * @param {ArrayBuffer|Uint8Array} buffer - the inflated container
+ * @returns {[string, Uint8Array][]} `base/…`, `core/…` and their bytes
+ */
+export function unpackLibrary(buffer) {
+  const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+  if (new TextDecoder().decode(bytes.subarray(0, 8)) !== PACK_MAGIC) {
+    throw new Error(`The standard library pack is not a ${PACK_MAGIC} container.`);
+  }
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const decoder = new TextDecoder();
+  const count = view.getUint32(8, true);
+  const entries = [];
+  let pos = 12;
+  for (let i = 0; i < count; i++) {
+    const pathLen = view.getUint32(pos, true);
+    pos += 4;
+    const path = decoder.decode(bytes.subarray(pos, pos + pathLen));
+    pos += pathLen;
+    const dataLen = view.getUint32(pos, true);
+    pos += 4;
+    entries.push([path, bytes.subarray(pos, pos + dataLen)]);
+    pos += dataLen;
+  }
+  return entries;
+}

package/src/packaged.node.js

@@ -0,0 +1,13 @@
+// The Node half of the packaged-asset source: reading the files that ship in
+// this package.
+//
+// This module is only reachable through the `node` condition in package.json,
+// so a browser bundle never includes it and never trips over `node:fs`.
+import { readFile } from 'node:fs/promises';
+
+export const PACKAGED_ROOT = new URL('../assets/', import.meta.url);
+
+export const packagedAssets = {
+  root: PACKAGED_ROOT,
+  readFile: async (name) => new Uint8Array(await readFile(new URL(name, PACKAGED_ROOT))),
+};

package/src/runtime.js

@@ -0,0 +1,40 @@
+// The engines, one per asset source.
+//
+// A source is keyed by where its bytes came from, so creating two compilers
+// against the same `baseUrl` — or the same package — compiles the compiler
+// module and unpacks the standard library once. Every compiler holds a
+// reference; the entry is dropped when the last one is disposed.
+
+import { inflateGzip } from './assets.js';
+import { createEngine } from './engine.js';
+import { unpackLibrary } from './pack.js';
+
+const engines = new Map();
+
+async function loadEngine(source, onProgress) {
+  const report = (name) => (loaded, total) => onProgress?.({ name, loaded, total });
+  const wasmBytes = await source.readAsset('odin.wasm', report('odin.wasm'));
+  const packBytes = await source.readAsset('odin-src.pack.gz', report('odin-src.pack.gz'));
+  const compilerModule = await WebAssembly.compile(wasmBytes);
+  const libraryEntries = unpackLibrary(await inflateGzip(packBytes));
+  return createEngine({ compilerModule, libraryEntries });
+}
+
+/** @returns {{ promise: Promise<object>, refs: number, key: string }} */
+export function acquireEngine(source, onProgress) {
+  let entry = engines.get(source.key);
+  if (entry === undefined) {
+    entry = { key: source.key, refs: 0, promise: loadEngine(source, onProgress) };
+    engines.set(source.key, entry);
+    // A failed load must not be cached, or every later attempt reports the same
+    // stale failure.
+    entry.promise.catch(() => engines.delete(source.key));
+  }
+  entry.refs += 1;
+  return entry;
+}
+
+export function releaseEngine(entry) {
+  entry.refs -= 1;
+  if (entry.refs <= 0) engines.delete(entry.key);
+}