@live-codes/odin-wasm 0.0.0-stage → 0.1.0

package/src/asset-receipts.js

@@ -0,0 +1,12 @@
+// Receipts for the assets this package ships in `assets/`, keyed by their path under it.
+//
+// Generated by scripts/sync-assets.mjs — do not edit by hand.
+//
+// Every read is checked against one of these before the bytes are used, so a truncated download or
+// a host serving something else is an error here rather than a wasm instantiation failure a step
+// later.
+
+export const ASSET_RECEIPTS = Object.freeze({
+	'odin.wasm': Object.freeze({ bytes: 3763439, sha256: '5044e383aa2415927cd518a4d3e6fdbcc6c99154ac90d212e154f18c092deef3' }),
+	'odin-src.pack.gz': Object.freeze({ bytes: 2502169, sha256: '567b1c0c1b50d3e557d119927ade3373dda91364acae3302f2cf01950a8dd8f9' })
+});

package/src/assets.js

@@ -0,0 +1,149 @@
+// Where the assets come from, and how they are read.
+//
+// Two sources behind one shape, because everything downstream only needs "give
+// me this asset, verified":
+//
+//   hosted   - a base URL. The page fetches from a server you control.
+//   packaged - the files that ship inside this package, on disk. Only reachable
+//              in Node, which is the only place there is a filesystem.
+//
+// Every asset is checked against a pinned SHA-256 receipt before it is used, so
+// a truncated download or a host serving something else is an error here rather
+// than a wasm instantiation failure a step later.
+
+import { ASSET_RECEIPTS } from './asset-receipts.js';
+
+/** The assets the package carries, in the order they are loaded. */
+export const ASSET_NAMES = Object.freeze(Object.keys(ASSET_RECEIPTS));
+
+const BASE_URL_REQUIRED =
+  'baseUrl is required here. The assets that ship in this package can only be read where there is a ' +
+  'filesystem, and a browser cannot reach a file inside an npm package — copy them somewhere your ' +
+  'page can fetch with `npx --package @live-codes/odin-wasm odin-wasm-copy-assets <dir>` and pass ' +
+  'that directory as baseUrl.';
+
+/**
+ * @param {object} options - the options `createCompiler` was given
+ * @param {object|null} packaged - the packaged-asset reader, or null where there is no filesystem
+ */
+export function resolveAssetSource(options, packaged) {
+  if (options.baseUrl != null && options.baseUrl !== '') return createHostedSource(options);
+  if (!packaged) throw new Error(BASE_URL_REQUIRED);
+  return createPackagedSource(packaged);
+}
+
+function createHostedSource(options) {
+  const baseUrl = resolveBaseUrl(options.baseUrl);
+  return {
+    kind: 'hosted',
+    key: `hosted\u0000${baseUrl}`,
+    description: baseUrl,
+    baseUrl,
+    async readAsset(name, onProgress) {
+      const url = new URL(name, baseUrl);
+      const response = await fetch(url);
+      if (!response.ok) {
+        throw new Error(`Failed to load the runtime asset ${url}: ${response.status}`);
+      }
+      return verifyReceipt(name, await readBody(response, onProgress));
+    },
+  };
+}
+
+function createPackagedSource(packaged) {
+  return {
+    kind: 'packaged',
+    key: `packaged\u0000${packaged.root.href}`,
+    description: `the assets packaged with this library (${packaged.root.href})`,
+    async readAsset(name) {
+      let bytes;
+      try {
+        bytes = await packaged.readFile(name);
+      } catch (error) {
+        throw new Error(
+          `Failed to read the packaged asset ${name} from ${packaged.root.href}: ${error.message}`,
+          { cause: error }
+        );
+      }
+      return verifyReceipt(name, bytes);
+    },
+  };
+}
+
+// Absolute http(s); in a browser a relative URL resolves against the page.
+function resolveBaseUrl(value) {
+  let url;
+  try {
+    url = value instanceof URL ? value : new URL(value, globalThis.document?.baseURI);
+  } catch (error) {
+    throw new Error(
+      `baseUrl must be an absolute http(s) URL, or relative to the page in a browser: ${error.message}`,
+      { cause: error }
+    );
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+    throw new Error(
+      `baseUrl must be an absolute http(s) URL, or relative to the page in a browser: got "${value}"`
+    );
+  }
+  return url.href.endsWith('/') ? url.href : `${url.href}/`;
+}
+
+async function readBody(response, onProgress) {
+  if (!onProgress || !response.body) {
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  const total = Number(response.headers.get('content-length')) || 0;
+  const reader = response.body.getReader();
+  const chunks = [];
+  let loaded = 0;
+  onProgress(0, total);
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+    loaded += value.length;
+    onProgress(loaded, total);
+  }
+  const bytes = new Uint8Array(loaded);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return bytes;
+}
+
+async function verifyReceipt(name, bytes) {
+  const receipt = ASSET_RECEIPTS[name];
+  if (!receipt) throw new Error(`No pinned receipt for the runtime asset ${name}`);
+  if (bytes.byteLength !== receipt.bytes) {
+    throw new Error(`The runtime asset ${name} is ${bytes.byteLength} bytes, expected ${receipt.bytes}`);
+  }
+  const digest = await sha256Hex(bytes);
+  if (digest !== receipt.sha256) {
+    throw new Error(
+      `The runtime asset ${name} failed SHA-256 verification: expected ${receipt.sha256}, got ${digest}`
+    );
+  }
+  return bytes;
+}
+
+export async function sha256Hex(bytes) {
+  const subtle = globalThis.crypto?.subtle;
+  if (!subtle) {
+    throw new Error(
+      'Verifying the runtime assets needs crypto.subtle: a secure context in the browser, or Node 20 and later.'
+    );
+  }
+  const digest = await subtle.digest('SHA-256', bytes);
+  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('');
+}
+
+export async function inflateGzip(bytes) {
+  if (typeof DecompressionStream !== 'function') {
+    throw new Error('Inflating the standard library pack needs DecompressionStream');
+  }
+  const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'));
+  return new Uint8Array(await new Response(stream).arrayBuffer());
+}

package/src/engine.js

@@ -0,0 +1,198 @@
+// The compile → run engine. It has no opinion about where the compiler module
+// and the standard library came from; `assets.js` reads them, this runs them.
+//
+// The compiler is a WASI command module, so every compile is a fresh instance
+// over a fresh in-memory file system: the standard library, the user's file,
+// and an `out/` to write into. The module it emits then runs on one of two
+// hosts, depending on the target it was built for.
+
+import { WasiFileSystem, WasiExit, createWasi, runWasi } from './wasi-host.js';
+import { createJsHost, JsExit } from './host-js.js';
+
+// Where the compiler looks for base/, core/ and vendor/.
+const ODIN_ROOT = '/odin';
+const PROGRAM = '/out/main.wasm';
+
+/** The targets the compiler can emit, and the page can run. */
+export const TARGETS = ['wasi_wasm32', 'js_wasm32'];
+
+const encoder = new TextEncoder();
+
+const capture = () => {
+  const stdoutParts = [];
+  const stderrParts = [];
+  const order = [];
+  return {
+    stdout: (text) => {
+      stdoutParts.push(text);
+      order.push(text);
+    },
+    stderr: (text) => {
+      stderrParts.push(text);
+      order.push(text);
+    },
+    stdoutText: () => stdoutParts.join(''),
+    stderrText: () => stderrParts.join(''),
+    outputText: () => order.join(''),
+  };
+};
+
+/** The lines a caller reads as diagnostics, with the empty ones dropped. */
+const diagnosticLines = (text) =>
+  text
+    .split('\n')
+    .map((line) => line.trimEnd())
+    .filter((line) => line.trim() !== '');
+
+/**
+ * @param {object} assets
+ * @param {WebAssembly.Module} assets.compilerModule - the Odin compiler
+ * @param {[string, Uint8Array][]} assets.libraryEntries - the packed standard library
+ */
+export function createEngine({ compilerModule, libraryEntries }) {
+  /** A fresh file system with the standard library under /odin. */
+  const libraryFs = () => {
+    const fs = new WasiFileSystem();
+    for (const [path, data] of libraryEntries) fs.writeFile(`${ODIN_ROOT}/${path}`, data);
+    // The compiler registers its collections (base, core, vendor) from
+    // directories and requires all three to exist; the pack ships no vendor
+    // packages, so the directory is empty.
+    fs.mkdirAll(`${ODIN_ROOT}/vendor`);
+    return fs;
+  };
+
+  const compile = ({ code, target, compileArgs, fileName }) => {
+    const fs = libraryFs();
+    fs.writeFile(`/src/${fileName}`, encoder.encode(code));
+    fs.mkdirAll('/out');
+
+    const compilerOut = [];
+    const compilerErr = [];
+    const args = [
+      'odin',
+      'build',
+      '/src',
+      `-out:${PROGRAM}`,
+      '-backend:wasm',
+      `-target:${target}`,
+      ...compileArgs,
+    ];
+
+    let exitCode = 1;
+    let crash = '';
+    try {
+      exitCode = runWasi(compilerModule, createWasi(fs, {
+        args,
+        env: [`ODIN_ROOT=${ODIN_ROOT}`],
+        stdout: (text) => compilerOut.push(text),
+        stderr: (text) => compilerErr.push(text),
+      }));
+    } catch (error) {
+      crash = error?.message ?? String(error);
+    }
+
+    return {
+      exitCode,
+      artifact: exitCode === 0 ? fs.readFile(PROGRAM) : null,
+      diagnostics: diagnosticLines(compilerOut.join('') + compilerErr.join('')),
+      crash,
+    };
+  };
+
+  const runWasiTarget = (artifact, { input, args }) => {
+    const captured = capture();
+    let exitCode = 0;
+    try {
+      exitCode = runWasi(new WebAssembly.Module(artifact.slice()), createWasi(new WasiFileSystem(), {
+        args: ['main.wasm', ...args],
+        stdin: input,
+        stdout: captured.stdout,
+        stderr: captured.stderr,
+      }));
+    } catch (error) {
+      if (error instanceof WasiExit) exitCode = error.code;
+      else {
+        captured.stderr(`The program trapped: ${error?.message ?? error}\n`);
+        exitCode = 1;
+      }
+    }
+    return { exitCode, ...capturedText(captured) };
+  };
+
+  const runJsTarget = async (artifact, { input, args }) => {
+    const captured = capture();
+    const host = createJsHost({ input, args: ['main.wasm', ...args], stdout: captured.stdout, stderr: captured.stderr });
+    let exitCode = 0;
+    try {
+      const { instance } = await WebAssembly.instantiate(artifact.slice(), host.imports);
+      host.setMemory(instance.exports.memory);
+      if (instance.exports._start) instance.exports._start();
+      if (instance.exports._end) instance.exports._end();
+    } catch (error) {
+      if (error instanceof JsExit) exitCode = error.code;
+      else {
+        captured.stderr(`The program trapped: ${error?.message ?? error}\n`);
+        exitCode = 1;
+      }
+    }
+    return { exitCode, ...capturedText(captured) };
+  };
+
+  /**
+   * Compile and run one program.
+   *
+   * @param {object} params
+   * @param {string} params.code
+   * @param {string|Uint8Array} params.input
+   * @param {string} params.target
+   * @param {string[]} params.args
+   * @param {string[]} params.compileArgs
+   * @param {string} params.fileName
+   */
+  const compileAndRun = async ({ code, input = '', target, args = [], compileArgs = [], fileName = 'main.odin' }) => {
+    const compileStarted = now();
+    const compiled = compile({ code, target, compileArgs: [...compileArgs], fileName });
+    const compileMs = Math.round(now() - compileStarted);
+
+    if (compiled.artifact === null || compiled.artifact.length === 0) {
+      const errors = compiled.crash
+        ? [compiled.crash]
+        : compiled.diagnostics.length > 0
+          ? compiled.diagnostics
+          : [`The compiler produced no module (exit code ${compiled.exitCode}).`];
+      return { stdout: '', stderr: '', output: '', errors, exitCode: null, compileMs, runMs: null };
+    }
+
+    const runStarted = now();
+    let ran;
+    try {
+      ran = target === 'js_wasm32'
+        ? await runJsTarget(compiled.artifact, { input, args })
+        : runWasiTarget(compiled.artifact, { input, args });
+    } catch (error) {
+      ran = { exitCode: 1, stdout: '', stderr: `The program could not run: ${error?.message ?? error}\n`, output: '', };
+      ran.output = ran.stderr;
+    }
+    const runMs = Math.round(now() - runStarted);
+
+    return {
+      stdout: ran.stdout,
+      stderr: ran.stderr,
+      output: ran.output,
+      errors: [],
+      exitCode: ran.exitCode,
+      compileMs,
+      runMs,
+    };
+  };
+
+  return { compileAndRun };
+}
+
+const capturedText = (captured) => ({
+  stdout: captured.stdoutText(),
+  stderr: captured.stderrText(),
+  output: captured.outputText(),
+});
+
+const now = () => (globalThis.performance?.now?.() ?? Date.now());

package/src/host-js.js

@@ -0,0 +1,112 @@
+// The host a `js_wasm32` program runs on, outside a page.
+//
+// A js_wasm32 module imports `odin_env` (the console, stdin, `evaluate`, the
+// math intrinsics), `odin_dom` (elements and events), `webgl` and `webgl2`.
+// Only `odin_env` can do anything without a DOM, so the other three are stubs
+// that accept any call and return 0 — a program that reaches for the DOM runs
+// and does nothing rather than failing to instantiate. A page that wants the
+// DOM and the canvas supplies its own host; see the repository's page, which
+// uses Odin's own `odin.js`.
+
+/** Thrown by `trap`/`abort`, so an exit is not confused with a real crash. */
+export class JsExit extends Error {
+  constructor(code) {
+    super(`The program exited (code ${code}).`);
+    this.name = 'JsExit';
+    this.code = code;
+  }
+}
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+// Any import the host does not define resolves to a no-op returning 0, so a
+// module written against a different set of host functions still instantiates.
+const withFallback = (target) =>
+  new Proxy(target, {
+    get: (object, property) =>
+      typeof property === 'string' && !(property in object) ? () => 0 : object[property],
+  });
+
+/**
+ * @param {object} options
+ * @param {string|Uint8Array} [options.input] - the program's stdin, handed out once
+ * @param {string[]} [options.args] - argv, including argv[0]
+ * @param {(text: string) => void} [options.stdout]
+ * @param {(text: string) => void} [options.stderr]
+ */
+export function createJsHost({ input = '', args = [], stdout = () => {}, stderr = () => {} } = {}) {
+  const inputBytes = typeof input === 'string' ? encoder.encode(input) : new Uint8Array(input);
+  let inputOffset = 0;
+
+  let memory = null;
+  const bytes = () => new Uint8Array(memory.buffer);
+  const readString = (ptr, len) => decoder.decode(bytes().subarray(ptr, ptr + len));
+
+  // Any call to a DOM or WebGL import is accepted and returns 0.
+  const inertNamespace = new Proxy(
+    {},
+    { get: (_target, property) => (typeof property === 'string' ? () => 0 : undefined) }
+  );
+
+  const odin_env = {
+    write(fd, ptr, len) {
+      const text = readString(ptr, len);
+      (fd === 2 ? stderr : stdout)(text);
+    },
+    // Standard input, for the standard library's js streams — see the
+    // repository's patches/core/os/file_js.odin, which is in the shipped pack.
+    read(fd, ptr, len) {
+      if (fd !== 0) return 0;
+      const count = Math.max(0, Math.min(len, inputBytes.length - inputOffset));
+      if (count > 0) {
+        bytes().set(inputBytes.subarray(inputOffset, inputOffset + count), ptr);
+        inputOffset += count;
+      }
+      return count;
+    },
+    trap() {
+      throw new JsExit(1);
+    },
+    abort() {
+      throw new JsExit(1);
+    },
+    alert() {},
+    // The one interop call that works without a DOM: it evaluates in the global
+    // scope, so a page that embeds this can draw through it.
+    evaluate(ptr, len) {
+      const source = readString(ptr, len);
+      (0, globalThis.eval)(source);
+    },
+    open() {},
+    time_now: () => BigInt(Date.now()),
+    tick_now: () => globalThis.performance?.now?.() ?? Date.now(),
+    time_sleep() {},
+    sqrt: Math.sqrt,
+    sin: Math.sin,
+    cos: Math.cos,
+    pow: Math.pow,
+    fmuladd: (x, y, z) => x * y + z,
+    ln: Math.log,
+    exp: Math.exp,
+    ldexp: (x, exponent) => x * 2 ** exponent,
+    rand_bytes(ptr, len) {
+      bytes().set(globalThis.crypto.getRandomValues(new Uint8Array(len)), ptr);
+    },
+  };
+
+  return {
+    imports: {
+      env: withFallback({}),
+      odin_env: withFallback(odin_env),
+      odin_dom: inertNamespace,
+      webgl: inertNamespace,
+      webgl2: inertNamespace,
+    },
+    /** The module exports its own memory, so it is set after instantiation. */
+    setMemory(mem) {
+      memory = mem;
+    },
+    args,
+  };
+}

package/src/index.js

@@ -0,0 +1,9 @@
+// The entry every environment gets unless something more specific matches, so
+// it has to work without a filesystem: no `node:fs` here, and `baseUrl` is
+// required because a browser cannot read a file that lives inside an npm package.
+import { createApi } from './api.js';
+
+const api = createApi({ packaged: null });
+
+export const createCompiler = api.createCompiler;
+export const { TARGETS } = api;

package/src/index.node.js

@@ -0,0 +1,9 @@
+// Node, where the assets that ship in this package can be read off disk, so
+// `baseUrl` becomes optional and `createCompiler()` is enough on its own.
+import { createApi } from './api.js';
+import { packagedAssets } from './packaged.node.js';
+
+const api = createApi({ packaged: packagedAssets });
+
+export const createCompiler = api.createCompiler;
+export const { TARGETS } = api;

package/src/pack.js

@@ -0,0 +1,40 @@
+// The packed standard library: Odin's `base/` and `core/`, in the container the
+// repository's packer writes.
+//
+// The compiler reads its standard library from files, so a page must give it
+// one. A flat, self-describing container is smaller than a tar (no per-entry
+// headers) and inflates with the one decompressor a browser and Node both have.
+//
+// Format (before gzip):
+//   "ODINPOC1"          8 bytes
+//   count               u32 little-endian
+//   count × { pathLen:u32, path:utf8, dataLen:u32, data:bytes }
+
+export const PACK_MAGIC = 'ODINPOC1';
+
+/**
+ * @param {ArrayBuffer|Uint8Array} buffer - the inflated container
+ * @returns {[string, Uint8Array][]} `base/…`, `core/…` and their bytes
+ */
+export function unpackLibrary(buffer) {
+  const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+  if (new TextDecoder().decode(bytes.subarray(0, 8)) !== PACK_MAGIC) {
+    throw new Error(`The standard library pack is not a ${PACK_MAGIC} container.`);
+  }
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const decoder = new TextDecoder();
+  const count = view.getUint32(8, true);
+  const entries = [];
+  let pos = 12;
+  for (let i = 0; i < count; i++) {
+    const pathLen = view.getUint32(pos, true);
+    pos += 4;
+    const path = decoder.decode(bytes.subarray(pos, pos + pathLen));
+    pos += pathLen;
+    const dataLen = view.getUint32(pos, true);
+    pos += 4;
+    entries.push([path, bytes.subarray(pos, pos + dataLen)]);
+    pos += dataLen;
+  }
+  return entries;
+}

package/src/packaged.node.js

@@ -0,0 +1,13 @@
+// The Node half of the packaged-asset source: reading the files that ship in
+// this package.
+//
+// This module is only reachable through the `node` condition in package.json,
+// so a browser bundle never includes it and never trips over `node:fs`.
+import { readFile } from 'node:fs/promises';
+
+export const PACKAGED_ROOT = new URL('../assets/', import.meta.url);
+
+export const packagedAssets = {
+  root: PACKAGED_ROOT,
+  readFile: async (name) => new Uint8Array(await readFile(new URL(name, PACKAGED_ROOT))),
+};

package/src/runtime.js

@@ -0,0 +1,40 @@
+// The engines, one per asset source.
+//
+// A source is keyed by where its bytes came from, so creating two compilers
+// against the same `baseUrl` — or the same package — compiles the compiler
+// module and unpacks the standard library once. Every compiler holds a
+// reference; the entry is dropped when the last one is disposed.
+
+import { inflateGzip } from './assets.js';
+import { createEngine } from './engine.js';
+import { unpackLibrary } from './pack.js';
+
+const engines = new Map();
+
+async function loadEngine(source, onProgress) {
+  const report = (name) => (loaded, total) => onProgress?.({ name, loaded, total });
+  const wasmBytes = await source.readAsset('odin.wasm', report('odin.wasm'));
+  const packBytes = await source.readAsset('odin-src.pack.gz', report('odin-src.pack.gz'));
+  const compilerModule = await WebAssembly.compile(wasmBytes);
+  const libraryEntries = unpackLibrary(await inflateGzip(packBytes));
+  return createEngine({ compilerModule, libraryEntries });
+}
+
+/** @returns {{ promise: Promise<object>, refs: number, key: string }} */
+export function acquireEngine(source, onProgress) {
+  let entry = engines.get(source.key);
+  if (entry === undefined) {
+    entry = { key: source.key, refs: 0, promise: loadEngine(source, onProgress) };
+    engines.set(source.key, entry);
+    // A failed load must not be cached, or every later attempt reports the same
+    // stale failure.
+    entry.promise.catch(() => engines.delete(source.key));
+  }
+  entry.refs += 1;
+  return entry;
+}
+
+export function releaseEngine(entry) {
+  entry.refs -= 1;
+  if (entry.refs <= 0) engines.delete(entry.key);
+}