beam.com 0.1.0-rc.1

package/js/beam.com.mjs

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+// npx beam.com ARGS: beam.com of the version of this package, with ARGS
+// (download.mjs). For example: npx beam.com app.erl -o app.com
+import { spawn } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { constants } from 'node:os';
+import { command, ensure } from './download.mjs';
+
+let release = null;
+try {
+  release = JSON.parse(readFileSync(new URL('../runtime/release.json', import.meta.url), 'utf8'));
+} catch {}
+
+let file;
+try {
+  file = await ensure(release, { log: (text) => console.error(text) });
+} catch (e) {
+  console.error(e.message);
+  process.exit(1);
+}
+const [program, args] = command(file, process.argv.slice(2));
+const child = spawn(program, args, { stdio: 'inherit' });
+// The terminal sends SIGINT to the process group, so beam.com gets it
+// already: a second one would go to the break handler of the BEAM.
+process.on('SIGINT', () => {});
+process.on('SIGTERM', () => child.kill('SIGTERM'));
+child.on('error', (e) => {
+  console.error(`beam.com: ${file}: ${e.message}`);
+  process.exit(1);
+});
+child.on('exit', (code, signal) => {
+  process.exit(signal ? 128 + (constants.signals[signal] ?? 0) : (code ?? 1));
+});

package/js/download.mjs

@@ -0,0 +1,78 @@
+// The beam.com of this package: the file of the GitHub release of the same
+// version, in the cache, with the SHA-256 that the package holds
+// (runtime/release.json, from scripts/npm.sh). The package does not hold
+// the file: a host that only uses the runtime never downloads it.
+//
+// - BEAM_COM: the path of a beam.com to use. Then nothing is downloaded.
+// - BEAM_COM_DOWNLOAD: the URL of the directory of the files, in place of
+//   the GitHub release (a mirror, or a test).
+// - BEAM_COM_CACHE: the cache directory, as for beam.com.
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export const REPOSITORY = 'sntran/BEAM.com';
+
+// The cache directory of beam.com (beam_com:cache_dir/0).
+export function cacheDir(env = process.env) {
+  if (env.BEAM_COM_CACHE) return env.BEAM_COM_CACHE;
+  if (env.XDG_CACHE_HOME) return path.join(env.XDG_CACHE_HOME, 'beam.com');
+  if (env.LOCALAPPDATA) return path.join(env.LOCALAPPDATA, 'beam.com');
+  return path.join(env.HOME ?? os.homedir(), '.cache', 'beam.com');
+}
+
+// The path of the file in the cache. The name stays beam.com, because
+// beam.com selects a tool by its name (mix.com is mix). Windows runs an
+// APE file as a PE executable only with an .exe name.
+export function cachePath({ version, sha256 }, env = process.env, platform = process.platform) {
+  const name = platform === 'win32' ? 'beam.exe' : 'beam.com';
+  return path.join(cacheDir(env), 'npm', `${version}-${sha256.slice(0, 12)}`, name);
+}
+
+export function downloadUrl({ version }, env = process.env) {
+  const base = env.BEAM_COM_DOWNLOAD || `https://github.com/${REPOSITORY}/releases/download/v${version}`;
+  return `${base.replace(/\/$/, '')}/beam.com`;
+}
+
+// The path of a beam.com that runs. The file in the cache was checked when
+// it was downloaded: a file is in the cache only after the check.
+export async function ensure(release, { env = process.env, platform = process.platform, fetch = globalThis.fetch, log = () => {} } = {}) {
+  if (env.BEAM_COM) return env.BEAM_COM;
+  if (!release?.version || !/^[0-9a-f]{64}$/.test(release?.sha256 ?? '')) {
+    throw new Error('this package has no release of beam.com (runtime/release.json): set BEAM_COM to the path of a beam.com');
+  }
+  const file = cachePath(release, env, platform);
+  if (fs.existsSync(file)) return file;
+  const url = downloadUrl(release, env);
+  log(`beam.com: downloading ${url}`);
+  const response = await fetch(url);
+  if (!response.ok) throw new Error(`beam.com: ${url}: status ${response.status}`);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const part = `${file}.${process.pid}.part`;
+  const out = fs.createWriteStream(part, { mode: 0o755 });
+  const hash = createHash('sha256');
+  try {
+    for await (const chunk of response.body) {
+      hash.update(chunk);
+      if (!out.write(chunk)) await new Promise((r) => out.once('drain', r));
+    }
+    await new Promise((resolve, reject) => out.end((e) => (e ? reject(e) : resolve())));
+    const got = hash.digest('hex');
+    if (got !== release.sha256) {
+      throw new Error(`beam.com: ${url}: the SHA-256 is ${got}, and this package expects ${release.sha256}`);
+    }
+    fs.chmodSync(part, 0o755);
+    fs.renameSync(part, file);
+  } finally {
+    out.destroy();
+    fs.rmSync(part, { force: true });
+  }
+  return file;
+}
+
+// The command that starts the file: Windows runs it as a PE executable,
+// and a Unix shell runs the shell script at the start of an APE file.
+export function command(file, args, platform = process.platform) {
+  return platform === 'win32' ? [file, args] : ['/bin/sh', [file, ...args]];
+}

package/js/node.mjs

@@ -0,0 +1,75 @@
+// The BEAM in Node.js (25 or later, for JSPI): a native app.com (beam.com
+// INPUT -o app.com) with the WebAssembly runtime of this package.
+//
+//   import { boot } from 'beam.com';
+//   const vm = await boot('app.com', { env: { PORT: '4000' } });
+//   const response = await vm.fetch(new Request('http://localhost/'));
+//
+// worker.js of the runtime is the code of the Workers. Its TCP sockets use
+// node:net. It imports beam.wasm as a WebAssembly.Module, as in Workers,
+// and Node.js imports a .wasm file as an instance: the hook gives the
+// module to the files of runtime/ only.
+import { registerHooks } from 'node:module';
+import fs from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+// app-com.js and runtime-id.js import nothing: they load before the hooks.
+import { appFiles, appRelease } from '../runtime/app-com.js';
+import runtimeId from '../runtime/runtime-id.js';
+
+const runtimeUrl = new URL('../runtime/', import.meta.url);
+const wasmUrl = new URL('beam.wasm', runtimeUrl);
+const WASM = 'beam-com:beam.wasm';
+
+registerHooks({
+  resolve(spec, ctx, next) {
+    if (spec === './beam.wasm' && ctx.parentURL?.startsWith(runtimeUrl.href)) {
+      return { url: WASM, shortCircuit: true };
+    }
+    return next(spec, ctx);
+  },
+  load(url, ctx, next) {
+    if (url !== WASM) return next(url, ctx);
+    return {
+      format: 'module',
+      shortCircuit: true,
+      source: 'import fs from "node:fs";\n'
+        + `export default await WebAssembly.compile(fs.readFileSync(${JSON.stringify(fileURLToPath(wasmUrl))}));\n`,
+    };
+  },
+});
+
+export { appRelease, runtimeId };
+
+// The release of an app.com for this runtime (release.bin as an
+// ArrayBuffer). The file must come from the beam.com of the version of
+// this package: app-com.js refuses a file for another runtime.
+export function release(app) {
+  return readApp(app, appRelease);
+}
+
+// The release of app with get (appRelease, or appFiles: no copy).
+async function readApp(app, get) {
+  const file = await fs.open(app);
+  try {
+    const { size } = await file.stat();
+    const read = async (at, n) => {
+      const b = new Uint8Array(n);
+      const { bytesRead } = await file.read(b, 0, n, at);
+      return b.subarray(0, bytesRead);
+    };
+    return await get(read, size, { runtime: runtimeId });
+  } finally {
+    await file.close();
+  }
+}
+
+// The VM of an app.com, when it is ready. env: the variables of the VM.
+// There is no cache for snapshots in Node.js, so BEAM_SNAPSHOT is "off".
+// The VM keeps its state between requests (plain: false), as in Deno.
+export async function boot(app, { env = {} } = {}) {
+  const bin = await readApp(app, appFiles);
+  const { Vm } = await import(new URL('worker.js', runtimeUrl).href);
+  const vm = new Vm({ BEAM_SNAPSHOT: 'off', ...env }, { release: bin, plain: false });
+  await vm.ready;
+  return vm;
+}

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "beam.com",
+  "version": "0.1.0-rc.1",
+  "description": "Erlang/OTP and Elixir in one executable file: npx beam.com, and the WebAssembly runtime for a native app.com",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sntran/BEAM.com.git"
+  },
+  "homepage": "https://github.com/sntran/BEAM.com#readme",
+  "engines": {
+    "node": ">=25"
+  },
+  "bin": {
+    "beam.com": "js/beam.com.mjs"
+  },
+  "exports": {
+    ".": {
+      "workerd": "./runtime/cloudflare/index.js",
+      "deno": "./runtime/deno.js",
+      "node": "./js/node.mjs",
+      "default": "./js/node.mjs"
+    },
+    "./app-com": "./runtime/app-com.js",
+    "./runtime-id": "./runtime/runtime-id.js",
+    "./worker": "./runtime/worker.js",
+    "./beam.wasm": "./runtime/beam.wasm",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "js/",
+    "runtime/",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/runtime/app-com.js

@@ -0,0 +1,203 @@
+// The release of a native app.com for the WebAssembly runtime: the same
+// file that runs natively runs in the VM of worker.js (Workers, Deno, a
+// web page). "beam.com INPUT -o app.com" writes the edge part of the file
+// into its zip, under .wasm/ (beam_com_wasm:overlay/4): .release.json
+// (the boot of the VM, and the runtime of the build), the application
+// wasm_host, the boot script with wasm_host, and the modules in the place
+// of NIFs.
+//
+// appRelease(read, size, { runtime }) gives the release as release.bin
+// gives it (an ArrayBuffer, see unpack in worker.js): the files of lib/
+// and releases/ of the zip, with the files of .wasm/ in their place, and
+// .release.json first.
+// - read(at, n): n bytes (a Uint8Array) at the offset at of app.com: a
+//   file, a range of a URL, or bytes in memory (bytesReader).
+// - size: the size of app.com in bytes.
+// - runtime: the identity of this runtime (the text of runtime-id.js of
+//   DIR, beam_com_wasm:runtime_id/1). With it, a file for another runtime
+//   is an error.
+// It reads three parts: the end of the file, the central directory, and
+// the span of the release (the edge part is at the end of it).
+//
+// appFiles(read, size, { runtime }) gives the same files with no copy:
+// {meta, files, byteLength, buffers, statics}, which worker.js takes in
+// the place of release.bin.
+// - files: [path, bytes] for each file of the release. A stored entry is a
+//   view of the bytes that read gives (with bytesReader, of app.com).
+// - buffers: the ArrayBuffers of these bytes. A file of the VM in another
+//   buffer is a file that the boot wrote (the snapshot keeps it).
+// - statics: the static files of a Phoenix app (lib/NAME-VSN/priv/static,
+//   NAME the name of the release), a Map of path to {data, size, crc, deflated}, which
+//   the host serves before the VM (worker.js). They are not in files,
+//   except cache_manifest.json, which Phoenix reads at its start.
+//
+// The zip of beam.com stores each .beam file as a gzip file (the loader of
+// ERTS reads it so, beam_com_zip), so it is a view too. A deflated .beam
+// entry (an older app.com) becomes a gzip file with the same data, with
+// no inflate. Each other entry is checked with its CRC-32.
+
+const text = new TextDecoder();
+const u16 = (b, i) => b[i] | (b[i + 1] << 8);
+const u32 = (b, i) => (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
+const fail = (message) => { throw new Error(`app.com: ${message}`); };
+
+const LOCAL = 0x04034b50, CENTRAL = 0x02014b50, END = 0x06054b50;
+const STORED = 0, DEFLATED = 8;
+// The end record (22 bytes) and a comment of at most 65535 bytes.
+const TAIL = 22 + 0xffff;
+
+const CRC = new Uint32Array(256).map((_, n) => {
+  let c = n;
+  for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+  return c >>> 0;
+});
+
+export function crc32(b) {
+  let c = 0xffffffff;
+  for (let i = 0; i < b.length; i++) c = CRC[(c ^ b[i]) & 0xff] ^ (c >>> 8);
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+// A reader of bytes in memory (an ArrayBuffer or a typed array).
+export function bytesReader(bytes) {
+  const b = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes.buffer ?? bytes, bytes.byteOffset ?? 0, bytes.byteLength);
+  return { size: b.length, read: async (at, n) => b.subarray(at, at + n) };
+}
+
+// The central directory: {entries, cdAt}, where each entry is {name,
+// method, crc, csize, size, offset}.
+export async function zipEntries(read, size) {
+  const n = Math.min(size, TAIL);
+  const tail = await read(size - n, n);
+  if (tail.length !== n) fail('a short read at the end of the file');
+  // The last end record whose comment ends the file.
+  let e = -1;
+  for (let i = n - 22; i >= 0; i--) {
+    if (u32(tail, i) === END && i + 22 + u16(tail, i + 20) === n) { e = i; break; }
+  }
+  if (e < 0) fail('not a zip file (no end of central directory)');
+  const count = u16(tail, e + 10), cdSize = u32(tail, e + 12), cdAt = u32(tail, e + 16);
+  if (count === 0xffff || cdAt === 0xffffffff || cdSize === 0xffffffff) fail('a zip64 file is not supported');
+  if (cdAt + cdSize > size) fail('the central directory is outside the file');
+  const cd = await read(cdAt, cdSize);
+  const entries = [];
+  for (let i = 0, k = 0; k < count; k++) {
+    if (i + 46 > cd.length || u32(cd, i) !== CENTRAL) fail('a bad central directory');
+    const nlen = u16(cd, i + 28), xlen = u16(cd, i + 30), clen = u16(cd, i + 32);
+    entries.push({
+      name: text.decode(cd.subarray(i + 46, i + 46 + nlen)), method: u16(cd, i + 10),
+      crc: u32(cd, i + 16), csize: u32(cd, i + 20), size: u32(cd, i + 24), offset: u32(cd, i + 42),
+    });
+    i += 46 + nlen + xlen + clen;
+  }
+  return { entries, cdAt };
+}
+
+async function inflate(raw) {
+  const s = new Blob([raw]).stream().pipeThrough(new DecompressionStream('deflate-raw'));
+  return new Uint8Array(await new Response(s).arrayBuffer());
+}
+
+// The same deflate data in a gzip file: a header of 10 bytes, the data,
+// the CRC-32 and the size.
+function gzipOf(raw, e) {
+  const out = new Uint8Array(10 + raw.length + 8);
+  out.set([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255]);
+  out.set(raw, 10);
+  const v = new DataView(out.buffer);
+  v.setUint32(10 + raw.length, e.crc, true);
+  v.setUint32(14 + raw.length, e.size, true);
+  return out;
+}
+
+// The data of the entry e, in span (the bytes of the file from offset base).
+async function entryData(span, base, e) {
+  const h = e.offset - base;
+  if (h < 0 || h + 30 > span.length || u32(span, h) !== LOCAL) fail(`${e.name}: no local header`);
+  const start = h + 30 + u16(span, h + 26) + u16(span, h + 28);
+  if (start + e.csize > span.length) fail(`${e.name}: the data is outside the file`);
+  const raw = span.subarray(start, start + e.csize);
+  if (e.method === DEFLATED && e.name.endsWith('.beam')) return gzipOf(raw, e);
+  let d;
+  if (e.method === STORED) d = raw;
+  else if (e.method === DEFLATED) d = await inflate(raw);
+  else fail(`${e.name}: the compression method ${e.method} is not supported`);
+  if (d.length !== e.size || crc32(d) !== e.crc) fail(`${e.name}: the CRC-32 does not match`);
+  return d;
+}
+
+const EDGE = '.wasm/';
+const inRelease = (name) => name.startsWith('lib/') || name.startsWith('releases/') || name.startsWith(EDGE);
+
+export async function appRelease(read, size, options = {}) {
+  const { meta, files } = await appFiles(read, size, { ...options, statics: false });
+  return pack(meta, files);
+}
+
+export async function appFiles(read, size, { runtime, statics: split = true } = {}) {
+  const { entries, cdAt } = await zipEntries(read, size);
+  const want = entries.filter((e) => inRelease(e.name) && !e.name.endsWith('/'));
+  const json = want.find((e) => e.name === `${EDGE}.release.json`);
+  if (!json) fail('no edge part (.wasm/.release.json): build it with a beam.com that has the WebAssembly runtime, and without --no-edge');
+  // One read: the entries of the release, up to the central directory.
+  const lo = Math.min(...want.map((e) => e.offset));
+  const span = await read(lo, cdAt - lo);
+  if (span.length !== cdAt - lo) fail('a short read of the release');
+  const meta = await entryData(span, lo, json);
+  const release = JSON.parse(text.decode(meta));
+  if (runtime && release.runtime !== runtime) {
+    fail(`it was built for the runtime ${String(release.runtime).slice(0, 12)}, and this runtime is ${runtime.slice(0, 12)}: `
+         + 'use the runtime of the beam.com that built it');
+  }
+  // The static files of a Phoenix app (PHX_SERVER in the environment of
+  // the release): lib/NAME-VSN/priv/static/, for the application with the
+  // name of the release. Plug.Static serves them at the root of the site.
+  const app = split && release.env?.PHX_SERVER === 'true' && typeof release.name === 'string'
+    && `lib/${release.name}-`;
+  const prefix = app && want.map((e) => /^lib\/[^/]+\/priv\/static\//.exec(e.name)?.[0])
+    .find((p) => p?.startsWith(app) && /^[0-9]/.test(p.slice(app.length)));
+  const statics = new Map();
+  const files = new Map();
+  for (const e of want) {
+    if (e.name.startsWith(EDGE)) continue;
+    if (prefix && e.name.startsWith(prefix) && e.name !== `${prefix}cache_manifest.json`) {
+      statics.set(e.name.slice(prefix.length - 1), staticEntry(span, lo, e));
+      continue;
+    }
+    files.set(e.name, await entryData(span, lo, e));
+  }
+  for (const e of want) if (e.name.startsWith(EDGE) && e !== json) files.set(e.name.slice(EDGE.length), await entryData(span, lo, e));
+  const list = [...files];
+  return {
+    meta, files: list, statics,
+    byteLength: list.reduce((n, [, d]) => n + d.length, meta.length),
+    buffers: new Set([meta.buffer, ...list.map(([, d]) => d.buffer)]),
+  };
+}
+
+// A static file: its bytes in the zip (stored or deflated), not checked
+// here (the host checks the CRC-32 when it inflates them).
+function staticEntry(span, base, e) {
+  const h = e.offset - base;
+  if (h < 0 || h + 30 > span.length || u32(span, h) !== LOCAL) fail(`${e.name}: no local header`);
+  const start = h + 30 + u16(span, h + 26) + u16(span, h + 28);
+  if (start + e.csize > span.length) fail(`${e.name}: the data is outside the file`);
+  if (e.method !== STORED && e.method !== DEFLATED) fail(`${e.name}: the compression method ${e.method} is not supported`);
+  return { data: span.subarray(start, start + e.csize), size: e.size, crc: e.crc, deflated: e.method === DEFLATED };
+}
+
+// release.bin: "BEAMFS1\n", then (length, path, length, data) for each
+// file, with .release.json first.
+function pack(meta, files) {
+  const enc = new TextEncoder();
+  const parts = [['.release.json', meta], ...files].map(([p, d]) => [enc.encode(p), d]);
+  const out = new Uint8Array(8 + parts.reduce((s, [p, d]) => s + 8 + p.length + d.length, 0));
+  const v = new DataView(out.buffer);
+  out.set(enc.encode('BEAMFS1\n'));
+  let i = 8;
+  for (const [p, d] of parts) {
+    v.setUint32(i, p.length); out.set(p, i + 4); i += 4 + p.length;
+    v.setUint32(i, d.length); out.set(d, i + 4); i += 4 + d.length;
+  }
+  return out.buffer;
+}