beam.com 0.1.0-rc.1

package/runtime/cloudflare/durable.js

@@ -0,0 +1,361 @@
+// The VM in one Durable Object, in place of one VM for each isolate: one
+// instance for all the requests (a server whose state must be in one
+// place), which runs all the time while it is in memory, and whose SQLite
+// storage is the database of Ecto SQLite. All the requests go to the object
+// "main" (BEAM_OBJECT names another one).
+//
+// Tenants: with the var BEAM_TENANTS, each tenant has its own object (its
+// own VM, state and SQLite storage), and the request header x-beam-tenant
+// gives its name to the app:
+// - "cookie": GET /.tenant/NAME sets the cookie beam_tenant, and the cookie
+//   names the object (for a workers.dev URL);
+// - "host": the first label of the host names the object (NAME.example.com);
+// - "path": /t/NAME/... names the object. The front Worker removes the
+//   prefix, and the VM gets BEAM_TENANT=NAME and BEAM_TENANT_PATH=/t/NAME,
+//   the base path of the URLs that the app writes. A request from an
+//   iframe of another site has no cookie, but it has its path.
+// A name has 1 to 32 characters: a-z, 0-9 and "-".
+//
+// Instances (path tenants, with the var BEAM_INSTANCES): each visitor can
+// start an instance (the button of the page of /), with a random name and
+// a time limit. At the limit, the object deletes its storage and stops.
+// The object ".registry" admits the visitors:
+// - BEAM_INSTANCES: the instances at one time (then a queue);
+// - BEAM_INSTANCE_TTL: the life of an instance in seconds (1800);
+// - BEAM_INSTANCE_HOURS: the hours of instances in each UTC day (24), a
+//   bound for the Durable Objects of the Free plan;
+// - BEAM_INSTANCES_PER_IP: the instances at one time for one address (2).
+// The VM gets BEAM_INSTANCE_EXPIRES (Unix seconds). A cron trigger (each
+// 30 minutes) calls sweep(): the registry deletes the storage of each
+// instance past its limit (if its alarm did not), and once the storage of
+// each object that BEAM_RETIRE names ("a,b,c": objects of an earlier mode,
+// which the registry did not make).
+//
+//   wrangler deploy -c wrangler.durable.jsonc
+import { DurableObject } from 'cloudflare:workers';
+import { Vm } from './worker.js';
+
+const valid = (name) => /^[a-z0-9-]{1,32}$/.test(name ?? '');
+const REGISTRY = '.registry';
+const EXPIRES = 'beam:expires';
+
+export class Beam extends DurableObject {
+  constructor(ctx, env) {
+    super(ctx, env);
+    this.vm = null;
+    this.expires = undefined;
+  }
+
+  // The VM starts at the first request: a path tenant gives its name there.
+  // The storage of the object keeps the secrets that the VM makes, and the
+  // host of the request is the default PHX_HOST (Vm.autoVars).
+  makeVm(vars, host) {
+    return new Vm(this.env, { plain: false, sql: this.ctx.storage.sql, id: this.ctx.id.toString(), vars,
+                              secrets: this.ctx.storage, host });
+  }
+
+  async fetch(request) {
+    if (this.env.BEAM_INSTANCES && this.env.BEAM_TENANTS === 'path') {
+      this.expires ??= (await this.ctx.storage.get(EXPIRES)) ?? null;
+      if (!this.expires || this.expires <= Date.now()) {
+        if (this.expires) await this.expire();
+        return page(410, 'This instance is not available', `<p>An instance stops at its time limit, and its files are deleted.</p>
+<form method="post" action="/.instance"><button>Start a new instance</button></form>`);
+      }
+    }
+    if (!this.vm) {
+      const vars = {};
+      const tenant = request.headers.get('x-beam-tenant');
+      if (this.env.BEAM_TENANTS === 'path' && valid(tenant)) {
+        vars.BEAM_TENANT = tenant;
+        vars.BEAM_TENANT_PATH = `/t/${tenant}`;
+      }
+      if (this.expires) vars.BEAM_INSTANCE_EXPIRES = String(Math.floor(this.expires / 1000));
+      this.vm = this.makeVm(vars, new URL(request.url).hostname);
+    }
+    return this.vm.fetch(request);
+  }
+
+  // An instance: its time limit (from the registry).
+  async start(expires) {
+    await this.ctx.storage.put(EXPIRES, expires);
+    await this.ctx.storage.setAlarm(expires);
+    this.expires = expires;
+  }
+
+  async alarm() {
+    await this.expire();
+  }
+
+  // The end of an instance: its storage (the files of BEAM_PERSIST, the
+  // database) is deleted, and the VM stops.
+  async expire() {
+    this.expires = null;
+    await this.ctx.storage.deleteAll();
+    if (this.vm) setTimeout(() => this.ctx.abort('the instance expired'), 0);
+  }
+
+  // The registry (the object ".registry") admits a visitor: a ticket and a
+  // hash of its address. It gives {name, expires} of a new or running
+  // instance, the place in the queue, or a limit.
+  async admit(ticket, address) {
+    const env = this.env, sql = this.ctx.storage.sql, now = Date.now();
+    // A limit that is not a number is the default (NaN would be no limit).
+    const max = limit(env.BEAM_INSTANCES, 1);
+    const ttl = limit(env.BEAM_INSTANCE_TTL, 1800);
+    const hours = limit(env.BEAM_INSTANCE_HOURS, 24);
+    const perAddress = limit(env.BEAM_INSTANCES_PER_IP, 2);
+    this.tables();
+    // A visitor that did not refresh its page for 60 s left the queue.
+    sql.exec('DELETE FROM beam_waiting WHERE seen < ?', now - 60000);
+    // The rows of the instances past their limit stay until sweep().
+    const one = (q, ...a) => [...sql.exec(q, ...a)][0];
+    const mine = one('SELECT name, expires FROM beam_instances WHERE ticket = ? AND expires > ?', ticket, now);
+    if (mine) return { name: mine.name, expires: mine.expires, running: true };
+    const count = one('SELECT count(*) AS n FROM beam_instances WHERE expires > ?', now).n;
+    const next = one('SELECT min(expires) AS t FROM beam_instances WHERE expires > ?', now).t;
+    if (one('SELECT count(*) AS n FROM beam_instances WHERE address = ? AND expires > ?', address, now).n >= perAddress) {
+      return { limit: 'address', next };
+    }
+    const day = new Date(now).toISOString().slice(0, 10);
+    const used = one('SELECT seconds FROM beam_budget WHERE day = ?', day)?.seconds ?? 0;
+    if (used + ttl > hours * 3600) return { limit: 'day' };
+    // An address has at most perAddress places in the queue, so that one
+    // client with many tickets cannot fill it.
+    const queued = one('SELECT count(*) AS n FROM beam_waiting WHERE address = ? AND ticket <> ?', address, ticket).n;
+    if (queued >= perAddress) return { limit: 'address', next };
+    sql.exec('INSERT INTO beam_waiting VALUES (?, ?, ?, ?) ON CONFLICT(ticket) DO UPDATE SET seen = excluded.seen',
+      ticket, address, now, now);
+    const since = one('SELECT since FROM beam_waiting WHERE ticket = ?', ticket).since;
+    const position = one('SELECT count(*) AS n FROM beam_waiting WHERE since < ? OR (since = ? AND ticket < ?)',
+      since, since, ticket).n + 1;
+    if (position > max - count) {
+      return { position, waiting: one('SELECT count(*) AS n FROM beam_waiting').n, next };
+    }
+    const name = randomName();
+    const expires = now + ttl * 1000;
+    sql.exec('INSERT INTO beam_instances VALUES (?, ?, ?, ?)', name, ticket, address, expires);
+    sql.exec('DELETE FROM beam_waiting WHERE ticket = ?', ticket);
+    sql.exec('DELETE FROM beam_budget WHERE day < ?', day);
+    sql.exec('INSERT INTO beam_budget VALUES (?, ?) ON CONFLICT(day) DO UPDATE SET seconds = seconds + excluded.seconds',
+      day, ttl);
+    return { name, expires };
+  }
+
+  // The registry: deletes the storage of the instances past their limit,
+  // and of the objects that BEAM_RETIRE names (once). Gives the counts.
+  async sweep() {
+    const env = this.env, sql = this.ctx.storage.sql, now = Date.now();
+    this.tables();
+    const stub = (name) => env.BEAM.get(env.BEAM.idFromName(name));
+    let expired = 0, retired = 0;
+    // An error of one object does not stop the sweep of the others.
+    for (const { name } of [...sql.exec('SELECT name FROM beam_instances WHERE expires <= ?', now)]) {
+      try {
+        await stub(name).expire();
+        sql.exec('DELETE FROM beam_instances WHERE name = ?', name);
+        expired++;
+      } catch (e) {
+        console.log(`beam: sweep: ${name}: ${e}`);
+      }
+    }
+    const done = new Set([...sql.exec('SELECT name FROM beam_retired')].map((r) => r.name));
+    for (const name of (env.BEAM_RETIRE ?? '').split(',').map((n) => n.trim()).filter(valid)) {
+      if (done.has(name)) continue;
+      try {
+        await stub(name).expire();
+        sql.exec('INSERT INTO beam_retired VALUES (?, ?)', name, now);
+        retired++;
+      } catch (e) {
+        console.log(`beam: sweep: ${name}: ${e}`);
+      }
+    }
+    return { expired, retired };
+  }
+
+  // The registry: the instances in use, for the page of /.
+  async usage() {
+    this.tables();
+    const n = [...this.ctx.storage.sql.exec('SELECT count(*) AS n FROM beam_instances WHERE expires > ?', Date.now())][0].n;
+    return { used: n, max: Number(this.env.BEAM_INSTANCES), ttl: Number(this.env.BEAM_INSTANCE_TTL ?? 1800) };
+  }
+
+  tables() {
+    const sql = this.ctx.storage.sql;
+    sql.exec('CREATE TABLE IF NOT EXISTS beam_instances (name TEXT PRIMARY KEY, ticket TEXT, address TEXT, expires INTEGER)');
+    sql.exec('CREATE TABLE IF NOT EXISTS beam_waiting (ticket TEXT PRIMARY KEY, address TEXT, since INTEGER, seen INTEGER)');
+    sql.exec('CREATE TABLE IF NOT EXISTS beam_budget (day TEXT PRIMARY KEY, seconds INTEGER)');
+    sql.exec('CREATE TABLE IF NOT EXISTS beam_retired (name TEXT PRIMARY KEY, time INTEGER)');
+  }
+}
+
+// 20 characters of a-z and 2-7: 100 random bits.
+function randomName() {
+  const abc = 'abcdefghijklmnopqrstuvwxyz234567';
+  return [...crypto.getRandomValues(new Uint8Array(20))].map((b) => abc[b & 31]).join('');
+}
+
+// A limit of the vars: a number of 0 or more, else the default.
+function limit(value, fallback) {
+  const n = Number(value);
+  return value !== undefined && value !== '' && Number.isFinite(n) && n >= 0 ? n : fallback;
+}
+
+// The address of a client for its limits: an IPv6 client has a whole /64,
+// so the limits count the /64.
+function clientKey(ip) {
+  if (!ip.includes(':')) return ip;
+  const [head, tail = ''] = ip.split('::');
+  const h = head ? head.split(':') : [], t = tail ? tail.split(':') : [];
+  const all = ip.includes('::') ? [...h, ...Array(8 - h.length - t.length).fill('0'), ...t] : h;
+  return all.slice(0, 4).map((x) => x.toLowerCase().replace(/^0+(?=.)/, '')).join(':') + '::/64';
+}
+
+const hex = (bytes) => [...bytes].map((b) => b.toString(16).padStart(2, '0')).join('');
+
+function cookie(request, name) {
+  return new RegExp(`(?:^|;\\s*)${name}=([^;]*)`).exec(request.headers.get('cookie') ?? '')?.[1];
+}
+
+function tenant(request, env) {
+  if (env.BEAM_TENANTS === 'host') return new URL(request.url).hostname.split('.')[0];
+  if (env.BEAM_TENANTS === 'cookie') return cookie(request, 'beam_tenant');
+  return null;
+}
+
+const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[c]);
+
+function page(status, title, body, headers = {}) {
+  return new Response(`<!doctype html><html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1"><title>${esc(title)}</title>
+<style>body{font:16px/1.5 system-ui,sans-serif;max-width:40rem;margin:4rem auto;padding:0 1rem;color:#1f2937}
+button{font:inherit;padding:.5rem 1rem;border-radius:.5rem;border:0;background:#4f46e5;color:#fff;cursor:pointer}</style>
+</head><body><h1>${esc(title)}</h1>${body}</body></html>`,
+  { status, headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store', ...headers } });
+}
+
+const minutes = (ms) => Math.max(1, Math.ceil(ms / 60000));
+
+// A static file of the app, in the front Worker (no request to the
+// object): BEAM_STATICS (serve(app) of the npm package gives it, for the
+// static files in app.com), else, with assets, the static assets of the
+// binding ASSETS (wasm/erts/host/static.mjs). Cloudflare serves the
+// assets of the root before the Worker runs, so only a path tenant asks
+// ASSETS (after its prefix).
+async function staticFile(env, request, assets = false) {
+  if (request.method !== 'GET' && request.method !== 'HEAD') return null;
+  if (env.BEAM_STATICS) return env.BEAM_STATICS(request);
+  if (!assets || !env.ASSETS) return null;
+  const asset = await env.ASSETS.fetch(request);
+  return asset.status !== 404 ? asset : null;
+}
+
+// The instances: the page of /, and POST (the button) or GET (the refresh
+// of the queue) of /.instance. Only a POST makes a ticket, so a crawler
+// that reads / starts no instance.
+async function instances(request, env, url) {
+  const registry = env.BEAM.get(env.BEAM.idFromName(REGISTRY));
+  if (url.pathname === '/' && request.method === 'GET') {
+    const { used, max, ttl } = await registry.usage();
+    return page(200, env.BEAM_INSTANCE_TITLE ?? 'Start an instance', `<p>Each visitor gets an instance of its own, for ${minutes(ttl * 1000)} minutes.
+At the end, the instance stops, and its files are deleted.</p>
+<p>${used} of ${max} instances are in use.</p>
+<form method="post" action="/.instance"><button>Start an instance</button></form>`);
+  }
+  if (url.pathname !== '/.instance' || (request.method !== 'POST' && request.method !== 'GET')) {
+    return new Response('not found\n', { status: 404 });
+  }
+  // Only the navigation of a page (the button, the refresh) gets the name
+  // of an instance: a script of a page of another instance, on this same
+  // origin, cannot read it with fetch().
+  const mode = request.headers.get('sec-fetch-mode');
+  if (mode && mode !== 'navigate') return new Response('forbidden\n', { status: 403 });
+  let ticket = cookie(request, 'beam_ticket');
+  const headers = {};
+  if (!/^[0-9a-f]{32}$/.test(ticket ?? '')) {
+    if (request.method === 'GET') return Response.redirect(new URL('/', url), 303);
+    ticket = hex(crypto.getRandomValues(new Uint8Array(16)));
+    headers['set-cookie'] = `beam_ticket=${ticket}; Path=/; Secure; HttpOnly; SameSite=Lax`;
+  }
+  // A hash of the address and the day, not the address.
+  const day = new Date().toISOString().slice(0, 10);
+  const digest = await crypto.subtle.digest('SHA-256',
+    new TextEncoder().encode(`${clientKey(request.headers.get('cf-connecting-ip') ?? '')} ${day}`));
+  const r = await registry.admit(ticket, hex(new Uint8Array(digest).slice(0, 16)));
+  if (r.name) {
+    if (!r.running) await env.BEAM.get(env.BEAM.idFromName(r.name)).start(r.expires);
+    return new Response(null, { status: 303, headers: { ...headers, location: `/t/${r.name}/` } });
+  }
+  if (r.limit === 'day') {
+    return page(503, 'No more instances today', '<p>The instances of today are used. They start again at 00:00 UTC.</p>', headers);
+  }
+  const wait = r.next ? `The next instance stops in about ${minutes(r.next - Date.now())} minutes.` : '';
+  if (r.limit === 'address') {
+    return page(429, 'Too many instances', `<p>Your address has the most instances that it can have at one time. ${wait}</p>`, headers);
+  }
+  return page(200, 'You are in the queue', `<p>Your place: ${r.position} of ${r.waiting}. ${wait}</p>
+<p>This page refreshes each 10 seconds. Keep it open to keep your place.</p>`,
+  { ...headers, refresh: '10; url=/.instance' });
+}
+
+export default {
+  // The cron trigger of the instances (see sweep()).
+  async scheduled(controller, env, ctx) {
+    if (!env.BEAM_INSTANCES) return;
+    const registry = env.BEAM.get(env.BEAM.idFromName(REGISTRY));
+    ctx.waitUntil(registry.sweep().then((r) => console.log(`beam: sweep: ${r.expired} expired, ${r.retired} retired`)));
+  },
+
+  async fetch(request, env) {
+    const url = new URL(request.url);
+    if (env.BEAM_TENANTS === 'path') {
+      const m = /^\/t\/([^/]+)(\/.*)?$/.exec(url.pathname);
+      if (!m) {
+        // A path of the app with no prefix (Livebook writes some so).
+        const asset = await staticFile(env, request);
+        if (asset) return asset;
+        if (env.BEAM_INSTANCES) return instances(request, env, url);
+        return new Response('not found\n', { status: 404 });
+      }
+      if (!valid(m[1])) return new Response('bad tenant name\n', { status: 400 });
+      if (m[2] === undefined) return Response.redirect(new URL(`/t/${m[1]}/`, url), 301);
+      // The path of the app, with no prefix, on the same origin (a path
+      // "//host/..." must not change the host).
+      const inner = new URL(url);
+      inner.pathname = m[2];
+      const asset = await staticFile(env, new Request(inner, request), true);
+      if (asset) return asset;
+      // All the tenants share this origin. A service worker of one tenant
+      // could take the requests of the others, so none can register.
+      if (request.headers.get('service-worker') === 'script') {
+        return new Response('no service workers for tenants\n', { status: 403 });
+      }
+      // The app can trust the header: a client cannot give it.
+      request = new Request(inner, request);
+      request.headers.set('x-beam-tenant', m[1]);
+      const response = await env.BEAM.get(env.BEAM.idFromName(m[1])).fetch(request);
+      if (!response.headers.has('service-worker-allowed')) return response;
+      const out = new Response(response.body, response);
+      out.headers.delete('service-worker-allowed');
+      return out;
+    }
+    const set = env.BEAM_TENANTS === 'cookie' && /^\/\.tenant\/([^/]+)$/.exec(url.pathname);
+    if (set) {
+      if (!valid(set[1])) return new Response('bad tenant name\n', { status: 400 });
+      return new Response(null, {
+        status: 303,
+        headers: { location: '/', 'set-cookie': `beam_tenant=${set[1]}; Path=/; Secure; HttpOnly; SameSite=Lax` },
+      });
+    }
+    const asset = await staticFile(env, request);
+    if (asset) return asset;
+    let name = env.BEAM_OBJECT ?? 'main';
+    const t = tenant(request, env);
+    if (t !== undefined && t !== null && !valid(t)) return new Response('bad tenant name\n', { status: 400 });
+    name = t ?? name;
+    request = new Request(request);
+    if (env.BEAM_TENANTS) request.headers.set('x-beam-tenant', name);
+    else request.headers.delete('x-beam-tenant');
+    return env.BEAM.get(env.BEAM.idFromName(name)).fetch(request);
+  },
+};

package/runtime/cloudflare/index.js

@@ -0,0 +1,76 @@
+// The module "beam.com" of the npm package in a Worker (the condition
+// "workerd" of its exports, which Wrangler uses): the engine that runs the
+// app.com of the project, with the runtime of this package. The entry of
+// the Worker:
+//
+//   import app from './app.com' with { type: 'bytes' };
+//   import { serve } from 'beam.com';
+//   export { Beam } from 'beam.com';   // stateful only
+//
+//   const beam = serve(app);
+//
+//   export default {
+//     fetch(request, env, ctx) {
+//       return beam.fetch(request, env, ctx);
+//     },
+//   };
+//
+// The app is stateful or stateless, as the Worker gives it:
+// - stateful: with the export Beam and its binding (BEAM) in wrangler.jsonc,
+//   each request goes to one Durable Object (durable.js): one VM for all
+//   the requests, whose timers run between requests, and its SQLite
+//   storage for Ecto SQLite;
+// - stateless: with no binding, each isolate runs its own VM (worker.js).
+//
+// wrangler.jsonc gives app.com to the Worker as a Data module:
+//   "rules": [{ "type": "Data", "globs": ["**/*.com"], "fallthrough": true }]
+import { release, use } from './release.js';
+import plain, { staticResponse } from './worker.js';
+import front from './durable.js';
+
+export { Beam } from './durable.js';
+
+// options.binding: the binding of the Durable Object (BEAM). options.name:
+// the name of the object, or a function of the request that gives it (a
+// tenant, for example); the var BEAM_OBJECT, else "main", by default.
+// options.nifs: the NIF libraries in WebAssembly of app.com, for an app
+// that has them. A Worker cannot compile WebAssembly at run time, so
+// "beam.com --nif-modules app.com ." writes nifs.js and nifs/, and the
+// entry gives them:
+//
+//   import nifs from './nifs.js';
+//   const beam = serve(app, { nifs });
+//
+// With the binding, the front of durable.js routes the requests: the
+// tenants and the instances of its vars (BEAM_TENANTS, BEAM_INSTANCES),
+// and the static assets of the binding ASSETS for path tenants. The
+// instances also need scheduled() for their cron trigger (the sweep):
+//
+//   scheduled(controller, env, ctx) {
+//     return beam.scheduled(controller, env, ctx);
+//   },
+export function serve(app, { binding = 'BEAM', name, nifs = null } = {}) {
+  use(app, nifs);
+  // The env of the front: the binding as BEAM, the name of the object, and
+  // the static files of app.com (the front serves them, with no request
+  // to the object).
+  const statics = async (request) => {
+    const files = (await release()).statics;
+    return files ? staticResponse(files, request) : null;
+  };
+  const frontEnv = (env) => ({
+    ...env, BEAM: env[binding], BEAM_OBJECT: name ?? env.BEAM_OBJECT, BEAM_STATICS: statics,
+  });
+  return {
+    fetch(request, env, ctx) {
+      const objects = env[binding];
+      if (!objects) return plain.fetch(request, env, ctx);
+      if (typeof name === 'function') return objects.getByName(name(request)).fetch(request);
+      return front.fetch(request, frontEnv(env), ctx);
+    },
+    scheduled(controller, env, ctx) {
+      if (!env[binding]) return;
+      return front.scheduled(controller, frontEnv(env), ctx);
+    },
+  };
+}

package/runtime/cloudflare/release.js

@@ -0,0 +1,42 @@
+// The release of a Worker that Wrangler bundles (cloudflare/worker.js and
+// cloudflare/durable.js): the edge part of the app.com of the project, as
+// appFiles gives it (views of the file, no copy). serve(app) of index.js
+// gives the file (use). It
+// runs at the top level of the entry of the Worker, so each isolate has
+// the file, also an isolate that runs only the Durable Object.
+//
+// The file must come from the beam.com of the version of this package:
+// app-com.js refuses a file for another runtime. worker.js reads the
+// release at the first request of an isolate, not in the global scope.
+import { appFiles, bytesReader } from '../app-com.js';
+import runtime from '../runtime-id.js';
+
+let app = null;
+let modules = null;
+
+// nifModules: the NIF libraries in WebAssembly of app.com, compiled by
+// Wrangler (nifs.js of "beam.com --nif-modules"), or null.
+export function use(bytes, nifModules = null) {
+  app = bytes;
+  modules = nifModules;
+}
+
+// For worker.js: the path of each NIF library in /app, and its module.
+export function nifs() {
+  return modules;
+}
+
+// One read of the file for each isolate: the front Worker and its VMs
+// share the files (they are views of app.com).
+let files = null;
+
+export function release() {
+  if (!app) {
+    throw new Error("beam.com: no app.com: call serve(app) in the entry of the Worker");
+  }
+  if (!files) {
+    const { read, size } = bytesReader(app);
+    files = appFiles(read, size, { runtime });
+  }
+  return files;
+}

package/runtime/cloudflare/snapshot.js

@@ -0,0 +1,3 @@
+// No snapshot of the build in a Worker that runs an app.com: the Worker
+// makes its own snapshot (the Cache API, or the R2 bucket SNAPSHOTS).
+export default null;