@coda-console/kernel 0.0.0 → 0.2.0

package/sim/world.mjs

@@ -0,0 +1,922 @@
+import assert from 'node:assert/strict';
+import { DatabaseSync } from 'node:sqlite';
+import { createKernel } from '../kernel.mjs';
+
+// A deterministic simulation: several kernel instances in one process, with the network, the clock, `schedule` and
+// crashes in the test's hands. Nothing here waits on real time; `flush()` only lets pending promise callbacks run.
+//
+// The world knows no host. Each instance runs whatever handlers its `setup` registers; the world wraps them to record,
+// in its own `sim_` tables and in the handler's own transaction, what committed. Its checks read those tables and the
+// kernel's, so they hold for any host; a host adds its own with onStep() and onQuiet(). A host with a layer of its own
+// around the kernel puts it between the world and the kernel with node.ports(), and the world checks that layer too.
+
+/** The instance died right after a commit: the commit stands, everything in memory is gone. */
+export class Crash extends Error {
+  name = 'Crash';
+}
+
+/** The COMMIT itself failed: the transaction is rolled back. */
+export class CommitFailed extends Error {
+  name = 'CommitFailed';
+}
+
+const lost = error => error instanceof Crash || error instanceof CommitFailed;
+
+export const flush = () => new Promise(resolve => setImmediate(resolve));
+
+/** @param {number} seed */
+export function random(seed) {
+  let state = seed >>> 0;
+  return () => {
+    state = (state + 0x6d2b79f5) >>> 0;
+    let t = state;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+export const NO_FAULTS = Object.freeze({ dropRequest: 0, dropReceipt: 0, duplicate: 0, commitFail: 0 });
+
+/** Every statement a kernel prepared, across all worlds of this process. */
+export const prepared = new Set();
+
+const ENDED = new Set(['completed', 'failed', 'canceled']);
+const UNENDED = "('sending', 'open')";
+
+// What the world records of the handlers it wraps, each row in the transaction that called the handler.
+const SIM_TABLES = `
+  CREATE TABLE IF NOT EXISTS sim_accepted (from_addr TEXT NOT NULL, id TEXT NOT NULL);
+  CREATE TABLE IF NOT EXISTS sim_results (from_addr TEXT NOT NULL, id TEXT NOT NULL);
+  CREATE TABLE IF NOT EXISTS sim_lates (from_addr TEXT NOT NULL, id TEXT NOT NULL);
+  CREATE TABLE IF NOT EXISTS sim_cancels (from_addr TEXT NOT NULL, id TEXT NOT NULL);
+  CREATE TABLE IF NOT EXISTS sim_statuses (from_addr TEXT NOT NULL, id TEXT NOT NULL, rev INTEGER NOT NULL);
+  CREATE TABLE IF NOT EXISTS sim_answers (run_from TEXT NOT NULL, run_id TEXT NOT NULL, asked INTEGER NOT NULL, from_addr TEXT NOT NULL,
+    id TEXT NOT NULL);`;
+
+const key = (from, id) => `${from}/${id}`;
+const ending = (row, column) => JSON.stringify([row[column], row.result, row.error, row.progress]);
+// An abandoned call's late outcome, in the form of ending().
+const lateEnding = row => {
+  const late = JSON.parse(row.late);
+  return JSON.stringify([late.outcome, ...[late.result, late.error, late.progress].map(value => value === undefined ? null : JSON.stringify(value))]);
+};
+const statusOf = row => JSON.stringify([row.status, row.rev, row.asked, row.asking, row.progress]);
+
+// The connection one generation of an instance gets, over the database it runs on: it counts commits and fails them
+// on purpose, and once that generation dies every call through it, or through a statement it prepared, throws Crash.
+// Everything else passes through. The kernel's copy also records what it prepares.
+function connection(node, generation, record) {
+  const alive = () => {
+    if (generation.dead) throw new Crash(`${node.name} is down`);
+  };
+  const through = (object, name) => {
+    const value = Reflect.get(object, name, object);
+    return typeof value === 'function' ? (...args) => (alive(), value.apply(object, args)) : value;
+  };
+  return new Proxy(node.raw, {
+    get(raw, name) {
+      if (name === 'exec') return sql => {
+        alive();
+        if (sql === 'BEGIN IMMEDIATE') {
+          node.called = new Set();
+          node.intents = new Set();
+        }
+        if (sql === 'COMMIT') node.commit();
+        else raw.exec(sql);
+      };
+      if (name === 'prepare') return sql => {
+        alive();
+        if (record) prepared.add(sql);
+        return new Proxy(raw.prepare(sql), { get: through });
+      };
+      return through(raw, name);
+    },
+  });
+}
+
+/**
+ * A generation's host puts a letter its kernel handed over into the network, for `to`, an address of some instance of
+ * the world. P-out counts by the letter's text, over the whole generation: a letter may be posted no more often than
+ * this generation's kernel has handed it over so far. A handing over the host did not post stays to its credit, so a
+ * letter handed over twice and posted once may be posted again later. Once that generation has died it throws Crash,
+ * whatever generation has come since.
+ * @param {SimNode} node
+ * @param {object} generation
+ * @param {string} to
+ * @param {object} envelope
+ * @returns {Promise<object>} the receipt; rejected when the letter or its receipt is lost
+ */
+function post(node, generation, to, envelope) {
+  if (generation.dead) throw new Crash(`${node.name} is down`);
+  const { handed } = generation;
+  const text = JSON.stringify(envelope);
+  const owed = handed.get(text) ?? 0;
+  if (owed === 0) node.world.violations.push(`P-out: ${node.name} posted a letter more often than its kernel handed it over: ${text}`);
+  else if (owed === 1) handed.delete(text);
+  else handed.set(text, owed - 1);
+  node.posted += 1;
+  return new Promise((resolve, reject) => node.world.net.push({ request: true, from: node, to, envelope: text, resolve, reject }));
+}
+
+/**
+ * What a host gives an instance: called on every generation inside its first transaction, with migrate(), so what it
+ * writes commits with it. It creates the host's own tables and registers its handlers with `kernel.handle` and
+ * `kernel.results`. Being inside a transaction, it cannot call what runs only outside one (recover(), tick(), pump(),
+ * inquire(), answer(), when()); it hands that to `node.recovered(fn)`. A host with a layer of its own around the kernel
+ * puts it between the world and the kernel with `node.ports()`.
+ * @typedef {(context: { kernel: object, node: SimNode, world: World }) => void} Setup
+ */
+
+/**
+ * Where a host stands between the world and its kernel, for one generation; each one left out is the world's own way.
+ * - receive: the world hands the host a letter for this instance; `from` is the instance that put it into the network,
+ *   which the world vouches for (P3). Returns the receipt.
+ * - send: this generation's kernel hands over a letter; the host puts it into the network with the `post` that
+ *   `node.ports()` returned, or not.
+ * - recover: the world recovers this generation; the host calls `kernel.recover()`.
+ * @typedef {{
+ *   receive?: (envelope: object, from: SimNode) => object,
+ *   send?: (to: string, envelope: object) => Promise<object>,
+ *   recover?: () => void,
+ * }} Ports
+ */
+
+/** One instance: a database that outlives crashes, and the kernel generation running on it. */
+export class SimNode {
+  up = false;
+  /** recover() has run on this generation: until then every letter to it gets not_ready */
+  ready = false;
+  /** @type {Array<unknown>} errors given to onError, other than crashes and failed commits */
+  errors = [];
+  seen = { run: 0, seq: 0 };
+  /** @type {Map<string, object>} the timers in its table at its last commit, by key */
+  timers = new Map();
+  /** the earliest due time this generation's kernel last gave to `wake`, null until it gives one (7) */
+  woke = null;
+  /** @type {Map<string, string[]>} call keys per lane, in outbox order */
+  lanes = new Map();
+  /** the checks' own statements on `raw` */
+  statements = new Map();
+  /** @type {Set<string>} addresses whose handlers the open transaction has called (S9) */
+  called = new Set();
+  /**
+   * @type {Set<string>} what the host asked of its kernel in the open transaction, joined to the world's at its commit.
+   * One asked in a savepoint that rolled back, inside a transaction that committed, still counts.
+   */
+  intents = new Set();
+  /** whatever the host's setup leaves here for its tests to reach; the world does not look at it */
+  host = undefined;
+  /** @type {(() => void) | undefined} what this generation's host does once its recover() has succeeded */
+  afterRecover = undefined;
+  /**
+   * calls of receive() on the host's handle while the world was handing this instance a letter, over all generations:
+   * the world's own call on its way, and every one of the host's, whatever the letter and at whatever depth. The kernel
+   * delivering to itself, as answer() does to a local address, does not go through the handle and does not count.
+   */
+  taken = 0;
+  /** letters put into the network with a generation's post, over all generations */
+  posted = 0;
+  /** the world is handing this instance a letter */
+  delivering = false;
+
+  /**
+   * @param {World} world
+   * @param {string} name
+   * @param {string[]} addresses
+   * @param {Setup} setup
+   * @param {{ open?: () => import('node:sqlite').DatabaseSync }} options open: the host's own database, opened once
+   */
+  constructor(world, name, addresses, setup, { open } = {}) {
+    this.world = world;
+    this.name = name;
+    this.addresses = addresses;
+    this.setup = setup;
+    /** the database itself, which outlives every generation; for checks only */
+    this.raw = open ? open() : new DatabaseSync(':memory:');
+  }
+
+  /**
+   * A new generation on the same database: migrate, let the host set up and, unless told otherwise, recover().
+   * `db` is this generation's connection, the one the host uses.
+   * @param {{ recover?: boolean }} [options]
+   */
+  boot({ recover = true } = {}) {
+    // A process that dies takes its connection's temporary tables, views and triggers with it; this connection outlives
+    // every generation, so the world drops them itself. The first generation too: nothing a host's open() made in temp
+    // is left for it. Triggers go first, as dropping a table drops its own. What SQLite keeps itself, such as the
+    // sqlite_sequence of an AUTOINCREMENT table, cannot be dropped and is no host's. Nor are a virtual table's shadow
+    // tables, which go with it: SQLite itself tells them apart, asking the table's module, in table_list. SQL functions
+    // cannot be taken back: a host registers its own again on every generation.
+    for (const { type, name } of this.q(`SELECT type, name FROM sqlite_temp_master AS m
+      WHERE type IN ('trigger', 'view', 'table') AND name NOT GLOB 'sqlite_*'
+        AND NOT EXISTS (SELECT 1 FROM pragma_table_list AS l WHERE l.schema = 'temp' AND l.name = m.name AND l.type = 'shadow')
+      ORDER BY CASE type WHEN 'trigger' THEN 0 WHEN 'view' THEN 1 ELSE 2 END`).all())
+      this.raw.exec(`DROP ${type.toUpperCase()} temp."${name.replaceAll('"', '""')}"`);
+    // handed: per letter text, how often this generation's kernel handed it over and its host has not posted it yet.
+    const generation = { dead: false, /** @type {Ports} */ ports: {}, handed: new Map(), recovers: 0 };
+    // Bound to this generation, so that a host of a generation that died cannot post through the next one.
+    generation.post = (to, envelope) => post(this, generation, to, envelope);
+    this.generation = generation;
+    this.db = connection(this, this.generation, false);
+    this.woke = null;
+    const kernel = createKernel({
+      db: connection(this, this.generation, true),
+      now: () => this.world.time,
+      send: (to, envelope) => this.world.send(this, to, envelope),
+      onError: error => this.report(error),
+      wake: at => {
+        this.woke = at;
+      },
+      schedule: fn => this.world.tasks.push(fn),
+    });
+    // What the host asks for, once it commits: a run asked to cancel, and a timer gone from its table without coming
+    // due, must be traceable to one of these (C7, 7).
+    for (const name of ['cancel', 'abandon', 'stop', 'unschedule']) {
+      const fn = kernel[name].bind(kernel);
+      kernel[name] = (ref, ...rest) => {
+        this.intents.add(`${name} ${key(ref.from, ref.id)}`);
+        return fn(ref, ...rest);
+      };
+    }
+    // Counted for P-rec.
+    const recovering = kernel.recover.bind(kernel);
+    kernel.recover = () => {
+      generation.recovers += 1;
+      return recovering();
+    };
+    this.wrap(kernel);
+    // The handle the host and the world hold, counted for P-in by where a call of receive() comes from, not by the letter
+    // it carries: one through the handle during a delivery is taken. Every other function is the kernel's own, bound to
+    // it, so the kernel calling its own receive() never comes back through the handle.
+    const receive = envelope => {
+      if (this.delivering) this.taken += 1;
+      return kernel.receive(envelope);
+    };
+    this.kernel = new Proxy(kernel, {
+      get(target, name) {
+        if (name === 'receive') return receive;
+        const value = Reflect.get(target, name);
+        return typeof value === 'function' ? value.bind(target) : value;
+      },
+    });
+    this.afterRecover = undefined;
+    this.up = true;
+    this.ready = false;
+    try {
+      this.kernel.transaction(() => {
+        this.kernel.migrate();
+        this.raw.exec(SIM_TABLES);
+        this.setup({ kernel: this.kernel, node: this, world: this.world });
+      });
+    } catch (error) {
+      if (!lost(error)) throw error;
+      if (this.up) this.down();
+    }
+    if (recover && this.up) this.recover();
+  }
+
+  /**
+   * Every handler the host registers is wrapped: once called, it marks its address as called in this transaction (S9)
+   * and records a row in the world's table, in the handler's own savepoint, so that the row commits exactly when what
+   * the handler did commits. The checks count those rows. An address the host registers anything for in this generation
+   * hears its statuses, results and late outcomes through the world, which records each whether or not the host gave a
+   * function for it. A cancel or answer function is wrapped only when given: without one, the kernel answers otherwise.
+   * The host's functions are called as methods of the object it registered (8).
+   */
+  wrap(kernel) {
+    const record = (address, sql, ...args) => {
+      this.touch(address);
+      this.q(sql).run(...args);
+    };
+    /** @type {Map<string, object | undefined>} per address registered, the result handlers the host gave */
+    const heard = new Map();
+    const results = kernel.results.bind(kernel);
+    const listen = address => {
+      if (heard.has(address)) return;
+      heard.set(address, undefined);
+      results(address, {
+        status: call => (record(address, 'INSERT INTO sim_statuses (from_addr, id, rev) VALUES (?, ?, ?)', call.from, call.id, call.rev),
+          heard.get(address)?.status?.(call)),
+        result: call => (record(address, 'INSERT INTO sim_results (from_addr, id) VALUES (?, ?)', call.from, call.id), heard.get(address)?.result?.(call)),
+        late: call => (record(address, 'INSERT INTO sim_lates (from_addr, id) VALUES (?, ?)', call.from, call.id), heard.get(address)?.late?.(call)),
+      });
+    };
+    kernel.results = (address, handlers) => {
+      listen(address);
+      heard.set(address, handlers);
+    };
+    const handle = kernel.handle.bind(kernel);
+    kernel.handle = (address, op, handlers) => {
+      listen(address);
+      handle(address, op, {
+        accept: handlers.accept && (call => (record(address, 'INSERT INTO sim_accepted (from_addr, id) VALUES (?, ?)', call.from, call.id), handlers.accept(call))),
+        cancel: handlers.cancel && ((run, reason) => (record(address, 'INSERT INTO sim_cancels (from_addr, id) VALUES (?, ?)', run.from, run.id),
+          handlers.cancel(run, reason))),
+        answer: handlers.answer && ((run, given) => (record(address, 'INSERT INTO sim_answers (run_from, run_id, asked, from_addr, id) VALUES (?, ?, ?, ?, ?)',
+          run.from, run.id, given.asked, given.from, given.id), handlers.answer(run, given))),
+      });
+    };
+  }
+
+  recover() {
+    const { generation } = this;
+    try {
+      if (generation.ports.recover) {
+        this.world.count('ports: a recovery through the host');
+        generation.ports.recover();
+        if (generation.recovers !== 1)
+          this.world.violations.push(`P-rec: the host of ${this.name} recovered, its kernel's recover() called ${generation.recovers} times, not once`);
+      } else this.kernel.recover();
+      this.ready = true;
+      this.afterRecover?.();
+    } catch (error) {
+      if (!lost(error)) throw error;
+      if (this.up) this.down();
+    }
+  }
+
+  down() {
+    this.up = false;
+    this.ready = false;
+    this.generation.dead = true;
+    this.kernel.close();
+  }
+
+  /**
+   * The host's part of starting up (10): `fn` runs once this generation's recover() has succeeded, at once and outside
+   * any transaction, where the host scans its runs and calls again (T4). A crash or a failed commit in it is handled as
+   * one in recover(). Called from the setup, once per generation.
+   * @param {() => void} fn
+   */
+  recovered(fn) {
+    this.afterRecover = fn;
+  }
+
+  /**
+   * Puts the host between the world and this generation's kernel (see Ports). Called from the setup, once per
+   * generation; the next generation starts without them.
+   * @param {Ports} ports
+   * @returns {(to: string, envelope: object) => Promise<object>} this generation's post (see post())
+   */
+  ports(ports) {
+    this.generation.ports = ports;
+    return this.generation.post;
+  }
+
+  /**
+   * The host's way to do something later: `fn` runs when the world runs its tasks, unless this generation has died by
+   * then. A crash or a failed commit in it is the world's business, as in act().
+   * @param {() => void} fn
+   */
+  schedule(fn) {
+    const generation = this.generation;
+    this.world.tasks.push(() => {
+      if (generation.dead) return;
+      try {
+        fn();
+      } catch (error) {
+        if (!lost(error)) throw error;
+      }
+    });
+  }
+
+  commit() {
+    const world = this.world;
+    if (world.chance(world.faults.commitFail)) throw new CommitFailed(`a commit of ${this.name} failed`);
+    this.raw.exec('COMMIT');
+    world.commits += 1;
+    for (const intent of this.intents) world.intents.add(intent);
+    world.observe(this);
+    if (world.commits === world.crashAt) {
+      world.crashed += 1;
+      this.down();
+      throw new Crash(`${this.name} crashed after commit ${world.commits}`);
+    }
+  }
+
+  report(error) {
+    if (!lost(error)) this.errors.push(error);
+  }
+
+  // The kernel called a handler of `address`; one transaction may call those of one address only (S9).
+  touch(address) {
+    this.called.add(address);
+    if (this.called.size > 1) this.world.violations.push(`S9: one transaction of ${this.name} called handlers of ${[...this.called].join(', ')}`);
+  }
+
+  migrated() {
+    return this.q("SELECT 1 FROM sqlite_master WHERE name = 'kernel_runs'").get() !== undefined;
+  }
+
+  // The checks' own statements, kept per database; they bypass the generation's connection.
+  q(sql) {
+    let statement = this.statements.get(sql);
+    if (!statement) this.statements.set(sql, statement = this.raw.prepare(sql));
+    return statement;
+  }
+
+  count(sql, ...args) {
+    return this.q(sql).get(...args).n;
+  }
+}
+
+export class World {
+  /** @type {SimNode[]} */
+  nodes = [];
+  /** @type {Map<string, SimNode>} */
+  owners = new Map();
+  /** Pending network events: requests not delivered yet, and receipts not handed back yet. */
+  net = [];
+  /** @type {Array<() => void>} what the kernels gave to `schedule` */
+  tasks = [];
+  time = 0;
+  commits = 0;
+  crashAt = Infinity;
+  crashed = 0;
+  faults = NO_FAULTS;
+  /** Hosts are to stop refusing on purpose: no Busy, no throwing handlers. Set when the world is brought to rest. */
+  calm = false;
+  /** @type {Map<string, number>} call key → the commit that first recorded its run on the callee */
+  order = new Map();
+  /** @type {Map<string, string>} ended rows as last seen (S1) */
+  ended = new Map();
+  /** @type {Map<string, number>} the version of each call's status as last seen */
+  revs = new Map();
+  /** @type {Map<string, { node: SimNode, calls: object[] }>} per run asked to cancel, the calls it awaited at that commit (L3) */
+  halted = new Map();
+  /** @type {Set<string>} the calls those runs awaited then: their own runs may be asked to cancel (C7) */
+  passed = new Set();
+  /** @type {Set<string>} what hosts asked of their kernels in transactions that committed: `<function> <ref>` for cancel, abandon,
+   * stop and unschedule */
+  intents = new Set();
+  /** @type {Map<string, number>} how often each of the rarer paths was taken */
+  tally = new Map();
+  violations = [];
+  /** @type {Array<() => void>} the hosts' own checks after every step, run by check() */
+  steps = [];
+  /** @type {Array<() => void>} the hosts' own checks at rest, run by checkQuiet() */
+  quiets = [];
+
+  /** @param {{ rng?: () => number, restart?: boolean }} [options] restart: a crashed instance comes back at once */
+  constructor({ rng = null, restart = false } = {}) {
+    this.rng = rng;
+    this.restart = restart;
+  }
+
+  chance(p) {
+    return p > 0 && this.rng() < p;
+  }
+
+  count(name) {
+    this.tally.set(name, (this.tally.get(name) ?? 0) + 1);
+  }
+
+  /**
+   * @param {string} name
+   * @param {string[]} addresses
+   * @param {Setup} setup
+   * @param {{ open?: () => import('node:sqlite').DatabaseSync }} [options]
+   */
+  node(name, addresses, setup, options) {
+    const node = new SimNode(this, name, addresses, setup, options);
+    this.nodes.push(node);
+    for (const address of addresses) this.owners.set(address, node);
+    return node;
+  }
+
+  start() {
+    for (const node of this.nodes) node.boot();
+  }
+
+  /** A host's own check, run after the world's by every check(). @param {() => void} fn */
+  onStep(fn) {
+    this.steps.push(fn);
+  }
+
+  /** A host's own check at rest, run after the world's by checkQuiet(). @param {() => void} fn */
+  onQuiet(fn) {
+    this.quiets.push(fn);
+  }
+
+  // After every commit: which runs it recorded, and which lane letters it owed, for S5; which runs it asked to cancel,
+  // with the calls they awaited then, for L3; and where each timer that left its table went (7).
+  observe(node) {
+    for (const row of node.q('SELECT rowid, from_addr, id FROM kernel_runs WHERE rowid > ?').all(node.seen.run)) {
+      node.seen.run = row.rowid;
+      this.order.set(key(row.from_addr, row.id), this.commits);
+    }
+    for (const row of node.q('SELECT seq, lane, kind, call_from, call_id FROM kernel_outbox WHERE seq > ?').all(node.seen.seq)) {
+      node.seen.seq = row.seq;
+      if (row.kind !== 'call' || row.lane === null) continue;
+      if (!node.lanes.has(row.lane)) node.lanes.set(row.lane, []);
+      node.lanes.get(row.lane).push(key(row.call_from, row.call_id));
+    }
+    const newly = new Set();
+    for (const run of node.q('SELECT from_addr, id FROM kernel_runs WHERE cancel_asked = 1').all()) {
+      const name = `${node.name} ${key(run.from_addr, run.id)}`;
+      if (this.halted.has(name)) continue;
+      const calls = node.q(`SELECT from_addr, id, to_addr FROM kernel_calls
+        WHERE by_from = ? AND by_id = ? AND awaited = 1 AND state IN ${UNENDED}`).all(run.from_addr, run.id);
+      this.halted.set(name, { node, calls });
+      for (const call of calls) this.passed.add(key(call.from_addr, call.id));
+      newly.add(key(run.from_addr, run.id));
+    }
+    // A timer leaves its table as a call of its id, once due; taken back by its host; or with its run asked to cancel
+    // in this very commit, which leaves none of the run's older timers behind.
+    const timers = new Map(node.q('SELECT from_addr, id, due, by_from, by_id FROM kernel_timers').all().map(row => [key(row.from_addr, row.id), row]));
+    for (const [name, timer] of node.timers) {
+      const by = timer.by_from === null ? undefined : key(timer.by_from, timer.by_id);
+      if (timers.has(name)) {
+        if (newly.has(by)) this.violations.push(`7: timer ${name} of ${node.name} outlived its run ${by} being asked to cancel`);
+        continue;
+      }
+      const taken = ['unschedule', 'cancel', 'abandon'].filter(how => this.intents.delete(`${how} ${name}`));
+      if (node.q('SELECT 1 FROM kernel_calls WHERE from_addr = ? AND id = ?').get(timer.from_addr, timer.id) && this.time >= timer.due) this.count('timer: came due and became a call');
+      else if (taken.length > 0) this.count(`timer: taken back by ${taken.join(', ')}`);
+      else if (newly.has(by)) this.count('timer: dropped with its run asked to cancel');
+      else this.violations.push(`7: timer ${name} of ${node.name} left its table at commit ${this.commits}: not due and called, not taken back, not dropped with its run`);
+    }
+    node.timers = timers;
+  }
+
+  /**
+   * The `send` every instance gets. It checks S6 at the moment of handing over; an answer is no letter of the outbox.
+   * The letter then goes to the host's send, or straight into the network.
+   */
+  send(node, to, envelope) {
+    const call = envelope.kind === 'call' ? envelope : envelope.call;
+    const text = JSON.stringify(envelope);
+    if (envelope.confirmed) this.count('inquiry sent');
+    const owed = envelope.kind === 'answer' || !node.raw.isTransaction && (envelope.confirmed
+      ? node.q("SELECT 1 FROM kernel_calls WHERE from_addr = ? AND id = ? AND state = 'open' AND to_addr = ? AND digest = ?")
+        .get(call.from, call.id, to, envelope.digest) !== undefined
+      : node.q('SELECT envelope FROM kernel_outbox WHERE call_from = ? AND call_id = ?').all(call.from, call.id).some(row => row.envelope === text));
+    if (!owed) this.violations.push(`S6: ${node.name} sent a letter that is no committed outbox row nor an inquiry about an open call: ${text}`);
+    const { handed, ports } = node.generation;
+    handed.set(text, (handed.get(text) ?? 0) + 1);
+    if (!ports.send) return node.generation.post(to, envelope);
+    this.count('ports: a letter out through the host');
+    return ports.send(to, envelope);
+  }
+
+  /**
+   * Handles one pending network event. A request is delivered now and its receipt queued as a later event; with
+   * faults, the request or the receipt may be lost, and the request may also be queued again as a late duplicate.
+   * @param {number} index
+   * @param {boolean} [faulty]
+   */
+  deliver(index, faulty = false) {
+    const [event] = this.net.splice(index, 1);
+    if (!event.request) {
+      if (event.receipt) event.resolve(event.receipt);
+      else event.reject(new Error('not delivered'));
+      return;
+    }
+    const faults = faulty ? this.faults : NO_FAULTS;
+    if (this.chance(faults.dropRequest)) {
+      this.net.push({ resolve: event.resolve, reject: event.reject });
+      return;
+    }
+    if (this.chance(faults.duplicate)) this.net.push({ ...event, resolve() {}, reject() {} });
+    const envelope = JSON.parse(event.envelope);
+    this.notice('letter', envelope);
+    let receipt = this.receive(event.to, envelope, event.from);
+    if (envelope.kind === 'answer') this.checkAnswer(envelope, receipt);
+    this.notice('receipt', envelope, receipt);
+    if (this.chance(faults.dropReceipt)) receipt = undefined;
+    this.net.push({ receipt, resolve: event.resolve, reject: event.reject });
+  }
+
+  // Counts the rarer paths a letter or its receipt is about to take, as the two sides' tables show them now.
+  notice(moment, envelope, receipt) {
+    const call = envelope.kind === 'call' ? envelope : envelope.call;
+    const row = (address, table) => {
+      const node = this.owners.get(address);
+      return node?.migrated() ? node.q(`SELECT * FROM ${table} WHERE from_addr = ? AND id = ?`).get(call.from, call.id) : undefined;
+    };
+    if (moment === 'letter') {
+      if (envelope.kind === 'answer') this.noticeAnswer(envelope);
+      if (envelope.kind === 'cancel' && !row(envelope.to, 'kernel_runs')) this.count('cancel before its call');
+      if (envelope.kind === 'status' && ENDED.has(row(envelope.to, 'kernel_calls')?.state)) this.count('status after the outcome');
+      if (envelope.kind === 'result' && row(envelope.to, 'kernel_calls')?.abandoned === 1) this.count('outcome letter after an abandon');
+      return;
+    }
+    if (!receipt?.ok || (envelope.kind !== 'call' && envelope.kind !== 'cancel')) {
+      if (envelope.confirmed && receipt?.code === 'unknown_call') this.count('inquiry: no record');
+      return;
+    }
+    const mine = row(envelope.from, 'kernel_calls');
+    if (receipt.view.outcome && mine?.abandoned === 1) this.count('outcome receipt after an abandon');
+    if (!envelope.confirmed) return;
+    if (receipt.view.outcome) this.count('inquiry: an outcome');
+    else if (receipt.view.rev > (mine?.rev ?? 0)) this.count('inquiry: a newer status');
+  }
+
+  // Counts the paths an answer is about to take that its receipt does not tell apart.
+  noticeAnswer(envelope) {
+    const node = this.owners.get(envelope.to);
+    if (!node?.up || !node.migrated()) return;
+    if (node.q('SELECT 1 FROM kernel_answers WHERE from_addr = ? AND id = ?').get(envelope.from, envelope.id)) this.count('answer: its number was taken before');
+    else if (node.q('SELECT 1 FROM kernel_runs WHERE from_addr = ? AND id = ? AND to_addr = ? AND status = ? AND asked = ?')
+      .get(envelope.call.from, envelope.call.id, envelope.to, 'input-required', envelope.asked)) this.count('answer: to its question');
+  }
+
+  /**
+   * `node` answers a run (C8). No instance keeps an answer: what comes back is the receipt, or undefined when the
+   * answer or its receipt was lost, or the answerer crashed.
+   * @param {SimNode} node
+   * @param {{ from: string, id: string, to: string, call: { from: string, id: string }, asked: number, answer: { mode: string } }} answer
+   * @returns {Promise<object | undefined>}
+   */
+  answer(node, answer) {
+    if (!node.up && this.restart) node.boot();
+    if (!node.up) return Promise.resolve(undefined);
+    const local = node.addresses.includes(answer.to);
+    this.count(local ? 'answer: to a local address' : 'answer: to another instance');
+    if (local) this.noticeAnswer({ ...answer, kind: 'answer' });
+    return node.kernel.answer(answer).then(receipt => {
+      if (local) this.checkAnswer(answer, receipt);
+      this.count(`answer receipt: ${receipt.ok ? (receipt.applied ? 'applied' : 'not applied') : receipt.code}`);
+      return receipt;
+    }, error => {
+      if (!lost(error) && error.message !== 'not delivered') throw error;
+      return undefined;
+    });
+  }
+
+  // A receipt that takes an answer says it was applied exactly when that answer's number has been taken, now or before
+  // (C8). The world's table says which numbers were taken: it records each answer function's call.
+  checkAnswer(answer, receipt) {
+    if (!receipt?.ok) return;
+    const taken = this.owners.get(answer.to).q('SELECT 1 FROM sim_answers WHERE from_addr = ? AND id = ?').get(answer.from, answer.id) !== undefined;
+    if (receipt.applied !== taken)
+      this.violations.push(`C8: an answer ${key(answer.from, answer.id)} got applied: ${receipt.applied}, but its number was ${taken ? '' : 'not '}taken`);
+  }
+
+  /**
+   * Hands a letter to the instance that owns `to`, through its host's receive when it gave one. The receipt comes back
+   * as it would over a wire; undefined when that instance is down or crashed meanwhile.
+   * @param {string} to
+   * @param {object} envelope
+   * @param {SimNode} [from] the instance that put the letter into the network; called directly, the owner of its `from`
+   */
+  receive(to, envelope, from = this.owners.get(envelope.from)) {
+    const node = this.owners.get(to);
+    if (!node?.up) return undefined;
+    const { ports } = node.generation;
+    const taken = node.taken;
+    node.delivering = true;
+    try {
+      if (!ports.receive) return JSON.parse(JSON.stringify(node.kernel.receive(envelope)));
+      this.count('ports: a letter in through the host');
+      return JSON.parse(JSON.stringify(ports.receive(envelope, from)));
+    } catch (error) {
+      if (!lost(error)) throw error;
+      return undefined;
+    } finally {
+      node.delivering = false;
+      if (node.taken - taken > 1)
+        this.violations.push(`P-in: the host of ${node.name} handed one ${envelope.kind} letter to its kernel ${node.taken - taken} times`);
+    }
+  }
+
+  /** The host of `node` runs `fn` in a transaction; a crash or a failed commit is the world's business, not an error. */
+  act(node, fn) {
+    if (!node.up && this.restart) node.boot();
+    if (!node.up) return undefined;
+    try {
+      return node.kernel.transaction(() => fn(node.kernel));
+    } catch (error) {
+      if (!lost(error)) throw error;
+      return undefined;
+    }
+  }
+
+  pump(node) {
+    if (!node.up) return;
+    try {
+      node.kernel.pump();
+    } catch (error) {
+      if (!lost(error)) throw error;
+    }
+  }
+
+  /** The host of `node` ticks, as its timer would make it (7). */
+  tick(node) {
+    if (!node.up && this.restart) node.boot();
+    if (!node.up) return;
+    try {
+      node.kernel.tick();
+    } catch (error) {
+      if (!lost(error)) throw error;
+    }
+  }
+
+  /** @param {SimNode} node @param {{ from: string, id: string }} call */
+  inquire(node, call) {
+    if (!node.up) return;
+    try {
+      node.kernel.inquire(call);
+    } catch (error) {
+      if (!lost(error)) throw error;
+    }
+  }
+
+  /** Runs everything pending, without faults, until nothing is left to do. */
+  async drain() {
+    for (;;) {
+      await flush();
+      const down = this.restart ? this.nodes.filter(node => !node.up) : [];
+      for (const node of down) node.boot();
+      if (down.length > 0) continue;
+      if (this.tasks.length > 0) this.tasks.shift()();
+      else if (this.net.length > 0) this.deliver(0);
+      else return;
+    }
+  }
+
+  /**
+   * Brings the world to rest as section 12 defines it: no more faults, everyone up and recovered, everyone pumped,
+   * idle() done, and the network and `schedule` queues empty. It first rests with what the hosts hold still working and
+   * checks their statuses there; then it rests again, calling `advance` every round, until a round finds nothing to
+   * advance. Otherwise it looks at none of the kernel's tables; what they hold at rest is for checkQuiet() to judge.
+   * @param {() => boolean} [advance] the hosts move on what they hold by a step; true while there was something to move
+   * @param {{ resend?: () => void }} [options] resend: how the hosts have every instance send what it keeps again, once a
+   *   round; without it the world pumps every instance itself, which would make up for a host that never does
+   */
+  async quiesce(advance = () => false, { resend } = {}) {
+    this.calm = true;
+    // Letters already in flight may still come back refused by faults chosen earlier; the pumps that count come after.
+    await this.drain();
+    await this.rest(() => false, resend);
+    this.checkStatus();
+    this.checkHalted();
+    await this.rest(advance, resend);
+  }
+
+  /** @param {() => boolean} advance @param {(() => void) | undefined} resend */
+  async rest(advance, resend) {
+    for (let round = 0; round < 50; round++) {
+      const crashed = this.crashed;
+      for (const node of this.nodes) {
+        if (!node.up) node.boot();
+        else if (!node.ready) node.recover();
+      }
+      const advanced = advance();
+      if (resend) resend();
+      else for (const node of this.nodes) this.pump(node);
+      // The clock passes every timer, and every instance ticks.
+      this.time = Math.max(this.time, ...this.nodes.filter(node => node.migrated()).map(node => node.q('SELECT max(due) AS due FROM kernel_timers').get().due ?? 0));
+      for (const node of this.nodes) this.tick(node);
+      await this.drain();
+      await Promise.all(this.nodes.filter(node => node.up).map(node => node.kernel.idle()));
+      if (this.crashed === crashed && !advanced && this.net.length === 0 && this.tasks.length === 0
+        && this.nodes.every(node => node.ready && node.count('SELECT count(*) AS n FROM kernel_timers') === 0)) return;
+    }
+    throw new Error(`the world did not come to rest: ${this.nodes.map(node => `${node.name} has ${node.count('SELECT count(*) AS n FROM kernel_outbox')} `
+      + `letters in its outbox and ${node.count(`SELECT count(*) AS n FROM kernel_calls WHERE state IN ${UNENDED}`)} calls not ended`).join(', ')}`);
+  }
+
+  /** At rest, before the held runs finish: a call that has not ended knows the status its run has now (C6). */
+  checkStatus() {
+    for (const node of this.nodes)
+      for (const call of node.q(`SELECT * FROM kernel_calls WHERE state IN ${UNENDED}`).all()) {
+        const name = key(call.from_addr, call.id);
+        const run = this.owners.get(call.to_addr).q('SELECT * FROM kernel_runs WHERE from_addr = ? AND id = ?').get(call.from_addr, call.id);
+        assert.ok(run, `call ${name} is ${call.state} at rest but its callee has no run`);
+        assert.equal(statusOf(call), statusOf(run), `the caller of ${name} does not have the status its run has at rest`);
+      }
+  }
+
+  /**
+   * L3, at rest: each call that a run awaited when it was asked to cancel has ended, or its callee has its run marked as
+   * asked to cancel.
+   */
+  checkHalted() {
+    for (const [name, { node, calls }] of this.halted)
+      for (const call of calls) {
+        if (ENDED.has(node.q('SELECT state FROM kernel_calls WHERE from_addr = ? AND id = ?').get(call.from_addr, call.id).state)) continue;
+        const run = this.owners.get(call.to_addr).q('SELECT cancel_asked FROM kernel_runs WHERE from_addr = ? AND id = ?').get(call.from_addr, call.id);
+        assert.equal(run?.cancel_asked, 1, `L3: run ${name} was asked to cancel, but the callee of ${key(call.from_addr, call.id)}, which it awaited, was not`);
+      }
+  }
+
+  /**
+   * The invariants that hold after every step: S1 to S10; that a request id names a timer or a call, not both; that
+   * a recovered instance last told its host of its earliest timer; that every timer left its table for a reason (see
+   * observe()); that a cancel function commits at most once per run,
+   * and a status handler at most once per version; that a caller's version of a status never goes back; that every run
+   * asked to cancel was asked by someone entitled to; and that every cancel owed is about a call that has not ended, or
+   * was abandoned. Then the hosts' own.
+   */
+  check() {
+    assert.deepEqual(this.violations, []);
+    for (const node of this.nodes) {
+      assert.ok(!node.raw.isTransaction, `${node.name} is inside a transaction between steps`);
+      if (!node.migrated()) continue;
+      const now = new Map();
+      for (const row of node.q('SELECT * FROM kernel_calls').all())
+        if (ENDED.has(row.state)) now.set(`${node.name} call ${key(row.from_addr, row.id)}`, ending(row, 'state'));
+      for (const row of node.q('SELECT * FROM kernel_runs').all())
+        if (ENDED.has(row.status)) now.set(`${node.name} run ${key(row.from_addr, row.id)}`, ending(row, 'status'));
+      for (const [name, text] of this.ended)
+        if (name.startsWith(`${node.name} `)) assert.equal(now.get(name), text, `S1: the outcome of ${name} changed`);
+      for (const [name, text] of now) this.ended.set(name, text);
+
+      for (const [table, what] of [['sim_accepted', 'S2: accept'], ['sim_results', 'S3: result'], ['sim_lates', 'S3: late'],
+        ['sim_cancels', 'the cancel function']]) {
+        const twice = node.q(`SELECT from_addr, id, count(*) AS n FROM ${table} GROUP BY from_addr, id HAVING n > 1`).all();
+        assert.deepEqual(twice, [], `${what} committed more than once on ${node.name}`);
+      }
+      const statuses = node.q('SELECT from_addr, id, rev, count(*) AS n FROM sim_statuses GROUP BY from_addr, id, rev HAVING n > 1').all();
+      assert.deepEqual(statuses, [], `the status handler committed one version twice on ${node.name}`);
+      for (const [columns, what] of [['run_from, run_id, asked', 'one question of a run took two answers'], ['from_addr, id', 'one answer was taken twice']]) {
+        const twice = node.q(`SELECT ${columns}, count(*) AS n FROM sim_answers GROUP BY ${columns} HAVING n > 1`).all();
+        assert.deepEqual(twice, [], `S8: ${what} on ${node.name}`);
+      }
+
+      // An outcome from the callee, or an abandoned call's late one, is the outcome of its run.
+      for (const call of node.q('SELECT * FROM kernel_calls').all()) {
+        if (!ENDED.has(call.state) || (call.local === 1 && call.late === null)) continue;
+        const run = this.owners.get(call.to_addr).q('SELECT * FROM kernel_runs WHERE from_addr = ? AND id = ?').get(call.from_addr, call.id);
+        assert.ok(run, `S4: call ${key(call.from_addr, call.id)} ended but its callee has no run`);
+        assert.equal(call.late === null ? ending(call, 'state') : lateEnding(call), ending(run, 'status'), `S4: call ${key(call.from_addr, call.id)} ended unlike its run`);
+      }
+
+      const orphans = node.q(`SELECT r.from_addr, r.id FROM kernel_runs AS r JOIN kernel_calls AS c ON c.by_from = r.from_addr AND c.by_id = r.id
+        WHERE r.status IN ('completed', 'failed', 'canceled') AND c.awaited = 1 AND c.state IN ${UNENDED}
+        UNION ALL SELECT r.from_addr, r.id FROM kernel_runs AS r JOIN kernel_timers AS t ON t.by_from = r.from_addr AND t.by_id = r.id
+        WHERE r.status IN ('completed', 'failed', 'canceled') AND t.awaited = 1`).all();
+      assert.deepEqual(orphans.map(run => key(run.from_addr, run.id)), [], `S7: runs of ${node.name} ended while awaiting calls or timers they made`);
+
+      const both = node.q('SELECT t.from_addr, t.id FROM kernel_timers AS t JOIN kernel_calls AS c ON c.from_addr = t.from_addr AND c.id = t.id').all();
+      assert.deepEqual(both.map(row => key(row.from_addr, row.id)), [], `7: request ids of ${node.name} name both a timer and a call`);
+      if (node.ready) assert.equal(node.woke, node.q('SELECT min(due) AS due FROM kernel_timers').get().due,
+        `7: the wake-up ${node.name} last gave its host is not its earliest timer`);
+
+      const silent = node.q(`SELECT from_addr, id FROM kernel_calls AS c WHERE state = 'sending' AND NOT EXISTS
+        (SELECT 1 FROM kernel_outbox AS o WHERE o.call_from = c.from_addr AND o.call_id = c.id AND o.kind IN ('call', 'cancel'))`).all();
+      assert.deepEqual(silent.map(call => key(call.from_addr, call.id)), [], `S10: sending calls of ${node.name} have no letter in its outbox`);
+
+      for (const letter of node.q(`SELECT o.call_from, o.call_id, c.state, c.abandoned FROM kernel_outbox AS o
+        LEFT JOIN kernel_calls AS c ON c.from_addr = o.call_from AND c.id = o.call_id WHERE o.kind = 'cancel'`).all())
+        assert.ok(letter.state !== null && (!ENDED.has(letter.state) || letter.abandoned === 1),
+          `${node.name} owes a cancel for ${key(letter.call_from, letter.call_id)}, which is ${letter.state ?? 'no call of its own'}`);
+
+      // A run is asked to cancel by its caller, by its own host, or because a run asked to cancel awaited its call (C7).
+      for (const run of node.q('SELECT from_addr, id FROM kernel_runs WHERE cancel_asked = 1').all()) {
+        const name = key(run.from_addr, run.id);
+        assert.ok(['cancel', 'abandon', 'stop'].some(how => this.intents.has(`${how} ${name}`)) || this.passed.has(name),
+          `run ${name} of ${node.name} was asked to cancel, but no host asked for it and no run asked to cancel awaited it`);
+      }
+
+      for (const call of node.q('SELECT from_addr, id, rev FROM kernel_calls').all()) {
+        const name = `${node.name} ${key(call.from_addr, call.id)}`;
+        assert.ok(call.rev >= (this.revs.get(name) ?? 0), `the status version of call ${name} went back`);
+        this.revs.set(name, call.rev);
+      }
+
+      for (const [lane, keys] of node.lanes) {
+        let last = -Infinity;
+        for (const name of keys) {
+          const at = this.order.get(name);
+          if (at === undefined) continue;
+          assert.ok(at > last, `S5: in lane ${lane} of ${node.name}, ${name} was recorded before an earlier call of the lane`);
+          last = at;
+        }
+      }
+    }
+    for (const fn of this.steps) fn();
+  }
+
+  /**
+   * The invariants of a world at rest: L1, no result still owed, L2, L4, and that every call and run has ended. L3 is
+   * quiesce()'s.
+   */
+  checkQuiet() {
+    for (const node of this.nodes) {
+      assert.equal(node.count('SELECT count(*) AS n FROM kernel_outbox'), 0, `L1: the outbox of ${node.name} is not empty`);
+      // With the clock past them all, every timer has become a call; the call has ended like any other, below.
+      assert.equal(node.count('SELECT count(*) AS n FROM kernel_timers'), 0, `L4: timers of ${node.name} are left at rest`);
+      assert.equal(node.count('SELECT count(*) AS n FROM kernel_calls WHERE owed = 1'), 0, `results of ${node.name} are still owed at rest`);
+      for (const call of node.q('SELECT * FROM kernel_calls').all())
+        assert.ok(ENDED.has(call.state), `call ${key(call.from_addr, call.id)} is still ${call.state} at rest`);
+      for (const run of node.q('SELECT * FROM kernel_runs').all()) {
+        const name = key(run.from_addr, run.id);
+        assert.ok(ENDED.has(run.status), `run ${name} is still ${run.status} at rest`);
+        if (run.root === 1) continue;
+        const caller = this.owners.get(run.from_addr);
+        const call = caller.q('SELECT * FROM kernel_calls WHERE from_addr = ? AND id = ?').get(run.from_addr, run.id);
+        // An abandoned call has its own outcome; the run's comes as the late one, to `late` instead of `result`.
+        const abandoned = call?.abandoned === 1;
+        assert.equal(call && (abandoned ? call.late && lateEnding(call) : ending(call, 'state')), ending(run, 'status'),
+          `L2: the caller of ${name} does not have its ${abandoned ? 'late ' : ''}outcome`);
+        for (const [table, what] of [['sim_results', 'result'], ['sim_lates', 'late']])
+          assert.equal(caller.count(`SELECT count(*) AS n FROM ${table} WHERE from_addr = ? AND id = ?`, run.from_addr, run.id),
+            (what === 'late') === abandoned ? 1 : 0, `L2: ${what} of ${name} was not committed exactly as often as it should`);
+      }
+    }
+    for (const fn of this.quiets) fn();
+  }
+}