@coda-console/kernel 0.0.0 → 0.1.0

package/sim/scenarios.mjs

@@ -0,0 +1,484 @@
+import assert from 'node:assert/strict';
+import { advance, CHANGES, held, host } from './host.mjs';
+import { flush, NO_FAULTS, random, World } from './world.mjs';
+
+const ENDED = new Set(['completed', 'failed', 'canceled']);
+
+// Small scenarios, each run once without a crash and then once per commit with a crash right after it (12. 怎么测).
+// The world restarts a crashed instance at once, so a scenario is the same script whatever crashed.
+export const SMALL = {
+  // Calls between two addresses of one instance: every letter goes through local delivery. A cancel function there
+  // ends its run at once, so the cancel's own receipt carries the outcome.
+  async local(world) {
+    const a = world.node('A', ['a1', 'a2'], host);
+    world.start();
+    world.act(a, kernel => kernel.call({ from: 'a1', id: 'l1', to: 'a2', op: 'work', input: { mode: 'complete', n: 1 }, lane: 'x' }));
+    world.act(a, kernel => kernel.call({ from: 'a1', id: 'l2', to: 'a2', op: 'work', input: { mode: 'hold', goal: 'failed' }, lane: 'x',
+      memo: { resultThrows: 1 } }));
+    await world.drain();
+    world.check();
+    world.act(a, kernel => kernel.call({ from: 'a2', id: 'l3', to: 'a1', op: 'nope' }));
+    await world.drain();
+    world.check();
+    world.inquire(a, { from: 'a1', id: 'l2' });
+    world.check();
+    world.act(a, kernel => kernel.call({ from: 'a1', id: 'l4', to: 'a2', op: 'work', input: { mode: 'hold', cancelSettles: true } }));
+    await world.drain();
+    world.act(a, kernel => kernel.cancel({ from: 'a1', id: 'l4' }));
+    await world.drain();
+    world.check();
+  },
+
+  // One lane to another instance, behind a busy callee, and a host that issues the same call again after a restart.
+  async lane(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1'], host);
+    world.start();
+    const first = { from: 'a1', id: 'r1', to: 'b1', op: 'work', input: { mode: 'busy', n: 1 }, lane: 'x' };
+    world.act(a, kernel => {
+      kernel.call(first);
+      kernel.call({ from: 'a1', id: 'r2', to: 'b1', op: 'work', input: { mode: 'hold', goal: 'completed' }, lane: 'x' });
+    });
+    world.act(a, kernel => kernel.call({ from: 'a1', id: 'r3', to: 'b1', op: 'work', input: { mode: 'fail' }, lane: 'x',
+      memo: { resultThrows: 2 } }));
+    await world.drain();
+    world.check();
+    world.act(a, kernel => kernel.call(first));
+    world.pump(a);
+    await world.drain();
+    world.check();
+    world.act(b, kernel => {
+      for (const row of held(b)) b.host.finish(kernel, row);
+    });
+    await world.drain();
+    world.check();
+  },
+
+  // Calls both ways between two instances, with the ways a callee can refuse.
+  async across(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1'], host);
+    world.start();
+    world.act(a, kernel => kernel.call({ from: 'a1', id: 'x1', to: 'b1', op: 'work', input: { mode: 'reject' } }));
+    world.act(b, kernel => kernel.call({ from: 'b1', id: 'x2', to: 'a1', op: 'work', input: { mode: 'throw' }, lane: 'y' }));
+    world.act(b, kernel => kernel.call({ from: 'b1', id: 'x3', to: 'a1', op: 'work', input: { mode: 'hold', goal: 'canceled' }, lane: 'y' }));
+    await world.drain();
+    world.check();
+  },
+
+  // A run reports its status as it goes, makes calls it awaits and one it does not, waits for one and stops waiting for
+  // the other, and ends with its last progress. Its caller's status handler throws while the world is not calm.
+  async tree(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1', 'b2'], host);
+    world.start();
+    const kids = [
+      { to: 'a1', mode: 'hold', goal: 'completed', awaited: true },
+      { to: 'b2', mode: 'hold', goal: 'canceled', awaited: true },
+      { to: 'b2', mode: 'complete', awaited: false },
+    ];
+    world.act(a, kernel => kernel.call({ from: 'a1', id: 't1', to: 'b1', op: 'work', input: { mode: 'parent', goal: 'failed', kids },
+      memo: { statusThrows: true } }));
+    await world.drain();
+    world.check();
+    for (const change of [CHANGES[1], CHANGES[2], CHANGES[4], CHANGES[3]]) {
+      for (const node of [a, b]) for (const row of held(node)) world.act(node, kernel => node.host.nudge(kernel, row, change));
+      await world.drain();
+      world.check();
+    }
+    const parent = () => held(b).filter(row => row.id === 't1');
+    for (const row of parent()) world.act(b, kernel => b.host.finish(kernel, row));
+    for (const row of held(a)) world.act(a, kernel => a.host.finish(kernel, row));
+    await world.drain();
+    world.check();
+    for (const row of parent()) world.act(b, kernel => b.host.finish(kernel, row, { detach: true }));
+    await world.drain();
+    world.check();
+  },
+
+  // A root run reports its status to no one, makes calls it awaits, and ends once they have.
+  async root(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1'], host);
+    world.start();
+    world.act(a, kernel => a.host.root(kernel, { address: 'a1', id: 'r1', goal: 'canceled', kids: [
+      { to: 'b1', mode: 'hold', goal: 'completed', awaited: true },
+      { to: 'a1', mode: 'complete', awaited: true },
+    ] }));
+    await world.drain();
+    world.check();
+    for (const node of [a, b]) for (const row of held(node)) world.act(node, kernel => node.host.nudge(kernel, row, CHANGES[2]));
+    await world.drain();
+    world.check();
+    for (const node of [a, b]) for (const row of held(node)) world.act(node, kernel => node.host.finish(kernel, row));
+    await world.drain();
+    world.check();
+  },
+
+  // Cancels at each point of a call's life: while sending, with another call behind it in its lane; in flight, with the
+  // cancel delivered first; open, where the cancel function throws; and ended, where it does nothing.
+  async cancel(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1'], host);
+    world.start();
+    const work = (id, input, more = {}) => ({ from: 'a1', id, to: 'b1', op: 'work', input, ...more });
+    world.act(a, kernel => {
+      kernel.call(work('s1', { mode: 'hold', goal: 'completed' }, { lane: 'x' }));
+      kernel.call(work('s2', { mode: 'hold', goal: 'completed' }, { lane: 'x' }));
+    });
+    world.act(a, kernel => kernel.cancel({ from: 'a1', id: 's1' }, 'no longer needed'));
+    await world.drain();
+    world.check();
+    world.act(a, kernel => kernel.call(work('f1', { mode: 'hold', goal: 'completed' })));
+    await flush();
+    world.act(a, kernel => kernel.cancel({ from: 'a1', id: 'f1' }));
+    await flush();
+    const overtaking = world.net.findIndex(event => event.request && JSON.parse(event.envelope).kind === 'cancel');
+    if (overtaking >= 0) world.deliver(overtaking);
+    await world.drain();
+    world.check();
+    world.act(a, kernel => {
+      kernel.call(work('o1', { mode: 'hold', goal: 'completed', cancelThrows: true }));
+      kernel.call(work('e1', { mode: 'complete' }));
+    });
+    await world.drain();
+    world.act(a, kernel => {
+      kernel.cancel({ from: 'a1', id: 'o1' });
+      kernel.cancel({ from: 'a1', id: 'e1' });
+    });
+    await world.drain();
+    world.check();
+  },
+
+  // A cancel goes down a call tree three levels deep, through a cancel function that throws; the run asked to cancel
+  // makes one more call, which it cannot await. A host stops its root run, and the call that run awaits is canceled.
+  async halt(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1', 'b2'], host);
+    world.start();
+    const kids = [{ to: 'a1', mode: 'parent', goal: 'canceled', awaited: true, cancelThrows: true,
+      kids: [{ to: 'b2', mode: 'hold', goal: 'completed', awaited: true }] }];
+    world.act(a, kernel => kernel.call({ from: 'a1', id: 'h1', to: 'b1', op: 'work', input: { mode: 'parent', goal: 'completed', kids } }));
+    world.act(a, kernel => a.host.root(kernel, { address: 'a1', id: 'r1', goal: 'canceled', kids: [{ to: 'b1', mode: 'hold', goal: 'completed', awaited: true }] }));
+    await world.drain();
+    world.check();
+    world.act(a, kernel => {
+      kernel.cancel({ from: 'a1', id: 'h1' }, 'stop it all');
+      kernel.stop({ from: 'a1', id: 'r1' });
+    });
+    await world.drain();
+    world.check();
+    world.act(b, kernel => kernel.call({ from: 'b1', id: 'after', to: 'a1', op: 'work', input: { mode: 'complete' }, by: { from: 'a1', id: 'h1' }, awaited: true }));
+    await world.drain();
+    world.check();
+  },
+
+  // A caller abandons an open call and one whose call letter is on its way, with another call behind it in its lane.
+  // The real outcomes come late: by result letter, through a late handler that throws once, and by receipt.
+  async abandon(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1'], host);
+    world.start();
+    const work = (id, input, more = {}) => ({ from: 'a1', id, to: 'b1', op: 'work', input, ...more });
+    world.act(a, kernel => kernel.call(work('g1', { mode: 'hold', goal: 'completed' }, { memo: { resultThrows: 1 } })));
+    await world.drain();
+    world.act(a, kernel => {
+      kernel.call(work('g2', { mode: 'complete', n: 2 }, { lane: 'x' }));
+      kernel.call(work('g3', { mode: 'complete', n: 3 }, { lane: 'x' }));
+    });
+    await flush();
+    world.act(a, kernel => {
+      kernel.abandon({ from: 'a1', id: 'g1' });
+      kernel.abandon({ from: 'a1', id: 'g2' });
+    });
+    await world.drain();
+    world.check();
+    world.act(b, kernel => {
+      for (const row of held(b)) b.host.finish(kernel, row);
+    });
+    await world.drain();
+    world.check();
+  },
+
+  // Inquiries find what letters lost on the way would have said: a newer status, and an outcome. Every crash restarts
+  // an instance, whose recover() inquires about its open calls.
+  async inquire(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1'], host);
+    world.start();
+    world.act(a, kernel => {
+      kernel.call({ from: 'a1', id: 'q1', to: 'b1', op: 'work', input: { mode: 'hold', goal: 'completed' } });
+      kernel.call({ from: 'a1', id: 'q2', to: 'b1', op: 'work', input: { mode: 'hold', goal: 'completed' } });
+    });
+    await world.drain();
+    world.act(b, kernel => kernel.update({ from: 'a1', id: 'q1' }, { progress: { step: 1 } }));
+    for (const row of held(b)) if (row.id === 'q2') world.act(b, kernel => b.host.finish(kernel, row));
+    await flush();
+    // The letters are lost: they wait in B's outbox for its next pump().
+    world.net = world.net.map(event => event.request ? { resolve: event.resolve, reject: event.reject } : event);
+    await world.drain();
+    world.inquire(a, { from: 'a1', id: 'q1' });
+    world.inquire(a, { from: 'a1', id: 'q2' });
+    await world.drain();
+    world.check();
+  },
+
+  // Answers to a run waiting for input, from another instance and from the run's own: one taken that asks again, the
+  // same number again, a stale question, a refusal, a handler that throws, one taken that ends the run, and answers to
+  // a run that has ended, a run that never was, and the wrong address. An answer lost in a crash is not sent again.
+  async answer(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1', 'b2'], host);
+    world.start();
+    const run = { from: 'a1', id: 'q1' };
+    world.act(a, kernel => kernel.call({ ...run, to: 'b1', op: 'work', input: { mode: 'hold', goal: 'completed' } }));
+    await world.drain();
+    world.act(b, kernel => kernel.update(run, { status: 'input-required', asking: { question: 1 } }));
+    await world.drain();
+    world.check();
+    const asked = () => b.q('SELECT asked FROM kernel_runs WHERE from_addr = ? AND id = ?').get(run.from, run.id)?.asked ?? 0;
+    const answer = (from, id, mode, more = {}) => ({ from, id, to: 'b1', call: run, asked: asked(), answer: { mode }, ...more });
+    const first = answer('a1', 'n1', 'ask');
+    for (const sent of [first, first, answer('a1', 'n2', 'take', { asked: asked() - 1 })]) {
+      const receipt = world.answer(a, sent);
+      await world.drain();
+      await receipt;
+      world.check();
+    }
+    for (const mode of ['refuse', 'throw', 'settle']) {
+      await world.answer(b, answer('b2', `n-${mode}`, mode));
+      world.check();
+    }
+    for (const sent of [answer('a1', 'n3', 'take'), answer('a1', 'n4', 'take', { call: { from: 'a1', id: 'none' } }), answer('a1', 'n5', 'take', { to: 'b2' })]) {
+      const receipt = world.answer(a, sent);
+      await world.drain();
+      await receipt;
+    }
+    await world.drain();
+    world.check();
+  },
+
+  // Timers: two come due at once and become calls in the order they were registered; one is unscheduled, one
+  // canceled, one abandoned and its id used again for another; a root run cannot end while it awaits a timer, until it
+  // detaches it; a run stopped loses its timers, due or not.
+  async timer(world) {
+    const a = world.node('A', ['a1'], host);
+    const b = world.node('B', ['b1'], host);
+    world.start();
+    const timer = (id, delayMs, more = {}) => ({ from: 'a1', id, delayMs, to: 'b1', op: 'work', input: { mode: 'complete', n: id }, lane: 'x', ...more });
+    world.act(a, kernel => {
+      for (const [id, delayMs] of [['t1', 10], ['t2', 10], ['t3', 20], ['t4', 5], ['t5', 5]]) kernel.after(timer(id, delayMs));
+      a.host.root(kernel, { address: 'a1', id: 'r1', goal: 'completed', kids: [{ to: 'b1', mode: 'complete', awaited: true, delayMs: 30 }] });
+      a.host.root(kernel, { address: 'a1', id: 'r2', goal: 'canceled', kids: [{ to: 'b1', mode: 'complete', awaited: true, delayMs: 0 },
+        { to: 'b1', mode: 'hold', awaited: false, delayMs: 40 }] });
+    });
+    world.check();
+    world.act(a, kernel => {
+      kernel.unschedule({ from: 'a1', id: 't3' });
+      kernel.cancel({ from: 'a1', id: 't4' });
+      kernel.abandon({ from: 'a1', id: 't5' });
+      kernel.after(timer('t5', 1, { input: { mode: 'hold', goal: 'completed' } }));
+      kernel.stop({ from: 'a1', id: 'r2' });
+    });
+    world.time = 10;
+    world.tick(a);
+    await world.drain();
+    world.check();
+    const r1 = () => held(a).filter(row => row.id === 'r1');
+    for (const row of r1()) world.act(a, kernel => a.host.finish(kernel, row));
+    world.act(a, kernel => kernel.detach({ from: 'a1', id: 'a1:r1>0' }));
+    for (const row of r1()) world.act(a, kernel => a.host.finish(kernel, row));
+    world.time = 30;
+    world.tick(a);
+    await world.drain();
+    world.check();
+  },
+};
+
+/** Every error a world's hosts were told of is one its handlers threw on purpose, or the C9 refusal of a hasty accept. */
+export function checkErrors(world) {
+  for (const node of world.nodes)
+    for (const error of node.errors)
+      assert.match(String(error?.message), /^((|result |status |cancel |late |answer )handler blew up|run \S+ still awaits a call it made)$/,
+        `unexpected error on ${node.name}`);
+}
+
+/**
+ * Runs a small scenario to rest and checks it.
+ * @param {(world: World) => Promise<void>} scenario
+ * @param {number} crashAt the commit after which an instance crashes, or Infinity
+ */
+export async function play(scenario, crashAt) {
+  const world = new World({ restart: true });
+  world.crashAt = crashAt;
+  await scenario(world);
+  await world.quiesce(() => advance(world));
+  world.check();
+  world.checkQuiet();
+  checkErrors(world);
+  return world;
+}
+
+const MODES = ['complete', 'complete', 'fail', 'hold', 'reject', 'busy', 'throw', 'parent', 'hold', 'hasty'];
+const KID_MODES = ['complete', 'fail', 'hold', 'reject', 'busy', 'throw'];
+const GOALS = ['completed', 'failed', 'canceled'];
+const LANES = [null, 'x', 'y'];
+const ANSWERS = ['take', 'take', 'take', 'ask', 'settle', 'refuse', 'throw'];
+const DELAYS = [0, 1, 5, 10, 30];
+
+/**
+ * A long scenario chosen by `seed`: three instances calling each other under every fault of the model, checked after
+ * every step and at rest.
+ * @param {number} seed
+ * @param {number} steps
+ */
+export async function wander(seed, steps) {
+  const rng = random(seed);
+  const pick = list => list[Math.floor(rng() * list.length)];
+  const world = new World({ rng });
+  world.faults = { dropRequest: 0.1, dropReceipt: 0.1, duplicate: 0.1, commitFail: 0.03 };
+  const nodes = [world.node('A', ['a1', 'a2'], host), world.node('B', ['b1'], host), world.node('C', ['c1'], host)];
+  const addresses = nodes.flatMap(node => node.addresses);
+  world.start();
+  const issued = [];
+  let roots = 0;
+  let later = 0;
+  /** every answer sent, those whose receipt never came, and those taken: an answerer may send one again, with its number (C8) */
+  const answers = [];
+  const unanswered = [];
+  const taken = [];
+  const answer = (node, sent) => world.answer(node, sent).then(receipt => {
+    if (!receipt) unanswered.push({ node, sent });
+    else if (receipt.applied) taken.push({ node, sent });
+  });
+  // Now and then a kid is a parent of its own, and the call tree is three levels deep.
+  const kids = (depth = 1) => Array.from({ length: 1 + Math.floor(rng() * 2) }, () => {
+    const kid = { to: pick(addresses), mode: pick(KID_MODES), goal: pick(GOALS), awaited: rng() < 0.6, cancelThrows: rng() < 0.2, cancelSettles: rng() < 0.2,
+      delayMs: rng() < 0.2 ? pick(DELAYS) : undefined };
+    return depth === 1 && rng() < 0.15 ? { ...kid, mode: 'parent', kids: kids(2) } : kid;
+  });
+  const unended = node => node.q("SELECT from_addr, id, state FROM kernel_calls WHERE state IN ('sending', 'open') ORDER BY rowid").all();
+  for (let step = 0; step < steps; step++) {
+    const roll = rng();
+    if (roll < 0.16) {
+      const node = pick(nodes);
+      const call = {
+        from: pick(node.addresses), id: `${seed}-${issued.length}`, to: pick(addresses), op: rng() < 0.05 ? 'nope' : 'work',
+        input: { mode: pick(MODES), goal: pick(GOALS), n: issued.length, kids: kids(), cancelThrows: rng() < 0.2, cancelSettles: rng() < 0.2 },
+        lane: pick(LANES),
+        memo: { resultThrows: rng() < 0.2 ? 1 + Math.floor(rng() * 2) : 0, statusThrows: rng() < 0.1 },
+        delayMs: rng() < 0.25 ? pick(DELAYS) : undefined,
+      };
+      issued.push(call);
+      world.act(node, kernel => call.delayMs === undefined ? kernel.call(call) : kernel.after(call));
+    } else if (roll < 0.18) {
+      const node = pick(nodes);
+      const run = { address: pick(node.addresses), id: `${seed}-r${roots++}`, goal: pick(GOALS), kids: kids() };
+      world.act(node, kernel => node.host.root(kernel, run));
+    } else if (roll < 0.22 && issued.length > 0) {
+      // The same request again, as a call or as a timer, whichever it was first: the two share their ids (7).
+      const call = pick(issued);
+      const how = rng() < 0.5 ? 'call' : 'after';
+      const seen = world.act(world.owners.get(call.from), kernel => kernel[how]({ ...call, delayMs: call.delayMs ?? 5 }));
+      if (seen) assert.deepEqual([seen.to, seen.op], [call.to, call.op]);
+      if (seen) world.count(`same id again: ${call.delayMs === undefined ? 'call' : 'after'} then ${how}, finds a ${seen.state === 'scheduled' ? 'timer' : 'call'}`);
+    } else if (roll < 0.45 && world.net.length > 0) world.deliver(Math.floor(rng() * world.net.length), true);
+    else if (roll < 0.47) {
+      // A host does what it may to a timer not yet due.
+      const node = pick(nodes);
+      const timer = node.migrated() ? pick(node.q('SELECT from_addr, id FROM kernel_timers ORDER BY rowid').all()) : undefined;
+      const how = pick(['unschedule', 'cancel', 'abandon', 'detach', 'inquire']);
+      const ref = timer && { from: timer.from_addr, id: timer.id };
+      if (ref && how === 'inquire' && node.up) {
+        world.inquire(node, ref);
+        world.count('timer: inquire');
+      } else if (ref && how !== 'inquire' && world.act(node, kernel => (kernel[how](ref), true))) world.count(`timer: ${how}`);
+    } else if (roll < 0.485) world.time += pick([1, 2, 5, 10, 20]);
+    else if (roll < 0.50) {
+      const node = pick(nodes);
+      if (node.up && node.woke !== null && node.woke <= world.time) world.count('tick: the host\'s timer went off');
+      world.tick(node);
+    }
+    else if (roll < 0.57) {
+      // Someone answers a run: mostly one waiting for input, now and then with a stale question, or a run that has
+      // ended, a run that never was, or the wrong address.
+      const node = pick(nodes);
+      const runs = status => nodes.filter(node => node.migrated())
+        .flatMap(node => node.q(`SELECT from_addr, id, to_addr, asked FROM kernel_runs WHERE status ${status} ORDER BY rowid`).all());
+      const run = (rng() < 0.85 ? pick(runs("= 'input-required'")) : undefined) ?? (rng() < 0.9 ? pick(runs('IS NOT NULL')) : undefined)
+        ?? { from_addr: 'a1', id: 'none', to_addr: pick(addresses), asked: 1 };
+      const sent = { from: pick(node.addresses), id: `${seed}-n${answers.length}`, to: rng() < 0.05 ? pick(addresses) : run.to_addr,
+        call: { from: run.from_addr, id: run.id }, asked: run.asked - (rng() < 0.1 ? 1 : 0), answer: { mode: pick(ANSWERS) } };
+      answers.push({ node, sent });
+      answer(node, sent);
+    } else if (roll < 0.585 && answers.length > 0) {
+      // Mostly one whose receipt was lost; now and then one taken already, or any, as an answerer unsure of what it sent.
+      const which = rng();
+      const { node, sent } = which < 0.5 && unanswered.length > 0 ? unanswered.splice(Math.floor(rng() * unanswered.length), 1)[0]
+        : which < 0.8 && taken.length > 0 ? pick(taken) : pick(answers);
+      world.count('answer sent again with its number');
+      answer(node, sent);
+    }
+    else if (roll < 0.64 && world.tasks.length > 0) world.tasks.shift()();
+    else if (roll < 0.71) world.pump(pick(nodes));
+    else if (roll < 0.74) {
+      const node = pick(nodes);
+      const row = node.up ? pick(held(node)) : undefined;
+      const detach = rng() < 0.3;
+      if (row) world.act(node, kernel => node.host.finish(kernel, row, { detach }));
+    } else if (roll < 0.79) {
+      const node = pick(nodes);
+      const row = node.up ? pick(held(node)) : undefined;
+      // Now and then twice, with the send loop let run in between: an older status may still be on its way. Now and
+      // then the run ends right after, so that its outcome may overtake its status.
+      for (let times = rng() < 0.5 ? 2 : 1; row && times > 0; times--) {
+        const change = pick(CHANGES);
+        world.act(node, kernel => node.host.nudge(kernel, row, change));
+        await flush();
+      }
+      if (row && rng() < 0.3) world.act(node, kernel => node.host.finish(kernel, row));
+    } else if (roll < 0.83) {
+      // Any call a node made, whatever its state.
+      const node = pick(nodes);
+      const call = node.migrated() ? pick(node.q('SELECT from_addr, id, state FROM kernel_calls ORDER BY rowid').all()) : undefined;
+      if (call && world.act(node, kernel => (kernel.cancel({ from: call.from_addr, id: call.id }, 'changed my mind'), true)))
+        world.count(`cancel a call that is ${ENDED.has(call.state) ? 'ended' : call.state}`);
+    } else if (roll < 0.85) {
+      const node = pick(nodes);
+      const call = node.migrated() ? pick(unended(node)) : undefined;
+      if (call && world.act(node, kernel => (kernel.abandon({ from: call.from_addr, id: call.id }), true))) world.count(`abandon a call that is ${call.state}`);
+    } else if (roll < 0.86) {
+      const node = pick(nodes);
+      const row = node.up ? pick(held(node)) : undefined;
+      if (row && world.act(node, kernel => (kernel.stop({ from: row.from_addr, id: row.id }), true)))
+        world.count(row.from_addr === row.addr ? 'stop a root run' : 'stop a run');
+    } else if (roll < 0.88) {
+      const node = pick(nodes);
+      const call = node.up ? pick(unended(node).filter(call => call.state === 'open')) : undefined;
+      if (call) world.inquire(node, { from: call.from_addr, id: call.id });
+    } else if (roll < 0.89) {
+      // A run the host keeps working makes another call it means to await.
+      const node = pick(nodes);
+      const row = node.up ? pick(held(node)) : undefined;
+      const asked = row && node.q('SELECT cancel_asked FROM kernel_runs WHERE from_addr = ? AND id = ?').get(row.from_addr, row.id).cancel_asked === 1;
+      const call = row && { from: row.addr, id: `${seed}-k${later++}`, to: pick(addresses), op: 'work', input: { mode: pick(KID_MODES), goal: pick(GOALS) },
+        by: { from: row.from_addr, id: row.id }, awaited: true, delayMs: rng() < 0.5 ? pick(DELAYS) : undefined };
+      if (call && world.act(node, kernel => call.delayMs === undefined ? kernel.call(call) : kernel.after(call)) && asked)
+        world.count(`${call.delayMs === undefined ? 'call' : 'timer'} from a run asked to cancel`);
+    } else if (roll < 0.92) {
+      const node = pick(nodes.filter(node => node.up).concat([undefined]));
+      node?.down();
+    } else if (roll < 0.945) pick(nodes.filter(node => !node.up).concat([undefined]))?.boot({ recover: false });
+    else if (roll < 0.97) pick(nodes.filter(node => node.up && !node.ready).concat([undefined]))?.recover();
+    else if (roll < 0.99) world.crashAt = world.commits + 1 + Math.floor(rng() * 4);
+    await flush();
+    world.check();
+  }
+  world.crashAt = Infinity;
+  world.faults = NO_FAULTS;
+  await world.quiesce(() => advance(world));
+  world.check();
+  world.checkQuiet();
+  checkErrors(world);
+  return { world, issued: issued.length };
+}

package/sim/solo.mjs

@@ -0,0 +1,47 @@
+import { DatabaseSync } from 'node:sqlite';
+import { createKernel } from '../kernel.mjs';
+
+/**
+ * One kernel on its own, for tests that play the other side by hand. Its connection logs every statement it executes
+ * and can be told to fail the next COMMIT. Its `send` records each letter and answers with `answer(to, envelope)`;
+ * by default it never answers. Its clock reads `clock.now`, and what it gives `wake` is kept in `woken`.
+ * @param {{ answer?: (to: string, envelope: object) => Promise<object>, schedule?: (fn: () => void) => void,
+ *   wake?: (at: number | null) => void }} [options]
+ */
+export function solo({ answer = () => new Promise(() => {}), schedule = fn => fn(), wake } = {}) {
+  const raw = new DatabaseSync(':memory:');
+  const log = [];
+  const db = {
+    failCommit: false,
+    get isTransaction() {
+      return raw.isTransaction;
+    },
+    exec(sql) {
+      log.push(sql);
+      if (sql === 'COMMIT' && db.failCommit) {
+        db.failCommit = false;
+        throw new Error('disk I/O error');
+      }
+      raw.exec(sql);
+    },
+    prepare: sql => raw.prepare(sql),
+  };
+  const sent = [];
+  const errors = [];
+  const clock = { now: 0 };
+  const woken = [];
+  const kernel = createKernel({
+    db,
+    now: () => clock.now,
+    send: (to, envelope) => {
+      sent.push({ to, envelope });
+      return answer(to, envelope);
+    },
+    onError: error => errors.push(error),
+    wake: wake ?? (at => woken.push(at)),
+    schedule,
+  });
+  kernel.transaction(() => kernel.migrate());
+  return { kernel, raw, db, log, sent, errors, clock, woken, row: (sql, ...args) => raw.prepare(sql).get(...args),
+    rows: (sql, ...args) => raw.prepare(sql).all(...args).map(row => ({ ...row })) };
+}