@rikalabs/akter 0.1.0-alpha.0

package/dist/testing/conformance/simulation.js

@@ -0,0 +1,167 @@
+import { Console, Effect, Layer, Schema } from "effect";
+import { Actor } from "../../index.js";
+import { ActorTest } from "../actor-test.js";
+import { ActorCluster } from "../cluster.js";
+import { clusterSimulationSeeds } from "../simulate-cluster.js";
+const Add = Actor.command("Add", { payload: Schema.Int, success: Schema.Int });
+const Credit = Actor.command("Credit", { payload: Schema.Int });
+const Pay = Actor.command("Pay", {
+    payload: Schema.Struct({ to: Schema.String, amount: Schema.Int }),
+});
+const total = Actor.state({ total: Schema.Int.pipe(Schema.withDecodingDefault(Effect.succeed(0))) });
+const Tally = Actor.make("SimClusterTally", { key: Schema.String, state: total, api: { Add } });
+const Wallet = Actor.make("SimClusterWallet", {
+    key: Schema.String,
+    state: total,
+    api: {},
+    internal: { Credit },
+});
+const Payer = Actor.make("SimClusterPayer", { key: Schema.String, api: { Pay } });
+export const simulationActors = Layer.mergeAll(Tally.toLayer(Effect.succeed({
+    Add: Effect.fnUntraced(function* (amount) {
+        const turn = yield* Tally.Turn;
+        yield* turn.state.set({ total: turn.state.total + amount });
+        return turn.state.total;
+    }),
+})), Wallet.toLayer(Effect.succeed({
+    Credit: Effect.fnUntraced(function* (amount) {
+        const turn = yield* Wallet.Turn;
+        yield* turn.state.set({ total: turn.state.total + amount });
+    }),
+})), Payer.toLayer(Effect.succeed({
+    Pay: Effect.fnUntraced(function* ({ to, amount }) {
+        yield* (yield* Wallet.intents(to)).Credit(amount);
+    }),
+})));
+/**
+ * Direct adds and relayed payments on a few actors, sent through `commands`
+ * commands of a seeded cluster simulation, then the totals each actor holds
+ * against the sums the script sent. Actor names carry `tag`, so scripts of
+ * several runs share a cluster without sharing state.
+ */
+export const clusterScript = ({ expect, options, commands, tag = options.seed, }) => Effect.gen(function* () {
+    const cluster = yield* ActorCluster;
+    const expected = new Map();
+    const report = yield* ActorTest.simulateCluster({ ...options, commands }, (sim) => Effect.gen(function* () {
+        for (let index = 0; index < commands; index++) {
+            const amount = yield* sim.int(1, 9);
+            const name = `${tag}-${yield* sim.pick(["a", "b", "c"])}`;
+            if ((yield* sim.int(0, 1)) === 0) {
+                const reply = yield* sim.command(`add ${name}`, Tally.get(name).pipe(Effect.flatMap((tally) => tally.Add(amount))));
+                expected.set(`tally:${name}`, (expected.get(`tally:${name}`) ?? 0) + amount);
+                expect(reply).toBe(expected.get(`tally:${name}`));
+            }
+            else {
+                yield* sim.command(`pay ${name}`, Payer.get(`${tag}-payer`).pipe(Effect.flatMap((payer) => payer.Pay({ to: name, amount }))));
+                expected.set(`wallet:${name}`, (expected.get(`wallet:${name}`) ?? 0) + amount);
+            }
+        }
+    }));
+    const held = new Map();
+    for (const key of expected.keys()) {
+        const [kind, name] = key.split(":");
+        held.set(key, (yield* cluster.on(0)(Effect.gen(function* () {
+            const test = yield* ActorTest;
+            const handle = kind === "tally" ? yield* Tally.get(name) : yield* Wallet.get(name);
+            return yield* test.inspect(handle.ref);
+        }))).state);
+    }
+    return { report, expected, held };
+});
+const clusterOn = (environment, body) => Effect.gen(function* () {
+    const database = yield* environment.freshDatabase;
+    const context = yield* Layer.build(ActorTest.cluster({
+        database,
+        runners: 3,
+        shardLockExpiration: "3 seconds",
+        actors: simulationActors,
+        relay: { poll: "100 millis" },
+    }));
+    return yield* body.pipe(Effect.provideContext(context));
+}).pipe(Effect.scoped);
+const withCluster = (environment, body) => environment.run(clusterOn(environment, body));
+export const sums = (expected) => new Map(Array.from(expected, ([key, sum]) => [key, { total: sum }]));
+/**
+ * Each seed gets a cluster of its own: settling moves the runners' clocks, and
+ * a runner restarted by one seed's kill would start the next one behind them.
+ * The seeded case logs each seed's schedule tagged `CLUSTER_SIMULATION`, so a
+ * night's schedules can be read from its log.
+ */
+export const simulationConformance = [
+    {
+        name: "keeps receipts and outbox delivery exactly once across seeded runner kills, lost heartbeats, crashes, and lost connections on three runners",
+        requiresIndependentConnections: true,
+        timeoutMs: 1_800_000,
+        run: ({ expect, environment }) => environment.run(Effect.gen(function* () {
+            for (const seed of yield* clusterSimulationSeeds)
+                yield* clusterOn(environment, Effect.gen(function* () {
+                    const { report, expected, held } = yield* clusterScript({
+                        expect,
+                        options: {
+                            seed: `c${seed}`,
+                            faults: [
+                                "crashBeforeCommit",
+                                "crashAfterCommit",
+                                "dropReply",
+                                "runnerKill",
+                                "heartbeatLoss",
+                                "connectionLoss",
+                            ],
+                        },
+                        commands: 8,
+                    });
+                    yield* Console.error(`CLUSTER_SIMULATION seed=${seed} ${report.steps
+                        .map(({ fault, point, landed }) => `${fault}${landed ? "+landed" : ""}@${point}`)
+                        .join(" ")}`);
+                    expect(report.steps.length).toBe(8);
+                    expect(held).toEqual(sums(expected));
+                }));
+        })),
+    },
+    {
+        name: "reruns a seed on three runners to the same fault schedule and outcome",
+        requiresIndependentConnections: true,
+        timeoutMs: 300_000,
+        run: ({ expect, environment }) => withCluster(environment, Effect.gen(function* () {
+            const options = {
+                seed: "repeat",
+                faults: ["crashBeforeCommit", "crashAfterCommit", "dropReply", "connectionLoss"],
+                faultRate: 0.8,
+            };
+            const first = yield* clusterScript({ expect, options, commands: 6, tag: "first" });
+            const second = yield* clusterScript({ expect, options, commands: 6, tag: "second" });
+            const schedule = (run) => run.report.steps.map(({ label, fault, runner, point, landed }) => [
+                label.split(" ")[0],
+                fault,
+                runner,
+                point,
+                landed,
+            ]);
+            expect(schedule(second)).toEqual(schedule(first));
+            expect(first.report.steps.some(({ fault }) => fault === "connectionLoss")).toBe(true);
+            expect(Array.from(second.expected.values())).toEqual(Array.from(first.expected.values()));
+            expect(second.held).toEqual(sums(second.expected));
+        })),
+    },
+    {
+        name: "dies with the seed when a cluster program sends no command",
+        requiresIndependentConnections: true,
+        timeoutMs: 60_000,
+        run: ({ expect, environment }) => withCluster(environment, Effect.gen(function* () {
+            const failure = yield* ActorTest.simulateCluster({ seed: "empty", faults: ["runnerKill"] }, () => Effect.void).pipe(Effect.exit);
+            expect(String(failure)).toContain("Simulation failed with seed empty");
+            expect(String(failure)).toContain("the program sent no command");
+        })),
+    },
+    {
+        name: "refuses a primary failover without a primary to fail over or a command count",
+        requiresIndependentConnections: true,
+        timeoutMs: 60_000,
+        run: ({ expect, environment }) => withCluster(environment, Effect.gen(function* () {
+            const unwired = yield* ActorTest.simulateCluster({ seed: "failover", faults: ["primaryFailover"], commands: 1 }, () => Effect.void).pipe(Effect.exit);
+            expect(String(unwired)).toContain("primaryFailover needs the primary option");
+            const uncounted = yield* ActorTest.simulateCluster({ seed: "failover", faults: ["primaryFailover"], primary: Effect.void }, () => Effect.void).pipe(Effect.exit);
+            expect(String(uncounted)).toContain("primaryFailover needs the number of commands");
+        })),
+    },
+];

package/dist/testing/conformance/single-shard.d.ts

@@ -0,0 +1,2 @@
+import type { ConformanceCase } from "../conformance.ts";
+export declare const singleShardConformance: ReadonlyArray<ConformanceCase>;

package/dist/testing/conformance/single-shard.js

@@ -0,0 +1,125 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+import { ActorTest } from "../actor-test.js";
+import { touchesOneShard } from "./neki/plan.js";
+import { mintSuite, mintWorkload, planAcrossShard } from "./mint.js";
+import { placementWorkload } from "./placement.js";
+import { scopeOf, statementLog } from "./statements.js";
+/**
+ * Drives a turn, a wake, a due-work scan, and the parent families of the
+ * placement fixtures, and returns every statement the runtime compiled while it
+ * did. The suite's runtime stops meanwhile: the first runtime creates the
+ * actors, and a second one wakes them from storage, so the recorder sits on
+ * runtimes of this case alone. The suite's runtime is running again on return.
+ * Start-up runs its own migrations and registry reads, so recording starts
+ * only once a runtime has finished starting.
+ */
+const frameworkStatements = (environment) => Effect.gen(function* () {
+    const log = statementLog();
+    const on = (effect, record) => Effect.acquireUseRelease(Effect.sync(() => environment.build({ observe: log.observe })), (runtime) => Effect.promise(() => runtime.runPromise(Effect.void)).pipe(Effect.andThen(Effect.sync(() => {
+        log.recording = record;
+    })), Effect.andThen(Effect.promise(() => runtime.runPromise(Effect.scoped(effect)))), Effect.ensuring(Effect.sync(() => {
+        log.recording = false;
+    }))), (runtime) => Effect.promise(() => runtime.dispose()));
+    yield* environment.stop;
+    yield* on(placementWorkload.create, false);
+    yield* on(Effect.gen(function* () {
+        yield* placementWorkload.wake;
+        yield* mintWorkload;
+        yield* (yield* ActorTest).advance("2 hours");
+    }), true).pipe(Effect.ensuring(environment.restart));
+    return Array.from(log.seen.values());
+});
+const byScope = (statements) => {
+    const scopes = {
+        "table-free": [],
+        keyed: [],
+        scan: [],
+        registry: [],
+        unkeyed: [],
+    };
+    for (const statement of statements)
+        scopes[scopeOf(statement.sql)].push(statement);
+    return scopes;
+};
+/** The statements whose Neki plan reaches anything other than one shard, with the plan. */
+const scatteredOf = (environment, statements) => Effect.promise(() => environment.run(Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const scattered = [];
+    for (const { sql: statement, params } of statements) {
+        const rows = yield* sql
+            .unsafe(`EXPLAIN (NEKI_PLAN, COSTS OFF, FORMAT TEXT) ${statement}`, params)
+            .values.pipe(Effect.orDie);
+        const plan = rows.map(([line]) => String(line)).join("\n");
+        if (!touchesOneShard(plan))
+            scattered.push({ statement: statement.slice(0, 160), plan });
+    }
+    return scattered;
+})));
+export const singleShardConformance = [
+    {
+        name: "single-shard: a minted child's first turn touches only its own routing key, not its parent's outbox",
+        run: ({ expect, environment, fixtureOf }) => Effect.runPromise(Effect.gen(function* () {
+            const log = statementLog();
+            const fixture = fixtureOf(mintSuite);
+            let child;
+            yield* environment.stop;
+            yield* Effect.acquireUseRelease(Effect.sync(() => environment.build({ observe: log.observe })), (runtime) => Effect.promise(() => runtime.runPromise(Effect.gen(function* () {
+                child = yield* planAcrossShard("child-statements");
+                fixture.onTurn = (point, request) => {
+                    if (request.ref.id !== child.ref.id)
+                        return;
+                    if (point === "afterClaim")
+                        log.recording = true;
+                    if (point === "beforeOutboxDelete")
+                        log.recording = false;
+                };
+                yield* (yield* ActorTest).advance("1 minute");
+            }))), (runtime) => Effect.promise(() => runtime.dispose()).pipe(Effect.ensuring(Effect.sync(() => {
+                fixture.onTurn = undefined;
+                log.recording = false;
+            })))).pipe(Effect.ensuring(environment.restart));
+            const statements = Array.from(log.seen.values());
+            const scopes = byScope(statements);
+            expect(scopes.keyed.length > 0).toBe(true);
+            expect(scopes.unkeyed).toEqual([]);
+            expect(scopes.scan).toEqual([]);
+            expect(scopes.registry).toEqual([]);
+            expect(scopes.keyed
+                .filter(({ params }) => !params.some((value) => String(value) === String(child.childKey)))
+                .map(({ sql }) => sql)).toEqual([]);
+            expect(statements
+                .filter(({ params }) => params.some((value) => String(value) === String(child.parentKey)))
+                .map(({ sql }) => sql)).toEqual([]);
+            expect(statements.filter(({ sql }) => /\bactor_outbox\b/.test(sql))).toEqual([]);
+        })),
+    },
+    {
+        name: "single-shard: every framework statement of a turn, a wake, and a due-work scan names its routing key or its bucket range",
+        run: ({ expect, environment }) => Effect.runPromise(Effect.gen(function* () {
+            const scopes = byScope(yield* frameworkStatements(environment));
+            expect(scopes.keyed.length > 10).toBe(true);
+            expect(scopes.scan.length).toBe(2);
+            expect(scopes.scan.every(({ sql }) => /generate_series\(\$\d+::int, \$\d+::int\)/.test(sql))).toBe(true);
+            expect(scopes.unkeyed.map(({ sql }) => sql)).toEqual([]);
+        })),
+    },
+    {
+        name: "single-shard: EXPLAIN (NEKI_PLAN) plans every keyed framework statement of a turn and a wake on one shard",
+        requiresNeki: true,
+        timeoutMs: 120_000,
+        run: ({ expect, environment }) => Effect.runPromise(Effect.gen(function* () {
+            const scopes = byScope(yield* frameworkStatements(environment));
+            expect(yield* scatteredOf(environment, scopes.keyed)).toEqual([]);
+        })),
+    },
+    {
+        name: "single-shard: EXPLAIN (NEKI_PLAN) plans each due-work scan statement on one shard",
+        requiresNeki: true,
+        timeoutMs: 120_000,
+        run: ({ expect, environment }) => Effect.runPromise(Effect.gen(function* () {
+            const scopes = byScope(yield* frameworkStatements(environment));
+            expect(yield* scatteredOf(environment, scopes.scan)).toEqual([]);
+        })),
+    },
+];

package/dist/testing/conformance/singleton.d.ts

@@ -0,0 +1,3 @@
+import type { ConformanceCase } from "../conformance.ts";
+/** Singleton cases: one activation and one background loop across three runners, and takeover by one survivor after a kill. */
+export declare const singletonConformance: ReadonlyArray<ConformanceCase>;

package/dist/testing/conformance/singleton.js

@@ -0,0 +1,293 @@
+import { Cause, DateTime, Effect, Exit, Layer, Option, Schedule, Schema } from "effect";
+import { SqlClient } from "effect/sql";
+import { Actor } from "../../index.js";
+import { InternalActors } from "../../runtime/actors.js";
+import { ActorTest } from "../actor-test.js";
+import { ActorCluster } from "../cluster.js";
+/** A tick from `loop`, committed by the activation whose loop is `by`. */
+const Ticked = Actor.event("Ticked", { loop: Schema.String, by: Schema.String });
+const Tick = Actor.command("Tick", { payload: Schema.String, success: Schema.Finite });
+const Log = Actor.query("Log", {
+    success: Schema.Array(Schema.Struct({
+        cursor: Schema.String,
+        loop: Schema.String,
+        by: Schema.String,
+        at: Schema.Finite,
+    })),
+});
+const Beacon = Actor.make("Beacon", {
+    key: Actor.singleton,
+    state: Actor.state({ ticks: Schema.Finite.pipe(Schema.withDecodingDefault(Effect.succeed(0))) }),
+    events: [Ticked],
+    api: { Tick, Log },
+});
+/** Every loop each cluster's singleton started, keyed by the cluster's tenant. */
+const loops = new Map();
+/** The most loops of one tenant live at once, checked whenever a loop starts. */
+const peaks = new Map();
+const peakOf = (tenant) => peaks.get(tenant) ?? 0;
+const startedLoops = () => [...loops.values()].reduce((total, started) => total + started.length, 0);
+const liveOf = (tenant) => loopsOf(tenant).filter(({ live }) => live);
+const loopsOf = (tenant) => loops.get(tenant) ?? [];
+const TICK_INTERVAL = "50 millis";
+/**
+ * A singleton whose build forks one background loop. The loop shows it ran
+ * only by sending `Tick` commands, so every tick it made is a committed turn.
+ */
+const BeaconLive = Layer.mergeAll(Beacon.toLayer(Effect.gen(function* () {
+    const beacon = yield* Beacon.get();
+    const started = loops.get(beacon.ref.tenant) ?? [];
+    loops.set(beacon.ref.tenant, started);
+    const loop = {
+        id: `loop-${started.length + 1}`,
+        startedAt: performance.now(),
+        stoppedAt: undefined,
+        live: true,
+        closed: false,
+        attempts: 0,
+        refused: [],
+    };
+    started.push(loop);
+    peaks.set(beacon.ref.tenant, Math.max(peakOf(beacon.ref.tenant), started.filter(({ live }) => live).length));
+    yield* Effect.forkScoped(Effect.gen(function* () {
+        loop.attempts += 1;
+        yield* beacon.Tick(loop.id).pipe(Effect.catch((error) => Effect.sync(() => {
+            loop.refused.push(error.reason);
+        })));
+    }).pipe(Effect.repeat(Schedule.spaced(TICK_INTERVAL)), Effect.ensuring(Effect.sync(() => {
+        loop.live = false;
+        loop.stoppedAt = performance.now();
+    }))), { startImmediately: true });
+    yield* Effect.addFinalizer(() => Effect.sync(() => {
+        loop.closed = true;
+    }));
+    return {
+        Tick: Effect.fnUntraced(function* (from) {
+            const turn = yield* Beacon.Turn;
+            yield* turn.state.set({ ticks: turn.state.ticks + 1 });
+            yield* turn.emit(Ticked.make({ loop: from, by: loop.id }));
+            return turn.state.ticks;
+        }),
+    };
+})), Beacon.toQueryLayer(Effect.succeed({
+    Log: Effect.fnUntraced(function* () {
+        const entries = yield* (yield* Beacon.Read).events(Ticked);
+        return entries.map(({ cursor, event, timestamp }) => ({
+            cursor,
+            loop: event.loop,
+            by: event.by,
+            at: DateTime.toEpochMillis(timestamp),
+        }));
+    }, Effect.orDie),
+})));
+const EXPIRATION_SECONDS = 3;
+const singletonCluster = (database, runners) => ActorTest.cluster({
+    database,
+    runners,
+    shardLockExpiration: `${EXPIRATION_SECONDS} seconds`,
+    actors: BeaconLive,
+});
+/** Builds a fresh database and a cluster of `runners` serving the singleton. */
+const withSingletonCluster = (environment, runners, body) => environment.run(Effect.gen(function* () {
+    const database = yield* environment.freshDatabase;
+    const context = yield* Layer.build(singletonCluster(database, runners));
+    return yield* body.pipe(Effect.provideContext(context));
+}));
+const on = (runner) => (effect) => Effect.gen(function* () {
+    const cluster = yield* ActorCluster;
+    return yield* cluster.on(runner)(effect);
+});
+const beaconRef = (runner) => on(runner)(Beacon.get().pipe(Effect.map((beacon) => beacon.ref)));
+const logOf = (runner) => on(runner)(Beacon.get().pipe(Effect.flatMap((beacon) => beacon.Log())));
+const tickFrom = (runner, from) => on(runner)(Beacon.get().pipe(Effect.flatMap((beacon) => beacon.Tick(from))));
+const inspectOn = (runner, ref) => on(runner)(ActorTest.use((test) => test.inspect(ref)));
+/** Polls `runner`'s view of the log until `accept` holds. */
+const awaitLog = (runner, accept, what) => logOf(runner).pipe(Effect.repeat({ schedule: Schedule.spaced("20 millis"), until: accept }), Effect.timeoutOrElse({
+    duration: "30 seconds",
+    orElse: () => Effect.die(new Error(`Timed out waiting for ${what}`)),
+}));
+/** Polls until `check` holds. */
+const awaitThat = (check, what) => Effect.sync(check).pipe(Effect.repeat({ schedule: Schedule.spaced("20 millis"), until: (held) => held }), Effect.timeoutOrElse({
+    duration: "30 seconds",
+    orElse: () => Effect.die(new Error(`Timed out waiting for ${what}`)),
+}));
+/** A shard lock row with database-clock times in epoch milliseconds. */
+const lockOf = (runner, ref) => on(runner)(Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const shard = yield* (yield* InternalActors).shardId(ref);
+    const [row] = yield* sql `
+        SELECT address,
+          (EXTRACT(EPOCH FROM acquired_at) * 1000)::float8 AS acquired,
+          (EXTRACT(EPOCH FROM LOCALTIMESTAMP) * 1000)::float8 AS now
+        FROM cluster_locks WHERE shard_id = ${shard}`.pipe(Effect.orDie);
+    return row;
+}));
+/**
+ * Waits for the cluster's shards to settle and returns the one live loop once
+ * it has committed a tick; rebalancing while runners join may move the
+ * singleton, and so restart its loop, before that.
+ */
+const settled = Effect.gen(function* () {
+    const cluster = yield* ActorCluster;
+    const ref = yield* beaconRef(0);
+    yield* cluster.ready;
+    yield* awaitThat(() => liveOf(ref.tenant).length === 1, "one live loop");
+    const loop = liveOf(ref.tenant)[0];
+    yield* awaitLog(0, (log) => log.some(({ by }) => by === loop.id), "a settled tick");
+    return { ref, loop, before: loopsOf(ref.tenant).length };
+});
+/**
+ * Waits until one loop is live and its activation committed the log's last
+ * five ticks, and returns that loop.
+ */
+const steady = (runner, tenant) => Effect.gen(function* () {
+    const [live] = liveOf(tenant);
+    const tail = (yield* logOf(runner)).slice(-5);
+    return live !== undefined &&
+        liveOf(tenant).length === 1 &&
+        tail.length === 5 &&
+        tail.every(({ by }) => by === live.id)
+        ? Option.some(live)
+        : Option.none();
+}).pipe(Effect.repeat({ schedule: Schedule.spaced("25 millis"), until: Option.isSome }), Effect.map(Option.getOrThrow), Effect.timeoutOrElse({
+    duration: "30 seconds",
+    orElse: () => Effect.die(new Error("Timed out waiting for one steady loop")),
+}));
+/**
+ * The activations that committed a log, in commit order with consecutive
+ * duplicates collapsed. Each activation's commits are one unbroken run when
+ * none commits after its successor has.
+ */
+const runs = (log) => log.flatMap(({ by }, index) => (index === 0 || log[index - 1].by !== by ? [by] : []));
+const unbroken = (log) => new Set(runs(log)).size === runs(log).length;
+const contiguous = (log) => log.every(({ cursor }, index) => cursor === String(index + 1));
+/** Singleton cases: one activation and one background loop across three runners, and takeover by one survivor after a kill. */
+export const singletonConformance = [
+    {
+        name: "keeps one singleton activation and one background loop across three runners",
+        requiresIndependentConnections: true,
+        timeoutMs: 60_000,
+        run: ({ expect, environment }) => withSingletonCluster(environment, 3, Effect.gen(function* () {
+            const cluster = yield* ActorCluster;
+            const { ref, loop, before } = yield* settled;
+            for (const runner of [0, 1, 2])
+                yield* tickFrom(runner, `runner-${runner}`);
+            yield* awaitLog(1, (log) => log.filter(({ loop: from }) => from === loop.id).length >= 20, "twenty ticks from the settled loop");
+            const log = yield* logOf(2);
+            expect(peakOf(ref.tenant)).toBe(1);
+            expect(loopsOf(ref.tenant).length).toBe(before);
+            expect(liveOf(ref.tenant)).toEqual([loop]);
+            expect(unbroken(log)).toBe(true);
+            expect(runs(log).at(-1)).toBe(loop.id);
+            expect(contiguous(log)).toBe(true);
+            for (const runner of [0, 1, 2])
+                expect(log.find(({ loop: from }) => from === `runner-${runner}`)?.by).toBe(loop.id);
+            expect(loop.refused).toEqual([]);
+            const inspection = yield* inspectOn(1, ref);
+            expect(inspection.receipts).toBe(inspection.events);
+            expect(inspection.receipts >= log.length).toBe(true);
+            expect((yield* cluster.owner(ref)) === undefined).toBe(false);
+        })),
+    },
+    {
+        name: "registers a singleton's type before its entity can activate",
+        requiresIndependentConnections: true,
+        timeoutMs: 60_000,
+        run: ({ expect, environment }) => environment.run(Effect.gen(function* () {
+            const database = yield* environment.freshDatabase;
+            yield* Effect.gen(function* () {
+                const context = yield* Layer.build(singletonCluster(database, 1));
+                yield* on(0)(Effect.gen(function* () {
+                    const sql = yield* SqlClient.SqlClient;
+                    yield* sql `INSERT INTO actor_routed_subscriptions (source_type, subscriber_type, subscription)
+                  VALUES ('Beacon', 'Absent', 'Watch')`.pipe(Effect.orDie);
+                })).pipe(Effect.provideContext(context));
+            }).pipe(Effect.scoped);
+            const before = startedLoops();
+            const exit = yield* Layer.build(singletonCluster(database, 1)).pipe(Effect.scoped, Effect.exit);
+            expect(Exit.isFailure(exit) &&
+                Cause.pretty(exit.cause).includes("Actor Beacon is registered without the subscriber types that route from it")).toBe(true);
+            expect(startedLoops()).toBe(before);
+        })),
+    },
+    {
+        name: "moves the singleton and its loop to exactly one survivor after its runner is killed",
+        requiresIndependentConnections: true,
+        timeoutMs: 90_000,
+        run: ({ expect, environment }) => withSingletonCluster(environment, 3, Effect.gen(function* () {
+            const cluster = yield* ActorCluster;
+            const { ref, loop, before } = yield* settled;
+            const owner = (yield* cluster.owner(ref));
+            const survivor = (owner + 1) % 3;
+            const killed = (yield* lockOf(survivor, ref)).now;
+            yield* cluster.kill(owner);
+            const held = yield* lockOf(survivor, ref);
+            const taken = yield* lockOf(survivor, ref).pipe(Effect.repeat({
+                schedule: Schedule.spaced("20 millis"),
+                until: ({ address }) => address !== held.address,
+            }), Effect.timeoutOrElse({
+                duration: "30 seconds",
+                orElse: () => Effect.die(new Error("Timed out waiting for the shard's takeover")),
+            }));
+            yield* awaitThat(() => loopsOf(ref.tenant).length > before, "a survivor's loop");
+            const successor = loopsOf(ref.tenant)[before];
+            const log = yield* awaitLog(survivor, (entries) => entries.filter(({ by }) => by === successor.id).length >= 10, "the survivor's loop to tick");
+            const next = (yield* cluster.owner(ref));
+            expect(next === owner).toBe(false);
+            const expired = held.acquired + EXPIRATION_SECONDS * 1000;
+            expect(taken.acquired >= expired).toBe(true);
+            expect(taken.acquired >= killed).toBe(true);
+            expect(loopsOf(ref.tenant).length).toBe(before + 1);
+            expect(unbroken(log)).toBe(true);
+            expect(runs(log).slice(-2)).toEqual([loop.id, successor.id]);
+            expect(contiguous(log)).toBe(true);
+            const resumed = log.find(({ by }) => by === successor.id);
+            expect(resumed.at >= expired).toBe(true);
+            yield* awaitThat(() => !loop.live, "the dead runner's loop to stop");
+            expect(liveOf(ref.tenant)).toEqual([successor]);
+            expect(successor.refused).toEqual([]);
+            const inspection = yield* inspectOn(next, ref);
+            expect(inspection.receipts).toBe(inspection.events);
+        })),
+    },
+    {
+        name: "stops a heartbeat-paused singleton owner's loop before its successor starts and fences its commits",
+        requiresIndependentConnections: true,
+        timeoutMs: 90_000,
+        run: ({ expect, environment }) => withSingletonCluster(environment, 2, Effect.gen(function* () {
+            const cluster = yield* ActorCluster;
+            const { ref, loop: zombie, before } = yield* settled;
+            const paused = (yield* cluster.owner(ref));
+            const rival = 1 - paused;
+            const first = Number((yield* inspectOn(rival, ref)).generation);
+            const heartbeat = yield* cluster.pauseHeartbeat(paused);
+            yield* awaitThat(() => loopsOf(ref.tenant).length > before, "the rival's loop");
+            const successor = loopsOf(ref.tenant)[before];
+            yield* awaitLog(rival, (entries) => entries.filter(({ by }) => by === successor.id).length >= 10, "the rival's activation to commit");
+            expect(yield* cluster.owner(ref)).toBe(rival);
+            expect(zombie.live).toBe(false);
+            expect(zombie.stoppedAt < successor.startedAt).toBe(true);
+            yield* Effect.sleep(`${EXPIRATION_SECONDS} seconds`);
+            expect(loopsOf(ref.tenant).length).toBe(before + 1);
+            expect(liveOf(ref.tenant)).toEqual([successor]);
+            expect(peakOf(ref.tenant)).toBe(1);
+            const split = yield* logOf(rival);
+            expect(unbroken(split)).toBe(true);
+            expect(contiguous(split)).toBe(true);
+            const moved = split.findIndex(({ by }) => by === successor.id);
+            expect(split.slice(moved).some(({ by }) => by === zombie.id)).toBe(false);
+            yield* heartbeat.resume;
+            yield* cluster.ready;
+            const live = yield* steady(rival, ref.tenant);
+            yield* tickFrom(paused, "after-resume");
+            const log = yield* logOf(rival);
+            expect(unbroken(log)).toBe(true);
+            expect(contiguous(log)).toBe(true);
+            expect(log.find(({ loop }) => loop === "after-resume")?.by).toBe(live.id);
+            expect(liveOf(ref.tenant)).toEqual([live]);
+            const inspection = yield* inspectOn(rival, ref);
+            expect(inspection.receipts).toBe(inspection.events);
+            expect(Number(inspection.generation) > first).toBe(true);
+        })),
+    },
+];

package/dist/testing/conformance/statements.d.ts

@@ -0,0 +1,60 @@
+import type { Statement } from "effect/sql";
+export interface RecordedStatement {
+    /** The compiled text, whitespace collapsed. */
+    readonly sql: string;
+    /** The parameters of the first run of this text. */
+    readonly params: ReadonlyArray<unknown>;
+}
+/**
+ * Tables that hold one row per deployment, per actor type, or per runner
+ * rather than per actor, so no `routing_key` names them. Cluster's own
+ * `cluster_*` tables are the same: they stay together in one shard group.
+ */
+export declare const REGISTRY_TABLES: ReadonlySet<string>;
+export type StatementScope = 
+/** Reads no table: a clock, a setting, or a transaction command. */
+"table-free"
+/** Names a `routing_key` in a predicate or an insert list. */
+ | "keyed"
+/** Probes an index range of `bucket`, the top bits of `routing_key`, without naming a key. */
+ | "scan"
+/** Touches only per-deployment tables. */
+ | "registry"
+/** Touches a table without naming a routing key. */
+ | "unkeyed";
+export declare const scopeOf: (sql: string) => StatementScope;
+/** The statements the runtime compiled while recording, first sighting of each text. */
+export interface StatementLog {
+    recording: boolean;
+    readonly seen: Map<string, RecordedStatement>;
+    readonly observe: (statement: Statement.Statement<unknown>) => void;
+}
+export declare const statementLog: () => StatementLog;
+/**
+ * Counts the statements a client sends on one Postgres connection from its
+ * protocol messages, for a relay that sees every byte the client writes:
+ * every extended-protocol statement ends its cycle with one Sync, and a
+ * simple query is one Query message. The startup and encryption requests
+ * that open a connection carry no type byte and are skipped.
+ */
+export declare const wireStatements: () => (chunk: Buffer) => number;
+/**
+ * What one external command to a warm activation costs across every pool,
+ * derived from the turn's two groups rather than measured. The admission
+ * flight sends `BEGIN`, the timeout `set_config`, and the fenced read that
+ * resolves the receipt. A command that runs its handler and dirties one key
+ * then sends the state upsert, the receipt insert, `COMMIT`, and the read of
+ * the commit version and the clock its expiry recheck uses: 2 flights and 7
+ * statements. A replay sends `ROLLBACK` and that same read instead: 2 flights
+ * and 5 statements. Nothing is read before delivery or after the turn.
+ */
+export declare const SERVED_COMMAND: {
+    readonly warm: {
+        readonly flights: 2;
+        readonly statements: 7;
+    };
+    readonly replay: {
+        readonly flights: 2;
+        readonly statements: 5;
+    };
+};