@rikalabs/akter 0.1.0-alpha.0

package/dist/testing/actor-test.js

@@ -0,0 +1,337 @@
+import { Context, Crypto, Data, DateTime, Deferred, Duration, Effect, FileSystem, Layer, Redacted, Schema, Option, Predicate, Stream, } from "effect";
+import { SqlClient } from "effect/sql";
+import { CurrentCaller, Tenant, System, principal, } from "../identity/caller.js";
+import { descriptorOf } from "../actor/descriptor.js";
+import { OpenRejected } from "../runtime/connections/holder.js";
+import { ClientMessage } from "../runtime/connections/protocol.js";
+import { WatchTap } from "../runtime/connections/watch.js";
+import {} from "../handles/actors.js";
+import { InternalActors } from "../runtime/actors.js";
+import {} from "../runtime/request.js";
+import { SeedJson } from "../runtime/operators/seed.js";
+import { Database, layer as runtimeLayer } from "../runtime/layer.js";
+import { LocalRequestSerialization } from "../runtime/runner.js";
+import { compress, decompress, routingKey } from "../runtime/storage/codec.js";
+import { recordedPlacement } from "../runtime/storage/placements.js";
+import { VERSION_KEY } from "../state/migration.js";
+import { CleanupHooks, RetryTurn, TurnHooks } from "../runtime/turn/hooks.js";
+import { ProgressTap } from "../runtime/jobs/progress.js";
+import { databaseTime, FrameworkClock } from "../runtime/turn/admission.js";
+import { ActorCluster, clusterLayer } from "./cluster.js";
+import { simulate } from "./simulate.js";
+import { simulateCluster, } from "./simulate-cluster.js";
+/**
+ * Present while `ActorTest.cluster` builds one of its runners: the runner
+ * shares the cluster's tenant and opens its database connections through
+ * `connect`, so killing it can cut them.
+ */
+export class ClusterMember extends Context.Service()("@rikalabs/akter/testing/actor-test/ClusterMember") {
+}
+/** The grant key tests sign content grants with unless they configure their own. */
+export const TEST_CONTENT_KEY = {
+    id: "test",
+    secret: Redacted.make("akter test content grant key, never for production"),
+};
+const decodeJson = Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown));
+const valueCodec = (schema) => Schema.fromJsonString(Schema.toCodecJson(Schema.Struct({ value: schema })));
+const testActors = new WeakMap();
+/** Package-internal escape hatch for deterministic runtime conformance cases. */
+export const executeForTest = (request) => Effect.gen(function* () {
+    const test = yield* ActorTest;
+    const actors = testActors.get(test);
+    if (actors === undefined)
+        return yield* Effect.die(new Error("ActorTest runtime unavailable"));
+    return (yield* actors.execute(request)).outcome;
+});
+/**
+ * Runs one retention sweep in the current runtime now, as its background loop
+ * does every minute; for benchmarks and tests that use the production layer.
+ */
+export const cleanup = Effect.gen(function* () {
+    return yield* (yield* InternalActors).cleanup;
+});
+/**
+ * Sweeps every tenant's unreferenced content now, however recently each was
+ * swept, and returns how many contents it deleted; for benchmarks and tests.
+ */
+export const sweepContent = Effect.gen(function* () {
+    return yield* (yield* InternalActors).sweepContent;
+});
+export const ProgressRecord = Data.taggedEnum();
+export class ActorTest extends Context.Service()("@rikalabs/akter/testing/actor-test/ActorTest") {
+    /**
+     * Runs `runners` runtimes in this process against one Postgres database,
+     * each a distinct Cluster runner with its own address, connection pool, and
+     * expiring shard locks. Provides `ActorCluster`; see its controls.
+     */
+    static cluster = (options) => clusterLayer(options);
+    /**
+     * Runs `program` on the current test runtime under a fault schedule drawn
+     * from `seed`, then checks exactly-once receipts and outbox delivery; a
+     * failure dies with the seed that reproduces it.
+     */
+    static simulate = (options, program) => Effect.gen(function* () {
+        return yield* simulate(yield* ActorTest)(options, program);
+    });
+    /**
+     * Runs `program` on the current `ActorTest.cluster` under a fault schedule
+     * drawn from `seed`, with runner kills, lost heartbeats, and primary
+     * failovers as well as crashes, then checks exactly-once receipts and
+     * outbox delivery; a failure dies with the seed that reproduces it.
+     */
+    static simulateCluster = (options, program) => Effect.gen(function* () {
+        return yield* simulateCluster(yield* ActorCluster)(options, program);
+    });
+    static layer = (options = {}) => Layer.unwrap(Effect.gen(function* () {
+        const crypto = yield* Crypto.Crypto;
+        const member = Option.getOrUndefined(yield* Effect.serviceOption(ClusterMember));
+        const tenant = member?.tenant ?? (yield* crypto.randomUUIDv4.pipe(Effect.orDie));
+        const faults = new Map();
+        const outer = yield* TurnHooks;
+        let clockOffset = 0;
+        const progress = [];
+        let dropProgress = () => false;
+        let dropCommitted = false;
+        const hooks = Layer.mergeAll(Layer.succeed(TurnHooks, {
+            at: (point, request) => Effect.suspend(() => {
+                const queue = faults.get(point) ?? [];
+                const index = queue.findIndex(({ commandId }) => commandId === undefined || commandId === request.commandId);
+                return index < 0 ? outer.at(point, request) : queue.splice(index, 1)[0].fault;
+            }),
+        }), Layer.succeed(FrameworkClock, { offsetMillis: () => clockOffset }), Layer.succeed(WatchTap, { dropsCommitted: () => dropCommitted }), Layer.succeed(ProgressTap, {
+            send: (message) => Effect.sync(() => {
+                const dropped = dropProgress(message);
+                progress.push(ProgressRecord.Progress({ ...message, dropped }));
+                return !dropped;
+            }),
+            closed: (message) => Effect.sync(() => {
+                const dropped = dropProgress(message);
+                progress.push(ProgressRecord.ProgressClosed({ ...message, dropped }));
+                return !dropped;
+            }),
+        }), Layer.succeed(CleanupHooks, {
+            batchSize: 1000,
+            afterBatch: Effect.void,
+            periodic: false,
+        }));
+        const addFault = (point, fault, options) => Effect.sync(() => {
+            const queue = faults.get(point) ?? [];
+            queue.push({ commandId: options?.commandId, fault });
+            faults.set(point, queue);
+        });
+        const test = Layer.effect(ActorTest, Effect.gen(function* () {
+            const sql = yield* SqlClient.SqlClient;
+            const storedRoutingKey = Effect.fnUntraced(function* (ref) {
+                const placement = yield* recordedPlacement(ref.actor).pipe(Effect.provideService(SqlClient.SqlClient, sql));
+                if (placement === undefined)
+                    return yield* Effect.die(new Error(`Actor ${ref.actor} is not registered`));
+                return routingKey({ ref, placement });
+            });
+            const internalActors = yield* InternalActors;
+            const service = ActorTest.of({
+                tenant,
+                actor: Effect.fnUntraced(function* (definition, id = "singleton", actorOptions) {
+                    const { as } = options;
+                    const caller = as === undefined
+                        ? System.make({ source: "actor" })
+                        : Schema.is(System)(as)
+                            ? as
+                            : System.make({
+                                source: "actor",
+                                onBehalfOf: Option.getOrUndefined(principal(as)),
+                            });
+                    const internal = descriptorOf(definition)?.internal;
+                    if (internal === undefined)
+                        return yield* Effect.die(new Error("Unknown actor definition"));
+                    const system = yield* internal
+                        .handle(id, tenant, caller)
+                        .pipe(Effect.provideService(InternalActors, internalActors));
+                    if (actorOptions?.seed !== undefined) {
+                        const fs = yield* FileSystem.FileSystem;
+                        const seed = yield* fs
+                            .readFileString(actorOptions.seed)
+                            .pipe(Effect.flatMap(Schema.decodeEffect(SeedJson)), Effect.orDie);
+                        yield* internalActors.seed({ ref: system.ref, caller, seed });
+                    }
+                    return { system, inspect: service.inspect(system.ref) };
+                }),
+                crashNext: (point, faultOptions) => addFault(point, Effect.die(RetryTurn.make({ message: `Injected ${point} crash` })), faultOptions),
+                clearFaults: Effect.sync(() => {
+                    const left = Array.from(faults, ([point, queue]) => queue.map(() => point)).flat();
+                    faults.clear();
+                    return left;
+                }),
+                pauseNext: Effect.fnUntraced(function* (point, faultOptions) {
+                    const reached = yield* Deferred.make();
+                    const release = yield* Deferred.make();
+                    yield* addFault(point, Deferred.succeed(reached, undefined).pipe(Effect.andThen(Deferred.await(release))), faultOptions);
+                    return {
+                        reached: Deferred.await(reached),
+                        release: Deferred.succeed(release, undefined).pipe(Effect.asVoid),
+                    };
+                }),
+                inspect: Effect.fnUntraced(function* (ref) {
+                    const routing = yield* storedRoutingKey(ref);
+                    const generations = yield* sql `SELECT generation::text AS generation FROM actor_generations
+            WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}`;
+                    const state = (yield* sql `SELECT key, value FROM actor_state
+            WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}
+              AND key <> ${VERSION_KEY}`).map(({ key, value }) => ({
+                        key,
+                        value: decompress(value),
+                    }));
+                    const [counted] = yield* sql `SELECT
+                  (SELECT count(*)::integer FROM actor_receipts
+                    WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}) AS receipts,
+                  (SELECT count(*)::integer FROM actor_events
+                    WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}) AS events`;
+                    const outbox = yield* sql `SELECT count(*) FILTER (WHERE kind = 'intent')::integer AS intents,
+              count(*) FILTER (WHERE kind = 'job')::integer AS jobs FROM actor_outbox
+            WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}`;
+                    const tables = yield* sql `
+                  SELECT table_schema, table_name, CASE WHEN table_schema = current_schema()
+                    THEN table_name ELSE table_schema || '.' || table_name END AS label
+                  FROM actor_tables WHERE actor_type = ${ref.actor} ORDER BY label`;
+                    const rows = {};
+                    for (const { table_schema, table_name, label } of tables)
+                        rows[label] = (yield* sql `
+                    SELECT count(*)::integer AS count FROM ${sql(table_schema)}.${sql(table_name)}
+                    WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_id = ${ref.id}`)[0].count;
+                    const declared = internalActors.declaredBlobs(ref.actor);
+                    const stored = yield* sql `
+                  SELECT blob, count(DISTINCT name)::integer AS entries FROM actor_blobs
+                  WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant}
+                    AND actor_type = ${ref.actor} AND actor_id = ${ref.id}
+                  GROUP BY blob`;
+                    const blobs = Object.fromEntries(declared.map((name) => [name, 0]));
+                    for (const { blob, entries } of stored)
+                        blobs[blob] = entries;
+                    const inspection = {
+                        generation: generations[0]?.generation,
+                        state: Object.fromEntries(yield* Effect.forEach(state, Effect.fnUntraced(function* ({ key, value }) {
+                            return [
+                                key,
+                                yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Json))(value).pipe(Effect.orDie),
+                            ];
+                        }))),
+                        receipts: counted.receipts,
+                        events: counted.events,
+                        outbox: outbox[0].intents,
+                        jobs: outbox[0].jobs,
+                    };
+                    const withRows = tables.length > 0 ? { ...inspection, rows } : inspection;
+                    return declared.length > 0 ? { ...withRows, blobs } : withRows;
+                }, Effect.orDie),
+                seed: Effect.fnUntraced(function* (ref, state, version) {
+                    const key = yield* storedRoutingKey(ref);
+                    yield* sql `INSERT INTO actor_generations (routing_key, tenant_id, actor_type, actor_id)
+            VALUES (${key}, ${ref.tenant}, ${ref.actor}, ${ref.id}) ON CONFLICT DO NOTHING`;
+                    const rows = Object.entries(state).map(([name, value]) => [name, JSON.stringify(value)]);
+                    if (version > 0)
+                        rows.push([VERSION_KEY, String(version)]);
+                    for (const [name, value] of rows)
+                        yield* sql `INSERT INTO actor_state (routing_key, tenant_id, actor_type, actor_id, key, value)
+            VALUES (${key}, ${ref.tenant}, ${ref.actor}, ${ref.id}, ${name}, ${compress(value)})`;
+                }, Effect.orDie),
+                advance: Effect.fnUntraced(function* (duration) {
+                    const millis = Duration.toMillis(duration);
+                    if (!Number.isFinite(millis) || millis < 0)
+                        return yield* Effect.die(new Error("advance needs a finite, non-negative duration"));
+                    yield* internalActors.extendOutboxLeases(millis, Effect.sync(() => {
+                        clockOffset += millis;
+                    }));
+                    yield* internalActors.drainOutbox;
+                }),
+                now: databaseTime.pipe(Effect.map((millis) => DateTime.makeUnsafe(millis)), Effect.provideService(SqlClient.SqlClient, sql), Effect.provideService(FrameworkClock, { offsetMillis: () => clockOffset }), Effect.orDie),
+                cleanup: internalActors.cleanup,
+                receiptsFor: Effect.fnUntraced(function* (ref, command) {
+                    const routing = yield* storedRoutingKey(ref);
+                    const rows = yield* sql `
+                  SELECT count(*)::integer AS count FROM actor_receipts
+                  WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor}
+                    AND actor_id = ${ref.id} AND command = ${command}`;
+                    return rows[0].count;
+                }, Effect.orDie),
+                connect: Effect.fnUntraced(function* (ref, member, params) {
+                    const server = valueCodec(member.server);
+                    const decodeServer = Schema.decodeEffect(server);
+                    const encodeClient = Schema.encodeEffect(valueCodec(member.client));
+                    const decodeError = Schema.decodeEffect(Schema.fromJsonString(Schema.toCodecJson(member.error)));
+                    const encoded = yield* Schema.encodeEffect(valueCodec(member.payload))({
+                        value: params,
+                    }).pipe(Effect.orDie);
+                    const held = yield* internalActors.holder
+                        .open({
+                        ref,
+                        member: member.tag,
+                        caller: yield* CurrentCaller,
+                        params: encoded,
+                    })
+                        .pipe(Effect.catchTag("OpenRejected", (rejected) => Effect.flatMap(decodeError(rejected.value).pipe(Effect.orDie), (error) => Effect.fail(error))));
+                    const messages = held.messages.pipe(Stream.mapEffect((message) => ClientMessage.guards.Frame(message)
+                        ? Effect.map(decodeServer(message.frame).pipe(Effect.orDie), ({ value }) => ({
+                            ...message,
+                            frame: value,
+                        }))
+                        : ClientMessage.guards.Progress(message)
+                            ? Effect.map(decodeJson(message.frame).pipe(Effect.orDie), (frame) => ({
+                                ...message,
+                                frame,
+                            }))
+                            : Effect.succeed(message)));
+                    const connection = {
+                        connectionId: held.connectionId,
+                        cursor: held.cursor,
+                        send: (frame) => Effect.flatMap(encodeClient({ value: frame }).pipe(Effect.orDie), held.send),
+                        frames: messages.pipe(Stream.filter((message) => Predicate.isTagged(message, "Frame")), Stream.map((message) => message.frame)),
+                        messages,
+                        resyncDone: held.resyncDone,
+                        close: held.close,
+                    };
+                    return connection;
+                }),
+                hibernate: internalActors.hibernate,
+                invalidate: Effect.fnUntraced(function* (ref) {
+                    const routing = yield* storedRoutingKey(ref);
+                    yield* sql `UPDATE actor_generations SET generation = generation + 1
+            WHERE routing_key = ${routing} AND tenant_id = ${ref.tenant} AND actor_type = ${ref.actor} AND actor_id = ${ref.id}`;
+                }, Effect.orDie),
+                progress: Effect.sync(() => [...progress]),
+                resendProgress: internalActors.deliverProgress,
+                dropProgress: (predicate) => Effect.sync(() => {
+                    dropProgress = predicate;
+                }),
+                dropCommitted: (dropping) => Effect.sync(() => {
+                    dropCommitted = dropping;
+                }),
+            });
+            testActors.set(service, internalActors);
+            return service;
+        }));
+        const runtime = runtimeLayer({
+            authorize: options.authorize,
+            retryWindowMs: options.retryWindowMs,
+            maxResidentActors: options.maxResidentActors,
+            admission: options.admission,
+            relay: options.relay,
+            executors: options.executors,
+            observability: options.observability,
+            rowLevelSecurity: options.rowLevelSecurity,
+            adoption: options.adoption,
+            payloadWriterWindow: options.payloadWriterWindow,
+            content: options.content ?? { keys: [TEST_CONTENT_KEY] },
+            fleet: options.fleet,
+        });
+        return Layer.mergeAll(runtime, test.pipe(Layer.provide(runtime)), Layer.succeed(CurrentCaller, options.as ?? System.make({ source: "process" })), Layer.succeed(Tenant, tenant), Layer.succeed(FrameworkClock, { offsetMillis: () => clockOffset })).pipe(Layer.provide(hooks), Layer.provide(Layer.succeed(LocalRequestSerialization, true)), Layer.provideMerge(options.database !== undefined && Redacted.isRedacted(options.database)
+            ? Database.postgres({
+                url: options.database,
+                maxConnections: options.maxConnections ?? 10,
+                offTurnConnections: options.maxConnections,
+                stream: member?.connect,
+                replica: options.replica === undefined
+                    ? undefined
+                    : { url: options.replica, maxConnections: 4 },
+            })
+            : Database.pglite(options.database)));
+    }));
+}

package/dist/testing/cluster.d.ts

@@ -0,0 +1,93 @@
+import { Context, Crypto, Duration, Effect, Layer, Scope } from "effect";
+import { SqlClient } from "effect/sql";
+import { InternalActors } from "../runtime/actors.ts";
+import type { ActorRef } from "../identity/caller.ts";
+import { ActorTest, type TestOptions } from "./actor-test.ts";
+/** Services each runner provides to effects run with `cluster.on(runner)`. */
+export type RunnerServices = Layer.Success<ReturnType<typeof ActorTest.layer>>;
+export interface ClusterOptions<ROut, E, RIn> extends TestOptions {
+    /** Number of runners, at least 1. */
+    readonly runners: number;
+    /** How long a runner's shard locks outlive its last heartbeat. Whole seconds; Cluster rounds up. */
+    readonly shardLockExpiration: Duration.Input;
+    /** Application layers (`X.toLayer`, `X.toQueryLayer`, ...) that every runner builds. */
+    readonly actors: Layer.Layer<ROut, E, RIn>;
+    /** Layers only some runners build, such as an effect layer one runner lacks. */
+    readonly runnerActors?: (runner: number) => Layer.Layer<never, never, RunnerServices>;
+    /**
+     * Runners that hold connections but are never assigned actor shards, so an
+     * actor is always owned by another runner and killing its owner leaves the
+     * holder alive.
+     */
+    readonly holdersOnly?: ReadonlyArray<number>;
+}
+declare const ActorCluster_base: Context.ServiceClass<ActorCluster, "@rikalabs/akter/testing/cluster/ActorCluster", {
+    readonly runners: number;
+    /** The tenant every runner of this cluster runs its actors under. */
+    readonly tenant: string;
+    /**
+     * A connection pool of the cluster's own to the shared database, for
+     * inspection: `kill` never cuts it.
+     */
+    readonly sql: SqlClient.SqlClient;
+    /**
+     * Runs `effect` on `runner`: its handles dispatch through that runner, and
+     * its `ActorTest` fault points and inspection belong to that runner. An
+     * effect already running when its runner is killed is not interrupted; its
+     * calls fail or time out, so call through a surviving runner.
+     */
+    readonly on: (runner: number) => <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, Exclude<R, RunnerServices>>;
+    /**
+     * Stops `runner` the way a crash does: its connections close, rolling back
+     * open turns, and its shard locks and heartbeat are left to expire.
+     */
+    readonly kill: (runner: number) => Effect.Effect<void>;
+    /**
+     * Stops `runner` the way a graceful process exit does: its layer closes,
+     * so its activations end and it releases its shard locks for the others
+     * to take at once. Drain it first to finish or interrupt its work.
+     */
+    readonly shutdown: (runner: number) => Effect.Effect<void>;
+    /**
+     * Delays every database reply to every runner, without dropping it, as a
+     * partition between the database and its clients would: statements a turn
+     * sends still run, but it hears nothing back until `failover`. A COMMIT
+     * sent meanwhile is made by the database and unknown to its runner.
+     */
+    readonly holdReplies: Effect.Effect<void>;
+    /**
+     * Cuts every runner's open database connections at once, discarding held
+     * replies, and lets the runners reconnect to the same database, as they
+     * do when a primary fails over to a promoted standby at the same address:
+     * open turns lose their connection, and shard locks, which are table
+     * rows, survive.
+     */
+    readonly failover: Effect.Effect<void>;
+    /** Starts a stopped runner again as a new process, under a new address. */
+    readonly restart: (runner: number) => Effect.Effect<void>;
+    /**
+     * Stops `runner` writing heartbeats and lock refreshes while it keeps
+     * serving the shards it holds: other runners take its shards once its
+     * locks expire, and only the database fence stands between the two.
+     */
+    readonly pauseHeartbeat: (runner: number) => Effect.Effect<{
+        readonly resume: Effect.Effect<void>;
+    }>;
+    /**
+     * The runner holding an unexpired lock on the shard that places `ref`.
+     * While a runner's heartbeat is paused, it may still serve the actor too.
+     */
+    readonly owner: (ref: ActorRef) => Effect.Effect<number | undefined>;
+    /** Waits until every running runner holds exactly the shards assigned to it. */
+    readonly ready: Effect.Effect<void>;
+}>;
+export declare class ActorCluster extends ActorCluster_base {
+}
+/**
+ * Provides an `ActorCluster` of `options.runners` real runners sharing one Postgres database, each serving runner RPCs on its own listener.
+ * The first runner migrates the database and the rest start together. Runner rows carry the cluster's tenant, so rows another cluster left in the same database are never mistaken for this one's.
+ * `kill` cuts a runner's database and runner connections at once so Postgres rolls back its open turns; a graceful exit closes its layer first so it releases its locks.
+ * Fails to build when `database` is missing or is not a Postgres URL, since PGlite has a single connection.
+ */
+export declare const clusterLayer: <ROut, E, RIn>(options: ClusterOptions<ROut, E, RIn>) => Layer.Layer<ActorCluster, never, Crypto.Crypto | Exclude<Exclude<RIn, ActorTest | import("../index.ts").Actors | import("../index.ts").ContentStore | import("../runtime/index.ts").DefectLog | InternalActors | import("../runtime/operators/repair.ts").OperatorRuntime | import("@effect/sql-pglite/PgliteClient").PgliteClient | import("../runtime/drain.ts").RuntimeControl | SqlClient.SqlClient | import("../runtime/index.ts").TelemetrySampler>, Scope.Scope>>;
+export {};