@rikalabs/akter 0.1.0-alpha.0

package/dist/testing/conformance/subscriptions/failures.js

@@ -0,0 +1,108 @@
+import { Effect } from "effect";
+import { ActorTest } from "../../actor-test.js";
+import { CLAIM_LEASE } from "../outbox.js";
+import { SubFollower, SubOrder } from "./actors.js";
+import { drain, followerLog, handlerRuns, logOf, run, sourceRows } from "./harness.js";
+/** Blocking, ordering, and backoff of failing subscription deliveries. */
+export const subscriptionFailureConformance = [
+    {
+        name: "holds later events behind a failing one on the same row only, and interleaves two sources",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            const follower = yield* SubFollower.get("hold-follower");
+            yield* follower.Follow({ source: "hold-a" });
+            yield* follower.Follow({ source: "hold-b" });
+            yield* drain;
+            fixture.behave = (entry) => (entry.includes("/hold-a#1:") ? "defect" : "apply");
+            yield* (yield* SubOrder.get("hold-a")).PlaceMany({ customerId: "h", count: 2 });
+            yield* (yield* SubOrder.get("hold-b")).PlaceMany({ customerId: "h", count: 2 });
+            yield* drain;
+            expect(yield* followerLog("hold-follower")).toEqual([
+                "hold-b#1:OrderPlaced",
+                "hold-b#2:OrderPlaced",
+            ]);
+            const [held] = yield* sourceRows("hold-a");
+            expect(held).toMatchObject({ delivered: "0", attempts: 1 });
+            expect(held.last_error?.includes("Handler defect")).toBe(true);
+            fixture.behave = () => "apply";
+            yield* test.advance(CLAIM_LEASE);
+            expect(yield* followerLog("hold-follower")).toEqual([
+                "hold-b#1:OrderPlaced",
+                "hold-b#2:OrderPlaced",
+                "hold-a#1:OrderPlaced",
+                "hold-a#2:OrderPlaced",
+            ]);
+        })),
+    },
+    {
+        name: "holds a routed source's later events for every routed subscriber while one is blocked, and lets other subscriptions of the same source proceed",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            const order = yield* SubOrder.get("blocked-order");
+            fixture.behave = (entry) => entry.startsWith("SubSummary/blocked-first/") ? "defect" : "apply";
+            yield* order.Place({ customerId: "blocked-first", amount: 1 });
+            yield* order.Place({ customerId: "blocked-second", amount: 1 });
+            yield* drain;
+            expect(yield* logOf("SubSummary", "blocked-first")).toEqual([]);
+            expect(yield* logOf("SubSummary", "blocked-second")).toEqual([]);
+            expect(yield* logOf("SubAuditor", "blocked-first")).toEqual([
+                "blocked-order#1:OrderPlaced",
+            ]);
+            expect(yield* logOf("SubAuditor", "blocked-second")).toEqual([
+                "blocked-order#2:OrderPlaced",
+            ]);
+            const [summary] = yield* sourceRows("blocked-order", "SubSummary");
+            expect(summary.attempts).toBe(1);
+            expect(summary.last_error?.includes("Handler defect")).toBe(true);
+            fixture.behave = () => "apply";
+            yield* test.advance(CLAIM_LEASE);
+            expect(yield* logOf("SubSummary", "blocked-first")).toEqual([
+                "blocked-order#1:OrderPlaced",
+            ]);
+            expect(yield* logOf("SubSummary", "blocked-second")).toEqual([
+                "blocked-order#2:OrderPlaced",
+            ]);
+        })),
+    },
+    {
+        name: "retries a defecting delivery with capped backoff and records last_error",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            yield* (yield* SubFollower.get("poison-follower")).Follow({ source: "poison-order" });
+            yield* drain;
+            fixture.behave = (entry) => entry.startsWith("SubFollower/poison-follower/") ? "defect" : "apply";
+            yield* (yield* SubOrder.get("poison-order")).Place({ customerId: "p", amount: 1 });
+            yield* drain;
+            for (let attempt = 1; attempt <= 3; attempt++) {
+                const [row] = yield* sourceRows("poison-order");
+                expect(row).toMatchObject({ attempts: attempt, delivered: "0", due: true });
+                expect(row.last_error?.includes("Handler defect on poison-order#1")).toBe(true);
+                yield* test.advance(CLAIM_LEASE);
+            }
+            expect(handlerRuns(fixture, "SubFollower/poison-follower")).toBe(4);
+            expect(yield* followerLog("poison-follower")).toEqual([]);
+            fixture.behave = () => "apply";
+            yield* test.advance("300 seconds");
+            expect(yield* followerLog("poison-follower")).toEqual(["poison-order#1:OrderPlaced"]);
+            expect(yield* sourceRows("poison-order")).toMatchObject([
+                { attempts: 0, last_error: null, delivered: "1" },
+            ]);
+        })),
+    },
+    {
+        name: "backs a routed row off with last_error when route returns an id the key schema rejects",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            const order = yield* SubOrder.get("badroute-order");
+            yield* order.Place({ customerId: "", amount: 1 });
+            yield* order.Place({ customerId: "badroute-ok", amount: 1 });
+            yield* drain;
+            const [summary] = yield* sourceRows("badroute-order", "SubSummary");
+            expect(summary.delivered).toBe("0");
+            expect(summary.last_error?.startsWith("Route failed")).toBe(true);
+            expect(yield* logOf("SubSummary", "badroute-ok")).toEqual([]);
+            yield* test.advance(CLAIM_LEASE);
+            expect(yield* logOf("SubSummary", "badroute-ok")).toEqual([]);
+        })),
+    },
+];

package/dist/testing/conformance/subscriptions/feeds.d.ts

@@ -0,0 +1,4 @@
+import type { ConformanceCase } from "../../conformance.ts";
+import { type SubscriptionsFixture } from "./actors.ts";
+/** Feed rows, the tag summary, and wake-ups of event subscriptions. */
+export declare const subscriptionFeedConformance: ReadonlyArray<ConformanceCase<SubscriptionsFixture>>;

package/dist/testing/conformance/subscriptions/feeds.js

@@ -0,0 +1,218 @@
+import { Deferred, Effect, Fiber, Schema } from "effect";
+import { SqlClient } from "effect/sql";
+import { ActorTest } from "../../actor-test.js";
+import { CLAIM_LEASE, ExplainOutput, planNodes } from "../outbox.js";
+import { SubFollower, SubOrder } from "./actors.js";
+import { crashOnce, drain, followerLog, handlerRuns, outboxOf, query, run, sourceRows, tagMismatches, } from "./harness.js";
+/** Feed rows, the tag summary, and wake-ups of event subscriptions. */
+export const subscriptionFeedConformance = [
+    {
+        name: "reactivates a subscription over its tombstone, so the source's next event is delivered",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const follower = yield* SubFollower.get("revive-follower");
+            yield* follower.Follow({ source: "revive-order" });
+            yield* drain;
+            yield* follower.Unfollow("revive-order");
+            yield* drain;
+            expect(yield* sourceRows("revive-order")).toMatchObject([
+                { subscriber_id: "revive-follower", epoch: "2", active: false, due: false },
+            ]);
+            expect(yield* tagMismatches).toEqual([]);
+            yield* follower.Follow({ source: "revive-order" });
+            yield* drain;
+            yield* (yield* SubOrder.get("revive-order")).Place({ customerId: "r", amount: 1 });
+            yield* drain;
+            expect(yield* followerLog("revive-follower")).toEqual(["revive-order#1:OrderPlaced"]);
+            expect(yield* sourceRows("revive-order")).toMatchObject([
+                { subscriber_id: "revive-follower", epoch: "3", active: true, delivered: "1" },
+            ]);
+            expect(yield* tagMismatches).toEqual([]);
+        })),
+    },
+    {
+        name: "tombstones a caught-up row and delivers Rejected when a resubscribe names a cursor above the source's head",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const follower = yield* SubFollower.get("above-follower");
+            const order = yield* SubOrder.get("above-order");
+            yield* follower.Follow({ source: "above-order" });
+            yield* drain;
+            yield* order.Place({ customerId: "a", amount: 1 });
+            yield* drain;
+            expect(yield* sourceRows("above-order")).toMatchObject([
+                { epoch: "1", active: true, delivered: "1", due: false },
+            ]);
+            yield* follower.Follow({ source: "above-order", from: "9" });
+            yield* drain;
+            yield* order.Place({ customerId: "a", amount: 2 });
+            yield* drain;
+            expect(yield* followerLog("above-follower")).toEqual([
+                "above-order#1:OrderPlaced",
+                "above-order!rejected:9",
+            ]);
+            expect(yield* sourceRows("above-order")).toMatchObject([
+                { subscriber_id: "above-follower", epoch: "2", active: false, due: false },
+            ]);
+            expect(yield* tagMismatches).toEqual([]);
+        })),
+    },
+    {
+        name: "expands a feed row once its lease ends when the relay dies after claiming it",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            yield* (yield* SubFollower.get("feeddie-follower")).Follow({ source: "feeddie-order" });
+            yield* drain;
+            const crash = crashOnce(fixture, "afterClaim", (request) => request.command === "$feed" && request.ref.id === "feeddie-order");
+            yield* (yield* SubOrder.get("feeddie-order")).Place({ customerId: "f", amount: 1 });
+            yield* drain;
+            expect(crash.crashed).toBe(true);
+            expect(yield* followerLog("feeddie-follower")).toEqual([]);
+            expect(yield* outboxOf("SubOrder", "feeddie-order", "feed")).toBe(1);
+            yield* test.advance(CLAIM_LEASE);
+            expect(yield* followerLog("feeddie-follower")).toEqual(["feeddie-order#1:OrderPlaced"]);
+            expect(handlerRuns(fixture, "SubFollower/feeddie-follower")).toBe(1);
+            expect(yield* outboxOf("SubOrder", "feeddie-order", "feed")).toBe(0);
+        })),
+    },
+    {
+        name: "writes one feed row per publishing turn whatever the subscriber count",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            for (const [source, followers] of [
+                ["feed-one", 1],
+                ["feed-many", 12],
+            ]) {
+                for (let index = 0; index < followers; index++)
+                    yield* (yield* SubFollower.get(`${source}-f${index}`)).Follow({ source });
+                yield* drain;
+                const pause = yield* test.pauseNext("afterClaim");
+                yield* (yield* SubOrder.get(source)).PlaceMany({ customerId: "f", count: 3 });
+                const draining = yield* drain.pipe(Effect.forkChild);
+                yield* pause.reached;
+                expect(yield* outboxOf("SubOrder", source, "feed")).toBe(1);
+                expect((yield* test.inspect({ tenant: test.tenant, actor: "SubOrder", id: source })).outbox).toBe(0);
+                yield* pause.release;
+                yield* Fiber.join(draining);
+                expect(yield* outboxOf("SubOrder", source, "feed")).toBe(0);
+                for (let index = 0; index < followers; index++)
+                    expect((yield* followerLog(`${source}-f${index}`)).length).toBe(3);
+            }
+        })),
+    },
+    {
+        name: "writes no feed row for an actor with no subscriptions",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            yield* (yield* SubOrder.get("quiet-order")).Note("nobody listens");
+            expect(yield* outboxOf("SubOrder", "quiet-order", "feed")).toBe(0);
+            expect(yield* sourceRows("quiet-order")).toEqual([]);
+        })),
+    },
+    {
+        name: "probes the tag summary by key with 10^5 non-matching rows",
+        requiresIndependentConnections: true,
+        timeoutMs: 120_000,
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            const sql = yield* SqlClient.SqlClient;
+            const routingKey = "1";
+            yield* sql `INSERT INTO actor_generations (routing_key, tenant_id, actor_type, actor_id)
+            SELECT ${routingKey}::bigint, ${test.tenant}, 'SubProbe', 'p' || n
+            FROM generate_series(1, 100000) AS n`.pipe(Effect.orDie);
+            yield* sql `INSERT INTO actor_subscription_tags (routing_key, tenant_id, source_type, source_id, event, rows)
+            SELECT ${routingKey}::bigint, ${test.tenant}, 'SubProbe', 'p' || n, 'OrderPlaced', 1
+            FROM generate_series(1, 100000) AS n`.pipe(Effect.orDie);
+            yield* sql `ANALYZE actor_subscription_tags`.pipe(Effect.orDie);
+            const [row] = yield* sql `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
+            SELECT 1 FROM actor_subscription_tags t
+            WHERE t.routing_key = ${routingKey}::bigint AND t.tenant_id = ${test.tenant}
+              AND t.source_type = 'SubProbe' AND t.source_id = 'missing' AND t.event IN ('OrderPlaced', 'OrderCancelled')`.pipe(Effect.orDie);
+            const [explained] = yield* Schema.decodeUnknownEffect(ExplainOutput)(row["QUERY PLAN"]).pipe(Effect.orDie);
+            const nodes = planNodes(explained.Plan);
+            expect(nodes.some((node) => node["Index Name"] === "actor_subscription_tags_pkey")).toBe(true);
+            expect(nodes.some((node) => node["Node Type"] === "Seq Scan")).toBe(false);
+            expect(explained.Plan["Actual Rows"]).toBe(0);
+            expect(nodes.reduce((sum, node) => sum + node["Shared Hit Blocks"] + node["Shared Read Blocks"], 0) < 16).toBe(true);
+            yield* sql `DELETE FROM actor_subscription_tags WHERE source_type = 'SubProbe'`.pipe(Effect.orDie);
+            yield* sql `DELETE FROM actor_generations WHERE actor_type = 'SubProbe'`.pipe(Effect.orDie);
+        })),
+    },
+    {
+        name: "leases the rows an expansion makes due and starts their delivery without a claim pass",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            yield* (yield* SubFollower.get("lease-follower")).Follow({ source: "lease-order" });
+            yield* drain;
+            const order = [];
+            const claimed = yield* Deferred.make();
+            fixture.hook = (point, request) => {
+                if (point === "afterClaim" && request.commandId === "lease-follower") {
+                    order.push("delivery");
+                    return Deferred.succeed(claimed, undefined).pipe(Effect.asVoid);
+                }
+                if (point === "afterExpand" && request.ref.id === "lease-order")
+                    return Deferred.await(claimed).pipe(Effect.timeout("2 seconds"), Effect.ignore, Effect.andThen(Effect.sync(() => order.push("expanded"))));
+                return Effect.void;
+            };
+            yield* (yield* SubOrder.get("lease-order")).Place({ customerId: "l", amount: 1 });
+            yield* drain;
+            expect(order).toEqual(["delivery", "expanded"]);
+            expect(yield* followerLog("lease-follower")).toEqual(["lease-order#1:OrderPlaced"]);
+            expect(yield* sourceRows("lease-order")).toMatchObject([
+                { delivered: "1", due: false, attempts: 0 },
+            ]);
+        })),
+    },
+    {
+        name: "loses no wake when a commit races a settle or an expansion",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            const order = yield* SubOrder.get("race-order");
+            yield* (yield* SubFollower.get("race-follower")).Follow({ source: "race-order" });
+            yield* drain;
+            const pause = yield* test.pauseNext("afterSettleSnapshot");
+            yield* order.Place({ customerId: "r", amount: 1 });
+            const draining = yield* drain.pipe(Effect.forkChild);
+            yield* pause.reached;
+            yield* order.Place({ customerId: "r", amount: 2 });
+            for (;;) {
+                if ((yield* outboxOf("SubOrder", "race-order", "feed")) === 0)
+                    break;
+                yield* Effect.sleep("20 millis");
+            }
+            yield* pause.release;
+            yield* Fiber.join(draining);
+            yield* drain;
+            expect(yield* followerLog("race-follower")).toEqual([
+                "race-order#1:OrderPlaced",
+                "race-order#2:OrderPlaced",
+            ]);
+            expect(yield* sourceRows("race-order")).toMatchObject([{ delivered: "2", due: false }]);
+        })),
+    },
+    {
+        name: "keeps a backing-off row's due time when new commits expand the feed",
+        run: ({ expect, environment, fixture }) => run(environment, fixture, Effect.gen(function* () {
+            const test = yield* ActorTest;
+            const order = yield* SubOrder.get("backoff-order");
+            yield* (yield* SubFollower.get("backoff-follower")).Follow({ source: "backoff-order" });
+            yield* drain;
+            fixture.behave = (entry) => entry.startsWith("SubFollower/backoff-follower/") ? "defect" : "apply";
+            yield* order.Place({ customerId: "b", amount: 1 });
+            yield* drain;
+            const due = () => query((sql) => sql `SELECT due_at_ms::text AS due_at_ms
+                FROM actor_subscriptions WHERE tenant_id = ${test.tenant}
+                  AND source_id = 'backoff-order' AND subscriber_id = 'backoff-follower'`).pipe(Effect.map((rows) => rows[0].due_at_ms));
+            const backingOff = yield* due();
+            yield* order.Place({ customerId: "b", amount: 2 });
+            yield* drain;
+            expect(yield* due()).toBe(backingOff);
+            const [backedOff] = yield* sourceRows("backoff-order");
+            expect(backedOff).toMatchObject({ delivered: "0", attempts: 1 });
+            expect(backedOff.last_error?.includes("Handler defect")).toBe(true);
+            fixture.behave = () => "apply";
+            yield* test.advance(CLAIM_LEASE);
+            expect(yield* followerLog("backoff-follower")).toEqual([
+                "backoff-order#1:OrderPlaced",
+                "backoff-order#2:OrderPlaced",
+            ]);
+        })),
+    },
+];

package/dist/testing/conformance/subscriptions/harness.d.ts

@@ -0,0 +1,124 @@
+import { Effect, Schema } from "effect";
+import { SqlClient, type SqlError } from "effect/sql";
+import { InternalActors } from "../../../runtime/actors.ts";
+import { Request } from "../../../runtime/request.ts";
+import { type TurnPoint } from "../../../runtime/turn/hooks.ts";
+import { ActorTest } from "../../actor-test.ts";
+import type { ConformanceEnvironment, ConformanceServices } from "../../conformance.ts";
+import { type SubscriptionsFixture } from "./actors.ts";
+export declare const LogState: Schema.Struct<{
+    readonly log: Schema.optional<Schema.$Array<Schema.String>>;
+}>;
+export declare const logOf: (actor: string, id: string) => Effect.Effect<readonly string[], never, ActorTest>;
+export declare const query: <A>(statement: (sql: SqlClient.SqlClient) => Effect.Effect<A, SqlError.SqlError>) => Effect.Effect<A, never, SqlClient.SqlClient>;
+/** The source-side rows of one order in the test tenant, of one subscriber type. */
+export declare const sourceRows: (sourceId: string, subscriberType?: string) => Effect.Effect<readonly {
+    subscriber_type: string;
+    subscription: string;
+    subscriber_id: string;
+    epoch: string;
+    active: boolean;
+    delivered: string;
+    due: boolean;
+    attempts: number;
+    last_error: string | null;
+}[], never, ActorTest | SqlClient.SqlClient>;
+/** The subscriber-side cursor rows of one follower in the test tenant. */
+export declare const cursorRows: (actor: string, id: string) => Effect.Effect<readonly {
+    subscription: string;
+    source_id: string;
+    epoch: string;
+    active: boolean;
+    applied: string;
+}[], never, ActorTest | SqlClient.SqlClient>;
+/** Outbox rows of one kind an actor of the test tenant holds. */
+export declare const outboxOf: (actor: string, id: string, kind: "feed" | "control") => Effect.Effect<number, never, ActorTest | SqlClient.SqlClient>;
+/**
+ * Tag summary rows that disagree with the active rows carrying each tag. A
+ * missed increment loses wakes, and a missed decrement costs feed writes forever.
+ */
+export declare const tagMismatches: Effect.Effect<readonly {
+    event: string;
+    counted: number | null;
+    summary: number | null;
+}[], never, SqlClient.SqlClient>;
+export declare const drain: Effect.Effect<void, never, ActorTest>;
+/** Makes the first `point` that `match` accepts die, as a crash there would; later ones pass. */
+export declare const crashOnce: (fixture: SubscriptionsFixture, point: TurnPoint, match: (request: Request) => boolean) => {
+    crashed: boolean;
+};
+/** Holds the first `point` that `match` accepts until `release`. */
+export declare const pauseOnce: (fixture: SubscriptionsFixture, point: TurnPoint, match: (request: Request) => boolean) => Effect.Effect<{
+    reached: Effect.Effect<void, never, never>;
+    release: Effect.Effect<void, never, never>;
+}, never, never>;
+/** A fault point of one subscription row's work: its tag and its subscriber, or the delivery command. */
+export declare const subscriber: (id: string) => (request: Request) => boolean;
+/** A follower's committed log, as entries of one source. */
+export declare const followerLog: (id: string) => Effect.Effect<readonly string[], never, ActorTest>;
+export declare const handlerRuns: (fixture: SubscriptionsFixture, prefix: string) => number;
+/** Builds a relay delivery request, as a runner would, for cases that play a stale runner. */
+export declare const deliveryRequest: (options: {
+    readonly subscriber: string;
+    readonly source: string;
+    readonly epoch: string;
+    readonly cursor: string;
+    /** Re-encodes the delivery with a field a newer schema added. */
+    readonly extra?: boolean;
+    /** The subscriber type, its subscription, its handler, and the source type; `SubFollower`'s `FollowedOrders` of `SubOrder` by default. */
+    readonly route?: {
+        readonly actor: string;
+        readonly subscription: string;
+        readonly command: string;
+        readonly source: string;
+    };
+}) => Effect.Effect<{
+    readonly ref: {
+        readonly tenant: string;
+        readonly actor: string;
+        readonly id: string;
+    };
+    readonly caller: {
+        readonly _tag: "Anonymous";
+    } | {
+        readonly _tag: "System";
+        readonly source: "actor" | "cron" | "job" | "process" | "subscription" | "timer" | "workflow";
+        readonly ref?: {
+            readonly tenant: string;
+            readonly actor: string;
+            readonly id: string;
+        } | undefined;
+        readonly onBehalfOf?: {
+            readonly subject: string;
+        } | undefined;
+        readonly mint?: {
+            readonly commandId: string;
+            readonly ordinal: number;
+        } | undefined;
+    } | {
+        readonly _tag: "User";
+        readonly subject: string;
+    };
+    readonly command: string;
+    readonly commandId: string;
+    readonly payload: string;
+    readonly external?: boolean | undefined;
+    readonly clockOffset?: number | undefined;
+    readonly redelivered?: boolean | undefined;
+    readonly intent?: {
+        readonly tenant: string;
+        readonly actor: string;
+        readonly id: string;
+    } | undefined;
+    readonly delivery?: {
+        readonly subscription: string;
+        readonly sourceType: string;
+        readonly sourceId: string;
+        readonly epoch: string;
+        readonly kind: "event" | "gap" | "rejected";
+        readonly position: string;
+    } | undefined;
+    readonly queuedAtMs?: number | undefined;
+    readonly usageToken?: string | undefined;
+}, Schema.SchemaError, ActorTest | InternalActors | SqlClient.SqlClient>;
+export declare const run: <A, E>(environment: ConformanceEnvironment, fixture: SubscriptionsFixture, body: Effect.Effect<A, E, ConformanceServices>) => Promise<A>;

package/dist/testing/conformance/subscriptions/harness.js

@@ -0,0 +1,143 @@
+import { DateTime, Deferred, Effect, Schema } from "effect";
+import { SqlClient } from "effect/sql";
+import { InternalActors } from "../../../runtime/actors.js";
+import { Request } from "../../../runtime/request.js";
+import { System } from "../../../identity/caller.js";
+import { deliveryCommandId } from "../../../runtime/subscriptions/identity.js";
+import { RetryTurn } from "../../../runtime/turn/hooks.js";
+import { ActorTest } from "../../actor-test.js";
+import { OrderDelivery, OrderPlaced, reset } from "./actors.js";
+export const LogState = Schema.Struct({ log: Schema.optional(Schema.Array(Schema.String)) });
+export const logOf = Effect.fnUntraced(function* (actor, id) {
+    const test = yield* ActorTest;
+    const { state } = yield* test.inspect({ tenant: test.tenant, actor, id });
+    return (yield* Schema.decodeUnknownEffect(LogState)(state).pipe(Effect.orDie)).log ?? [];
+});
+export const query = (statement) => Effect.gen(function* () {
+    return yield* statement(yield* SqlClient.SqlClient);
+}).pipe(Effect.orDie);
+/** The source-side rows of one order in the test tenant, of one subscriber type. */
+export const sourceRows = (sourceId, subscriberType = "SubFollower") => Effect.gen(function* () {
+    const test = yield* ActorTest;
+    return yield* query((sql) => sql `SELECT subscriber_type, subscription, subscriber_id, epoch::text AS epoch, active,
+          delivered::text AS delivered, due_at_ms IS NOT NULL AS due, attempts, last_error
+        FROM actor_subscriptions WHERE tenant_id = ${test.tenant} AND source_type = 'SubOrder'
+          AND source_id = ${sourceId} AND subscriber_type = ${subscriberType}
+        ORDER BY subscription, subscriber_id`);
+});
+/** The subscriber-side cursor rows of one follower in the test tenant. */
+export const cursorRows = (actor, id) => Effect.gen(function* () {
+    const test = yield* ActorTest;
+    return yield* query((sql) => sql `SELECT subscription, source_id, epoch::text AS epoch, active, applied::text AS applied
+        FROM actor_subscription_cursors WHERE tenant_id = ${test.tenant} AND actor_type = ${actor}
+          AND actor_id = ${id}
+        ORDER BY subscription, source_id`);
+});
+/** Outbox rows of one kind an actor of the test tenant holds. */
+export const outboxOf = (actor, id, kind) => Effect.gen(function* () {
+    const test = yield* ActorTest;
+    return (yield* query((sql) => sql `SELECT count(*)::int AS count FROM actor_outbox
+        WHERE tenant_id = ${test.tenant} AND actor_type = ${actor} AND actor_id = ${id}
+          AND kind = ${kind}`))[0].count;
+});
+/**
+ * Tag summary rows that disagree with the active rows carrying each tag. A
+ * missed increment loses wakes, and a missed decrement costs feed writes forever.
+ */
+export const tagMismatches = query((sql) => sql `
+    SELECT event, a.rows AS counted, t.rows AS summary FROM (
+      SELECT routing_key, tenant_id, source_type, source_id, x.tag AS event, count(*)::int AS rows
+      FROM actor_subscriptions, unnest(events) AS x(tag)
+      WHERE active GROUP BY routing_key, tenant_id, source_type, source_id, x.tag
+    ) a FULL JOIN actor_subscription_tags t USING (routing_key, tenant_id, source_type, source_id, event)
+    WHERE a.rows IS DISTINCT FROM t.rows`);
+export const drain = ActorTest.use((test) => test.advance(0));
+/** Makes the first `point` that `match` accepts die, as a crash there would; later ones pass. */
+export const crashOnce = (fixture, point, match) => {
+    const state = { crashed: false };
+    fixture.hook = (reached, request) => {
+        if (state.crashed || reached !== point || !match(request))
+            return Effect.void;
+        state.crashed = true;
+        return Effect.die(RetryTurn.make({ message: `Injected ${point} crash` }));
+    };
+    return state;
+};
+/** Holds the first `point` that `match` accepts until `release`. */
+export const pauseOnce = Effect.fnUntraced(function* (fixture, point, match) {
+    const reached = yield* Deferred.make();
+    const release = yield* Deferred.make();
+    let paused = false;
+    fixture.hook = (at, request) => {
+        if (paused || at !== point || !match(request))
+            return Effect.void;
+        paused = true;
+        return Deferred.succeed(reached, undefined).pipe(Effect.andThen(Deferred.await(release)));
+    };
+    return {
+        reached: Deferred.await(reached),
+        release: Deferred.succeed(release, undefined).pipe(Effect.asVoid),
+    };
+});
+/** A fault point of one subscription row's work: its tag and its subscriber, or the delivery command. */
+export const subscriber = (id) => (request) => request.commandId === id || request.ref.id === id;
+/** A follower's committed log, as entries of one source. */
+export const followerLog = (id) => logOf("SubFollower", id);
+export const handlerRuns = (fixture, prefix) => fixture.runs.filter((run) => run.startsWith(prefix)).length;
+const DeliveredOrder = OrderDelivery.members[0];
+const encodeDelivery = Schema.encodeEffect(Schema.fromJsonString(Schema.toCodecJson(Schema.Struct({ value: OrderDelivery }))));
+const encodeWithExtra = Schema.encodeEffect(Schema.fromJsonString(Schema.toCodecJson(Schema.Struct({ value: OrderDelivery, extra: Schema.Boolean }))));
+/** Builds a relay delivery request, as a runner would, for cases that play a stale runner. */
+export const deliveryRequest = Effect.fnUntraced(function* (options) {
+    const test = yield* ActorTest;
+    const internal = yield* InternalActors;
+    const route = options.route ?? {
+        actor: "SubFollower",
+        subscription: "FollowedOrders",
+        command: "OnOrder",
+        source: "SubOrder",
+    };
+    const subscriber = { tenant: test.tenant, actor: route.actor, id: options.subscriber };
+    const source = { tenant: test.tenant, actor: route.source, id: options.source };
+    const envelope = {
+        subscription: route.subscription,
+        sourceType: route.source,
+        sourceId: options.source,
+        epoch: options.epoch,
+        kind: "event",
+        position: options.cursor,
+    };
+    const [event] = yield* query((sql) => sql `
+      SELECT emitted_at_ms::text AS emitted_at_ms, command_id FROM actor_events
+      WHERE tenant_id = ${test.tenant} AND actor_type = ${route.source} AND actor_id = ${options.source}
+        AND sequence = ${options.cursor}`);
+    const value = DeliveredOrder.make({
+        subscription: route.subscription,
+        source,
+        cursor: options.cursor,
+        event: OrderPlaced.make({ customerId: "c", amount: 1 }),
+        commandId: event.command_id,
+        timestamp: DateTime.makeUnsafe(Number(event.emitted_at_ms)),
+    });
+    return Request.make({
+        ref: subscriber,
+        caller: System.make({ source: "subscription", ref: source }),
+        command: route.command,
+        commandId: yield* deliveryCommandId({
+            subscriber,
+            envelope,
+            issuedAt: Number(event.emitted_at_ms),
+            retryWindowMs: internal.retryWindowMs,
+        }),
+        payload: options.extra === true
+            ? yield* encodeWithExtra({ value, extra: true })
+            : yield* encodeDelivery({ value }),
+        delivery: envelope,
+    });
+});
+export const run = (environment, fixture, body) => environment.run(Effect.gen(function* () {
+    yield* reset(fixture);
+    const result = yield* body;
+    yield* reset(fixture);
+    return result;
+}));

package/dist/testing/conformance/subscriptions/operator.d.ts

@@ -0,0 +1,4 @@
+import type { ConformanceCase } from "../../conformance.ts";
+import { type SubscriptionsFixture } from "./actors.ts";
+/** Operator skip and listing of stuck subscription rows. */
+export declare const subscriptionOperatorConformance: ReadonlyArray<ConformanceCase<SubscriptionsFixture>>;