@rikalabs/akter 0.1.0-alpha.0

package/dist/runtime/runner.js

@@ -0,0 +1,122 @@
+import { Context, Duration, Effect, HashRing, Layer, Option, PrimaryKey } from "effect";
+import { MessageStorage, RunnerAddress, RunnerHealth, Runners, RunnerServer, RunnerStorage, Sharding, ShardingConfig, ShardId, } from "effect/cluster";
+import { RpcSerialization, RpcServer } from "effect/rpc";
+import { SocketServer } from "effect/socket";
+import { SqlClient } from "effect/sql";
+import { identity } from "./peering/credentials.js";
+import { mtls } from "./peering/transport.js";
+import { admissionSharding, MailboxRefusals } from "./topology/admission.js";
+/**
+ * Whether a command to an entity on this runner is encoded and decoded the
+ * way a remote delivery is. Off by default: a local hop hands the owner the
+ * request the runtime built and validated, and only a remote hop needs the
+ * bytes. Test runtimes turn it on so every delivery still checks the request
+ * schema.
+ */
+export const LocalRequestSerialization = Context.Reference("@rikalabs/akter/runtime/runner/LocalRequestSerialization", { defaultValue: () => false });
+/** Package-internal wiring shared by production runners and fault-injecting test runners. */
+export class RunnerWiring extends Context.Service()("@rikalabs/akter/runtime/runner/RunnerWiring") {
+}
+/** The public runner's readiness checks acquired shards, not just a reachable database. */
+export class RunnerReadiness extends Context.Service()("@rikalabs/akter/runtime/runner/RunnerReadiness") {
+}
+/** Builds assignment expectations independently of the runner's local acquisition state. */
+export const acquiredShards = Effect.gen(function* () {
+    const config = yield* ShardingConfig.ShardingConfig;
+    const storage = yield* RunnerStorage.RunnerStorage;
+    const self = Option.getOrThrow(config.runnerAddress);
+    const key = PrimaryKey.value(self);
+    return RunnerReadiness.of({
+        acquired: (sharding) => Effect.gen(function* () {
+            const runners = yield* storage.getRunners;
+            if (!runners.some(([runner, healthy]) => healthy && PrimaryKey.value(runner.address) === key))
+                return false;
+            for (const group of config.assignedShardGroups) {
+                const ring = HashRing.make();
+                for (const [runner, healthy] of runners)
+                    if (healthy && runner.groups.includes(group))
+                        HashRing.add(ring, runner.address, { weight: runner.weight });
+                const expected = HashRing.getShards(ring, config.shardsPerGroup);
+                if (expected === undefined)
+                    return false;
+                for (const [index, address] of expected.entries())
+                    if (PrimaryKey.value(address) === key &&
+                        !sharding.hasShardId(ShardId.make(group, index + 1)))
+                        return false;
+            }
+            return true;
+        }).pipe(Effect.timeoutOption("2 seconds"), Effect.map(Option.getOrElse(() => false)), Effect.orElseSucceed(() => false)),
+    });
+});
+const address = (value, advertise) => {
+    if (value.host.trim().length === 0 ||
+        (advertise && ["0.0.0.0", "::", "*"].includes(value.host)) ||
+        !Number.isInteger(value.port) ||
+        value.port < 1 ||
+        value.port > 65535)
+        throw new Error("Runner address needs a host and a port between 1 and 65535; advertise a reachable host, not a wildcard");
+    return RunnerAddress.RunnerAddress.make(value);
+};
+const duration = (value, name, minimum = 1) => {
+    const result = Duration.fromInputUnsafe(value);
+    if (!Number.isFinite(Duration.toMillis(result)) || Duration.toMillis(result) < minimum)
+        throw new Error(`${name} must be finite and at least ${minimum} milliseconds`);
+    return result;
+};
+/**
+ * Joins separate processes over TCP sockets. Provide this layer to
+ * `Actors.layer`; the framework keeps direct commands, SQL ownership, receipts,
+ * and outbox recovery, rather than enabling Cluster's persisted message store.
+ * A peer can deliver any runner message, so use `Runner.mtls` as the
+ * transport unless the network is isolated to this deployment's runners. The
+ * transport's own services stay in the runtime, so readiness sees whether a
+ * mutual TLS transport can still peer.
+ */
+export const socket = (options) => {
+    const shardsPerGroup = options.shardsPerGroup ?? 256;
+    if (!Number.isSafeInteger(shardsPerGroup) || shardsPerGroup < 1 || shardsPerGroup > 65536)
+        throw new Error("shardsPerGroup must be an integer between 1 and 65536");
+    const config = {
+        runnerAddress: Option.some(address(options.address, true)),
+        runnerListenAddress: Option.some(address(options.listenAddress ?? options.address, false)),
+        shardsPerGroup,
+        shardLockDisableAdvisory: true,
+        shardLockExpiration: duration(options.shardLockExpiration ?? "35 seconds", "shardLockExpiration", 3000),
+        shardLockRefreshInterval: duration(options.shardLockRefreshInterval ?? "10 seconds", "shardLockRefreshInterval"),
+        refreshAssignmentsInterval: duration(options.refreshAssignmentsInterval ?? "1 second", "refreshAssignmentsInterval"),
+        entityMessagePollInterval: "1 second",
+        entityTerminationTimeout: duration(options.entityTerminationTimeout ?? "15 seconds", "entityTerminationTimeout"),
+    };
+    const sharding = RunnerServer.layer.pipe(Layer.provideMerge(admissionSharding), Layer.provideMerge(Runners.layerRpc), Layer.provide(RpcServer.layerProtocolSocketServer), Layer.provideMerge(options.transport), Layer.provide(RpcSerialization.layerNdjson), Layer.orDie);
+    return Layer.succeed(RunnerWiring, {
+        config,
+        sharding,
+        storage: (storage) => storage,
+        production: true,
+    });
+};
+/**
+ * Runner-to-runner transport and ownership configuration, provided to
+ * `Actors.layer`: `socket` wires a runner, `mtls` is its authenticated
+ * transport, and `identity` is the URI subject alternative name a
+ * deployment's runner certificates carry.
+ */
+export const Runner = { socket, mtls, identity };
+/** Refuses a routing layout mismatch before any runner registers or takes shards. */
+export const checkRunnerConfiguration = Effect.fnUntraced(function* (wiring) {
+    const sql = yield* SqlClient.SqlClient;
+    if (wiring?.production === true) {
+        const shards = wiring.config.shardsPerGroup;
+        const expiration = Duration.toMillis(Duration.fromInputUnsafe(wiring.config.shardLockExpiration));
+        yield* sql `UPDATE actor_deployment SET runner_shards = ${shards}, runner_lock_expiration_ms = ${expiration}
+      WHERE runner_shards IS NULL`;
+        const [row] = yield* sql `SELECT runner_shards, runner_lock_expiration_ms::text FROM actor_deployment`;
+        if (row.runner_shards !== shards || Number(row.runner_lock_expiration_ms) !== expiration)
+            return yield* Effect.die(new Error("Runner shard count or lock expiration differs from the deployment; stop all runners and migrate explicitly"));
+    }
+    else if (wiring === undefined) {
+        const [row] = yield* sql `SELECT runner_shards FROM actor_deployment`;
+        if (row.runner_shards !== null)
+            return yield* Effect.die(new Error("This database requires Runner.socket configuration; a single embedded runner cannot join it"));
+    }
+});

package/dist/runtime/storage/codec.d.ts

@@ -0,0 +1,39 @@
+import type { ActorRef } from "../../identity/caller.ts";
+/** Increment only with a migration: every stored `routing_key` depends on this encoding. */
+export declare const PLACEMENT_ENCODING = 1;
+/**
+ * An actor type's placement. A parent-placed type names its parent type and
+ * that type's own placement, down to an actor-placed root.
+ */
+export type Placement = "tenant" | "actor" | {
+    readonly parent: string;
+    readonly placement: Placement;
+};
+/** A parent-placed type's parent and the parent's placement; `undefined` for any other. */
+export declare const parentPlacement: (placement: Placement) => {
+    readonly parent: string;
+    readonly placement: Placement;
+} | undefined;
+/** The value `actor_placements.placement` records for `placement`. */
+export declare const placementKind: (placement: Placement) => "actor" | "parent" | "tenant";
+/**
+ * The 64-bit shard key shared by every row an actor owns. Tenant
+ * placement colocates a tenant's actors; actor placement spreads them; a
+ * parent-placed actor takes its root's key, read from the parent id its own
+ * id carries, so a family shares one shard. A parent-placed id without a
+ * parent id throws: ids are decoded against their actor's key before any
+ * routing, so that is a defect, not an input error.
+ */
+export declare const routingKey: ({ ref, placement, }: {
+    readonly ref: ActorRef;
+    readonly placement: Placement;
+}) => bigint;
+/** The shard key of a tenant: its tenant-placed actors and its shared content live there. */
+export declare const tenantRoutingKey: (tenant: string) => bigint;
+/**
+ * Stored state values are opaque zstd-compressed JSON: SQL never reads state,
+ * and anything worth querying belongs in an actor table.
+ */
+export declare const compress: (json: string) => Uint8Array;
+/** Inverse of `compress`; throws on bytes that are not zstd-compressed UTF-8. */
+export declare const decompress: (bytes: Uint8Array) => string;

package/dist/runtime/storage/codec.js

@@ -0,0 +1,41 @@
+import { parseChildId } from "../../identity/child.js";
+import { compressBytes, decompressBytes, hash64 } from "./platform.js";
+/** Increment only with a migration: every stored `routing_key` depends on this encoding. */
+export const PLACEMENT_ENCODING = 1;
+/** A parent-placed type's parent and the parent's placement; `undefined` for any other. */
+export const parentPlacement = (placement) => placement === "tenant" || placement === "actor" ? undefined : placement;
+/** The value `actor_placements.placement` records for `placement`. */
+export const placementKind = (placement) => placement === "tenant" || placement === "actor" ? placement : "parent";
+/**
+ * The 64-bit shard key shared by every row an actor owns. Tenant
+ * placement colocates a tenant's actors; actor placement spreads them; a
+ * parent-placed actor takes its root's key, read from the parent id its own
+ * id carries, so a family shares one shard. A parent-placed id without a
+ * parent id throws: ids are decoded against their actor's key before any
+ * routing, so that is a defect, not an input error.
+ */
+export const routingKey = ({ ref, placement, }) => {
+    const above = parentPlacement(placement);
+    if (above !== undefined) {
+        const parent = parseChildId(ref.id)?.parent;
+        if (parent === undefined)
+            throw new Error(`Actor ${ref.actor} id ${ref.id} carries no ${above.parent} parent id`);
+        return routingKey({
+            ref: { tenant: ref.tenant, actor: above.parent, id: parent },
+            placement: above.placement,
+        });
+    }
+    return placement === "tenant"
+        ? tenantRoutingKey(ref.tenant)
+        : placementHash(["actor", ref.tenant, ref.actor, ref.id]);
+};
+/** The shard key of a tenant: its tenant-placed actors and its shared content live there. */
+export const tenantRoutingKey = (tenant) => placementHash(["tenant", tenant]);
+const placementHash = (parts) => BigInt.asIntN(64, hash64(JSON.stringify([PLACEMENT_ENCODING, ...parts])));
+/**
+ * Stored state values are opaque zstd-compressed JSON: SQL never reads state,
+ * and anything worth querying belongs in an actor table.
+ */
+export const compress = (json) => compressBytes(new TextEncoder().encode(json));
+/** Inverse of `compress`; throws on bytes that are not zstd-compressed UTF-8. */
+export const decompress = (bytes) => new TextDecoder().decode(decompressBytes(bytes));

package/dist/runtime/storage/generation.d.ts

@@ -0,0 +1,69 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+import type { ActorRef } from "../../identity/caller.ts";
+/**
+ * What one activation remembers of its authority between writes. `generation`
+ * is the fenced epoch it acquired and `state` the committed state it last read
+ * or wrote. Memory is never authority: every durable writer proves in its own
+ * transaction that the stored generation still equals `generation`, and only
+ * that proof lets a turn skip reading state again. Command turns, connection
+ * sessions and workflow runs of one activation share this one object.
+ */
+export interface ActivationCache {
+    generation: string | undefined;
+    state: ReadonlyMap<string, string> | undefined;
+}
+/** A fresh activation has acquired no generation and read no state. */
+export declare const emptyActivationCache: () => ActivationCache;
+/**
+ * Drops what an activation remembers once a writer found its generation stale,
+ * so no later writer trusts that generation or the state read under it; the
+ * next one acquires a new generation and reads state again.
+ */
+export declare const forget: (cache: ActivationCache) => void;
+/** One actor's stored identity: the routing key its rows live under and its address. */
+export interface OwnedActor {
+    readonly key: bigint;
+    readonly ref: ActorRef;
+}
+/**
+ * The predicate naming `actor`'s rows in any table keyed by actor identity,
+ * with columns qualified by `alias` when the statement joins another such table.
+ */
+export declare const actorRow: ({ sql, actor: { key, ref }, alias, }: {
+    readonly sql: SqlClient.SqlClient;
+    readonly actor: OwnedActor;
+    readonly alias?: string;
+}) => import("effect/sql/Statement").Statement<import("effect/sql/SqlConnection").Row>;
+/**
+ * How a writer outside the turn's own admission holds the generation row until
+ * it commits. `UPDATE` serializes the writer with command turns, which lock the
+ * row the same way, for a multi-statement write that reads actor data. `SHARE`
+ * only stops another activation advancing the generation until the writer
+ * commits, for one statement that writes rows of its own.
+ */
+export type GenerationLock = "UPDATE" | "SHARE";
+/**
+ * The actor's generation row while it is still at `generation`, locked by
+ * `lock`, as a query or a derived table a single write statement joins. No row
+ * means another activation took over and the statement must write nothing. The
+ * lock waits out a concurrent takeover and then rechecks the generation, so a
+ * write that commits was ordered before any newer generation was acquired.
+ */
+export declare const heldGeneration: ({ sql, actor, generation, lock, }: {
+    readonly sql: SqlClient.SqlClient;
+    readonly actor: OwnedActor;
+    readonly generation: string;
+    readonly lock: GenerationLock;
+}) => import("effect/sql/Statement").Statement<import("effect/sql/SqlConnection").Row>;
+/**
+ * Proves inside the caller's transaction that `cache` still holds the actor's
+ * generation, and holds the row by `lock` until that transaction ends. A stale
+ * or never-fenced activation answers false and has forgotten its generation
+ * and state; each caller maps that answer to its own retryable failure.
+ */
+export declare const fence: (args_0: {
+    readonly actor: OwnedActor;
+    readonly cache: ActivationCache;
+    readonly lock: GenerationLock;
+}) => Effect.Effect<boolean, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;

package/dist/runtime/storage/generation.js

@@ -0,0 +1,49 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+/** A fresh activation has acquired no generation and read no state. */
+export const emptyActivationCache = () => ({
+    generation: undefined,
+    state: undefined,
+});
+/**
+ * Drops what an activation remembers once a writer found its generation stale,
+ * so no later writer trusts that generation or the state read under it; the
+ * next one acquires a new generation and reads state again.
+ */
+export const forget = (cache) => {
+    cache.generation = undefined;
+    cache.state = undefined;
+};
+/**
+ * The predicate naming `actor`'s rows in any table keyed by actor identity,
+ * with columns qualified by `alias` when the statement joins another such table.
+ */
+export const actorRow = ({ sql, actor: { key, ref }, alias, }) => {
+    const at = sql.literal(alias === undefined ? "" : `${alias}.`);
+    return sql `${at}routing_key = ${key} AND ${at}tenant_id = ${ref.tenant}
+    AND ${at}actor_type = ${ref.actor} AND ${at}actor_id = ${ref.id}`;
+};
+/**
+ * The actor's generation row while it is still at `generation`, locked by
+ * `lock`, as a query or a derived table a single write statement joins. No row
+ * means another activation took over and the statement must write nothing. The
+ * lock waits out a concurrent takeover and then rechecks the generation, so a
+ * write that commits was ordered before any newer generation was acquired.
+ */
+export const heldGeneration = ({ sql, actor, generation, lock, }) => sql `SELECT routing_key, tenant_id, actor_type, actor_id FROM actor_generations
+    WHERE ${actorRow({ sql, actor })} AND generation = ${generation} FOR ${sql.literal(lock)}`;
+/**
+ * Proves inside the caller's transaction that `cache` still holds the actor's
+ * generation, and holds the row by `lock` until that transaction ends. A stale
+ * or never-fenced activation answers false and has forgotten its generation
+ * and state; each caller maps that answer to its own retryable failure.
+ */
+export const fence = Effect.fnUntraced(function* ({ actor, cache, lock, }) {
+    if (cache.generation === undefined)
+        return false;
+    const sql = yield* SqlClient.SqlClient;
+    const held = yield* heldGeneration({ sql, actor, generation: cache.generation, lock });
+    if (held.length === 0)
+        forget(cache);
+    return held.length > 0;
+});

package/dist/runtime/storage/placements.d.ts

@@ -0,0 +1,15 @@
+import { Effect } from "effect";
+import { SqlClient, type SqlError } from "effect/sql";
+import { type Placement } from "./codec.ts";
+/**
+ * Records `name`'s placement on first registration and refuses one that
+ * differs from the record, for the type and every ancestor it routes through:
+ * a changed placement, encoding, or parent would read and write under
+ * different routing keys.
+ */
+export declare const checkPlacement: ({ name, placement, }: {
+    readonly name: string;
+    readonly placement: Placement;
+}) => Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient>;
+/** The placement recorded for `actorType`, resolved through its parents, or `undefined` if none is. */
+export declare const recordedPlacement: (actorType: string) => Effect.Effect<Placement | undefined, SqlError.SqlError, SqlClient.SqlClient>;

package/dist/runtime/storage/placements.js

@@ -0,0 +1,38 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+import { parentPlacement, PLACEMENT_ENCODING, placementKind } from "./codec.js";
+/**
+ * Records `name`'s placement on first registration and refuses one that
+ * differs from the record, for the type and every ancestor it routes through:
+ * a changed placement, encoding, or parent would read and write under
+ * different routing keys.
+ */
+export const checkPlacement = ({ name, placement, }) => Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const kind = placementKind(placement);
+    const above = parentPlacement(placement);
+    const parent = above?.parent ?? null;
+    yield* sql `INSERT INTO actor_placements (actor_type, placement, encoding, parent_type)
+    VALUES (${name}, ${kind}, ${PLACEMENT_ENCODING}, ${parent})
+    ON CONFLICT DO NOTHING`;
+    const [recorded] = yield* sql `SELECT placement, encoding, parent_type FROM actor_placements WHERE actor_type = ${name}`;
+    if (recorded?.placement !== kind ||
+        recorded.encoding !== PLACEMENT_ENCODING ||
+        recorded.parent_type !== parent)
+        return yield* Effect.die(new Error(`Actor ${name} placement differs from the deployment; migrate explicitly`));
+    if (above !== undefined)
+        yield* checkPlacement({ name: above.parent, placement: above.placement });
+});
+/** The placement recorded for `actorType`, resolved through its parents, or `undefined` if none is. */
+export const recordedPlacement = (actorType) => Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const [recorded] = yield* sql `SELECT placement, parent_type FROM actor_placements WHERE actor_type = ${actorType}`;
+    if (recorded === undefined)
+        return undefined;
+    if (recorded.placement !== "parent")
+        return recorded.placement;
+    const parent = yield* recordedPlacement(recorded.parent_type);
+    if (parent === undefined)
+        return yield* Effect.die(new Error(`Actor ${actorType}'s parent ${recorded.parent_type} has no recorded placement`));
+    return { parent: recorded.parent_type, placement: parent };
+});

package/dist/runtime/storage/platform.d.ts

@@ -0,0 +1,6 @@
+/** Preserves persisted xxHash3 routing keys on both supported runtimes. */
+export declare const hash64: (text: string) => bigint;
+/** Uses each runtime's native zstd implementation without changing the stored format. */
+export declare const compressBytes: (bytes: Uint8Array) => Uint8Array;
+/** Reads zstd frames written by either runtime. */
+export declare const decompressBytes: (bytes: Uint8Array) => Uint8Array;

package/dist/runtime/storage/platform.js

@@ -0,0 +1,9 @@
+import { zstdCompressSync, zstdDecompressSync } from "node:zlib";
+const bun = globalThis.Bun;
+const xxhash = bun === undefined ? (await import("@node-rs/xxhash")).xxh3.xxh64 : bun.hash.xxHash3;
+/** Preserves persisted xxHash3 routing keys on both supported runtimes. */
+export const hash64 = (text) => xxhash(text);
+/** Uses each runtime's native zstd implementation without changing the stored format. */
+export const compressBytes = (bytes) => bun === undefined ? zstdCompressSync(bytes) : bun.zstdCompressSync(bytes);
+/** Reads zstd frames written by either runtime. */
+export const decompressBytes = (bytes) => bun === undefined ? zstdDecompressSync(bytes) : bun.zstdDecompressSync(bytes);

package/dist/runtime/storage/retention.d.ts

@@ -0,0 +1,73 @@
+import { Effect } from "effect";
+import { SqlClient, type SqlError } from "effect/sql";
+/** One actor type's retention horizons, in milliseconds on the framework clock, and whether it has workflows to sweep. */
+interface RetentionPolicy {
+    readonly actorType: string;
+    readonly keepReceiptsMs: number;
+    readonly keepEventsMs: number;
+    /** How long past `keepEventsMs` subscriptions may hold an event back. */
+    readonly holdEventsMs: number;
+    readonly deliveryMs: number;
+    readonly keepWorkflowsMs: number;
+    readonly workflows: boolean;
+}
+/**
+ * How long after its id expires a receipt stays. An id expires one retry
+ * window after it is issued (a timer's after its due time), so this keeps
+ * receipts `keepReceipts` from issue; equivalently, it retains an expired id
+ * for `max(keepReceipts, retryWindow + deliveryTimeout)` after issue, so a
+ * command admitted just before expiry can still run and find the receipt of an
+ * attempt that committed meanwhile.
+ */
+export declare const receiptMarginMs: (horizon: {
+    readonly keepReceiptsMs: number;
+    readonly deliveryMs: number;
+    readonly retryWindowMs: number;
+}) => number;
+/** Rows deleted by a sweep, by kind. */
+export interface Swept {
+    readonly receipts: number;
+    readonly events: number;
+    readonly workflows: number;
+    /** Unreferenced content rows deleted after their grant horizon. */
+    readonly contents: number;
+}
+/**
+ * Deletes receipts and events past each actor type's horizon, in batches that
+ * each commit in their own transaction, so an interrupted sweep leaves only whole batches
+ * behind and the next sweep continues from there.
+ *
+ * A receipt goes only once its command id has expired, so external admission
+ * already rejects the id without it, and only while no outbox row carries its
+ * id, so a redelivered intent or effect route still finds it. A receipt can
+ * only gain such a row before it exists, never after.
+ *
+ * Events go as a prefix: each batch removes, per actor, every event up to the
+ * newest one it picked, so a retained event always has every later one after
+ * it. `event_sequence` lives on the generation row and is never touched, so
+ * no cursor is reissued. The prefix stops at the oldest cursor or pending
+ * wait of an open workflow execution of that actor, which still reads past it,
+ * and at the lowest position an active subscription to that actor has
+ * settled through, unless the events are older than `keepEvents` plus
+ * `holdEventsForSubscribers`.
+ *
+ * A finished workflow execution goes `keepWorkflows` after it finished; its
+ * steps went when it finished. A workflow manifest goes once it is neither
+ * the most recently accepted one for its workflow nor the start manifest of
+ * an open execution.
+ *
+ * Sweeps of one actor type take turns on an authoritative row lock, so two runners, or
+ * a sweep and `ActorTest.cleanup`, never lock overlapping event prefixes in
+ * opposite orders. Each batch starts at the newest age the previous one took,
+ * so it never walks index entries of rows earlier batches deleted and vacuum
+ * has not yet removed.
+ *
+ * The sweep yields after each batch, so a turn waiting for PGlite's one
+ * connection runs between batches instead of after the whole sweep.
+ */
+export declare const sweep: (policies: Iterable<RetentionPolicy>, retryWindowMs: number) => Effect.Effect<{
+    receipts: number;
+    events: number;
+    workflows: number;
+}, SqlError.SqlError, SqlClient.SqlClient>;
+export {};

package/dist/runtime/storage/retention.js

@@ -0,0 +1,158 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+import { databaseTime } from "../turn/admission.js";
+import { CleanupHooks } from "../turn/hooks.js";
+import { count, Metrics } from "../telemetry/metrics.js";
+import { coordinated } from "../database/coordination.js";
+/**
+ * How long after its id expires a receipt stays. An id expires one retry
+ * window after it is issued (a timer's after its due time), so this keeps
+ * receipts `keepReceipts` from issue; equivalently, it retains an expired id
+ * for `max(keepReceipts, retryWindow + deliveryTimeout)` after issue, so a
+ * command admitted just before expiry can still run and find the receipt of an
+ * attempt that committed meanwhile.
+ */
+export const receiptMarginMs = (horizon) => Math.max(horizon.keepReceiptsMs - horizon.retryWindowMs, horizon.deliveryMs);
+/**
+ * Deletes receipts and events past each actor type's horizon, in batches that
+ * each commit in their own transaction, so an interrupted sweep leaves only whole batches
+ * behind and the next sweep continues from there.
+ *
+ * A receipt goes only once its command id has expired, so external admission
+ * already rejects the id without it, and only while no outbox row carries its
+ * id, so a redelivered intent or effect route still finds it. A receipt can
+ * only gain such a row before it exists, never after.
+ *
+ * Events go as a prefix: each batch removes, per actor, every event up to the
+ * newest one it picked, so a retained event always has every later one after
+ * it. `event_sequence` lives on the generation row and is never touched, so
+ * no cursor is reissued. The prefix stops at the oldest cursor or pending
+ * wait of an open workflow execution of that actor, which still reads past it,
+ * and at the lowest position an active subscription to that actor has
+ * settled through, unless the events are older than `keepEvents` plus
+ * `holdEventsForSubscribers`.
+ *
+ * A finished workflow execution goes `keepWorkflows` after it finished; its
+ * steps went when it finished. A workflow manifest goes once it is neither
+ * the most recently accepted one for its workflow nor the start manifest of
+ * an open execution.
+ *
+ * Sweeps of one actor type take turns on an authoritative row lock, so two runners, or
+ * a sweep and `ActorTest.cleanup`, never lock overlapping event prefixes in
+ * opposite orders. Each batch starts at the newest age the previous one took,
+ * so it never walks index entries of rows earlier batches deleted and vacuum
+ * has not yet removed.
+ *
+ * The sweep yields after each batch, so a turn waiting for PGlite's one
+ * connection runs between batches instead of after the whole sweep.
+ */
+export const sweep = Effect.fnUntraced(function* (policies, retryWindowMs) {
+    const sql = yield* SqlClient.SqlClient;
+    const hooks = yield* CleanupHooks;
+    let receipts = 0;
+    let events = 0;
+    let workflows = 0;
+    for (const policy of policies) {
+        const now = yield* databaseTime;
+        const receiptCutoff = now - receiptMarginMs({ ...policy, retryWindowMs });
+        const eventCutoff = now - policy.keepEventsMs;
+        const holdCutoff = eventCutoff - policy.holdEventsMs;
+        const batch = (statement) => coordinated({ resource: `akter/retention/${policy.actorType}`, work: statement });
+        /** Runs `prune` in batches, each from the age the previous one reached, until one deletes nothing. */
+        const pruneAll = Effect.fnUntraced(function* (prune, metric) {
+            let from = "0";
+            let total = 0;
+            for (;;) {
+                const [pruned] = yield* batch(prune(from));
+                total += pruned.count;
+                if (metric !== undefined)
+                    yield* count(metric, { actor_type: policy.actorType }, pruned.count);
+                if (pruned.count === 0)
+                    return total;
+                from = pruned.last ?? from;
+                yield* hooks.afterBatch;
+                yield* Effect.yieldNow;
+            }
+        });
+        receipts += yield* pruneAll((from) => sql `
+        WITH doomed AS (
+          SELECT r.routing_key, r.tenant_id, r.actor_type, r.actor_id, r.command_id, r.expires_at_ms
+          FROM actor_receipts r
+          WHERE r.actor_type = ${policy.actorType} AND r.expires_at_ms >= ${from}::bigint
+            AND r.expires_at_ms <= ${receiptCutoff}
+            AND NOT EXISTS (
+              SELECT 1 FROM actor_outbox o
+              WHERE o.intent_id = r.command_id AND o.tenant_id = r.tenant_id
+                AND o.target_type = r.actor_type AND o.target_id = r.actor_id
+            )
+          ORDER BY r.expires_at_ms
+          LIMIT ${hooks.batchSize}
+          FOR UPDATE SKIP LOCKED),
+        gone AS (
+          DELETE FROM actor_receipts r USING doomed d
+          WHERE r.routing_key = d.routing_key AND r.tenant_id = d.tenant_id
+            AND r.actor_type = d.actor_type AND r.actor_id = d.actor_id AND r.command_id = d.command_id
+          RETURNING 1)
+        SELECT count(*)::integer AS count, (SELECT max(expires_at_ms)::text FROM doomed) AS last
+        FROM gone`, Metrics.receiptsPruned);
+        events += yield* pruneAll((from) => sql `
+        WITH picked AS (
+          SELECT routing_key, tenant_id, actor_type, actor_id, sequence, emitted_at_ms FROM actor_events
+          WHERE actor_type = ${policy.actorType} AND emitted_at_ms >= ${from}::bigint
+            AND emitted_at_ms <= ${eventCutoff}
+          ORDER BY emitted_at_ms
+          LIMIT ${hooks.batchSize}),
+        upto AS (
+          SELECT p.routing_key, p.tenant_id, p.actor_type, p.actor_id,
+            LEAST(max(p.sequence),
+              -- Active subscriptions hold what they haven't settled, until
+              -- the hold ends; then their subscribers get a RetentionGap.
+              GREATEST(
+                COALESCE((SELECT min(s.delivered) FROM actor_subscriptions s
+                  WHERE s.routing_key = p.routing_key AND s.tenant_id = p.tenant_id
+                    AND s.source_type = p.actor_type AND s.source_id = p.actor_id AND s.active),
+                  max(p.sequence)),
+                COALESCE(max(p.sequence) FILTER (WHERE p.emitted_at_ms <= ${holdCutoff}), 0)),
+              COALESCE((
+              SELECT min(LEAST(x.event_cursor, COALESCE(w.wait_after, x.event_cursor)))
+              FROM actor_workflow_executions x
+              LEFT JOIN actor_workflow_step w ON w.routing_key = x.routing_key
+                AND w.execution_id = x.execution_id AND w.kind = 'wait' AND w.exit IS NULL
+              WHERE x.routing_key = p.routing_key AND x.tenant_id = p.tenant_id
+                AND x.actor_type = p.actor_type AND x.actor_id = p.actor_id AND x.status <> 'finished'
+            ), max(p.sequence))) AS last
+          FROM picked p GROUP BY p.routing_key, p.tenant_id, p.actor_type, p.actor_id),
+        gone AS (
+          DELETE FROM actor_events e USING upto u
+          WHERE e.routing_key = u.routing_key AND e.tenant_id = u.tenant_id
+            AND e.actor_type = u.actor_type AND e.actor_id = u.actor_id AND e.sequence <= u.last
+          RETURNING 1)
+        SELECT count(*)::integer AS count, (SELECT max(emitted_at_ms)::text FROM picked) AS last
+        FROM gone`, Metrics.eventsPruned);
+        if (policy.workflows) {
+            const workflowCutoff = now - policy.keepWorkflowsMs;
+            workflows += yield* pruneAll(() => sql `
+        WITH doomed AS (
+          SELECT routing_key, execution_id FROM actor_workflow_executions
+          WHERE actor_type = ${policy.actorType} AND status = 'finished'
+            AND finished_at_ms <= ${workflowCutoff}
+          ORDER BY finished_at_ms
+          LIMIT ${hooks.batchSize}
+          FOR UPDATE SKIP LOCKED),
+        gone AS (
+          DELETE FROM actor_workflow_executions x USING doomed d
+          WHERE x.routing_key = d.routing_key AND x.execution_id = d.execution_id
+          RETURNING 1)
+        SELECT count(*)::integer AS count FROM gone`);
+            yield* batch(sql `
+        DELETE FROM actor_workflow_manifests m
+        WHERE m.actor_type = ${policy.actorType}
+          AND m.accepted_at_ms < (SELECT max(l.accepted_at_ms) FROM actor_workflow_manifests l
+            WHERE l.actor_type = m.actor_type AND l.workflow = m.workflow)
+          AND NOT EXISTS (SELECT 1 FROM actor_workflow_executions x
+            WHERE x.actor_type = m.actor_type AND x.workflow = m.workflow
+              AND x.manifest_hash = m.manifest_hash AND x.status <> 'finished')`);
+        }
+    }
+    return { receipts, events, workflows };
+});