@rikalabs/akter 0.1.0-alpha.0

package/dist/runtime/operators/export.js

@@ -0,0 +1,137 @@
+import { Effect, Option, Schema } from "effect";
+import { SqlClient } from "effect/sql";
+import { VERSION_KEY } from "../../state/migration.js";
+import { JOB_KEY_PREFIX } from "../../handles/intents.js";
+import { decodeBytes, decodeText } from "../inspector/queries.js";
+import { inReadOnlySnapshot } from "../database/snapshot.js";
+import { TenantScope, tenantSettings } from "../database/tenancy.js";
+import { databaseTime } from "../turn/admission.js";
+import { SEED_FORMAT } from "./seed.js";
+/** The most pending intents, and the most pending jobs, one export carries. */
+export const MAX_EXPORT_ROWS = 10_000;
+/**
+ * The actor cannot be exported faithfully: a stored value does not decode, or
+ * it holds more pending work than one export carries. `detail` names the row,
+ * never its contents.
+ */
+export class ExportRefused extends Schema.TaggedError()("ExportRefused", {
+    reason: Schema.Literals(["undecodable", "too_large"]),
+    detail: Schema.String,
+}) {
+}
+const decodeVersion = Schema.decodeUnknownOption(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)));
+/**
+ * Runs `effect` in a read-only snapshot bound to `tenant`: as the runtime's
+ * tenant role when row-level security is on, so the policies confine every
+ * table read to the tenant, and by the `durable.tenant` setting otherwise.
+ */
+const tenantSnapshot = (tenant) => (effect) => Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const { role } = yield* TenantScope;
+    yield* role === undefined
+        ? sql `SELECT set_config('durable.tenant', ${tenant}, true)`
+        : sql `SELECT ${tenantSettings({ sql, role, tenant })}`;
+    return yield* effect;
+}).pipe(inReadOnlySnapshot);
+/**
+ * Reads one actor's seed in a tenant-bound read-only snapshot: current
+ * state, the pending intents and timers, and the jobs not yet settled.
+ * Callers, credentials, receipts, event history, workflow executions, dead
+ * letters, owned-table rows, and blob entries are not carried; the seed counts
+ * them in `omitted`. Subscription cursors and connections are neither carried
+ * nor counted. A job whose attempt has been cancelled or has finally failed
+ * is not pending and is left out. It reads the runtime tables, not the
+ * inspection views, because the views do not show a cancelled job. `None`
+ * when the tenant has no such actor.
+ */
+export const exportActor = (page) => tenantSnapshot(page.tenant)(Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const [found] = yield* sql `
+        SELECT routing_key::text AS "routingKey", created FROM actor_generations
+        WHERE tenant_id = ${page.tenant} AND actor_type = ${page.actorType}
+          AND actor_id = ${page.actorId}`;
+    if (found === undefined)
+        return Option.none();
+    const { routingKey, created } = found;
+    const owned = sql `routing_key = ${routingKey}::int8 AND tenant_id = ${page.tenant}
+        AND actor_type = ${page.actorType} AND actor_id = ${page.actorId}`;
+    const exportedAtMs = yield* databaseTime;
+    const stored = yield* sql `
+        SELECT key, value FROM actor_state WHERE ${owned} ORDER BY key COLLATE "C"`;
+    const intents = yield* sql `
+        SELECT target_type AS "targetType", target_id AS "targetId", command, payload,
+          timer_key AS "timerKey", due_at_ms::float8 AS "dueAtMs"
+        FROM actor_outbox
+        WHERE ${owned} AND kind = 'intent'
+        ORDER BY due_at_ms, intent_id COLLATE "C" LIMIT ${MAX_EXPORT_ROWS + 1}`;
+    const jobs = yield* sql `
+        SELECT command AS job, payload, payload_version AS "payloadVersion",
+          timer_key AS "timerKey", due_at_ms::float8 AS "dueAtMs"
+        FROM actor_outbox
+        WHERE ${owned} AND kind = 'job' AND cancelled_at_ms IS NULL AND NOT final_failure
+        ORDER BY due_at_ms, intent_id COLLATE "C" LIMIT ${MAX_EXPORT_ROWS + 1}`;
+    if (intents.length > MAX_EXPORT_ROWS)
+        return yield* ExportRefused.make({ reason: "too_large", detail: "pending intents" });
+    if (jobs.length > MAX_EXPORT_ROWS)
+        return yield* ExportRefused.make({ reason: "too_large", detail: "pending jobs" });
+    const [counted] = yield* sql `
+        SELECT
+          (SELECT count(*)::int FROM actor_receipts WHERE ${owned}) AS receipts,
+          (SELECT count(*)::int FROM actor_events WHERE ${owned}) AS events,
+          (SELECT count(*)::int FROM actor_workflow_executions WHERE ${owned}) AS workflows,
+          (SELECT count(*)::int FROM actor_dead_letters WHERE ${owned}) AS "deadLetters",
+          (SELECT count(DISTINCT (blob, name))::int FROM actor_blobs WHERE ${owned})
+            + (SELECT count(DISTINCT (blob, name))::int FROM actor_content_refs WHERE ${owned}) AS blobs`;
+    const tables = yield* sql `
+        SELECT table_schema AS schema, table_name AS "table" FROM actor_tables
+        WHERE actor_type = ${page.actorType}`;
+    let tableRows = 0;
+    for (const { schema, table } of tables) {
+        const [rows] = yield* sql `
+          SELECT count(*)::int AS count FROM ${sql(schema)}.${sql(table)}
+          WHERE routing_key = ${routingKey}::int8 AND tenant_id = ${page.tenant}
+            AND actor_id = ${page.actorId}`;
+        tableRows += rows.count;
+    }
+    const state = {};
+    let stateVersion = 0;
+    for (const row of stored) {
+        const decoded = decodeBytes(row.value);
+        if (decoded === null || "undecodable" in decoded)
+            return yield* ExportRefused.make({ reason: "undecodable", detail: `state ${row.key}` });
+        if (row.key !== VERSION_KEY)
+            state[row.key] = decoded.json;
+        else
+            stateVersion = Option.getOrElse(decodeVersion(decoded.json), () => 0);
+    }
+    const payloadOf = Effect.fnUntraced(function* (text, detail) {
+        const decoded = decodeText(text);
+        if (decoded === null || "undecodable" in decoded)
+            return yield* ExportRefused.make({ reason: "undecodable", detail });
+        return decoded.json;
+    });
+    const dueIn = (dueAtMs) => Math.max(0, Math.round(dueAtMs - exportedAtMs));
+    const seed = {
+        format: SEED_FORMAT,
+        actor: { type: page.actorType, id: page.actorId },
+        created,
+        stateVersion,
+        state,
+        intents: yield* Effect.forEach(intents, (row) => Effect.map(payloadOf(row.payload, `intent ${row.command}`), (payload) => ({
+            target: { actor: row.targetType, id: row.targetId },
+            command: row.command,
+            payload,
+            key: row.timerKey ?? undefined,
+            dueInMs: dueIn(row.dueAtMs),
+        }))),
+        jobs: yield* Effect.forEach(jobs, (row) => Effect.map(payloadOf(row.payload, `job ${row.job}`), (payload) => ({
+            job: row.job,
+            payload,
+            payloadVersion: row.payloadVersion,
+            key: row.timerKey?.replace(JOB_KEY_PREFIX, "") ?? undefined,
+            dueInMs: dueIn(row.dueAtMs),
+        }))),
+        omitted: { ...counted, tableRows },
+    };
+    return Option.some(seed);
+}));

package/dist/runtime/operators/grants.d.ts

@@ -0,0 +1,45 @@
+import { Option, Schema } from "effect";
+/** What an operator may do; each action is granted separately. */
+export declare const OperatorAction: Schema.Literals<readonly ["inspect", "export", "receipts.read", "defects.read", "dead-letters.retry", "dead-letters.discard", "subscriptions.skip", "audit.read"]>;
+/** One grantable operator action. */
+export type OperatorAction = typeof OperatorAction.Type;
+/**
+ * One action on one resource scope. `tenant` is a tenant id or `"*"` for
+ * every tenant; an omitted field matches any value, and `commandId` narrows
+ * `receipts.read` to one receipt.
+ */
+export declare const Capability: Schema.Struct<{
+    readonly action: Schema.Literals<readonly ["inspect", "export", "receipts.read", "defects.read", "dead-letters.retry", "dead-letters.discard", "subscriptions.skip", "audit.read"]>;
+    readonly tenant: Schema.NonEmptyString;
+    readonly actorType: Schema.optionalKey<Schema.NonEmptyString>;
+    readonly actorId: Schema.optionalKey<Schema.NonEmptyString>;
+    readonly commandId: Schema.optionalKey<Schema.NonEmptyString>;
+}>;
+/** One capability of a grant. */
+export type Capability = typeof Capability.Type;
+/** Who the operator is and every capability they hold. */
+export declare const OperatorGrant: Schema.Struct<{
+    readonly operator: Schema.NonEmptyString;
+    readonly capabilities: Schema.$Array<Schema.Struct<{
+        readonly action: Schema.Literals<readonly ["inspect", "export", "receipts.read", "defects.read", "dead-letters.retry", "dead-letters.discard", "subscriptions.skip", "audit.read"]>;
+        readonly tenant: Schema.NonEmptyString;
+        readonly actorType: Schema.optionalKey<Schema.NonEmptyString>;
+        readonly actorId: Schema.optionalKey<Schema.NonEmptyString>;
+        readonly commandId: Schema.optionalKey<Schema.NonEmptyString>;
+    }>>;
+}>;
+/** An operator's identity and capabilities. */
+export type OperatorGrant = typeof OperatorGrant.Type;
+/** What one operator request touches; fields it doesn't name are tenant-wide. */
+export interface Resource {
+    readonly tenant: string;
+    readonly actorType?: string | undefined;
+    readonly actorId?: string | undefined;
+    readonly commandId?: string | undefined;
+}
+/** The first capability of `grant` that allows `action` on `resource`. */
+export declare const authorizing: ({ grant, action, resource, }: {
+    readonly grant: OperatorGrant;
+    readonly action: OperatorAction;
+    readonly resource: Resource;
+}) => Option.Option<Capability>;

package/dist/runtime/operators/grants.js

@@ -0,0 +1,40 @@
+import { Option, Schema } from "effect";
+/** What an operator may do; each action is granted separately. */
+export const OperatorAction = Schema.Literals([
+    "inspect",
+    "export",
+    "receipts.read",
+    "defects.read",
+    "dead-letters.retry",
+    "dead-letters.discard",
+    "subscriptions.skip",
+    "audit.read",
+]);
+/**
+ * One action on one resource scope. `tenant` is a tenant id or `"*"` for
+ * every tenant; an omitted field matches any value, and `commandId` narrows
+ * `receipts.read` to one receipt.
+ */
+export const Capability = Schema.Struct({
+    action: OperatorAction,
+    tenant: Schema.NonEmptyString,
+    actorType: Schema.optionalKey(Schema.NonEmptyString),
+    actorId: Schema.optionalKey(Schema.NonEmptyString),
+    commandId: Schema.optionalKey(Schema.NonEmptyString),
+});
+/** Who the operator is and every capability they hold. */
+export const OperatorGrant = Schema.Struct({
+    operator: Schema.NonEmptyString,
+    capabilities: Schema.Array(Capability),
+});
+/**
+ * A capability field left open matches anything; a set one must name the
+ * resource's own value, so an actor-scoped grant never covers a tenant-wide read.
+ */
+const covers = (granted, requested) => granted === undefined || granted === requested;
+/** The first capability of `grant` that allows `action` on `resource`. */
+export const authorizing = ({ grant, action, resource, }) => Option.fromUndefinedOr(grant.capabilities.find((capability) => capability.action === action &&
+    (capability.tenant === "*" || capability.tenant === resource.tenant) &&
+    covers(capability.actorType, resource.actorType) &&
+    covers(capability.actorId, resource.actorId) &&
+    covers(capability.commandId, resource.commandId)));

package/dist/runtime/operators/repair.d.ts

@@ -0,0 +1,206 @@
+import { Context, type Crypto, Effect, Option, Schema } from "effect";
+import { SqlClient } from "effect/sql";
+import type { RegisteredJob } from "../members.ts";
+import * as Queries from "../inspector/queries.ts";
+import { TenantScope } from "../database/tenancy.ts";
+import { type Placement } from "../storage/codec.ts";
+import { FrameworkClock } from "../turn/admission.ts";
+import { OutboxRuntime } from "../turn/outbox.ts";
+import { type ExportRefused } from "./export.ts";
+import type { Seed } from "./seed.ts";
+import { type AuditEntry, type AuditRecord } from "./audit.ts";
+declare const OperatorNotFound_base: Schema.Class<OperatorNotFound, Schema.TaggedStruct<"OperatorNotFound", {}>, import("effect/Cause").YieldableError>;
+/** The dead letter, receipt, or actor the request names does not exist. */
+export declare class OperatorNotFound extends OperatorNotFound_base {
+}
+declare const ProviderOutcomeUnknown_base: Schema.Class<ProviderOutcomeUnknown, Schema.TaggedStruct<"ProviderOutcomeUnknown", {
+    readonly jobId: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * The dead letter's last attempt may have applied the call. Retrying runs
+ * the provider call again under a new idempotency key, so the operator must
+ * state that they checked the provider first.
+ */
+export declare class ProviderOutcomeUnknown extends ProviderOutcomeUnknown_base {
+}
+declare const JobNotServed_base: Schema.Class<JobNotServed, Schema.TaggedStruct<"JobNotServed", {
+    readonly job: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+/** This runner has no executor for the job, so it can't know how to queue a retry. */
+export declare class JobNotServed extends JobNotServed_base {
+}
+/** Why a repair or operator read was refused. */
+export type RepairError = OperatorNotFound | ProviderOutcomeUnknown | JobNotServed;
+/** What an actor-scoped operator request names. */
+export interface ActorTarget {
+    readonly tenant: string;
+    readonly actorType: string;
+    readonly actorId: string;
+}
+/** A stored outcome as an operator reads it: never re-executed. */
+export interface ReceiptView {
+    readonly commandId: string;
+    readonly command: string;
+    readonly outcomeTag: string;
+    readonly outcome: Queries.Decoded | null;
+    readonly expiresAtMs: number;
+}
+declare const OperatorRuntime_base: Context.ServiceClass<OperatorRuntime, "@rikalabs/akter/runtime/operators/repair/OperatorRuntime", {
+    /** The actor type's recorded placement, or none when no runner registered it. */
+    readonly placement: (actorType: string) => Effect.Effect<Option.Option<Placement>>;
+    readonly inspect: (target: ActorTarget & {
+        readonly limit: number;
+        readonly outcomes: boolean;
+    }) => Effect.Effect<Option.Option<unknown>>;
+    readonly receipt: (target: ActorTarget & {
+        readonly commandId: string;
+    }) => Effect.Effect<Option.Option<ReceiptView>>;
+    /**
+     * One actor's seed, read in a single read-only snapshot; `None` when the
+     * tenant has no such actor.
+     */
+    readonly exportSeed: (target: ActorTarget) => Effect.Effect<Option.Option<Seed>, ExportRefused>;
+    readonly retry: (request: {
+        readonly target: ActorTarget;
+        readonly jobId: string;
+        readonly providerChecked: boolean;
+        readonly audit: AuditEntry;
+    }) => Effect.Effect<{
+        readonly jobId: string;
+    }, RepairError>;
+    readonly discard: (request: {
+        readonly target: ActorTarget;
+        readonly jobId: string;
+        readonly audit: AuditEntry;
+    }) => Effect.Effect<void, RepairError>;
+    /**
+     * Skips a stuck subscription row's events through `through`: the row
+     * moves past them, and its subscriber is sent a marker for the range.
+     */
+    readonly skip: (request: {
+        readonly target: ActorTarget;
+        readonly subscriberType: string;
+        readonly subscription: string;
+        readonly subscriberId: string;
+        readonly through: string;
+        readonly audit: AuditEntry;
+    }) => Effect.Effect<{
+        readonly through: string;
+    }, RepairError>;
+    /**
+     * A tenant's active subscription rows that have failed at least
+     * `minAttempts` deliveries in a row, most attempts first.
+     */
+    readonly lagging: (page: {
+        readonly tenant: string;
+        readonly minAttempts: number;
+        readonly limit: number;
+    }) => Effect.Effect<ReadonlyArray<LaggingSubscription>>;
+    /** A tenant's newest audit rows, or every tenant's for `"*"`. */
+    readonly audit: (page: {
+        readonly tenant: string;
+        readonly limit: number;
+    }) => Effect.Effect<ReadonlyArray<AuditRecord>>;
+    /** Writes one audit row on its own; a read's row goes in before its answer. */
+    readonly record: (entry: AuditEntry, outcome: Schema.Json) => Effect.Effect<void>;
+}>;
+/**
+ * Operator reads and repairs that need the runtime's outbox and executors.
+ * Every repair writes its audit row in its own transaction.
+ */
+export declare class OperatorRuntime extends OperatorRuntime_base {
+}
+/** A subscription row whose deliveries keep failing, and how far behind its source it is. */
+export interface LaggingSubscription {
+    readonly sourceType: string;
+    readonly sourceId: string;
+    readonly subscriberType: string;
+    readonly subscription: string;
+    readonly subscriberId: string;
+    readonly delivered: string;
+    readonly head: string;
+    readonly lag: string;
+    readonly attempts: number;
+    readonly lastError: string;
+    readonly dueAtMs: number | null;
+}
+/** What the runtime's clock and outbox references hold on this runner. */
+type ReferenceOf<T> = T extends Context.Reference<infer S> ? S : never;
+/**
+ * Builds the service over the runtime's SQL, crypto, clock, and outbox, and
+ * its job registrations, which say whether a retried job is capped.
+ *
+ * Repairs lock the dead letter they act on, so two repairs of it serialize and the
+ * second finds nothing. Without `receipts.read` an operator sees that a command
+ * ran, not what it returned. A retry is the operator's act and carries no end
+ * user's principal. The payload may hold customer data, so a discard's record
+ * keeps what failed, not what was sent.
+ */
+export declare const operatorRuntime: (deps: {
+    readonly services: Context.Context<SqlClient.SqlClient | Crypto.Crypto>;
+    readonly clock: ReferenceOf<typeof FrameworkClock>;
+    readonly outbox: ReferenceOf<typeof OutboxRuntime>;
+    readonly jobOf: (actorType: string, job: string) => RegisteredJob | undefined;
+    readonly wake: Effect.Effect<void>;
+    /** The tenant and adoption writer roles the operator's turns take, as the runtime's own turns do. */
+    readonly tenantScope: ReferenceOf<typeof TenantScope>;
+}) => {
+    /** The actor type's recorded placement, or none when no runner registered it. */
+    readonly placement: (actorType: string) => Effect.Effect<Option.Option<Placement>>;
+    readonly inspect: (target: ActorTarget & {
+        readonly limit: number;
+        readonly outcomes: boolean;
+    }) => Effect.Effect<Option.Option<unknown>>;
+    readonly receipt: (target: ActorTarget & {
+        readonly commandId: string;
+    }) => Effect.Effect<Option.Option<ReceiptView>>;
+    /**
+     * One actor's seed, read in a single read-only snapshot; `None` when the
+     * tenant has no such actor.
+     */
+    readonly exportSeed: (target: ActorTarget) => Effect.Effect<Option.Option<Seed>, ExportRefused>;
+    readonly retry: (request: {
+        readonly target: ActorTarget;
+        readonly jobId: string;
+        readonly providerChecked: boolean;
+        readonly audit: AuditEntry;
+    }) => Effect.Effect<{
+        readonly jobId: string;
+    }, RepairError>;
+    readonly discard: (request: {
+        readonly target: ActorTarget;
+        readonly jobId: string;
+        readonly audit: AuditEntry;
+    }) => Effect.Effect<void, RepairError>;
+    /**
+     * Skips a stuck subscription row's events through `through`: the row
+     * moves past them, and its subscriber is sent a marker for the range.
+     */
+    readonly skip: (request: {
+        readonly target: ActorTarget;
+        readonly subscriberType: string;
+        readonly subscription: string;
+        readonly subscriberId: string;
+        readonly through: string;
+        readonly audit: AuditEntry;
+    }) => Effect.Effect<{
+        readonly through: string;
+    }, RepairError>;
+    /**
+     * A tenant's active subscription rows that have failed at least
+     * `minAttempts` deliveries in a row, most attempts first.
+     */
+    readonly lagging: (page: {
+        readonly tenant: string;
+        readonly minAttempts: number;
+        readonly limit: number;
+    }) => Effect.Effect<ReadonlyArray<LaggingSubscription>>;
+    /** A tenant's newest audit rows, or every tenant's for `"*"`. */
+    readonly audit: (page: {
+        readonly tenant: string;
+        readonly limit: number;
+    }) => Effect.Effect<ReadonlyArray<AuditRecord>>;
+    /** Writes one audit row on its own; a read's row goes in before its answer. */
+    readonly record: (entry: AuditEntry, outcome: Schema.Json) => Effect.Effect<void>;
+};
+export {};

package/dist/runtime/operators/repair.js

@@ -0,0 +1,226 @@
+import { Context, Effect, Option, Schema } from "effect";
+import { SqlClient, SqlError } from "effect/sql";
+import { emptyOutbox } from "../../handles/intents.js";
+import { ActorRef, System } from "../../identity/caller.js";
+import * as Queries from "../inspector/queries.js";
+import { TenantScope, withTenant } from "../database/tenancy.js";
+import { routingKey } from "../storage/codec.js";
+import { recordedPlacement } from "../storage/placements.js";
+import { databaseTime, FrameworkClock } from "../turn/admission.js";
+import { OutboxRuntime, outboxStatements } from "../turn/outbox.js";
+import { exportActor } from "./export.js";
+import { auditRoutingKey, listAudit, writeAudit, } from "./audit.js";
+/** The dead letter, receipt, or actor the request names does not exist. */
+export class OperatorNotFound extends Schema.TaggedError()("OperatorNotFound", {}) {
+}
+/**
+ * The dead letter's last attempt may have applied the call. Retrying runs
+ * the provider call again under a new idempotency key, so the operator must
+ * state that they checked the provider first.
+ */
+export class ProviderOutcomeUnknown extends Schema.TaggedError()("ProviderOutcomeUnknown", { jobId: Schema.String }) {
+}
+/** This runner has no executor for the job, so it can't know how to queue a retry. */
+export class JobNotServed extends Schema.TaggedError()("JobNotServed", {
+    job: Schema.String,
+}) {
+}
+/**
+ * Operator reads and repairs that need the runtime's outbox and executors.
+ * Every repair writes its audit row in its own transaction.
+ */
+export class OperatorRuntime extends Context.Service()("@rikalabs/akter/runtime/operators/repair/OperatorRuntime") {
+}
+/**
+ * Builds the service over the runtime's SQL, crypto, clock, and outbox, and
+ * its job registrations, which say whether a retried job is capped.
+ *
+ * Repairs lock the dead letter they act on, so two repairs of it serialize and the
+ * second finds nothing. Without `receipts.read` an operator sees that a command
+ * ran, not what it returned. A retry is the operator's act and carries no end
+ * user's principal. The payload may hold customer data, so a discard's record
+ * keeps what failed, not what was sent.
+ */
+export const operatorRuntime = (deps) => {
+    const placement = (actorType) => recordedPlacement(actorType).pipe(Effect.map(Option.fromUndefinedOr), Effect.orDie, Effect.provideContext(deps.services));
+    const keyOf = (target) => Effect.flatMap(placement(target.actorType), (found) => Option.match(found, {
+        onNone: () => Effect.fail(OperatorNotFound.make({})),
+        onSome: (kind) => Effect.succeed(routingKey({
+            ref: { tenant: target.tenant, actor: target.actorType, id: target.actorId },
+            placement: kind,
+        })),
+    }));
+    const provided = (effect) => effect.pipe(Effect.catchIf(SqlError.isSqlError, Effect.die), Effect.provideService(FrameworkClock, deps.clock), Effect.provideService(OutboxRuntime, deps.outbox), Effect.provideService(TenantScope, deps.tenantScope), Effect.provideContext(deps.services));
+    /**
+     * Runs one repair of `target` in its own tenant transaction, and writes the
+     * repair's audit row, with the outcome it reports, in that same transaction.
+     */
+    const auditedRepair = (target, audit, repair) => Effect.gen(function* () {
+        const key = yield* keyOf(target);
+        const sql = yield* SqlClient.SqlClient;
+        const { result } = yield* withTenant(target.tenant)(sql.withTransaction(Effect.tap(repair(key), ({ outcome }) => writeAudit({ entry: audit, key, outcome }))));
+        return result;
+    });
+    const lockLetter = (key, target, jobId) => Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        const [letter] = yield* sql `
+        SELECT job, payload, payload_version AS "payloadVersion", attempts::int AS attempts, cause, ambiguous
+        FROM actor_dead_letters
+        WHERE routing_key = ${key} AND job_id = ${jobId} AND tenant_id = ${target.tenant}
+          AND actor_type = ${target.actorType} AND actor_id = ${target.actorId}
+        FOR UPDATE`;
+        if (letter === undefined)
+            return yield* OperatorNotFound.make({});
+        yield* sql `DELETE FROM actor_dead_letters WHERE routing_key = ${key} AND job_id = ${jobId}`;
+        return letter;
+    });
+    return OperatorRuntime.of({
+        placement,
+        inspect: ({ limit, outcomes, ...target }) => Queries.readOnly(target.tenant)(Queries.actor({ ...target, limit })).pipe(Effect.map(Option.map((page) => outcomes
+            ? page
+            : {
+                ...page,
+                receipts: page.receipts.map(({ outcome: _outcome, ...receipt }) => receipt),
+            })), Effect.orDie, provided),
+        receipt: ({ commandId, ...target }) => Effect.gen(function* () {
+            const key = yield* keyOf(target);
+            const sql = yield* SqlClient.SqlClient;
+            const [row] = yield* Queries.readOnly(target.tenant)(sql `SELECT command, outcome_tag, outcome, expires_at_ms::float8 AS expires_at_ms
+          FROM durable.receipts
+          WHERE routing_key = ${key} AND tenant_id = ${target.tenant}
+            AND actor_type = ${target.actorType} AND actor_id = ${target.actorId}
+            AND command_id = ${commandId}`);
+            return Option.map(Option.fromUndefinedOr(row), (found) => ({
+                commandId,
+                command: found.command,
+                outcomeTag: found.outcome_tag,
+                outcome: Queries.decodeText(found.outcome),
+                expiresAtMs: found.expires_at_ms,
+            }));
+        }).pipe(Effect.catchTag("OperatorNotFound", () => Effect.succeedNone), provided),
+        exportSeed: (target) => exportActor(target).pipe(Effect.catchIf(SqlError.isSqlError, Effect.die), provided),
+        retry: ({ target, jobId, providerChecked, audit }) => Effect.gen(function* () {
+            const ref = ActorRef.make({
+                tenant: target.tenant,
+                actor: target.actorType,
+                id: target.actorId,
+            });
+            const retried = yield* auditedRepair(target, audit, (key) => Effect.gen(function* () {
+                const letter = yield* lockLetter(key, target, jobId);
+                if (letter.ambiguous && !providerChecked)
+                    return yield* ProviderOutcomeUnknown.make({ jobId });
+                const registered = deps.jobOf(target.actorType, letter.job);
+                if (registered === undefined)
+                    return yield* JobNotServed.make({ job: letter.job });
+                const staged = yield* outboxStatements(key, ref, {
+                    ...emptyOutbox,
+                    jobs: [
+                        {
+                            job: letter.job,
+                            payload: letter.payload,
+                            version: letter.payloadVersion,
+                            caller: System.make({ source: "actor", ref }),
+                            due: undefined,
+                            key: undefined,
+                            capped: registered.perActor !== undefined,
+                        },
+                    ],
+                }, databaseTime);
+                for (const statement of staged.statements)
+                    yield* statement;
+                const retriedId = staged.jobIds[0];
+                return {
+                    result: { jobId: retriedId },
+                    outcome: {
+                        retried: jobId,
+                        jobId: retriedId,
+                        job: letter.job,
+                        attempts: letter.attempts,
+                        ambiguous: letter.ambiguous,
+                        providerChecked,
+                    },
+                };
+            }));
+            yield* deps.wake;
+            return retried;
+        }).pipe(provided),
+        discard: ({ target, jobId, audit }) => auditedRepair(target, audit, (key) => Effect.map(lockLetter(key, target, jobId), (letter) => ({
+            result: undefined,
+            outcome: {
+                discarded: jobId,
+                job: letter.job,
+                attempts: letter.attempts,
+                ambiguous: letter.ambiguous,
+                cause: letter.cause,
+            },
+        }))).pipe(provided),
+        skip: ({ target, subscriberType, subscription, subscriberId, through, audit }) => Effect.gen(function* () {
+            yield* auditedRepair(target, audit, (key) => Effect.gen(function* () {
+                const sql = yield* SqlClient.SqlClient;
+                const now = yield* databaseTime;
+                const [skipped] = yield* sql `
+              UPDATE actor_subscriptions s
+              SET gap_at_ms = ${now}, gap_through = ${BigInt(through)}, attempts = 0,
+                last_error = NULL, due_at_ms = ${now}
+              FROM (SELECT delivered, last_error FROM actor_subscriptions
+                WHERE routing_key = ${key} AND tenant_id = ${target.tenant}
+                  AND source_type = ${target.actorType} AND source_id = ${target.actorId}
+                  AND subscriber_type = ${subscriberType} AND subscription = ${subscription}
+                  AND subscriber_id = ${subscriberId}
+                  AND active AND last_error IS NOT NULL AND delivered < ${BigInt(through)}
+                FOR UPDATE) old
+              WHERE s.routing_key = ${key} AND s.tenant_id = ${target.tenant}
+                AND s.source_type = ${target.actorType} AND s.source_id = ${target.actorId}
+                AND s.subscriber_type = ${subscriberType} AND s.subscription = ${subscription}
+                AND s.subscriber_id = ${subscriberId}
+                AND ${BigInt(through)} <= (SELECT g.event_sequence FROM actor_generations g
+                  WHERE g.routing_key = ${key} AND g.tenant_id = ${target.tenant}
+                    AND g.actor_type = ${target.actorType} AND g.actor_id = ${target.actorId})
+              RETURNING old.delivered::text AS delivered, old.last_error AS "lastError"`;
+                if (skipped === undefined)
+                    return yield* OperatorNotFound.make({});
+                return {
+                    result: undefined,
+                    outcome: {
+                        subscriber: `${subscriberType}.${subscription}/${subscriberId}`,
+                        after: skipped.delivered,
+                        through,
+                        lastError: skipped.lastError,
+                    },
+                };
+            }));
+            yield* deps.wake;
+            return { through };
+        }).pipe(provided),
+        lagging: ({ tenant, minAttempts, limit }) => Queries.readOnly(tenant)(Effect.gen(function* () {
+            const sql = yield* SqlClient.SqlClient;
+            return yield* sql `
+            SELECT s.source_type AS "sourceType", s.source_id AS "sourceId",
+              s.subscriber_type AS "subscriberType", s.subscription,
+              s.subscriber_id AS "subscriberId", s.delivered::text AS delivered,
+              g.event_sequence::text AS head, (g.event_sequence - s.delivered)::text AS lag,
+              s.attempts::int AS attempts, s.last_error AS "lastError",
+              s.due_at_ms::float8 AS "dueAtMs"
+            FROM actor_subscriptions s
+            JOIN actor_generations g ON g.routing_key = s.routing_key
+              AND g.tenant_id = s.tenant_id AND g.actor_type = s.source_type
+              AND g.actor_id = s.source_id
+            WHERE s.tenant_id = ${tenant} AND s.active AND s.last_error IS NOT NULL
+              AND s.attempts >= ${minAttempts}
+            ORDER BY s.attempts DESC, g.event_sequence - s.delivered DESC,
+              s.source_type COLLATE "C", s.source_id COLLATE "C",
+              s.subscriber_type COLLATE "C", s.subscription COLLATE "C",
+              s.subscriber_id COLLATE "C"
+            LIMIT ${limit}`;
+        })).pipe(provided, Effect.orDie),
+        audit: (page) => Queries.readOnly(page.tenant)(listAudit(page)).pipe(provided, Effect.orDie),
+        record: (entry, outcome) => Effect.gen(function* () {
+            const found = entry.actorType === undefined ? Option.none() : yield* placement(entry.actorType);
+            yield* writeAudit({
+                entry,
+                key: auditRoutingKey({ entry, placement: Option.getOrUndefined(found) }),
+                outcome,
+            });
+        }).pipe(provided, Effect.orDie),
+    });
+};