@rikalabs/akter 0.1.0-alpha.0

package/dist/runtime/subscriptions/storage.d.ts

@@ -0,0 +1,40 @@
+import { Effect } from "effect";
+import { SqlClient, type Statement } from "effect/sql";
+/** The row columns of the subscription row aliased `alias`, for a select list or `RETURNING`. */
+export declare const rowColumns: ({ sql, alias, }: {
+    readonly sql: SqlClient.SqlClient;
+    readonly alias: string;
+}) => Statement.Fragment;
+/**
+ * CTEs that move the tag summary by the rows of the CTE named `changed`, whose
+ * columns name a source and the event tags one row followed before (`was`)
+ * and after (`now`) the change, each empty for an inactive or absent row. A
+ * summary row counts the active rows of its source that follow its tag, so
+ * each count moves by the rows that gained the tag less the rows that lost
+ * it. Every count moves in one upsert that takes its rows in key order, so
+ * concurrent writers of one source's counts lock them in the same order and
+ * never deadlock; an update whose lock order followed the plan would, and its
+ * victim's change would wait out a retry. The proposed row holds no negative
+ * count, which the summary's check refuses before the conflict is found, so
+ * the update reads the signed delta itself. A decrement upserts against the
+ * latest version of its row, and `tags_moved` returns every count it set,
+ * since one that reached 0 must be deleted before the transaction commits.
+ */
+export declare const summarize: ({ sql, changed, }: {
+    readonly sql: SqlClient.SqlClient;
+    readonly changed: string;
+}) => Statement.Statement<import("effect/sql/SqlConnection").Row>;
+/**
+ * Changes subscription rows and the tag summary in one transaction, so no
+ * caller maintains the summary. `old` locks the rows about to change and
+ * returns their row columns; `change` writes them, reading those rows as
+ * `old_rows`, and returns their row columns as they are now, with `active`
+ * false for a deleted row. The lock is its own statement: a lock in the
+ * changing statement would skip a row that statement already changed, and
+ * its rows would then count as new. Locking reads the latest committed
+ * version of each row, so a concurrent change that committed first is never
+ * counted twice. A summary count the change lowered to 0 is deleted in the
+ * same transaction, so no committed summary row counts 0. Returns how many
+ * rows `change` wrote.
+ */
+export declare const changeRows: (old: Statement.Fragment, change: Statement.Fragment) => Effect.Effect<number, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;

package/dist/runtime/subscriptions/storage.js

@@ -0,0 +1,98 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+/** A subscription row's key, then what it follows: the columns every row change reads and returns. */
+const ROW = [
+    "routing_key",
+    "tenant_id",
+    "source_type",
+    "source_id",
+    "subscriber_type",
+    "subscription",
+    "subscriber_id",
+    "active",
+    "events",
+];
+/** The row columns of the subscription row aliased `alias`, for a select list or `RETURNING`. */
+export const rowColumns = ({ sql, alias, }) => sql.literal(ROW.map((column) => `${alias}.${column}`).join(", "));
+/**
+ * CTEs that move the tag summary by the rows of the CTE named `changed`, whose
+ * columns name a source and the event tags one row followed before (`was`)
+ * and after (`now`) the change, each empty for an inactive or absent row. A
+ * summary row counts the active rows of its source that follow its tag, so
+ * each count moves by the rows that gained the tag less the rows that lost
+ * it. Every count moves in one upsert that takes its rows in key order, so
+ * concurrent writers of one source's counts lock them in the same order and
+ * never deadlock; an update whose lock order followed the plan would, and its
+ * victim's change would wait out a retry. The proposed row holds no negative
+ * count, which the summary's check refuses before the conflict is found, so
+ * the update reads the signed delta itself. A decrement upserts against the
+ * latest version of its row, and `tags_moved` returns every count it set,
+ * since one that reached 0 must be deleted before the transaction commits.
+ */
+export const summarize = ({ sql, changed, }) => {
+    const rows = sql.literal(changed);
+    return sql `tag_delta AS (
+      SELECT routing_key, tenant_id, source_type, source_id, tag, sum(d)::int AS d FROM (
+        SELECT c.routing_key, c.tenant_id, c.source_type, c.source_id, x.tag, 1 AS d
+        FROM ${rows} c CROSS JOIN LATERAL unnest(c.now) AS x(tag)
+        WHERE NOT x.tag = ANY(c.was)
+        UNION ALL
+        SELECT c.routing_key, c.tenant_id, c.source_type, c.source_id, x.tag, -1 AS d
+        FROM ${rows} c CROSS JOIN LATERAL unnest(c.was) AS x(tag)
+        WHERE NOT x.tag = ANY(c.now)) moved
+      GROUP BY routing_key, tenant_id, source_type, source_id, tag
+      HAVING sum(d) <> 0),
+    tags_moved AS (
+      INSERT INTO actor_subscription_tags AS t (routing_key, tenant_id, source_type, source_id, event, rows)
+      SELECT routing_key, tenant_id, source_type, source_id, tag, greatest(d, 0) FROM tag_delta
+      ORDER BY routing_key, tenant_id, source_type, source_id, tag
+      ON CONFLICT (routing_key, tenant_id, source_type, source_id, event)
+      DO UPDATE SET rows = t.rows + (SELECT d.d FROM tag_delta d
+        WHERE d.routing_key = EXCLUDED.routing_key AND d.tenant_id = EXCLUDED.tenant_id
+          AND d.source_type = EXCLUDED.source_type AND d.source_id = EXCLUDED.source_id
+          AND d.tag = EXCLUDED.event)
+      RETURNING t.routing_key, t.tenant_id, t.source_type, t.source_id, t.event, t.rows)`;
+};
+/**
+ * Changes subscription rows and the tag summary in one transaction, so no
+ * caller maintains the summary. `old` locks the rows about to change and
+ * returns their row columns; `change` writes them, reading those rows as
+ * `old_rows`, and returns their row columns as they are now, with `active`
+ * false for a deleted row. The lock is its own statement: a lock in the
+ * changing statement would skip a row that statement already changed, and
+ * its rows would then count as new. Locking reads the latest committed
+ * version of each row, so a concurrent change that committed first is never
+ * counted twice. A summary count the change lowered to 0 is deleted in the
+ * same transaction, so no committed summary row counts 0. Returns how many
+ * rows `change` wrote.
+ */
+export const changeRows = Effect.fnUntraced(function* (old, change) {
+    const sql = yield* SqlClient.SqlClient;
+    return yield* sql.withTransaction(Effect.gen(function* () {
+        const [locked] = yield* sql `
+        SELECT coalesce(json_agg(o), '[]')::text AS rows FROM (${old}) o`;
+        const [result] = yield* sql `
+        WITH old_rows AS (
+          SELECT * FROM json_to_recordset(${locked.rows}::json) AS o(routing_key bigint,
+            tenant_id text, source_type text, source_id text, subscriber_type text,
+            subscription text, subscriber_id text, active boolean, events text[])),
+        new_rows AS (${change}),
+        changed AS (
+          SELECT routing_key, tenant_id, source_type, source_id,
+            CASE WHEN o.active THEN o.events ELSE '{}'::text[] END AS was,
+            CASE WHEN n.active THEN n.events ELSE '{}'::text[] END AS now
+          FROM old_rows o FULL JOIN new_rows n USING (routing_key, tenant_id, source_type,
+            source_id, subscriber_type, subscription, subscriber_id)),
+        ${summarize({ sql, changed: "changed" })}
+        SELECT (SELECT count(*) FROM new_rows)::int AS rows,
+          (SELECT json_agg(json_build_array(routing_key::text, tenant_id, source_type, source_id,
+            event)) FROM tags_moved WHERE rows = 0)::text AS emptied`;
+        if (result.emptied !== null)
+            yield* sql `DELETE FROM actor_subscription_tags t
+          USING json_array_elements(${result.emptied}::json) AS e
+          WHERE t.routing_key = (e->>0)::bigint AND t.tenant_id = e->>1
+            AND t.source_type = e->>2 AND t.source_id = e->>3 AND t.event = e->>4
+            AND t.rows = 0`;
+        return result.rows;
+    }));
+});

package/dist/runtime/sweeps.d.ts

@@ -0,0 +1,39 @@
+import { Context, Crypto, Effect, Scope } from "effect";
+import { Sharding } from "effect/cluster";
+import { SqlClient, SqlError } from "effect/sql";
+import type { Registration } from "./members.ts";
+import type { tenantContent } from "./content/store.ts";
+import type { SubscriptionRelay } from "./subscriptions/relay.ts";
+import { FrameworkClock } from "./turn/admission.ts";
+/**
+ * Starts the runtime's periodic background work in its scope: the payload
+ * writer heartbeat every half window, retention and content cleanup every
+ * minute when the cleanup hooks allow it, and the telemetry sampler as one
+ * singleton per Sharding. The writer rows are dropped when the scope closes.
+ * Returns the cleanup pass for callers that sweep on demand, the sweeping
+ * fiber a drain interrupts, and one sampling pass.
+ */
+export declare const startSweeps: (args_0: {
+    readonly registrations: ReadonlyMap<string, Registration>;
+    readonly sweepsWorkflows: ReadonlySet<string>;
+    readonly retryWindowMs: number;
+    readonly subscriptions: SubscriptionRelay;
+    readonly content: ReturnType<typeof tenantContent> | undefined;
+    readonly services: Context.Context<SqlClient.SqlClient | Crypto.Crypto | Sharding.Sharding>;
+    readonly frameworkClock: (typeof FrameworkClock)["Service"];
+    readonly refreshPayloadWriters: Effect.Effect<void, SqlError.SqlError>;
+    readonly writerWindowMs: number;
+    readonly runtimeId: string;
+    readonly sampleEveryMs: number;
+    readonly sharding: Sharding.Sharding["Service"];
+    readonly scope: Scope.Scope;
+}) => Effect.Effect<{
+    cleanup: Effect.Effect<{
+        receipts: number;
+        events: number;
+        workflows: number;
+        contents: number;
+    }, import("./subscriptions/relay.ts").SubscriptionError, never>;
+    sweeping: import("effect/Fiber").Fiber<never, never> | undefined;
+    sample: Effect.Effect<void, never, never>;
+}, never, Scope.Scope>;

package/dist/runtime/sweeps.js

@@ -0,0 +1,54 @@
+import { Cause, Context, Crypto, Effect, Scope } from "effect";
+import { Sharding } from "effect/cluster";
+import { SqlClient, SqlError } from "effect/sql";
+import { dropWriters } from "./payloads/versions.js";
+import { sweep } from "./storage/retention.js";
+import { databaseSampler } from "./telemetry/sampler.js";
+import { FrameworkClock } from "./turn/admission.js";
+import { CleanupHooks } from "./turn/hooks.js";
+/** Shardings whose telemetry sampler is registered; Cluster allows one per name. */
+const sampled = new WeakSet();
+/** Pause between retention sweeps. */
+const CLEANUP_INTERVAL = "1 minute";
+/**
+ * Starts the runtime's periodic background work in its scope: the payload
+ * writer heartbeat every half window, retention and content cleanup every
+ * minute when the cleanup hooks allow it, and the telemetry sampler as one
+ * singleton per Sharding. The writer rows are dropped when the scope closes.
+ * Returns the cleanup pass for callers that sweep on demand, the sweeping
+ * fiber a drain interrupts, and one sampling pass.
+ */
+export const startSweeps = Effect.fnUntraced(function* ({ registrations, sweepsWorkflows, retryWindowMs, subscriptions, content, services, frameworkClock, refreshPayloadWriters, writerWindowMs, runtimeId, sampleEveryMs, sharding, scope, }) {
+    const cleanupHooks = yield* CleanupHooks;
+    const cleanup = Effect.suspend(() => refreshPayloadWriters.pipe(Effect.orDie, Effect.andThen(sweep(Array.from(registrations.values(), ({ name, policy }) => ({
+        actorType: name,
+        keepReceiptsMs: policy.keepReceiptsMs,
+        keepEventsMs: policy.keepEventsMs,
+        holdEventsMs: policy.holdEventsMs,
+        deliveryMs: policy.deliveryMs,
+        keepWorkflowsMs: policy.keepWorkflowsMs,
+        workflows: sweepsWorkflows.has(name),
+    })), retryWindowMs).pipe(Effect.tap(() => Effect.forEach([...registrations.values()], (registration) => subscriptions.cleanupRemoved(registration.name, registration.subscriptions.map((declared) => declared.tag)), { discard: true })))), Effect.flatMap((swept) => Effect.map(content === undefined ? Effect.succeed(0) : content.sweep(false), (contents) => ({ ...swept, contents }))))).pipe(Effect.provideContext(services), Effect.provideService(FrameworkClock, frameworkClock), Effect.provideService(CleanupHooks, cleanupHooks));
+    yield* Effect.sleep(writerWindowMs / 2).pipe(Effect.andThen(refreshPayloadWriters.pipe(Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause)
+        ? Effect.interrupt
+        : Effect.logWarning("Payload writer refresh failed", cause)))), Effect.forever, Effect.forkIn(scope));
+    yield* Effect.addFinalizer(() => dropWriters(runtimeId).pipe(Effect.provideContext(services), Effect.catchCause((cause) => Effect.logWarning("Payload writer rows not dropped", cause))));
+    const sweeping = cleanupHooks.periodic
+        ? yield* Effect.sleep(CLEANUP_INTERVAL).pipe(Effect.andThen(cleanup.pipe(Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause)
+            ? Effect.interrupt
+            : Effect.logWarning("Retention cleanup failed", cause)))), Effect.forever, Effect.forkIn(scope))
+        : undefined;
+    const sampler = databaseSampler();
+    const sample = Effect.suspend(() => sampler(Array.from(registrations.values(), ({ name, policy }) => ({
+        actorType: name,
+        keepEventsMs: policy.keepEventsMs,
+        holdEventsMs: policy.holdEventsMs,
+    })))).pipe(Effect.provideContext(services), Effect.provideService(FrameworkClock, frameworkClock), Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause)
+        ? Effect.interrupt
+        : Effect.logWarning("Telemetry sampling failed", cause)));
+    if (!sampled.has(sharding)) {
+        sampled.add(sharding);
+        yield* sharding.registerSingleton("akter/telemetry", Effect.sleep(sampleEveryMs).pipe(Effect.andThen(sample), Effect.forever));
+    }
+    return { cleanup, sweeping, sample };
+});

package/dist/runtime/telemetry/defects.d.ts

@@ -0,0 +1,57 @@
+import { Context, Effect, Schema } from "effect";
+/**
+ * One deterministic defect, as its failed turn span recorded it: the span's
+ * name, trace, and attributes, and the cause the span carries.
+ */
+export declare const DefectRecord: Schema.Struct<{
+    readonly span: Schema.String;
+    readonly traceId: Schema.String;
+    readonly spanId: Schema.String;
+    readonly atMs: Schema.Finite;
+    readonly tenant: Schema.String;
+    readonly actorType: Schema.String;
+    readonly actorId: Schema.String;
+    readonly command: Schema.String;
+    readonly commandId: Schema.String;
+    readonly trigger: Schema.String;
+    readonly cause: Schema.String;
+}>;
+/** The decoded shape of a `DefectRecord`. */
+export type DefectRecord = typeof DefectRecord.Type;
+/** An array of defect records, as the operator read returns them. */
+export declare const DefectRecords: Schema.$Array<Schema.Struct<{
+    readonly span: Schema.String;
+    readonly traceId: Schema.String;
+    readonly spanId: Schema.String;
+    readonly atMs: Schema.Finite;
+    readonly tenant: Schema.String;
+    readonly actorType: Schema.String;
+    readonly actorId: Schema.String;
+    readonly command: Schema.String;
+    readonly commandId: Schema.String;
+    readonly trigger: Schema.String;
+    readonly cause: Schema.String;
+}>>;
+/** Narrows a defect listing; `limit` keeps the newest matches. */
+interface DefectFilter {
+    readonly actorType?: string | undefined;
+    readonly sinceMs?: number | undefined;
+    readonly limit?: number | undefined;
+}
+declare const DefectLog_base: Context.ServiceClass<DefectLog, "@rikalabs/akter/runtime/telemetry/defects/DefectLog", {
+    readonly record: (defect: DefectRecord) => Effect.Effect<void>;
+    readonly list: (filter: DefectFilter) => Effect.Effect<ReadonlyArray<DefectRecord>>;
+}>;
+/**
+ * The defect spans this runner's exporter saw most recently, newest last.
+ * The log is bounded and in memory: the telemetry backend keeps the history,
+ * and a restarted runner starts empty.
+ */
+export declare class DefectLog extends DefectLog_base {
+}
+/** An in-memory `DefectLog` that keeps the newest `capacity` records. */
+export declare const boundedDefectLog: (capacity: number) => {
+    readonly record: (defect: DefectRecord) => Effect.Effect<void>;
+    readonly list: (filter: DefectFilter) => Effect.Effect<ReadonlyArray<DefectRecord>>;
+};
+export {};

package/dist/runtime/telemetry/defects.js

@@ -0,0 +1,43 @@
+import { Context, Effect, Schema } from "effect";
+/**
+ * One deterministic defect, as its failed turn span recorded it: the span's
+ * name, trace, and attributes, and the cause the span carries.
+ */
+export const DefectRecord = Schema.Struct({
+    span: Schema.String,
+    traceId: Schema.String,
+    spanId: Schema.String,
+    atMs: Schema.Finite,
+    tenant: Schema.String,
+    actorType: Schema.String,
+    actorId: Schema.String,
+    command: Schema.String,
+    commandId: Schema.String,
+    trigger: Schema.String,
+    cause: Schema.String,
+});
+/** An array of defect records, as the operator read returns them. */
+export const DefectRecords = Schema.Array(DefectRecord);
+/**
+ * The defect spans this runner's exporter saw most recently, newest last.
+ * The log is bounded and in memory: the telemetry backend keeps the history,
+ * and a restarted runner starts empty.
+ */
+export class DefectLog extends Context.Service()("@rikalabs/akter/runtime/telemetry/defects/DefectLog") {
+}
+/** An in-memory `DefectLog` that keeps the newest `capacity` records. */
+export const boundedDefectLog = (capacity) => {
+    const records = [];
+    return DefectLog.of({
+        record: (defect) => Effect.sync(() => {
+            records.push(defect);
+            if (records.length > capacity)
+                records.splice(0, records.length - capacity);
+        }),
+        list: ({ actorType, sinceMs, limit }) => Effect.sync(() => {
+            const matching = records.filter((defect) => (actorType === undefined || defect.actorType === actorType) &&
+                (sinceMs === undefined || defect.atMs >= sinceMs));
+            return limit === undefined ? matching : matching.slice(-limit);
+        }),
+    });
+};

package/dist/runtime/telemetry/metrics.d.ts

@@ -0,0 +1,52 @@
+import { Effect, Metric } from "effect";
+/**
+ * Metric names and attribute keys are public, like span names. Prometheus
+ * exposition replaces `.` and `-` with `_`, so `akter.turns` is
+ * scraped as `akter_turns`. Attributes are bounded by the
+ * deployment's declarations (actor types, commands, jobs, subscriptions)
+ * or by fixed sets; no metric carries a tenant, actor id, or command id.
+ *
+ * The database gauges (`outboxRows` and the other sampled series) are read
+ * from the database by one runner of the deployment.
+ */
+export declare const Metrics: {
+    readonly turns: Metric.Counter<number>;
+    readonly turnDuration: Metric.Histogram<number>;
+    readonly mailboxAge: Metric.Histogram<number>;
+    readonly poolWait: Metric.Histogram<number>;
+    readonly activations: Metric.Gauge<number>;
+    readonly activationsStarted: Metric.Counter<number>;
+    readonly receiptsWritten: Metric.Counter<number>;
+    readonly receiptsReplayed: Metric.Counter<number>;
+    readonly receiptsPruned: Metric.Counter<number>;
+    readonly eventsAppended: Metric.Counter<number>;
+    readonly eventsPruned: Metric.Counter<number>;
+    readonly outboxStaged: Metric.Counter<number>;
+    readonly relayDelivered: Metric.Counter<number>;
+    readonly relayRetried: Metric.Counter<number>;
+    readonly deadLetters: Metric.Counter<number>;
+    readonly undeliverableGaps: Metric.Counter<number>;
+    readonly outboxRows: Metric.Gauge<number>;
+    readonly relayLag: Metric.Gauge<number>;
+    readonly stuckRows: Metric.Gauge<number>;
+    readonly subscriptionLagEvents: Metric.Gauge<number>;
+    readonly subscriptionLag: Metric.Gauge<number>;
+    readonly subscriptionPinned: Metric.Gauge<number>;
+    readonly workflowPinned: Metric.Gauge<number>;
+    readonly fleetLagBytes: Metric.Gauge<number>;
+    readonly fleetLag: Metric.Gauge<number>;
+    readonly fleetGroupsRecomputed: Metric.Counter<number>;
+    readonly fleetBatches: Metric.Counter<number>;
+};
+type Attributes = Readonly<Record<string, string>>;
+/** Adds `value` to `metric` under `attributes`; a zero adds nothing. Never fails. */
+export declare const count: {
+    (attributes: Attributes, value: number): (metric: Metric.Metric<number, unknown>) => Effect.Effect<void>;
+    (metric: Metric.Metric<number, unknown>, attributes: Attributes, value: number): Effect.Effect<void>;
+};
+/** Sets a gauge, or records a histogram observation, under `attributes`. */
+export declare const record: {
+    (attributes: Attributes, value: number): (metric: Metric.Metric<number, unknown>) => Effect.Effect<void>;
+    (metric: Metric.Metric<number, unknown>, attributes: Attributes, value: number): Effect.Effect<void>;
+};
+export {};

package/dist/runtime/telemetry/metrics.js

@@ -0,0 +1,138 @@
+import { Effect, Metric } from "effect";
+import { dual } from "effect/Function";
+const milliseconds = [1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000];
+/**
+ * Metric names and attribute keys are public, like span names. Prometheus
+ * exposition replaces `.` and `-` with `_`, so `akter.turns` is
+ * scraped as `akter_turns`. Attributes are bounded by the
+ * deployment's declarations (actor types, commands, jobs, subscriptions)
+ * or by fixed sets; no metric carries a tenant, actor id, or command id.
+ *
+ * The database gauges (`outboxRows` and the other sampled series) are read
+ * from the database by one runner of the deployment.
+ */
+export const Metrics = {
+    turns: Metric.counter("akter.turns", {
+        description: "Turns finished on this runner, by actor_type and outcome (success, failure, defect, replay, acknowledged).",
+        incremental: true,
+    }),
+    turnDuration: Metric.histogram("akter.turn.duration_ms", {
+        description: "Turn time from the mailbox to the reply, by actor_type.",
+        boundaries: milliseconds,
+    }),
+    mailboxAge: Metric.histogram("akter.mailbox.age_ms", {
+        description: "How long a command waited between its admission and the start of its turn, by actor_type.",
+        boundaries: milliseconds,
+    }),
+    poolWait: Metric.histogram("akter.pool.wait_ms", {
+        description: "How long a turn waited for a session from the turn pool (Postgres only).",
+        boundaries: milliseconds,
+    }),
+    activations: Metric.gauge("akter.activations", {
+        description: "Activations resident on this runner, by actor_type.",
+    }),
+    activationsStarted: Metric.counter("akter.activations.started", {
+        description: "Activations built on this runner, by actor_type, including rebuilds after a defect.",
+        incremental: true,
+    }),
+    receiptsWritten: Metric.counter("akter.receipts.written", {
+        description: "Receipts committed by turns on this runner, by actor_type.",
+        incremental: true,
+    }),
+    receiptsReplayed: Metric.counter("akter.receipts.replayed", {
+        description: "Commands answered from a stored receipt without running their handler, by actor_type.",
+        incremental: true,
+    }),
+    receiptsPruned: Metric.counter("akter.receipts.pruned", {
+        description: "Receipts retention deleted on this runner, by actor_type.",
+        incremental: true,
+    }),
+    eventsAppended: Metric.counter("akter.events.appended", {
+        description: "Events committed by turns on this runner, by actor_type.",
+        incremental: true,
+    }),
+    eventsPruned: Metric.counter("akter.events.pruned", {
+        description: "Events retention deleted on this runner, by actor_type.",
+        incremental: true,
+    }),
+    outboxStaged: Metric.counter("akter.outbox.staged", {
+        description: "Outbox rows committed by turns on this runner, by kind (intent, job).",
+        incremental: true,
+    }),
+    relayDelivered: Metric.counter("akter.relay.delivered", {
+        description: "Outbox and subscription deliveries this runner settled, by kind (intent, job, subscription).",
+        incremental: true,
+    }),
+    relayRetried: Metric.counter("akter.relay.retried", {
+        description: "Deliveries this runner backed off to retry, by kind (intent, job, subscription).",
+        incremental: true,
+    }),
+    deadLetters: Metric.counter("akter.job.dead_letters", {
+        description: "Jobs dead-lettered on this runner, by actor_type and job.",
+        incremental: true,
+    }),
+    undeliverableGaps: Metric.counter("akter.subscription.undeliverable_gaps", {
+        description: "Retention gaps an id-routed subscription row counted without a recipient, by subscriber_type and subscription.",
+        incremental: true,
+    }),
+    outboxRows: Metric.gauge("akter.outbox.rows", {
+        description: "Outbox rows waiting, by kind (intent, job, feed, control).",
+    }),
+    relayLag: Metric.gauge("akter.relay.lag_ms", {
+        description: "How long the oldest due, unclaimed row has been due, by kind (intent, job, subscription); 0 when none is due.",
+    }),
+    stuckRows: Metric.gauge("akter.relay.stuck_rows", {
+        description: "Rows claimed at least 8 times and still pending, by kind (intent, subscription).",
+    }),
+    subscriptionLagEvents: Metric.gauge("akter.subscription.lag_events", {
+        description: "The most source events any row of a subscription is behind, by subscriber_type and subscription.",
+    }),
+    subscriptionLag: Metric.gauge("akter.subscription.lag_ms", {
+        description: "The age of the oldest undelivered event of any row of a subscription, by subscriber_type and subscription.",
+    }),
+    subscriptionPinned: Metric.gauge("akter.subscription.pinned_events", {
+        description: "Events past keepEvents that subscriptions still hold back from retention, by source actor_type.",
+    }),
+    workflowPinned: Metric.gauge("akter.workflow.pinned_events", {
+        description: "Events past keepEvents that open workflow executions still hold back from retention, by actor_type.",
+    }),
+    fleetLagBytes: Metric.gauge("akter.fleet.lag_bytes", {
+        description: "WAL bytes between the server's flush position and what the fleet maintainer has applied, by view; on the runner that maintains.",
+    }),
+    fleetLag: Metric.gauge("akter.fleet.lag_ms", {
+        description: "Time since the fleet maintainer last drained the change feed, by view; on the runner that maintains.",
+    }),
+    fleetGroupsRecomputed: Metric.counter("akter.fleet.groups_recomputed", {
+        description: "Fleet view groups the maintainer recomputed, by view.",
+        incremental: true,
+    }),
+    fleetBatches: Metric.counter("akter.fleet.batches", {
+        description: "Change-feed batches the fleet maintainer applied and advanced past.",
+        incremental: true,
+    }),
+};
+const attributedMetrics = new WeakMap();
+/**
+ * The metric under `attributes`, made once per attribute set and reused. A
+ * fresh `Metric.withAttributes` re-derives its series key, sorting and
+ * serializing the attributes, on every update. Attribute sets are bounded by
+ * the deployment's declarations, so the cache is too.
+ */
+const attributed = (metric, attributes) => {
+    let byKey = attributedMetrics.get(metric);
+    if (byKey === undefined) {
+        byKey = new Map();
+        attributedMetrics.set(metric, byKey);
+    }
+    const key = JSON.stringify(Object.entries(attributes).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0));
+    let cached = byKey.get(key);
+    if (cached === undefined) {
+        cached = Metric.withAttributes(metric, attributes);
+        byKey.set(key, cached);
+    }
+    return cached;
+};
+/** Adds `value` to `metric` under `attributes`; a zero adds nothing. Never fails. */
+export const count = dual(3, (metric, attributes, value) => value === 0 ? Effect.void : Metric.update(attributed(metric, attributes), value));
+/** Sets a gauge, or records a histogram observation, under `attributes`. */
+export const record = dual(3, (metric, attributes, value) => Metric.update(attributed(metric, attributes), value));

package/dist/runtime/telemetry/routes.d.ts

@@ -0,0 +1,19 @@
+import { HttpRouter } from "effect/http";
+/** Configuration for `Telemetry.serve`. */
+export interface TelemetryOptions {
+    /** Prefix for the route. Default none: `/metrics`. */
+    readonly basePath?: `/${string}`;
+}
+/**
+ * Serves this runner's metrics on the application's `HttpRouter` as
+ * `GET /metrics`, in Prometheus text exposition. Metric attributes carry no
+ * tenant or actor id, so the route is unauthenticated, like any scrape
+ * target; keep it on a private listener. Defect spans are an operator read,
+ * served by `Operators.serve`.
+ */
+declare const serve: (options?: TelemetryOptions) => import("effect/Layer").Layer<never, never, HttpRouter.HttpRouter>;
+/** Runner telemetry: `serve` mounts the Prometheus route. */
+export declare const Telemetry: {
+    serve: typeof serve;
+};
+export {};

package/dist/runtime/telemetry/routes.js

@@ -0,0 +1,18 @@
+import { Effect } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/http";
+import { PrometheusMetrics } from "effect/observability";
+/**
+ * Serves this runner's metrics on the application's `HttpRouter` as
+ * `GET /metrics`, in Prometheus text exposition. Metric attributes carry no
+ * tenant or actor id, so the route is unauthenticated, like any scrape
+ * target; keep it on a private listener. Defect spans are an operator read,
+ * served by `Operators.serve`.
+ */
+const serve = (options = {}) => HttpRouter.use(Effect.fnUntraced(function* (router) {
+    const basePath = (options.basePath ?? "").replace(/\/+$/, "");
+    yield* router.add("GET", `${basePath}/metrics`, () => PrometheusMetrics.format().pipe(Effect.map((body) => HttpServerResponse.text(body, {
+        contentType: "text/plain; version=0.0.4; charset=utf-8",
+    }))));
+}));
+/** Runner telemetry: `serve` mounts the Prometheus route. */
+export const Telemetry = { serve };

package/dist/runtime/telemetry/sampler.d.ts

@@ -0,0 +1,26 @@
+import { Context, Effect } from "effect";
+import { SqlClient } from "effect/sql";
+/** An actor type's event-retention settings, in milliseconds, as the sampler reads them. */
+interface SampledType {
+    readonly actorType: string;
+    readonly keepEventsMs: number;
+    readonly holdEventsMs: number;
+}
+/**
+ * Reports the gauges only the database knows. Each statement reads pending
+ * work or retained history past its horizon, never every stored actor: the
+ * outbox and due subscription rows are the relay's backlog, and events past
+ * `keepEvents` stay only while something pins them. A series that stops
+ * appearing is reported as 0 once, so a recovered subscription or an emptied
+ * kind does not keep its last value. Only due or backing-off subscription rows
+ * have undelivered events, which the partial due index covers, and pinned
+ * events past `keepEvents` are read from the pinned rows alone.
+ */
+export declare const databaseSampler: () => (types: readonly SampledType[]) => Effect.Effect<void, import("effect/sql/SqlError").SqlError, SqlClient.SqlClient>;
+declare const TelemetrySampler_base: Context.ServiceClass<TelemetrySampler, "@rikalabs/akter/runtime/telemetry/sampler/TelemetrySampler", {
+    readonly sample: Effect.Effect<void>;
+}>;
+/** Samples the database gauges at once; the runtime also samples every `observability.sampleEvery`. */
+export declare class TelemetrySampler extends TelemetrySampler_base {
+}
+export {};