@cosmicdrift/kumiko-framework 0.383.1 → 0.385.0

package/dist/jobs/job-runner.js

@@ -1,6 +1,5 @@
 /// <reference types="temporal-polyfill/global" preserve="true" />
 import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
-import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
 import { DelayedError, Queue, UnrecoverableError, Worker } from "bullmq";
 import { Redis } from "ioredis";
 import { requestContext } from "../api/request-context.js";
@@ -9,398 +8,22 @@ import { createDerivativesContext } from "../derivatives/derivatives-context.js"
 import { bridgeStub } from "../engine/bridge-stub.js";
 import { qnScope } from "../engine/qualified-name.js";
 import { createSystemUser } from "../engine/system-user.js";
-import { SYSTEM_TENANT_ID, } from "../engine/types/index.js";
+import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
 import { InternalError } from "../errors/index.js";
-import { isKumikoError } from "../errors/kumiko-error.js";
 import { createTenantFileContext } from "../files/file-handle.js";
 import { createFallbackLogger, scrubLogText } from "../logging/index.js";
-import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
+import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { configAccessorFactoryOf } from "../pipeline/config-accessor-factory.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
 import { RedisKeys } from "../pipeline/redis-keys.js";
 import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js";
-import { TENANT_TIMEZONE_CONFIG_KEY } from "../pipeline/tenant-timezone-config-key.js";
-import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
-import { createTzContext, isValidIanaTimeZone } from "../time/index.js";
+import { buildPersonalDataGate, parseWriteOrigin } from "../pipeline/write-origin.js";
 import { cronRepeatOptions } from "./cron-repeat-options.js";
+import { BOOT_GATE_SLOW_WARN_MS, BOOT_REDIS_TIMEOUT_MS, buildRetryBullOpts, COMPLETED_JOB_RETENTION_AGE_SEC, captureTraceContext, createCrossTenantReads, createJobLogger, DEFAULT_MAX_DEFERS, DEFER_BACKOFF_MAX_DOUBLINGS, DEFER_COUNT_KEY, DEFER_DELAY_CAP_MS, FAILED_JOB_RETENTION_AGE_SEC, HANDLER_DEFER_COUNT_KEY, isJobTriggerEvent, jobSubjectKey, maxRetryWindowMs, missingDbProxy, parseRedisOpts, payloadMatchesTriggerFilter, priorAttemptsOf, readTraceContext, resolveJobTz, STOP_QUEUE_READY_TIMEOUT_MS, stampGatedWriteOrigin, startQueueDepthPolling, TRACE_CONTEXT_KEY, tenantFailureMessageKey, timeoutReject, } from "./job-runner-helpers.js";
+import { ACTIVE_TENANT_IDS_QUERY_NAME, bootJobIdForJobName, DEFAULT_QUEUE_NAME_PREFIX, markSchedulersSeen, ORPHAN_SCHEDULER_GRACE_MS, perTenantChildJobId, queueNameFor, removeOrphanJobSchedulers, schedulerIdForJobName, } from "./job-runner-schedulers.js";
 import { resolveMaxPerTenant } from "./max-per-tenant.js";
-// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
-// (a security-relevant escape hatch that must fail the run closed), a
-// malformed `_triggerEvent` only costs an idempotency-key convenience — the
-// run proceeds without it rather than failing over what would be a framework
-// bug, not untrusted input.
-function isJobTriggerEvent(value) {
-    if (!isPlainObject(value))
-        return false;
-    if (typeof value["id"] !== "string")
-        return false;
-    const headers = value["headers"];
-    if (typeof headers !== "object" || headers === null)
-        return false;
-    return Object.values(headers).every((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean");
-}
-// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
-function stampGatedWriteOrigin(data) {
-    delete data["_writeOrigin"];
-    const origin = requestContext.get()?.writeOrigin;
-    if (origin && isPersonalDataGated(origin)) {
-        data["_writeOrigin"] = origin;
-    }
-}
-// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
-// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
-// accidental drift would silently drop jobs. Tests override via
-// `queueNamePrefix` for per-run isolation (stale jobs from a prior run
-// don't leak into a new test because the queue name includes a timestamp).
-const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
-function queueNameFor(prefix, lane) {
-    return `${prefix}-${lane}`;
-}
-// perTenant jobs need a source of truth for "which tenants are active" —
-// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
-// (bundled-features package) registers a query under this exact name when
-// mounted. framework must not depend on bundled-features (wrong direction —
-// bundled-features already depends on framework), so this is a name-only
-// coupling: createJobRunner looks the handler up in the registry by string
-// instead of importing it.
-const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
-/**
- * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
- * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
- * treated as old repeatables — spawning a new scheduler entry every tick and
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
- * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
- */
-export function schedulerIdForJobName(jobName) {
-    return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
-}
-// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
-// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
-export function bootJobIdForJobName(jobName) {
-    return `boot-${jobName.replace(/[.:]/g, "-")}`;
-}
-const SCHEDULER_PAGE_SIZE = 100;
-// Rolling deploys run old and new pods against the same Redis for a while; an
-// old runner restarting in that window must not delete the scheduler of a
-// cron job the new version just added.
-export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
-function schedulerSeenKey(queue) {
-    return queue.toKey("kumiko-scheduler-seen");
-}
-// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
-// their own ids periodically; a single stamp at start() would let a runner
-// restarting during a mixed-version window longer than the grace period delete
-// a scheduler a healthy newer pod still owns.
-async function markSchedulersSeen(queue, schedulerIds) {
-    // skip: no cron scheduler on this lane, nothing to stamp
-    if (schedulerIds.size === 0)
-        return;
-    const stamp = String(Date.now());
-    const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
-    await (await queue.client).hset(schedulerSeenKey(queue), entries);
-}
-// A cron job dropped from the app (or an older scheduler-id format) leaves its
-// scheduler in Redis, where it keeps firing jobs the worker no longer knows
-// ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
-async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
-    try {
-        // Collect first: removing while paging would shift the indices.
-        const existingKeys = [];
-        for (let start = 0;; start += SCHEDULER_PAGE_SIZE) {
-            const page = await queue.getJobSchedulers(start, start + SCHEDULER_PAGE_SIZE - 1);
-            for (const scheduler of page)
-                existingKeys.push(scheduler.key);
-            if (page.length < SCHEDULER_PAGE_SIZE)
-                break;
-        }
-        const client = await queue.client;
-        const seenAt = await client.hgetall(schedulerSeenKey(queue));
-        const now = Date.now();
-        for (const key of existingKeys) {
-            if (expectedSchedulerIds.has(key))
-                continue;
-            const lastSeen = Number(seenAt[key]);
-            if (key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS)
-                continue;
-            await queue.removeJobScheduler(key);
-            await client.hdel(schedulerSeenKey(queue), key);
-            log.info("removed orphan job scheduler", { schedulerId: key, lane });
-        }
-    }
-    catch (err) {
-        log.warn("orphan job scheduler cleanup failed", {
-            lane,
-            error: err instanceof Error ? err.message : String(err),
-        });
-    }
-}
-// Deterministic per-tenant child job id, derived from the perTenant
-// wrapper's own BullMQ job id. A cron tick's wrapper id is
-// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
-// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
-// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
-// across retries of that job. Reusing it means a wrapper that runs twice
-// for the same trigger (retry after failure, a Redis drop, an instance
-// restarting mid-run) re-derives the *same* child ids, and BullMQ's
-// existing-jobId add() no-ops the second batch instead of creating
-// duplicates — safe only as long as the child job hash outlives the
-// wrapper's retry window, enforced at construction against
-// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
-// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
-// (fw#1603/#1604) — the wrapper id already contains ":", so strip
-// separators from both halves before joining instead of interpolating raw.
-function perTenantChildJobId(wrapperJobId, tenantId) {
-    const sanitize = (value) => value.replace(/[.:]/g, "-");
-    return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
-}
-function createJobLogger(logs, baseContext = {}) {
-    function push(level, msg, data) {
-        const merged = { ...baseContext, ...data };
-        const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
-        logs.push({ level, message, timestamp: Temporal.Now.instant() });
-    }
-    const logger = {
-        info(msg, data) {
-            push("info", msg, data);
-        },
-        warn(msg, data) {
-            push("warn", msg, data);
-        },
-        error(msg, data) {
-            push("error", msg, data);
-        },
-        // No-op by design, not missing coverage: JobLogEntry.level (and the
-        // persisted run-completed/-failed event schema in bundled-features/
-        // jobs/events.ts) only has info|warn|error — a job's debug() calls have
-        // nowhere durable to land.
-        debug() { },
-        child(context) {
-            return createJobLogger(logs, { ...baseContext, ...context });
-        },
-    };
-    return logger;
-}
-// Single source of the stored subject string: the writer (jobSubjectKey) and
-// the tenant failures query both go through it. Matching is exact — field names
-// and JSON types must equal what the writer stored (`"1"` is not `1`).
-export function serializeJobSubject(subject) {
-    return JSON.stringify(Object.keys(subject)
-        .sort()
-        .map((field) => [field, subject[field] ?? null]));
-}
-// Stable identity for the declared payload fields: sorted keys, so two runs
-// with the same subject values produce the same string and the later one
-// replaces the earlier record. Non-primitives would serialize into something
-// no caller can reconstruct for a lookup, so they fail the run loudly.
-function jobSubjectKey(jobName, payload, fields, declaration) {
-    if (fields === undefined || fields.length === 0)
-        return null;
-    const subject = {};
-    for (const field of fields) {
-        const value = payload[field] ?? null;
-        if (value !== null && typeof value === "object") {
-            throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
-        }
-        subject[field] = value;
-    }
-    return serializeJobSubject(subject);
-}
-async function resolveJobTz(config, context) {
-    const tenantTz = config ? await config(TENANT_TIMEZONE_CONFIG_KEY) : undefined;
-    return createTzContext({
-        ...(context.geoTzProvider !== undefined && { geoTz: context.geoTzProvider }),
-        ...(context.clock !== undefined && { clock: context.clock }),
-        tenant: typeof tenantTz === "string" && isValidIanaTimeZone(tenantTz) ? tenantTz : "UTC",
-    });
-}
-// Jobs that never touch ctx.db run fine on a runner built without one; a job
-// that does gets a named error instead of a bare TypeError on undefined.
-function missingDbProxy(jobName) {
-    // @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
-    return new Proxy({}, {
-        get() {
-            throw new InternalError({
-                message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
-            });
-        },
-    });
-}
-function createCrossTenantReads(systemModeDb, reason, report) {
-    return {
-        selectMany(table, where, options) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.selectMany(table, where, options);
-        },
-        fetchOne(table, where) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.fetchOne(table, where);
-        },
-        count(table, where) {
-            report("cross-tenant-read", reason);
-            return systemModeDb.count(table, where);
-        },
-    };
-}
-function priorAttemptsOf(data) {
-    const value = data["_priorAttempts"];
-    return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
-}
-// Only a translation key ever travels to the tenant: the thrown error's own
-// i18nKey when it carries one, otherwise the key declared at the job. The
-// error message itself can echo provider or payload content and stays on the
-// run row and in the run log.
-function tenantFailureMessageKey(err, declaration) {
-    // skip: job did not opt in — nothing tenant-visible to record
-    if (!declaration)
-        return null;
-    if (isKumikoError(err) && err.i18nKey.length > 0)
-        return err.i18nKey;
-    return declaration.messageKey;
-}
-// Serialized trace context lives under this key in the BullMQ job data.
-// Leading underscore matches the existing internal-meta convention
-// (_triggeredById, _tenantId, _payload).
-const TRACE_CONTEXT_KEY = "_traceContext";
-const DEFER_COUNT_KEY = "_deferCount";
-// Separate from the maxPerTenant counter so tenant-limit deferrals do not eat into maxDefers.
-const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
-const DEFAULT_MAX_DEFERS = 100;
-const DEFER_DELAY_CAP_MS = 30_000;
-const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
-function readTraceContext(data) {
-    const raw = data[TRACE_CONTEXT_KEY];
-    if (!raw || typeof raw !== "object")
-        return undefined;
-    const ctx = raw; // @cast-boundary engine-payload
-    if (!ctx.traceId || !ctx.spanId)
-        return undefined;
-    return { traceId: ctx.traceId, spanId: ctx.spanId };
-}
-function captureTraceContext(tracer) {
-    const span = tracer.getActiveSpan();
-    if (!span?.traceId)
-        return undefined;
-    return { traceId: span.traceId, spanId: span.spanId };
-}
-const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
-const QUEUE_DEPTH_WARN_INTERVAL_MS = 5 * 60_000;
-// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
-// a global Redis-backed value (BullMQ, not per-process), so one reporter per
-// lane is enough. Fires once immediately (metrics exist before the first
-// poll tick) then on an interval; the caller stores + clears the handle.
-async function startQueueDepthPolling(queue, lane, meter) {
-    let lastWarnAt;
-    const pollQueueDepth = async () => {
-        try {
-            const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
-            emitJobQueueDepth(meter, lane, counts);
-        }
-        catch (err) {
-            // Next poll retries; the warn is throttled so a Redis outage logs once per interval, not once per poll.
-            const nowMs = Date.now();
-            if (lastWarnAt === undefined || nowMs - lastWarnAt >= QUEUE_DEPTH_WARN_INTERVAL_MS) {
-                lastWarnAt = nowMs;
-                createFallbackLogger("job-runner").warn("queue depth poll failed", {
-                    lane,
-                    error: scrubLogText(err instanceof Error ? err.message : String(err)),
-                });
-            }
-        }
-    };
-    await pollQueueDepth();
-    return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
-}
-function parseRedisOpts(url) {
-    const parsed = new URL(url);
-    const result = {
-        host: parsed.hostname,
-        port: Number(parsed.port) || 6379,
-    };
-    if (parsed.pathname.length > 1) {
-        result.db = Number(parsed.pathname.slice(1));
-    }
-    return result;
-}
-// redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
-// would otherwise hang start() forever with no health endpoint to notice.
-const BOOT_REDIS_TIMEOUT_MS = 10_000;
-// Shutdown must fit a k8s grace period even when Redis is down, so the
-// readiness wait in stop() is capped well below the boot timeout.
-const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
-// A gate without an explicit `timeout` may legitimately run long (migration-style
-// checks), so it is never rejected; a hang just must not stay silent.
-const BOOT_GATE_SLOW_WARN_MS = 60_000;
-function timeoutReject(ms, message) {
-    let timer;
-    const promise = new Promise((_, reject) => {
-        timer = setTimeout(() => reject(new Error(message)), ms);
-    });
-    return {
-        promise,
-        cancel: () => {
-            if (timer !== undefined)
-                clearTimeout(timer);
-        },
-    };
-}
-// Default base delay when a job opts into backoff without an explicit
-// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
-// NaN/undefined, which is falsy — the job retries immediately instead of
-// waiting.
-const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
-// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
-// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
-// set must retry the same way regardless of how it got enqueued, or it
-// fails for good on the very first error on whichever path skips this.
-function buildRetryBullOpts(jobDef, priorAttempts = 0) {
-    const opts = {};
-    if (jobDef.retries !== undefined)
-        opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
-    if (jobDef.backoff) {
-        opts.backoff =
-            typeof jobDef.backoff === "string"
-                ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
-                : {
-                    type: jobDef.backoff.type,
-                    delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
-                };
-    }
-    return opts;
-}
-// Equality filter on top-level payload fields for a job's trigger.where —
-// lets N jobs share one broad event QN, partitioned by a payload
-// discriminant instead of needing N distinct event types.
-function payloadMatchesTriggerFilter(where, payload) {
-    if (!where)
-        return true;
-    return Object.entries(where).every(([key, value]) => payload[key] === value);
-}
-// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
-// OTHER jobs in the same completed/failed zset (moveToFinished lua,
-// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
-// the sweep only runs lazily on a later finish into that set. Per-job
-// retention is therefore meaningless on a shared queue — only queue-wide,
-// age-only retention (no count) is safe: a count would evict perTenant
-// children still inside their wrapper's retry window, and boot jobs. The
-// completed age must stay above every perTenant wrapper's own worst-case
-// retry window (see maxRetryWindowMs and the invariant check in
-// createJobRunner below), because child dedup (perTenantChildJobId) relies
-// on the children still existing in Redis when a wrapper retry re-derives
-// their ids. The margin here is generous — queue wait time isn't bounded by
-// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
-// not BullMQ (fw#3199).
-const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
-const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
-// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
-// retries, consistent with buildRetryBullOpts's own backoff computation.
-function maxRetryWindowMs(jobDef) {
-    if (!jobDef.backoff || jobDef.retries === undefined)
-        return 0;
-    const delayMs = (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
-        DEFAULT_JOB_BACKOFF_DELAY_MS;
-    const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
-    return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
-}
+export { serializeJobSubject } from "./job-runner-helpers.js";
+export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.js";
 export function createJobRunner(options) {
     const { registry, context, redisUrl, consumerLane } = options;
     const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;

package/dist/pipeline/dispatch-shared.js

@@ -576,8 +576,8 @@ readOnlyForTenant = false) {
             await appendDomainEvent(ctx, args, user, tx, registry.getHandlerFeature(type));
         },
         // Savepoint-scoped append: catches a losing writer's VersionConflictError
-        // without poisoning the rest of the handler's transaction. Bun.SQL/
-        // postgres.js abort the WHOLE begin() on an uncaught statement error
+        // without poisoning the rest of the handler's transaction. postgres.js
+        // aborts the WHOLE begin() on an uncaught statement error
         // (SQLSTATE 25P02) even if the JS error is caught — runInSavepoint
         // wraps the append in a real SAVEPOINT so only that nested scope rolls
         // back on conflict, and subsequent statements in the outer tx (e.g. a

package/dist/pipeline/event-dispatcher-admin.d.ts

@@ -1,5 +1,10 @@
 /// <reference types="temporal-polyfill/global" preserve="true" />
-import type { DbConnection } from "../db/connection.ts";
+import type { DbRunner } from "../db/connection.ts";
+export type ConsumerTransitionRejection = "unknown-consumer" | "wrong-status";
+export declare class ConsumerTransitionError extends Error {
+    readonly rejection: ConsumerTransitionRejection;
+    constructor(rejection: ConsumerTransitionRejection, message: string);
+}
 export type ConsumerRecoveryState = {
     readonly name: string;
     readonly instanceId: string;
@@ -10,13 +15,15 @@ export type ConsumerRecoveryState = {
     readonly lastError: string | null;
     readonly updatedAt: Temporal.Instant;
 };
-export declare function restartConsumer(db: DbConnection, name: string, instanceId?: string): Promise<ConsumerRecoveryState>;
-export declare function disableConsumer(db: DbConnection, name: string, instanceId?: string): Promise<ConsumerRecoveryState>;
-export declare function enableConsumer(db: DbConnection, name: string, instanceId?: string): Promise<ConsumerRecoveryState>;
-export declare function skipPoisonEvent(db: DbConnection, name: string, instanceId?: string): Promise<ConsumerRecoveryState & {
+export declare function restartConsumer(db: DbRunner, name: string, instanceId?: string): Promise<ConsumerRecoveryState>;
+export declare function disableConsumer(db: DbRunner, name: string, instanceId?: string): Promise<ConsumerRecoveryState>;
+export declare function enableConsumer(db: DbRunner, name: string, instanceId?: string): Promise<ConsumerRecoveryState>;
+export declare function skipPoisonEvent(db: DbRunner, name: string, instanceId?: string, options?: {
+    readonly requireStatus?: string;
+}): Promise<ConsumerRecoveryState & {
     readonly skippedEventId: bigint | null;
 }>;
-export declare function getConsumerState(db: DbConnection, name: string, instanceId?: string): Promise<{
+export declare function getConsumerState(db: DbRunner, name: string, instanceId?: string): Promise<{
     readonly name: string;
     readonly instanceId: string;
     readonly status: string;
@@ -26,7 +33,7 @@ export declare function getConsumerState(db: DbConnection, name: string, instanc
     readonly lastError: string | null;
     readonly updatedAt: Temporal.Instant;
 } | null>;
-export declare function listConsumersWithState(db: DbConnection, registeredNames: readonly string[]): Promise<ReadonlyArray<{
+export declare function listConsumersWithState(db: DbRunner, registeredNames: readonly string[]): Promise<ReadonlyArray<{
     readonly name: string;
     readonly instanceId: string;
     readonly status: string;
@@ -44,4 +51,4 @@ export type ConsumerProgress = {
     readonly highWaterMark: bigint;
     readonly lag: bigint;
 };
-export declare function getAllConsumerProgress(db: DbConnection, registeredNames: readonly string[]): Promise<readonly ConsumerProgress[]>;
+export declare function getAllConsumerProgress(db: DbRunner, registeredNames: readonly string[]): Promise<readonly ConsumerProgress[]>;

package/dist/pipeline/event-dispatcher-admin.js

@@ -1,6 +1,6 @@
 import { advanceConsumerPastEventReturning, removePendingGapReturning, selectConsumerForUpdate, selectSnapshotXmax, updateConsumerStatusReturning, } from "../db/queries/event-consumer.js";
 import { selectNextEventIdAfter, selectSmallestVisibleIdInRanges, } from "../db/queries/event-store.js";
-import { coerceRow, extractTableInfo, selectMany } from "../db/query.js";
+import { coerceRow, extractTableInfo, runInNewTransaction, runInSavepointIfSupported, runnerCanBeginTransaction, selectMany, } from "../db/query.js";
 import { getEventsHighWaterMark } from "../event-store/index.js";
 import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.js";
 import { mergeContiguousGapRanges, rangeContainsId, splitRangeExcludingIds, toIdRanges, } from "./pending-gap-ranges.js";
@@ -33,6 +33,14 @@ function normalizeConsumerState(row) {
         updatedAt: row.updatedAt,
     };
 }
+export class ConsumerTransitionError extends Error {
+    rejection;
+    constructor(rejection, message) {
+        super(message);
+        this.rejection = rejection;
+        this.name = "ConsumerTransitionError";
+    }
+}
 // Ops calls default to the SHARED_INSTANCE_SENTINEL row — that's the only
 // row shared-delivery consumers have, so legacy CLI invocations without
 // --instance-id keep working. Per-instance consumers require an explicit
@@ -45,7 +53,7 @@ async function requireConsumerRow(db, name, instanceId) {
         instanceId,
     });
     if (!row) {
-        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") has no state row — it hasn't run yet, the name is misspelled, or the instance is misspelled. ` +
+        throw new ConsumerTransitionError("unknown-consumer", `Consumer "${name}" (instance_id="${instanceId}") has no state row — it hasn't run yet, the name is misspelled, or the instance is misspelled. ` +
             `For per-instance consumers pass the instance_id explicitly; shared consumers use the default.`);
     }
     return row;
@@ -53,7 +61,7 @@ async function requireConsumerRow(db, name, instanceId) {
 async function requireConsumerRowForUpdate(tx, name, instanceId) {
     const raw = await selectConsumerForUpdate(tx, name, instanceId);
     if (!raw) {
-        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") has no state row — it hasn't run yet, the name is misspelled, or the instance is misspelled. ` +
+        throw new ConsumerTransitionError("unknown-consumer", `Consumer "${name}" (instance_id="${instanceId}") has no state row — it hasn't run yet, the name is misspelled, or the instance is misspelled. ` +
             `For per-instance consumers pass the instance_id explicitly; shared consumers use the default.`);
     }
     return coerceRow(raw, extractTableInfo(eventConsumerStateTable));
@@ -69,7 +77,7 @@ async function applyConsumerStatusTransition(db, name, instanceId, targetStatus)
 export async function restartConsumer(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
     const before = await requireConsumerRow(db, name, instanceId);
     if (before.status !== "dead") {
-        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") is not dead (status="${before.status}"). Restart only applies to dead consumers; use "enable" for a disabled one.`);
+        throw new ConsumerTransitionError("wrong-status", `Consumer "${name}" (instance_id="${instanceId}") is not dead (status="${before.status}"). Restart only applies to dead consumers; use "enable" for a disabled one.`);
     }
     return applyConsumerStatusTransition(db, name, instanceId, "idle");
 }
@@ -80,10 +88,17 @@ export async function disableConsumer(db, name, instanceId = SHARED_INSTANCE_SEN
 export async function enableConsumer(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
     const before = await requireConsumerRow(db, name, instanceId);
     if (before.status !== "disabled") {
-        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") is not disabled (status="${before.status}"). Enable only flips disabled → idle; use "restart" for a dead consumer.`);
+        throw new ConsumerTransitionError("wrong-status", `Consumer "${name}" (instance_id="${instanceId}") is not disabled (status="${before.status}"). Enable only flips disabled → idle; use "restart" for a dead consumer.`);
     }
     return applyConsumerStatusTransition(db, name, instanceId, "idle");
 }
+// A pool connection opens its own transaction; inside a handler transaction the
+// locked read and writes join it, confined to a savepoint.
+function runInOwnTransactionOrSavepoint(db, fn) {
+    return runnerCanBeginTransaction(db)
+        ? runInNewTransaction(db, fn)
+        : runInSavepointIfSupported(db, fn);
+}
 // Delivery stops at the poison event, so ids between the cursor and it were
 // invisible during that pass (in-flight commits); keeping them as a pending gap
 // avoids losing them when the cursor jumps over.
@@ -97,6 +112,11 @@ async function invisibleWindowGap(tx, cursor, poisonId) {
         xmax: await selectSnapshotXmax(tx),
     };
 }
+function assertRequiredStatus(row, name, instanceId, requiredStatus) {
+    if (requiredStatus !== undefined && row.status !== requiredStatus) {
+        throw new ConsumerTransitionError("wrong-status", `Consumer "${name}" (instance_id="${instanceId}") is "${row.status}", not "${requiredStatus}"; nothing skipped.`);
+    }
+}
 // skipPoisonEvent skips the event whose handler failed last
 // (last_failed_event_id, set by the delivery's halt-on-poison branch): inside
 // pending_gaps it is split out of its range, otherwise the cursor moves onto
@@ -104,11 +124,12 @@ async function invisibleWindowGap(tx, cursor, poisonId) {
 // falls back to a guess: the smallest visible pending-gap id, else the next
 // event after the cursor. Single TX so concurrent dispatcher passes can't
 // double-advance. Nothing to skip → idempotent no-op.
-export async function skipPoisonEvent(db, name, instanceId = SHARED_INSTANCE_SENTINEL) {
-    return db.begin(async (tx) => {
+export async function skipPoisonEvent(db, name, instanceId = SHARED_INSTANCE_SENTINEL, options = {}) {
+    return runInOwnTransactionOrSavepoint(db, async (tx) => {
         // Locked read: a dispatcher turn committing new gaps between an unlocked
         // read and the pending_gaps overwrite below would have them dropped.
         const before = await requireConsumerRowForUpdate(tx, name, instanceId);
+        assertRequiredStatus(before, name, instanceId, options.requireStatus);
         const pendingGaps = before.pendingGaps;
         const failedEventId = before.lastFailedEventId;
         const failedIsPending = failedEventId !== null && pendingGaps.some((gap) => rangeContainsId(gap, failedEventId));

package/dist/pipeline/event-dispatcher.d.ts

@@ -49,5 +49,5 @@ export type EventDispatcherOptions = {
     readonly pgClient?: PgListenClient;
 };
 export declare function createEventDispatcher(options: EventDispatcherOptions): EventDispatcher;
-export type { ConsumerProgress, ConsumerRecoveryState } from "./event-dispatcher-admin.ts";
-export { disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher-admin.ts";
+export type { ConsumerProgress, ConsumerRecoveryState, ConsumerTransitionRejection, } from "./event-dispatcher-admin.ts";
+export { ConsumerTransitionError, disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher-admin.ts";

package/dist/pipeline/event-dispatcher.js

@@ -21,7 +21,6 @@ const CONNECTION_LOSS_CODES = new Set([
     "CONNECT_TIMEOUT",
     "ECONNREFUSED",
     "ECONNRESET",
-    "ERR_POSTGRES_CONNECTION_CLOSED",
     "57P01",
 ]);
 function isConnectionLossError(e) {
@@ -506,7 +505,7 @@ export function createEventDispatcher(options) {
                         // Fires on initial connect and again when the driver's own
                         // re-LISTEN lands after a drop; during an outage that re-LISTEN
                         // waits in its connection backoff (postgres-js: measured 10-30s
-                        // after the DB returns, Bun.SQL: up to 32s), polling covers
+                        // after the DB returns), polling covers
                         // delivery meanwhile. Never unlisten()
                         // + listen() by hand: that leaves duplicate listeners.
                         // ponytail: no own LISTEN reconnect, the backoff gap is polled;
@@ -534,7 +533,7 @@ export function createEventDispatcher(options) {
                 timer = null;
             }
             if (pgSubscription) {
-                // Called on the subscription: Bun.SQL's unlisten is a prototype method.
+                // Called on the subscription, not detached: unlisten may rely on `this`.
                 await pgSubscription.unlisten().catch(() => {
                     // skip: unlisten failure only matters during shutdown — the
                     // subscription is being torn down anyway.
@@ -573,4 +572,4 @@ export function createEventDispatcher(options) {
         runOnce,
     };
 }
-export { disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher-admin.js";
+export { ConsumerTransitionError, disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher-admin.js";

package/dist/pipeline/index.d.ts

@@ -19,8 +19,8 @@ export type { ConsumerStatus, PendingGapEntry } from "./event-consumer-state.ts"
 export { ConsumerStatuses, createEventConsumerStateTable, eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.ts";
 export type { EventDedup } from "./event-dedup.ts";
 export { createEventDedup } from "./event-dedup.ts";
-export type { ConsumerProgress, ConsumerRecoveryState, DispatcherPassResult, EventConsumer, EventConsumerBatchHandler, EventConsumerHandler, EventDispatcher, EventDispatcherOptions, } from "./event-dispatcher.ts";
-export { createEventDispatcher, disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher.ts";
+export type { ConsumerProgress, ConsumerRecoveryState, ConsumerTransitionRejection, DispatcherPassResult, EventConsumer, EventConsumerBatchHandler, EventConsumerHandler, EventDispatcher, EventDispatcherOptions, } from "./event-dispatcher.ts";
+export { ConsumerTransitionError, createEventDispatcher, disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher.ts";
 export type { ConsumerCursor } from "./event-dispatcher-delivery.ts";
 export { selectConsumerCursorForUpdate } from "./event-dispatcher-delivery.ts";
 export type { PruneEventsOptions, PruneEventsResult } from "./event-retention.ts";

package/dist/pipeline/index.js

@@ -10,7 +10,7 @@ export { createDistributedLock } from "./distributed-lock.js";
 export { createEntityCache } from "./entity-cache.js";
 export { ConsumerStatuses, createEventConsumerStateTable, eventConsumerStateTable, SHARED_INSTANCE_SENTINEL, } from "./event-consumer-state.js";
 export { createEventDedup } from "./event-dedup.js";
-export { createEventDispatcher, disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher.js";
+export { ConsumerTransitionError, createEventDispatcher, disableConsumer, enableConsumer, getAllConsumerProgress, getConsumerState, listConsumersWithState, restartConsumer, skipPoisonEvent, } from "./event-dispatcher.js";
 export { selectConsumerCursorForUpdate } from "./event-dispatcher-delivery.js";
 export { ConsumerLagError, pruneEvents } from "./event-retention.js";
 export { createIdempotencyGuard } from "./idempotency.js";