@cosmicdrift/kumiko-framework 0.382.0 → 0.383.0

package/dist/jobs/job-runner.js

@@ -13,7 +13,7 @@ import { SYSTEM_TENANT_ID, } from "../engine/types/index.js";
 import { InternalError } from "../errors/index.js";
 import { isKumikoError } from "../errors/kumiko-error.js";
 import { createTenantFileContext } from "../files/file-handle.js";
-import { createFallbackLogger } from "../logging/index.js";
+import { createFallbackLogger, scrubLogText } from "../logging/index.js";
 import { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, createMetricsForFactory, emitJobLastSuccess, emitJobQueueDepth, flushEscapeHatchAudits, getFallbackTracer, registerStandardMetrics, withEscapeHatchAuditScope, } from "../observability/index.js";
 import { configAccessorFactoryOf } from "../pipeline/config-accessor-factory.js";
 import { createDistributedLock } from "../pipeline/distributed-lock.js";
@@ -262,6 +262,9 @@ function tenantFailureMessageKey(err, declaration) {
 // (_triggeredById, _tenantId, _payload).
 const TRACE_CONTEXT_KEY = "_traceContext";
 const DEFER_COUNT_KEY = "_deferCount";
+// Separate from the maxPerTenant counter so tenant-limit deferrals do not eat into maxDefers.
+const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
+const DEFAULT_MAX_DEFERS = 100;
 const DEFER_DELAY_CAP_MS = 30_000;
 const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
 function readTraceContext(data) {
@@ -280,18 +283,28 @@ function captureTraceContext(tracer) {
     return { traceId: span.traceId, spanId: span.spanId };
 }
 const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
+const QUEUE_DEPTH_WARN_INTERVAL_MS = 5 * 60_000;
 // kumiko_job_queue_depth: only the lane's own consumer polls — the count is
 // a global Redis-backed value (BullMQ, not per-process), so one reporter per
 // lane is enough. Fires once immediately (metrics exist before the first
 // poll tick) then on an interval; the caller stores + clears the handle.
 async function startQueueDepthPolling(queue, lane, meter) {
+    let lastWarnAt;
     const pollQueueDepth = async () => {
         try {
             const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
             emitJobQueueDepth(meter, lane, counts);
         }
-        catch {
-            // skip: transient Redis hiccup — next poll retries
+        catch (err) {
+            // Next poll retries; the warn is throttled so a Redis outage logs once per interval, not once per poll.
+            const nowMs = Date.now();
+            if (lastWarnAt === undefined || nowMs - lastWarnAt >= QUEUE_DEPTH_WARN_INTERVAL_MS) {
+                lastWarnAt = nowMs;
+                createFallbackLogger("job-runner").warn("queue depth poll failed", {
+                    lane,
+                    error: scrubLogText(err instanceof Error ? err.message : String(err)),
+                });
+            }
         }
     };
     await pollQueueDepth();
@@ -614,6 +627,16 @@ export function createJobRunner(options) {
         const grown = Math.min(deferMs * 2 ** Math.min(deferCount, DEFER_BACKOFF_MAX_DOUBLINGS), DEFER_DELAY_CAP_MS);
         return grown + Math.floor(Math.random() * deferMs);
     }
+    function isJobDeferral(value) {
+        return (typeof value === "object" &&
+            value !== null &&
+            "kind" in value &&
+            value.kind === "job-deferral");
+    }
+    function priorDeferCountOf(data, key) {
+        const prior = data[key];
+        return typeof prior === "number" ? prior : 0;
+    }
     // Only err.message, never payload or the logs array: job payloads can carry PII.
     function logFailedAttempt(entry) {
         errorLogger.error("job failed", {
@@ -622,7 +645,9 @@ export function createJobRunner(options) {
             attempt: entry.attempt,
             // BullMQ never retries an UnrecoverableError, whatever attempts remain.
             final: entry.finalAttempt || entry.err instanceof UnrecoverableError,
-            error: entry.err instanceof Error ? entry.err.message : String(entry.err),
+            error: scrubLogText(entry.err instanceof Error ? entry.err.message : String(entry.err)),
+            ...(entry.err instanceof Error &&
+                entry.err.stack && { stack: scrubLogText(entry.err.stack) }),
         });
     }
     async function reportPreRunFailure(jobName, bullJob, err) {
@@ -779,8 +804,7 @@ export function createJobRunner(options) {
                 activeOnly: true,
                 self: { id: bullJob.id, processedOn: bullJob.processedOn },
             }))) {
-            const priorDeferCount = bullJob.data[DEFER_COUNT_KEY];
-            const deferCount = typeof priorDeferCount === "number" ? priorDeferCount : 0;
+            const deferCount = priorDeferCountOf(bullJob.data, DEFER_COUNT_KEY);
             await bullJob.updateData?.({ ...bullJob.data, [DEFER_COUNT_KEY]: deferCount + 1 });
             await bullJob.moveToDelayed(Date.now() + deferDelayMs(maxPerTenant.deferMs, deferCount), token);
             throw new DelayedError();
@@ -1008,6 +1032,17 @@ export function createJobRunner(options) {
             },
         }, { reason: identitySwitchGrantReason, report: reportEscapeHatch });
         const jobContextWithoutTz = {
+            defer: (delayMs) => {
+                if (!Number.isInteger(delayMs) || delayMs <= 0) {
+                    throw new RangeError(`Job "${jobName}": ctx.defer needs a positive integer delay in ms (got ${delayMs})`);
+                }
+                if (token === undefined || !bullJob.moveToDelayed) {
+                    throw new InternalError({
+                        message: `Job "${jobName}": ctx.defer needs a queued run; it is not available inline (boot gate).`,
+                    });
+                }
+                return { kind: "job-deferral", delayMs };
+            },
             ...context,
             db: jobDb ?? missingDbProxy(jobName),
             files,
@@ -1064,6 +1099,7 @@ export function createJobRunner(options) {
         const inheritedCorrelationId = rawData["_correlationId"] ?? undefined; // @cast-boundary dynamic-key
         const jobRequestId = requestContext.generateId();
         const jobCorrelationId = inheritedCorrelationId ?? jobRequestId;
+        const deferred = {};
         const runInSpan = async () => {
             try {
                 if (subjectConfigError) {
@@ -1076,17 +1112,29 @@ export function createJobRunner(options) {
                             "a trustworthy anonymous-root gate.",
                     });
                 }
-                await requestContext.run({
+                const handlerResult = await requestContext.run({
                     requestId: jobRequestId,
                     correlationId: jobCorrelationId,
                     // #3043 — events a job writes carry the job as their origin.
                     handler: jobName,
                     feature: registry.getJobFeature(jobName) ?? qnScope(jobName),
                     writeOrigin: jobOrigin,
+                    tenantId,
+                    ...(triggeredById !== null && { userId: triggeredById }),
                 }, async () => jobDef.handler(payload, {
                     ...jobContextWithoutTz,
                     tz: await resolveJobTz(config, context),
                 }));
+                if (isJobDeferral(handlerResult)) {
+                    const maxDefers = jobDef.maxDefers ?? DEFAULT_MAX_DEFERS;
+                    if (priorDeferCountOf(bullJob.data, HANDLER_DEFER_COUNT_KEY) >= maxDefers) {
+                        // The counter never resets, so a retry would hit the cap again.
+                        throw new UnrecoverableError(`Job "${jobName}" was deferred ${maxDefers} times (maxDefers); giving up.`);
+                    }
+                    deferred.deferral = handlerResult;
+                    // skip: the deferral is applied after the span closes, below
+                    return;
+                }
                 // Stamped before the observer hook: a throwing onJobComplete must not
                 // make a run that actually succeeded look dead to the liveness alert.
                 if (context.meter) {
@@ -1135,6 +1183,25 @@ export function createJobRunner(options) {
                 },
                 ...(parentContext ? { parent: parentContext } : {}),
             }, runInSpan));
+            if (deferred.deferral && token !== undefined && bullJob.moveToDelayed) {
+                // Run-log bookkeeping must not turn a defer into a failure.
+                try {
+                    await options.onJobDeferred?.(jobName, jobId, meta);
+                }
+                catch (err) {
+                    errorLogger.warn("job deferred hook failed", {
+                        job: jobName,
+                        jobId,
+                        error: scrubLogText(err instanceof Error ? err.message : String(err)),
+                    });
+                }
+                await bullJob.updateData?.({
+                    ...bullJob.data,
+                    [HANDLER_DEFER_COUNT_KEY]: priorDeferCountOf(bullJob.data, HANDLER_DEFER_COUNT_KEY) + 1,
+                });
+                await bullJob.moveToDelayed(Date.now() + deferred.deferral.delayMs, token);
+                throw new DelayedError();
+            }
         }
         finally {
             // Release the sequential lock value-matched (Lua compare-and-delete

package/dist/logging/index.d.ts

@@ -1,4 +1,5 @@
 export type { LoggerOptions } from "./pino-logger.ts";
 export { createLogger } from "./pino-logger.ts";
+export { scrubLogText, scrubLogValue } from "./scrub.ts";
 export type { Logger } from "./types.ts";
-export { createFallbackLogger } from "./utils.ts";
+export { createFallbackLogger, setProcessLogger } from "./utils.ts";

package/dist/logging/index.js

@@ -1,2 +1,3 @@
 export { createLogger } from "./pino-logger.js";
-export { createFallbackLogger } from "./utils.js";
+export { scrubLogText, scrubLogValue } from "./scrub.js";
+export { createFallbackLogger, setProcessLogger } from "./utils.js";

package/dist/logging/pino-logger.d.ts

@@ -1,5 +1,6 @@
 import { type DestinationStream } from "pino";
 import type { Logger } from "./types.ts";
+export declare const REDACT_PATHS: readonly string[];
 export type LoggerOptions = {
     level?: "debug" | "info" | "warn" | "error";
     pretty?: boolean;

package/dist/logging/pino-logger.js

@@ -1,10 +1,45 @@
 import pino, {} from "pino";
 import { observabilityContext } from "../observability/index.js";
+import { requestLogFields } from "./request-log-fields.js";
+const REDACTED_KEYS = [
+    "password",
+    "passwordHash",
+    "newPassword",
+    "token",
+    "accessToken",
+    "refreshToken",
+    "idToken",
+    "secret",
+    "clientSecret",
+    "apiKey",
+    "authorization",
+    "cookie",
+    "iban",
+    "email",
+    "set-cookie",
+    "x-api-key",
+    "sessionToken",
+    "privateKey",
+    "credentials",
+    "otp",
+];
+// pino/fast-redact needs bracket notation for keys that are not plain identifiers.
+function redactPathSegment(key) {
+    return /^[A-Za-z_$][\w$]*$/.test(key) ? key : `["${key}"]`;
+}
+// Depth 0-2: pino wildcards match exactly one level, so each depth needs its own path.
+export const REDACT_PATHS = REDACTED_KEYS.flatMap((key) => {
+    const segment = redactPathSegment(key);
+    const dotted = segment.startsWith("[") ? segment : `.${segment}`;
+    return [segment, `*${dotted}`, `*.*${dotted}`];
+});
 export function createLogger(options = {}) {
     const level = options.level ?? process.env["LOG_LEVEL"] ?? "info"; // @cast-boundary dynamic-key
     const pretty = options.pretty ?? process.env["LOG_FORMAT"] === "pretty";
     const pinoConfig = {
         level,
+        redact: { paths: [...REDACT_PATHS], censor: "[redacted]" },
+        mixin: requestLogFields,
         ...(pretty && !options.destination
             ? { transport: { target: "pino-pretty", options: { colorize: true } } }
             : {}),

package/dist/logging/request-log-fields.d.ts

@@ -0,0 +1 @@
+export declare function requestLogFields(): Record<string, string>;

package/dist/logging/request-log-fields.js

@@ -0,0 +1,21 @@
+import { requestContext } from "../api/request-context.js";
+// Only set fields are emitted so log lines outside a request stay lean.
+export function requestLogFields() {
+    const store = requestContext.get();
+    if (!store)
+        return {};
+    const fields = {};
+    if (store.requestId)
+        fields["requestId"] = store.requestId;
+    if (store.correlationId)
+        fields["correlationId"] = store.correlationId;
+    if (store.tenantId)
+        fields["tenantId"] = store.tenantId;
+    if (store.userId)
+        fields["userId"] = store.userId;
+    if (store.feature)
+        fields["feature"] = store.feature;
+    if (store.handler)
+        fields["handler"] = store.handler;
+    return fields;
+}

package/dist/logging/scrub.d.ts

@@ -0,0 +1,2 @@
+export declare function scrubLogText(text: string): string;
+export declare function scrubLogValue(value: unknown): unknown;

package/dist/logging/scrub.js

@@ -0,0 +1,21 @@
+const EMAIL_PATTERN = /([A-Za-z0-9._%+-])[A-Za-z0-9._%+-]*@([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,})/g;
+// Postgres echoes the offending row values in constraint-violation details.
+const PG_KEY_DETAIL_PATTERN = /Key \(([^)]*)\)=\(.*?\)(?=\s|$|[.,;])/g;
+const PG_FAILING_ROW_PATTERN = /Failing row contains \(.*\)/g;
+const JWT_PATTERN = /eyJ[\w-]+\.[\w-]+\.[\w-]+/g;
+const BEARER_PATTERN = /Bearer\s+\S+/g;
+export function scrubLogText(text) {
+    return text
+        .replace(PG_KEY_DETAIL_PATTERN, "Key ($1)=(***)")
+        .replace(PG_FAILING_ROW_PATTERN, "Failing row contains (***)")
+        .replace(EMAIL_PATTERN, "$1***@$2")
+        .replace(JWT_PATTERN, "[jwt]")
+        .replace(BEARER_PATTERN, "Bearer [redacted]");
+}
+export function scrubLogValue(value) {
+    if (typeof value === "string")
+        return scrubLogText(value);
+    if (value instanceof Error)
+        return scrubLogText(value.message);
+    return value;
+}

package/dist/logging/utils.d.ts

@@ -4,5 +4,6 @@ type FallbackLogger = {
     warn(msg: string, data?: Record<string, unknown>): void;
     debug(msg: string, data?: Record<string, unknown>): void;
 };
+export declare function setProcessLogger(logger: Logger | null): void;
 export declare function createFallbackLogger(namespace: string, logger?: (Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>) | undefined): FallbackLogger;
 export {};

package/dist/logging/utils.js

@@ -1,3 +1,8 @@
+let processLogger = null;
+// Fallback loggers created at module level before boot must pick this up, so it is read per call.
+export function setProcessLogger(logger) {
+    processLogger = logger;
+}
 export function createFallbackLogger(namespace, logger) {
     if (logger) {
         return {
@@ -24,18 +29,47 @@ export function createFallbackLogger(namespace, logger) {
             },
         };
     }
+    let cachedParent = null;
+    let cachedChild = null;
+    const resolveProcessChild = () => {
+        if (!processLogger)
+            return null;
+        if (cachedParent !== processLogger || !cachedChild) {
+            cachedParent = processLogger;
+            cachedChild = processLogger.child({ component: namespace });
+        }
+        return cachedChild;
+    };
     return {
         error(msg, data) {
-            // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
-            console.error(`[${namespace}] ${msg}`, data);
+            const child = resolveProcessChild();
+            if (child) {
+                child.error(msg, data);
+            }
+            else {
+                // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
+                console.error(`[${namespace}] ${msg}`, data);
+            }
         },
         warn(msg, data) {
-            // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
-            console.warn(`[${namespace}] ${msg}`, data);
+            const child = resolveProcessChild();
+            if (child) {
+                child.warn(msg, data);
+            }
+            else {
+                // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
+                console.warn(`[${namespace}] ${msg}`, data);
+            }
         },
         debug(msg, data) {
-            // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
-            console.debug(`[${namespace}] ${msg}`, data);
+            const child = resolveProcessChild();
+            if (child) {
+                child.debug(msg, data);
+            }
+            else {
+                // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
+                console.debug(`[${namespace}] ${msg}`, data);
+            }
         },
     };
 }

package/dist/migrations/pii-event-backfill.js

@@ -18,6 +18,7 @@ import { alterTableAddColumn } from "../db/queries/ddl.js";
 import { persistPiiBackfillState, readPiiBackfillState, } from "../db/queries/pii-backfill-state.js";
 import { upsertOnConflict } from "../db/query.js";
 import { tableExists } from "../db/schema-inspection.js";
+import { withoutStatementTimeout } from "../db/statement-timeout.js";
 import { unsafePushTables } from "../db/table-push.js";
 import { createFallbackLogger } from "../logging/utils.js";
 import { clearPendingRebuilds, createPendingRebuildsTable, listPendingRebuildRows, pendingRebuildsTable, rebuildProjectionOrMultiStream, } from "./pending-rebuilds.js";
@@ -207,7 +208,10 @@ function reportRunOutcome(log, progress) {
         });
     }
 }
-export async function runPiiEventBackfill(db, registry, options = {}) {
+export function runPiiEventBackfill(db, registry, options = {}) {
+    return withoutStatementTimeout(db, (unboundedDb) => backfillPiiEvents(unboundedDb, registry, options));
+}
+async function backfillPiiEvents(db, registry, options) {
     const log = createFallbackLogger("pii-event-backfill", options.logger);
     const kms = configuredPiiSubjectKms();
     if (!kms || !isLocalKeyKmsAdapter(kms))

package/dist/observability/standard-metrics.d.ts

@@ -32,6 +32,7 @@ export declare function emitEventConsumerPendingGaps(meter: Meter, labels: {
     readonly consumer: string;
     readonly instanceId: string;
 }, gapRanges: number): void;
+export declare function emitRateLimitFailOpen(meter: Meter, label: string): void;
 export declare function emitEventConsumerRearmExhausted(meter: Meter, labels: {
     readonly consumer: string;
     readonly instanceId: string;

package/dist/observability/standard-metrics.js

@@ -3,6 +3,12 @@
 // Feature code never emits these — the Framework wires them from the HTTP
 // middleware, dispatcher, and DB wrapper.
 export const STANDARD_METRIC_DEFS = [
+    {
+        name: "kumiko_rate_limit_fail_open_total",
+        type: "counter",
+        description: "Requests let through because the rate-limit backend (Redis) failed.",
+        labels: ["label"],
+    },
     {
         name: "kumiko_http_requests_total",
         type: "counter",
@@ -227,6 +233,9 @@ export function emitEventConsumerPendingGaps(meter, labels, gapRanges) {
 // Caller (event-dispatcher.ts) is responsible for calling this exactly
 // once per (consumer, instanceId) transition into exhausted-dead — not
 // once per poll pass while it stays dead.
+export function emitRateLimitFailOpen(meter, label) {
+    meter.counter("kumiko_rate_limit_fail_open_total").inc(1, { label });
+}
 export function emitEventConsumerRearmExhausted(meter, labels) {
     meter.counter("kumiko_event_consumer_rearm_exhausted_total").inc(1, {
         consumer: labels.consumer,

package/dist/pipeline/msp-rebuild.js

@@ -1,6 +1,7 @@
 import { markConsumerRebuildFailed, resetConsumerForMspRebuild, selectConsumerForUpdate, selectEventIdHorizonWithMissingRanges, selectSnapshotXmax, updateConsumerRebuildCursor, } from "../db/queries/event-consumer.js";
 import { assertLiveColumnsMatchMeta, assertLiveTableHasNoRowLevelSecurity, buildShadowTable, ensureRebuildSchema, rebuildMetaOrThrow, swapShadowIntoLive, } from "../db/queries/shadow-swap.js";
 import { runInSavepoint, selectMany } from "../db/query.js";
+import { withoutStatementTimeout } from "../db/statement-timeout.js";
 import { withExecutorManagedWrites } from "../db/tenant-db.js";
 import { InternalError } from "../errors/index.js";
 import { loadAggregate, loadAggregateAsOf } from "../event-store/event-store.js";
@@ -29,7 +30,10 @@ function createRebuildCtx(registry, db, tenantId) {
         },
     };
 }
-export async function rebuildMultiStreamProjection(mspName, deps) {
+export function rebuildMultiStreamProjection(mspName, deps) {
+    return withoutStatementTimeout(deps.db, (db) => replayMultiStreamProjectionIntoShadow(mspName, { ...deps, db }));
+}
+async function replayMultiStreamProjectionIntoShadow(mspName, deps) {
     const { db, registry } = deps;
     const msp = registry.getAllMultiStreamProjections().get(mspName);
     if (!msp) {

package/dist/pipeline/projection-rebuild.js

@@ -1,6 +1,7 @@
 import { countSubscribedEvents, finalizeProjectionRebuild, markProjectionRebuildFailed, markProjectionRebuilding, selectEventsForProjectionRebuildBatch, } from "../db/queries/projection-rebuild.js";
 import { assertLiveColumnsMatchMeta, assertLiveTableHasNoRowLevelSecurity, assertNoBlindIndexLoss, assertNoUnreachableLiveRows, buildShadowTable, countColumnDrift, ensureRebuildSchema, fenceLiveTable, rebuildMetaOrThrow, swapShadowIntoLive, } from "../db/queries/shadow-swap.js";
 import { coerceRow, extractTableInfo, runInSavepoint, selectMany } from "../db/query.js";
+import { withoutStatementTimeout } from "../db/statement-timeout.js";
 import { withExecutorManagedWrites } from "../db/tenant-db.js";
 import { eventsTable, getEventsHighWaterMark, upcastStoredEvent, } from "../event-store/index.js";
 import { createRebuildDeadLetterTable, recordRebuildDeadLetters, } from "../event-store/rebuild-dead-letter.js";
@@ -45,7 +46,10 @@ function warnOnColumnDrift(projectionName, tableName, drift) {
         "Not blocking — replay wins on swap. Likely cause: a legacy direct-write predating the row's " +
         "event history (#494), or a sensitive custom field pending #972. Investigate if unexpected.");
 }
-export async function rebuildProjection(projectionName, deps) {
+export function rebuildProjection(projectionName, deps) {
+    return withoutStatementTimeout(deps.db, (db) => replayProjectionIntoShadow(projectionName, { ...deps, db }));
+}
+async function replayProjectionIntoShadow(projectionName, deps) {
     const { db, registry } = deps;
     const projection = registry.getAllProjections().get(projectionName);
     if (!projection) {

package/dist/rate-limit/middleware.d.ts

@@ -1,7 +1,8 @@
 import type { Context, MiddlewareHandler } from "hono";
 import { type ClientIpResolver } from "../api/client-ip.ts";
 import type { HttpRouteRateLimit } from "../engine/types/http-route.ts";
-import type { RateLimitResolver } from "./resolver.ts";
+import type { Meter } from "../observability/types/index.ts";
+import type { RateLimitDecision, RateLimitResolver } from "./resolver.ts";
 export type GlobalIpRateLimitOptions = {
     readonly resolver: RateLimitResolver;
     readonly limit?: number;
@@ -9,7 +10,8 @@ export type GlobalIpRateLimitOptions = {
     readonly trustedProxyHops?: number;
     readonly clientIpResolver?: ClientIpResolver;
     readonly extractIp?: (c: Context) => string | undefined;
-    readonly onFailClosed?: (err: unknown) => void;
+    readonly onFailOpen?: (err: unknown) => void;
+    readonly meter?: Meter;
 };
 export declare const DEFAULT_GLOBAL_RATE_LIMIT: {
     readonly limit: 600;
@@ -21,7 +23,8 @@ export type HttpRouteRateLimitOptions = {
     readonly rateLimit: HttpRouteRateLimit;
     readonly routeKey: string;
     readonly clientIpResolver: ClientIpResolver;
-    readonly onFailClosed?: (err: unknown) => void;
+    readonly onFailOpen?: (err: unknown) => void;
+    readonly meter?: Meter;
 };
 export declare function httpRouteRateLimit(opts: HttpRouteRateLimitOptions): MiddlewareHandler;
 export type AuthEndpointRateLimitOptions = {
@@ -35,3 +38,5 @@ export type AuthEndpointRateLimitOptions = {
     readonly onFailClosed?: (err: unknown) => void;
 };
 export declare function authEndpointRateLimit(opts: AuthEndpointRateLimitOptions): MiddlewareHandler;
+export declare function createFailOpenReporter(label: string, meter?: Meter, now?: () => number): (err: unknown) => void;
+export declare function respondRateLimited(c: Context, decision: RateLimitDecision, bucket: string): Response;

package/dist/rate-limit/middleware.js

@@ -2,6 +2,8 @@ import { isAuthRateLimitExempt } from "../api/api-constants.js";
 import { clientIpSourceFromHonoContext, createClientIpResolver, } from "../api/client-ip.js";
 import { requestContext } from "../api/request-context.js";
 import { RateLimitError, serializeError } from "../errors/index.js";
+import { createFallbackLogger, scrubLogText } from "../logging/index.js";
+import { emitRateLimitFailOpen } from "../observability/standard-metrics.js";
 import { toPublicBucketName } from "./bucket.js";
 // Shared with runProdApp (server-runtime re-exports it). The auth middleware default stays
 // stricter than runProdApp's auth limit (30): a bare authEndpointRateLimit guards brute force.
@@ -12,7 +14,7 @@ export function globalIpRateLimit(opts) {
     const clientIpResolver = opts.clientIpResolver ??
         createClientIpResolver(opts.trustedProxyHops ?? 0, "globalIpRateLimit");
     const extractIp = opts.extractIp ?? ((c) => clientIpResolver.resolve(clientIpSourceFromHonoContext(c)));
-    const onFailClosed = opts.onFailClosed ?? defaultOnFailClosed("l1-global-ip");
+    const onFailOpen = opts.onFailOpen ?? createFailOpenReporter("l1-global-ip", opts.meter);
     return async (c, next) => {
         const ip = extractIp(c);
         if (!ip) {
@@ -28,20 +30,18 @@ export function globalIpRateLimit(opts) {
             setRateLimitHeaders(c, decision);
         }
         catch (e) {
-            // Fail-closed: refuse rather than let a flood through with no cap.
             // resolver.check never throws RateLimitError (only enforce does),
             // so any throw here is an infrastructure failure (Redis down).
-            onFailClosed(e);
-            return c.json({ error: { code: "rate_limit_unavailable", message: "Rate limiter unavailable" } }, 503);
+            onFailOpen(e);
         }
         await next();
     };
 }
-// Per-route limit for r.httpRoute({ rateLimit }). Fail-closed like L1: these
-// routes are anonymous, so an outage must not turn into an unbounded flood.
+// Per-route limit for r.httpRoute({ rateLimit }). Fail-open like L1: a Redis outage must not
+// take anonymous routes (status pages, webhooks) down with it.
 export function httpRouteRateLimit(opts) {
     const { rateLimit, routeKey } = opts;
-    const onFailClosed = opts.onFailClosed ?? defaultOnFailClosed("http-route");
+    const onFailOpen = opts.onFailOpen ?? createFailOpenReporter("http-route", opts.meter);
     const config = {
         limit: rateLimit.limit,
         windowSeconds: rateLimit.windowSeconds,
@@ -59,8 +59,7 @@ export function httpRouteRateLimit(opts) {
             setRateLimitHeaders(c, decision);
         }
         catch (e) {
-            onFailClosed(e);
-            return c.json({ error: { code: "rate_limit_unavailable", message: "Rate limiter unavailable" } }, 503);
+            onFailOpen(e);
         }
         await next();
     };
@@ -95,6 +94,25 @@ export function authEndpointRateLimit(opts) {
         await next();
     };
 }
+const FAIL_OPEN_WARN_INTERVAL_MS = 30_000;
+export function createFailOpenReporter(label, meter, now = Date.now) {
+    const logger = createFallbackLogger("rate-limit");
+    let lastWarnAt;
+    return (err) => {
+        if (meter)
+            emitRateLimitFailOpen(meter, label);
+        const nowMs = now();
+        // skip: throttled, the counter above still records every fail-open
+        if (lastWarnAt !== undefined && nowMs - lastWarnAt < FAIL_OPEN_WARN_INTERVAL_MS)
+            return;
+        lastWarnAt = nowMs;
+        logger.warn("rate limiter unavailable, failing open", {
+            label,
+            // ioredis errors can carry the command args, i.e. the bucket key with IP/target.
+            error: scrubLogText(err instanceof Error ? err.message : String(err)),
+        });
+    };
+}
 function defaultOnFailClosed(label) {
     return (err) => {
         // Loud by default — fail-closed for an unknown reason is an ops
@@ -116,7 +134,7 @@ function setRateLimitHeaders(c, decision) {
     // client libs that parse as integer would silently see NaN.
     c.header("X-RateLimit-Reset", String(Math.floor(decision.resetAt.epochMilliseconds / 1000)));
 }
-function respondRateLimited(c, decision, bucket) {
+export function respondRateLimited(c, decision, bucket) {
     // Build a RateLimitError so the wire shape is identical to the L3
     // dispatcher path. serializeError adds i18nKey, requestId, timestamp —
     // fields a hand-rolled body would silently miss.

package/dist/rate-limit/resolver.js

@@ -179,11 +179,19 @@ function ensureCommand(redis) {
     }
     return redis; // @cast-boundary engine-bridge
 }
+// ioredis queues commands while reconnecting; waiting out commandTimeout per request would
+// stall every caller, so a non-ready connection counts as a failure immediately.
+function assertRedisReady(redis) {
+    if (redis.status !== "ready") {
+        throw new Error(`rate-limit redis connection not ready (status: ${redis.status})`);
+    }
+}
 export function createRateLimitResolver(opts) {
     const client = ensureCommand(opts.redis);
     const prefix = opts.keyPrefix ?? RedisKeys.rateLimit;
     const now = opts.nowMs ?? (() => Date.now());
     async function check(bucket, config) {
+        assertRedisReady(opts.redis);
         const cost = config.cost ?? 1;
         const refillRatePerMs = config.limit / (config.windowSeconds * 1000);
         const ttlSeconds = Math.max(1, config.windowSeconds * 2);
@@ -216,6 +224,7 @@ export function createRateLimitResolver(opts) {
     async function enforceAll(requests) {
         if (requests.length === 0)
             return [];
+        assertRedisReady(opts.redis);
         const nowMs = now();
         const keys = requests.map(({ bucket }) => `${prefix}${bucket}`);
         // The script writes each key back once, so a repeated key would deduct only one of its costs.
@@ -257,6 +266,7 @@ export function createRateLimitResolver(opts) {
         });
     }
     async function peek(bucket, config) {
+        assertRedisReady(opts.redis);
         const refillRatePerMs = config.limit / (config.windowSeconds * 1000);
         const nowMs = now();
         const [remaining, retryAfterMs] = await client.kumikoRateLimitPeek(`${prefix}${bucket}`, String(config.limit), String(refillRatePerMs), String(nowMs));

package/dist/redis/client.d.ts

@@ -1,4 +1,5 @@
 import { Redis, type RedisOptions } from "ioredis";
+import type { Logger } from "../logging/types.ts";
 export type RedisClientOptions = {
     readonly connectTimeoutMs?: number;
     readonly commandTimeoutMs?: number;
@@ -6,5 +7,6 @@ export type RedisClientOptions = {
     readonly extra?: Readonly<Omit<RedisOptions, "connectTimeout" | "commandTimeout" | "maxRetriesPerRequest">>;
 };
 export declare function createRedisClient(url: string, options?: RedisClientOptions): Redis;
+export declare function logRedisConnectionState(redis: Redis, logger: Pick<Logger, "warn" | "info">): void;
 export declare function redisChannelPrefixFromEnv(env?: Readonly<Record<string, string | undefined>>): string;
 export declare function redisClientOptionsFromEnv(env?: Readonly<Record<string, string | undefined>>): RedisClientOptions;