@cosmicdrift/kumiko-framework 0.352.0 → 0.354.0

package/dist/migrations/pii-event-backfill.js

@@ -14,6 +14,7 @@ import { bigint, instant, table as pgTable, sql, text } from "../db/dialect.js";
 import { extractTableName } from "../db/index.js";
 import { acquireNamespacedAdvisoryLock } from "../db/queries/advisory-lock.js";
 import { backfillEventPiiEncryptionBatch, } from "../db/queries/backfill-pii.js";
+import { alterTableAddColumn } from "../db/queries/ddl.js";
 import { persistPiiBackfillState, readPiiBackfillState, } from "../db/queries/pii-backfill-state.js";
 import { upsertOnConflict } from "../db/query.js";
 import { tableExists } from "../db/schema-inspection.js";
@@ -37,10 +38,12 @@ export const piiBackfillStateTable = pgTable("kumiko_pii_backfill_state", {
     updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql `now()`),
 });
 export async function createPiiBackfillStateTable(db) {
-    // skip: table already exists — boot runs on every replica
-    if (await tableExists(db, "public.kumiko_pii_backfill_state"))
-        return;
-    await unsafePushTables(db, { kumikoPiiBackfillState: piiBackfillStateTable });
+    if (!(await tableExists(db, "public.kumiko_pii_backfill_state"))) {
+        await unsafePushTables(db, { kumikoPiiBackfillState: piiBackfillStateTable });
+    }
+    // Added after the table's first deploy; IF NOT EXISTS keeps concurrent replica boots safe.
+    await alterTableAddColumn(db, "kumiko_pii_backfill_state", "failed_event_ids", "bigint[]", " DEFAULT '{}'::bigint[]", " NOT NULL", 
+    /* ifNotExists */ true);
 }
 // Replicas boot concurrently: two of them can both pass the tableExists gate and
 // one CREATE then fails. The table existing afterwards means the other won.
@@ -107,17 +110,19 @@ function initialScanCursor(state) {
 // One transaction per batch: the advisory lock serializes pods, and the payload
 // is re-read inside it so a value is never encrypted twice and two backfills
 // never mint the same DEK. Returns null when another pod already finished a full run.
-async function runOneBatch(ctx, progress) {
+async function runOneBatch(ctx, progress, retryEventIds) {
     return ctx.db.begin(async (tx) => {
         await acquireNamespacedAdvisoryLock(tx, PII_EVENT_BACKFILL_LOCK_NAMESPACE, PII_EVENT_BACKFILL_LOCK_KEY);
         const state = await readPiiBackfillState(tx, ctx.fingerprint);
-        if (ctx.mode === "full" && state?.completed === true)
+        if (retryEventIds === undefined && ctx.mode === "full" && state?.completed === true) {
             return null;
+        }
         const startCursor = ctx.mode === "full" && state
             ? maxBigInt(progress.scanCursor, BigInt(state.cursor))
             : progress.scanCursor;
         const batch = await backfillEventPiiEncryptionBatch(tx, ctx.registry, {
             afterEventId: startCursor,
+            ...(retryEventIds && { eventIds: retryEventIds }),
             batchSize: ctx.batchSize,
             resolveOwnerFromProjection: true,
             scanCache: progress.scanCache,
@@ -128,10 +133,15 @@ async function runOneBatch(ctx, progress) {
             : [];
         if (tables.length > 0)
             await queueRebuildTables(tx, tables, ctx.migrationId);
-        if (!progress.failureSeen) {
-            const completesRun = batch.scannedAll && batch.failures.length === 0;
-            await persistPiiBackfillState(tx, ctx.fingerprint, persistableCursor(batch, startCursor), completesRun);
-        }
+        // The cursor moves past failed events: they are recorded here and retried on their own,
+        // so one permanently failing event no longer forces a rescan from it on every boot.
+        const failedIds = batch.failures.map((failure) => failure.eventId);
+        await persistPiiBackfillState(tx, ctx.fingerprint, {
+            cursor: retryEventIds ? 0n : advancedCursor(batch, startCursor),
+            markCompleted: retryEventIds === undefined && batch.scannedAll,
+            addFailedEventIds: failedIds,
+            removeFailedEventIds: retryEventIds?.filter((id) => !failedIds.includes(id)) ?? [],
+        });
         return { batch, tables };
     });
 }
@@ -141,14 +151,25 @@ function recordBatch(progress, step) {
     progress.totals.updatedEvents += batch.updatedEvents;
     progress.totals.encryptedFields += batch.encryptedFields;
     progress.totals.erasedFields += batch.erasedFields;
-    progress.failures.push(...batch.failures);
+    for (const failure of batch.failures) {
+        // The catch-up overlap and the per-event retry can hit the same failed event twice in one run.
+        if (progress.failedIdsSeen.has(failure.eventId))
+            continue;
+        progress.failedIdsSeen.add(failure.eventId);
+        progress.failures.push(failure);
+    }
     for (const table of step.tables)
         progress.queuedTables.add(table);
-    if (batch.failures.length > 0)
-        progress.failureSeen = true;
 }
 // Returns true when the run was aborted between batches.
-async function scanAllBatches(ctx, progress, signal) {
+async function scanAllBatches(ctx, progress, signal, retryEventIds) {
+    for (let from = 0; from < retryEventIds.length; from += ctx.batchSize) {
+        if (signal?.aborted)
+            return true;
+        const step = await runOneBatch(ctx, progress, retryEventIds.slice(from, from + ctx.batchSize));
+        if (step !== null)
+            recordBatch(progress, step);
+    }
     for (;;) {
         if (signal?.aborted)
             return true;
@@ -163,7 +184,9 @@ async function scanAllBatches(ctx, progress, signal) {
 }
 function reportRunOutcome(log, progress) {
     if (progress.failures.length > 0) {
-        log.error(`${progress.failures.length} event(s) could not be PII-encrypted and will be retried on the next boot`, {
+        log.error(`${progress.failures.length} event(s) could not be PII-encrypted; the scan moved past them and ` +
+            "only these events are retried on each boot. Fix the cause, or erase subjects that stay unresolvable via " +
+            "backfillEventPiiEncryption(..., { eraseUnresolvableSubjects: true })", {
             failureCount: progress.failures.length,
             eventIds: progress.failures.slice(0, MAX_LOGGED_FAILURE_IDS).map((f) => f.eventId),
         });
@@ -180,24 +203,27 @@ export async function runPiiEventBackfill(db, registry, options = {}) {
     const fingerprint = piiAnnotationFingerprint(registry);
     await ensureInfraTable(db, "kumiko_pii_backfill_state", createPiiBackfillStateTable);
     await ensureInfraTable(db, "kumiko_pending_rebuilds", createPendingRebuildsTable);
-    const { mode, scanCursor } = initialScanCursor(await readPiiBackfillState(db, fingerprint));
+    const initialState = await readPiiBackfillState(db, fingerprint);
+    const { mode, scanCursor } = initialScanCursor(initialState);
     const ctx = {
         db,
         registry,
         fingerprint,
-        migrationId: `${PII_BACKFILL_MIGRATION_PREFIX}${fingerprint.slice(0, 12)}`,
+        // The per-run suffix keeps a peer replica's re-queue of the same table (new
+        // migration_id via the upsert) from being deleted by this run's clear.
+        migrationId: `${PII_BACKFILL_MIGRATION_PREFIX}${fingerprint.slice(0, 12)}:${crypto.randomUUID()}`,
         mode,
         batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE,
     };
     const progress = {
         scanCursor,
-        failureSeen: false,
         scanCache: {},
         totals: { scannedEvents: 0, updatedEvents: 0, encryptedFields: 0, erasedFields: 0 },
         failures: [],
         queuedTables: new Set(),
+        failedIdsSeen: new Set(),
     };
-    const aborted = await scanAllBatches(ctx, progress, options.signal);
+    const aborted = await scanAllBatches(ctx, progress, options.signal, initialState?.failedEventIds ?? []);
     reportRunOutcome(log, progress);
     const rebuild = aborted ? null : await rebuildQueuedTables(db, registry, options.signal, log);
     const queuedTables = [...progress.queuedTables];
@@ -218,12 +244,7 @@ export async function runPiiEventBackfill(db, registry, options = {}) {
         rebuild,
     };
 }
-// Never advance past a failed event so the next boot retries it.
-function persistableCursor(batch, startCursor) {
-    if (batch.firstFailedEventId !== null) {
-        const beforeFailure = BigInt(batch.firstFailedEventId) - 1n;
-        return beforeFailure > startCursor ? beforeFailure : startCursor;
-    }
+function advancedCursor(batch, startCursor) {
     return batch.lastEventId === null ? startCursor : BigInt(batch.lastEventId);
 }
 function maxBigInt(a, b) {

package/dist/observability/index.d.ts

@@ -12,5 +12,5 @@ export { type MetricEvent, type MetricEventHandler, RecordingMeter, } from "./re
 export { type RecordedSpan, RecordingTracer, type RecordingTracerOptions, serializeSpanContext, } from "./recording-tracer.js";
 export { wrapRedisClient } from "./redis-wrapper.js";
 export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
-export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
+export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerPendingGaps, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
 export type { Counter, Gauge, Histogram, Meter, MetricDefinition, MetricLabels, MetricsHandle, MetricType, ObservabilityOptions, ObservabilityProvider, SamplingConfig, SensitiveFilterConfig, SerializedTraceContext, Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatus, StartSpanOptions, Tracer, } from "./types/index.js";

package/dist/observability/index.js

@@ -13,4 +13,4 @@ export { RecordingMeter, } from "./recording-meter.js";
 export { RecordingTracer, serializeSpanContext, } from "./recording-tracer.js";
 export { wrapRedisClient } from "./redis-wrapper.js";
 export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
-export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
+export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerPendingGaps, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";

package/dist/observability/metrics-wiring.d.ts

@@ -2,7 +2,7 @@ import * as z from "zod";
 import { type PrometheusMeter } from "./prometheus-meter.js";
 import type { ObservabilityProvider } from "./types/index.js";
 export declare const prometheusMetricsEnvSchema: z.ZodObject<{
-    PROMETHEUS_METRICS_TOKEN: z.ZodOptional<z.ZodString>;
+    PROMETHEUS_METRICS_TOKEN: z.ZodOptional<z.ZodUnion<readonly [z.ZodPipe<z.ZodLiteral<"">, z.ZodTransform<undefined, "">>, z.ZodString]>>;
 }, z.core.$strip>;
 type PrometheusObservabilityProvider = ObservabilityProvider & {
     readonly meter: PrometheusMeter;

package/dist/observability/metrics-wiring.js

@@ -2,9 +2,9 @@ import * as z from "zod";
 import { createNoopProvider } from "./noop-provider.js";
 import { createPrometheusMeter } from "./prometheus-meter.js";
 export const prometheusMetricsEnvSchema = z.object({
+    // An empty value (blank .env line, empty k8s secret) means "unset", not "invalid".
     PROMETHEUS_METRICS_TOKEN: z
-        .string()
-        .min(32)
+        .union([z.literal("").transform(() => undefined), z.string().min(32)])
         .optional()
         .describe("Bearer token for /metrics; unset keeps the endpoint off.")
         .meta({ kumiko: { pulumi: { secret: true, generator: "openssl rand -base64 32" } } }),

package/dist/observability/standard-metrics.d.ts

@@ -28,6 +28,10 @@ export declare function emitEventConsumerLag(meter: Meter, labels: {
     readonly consumer: string;
     readonly instanceId: string;
 }, lagEvents: number): void;
+export declare function emitEventConsumerPendingGaps(meter: Meter, labels: {
+    readonly consumer: string;
+    readonly instanceId: string;
+}, gapRanges: number): void;
 export declare function emitEventConsumerRearmExhausted(meter: Meter, labels: {
     readonly consumer: string;
     readonly instanceId: string;

package/dist/observability/standard-metrics.js

@@ -61,6 +61,12 @@ export const STANDARD_METRIC_DEFS = [
     // per-instance consumers (one cursor per dispatcher). Without this,
     // Prometheus collapses per-instance shards into last-writer-wins and
     // per-instance lag is invisible to alerting.
+    {
+        name: "kumiko_event_consumer_pending_gaps",
+        type: "gauge",
+        description: "Number of unresolved id ranges below the consumer's cursor (ids not yet visible or proven rolled back).",
+        labels: ["consumer", "instance_id"],
+    },
     {
         name: "kumiko_event_consumer_lag_events",
         type: "gauge",
@@ -212,6 +218,12 @@ export function emitEventConsumerLag(meter, labels, lagEvents) {
         instance_id: labels.instanceId,
     });
 }
+export function emitEventConsumerPendingGaps(meter, labels, gapRanges) {
+    meter.gauge("kumiko_event_consumer_pending_gaps").set(gapRanges, {
+        consumer: labels.consumer,
+        instance_id: labels.instanceId,
+    });
+}
 // Caller (event-dispatcher.ts) is responsible for calling this exactly
 // once per (consumer, instanceId) transition into exhausted-dead — not
 // once per poll pass while it stays dead.

package/dist/pipeline/dispatch-shared.d.ts

@@ -46,7 +46,9 @@ export type DispatchContext = {
     tracer: ReturnType<typeof getFallbackTracer>;
     meter: ReturnType<typeof getFallbackMeter>;
     membershipQuery: string;
+    memberReadTimeoutMs: number;
 };
+export declare const DEFAULT_MEMBER_READ_TIMEOUT_MS = 10000;
 export declare function resolveDbSource(ctx: DispatchContext, tx: DbTx | undefined): DbConnection | DbTx | undefined;
 export declare function isMemberResolutionPrincipal(user: SessionUser): boolean;
 export declare function applyMemberResolutionReadOnly(handlerContext: HandlerContext): HandlerContext;

package/dist/pipeline/dispatch-shared.js

@@ -1,7 +1,7 @@
 import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
 import { requestContext, runWithOrigin } from "../api/request-context.js";
 import { runInSavepoint, selectMany } from "../db/query.js";
-import { createTenantDb, createUncheckedSystemDb, hasTenantColumn, } from "../db/tenant-db.js";
+import { createTenantDb, createUncheckedSystemDb, hasTenantColumn, runInOwnTransaction, } from "../db/tenant-db.js";
 import { createDerivativesContext } from "../derivatives/derivatives-context.js";
 import { createSystemUser } from "../engine/system-user.js";
 import { isRateLimitDisabled, resolveAgentExposure } from "../engine/types/index.js";
@@ -40,6 +40,7 @@ export const TENANT_TIMEZONE_CONFIG_KEY = "tenant:config:timezone";
 // exercises both against the real config feature.
 export const CONFIG_WRITE_SET_TYPE = "config:write:set";
 export const CONFIG_WRITE_RESET_TYPE = "config:write:reset";
+export const DEFAULT_MEMBER_READ_TIMEOUT_MS = 10_000;
 // Narrowing-helper: AppContext.db ist DbConnection|TenantDb|undefined. Die
 // dispatch-Pfade brauchen DbConnection (oder DbTx aus Caller-Scope) für
 // appendEvent/projection-writes; TenantDb-Branch wird hier ausgeschlossen.
@@ -113,14 +114,57 @@ function denyingJobRunnerProxy() {
         },
     });
 }
+// Allowlist (not denylist): a method added to the target later stays denied
+// until someone decides it is read-only.
+function allowlistedReadOnlyProxy(target, allowedMethods) {
+    return new Proxy(target, {
+        get(inner, prop) {
+            if (typeof prop === "string" && allowedMethods.has(prop)) {
+                const method = Reflect.get(inner, prop, inner);
+                return typeof method === "function" ? method.bind(inner) : method;
+            }
+            // Symbol reads and "then" come from await/Promise.resolve/inspect, not from handler code.
+            if (typeof prop === "symbol" || prop === "then")
+                return undefined;
+            throw memberResolutionReadOnlyDenied();
+        },
+    });
+}
+const MEMBER_READ_SEARCH_METHODS = new Set(["search"]);
+const MEMBER_READ_ENTITY_CACHE_METHODS = new Set(["get", "mget"]);
+const MEMBER_READ_REDIS_METHODS = new Set([
+    "get",
+    "mget",
+    "exists",
+    "ttl",
+    "pttl",
+    "strlen",
+    "type",
+    "hget",
+    "hmget",
+    "hgetall",
+    "hexists",
+    "hlen",
+    "sismember",
+    "smembers",
+    "scard",
+    "llen",
+    "lrange",
+    "zscore",
+    "zcard",
+    "zrange",
+]);
 // Every write/side-effect surface throws, so "no writeAsMember" holds even
 // if a handler tries ctx.write directly on a resolved member principal.
+// Deliberately open: `db` (guarded by the READ ONLY transaction + memberReadOnly grant),
+// and read-only methods of searchAdapter / redis / entityCache (allowlisted).
 export function applyMemberResolutionReadOnly(handlerContext) {
     return {
         ...handlerContext,
         // `db` stays open — executeQuery runs the whole handler in a Postgres READ ONLY
         // transaction, and its `memberReadOnly` grant denies every raw-runner handout.
         dbOutsideTransaction: undefined,
+        outsideTransaction: undefined,
         write: denyMemberResolutionWrite,
         writeAs: denyMemberResolutionWrite,
         // A resolved member cannot switch identity either — the target would get a normal, writable context.
@@ -144,6 +188,15 @@ export function applyMemberResolutionReadOnly(handlerContext) {
         derivatives: undefined,
         ...(handlerContext.jobRunner && { jobRunner: denyingJobRunnerProxy() }),
         ...(handlerContext.notify && { notify: denyMemberResolutionWrite }),
+        ...(handlerContext.searchAdapter && {
+            searchAdapter: allowlistedReadOnlyProxy(handlerContext.searchAdapter, MEMBER_READ_SEARCH_METHODS),
+        }),
+        ...(handlerContext.redis && {
+            redis: allowlistedReadOnlyProxy(handlerContext.redis, MEMBER_READ_REDIS_METHODS),
+        }),
+        ...(handlerContext.entityCache && {
+            entityCache: allowlistedReadOnlyProxy(handlerContext.entityCache, MEMBER_READ_ENTITY_CACHE_METHODS),
+        }),
     };
 }
 export async function buildHandlerContext(ctx, type, user, origin, tx, afterCommitHooks, includeDeleted, isPostCommit) {
@@ -210,6 +263,11 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         ? createSystemScopedDbGuard(type, "dbOutsideTransaction", "ctx.systemDb.outsideTransaction.assertTenantMatch(...) or " +
             "ctx.systemDb.outsideTransaction.acknowledgeCrossTenant(...)")
         : rawDbOutsideTransaction;
+    // Not offered to r.systemScope() handlers: the raw outside-tx db is unfiltered in
+    // system mode, so fn would hand them the same cross-tenant door the guards above close.
+    const outsideTransaction = !isSystem && rawDbOutsideTransaction
+        ? (fn) => runInOwnTransaction(rawDbOutsideTransaction, fn)
+        : undefined;
     const log = context.log?.child({
         handler: type,
         tenantId: user.tenantId,
@@ -255,7 +313,9 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
     // _fileProviderResolver when a file-provider plugin is mounted; falls back
     // to a statically-injected context.files (tests).
     const fileResolver = context._fileProviderResolver;
-    const files = fileResolver ? createFileContext(() => fileResolver(user.tenantId)) : context.files;
+    const files = fileResolver
+        ? createFileContext(() => fileResolver(user.tenantId), user.tenantId)
+        : context.files;
     // ctx.derivatives builds on ctx.files (needs a FileContext to read the
     // original + write the variant) plus db (to look up the FileRef row) — so
     // it can only be constructed exactly when files+db both resolved; falls
@@ -302,13 +362,13 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         ungatedMemberReader ??= createMemberReaderFn(ctx, user.tenantId, tx);
         return ungatedMemberReader(userId, qn, payload);
     }, { ...identitySwitchAudit, tenantId: user.tenantId });
+    // queryProjection works against both single-stream and multi-stream
+    // projections. MSPs without a table cannot be queried — those are
+    // side-effect-only consumers (no state to read back).
+    const findProjectionTable = (qualifiedName) => registry.getAllProjections().get(qualifiedName)?.table ??
+        registry.getAllMultiStreamProjections().get(qualifiedName)?.table;
     const ungatedQueryProjection = async (qualifiedName, queryOptions) => {
-        // queryProjection works against both single-stream and multi-stream
-        // projections. MSPs without a table cannot be queried — those are
-        // side-effect-only consumers (no state to read back).
-        const singleProj = registry.getAllProjections().get(qualifiedName);
-        const mspProj = registry.getAllMultiStreamProjections().get(qualifiedName);
-        const projTable = singleProj?.table ?? mspProj?.table;
+        const projTable = findProjectionTable(qualifiedName);
         if (!projTable) {
             const singleNames = [...registry.getAllProjections().keys()];
             const mspNames = [...registry.getAllMultiStreamProjections().keys()].filter((n) => registry.getAllMultiStreamProjections().get(n)?.table);
@@ -330,7 +390,11 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         const rows = await selectMany(dbSource, projTable, where);
         return rows; // @cast-boundary engine-payload
     };
-    const queryProjection = createGatedProjectionReader(`handler "${type}"`, hasIdentitySwitchGrant, ungatedQueryProjection, identitySwitchAudit);
+    const queryProjection = createGatedProjectionReader(`handler "${type}"`, hasIdentitySwitchGrant, ungatedQueryProjection, identitySwitchAudit, (qualifiedName) => {
+        const projTable = findProjectionTable(qualifiedName);
+        // Unknown projection: keep the grant check first, the ungated reader reports the real error.
+        return projTable === undefined || hasTenantColumn(projTable);
+    });
     const bridge = {
         query: (targetType, payload) => executeQuery(ctx, targetType, payload, user, origin, tx), // @wrapper-known semantic-alias
         queryAs: identitySwitch.queryAs,
@@ -624,6 +688,7 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
         registry,
         db: exposedDb,
         dbOutsideTransaction,
+        ...(outsideTransaction && { outsideTransaction }),
         ...(systemDb && { systemDb }),
         log,
         notify,

package/dist/pipeline/dispatch-stream.d.ts

@@ -1,3 +1,7 @@
 import type { SessionUser } from "../engine/types/index.js";
 import { type DispatchContext, type WriteOrigin } from "./dispatch-shared.js";
-export declare function executeStream(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin): AsyncGenerator<unknown>;
+export type StreamOptions = {
+    /** JWT exp (epoch seconds) the stream was opened with; the stream ends when it passes. */
+    readonly tokenExpiresAtSec?: number | undefined;
+};
+export declare function executeStream(ctx: DispatchContext, type: string, payload: unknown, user: SessionUser, origin: WriteOrigin, options?: StreamOptions): AsyncGenerator<unknown>;

package/dist/pipeline/dispatch-stream.js

@@ -1,4 +1,5 @@
 import { accessInvalidationCredentialFor } from "../api/sse-broker.js";
+import { scheduleTokenExpiry } from "../api/token-expiry-timer.js";
 import { hasAccess } from "../engine/access.js";
 import { AccessDeniedError, memberResolutionReadOnlyDenied, NotFoundError, validationErrorFromZod, } from "../errors/index.js";
 import { assertNoSecretLeak } from "../secrets/index.js";
@@ -10,10 +11,10 @@ import { handlerAccessError } from "./handler-access-error.js";
 // of returning a single response. streamHandler never entity-maps (unlike
 // write/queryHandler — see feature-entity-handlers.ts), so there's no
 // field-access filter or postQuery-hook stage to run here.
-export async function* executeStream(ctx, type, payload, user, origin) {
-    yield* runStreamInstrumented(ctx, type, user, () => executeStreamInner(ctx, type, payload, user, origin));
+export async function* executeStream(ctx, type, payload, user, origin, options) {
+    yield* runStreamInstrumented(ctx, type, user, () => executeStreamInner(ctx, type, payload, user, origin, options));
 }
-async function* executeStreamInner(ctx, type, payload, user, origin) {
+async function* executeStreamInner(ctx, type, payload, user, origin, options) {
     const { registry } = ctx;
     const handler = registry.getStreamHandler(type);
     if (!handler)
@@ -32,15 +33,18 @@ async function* executeStreamInner(ctx, type, payload, user, origin) {
     if (!parsed.success) {
         throw validationErrorFromZod(parsed.error);
     }
-    // Idle (heartbeat-only) streams must also cut on access revoke — race each
-    // pull against an invalidated Deferred instead of a post-chunk boolean.
-    let resolveInvalidated;
-    const invalidated = new Promise((resolve) => {
-        resolveInvalidated = resolve;
+    // Idle (heartbeat-only) streams must also cut on access revoke or token expiry — race each
+    // pull against an ended Deferred instead of a post-chunk boolean.
+    let resolveEnded;
+    const ended = new Promise((resolve) => {
+        resolveEnded = resolve;
     });
     const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(user.id, () => {
-        resolveInvalidated?.();
+        resolveEnded?.("revoked");
     }, accessInvalidationCredentialFor(user));
+    const expiryTimer = scheduleTokenExpiry(options?.tokenExpiresAtSec, () => {
+        resolveEnded?.("expired");
+    });
     let iterator;
     // When access is revoked mid-pull, `iterator.next()` is still in flight.
     // Awaiting `iterator.return()` in that state deadlocks async generators in
@@ -55,14 +59,16 @@ async function* executeStreamInner(ctx, type, payload, user, origin) {
             const nextPull = iterator.next();
             const outcome = await Promise.race([
                 nextPull.then((result) => ({ kind: "chunk", result })),
-                invalidated.then(() => ({ kind: "invalidated" })),
+                ended.then((reason) => ({ kind: "ended", reason })),
             ]);
-            if (outcome.kind === "invalidated") {
+            if (outcome.kind === "ended") {
                 abandonedForInvalidation = true;
                 void nextPull.catch(() => { });
                 void iterator.return?.(undefined)?.then(undefined, () => { });
                 throw new AccessDeniedError({
-                    message: `access revoked mid-stream for ${type}`,
+                    message: outcome.reason === "expired"
+                        ? `token expired mid-stream for ${type}`
+                        : `access revoked mid-stream for ${type}`,
                     details: { handler: type },
                 });
             }
@@ -75,6 +81,7 @@ async function* executeStreamInner(ctx, type, payload, user, origin) {
     }
     finally {
         unsubscribeAccessInvalidation?.();
+        clearTimeout(expiryTimer);
         // Close the generator so cleanup runs; skip awaiting return() after
         // access-revoke abandonment — overlapping next()+return() can deadlock.
         if (iterator !== undefined && !abandonedForInvalidation) {

package/dist/pipeline/dispatcher.d.ts

@@ -4,6 +4,7 @@ import type { EffectiveFeaturesResolver } from "../engine/tier-resolver-extensio
 import type { TenantId } from "../engine/types/identifiers.js";
 import type { ActiveMembershipResult, AppContext, DispatchWriteRef, JobRunnerRef, MemberReader, Registry, SessionUser, WriteResult } from "../engine/types/index.js";
 import type { BatchCommand, BatchResult } from "./dispatch-shared.js";
+import { type StreamOptions } from "./dispatch-stream.js";
 import { type HandlerType } from "./dispatcher-utils.js";
 import type { IdempotencyGuard } from "./idempotency.js";
 import type { LifecycleHooks } from "./lifecycle-pipeline.js";
@@ -14,6 +15,7 @@ export type DispatcherOptions = {
     lifecycle?: LifecycleHooks;
     jobRunner?: JobRunnerRef;
     membershipQuery?: string;
+    memberReadTimeoutMs?: number;
     effectiveFeatures?: EffectiveFeaturesResolver;
     sseBroker?: SseBroker;
     cacheSync?: CacheSyncBus | null;
@@ -21,7 +23,7 @@ export type DispatcherOptions = {
 export type Dispatcher = {
     write(type: HandlerType, payload: unknown, user: SessionUser, requestId?: string): Promise<WriteResult>;
     query(type: HandlerType, payload: unknown, user: SessionUser): Promise<unknown>;
-    stream(type: HandlerType, payload: unknown, user: SessionUser): AsyncGenerator<unknown>;
+    stream(type: HandlerType, payload: unknown, user: SessionUser, options?: StreamOptions): AsyncGenerator<unknown>;
     command(type: HandlerType, payload: unknown, user: SessionUser): Promise<void>;
     batch(commands: readonly BatchCommand[], user: SessionUser, requestId?: string): Promise<BatchResult>;
     resolveAuthClaims(user: SessionUser): Promise<Record<string, unknown>>;

package/dist/pipeline/dispatcher.js

@@ -1,5 +1,4 @@
 import { runWithWriteOrigin } from "../api/request-context.js";
-import { DEPRECATED_CROSS_TENANT_SIGNAL, isDeprecatedCrossTenantHandler, } from "../engine/entity-handlers.js";
 import { TENANT_MEMBERSHIPS_QUERY } from "../engine/extension-names.js";
 import { InternalError, reraiseAsKumikoError } from "../errors/index.js";
 import { createEscapeHatchReportWindow, getFallbackMeter, getFallbackTracer, registerStandardMetrics, } from "../observability/index.js";
@@ -7,7 +6,7 @@ import { CACHE_SYNC_TOPICS, isTenantConfigSyncMessage } from "../redis/cache-syn
 import { INTERACTIVE_SIGN_IN_POLICY, resolveActiveMembershipFn } from "./active-membership.js";
 import { runBatch, unwrapSingle } from "./dispatch-batch.js";
 import { executeQuery } from "./dispatch-query.js";
-import { resolveAuthClaimsFn, TENANT_TIMEZONE_CONFIG_KEY } from "./dispatch-shared.js";
+import { DEFAULT_MEMBER_READ_TIMEOUT_MS, resolveAuthClaimsFn, TENANT_TIMEZONE_CONFIG_KEY, } from "./dispatch-shared.js";
 import { executeStream } from "./dispatch-stream.js";
 import { resolveType } from "./dispatcher-utils.js";
 import { createMemberReaderFn } from "./member-reader.js";
@@ -49,6 +48,12 @@ export function dispatcherToWriteRef(dispatcher) {
 export function createDispatcher(registry, context, options = {}) {
     const { idempotency, lifecycle, jobRunner, effectiveFeatures, sseBroker } = options;
     const membershipQuery = options.membershipQuery ?? TENANT_MEMBERSHIPS_QUERY;
+    const memberReadTimeoutMs = options.memberReadTimeoutMs ?? DEFAULT_MEMBER_READ_TIMEOUT_MS;
+    if (!Number.isSafeInteger(memberReadTimeoutMs) || memberReadTimeoutMs <= 0) {
+        throw new InternalError({
+            message: `DispatcherOptions.memberReadTimeoutMs must be a positive integer, got ${memberReadTimeoutMs}`,
+        });
+    }
     // Pre-build tables and transition maps for auto-guard (avoid per-request allocation)
     const tableCache = new Map();
     const transitionCache = new Map();
@@ -75,17 +80,6 @@ export function createDispatcher(registry, context, options = {}) {
     // Ensure standard metrics exist on whatever meter we ended up with.
     // Idempotent: buildServer may have registered them already.
     registerStandardMetrics(dispatcherMeter);
-    for (const def of [
-        ...registry.getAllWriteHandlers().values(),
-        ...registry.getAllQueryHandlers().values(),
-    ]) {
-        if (isDeprecatedCrossTenantHandler(def.handler)) {
-            context.log?.warn(DEPRECATED_CROSS_TENANT_SIGNAL, {
-                handler: def.name,
-                migration: 'replace crossTenant: true with escapeHatch: { reason: "<why this operator handler reads/writes every tenant>" } (bun node_modules/@cosmicdrift/kumiko-framework/src/scripts/codemod/migrate-cross-tenant.ts)',
-            });
-        }
-    }
     const ctx = {
         registry,
         appContext: context,
@@ -102,6 +96,7 @@ export function createDispatcher(registry, context, options = {}) {
         tracer: dispatcherTracer,
         meter: dispatcherMeter,
         membershipQuery,
+        memberReadTimeoutMs,
     };
     const dispatcher = {
         async write(typeOrRef, payload, user, requestId) {
@@ -116,9 +111,9 @@ export function createDispatcher(registry, context, options = {}) {
             const origin = rootWriteOrigin(registry, type, user);
             return runWithWriteOrigin(origin, () => executeQuery(ctx, type, payload, user, origin));
         },
-        stream: (typeOrRef, payload, user) => {
+        stream: (typeOrRef, payload, user, options) => {
             const type = resolveType(typeOrRef);
-            return executeStream(ctx, type, payload, user, rootWriteOrigin(registry, type, user));
+            return executeStream(ctx, type, payload, user, rootWriteOrigin(registry, type, user), options);
         },
         async command(typeOrRef, payload, user) {
             const type = resolveType(typeOrRef);

package/dist/pipeline/event-consumer-state.js

@@ -1,6 +1,5 @@
 import { bigint, index, instant, integer, jsonb, table as pgTable, primaryKey, sql, text, } from "../db/dialect.js";
 import { alterTableAddColumn } from "../db/queries/ddl.js";
-import { deleteOrphanedPerInstanceConsumerRows } from "../db/queries/event-consumer.js";
 import { tableExists } from "../db/schema-inspection.js";
 import { unsafePushTables } from "../stack/index.js";
 // Reserved sentinel used in the instance_id column for consumers whose
@@ -39,20 +38,6 @@ export const eventConsumerStateTable = pgTable("kumiko_event_consumers", {
     pk: primaryKey({ columns: [t.name, t.instanceId] }),
     statusIdx: index("kumiko_event_consumers_status_idx").on(t.status),
 }));
-// fw#2625: these three consumers moved from delivery: "per-instance" to
-// "shared" (see system-hooks.ts and bundled-features/feature-toggles/
-// feature.ts). Their old per-instance rows are now orphaned — no dispatcher
-// will ever advance them again — and pruneEvents stays pinned to whichever
-// one has the lowest cursor forever. Named here (not imported from
-// system-hooks.ts / feature.ts) to avoid a pipeline → system-hooks /
-// bundled-features dependency for three string literals; those modules'
-// own exported constants (or, for feature-toggles, the qualified consumer
-// name qualify() produces) are the source of truth these must match.
-const ORPHANED_PER_INSTANCE_CONSUMER_NAMES = [
-    "system:consumer:sse-broadcast",
-    "system:consumer:access-invalidation",
-    "feature-toggles:projection:toggle-cache-sync",
-];
 // Object-const form lets call sites write `ConsumerStatuses.disabled` instead
 // of the raw string, which keeps status checks refactor-safe. The runtime
 // value on each field is the same string the DB stores — no mapping needed.
@@ -84,19 +69,8 @@ export async function createEventConsumerStateTable(db) {
         /* ifNotExists */ true);
         await alterTableAddColumn(db, "kumiko_event_consumers", "last_failed_event_id", "bigint", "", "", 
         /* ifNotExists */ true);
-        // Runs on every boot, including mid-rolling-deploy: a new pod can delete
-        // an old pod's still-in-use per-instance row out from under it. That old
-        // pod's next acquireConsumer then finds no row, delivers no more SSE/
-        // access-invalidation events for the rest of its lifetime — a one-time,
-        // few-second gap that resolves itself as clients reconnect onto new pods
-        // during the same rollout. Not a replay risk: preRegisterConsumers only
-        // (re-)inserts a row at dispatcher startup, so the deleted pod won't
-        // silently get a fresh cursor-0 row and replay history mid-flight.
-        await deleteOrphanedPerInstanceConsumerRows(db, ORPHANED_PER_INSTANCE_CONSUMER_NAMES, SHARED_INSTANCE_SENTINEL);
         // skip: table (+ any missing column) is already up to date
         return;
     }
-    // A freshly created table has no rows to clean up — the DELETE only
-    // matters for a DB that already ran the old per-instance consumers.
     await unsafePushTables(db, { kumikoEventConsumers: eventConsumerStateTable });
 }