@cosmicdrift/kumiko-framework 0.328.0 → 0.329.0

package/dist/pipeline/member-reader.js

@@ -0,0 +1,87 @@
+// ctx.queryAsMember — reads a query handler as a stored tenant member. The
+// resolved SessionUser never leaves this module.
+import { buildSessionRoles } from "../engine/membership-roles.js";
+import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
+import { AccessDeniedError, FrameworkReasons, InternalError } from "../errors/index.js";
+import { resolveActiveMembershipFn, resolvePrincipalPlugin, } from "./active-membership.js";
+import { executeQuery } from "./dispatch-query.js";
+import { resolveAuthClaimsFn, resolveDbSource } from "./dispatch-shared.js";
+import { isSystemIdentity } from "./system-identity-switch.js";
+import { rootWriteOrigin } from "./write-origin.js";
+// Stricter than interactive sign-in: an unknown principal or a tenant
+// mid-teardown must not resolve — there is no user-facing flow to recover.
+export const BACKGROUND_READ_POLICY = {
+    allowPendingDestruction: false,
+    allowUnknownPrincipal: false,
+};
+// Generic on purpose — details never carry which check failed, so a caller
+// can't probe another user's membership state via the error shape.
+function memberResolutionDenied() {
+    return new AccessDeniedError({
+        message: "not permitted to read as this member",
+        details: { reason: FrameworkReasons.memberResolutionDenied },
+    });
+}
+// Membership/principal/lifecycle resolve on the root connection, outside the
+// caller's tx — only the target query runs inside it (via the `tx` argument).
+async function resolveMember(ctx, tenantId, userId) {
+    const membership = await resolveActiveMembershipFn(ctx, userId, tenantId, BACKGROUND_READ_POLICY);
+    if (membership.kind === "rejected")
+        throw memberResolutionDenied();
+    const principalPlugin = resolvePrincipalPlugin(ctx.registry);
+    const dbSource = resolveDbSource(ctx, undefined);
+    if (!dbSource) {
+        throw new InternalError({
+            message: "ctx.queryAsMember requires a database connection — none is configured.",
+        });
+    }
+    // resolveDbSource(ctx, undefined) never returns a tx, only AppContext.db.
+    const db = dbSource; // @cast-boundary db-operator
+    const profile = await principalPlugin.resolveProfile(userId, { db });
+    if (!profile)
+        throw memberResolutionDenied();
+    // buildSessionRoles runs stripForbiddenMembershipRoles on the membership half (rebuild-resurrected roles).
+    const roles = buildSessionRoles(profile.globalRoles, membership.membership.roles);
+    const base = {
+        id: userId,
+        tenantId,
+        roles,
+        ...(profile.timezone !== undefined && { timezone: profile.timezone }),
+        ...(profile.locale !== undefined && { locale: profile.locale }),
+        origin: "member-resolution",
+    };
+    // Defense in depth: buildSessionRoles strips forbidden roles only from the
+    // membership half, not globalRoles — a resolved principal must never be SYSTEM.
+    if (isSystemIdentity(base))
+        throw memberResolutionDenied();
+    const claims = await resolveAuthClaimsFn(ctx, base);
+    return Object.keys(claims).length > 0 ? { ...base, claims } : base;
+}
+// One reader per handler invocation (shared with its hooks) or job run; a
+// nested dispatch builds its own reader with its own cache.
+export function createMemberReaderFn(ctx, tenantId, tx) {
+    if (tenantId === SYSTEM_TENANT_ID) {
+        throw new InternalError({
+            message: "queryAsMember needs a tenant-scoped context — got SYSTEM_TENANT_ID.",
+        });
+    }
+    // Only successful resolutions are cached — a rejected/failed resolution
+    // deletes its own entry so a transient error can't poison the rest of the run.
+    const cache = new Map();
+    function resolve(userId) {
+        const cached = cache.get(userId);
+        if (cached)
+            return cached;
+        const pending = resolveMember(ctx, tenantId, userId);
+        cache.set(userId, pending);
+        pending.catch(() => {
+            cache.delete(userId);
+        });
+        return pending;
+    }
+    return async (userId, qn, payload) => {
+        const user = await resolve(userId);
+        // A resolved member is never anonymous and read-only, so it may start its own root.
+        return executeQuery(ctx, qn, payload, user, rootWriteOrigin(ctx.registry, qn, user), tx);
+    };
+}

package/dist/pipeline/msp-rebuild.d.ts

@@ -0,0 +1,12 @@
+import type { DbConnection } from "../db/connection.js";
+import type { Registry } from "../engine/types/index.js";
+import type { Meter } from "../observability/types/metric.js";
+import type { RebuildResult } from "./projection-rebuild.js";
+export type MspRebuildDeps = {
+    readonly db: DbConnection;
+    readonly registry: Registry;
+    readonly meter?: Meter;
+    readonly onMetrics?: (result: RebuildResult) => void;
+    readonly markDeadOnFailure?: boolean;
+};
+export declare function rebuildMultiStreamProjection(mspName: string, deps: MspRebuildDeps): Promise<RebuildResult>;

package/dist/pipeline/msp-rebuild.js

@@ -0,0 +1,163 @@
+import { markConsumerRebuildFailed, resetConsumerForMspRebuild, selectConsumerForUpdate, selectEventIdHorizonWithMissingRanges, selectSnapshotXmax, updateConsumerRebuildCursor, } from "../db/queries/event-consumer.js";
+import { assertLiveColumnsMatchMeta, assertLiveTableHasNoRowLevelSecurity, buildShadowTable, ensureRebuildSchema, rebuildMetaOrThrow, swapShadowIntoLive, } from "../db/queries/shadow-swap.js";
+import { runInSavepoint, selectMany } from "../db/query.js";
+import { InternalError } from "../errors/index.js";
+import { loadAggregate, loadAggregateAsOf } from "../event-store/event-store.js";
+import { eventsTable, upcastStoredEvent } from "../event-store/index.js";
+import { createRebuildDeadLetterTable, recordRebuildDeadLetters, } from "../event-store/rebuild-dead-letter.js";
+import { upcastStoredEvents } from "../event-store/upcaster.js";
+import { emitProjectionRebuild } from "../observability/standard-metrics.js";
+import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
+import { capPendingGapsBelowCursor, subtractSortedIdsFromRanges } from "./pending-gap-ranges.js";
+function createRebuildCtx(registry, db, tenantId) {
+    // Both surfaces throw — rebuild MUST NOT emit. We share one impl.
+    const refuseAppend = async (args) => {
+        throw new InternalError({
+            message: `rebuildMultiStreamProjection: ctx.appendEvent("${args.type}") is not supported during rebuild. The events your saga would emit already live in the event log — rebuild only derives read-model state. If you need to retroactively emit events, do so via a dedicated write-handler, not via the apply path.`,
+        });
+    };
+    return {
+        registry,
+        appendEvent: refuseAppend, // @cast-boundary engine-bridge
+        unsafeAppendEvent: refuseAppend,
+        loadAggregate: async (aggregateId, options) => {
+            const events = options?.asOf
+                ? await loadAggregateAsOf(db, aggregateId, tenantId, options.asOf)
+                : await loadAggregate(db, aggregateId, tenantId);
+            return upcastStoredEvents(events, registry.getEventUpcasters(), { db, tenantId });
+        },
+    };
+}
+export async function rebuildMultiStreamProjection(mspName, deps) {
+    const { db, registry } = deps;
+    const msp = registry.getAllMultiStreamProjections().get(mspName);
+    if (!msp) {
+        throw new InternalError({
+            message: `MultiStreamProjection "${mspName}" is not registered. Known: ${[...registry.getAllMultiStreamProjections().keys()].join(", ") || "(none)"}`,
+        });
+    }
+    if (!msp.table) {
+        throw new InternalError({
+            message: `MultiStreamProjection "${mspName}" has no backing table — it is a pure side-effect consumer (webhooks, notifications, external sync). Rebuild would re-invoke those side-effects by replaying the log. For poison events use bun kumiko consumer skip / restart; there is no analogous "rebuild" concept for side-effect sinks.`,
+        });
+    }
+    const meta = rebuildMetaOrThrow(msp.table, mspName);
+    // Declared per-projection (MspErrorMode): the rebuild policy falls back to
+    // continuous when omitted — previously declared but never honored here.
+    const skipApplyErrors = (msp.errorMode?.rebuild ?? msp.errorMode?.continuous)?.skipApplyErrors === true;
+    const startedAt = Date.now();
+    let eventsProcessed = 0;
+    let lastProcessedEventId = 0n;
+    const skipped = [];
+    try {
+        await ensureRebuildSchema(db);
+        // Outside the rebuild tx, like the schema: idempotent DDL colliding
+        // inside the tx would roll the whole replay back.
+        if (skipApplyErrors)
+            await createRebuildDeadLetterTable(db);
+        const replayedIds = [];
+        await db.begin(async (tx) => {
+            await resetConsumerForMspRebuild(tx, mspName, SHARED_INSTANCE_SENTINEL);
+            await selectConsumerForUpdate(tx, mspName, SHARED_INSTANCE_SENTINEL);
+            const { horizon, gaps: seededGaps } = await selectEventIdHorizonWithMissingRanges(tx);
+            await assertLiveColumnsMatchMeta(tx, meta, mspName);
+            await assertLiveTableHasNoRowLevelSecurity(tx, meta.tableName, mspName);
+            await buildShadowTable(tx, meta);
+            const subscribedTypes = Object.keys(msp.apply);
+            if (subscribedTypes.length > 0) {
+                // Events past `horizon` are the live dispatcher's to deliver via
+                // `id > cursor` — bounding the replay here is what keeps the two
+                // paths from ever double-applying the same event.
+                const events = await selectMany(tx, eventsTable, { type: [...subscribedTypes], id: { lte: horizon } }, { orderBy: { col: "id", direction: "asc" } });
+                const upcasters = registry.getEventUpcasters();
+                for (const row of events) {
+                    const raw = {
+                        id: String(row.id),
+                        aggregateId: row.aggregateId,
+                        aggregateType: row.aggregateType,
+                        tenantId: row.tenantId,
+                        version: row.version,
+                        type: row.type,
+                        eventVersion: row.eventVersion,
+                        payload: row.payload,
+                        metadata: row.metadata,
+                        createdAt: row.createdAt,
+                        createdBy: row.createdBy,
+                    };
+                    const storedEvent = await upcastStoredEvent(raw, upcasters, {
+                        db: tx,
+                        tenantId: row.tenantId,
+                    });
+                    replayedIds.push(row.id);
+                    const applyFn = msp.apply[row.type];
+                    if (!applyFn)
+                        continue;
+                    if (skipApplyErrors) {
+                        // Driver-native savepoint: a throwing SQL statement would
+                        // otherwise poison the rebuild tx (25P02) AND make the driver
+                        // reject the whole begin() even after a caught error. Apply +
+                        // ctx run on the savepoint-scoped handle.
+                        try {
+                            await runInSavepoint(tx, async (sp) => {
+                                const rebuildCtx = createRebuildCtx(registry, sp, row.tenantId);
+                                await applyFn(storedEvent, sp, rebuildCtx);
+                            });
+                        }
+                        catch (e) {
+                            skipped.push({ event: storedEvent, error: e });
+                        }
+                    }
+                    else {
+                        const rebuildCtx = createRebuildCtx(registry, tx, row.tenantId);
+                        await applyFn(storedEvent, tx, rebuildCtx);
+                    }
+                    eventsProcessed++;
+                    lastProcessedEventId = row.id;
+                }
+            }
+            if (skipped.length > 0) {
+                await recordRebuildDeadLetters(tx, mspName, skipped);
+            }
+            // Every id actually read this replay (applied or skip-applied) is
+            // resolved — a live pass must never redeliver it. Anything surviving
+            // that subtraction is capped below the written cursor: a gap at or
+            // above it is the live dispatcher's own `id > cursor` territory, not
+            // pending_gaps'. No events replayed (cursor stays 0) means no gap can
+            // sit below it either.
+            const survivingGaps = subtractSortedIdsFromRanges(seededGaps, replayedIds);
+            const cappedGaps = capPendingGapsBelowCursor(survivingGaps, lastProcessedEventId);
+            const postReplayXmax = cappedGaps.length > 0 ? await selectSnapshotXmax(tx) : null;
+            const pendingGaps = postReplayXmax === null
+                ? cappedGaps
+                : cappedGaps.map((gap) => ({ ...gap, xmax: postReplayXmax }));
+            await updateConsumerRebuildCursor(tx, mspName, SHARED_INSTANCE_SENTINEL, lastProcessedEventId, pendingGaps);
+            await swapShadowIntoLive(tx, meta.tableName);
+        });
+    }
+    catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        if (deps.markDeadOnFailure !== false) {
+            await markConsumerRebuildFailed(db, mspName, SHARED_INSTANCE_SENTINEL, message);
+        }
+        if (deps.meter) {
+            emitProjectionRebuild(deps.meter, { projection: mspName, success: false }, (Date.now() - startedAt) / 1000, 0);
+        }
+        throw e;
+    }
+    const result = {
+        projection: mspName,
+        eventsProcessed,
+        eventsSkipped: skipped.length,
+        lastProcessedEventId,
+        durationMs: Date.now() - startedAt,
+        // countColumnDrift is single-table (public.<t> vs kumiko_rebuild.<t>) and
+        // not wired into the multi-stream rebuild path — MSPs don't run the
+        // implicit-projection ghost-row guard either. #916 scope is single-table.
+        columnDriftCount: 0,
+    };
+    if (deps.meter) {
+        emitProjectionRebuild(deps.meter, { projection: mspName, success: true }, result.durationMs / 1000, eventsProcessed);
+    }
+    deps.onMetrics?.(result);
+    return result;
+}

package/dist/pipeline/multi-stream-apply-context.d.ts

@@ -0,0 +1,18 @@
+import type { DerivativesContext } from "@cosmicdrift/kumiko-types/derivatives-types";
+import type { MultiStreamApplyContext } from "@cosmicdrift/kumiko-types/multi-stream-apply-context-types";
+import type { SecretsContext } from "@cosmicdrift/kumiko-types/secrets-types";
+import type { DbRunner } from "../db/connection.js";
+import type { Registry, TenantId } from "../engine/types/index.js";
+import type { FileContext } from "../files/file-handle.js";
+export type { MultiStreamApplyContext } from "@cosmicdrift/kumiko-types/multi-stream-apply-context-types";
+export type MultiStreamApplyContextDeps = {
+    readonly registry: Registry;
+    readonly db: DbRunner;
+    readonly tenantId: TenantId;
+    readonly userId: string;
+    readonly callerFeature?: string;
+    readonly files?: FileContext;
+    readonly derivatives?: DerivativesContext;
+    readonly secrets?: SecretsContext;
+};
+export declare function createMultiStreamApplyContext(deps: MultiStreamApplyContextDeps): MultiStreamApplyContext;

package/dist/pipeline/multi-stream-apply-context.js

@@ -0,0 +1,40 @@
+import { loadAggregate, loadAggregateAsOf } from "../event-store/event-store.js";
+import { upcastStoredEvents } from "../event-store/upcaster.js";
+import { appendDomainEventCore } from "./append-event-core.js";
+export function createMultiStreamApplyContext(deps) {
+    return {
+        registry: deps.registry,
+        ...(deps.files ? { files: deps.files } : {}),
+        ...(deps.derivatives ? { derivatives: deps.derivatives } : {}),
+        ...(deps.secrets ? { secrets: deps.secrets } : {}),
+        appendEvent: (async (args) => {
+            await appendDomainEventCore({
+                registry: deps.registry,
+                db: deps.db,
+                tenantId: deps.tenantId,
+                userId: deps.userId,
+                callSiteLabel: "MSP-apply ctx.appendEvent",
+                ...(deps.callerFeature && { callerFeature: deps.callerFeature }),
+            }, args);
+        }), // @cast-boundary engine-bridge
+        unsafeAppendEvent: async (args) => {
+            await appendDomainEventCore({
+                registry: deps.registry,
+                db: deps.db,
+                tenantId: deps.tenantId,
+                userId: deps.userId,
+                callSiteLabel: "MSP-apply ctx.unsafeAppendEvent",
+                ...(deps.callerFeature && { callerFeature: deps.callerFeature }),
+            }, args);
+        },
+        loadAggregate: async (aggregateId, options) => {
+            const events = options?.asOf
+                ? await loadAggregateAsOf(deps.db, aggregateId, deps.tenantId, options.asOf)
+                : await loadAggregate(deps.db, aggregateId, deps.tenantId);
+            return upcastStoredEvents(events, deps.registry.getEventUpcasters(), {
+                db: deps.db,
+                tenantId: deps.tenantId,
+            });
+        },
+    };
+}

package/dist/pipeline/pending-gap-ranges.d.ts

@@ -0,0 +1,12 @@
+import type { PendingIdRange } from "../db/queries/event-store.js";
+import type { PendingGapEntry } from "./event-consumer-state.js";
+export declare function toIdRanges(gaps: readonly PendingGapEntry[]): PendingIdRange[];
+export declare function rangeContainsId(range: PendingGapEntry, id: bigint): boolean;
+export declare function isRangeFullyCoveredByFetch(range: PendingGapEntry, truncated: boolean, maxFetchedId: bigint | null): boolean;
+export declare function partitionBurntGaps(gaps: readonly PendingGapEntry[], fetchedIds: readonly bigint[], truncated: boolean, xminNow: string): {
+    readonly burnt: PendingGapEntry[];
+    readonly surviving: PendingGapEntry[];
+};
+export declare function splitRangeExcludingIds(range: PendingGapEntry, excludeIds: readonly bigint[]): PendingGapEntry[];
+export declare function subtractSortedIdsFromRanges(ranges: readonly PendingGapEntry[], sortedIds: readonly bigint[]): PendingGapEntry[];
+export declare function capPendingGapsBelowCursor(gaps: readonly PendingGapEntry[], cursor: bigint): PendingGapEntry[];

package/dist/pipeline/pending-gap-ranges.js

@@ -0,0 +1,99 @@
+export function toIdRanges(gaps) {
+    return gaps.map((g) => ({ from: BigInt(g.from), to: BigInt(g.to) }));
+}
+export function rangeContainsId(range, id) {
+    return id >= BigInt(range.from) && id <= BigInt(range.to);
+}
+// A range is only provably burnt (rolled back, not just slow) once it's
+// certain the fetch would have surfaced any of its ids had they been
+// visible. LIMIT can cut a fetch off before reaching the range's ids —
+// `to < maxFetchedId` proves the scan passed the range regardless of the
+// cutoff, since ORDER BY id ASC LIMIT N always includes every matching id
+// below the largest one it did return.
+export function isRangeFullyCoveredByFetch(range, truncated, maxFetchedId) {
+    if (!truncated)
+        return true;
+    if (maxFetchedId === null)
+        return false;
+    return BigInt(range.to) < maxFetchedId;
+}
+// A gap is burnt when none of its ids showed up in this turn's fetch, the
+// fetch is proven to have covered it, and its recorded xmax is behind this
+// turn's xmin (every xact that could still produce a row has finished).
+export function partitionBurntGaps(gaps, fetchedIds, truncated, xminNow) {
+    const maxFetchedId = fetchedIds.at(-1) ?? null;
+    const burnt = [];
+    const surviving = [];
+    for (const gap of gaps) {
+        const hasVisibleId = fetchedIds.some((id) => rangeContainsId(gap, id));
+        const isBurnt = !hasVisibleId &&
+            isRangeFullyCoveredByFetch(gap, truncated, maxFetchedId) &&
+            BigInt(gap.xmax) <= BigInt(xminNow);
+        (isBurnt ? burnt : surviving).push(gap);
+    }
+    return { burnt, surviving };
+}
+// Carves `excludeIds` (delivered or skip-applied this turn) out of `range`,
+// yielding 0-2 sub-ranges. xmax is preserved on every surviving piece.
+export function splitRangeExcludingIds(range, excludeIds) {
+    const from = BigInt(range.from);
+    const to = BigInt(range.to);
+    const inRange = excludeIds
+        .filter((id) => id >= from && id <= to)
+        .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
+    if (inRange.length === 0)
+        return [range];
+    const parts = [];
+    let cursor = from;
+    for (const id of inRange) {
+        if (id > cursor)
+            parts.push({ from: cursor.toString(), to: (id - 1n).toString(), xmax: range.xmax });
+        cursor = id + 1n;
+    }
+    if (cursor <= to)
+        parts.push({ from: cursor.toString(), to: to.toString(), xmax: range.xmax });
+    return parts;
+}
+// Same carve-out as splitRangeExcludingIds, but for many ranges against many
+// ids at once — an MSP rebuild's replay reads events in ascending id order,
+// so a single forward pointer over `sortedIds` (both inputs already sorted,
+// ranges non-overlapping) avoids scanning the full id list per range.
+export function subtractSortedIdsFromRanges(ranges, sortedIds) {
+    if (sortedIds.length === 0)
+        return [...ranges];
+    const result = [];
+    let cursor = 0;
+    for (const range of ranges) {
+        const from = BigInt(range.from);
+        const to = BigInt(range.to);
+        let id = sortedIds[cursor];
+        while (id !== undefined && id < from) {
+            cursor++;
+            id = sortedIds[cursor];
+        }
+        const idsInRange = [];
+        while (id !== undefined && id <= to) {
+            idsInRange.push(id);
+            cursor++;
+            id = sortedIds[cursor];
+        }
+        result.push(...splitRangeExcludingIds(range, idsInRange));
+    }
+    return result;
+}
+// Ranges above the written cursor belong to the live dispatcher's own
+// `id > cursor` territory; a range straddling it is truncated at cursor - 1.
+export function capPendingGapsBelowCursor(gaps, cursor) {
+    if (cursor <= 0n)
+        return [];
+    const capped = [];
+    for (const gap of gaps) {
+        const from = BigInt(gap.from);
+        if (from >= cursor)
+            continue;
+        const to = BigInt(gap.to);
+        const cappedTo = to >= cursor ? cursor - 1n : to;
+        capped.push({ ...gap, to: cappedTo.toString() });
+    }
+    return capped;
+}

package/dist/pipeline/projection-rebuild.d.ts

@@ -0,0 +1,55 @@
+import type { DbConnection } from "../db/connection.js";
+import type { Registry } from "../engine/types/index.js";
+import type { Meter } from "../observability/types/metric.js";
+export type RebuildResult = {
+    readonly projection: string;
+    readonly eventsProcessed: number;
+    readonly eventsSkipped: number;
+    readonly lastProcessedEventId: bigint;
+    readonly durationMs: number;
+    readonly columnDriftCount: number;
+};
+type RebuildDeps = {
+    readonly db: DbConnection;
+    readonly registry: Registry;
+    readonly meter?: Meter;
+    readonly onMetrics?: (result: RebuildResult) => void;
+    readonly signal?: AbortSignal;
+    readonly fenceLockTimeoutMs?: number;
+    readonly errorPolicy?: {
+        readonly skipApplyErrors?: boolean;
+    };
+    readonly __test_onBeforeFence?: () => void | Promise<void>;
+    readonly __test_onBuildShadowTable?: () => void;
+};
+export declare function rebuildProjection(projectionName: string, deps: RebuildDeps): Promise<RebuildResult>;
+export declare function getProjectionState(db: DbConnection, projectionName: string): Promise<{
+    readonly name: string;
+    readonly status: string;
+    readonly lastProcessedEventId: bigint;
+    readonly lastRebuildAt: Temporal.Instant | null;
+    readonly lastError: string | null;
+    readonly updatedAt: Temporal.Instant;
+} | null>;
+export declare function listProjectionsWithState(db: DbConnection, registry: Registry, options?: {
+    readonly includeImplicit?: boolean;
+}): Promise<ReadonlyArray<{
+    readonly name: string;
+    readonly sources: readonly string[];
+    readonly status: string;
+    readonly lastProcessedEventId: bigint;
+    readonly lastRebuildAt: Temporal.Instant | null;
+    readonly lastError: string | null;
+}>>;
+export type ProjectionProgress = {
+    readonly name: string;
+    readonly sources: readonly string[];
+    readonly status: string;
+    readonly lastProcessedEventId: bigint;
+    readonly lastRebuildAt: Temporal.Instant | null;
+    readonly lastError: string | null;
+    readonly highWaterMark: bigint;
+    readonly lag: bigint;
+};
+export declare function getAllProjectionProgress(db: DbConnection, registry: Registry): Promise<readonly ProjectionProgress[]>;
+export {};