@cosmicdrift/kumiko-framework 0.328.0 → 0.329.0

package/dist/db/queries/shadow-swap.js

@@ -0,0 +1,307 @@
+// Online projection rebuild via a transient shadow schema.
+//
+// Why a shadow SCHEMA and not a `read_foo__rebuild` table: apply(event, tx)
+// writes through the projection's canonical table object → an unqualified
+// `read_foo`. A captured table reference can't be re-pointed, so we redirect
+// NAME RESOLUTION instead of the write: build the shadow under the SAME name
+// in a private schema, point `search_path` there for the rebuild tx, and apply
+// lands in the shadow untouched. The live `read_foo` keeps serving reads and
+// writes for the whole replay — only the final swap takes a brief ACCESS
+// EXCLUSIVE lock, instead of holding it for the entire replay like an
+// in-place TRUNCATE + replay would.
+//
+// The shadow-schema choice also dissolves the index-rename problem: indexes
+// built in the shadow carry their canonical `read_foo_*` names and move intact
+// when the table is moved to public via SET SCHEMA.
+//
+// Boundary: the shadow table is rebuilt from EntityTableMeta, so any index NOT
+// expressed in meta (hand-added in a migration) is not reconstructed, and a
+// partial index whose WHERE the renderer can't express is rejected up-front.
+import { configuredBlindIndexKey } from "../../crypto/index.js";
+import { asEntityTableMeta, asRawClient } from "../query.js";
+import { renderTableDdl } from "../render-ddl.js";
+import { columnNamesOf, tableExists } from "../schema-inspection.js";
+import { quoteTableIdent } from "./table-ops.js";
+export const PROJECTION_REBUILD_SCHEMA = "kumiko_rebuild";
+const SCHEMA_IDENT = quoteTableIdent(PROJECTION_REBUILD_SCHEMA);
+function isDuplicateSchemaError(e) {
+    if (typeof e !== "object" || e === null || !("code" in e))
+        return false;
+    const { code } = e;
+    // 42P06 duplicate_schema, 23505 unique_violation on pg_namespace — both
+    // surface from the well-known CREATE SCHEMA IF NOT EXISTS race.
+    return code === "42P06" || code === "23505";
+}
+// Idempotent. MUST run OUTSIDE the rebuild tx: CREATE SCHEMA IF NOT EXISTS is
+// not race-free (two concurrent rebuilds of DIFFERENT projections can collide
+// on pg_namespace), and a collision inside the rebuild tx would roll the whole
+// replay back. The dup race is swallowed; anything else (e.g. a role without
+// CREATE privilege) rethrows so ops sees it loud.
+export async function ensureRebuildSchema(db) {
+    try {
+        await asRawClient(db).unsafe(`CREATE SCHEMA IF NOT EXISTS ${SCHEMA_IDENT}`);
+    }
+    catch (e) {
+        if (!isDuplicateSchemaError(e))
+            throw e;
+    }
+}
+// Resolve the canonical EntityTableMeta a projection's table object carries,
+// or throw. Online rebuild needs the full column+index shape to build the
+// shadow table; a meta-inexpressible partial index would be silently dropped
+// by the renderer, so reject it up-front instead of swapping in a table that
+// is missing an index.
+export function rebuildMetaOrThrow(table, projectionName) {
+    const meta = asEntityTableMeta(table);
+    if (!meta) {
+        throw new Error(`Projection "${projectionName}" has no resolvable EntityTableMeta — online rebuild needs it to build the shadow table.`);
+    }
+    if (meta.indexes.some((idx) => idx.needsManualWhere === true)) {
+        throw new Error(`Projection "${projectionName}" has a partial index whose WHERE clause the schema renderer can't express (drizzle sql\`…\`). Online rebuild reconstructs the table from meta and would silently drop that index. Make the WHERE renderable or rebuild this projection offline.`);
+    }
+    return meta;
+}
+// Fence against a rebuild running with a registry that does not match the
+// migrated live schema (#835): during a rolling deploy, a pod still running
+// the previous build can pick up an async rebuild job; its shadow — built from
+// the stale EntityTableMeta — would swap away a freshly-migrated column
+// (recurrence class of #494). Compares COLUMN NAMES only; a type-/nullability-
+// only drift passes (schema regression, not data loss — the boot gate of the
+// next deploy catches it). A missing live table is fine: nothing to wipe.
+// Must run BEFORE buildShadowTable; columnNamesOf pins table_schema='public',
+// so the shadow search_path could not redirect it anyway.
+export async function assertLiveColumnsMatchMeta(db, meta, projectionName) {
+    // skip: no live table yet — nothing a stale-meta shadow could wipe
+    if (!(await tableExists(db, `public.${meta.tableName}`)))
+        return;
+    const live = await columnNamesOf(db, meta.tableName);
+    const metaNames = new Set(meta.columns.map((c) => c.name));
+    const onlyLive = [...live].filter((c) => !metaNames.has(c));
+    const onlyMeta = [...metaNames].filter((c) => !live.has(c));
+    // skip: column sets match — this process's registry is in sync with the migrated table
+    if (onlyLive.length === 0 && onlyMeta.length === 0)
+        return;
+    const detail = [
+        onlyLive.length > 0 ? `live-only: ${onlyLive.join(", ")}` : "",
+        onlyMeta.length > 0 ? `meta-only: ${onlyMeta.join(", ")}` : "",
+    ]
+        .filter(Boolean)
+        .join("; ");
+    throw new Error(`projection-rebuild "${projectionName}": columns of live table "${meta.tableName}" do not match this process's EntityTableMeta (${detail}). ` +
+        "Rebuilding would swap away the difference. Likely cause: this pod runs a build whose registry is behind (or ahead of) the applied migrations — rolling deploy in progress? — or DDL was applied by hand. " +
+        "Rebuild aborted; retry from a pod whose code matches the migrated schema.");
+}
+// EntityTableMeta carries no RLS/policies, so the swap would silently drop them (#2907).
+export async function assertLiveTableHasNoRowLevelSecurity(tx, tableName, projectionName) {
+    const rows = await asRawClient(tx).unsafe(`SELECT c.relrowsecurity AS rls_enabled, c.relforcerowsecurity AS rls_forced,
+       (SELECT count(*)::int FROM pg_policy p WHERE p.polrelid = c.oid) AS policy_count
+     FROM pg_class c
+     JOIN pg_namespace n ON n.oid = c.relnamespace
+     WHERE n.nspname = 'public' AND c.relname = $1`, [tableName]);
+    const row = rows[0];
+    // skip: no live table yet — nothing to have RLS on
+    if (!row)
+        return;
+    // skip: no RLS flags and no policies — nothing the swap could drop
+    if (!row.rls_enabled && !row.rls_forced && row.policy_count === 0)
+        return;
+    const context = projectionName !== undefined ? `projection-rebuild "${projectionName}"` : "shadow swap";
+    throw new Error(`${context}: live table "${tableName}" has row level security (enabled: ${row.rls_enabled}, ` +
+        `forced: ${row.rls_forced}, policies: ${row.policy_count}). The shadow swap rebuilds the table ` +
+        "from EntityTableMeta and would silently drop RLS and every policy. Kumiko does not support RLS " +
+        "on rebuildable tables — see kumiko-platform docs/plans/rls-evaluation.md. Rebuild aborted; live " +
+        "table untouched.");
+}
+// Runs INSIDE the rebuild tx, AFTER the state/consumer row lock is taken.
+// Points search_path at the shadow schema (SET LOCAL → auto-reset on commit or
+// rollback), drops any leftover shadow from a crashed run, then builds the
+// shadow table + indexes under their canonical names. renderTableDdl output and
+// apply-writes are unqualified and resolve into the shadow; kumiko_events /
+// kumiko_projections live only in public and fall through to it.
+//
+// Boundary: an apply that writes to a table OTHER than its own projection (e.g.
+// an MSP saga touching a second read-model) writes UNQUALIFIED → resolves to
+// public, i.e. the live secondary table, not a shadow of it. Online rebuild is
+// safe for self-table-only apply (the common case); a multi-table apply would
+// mutate live state during replay.
+export async function buildShadowTable(tx, meta) {
+    const raw = asRawClient(tx);
+    await raw.unsafe(`SET LOCAL search_path TO ${SCHEMA_IDENT}, public`);
+    await raw.unsafe(`DROP TABLE IF EXISTS ${SCHEMA_IDENT}.${quoteTableIdent(meta.tableName)}`);
+    for (const stmt of renderTableDdl(meta)) {
+        await raw.unsafe(stmt);
+    }
+}
+// Fence the live table before the cutover: take ACCESS EXCLUSIVE on
+// public.<tableName> so no concurrent synchronous projection apply (a command
+// handler's append+apply) can commit a new event-derived row past the rebuild's
+// final catch-up. Schema-qualified so the active shadow search_path can't
+// redirect it. lock_timeout (SET LOCAL → auto-reset) bounds the wait: under a
+// pathological long-running writer the fence fails loud and the rebuild rolls
+// back rather than hanging indefinitely.
+export async function fenceLiveTable(tx, tableName, lockTimeoutMs) {
+    // Postgres treats lock_timeout = 0 as "no timeout" (wait forever) — the
+    // opposite of fail-fast. Reject it so a 0/negative value can't silently
+    // turn the fence into an unbounded wait.
+    if (Math.trunc(lockTimeoutMs) <= 0) {
+        throw new Error(`fenceLockTimeoutMs must be > 0, got ${lockTimeoutMs}`);
+    }
+    const raw = asRawClient(tx);
+    await raw.unsafe(`SET LOCAL lock_timeout = ${Math.trunc(lockTimeoutMs)}`);
+    await raw.unsafe(`LOCK TABLE public.${quoteTableIdent(tableName)} IN ACCESS EXCLUSIVE MODE`);
+}
+// Ids reported when the swap is aborted — enough to locate the ghost rows
+// without dumping an unbounded set into the log.
+const UNREACHABLE_SAMPLE_LIMIT = 20;
+// Runs INSIDE the rebuild tx, under the fence, before swapShadowIntoLive. A live
+// row whose aggregate id has NO event in the projection's source streams is
+// UNREACHABLE: no replay can ever reconstruct it, so the swap would silently
+// drop it. That is the #498 ghost — a row direct-inserted without ever emitting
+// a .created event. The static CI guard cannot see it in data that already
+// exists in production, or on table identifiers it couldn't resolve; this
+// catches it at cutover and aborts (tx rolls back, live untouched).
+//
+// Deliberately NARROW — event EXISTENCE only, not a column or row-vs-shadow
+// diff. The framework legitimately makes live diverge from a fresh replay in
+// several SHIPPED ways, none of which is drift:
+//   - a blind-index column recomputed to NULL after the subject's key is
+//     shredded (GDPR erase) — the NULL is the intended end state;
+//   - a `sensitive` column stripped from the event log by design;
+//   - an archived stream that stops replaying (fw#832) — the row's wipe is the
+//     intended tombstone behavior, reported via backfill's `failed` list;
+//   - a legacy column direct-written before its handler emitted events, healed
+//     by the #494 backfill-then-rebuild flow.
+// Checking event existence INCLUDING archived streams leaves every one of them
+// alone: those rows all have a real event, so they are not ghosts. Column-level
+// drift is a SEPARATE, non-blocking check — see countColumnDrift below (#916,
+// resolves the #722 open question: observe, don't block).
+//
+// Implicit projections only (caller-gated). aggregate_id and the entity id are
+// both uuid, so the anti-join probes the events index without a cast.
+export async function assertNoUnreachableLiveRows(tx, projectionName, tableName, aggregateTypes) {
+    // skip: no source streams → no events could back any row anyway; a rebuild
+    // of a subscription-less projection swaps an empty shadow (handled upstream).
+    if (aggregateTypes.length === 0)
+        return;
+    const raw = asRawClient(tx);
+    const t = quoteTableIdent(tableName);
+    const ghosts = await raw.unsafe(`SELECT l."id" FROM public.${t} l
+     WHERE NOT EXISTS (
+       SELECT 1 FROM "kumiko_events" e
+        WHERE e."aggregate_id" = l."id" AND e."aggregate_type" = ANY($1::text[])
+     )
+     LIMIT ${UNREACHABLE_SAMPLE_LIMIT}`, [aggregateTypes]);
+    // skip: every live row has a backing event — nothing unreachable, swap is safe
+    if (ghosts.length === 0)
+        return;
+    const ids = ghosts.map((r) => String(r.id));
+    const countLabel = ids.length === UNREACHABLE_SAMPLE_LIMIT ? `${ids.length}+` : String(ids.length);
+    throw new Error(`projection-rebuild "${projectionName}": ${countLabel} live rows in "${tableName}" have no ` +
+        `event in the projection's source streams and cannot be reconstructed by replay — the swap ` +
+        `would silently drop them (ids: ${ids.join(", ")}). A handler direct-inserted these rows ` +
+        `without emitting a .created event. Fix: register the table with r.storeTable(meta, ` +
+        `{ reason }) to opt out of rebuild, or emit the missing events. See ` +
+        `docs/reference/entity-write-patterns.md. Rebuild aborted; live table untouched.`);
+}
+// The bidx column is schema-driven, not key-driven: it exists NULL in a
+// plaintext install and in the fw#1610 case (KMS configured, no index key),
+// both of which are correct as-is. Only a POPULATED column with no key
+// configured in THIS process means the rebuild is about to overwrite proof
+// of a real index with NULL — that's the one provable data-loss case.
+export async function assertNoBlindIndexLoss(tx, tableName, meta, projectionName) {
+    // skip: a key is configured — the replay recomputes every bidx column with it
+    if (configuredBlindIndexKey() !== undefined)
+        return;
+    const bidxCols = meta.columns.filter((c) => c.name.endsWith("_bidx"));
+    // skip: no blind-index column on this table — nothing the rebuild could lose
+    if (bidxCols.length === 0)
+        return;
+    const t = quoteTableIdent(tableName);
+    const raw = asRawClient(tx);
+    const where = bidxCols.map((c) => `${quoteTableIdent(c.name)} IS NOT NULL`).join(" OR ");
+    const rows = await raw.unsafe(`SELECT count(*)::text AS total FROM public.${t} WHERE ${where}`);
+    const count = Number(rows[0]?.total ?? "0");
+    // skip: every bidx column is already NULL — nothing for the rebuild to lose
+    if (count === 0)
+        return;
+    throw new Error(`projection-rebuild "${projectionName}": "${tableName}" has ${count} row(s) with a populated ` +
+        `blind-index column, but KUMIKO_BLIND_INDEX_KEY is not configured in this process. The rebuild ` +
+        `would recompute those columns to NULL, and equality lookups on that field (login, password ` +
+        `reset) would stop matching afterward. Configure the blind-index key before this apply/rebuild ` +
+        `runs. See fw#3091. Rebuild aborted; live table untouched.`);
+}
+// Columns ignored by countColumnDrift — the one PROVABLY legitimate class of
+// live-vs-shadow divergence. A blind-index column (`<field>_bidx`) is
+// recomputed to NULL on GDPR key-shredding; the NULL is the intended end
+// state, not drift. Everything else that legitimately diverges (archived
+// streams, #494 backfill) either never reaches this comparison (archived rows
+// are absent from the shadow entirely, see swapShadowIntoLive) or IS real
+// column drift that the #494 backfill-then-rebuild flow relies on replay to
+// heal — reporting it (without blocking) is correct, not a false positive.
+const COLUMN_DRIFT_SAMPLE_LIMIT = 20;
+// Runs INSIDE the rebuild tx, in the same slot as assertNoUnreachableLiveRows
+// (after replay settles, before swapShadowIntoLive). Non-blocking counterpart
+// to the ghost-row guard: reports live rows whose column values differ from
+// the freshly-replayed shadow, WITHOUT aborting the swap (#916, resolves the
+// #722 open question in favor of observe-not-block).
+//
+// Why non-blocking: a legacy column direct-written before its handler emitted
+// events (#494) diverges from replay by design — that divergence is exactly
+// what the backfill-then-rebuild flow relies on replay to heal. Failing hard
+// here would make rebuild mutually exclusive with that shipped healing path.
+// There is no reliable metadata to distinguish "#494 healing in progress" from
+// "someone else corrupted this row" short of an open-ended per-column policy
+// blocklist — wrong-by-default whenever a class is missed. So: surface it,
+// don't police it. The caller logs the result; ops decides.
+//
+// Caveat: sensitive CUSTOM fields still diverge until #972 (Subject-DEK
+// design) — regular sensitive fields carry event-payload ciphertext parity
+// post-#973 and don't drift. Both are reported the same as any other column
+// drift; this is deliberate (see module comment above), not an oversight.
+//
+// Relies on assertLiveColumnsMatchMeta having already run: live/shadow/meta
+// column sets are known to match, so the diff can walk meta.columns directly.
+export async function countColumnDrift(tx, tableName, meta) {
+    const comparable = meta.columns.filter((c) => c.primaryKey !== true && !c.name.endsWith("_bidx"));
+    // skip: nothing to compare (id-only or all-bidx table) — no drift is possible
+    if (comparable.length === 0)
+        return { rowCount: 0, sample: [] };
+    const t = quoteTableIdent(tableName);
+    const raw = asRawClient(tx);
+    const driftCte = `WITH drifted AS (
+     SELECT l."id" AS id, string_agg(diff.col, ',') AS drifted_columns
+     FROM public.${t} l
+     JOIN ${SCHEMA_IDENT}.${t} s ON s."id" = l."id"
+     CROSS JOIN LATERAL (
+       VALUES ${comparable
+        .map((c) => `('${c.name}', l.${quoteTableIdent(c.name)} IS DISTINCT FROM s.${quoteTableIdent(c.name)})`)
+        .join(", ")}
+     ) AS diff(col, differs)
+     WHERE diff.differs
+     GROUP BY l."id"
+   )`;
+    // Two passes over `drifted`: an unbounded COUNT for the true total (replay
+    // already scanned every row this run, so a second scan here is cheap by
+    // comparison) plus a capped sample for the log. rowCount must NEVER be
+    // min(actual, LIMIT) — that would silently understate severity to ops.
+    const totalRows = await raw.unsafe(`${driftCte} SELECT count(*)::text AS total FROM drifted`);
+    const rowCount = Number(totalRows[0]?.total ?? "0");
+    // skip: no drift — nothing to sample
+    if (rowCount === 0)
+        return { rowCount: 0, sample: [] };
+    const rows = await raw.unsafe(`${driftCte} SELECT id, drifted_columns FROM drifted LIMIT ${COLUMN_DRIFT_SAMPLE_LIMIT}`);
+    const sample = rows.flatMap((r) => r.drifted_columns.split(",").map((col) => `${String(r.id)}.${col}`));
+    return { rowCount, sample };
+}
+// Atomic swap, INSIDE the rebuild tx, AFTER replay. Schema-qualified so the
+// active shadow search_path can't redirect them. DROP without CASCADE: if any
+// object depends on the live table the swap fails loud and the whole rebuild
+// rolls back, leaving the old table untouched.
+export async function swapShadowIntoLive(tx, tableName) {
+    // Re-checked right before DROP: RLS could have been enabled after the early check.
+    await assertLiveTableHasNoRowLevelSecurity(tx, tableName);
+    const raw = asRawClient(tx);
+    const ident = quoteTableIdent(tableName);
+    await raw.unsafe(`DROP TABLE public.${ident}`);
+    await raw.unsafe(`ALTER TABLE ${SCHEMA_IDENT}.${ident} SET SCHEMA public`);
+}

package/dist/db/queries/table-ops.d.ts

@@ -0,0 +1,4 @@
+import type { AnyDb } from "../query.js";
+/** Double-quote + escape — Postgres identifier rules. */
+export declare function quoteTableIdent(name: string): string;
+export declare function truncateTable(db: AnyDb, tableName: string): Promise<void>;

package/dist/db/queries/table-ops.js

@@ -0,0 +1,8 @@
+import { asRawClient } from "../query.js";
+/** Double-quote + escape — Postgres identifier rules. */
+export function quoteTableIdent(name) {
+    return `"${name.replace(/"/g, '""')}"`;
+}
+export async function truncateTable(db, tableName) {
+    await asRawClient(db).unsafe(`TRUNCATE TABLE ${quoteTableIdent(tableName)}`);
+}

package/dist/db/queries/test-stack.d.ts

@@ -0,0 +1,6 @@
+import type { AnyDb } from "../query.js";
+export { alterTableAddColumn, createIndexIfNotExists, executeDdlStatement } from "./ddl.js";
+export declare function truncateTablesRestartIdentity(db: AnyDb, tableNames: readonly string[]): Promise<void>;
+export declare function databaseExists(db: AnyDb, dbName: string): Promise<boolean>;
+export declare function createDatabase(db: AnyDb, dbName: string): Promise<void>;
+export declare function dropDatabaseIfExists(db: AnyDb, dbName: string): Promise<void>;

package/dist/db/queries/test-stack.js

@@ -0,0 +1,20 @@
+import { asRawClient } from "../query.js";
+import { quoteTableIdent } from "./table-ops.js";
+// Re-exported for back-compat — the generic DDL helpers moved to ./ddl so
+// the prod-boot path (stack/table-helpers.ts, pipeline/event-consumer-state.ts)
+// doesn't import from a module named for test-only concerns.
+export { alterTableAddColumn, createIndexIfNotExists, executeDdlStatement } from "./ddl.js";
+export async function truncateTablesRestartIdentity(db, tableNames) {
+    const quoted = tableNames.map((name) => quoteTableIdent(name)).join(", ");
+    await asRawClient(db).unsafe(`TRUNCATE ${quoted} RESTART IDENTITY CASCADE`);
+}
+export async function databaseExists(db, dbName) {
+    const rows = (await asRawClient(db).unsafe(`SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1) AS exists`, [dbName]));
+    return rows[0]?.exists === true;
+}
+export async function createDatabase(db, dbName) {
+    await asRawClient(db).unsafe(`CREATE DATABASE ${quoteTableIdent(dbName)}`);
+}
+export async function dropDatabaseIfExists(db, dbName) {
+    await asRawClient(db).unsafe(`DROP DATABASE IF EXISTS ${quoteTableIdent(dbName)}`);
+}

package/dist/db/query-api.d.ts

@@ -0,0 +1,2 @@
+export type { SelectOptions, WhereObject, WhereOperator, WhereValue, } from "../db/query.js";
+export { asRawClient, countWhere, deleteMany, fetchOne, insertMany, insertOne, runInSavepoint, selectMany, transaction, updateMany, } from "../db/query.js";

package/dist/db/query-api.js

@@ -0,0 +1,6 @@
+// Legacy re-export shim — query-api.ts hat früher drizzle gewrapped.
+// Heute liegen die Helpers in src/bun-db/. Wir re-exportieren von dort
+// damit existing imports `@cosmicdrift/kumiko-framework/db` weiterhin
+// funktionieren während wir die Callers schrittweise auf den direkten
+// bun-db-Import migrieren.
+export { asRawClient, countWhere, deleteMany, fetchOne, insertMany, insertOne, runInSavepoint, selectMany, transaction, updateMany, } from "../db/query.js";

package/dist/db/query.d.ts

@@ -0,0 +1 @@
+export { type AnyDb, asEntityTableMeta, asRawClient, coerceRow, countWhere, type DeleteManyBatchedOptions, type DeleteManyBatchedResult, deleteMany, deleteManyBatched, extractTableInfo, fetchOne, type IncrementCounterOptions, incrementCounter, insertMany, insertOne, type OrderByClause, runInNewTransaction, runInSavepoint, runInSavepointIfSupported, type SelectOptions, selectMany, type TableInfo, transaction, type UpsertOnConflictOptions, unsafeReadRetrying, updateMany, upsertByPk, upsertOnConflict, type WhereObject, type WhereOperator, type WhereValue, } from "../bun-db/query.js";

package/dist/db/query.js

@@ -0,0 +1,3 @@
+// Provider-neutrale Query-API. Re-exported aus bun-db/query.
+// Alle Consumer importieren von hier, nicht direkt aus bun-db/.
+export { asEntityTableMeta, asRawClient, coerceRow, countWhere, deleteMany, deleteManyBatched, extractTableInfo, fetchOne, incrementCounter, insertMany, insertOne, runInNewTransaction, runInSavepoint, runInSavepointIfSupported, selectMany, transaction, unsafeReadRetrying, updateMany, upsertByPk, upsertOnConflict, } from "../bun-db/query.js";

package/dist/db/rebuild-marker.d.ts

@@ -0,0 +1,4 @@
+import { type SchemaDiff } from "./migrate-generator.js";
+export declare function rebuildTablesFromDiff(diff: SchemaDiff): readonly string[];
+export declare function writeRebuildMarker(migrationsDir: string, sqlFilename: string, tables: readonly string[]): void;
+export declare function readRebuildMarker(migrationsDir: string, migrationId: string): readonly string[];

package/dist/db/rebuild-marker.js

@@ -0,0 +1,96 @@
+// Rebuild-Marker für den drizzle-freien Migrations-Pfad.
+//
+// `kumiko schema generate` schreibt neben jedes `NNNN_<name>.sql` einen
+// Sibling-Marker `NNNN_<name>.rebuild.json`, der die in dieser Migration
+// geänderten/neu angelegten Tabellen listet. `kumiko schema apply` liest den
+// Marker für jede frisch applizierte Migration und rebuildet die betroffenen
+// Projektionen.
+//
+// Bewusst nur **Tabellennamen** (kein Projection-Name): der Generator ist
+// registry-frei (kennt die App-Projektionen nicht). Die Auflösung
+// Tabelle→Projection passiert app-seitig beim Apply via
+// `buildProjectionTableIndex(registry)`. Tabellen ohne zugehörige Projektion
+// werden dort einfach übersprungen.
+//
+// Marker werden zum generate-Zeitpunkt aus dem strukturierten `SchemaDiff`
+// geschrieben — nicht beim Apply aus dem (ggf. hand-editierten) SQL
+// re-derived.
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { managedChangeRequiresRecreate } from "./migrate-generator.js";
+const MARKER_VERSION = 1;
+function markerPathFor(migrationsDir, migrationId) {
+    return join(migrationsDir, `${migrationId}.rebuild.json`);
+}
+// Only managed tables (event-stream derivatives) get rebuild markers — unmanaged
+// carry real data, never rebuilt from events; sorted+deduped for stable PR diff.
+//
+// A changed table needs a rebuild ONLY when the generated SQL RECREATES it
+// (managedChangeRequiresRecreate: drop/NOT-NULL-w/o-default/unique-index/type-
+// or nullability-change) — then the table is emptied and must be re-derived from
+// events. A pure additive nullable column is an in-place `ADD COLUMN` ALTER that
+// already brings the table to the target state (same logic #181 applied to
+// index-/default-only changes). Rebuilding it anyway is wasted replay AND — the
+// bug this closes — a co-triggered rebuild can drop the fresh column (rebuild
+// runs from the rebuilding process's registry meta; on a rolling deploy an older
+// pod's meta lacks the column → shadow-swap wipes it → phantom migration + boot-
+// drift crash; see 0008_add_pending_deletion_request_id / #494 / #835).
+//
+// If a NEW additive column genuinely needs value-backfill from historical events
+// (a column DERIVED from existing event fields), opt in explicitly by hand-adding
+// a `NNNN_<name>.rebuild.json` next to the migration (readRebuildMarker reads it).
+//
+// One built-in exception to the additive-column rule: a new `_bidx` column
+// (blind-index for a field that just became `lookupable: true`, naming
+// convention `${snake}_bidx` from entity-table-meta.ts:111) is always
+// additive SQL-wise but still needs a rebuild — the index is an HMAC over
+// the decrypted value, which SQL cannot backfill.
+export function rebuildTablesFromDiff(diff) {
+    const names = new Set();
+    for (const t of diff.changedTables) {
+        if (t.nextMeta.source !== "managed")
+            continue;
+        if (managedChangeRequiresRecreate(t) || t.newColumns.some((c) => c.name.endsWith("_bidx"))) {
+            names.add(t.tableName);
+        }
+    }
+    for (const t of diff.newTables) {
+        if (t.source === "managed")
+            names.add(t.tableName);
+    }
+    return [...names].sort();
+}
+// Schreibt `<sqlFilename ohne .sql>.rebuild.json`. Leere Tabellen-Liste →
+// kein Marker (z.B. reine Drop-Migration).
+export function writeRebuildMarker(migrationsDir, sqlFilename, tables) {
+    // skip: leere Tabellen-Liste → kein Marker (z.B. reine Drop-Migration).
+    if (tables.length === 0)
+        return;
+    const migrationId = sqlFilename.replace(/\.sql$/, "");
+    const marker = { version: MARKER_VERSION, tables };
+    writeFileSync(markerPathFor(migrationsDir, migrationId), `${JSON.stringify(marker, null, 2)}\n`);
+}
+// Liest die Tabellen-Liste für eine applizierte Migration. Kein Marker /
+// kaputtes File → leere Liste (Migration ohne Projection-Impact).
+export function readRebuildMarker(migrationsDir, migrationId) {
+    const path = markerPathFor(migrationsDir, migrationId);
+    if (!existsSync(path))
+        return [];
+    let parsed;
+    try {
+        parsed = JSON.parse(readFileSync(path, "utf8"));
+    }
+    catch {
+        return [];
+    }
+    if (typeof parsed !== "object" || parsed === null)
+        return [];
+    if (!("version" in parsed) || parsed.version !== MARKER_VERSION)
+        return [];
+    if (!("tables" in parsed))
+        return [];
+    const { tables } = parsed;
+    if (!Array.isArray(tables))
+        return [];
+    return tables.filter((t) => typeof t === "string");
+}

package/dist/db/reference-data.d.ts

@@ -0,0 +1,14 @@
+import type { ReferenceDataDef } from "../engine/types/index.js";
+import type { DbConnection } from "./connection.js";
+import type { TableColumns } from "./dialect.js";
+type Table = TableColumns<any>;
+/**
+ * Seed reference data at boot time.
+ * For each ReferenceDataDef: upsert rows (insert missing, update changed, never delete).
+ * Upsert key defaults to the first field in the data object.
+ */
+export declare function seedReferenceData(defs: readonly ReferenceDataDef[], tables: ReadonlyMap<string, Table>, db: DbConnection): Promise<{
+    inserted: number;
+    updated: number;
+}>;
+export {};

package/dist/db/reference-data.js

@@ -0,0 +1,66 @@
+import { KUMIKO_COLUMNS_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types";
+import { fetchOne, insertOne, updateMany } from "../db/query.js";
+import { SYSTEM_TENANT_ID } from "../engine/types/index.js";
+function hasColumn(table, field) {
+    const cols = table[KUMIKO_COLUMNS_SYMBOL];
+    if (typeof cols !== "object" || cols === null)
+        return false;
+    return field in cols;
+}
+/**
+ * Seed reference data at boot time.
+ * For each ReferenceDataDef: upsert rows (insert missing, update changed, never delete).
+ * Upsert key defaults to the first field in the data object.
+ */
+export async function seedReferenceData(defs, tables, db) {
+    let inserted = 0;
+    let updated = 0;
+    for (const def of defs) {
+        const table = tables.get(def.entityName);
+        if (!table)
+            continue;
+        if (def.data.length === 0)
+            continue;
+        const firstRow = def.data[0];
+        if (!firstRow)
+            continue;
+        const firstKey = Object.keys(firstRow)[0];
+        if (!firstKey)
+            continue;
+        const upsertKey = def.upsertKey ?? firstKey;
+        for (const row of def.data) {
+            const keyValue = row[upsertKey];
+            if (keyValue === undefined)
+                continue;
+            const existing = (await fetchOne(db, table, { [upsertKey]: keyValue }));
+            if (existing) {
+                const changes = {};
+                for (const [field, value] of Object.entries(row)) {
+                    if (field === upsertKey)
+                        continue;
+                    if (existing[field] !== value) {
+                        changes[field] = value;
+                    }
+                }
+                if (Object.keys(changes).length > 0) {
+                    await updateMany(db, table, changes, { [upsertKey]: keyValue });
+                    updated++;
+                }
+            }
+            else {
+                // Only add framework columns if the table actually has them.
+                // Drizzle used to filter extra fields silently; bunInsertOne doesn't.
+                const values = { ...row };
+                if (hasColumn(table, "tenantId"))
+                    values["tenantId"] = SYSTEM_TENANT_ID;
+                if (hasColumn(table, "version"))
+                    values["version"] = 1;
+                if (hasColumn(table, "insertedAt"))
+                    values["insertedAt"] = Temporal.Now.instant();
+                await insertOne(db, table, values);
+                inserted++;
+            }
+        }
+    }
+    return { inserted, updated };
+}

package/dist/db/render-ddl.d.ts

@@ -0,0 +1,4 @@
+import type { EntityTableMeta, IndexMeta } from "./entity-table-meta.js";
+export declare function renderIndex(tableName: string, idx: IndexMeta): string;
+export declare function renderTableDdl(meta: EntityTableMeta): readonly string[];
+export declare function renderTablesDdl(metas: readonly EntityTableMeta[]): readonly string[];

package/dist/db/render-ddl.js

@@ -0,0 +1,55 @@
+// Pure renderer: EntityTableMeta → SQL DDL statements.
+// Wird vom Migrate-Generator (Phase 2 — CLI-Tool `kumiko migrate generate`)
+// genutzt um initial-SQL-Files zu schreiben. Output ist Start-Form für
+// User-Review — App-Author darf das SQL danach hand-editieren (extra-Index,
+// partial-Index, BRIN, custom-clauses) bevor committed wird.
+//
+// NO-MAGIC-ON-DATA: dieser Renderer wird NIE zur App-Runtime aufgerufen.
+// Nur Build-Step. Runner liest checked-in SQL, nicht Renderer-Output.
+import { pgTypeToSqlType } from "./dialect.js";
+function quoteIdent(name) {
+    return `"${name.replace(/"/g, '""')}"`;
+}
+function renderColumn(col) {
+    const parts = [quoteIdent(col.name), pgTypeToSqlType(col.pgType)];
+    if (col.identity)
+        parts.push("GENERATED ALWAYS AS IDENTITY");
+    if (col.primaryKey)
+        parts.push("PRIMARY KEY");
+    if (col.defaultSql !== undefined)
+        parts.push(`DEFAULT ${col.defaultSql}`);
+    if (col.notNull && !col.primaryKey)
+        parts.push("NOT NULL");
+    return parts.join(" ");
+}
+export function renderIndex(tableName, idx) {
+    const kind = idx.unique === true ? "UNIQUE INDEX" : "INDEX";
+    const colList = idx.columns.map(quoteIdent).join(", ");
+    if (idx.needsManualWhere === true) {
+        return [
+            `-- WARN: partial-index "${idx.name}" needs a WHERE clause that the`,
+            `--       generator can't render (entity uses drizzle sql\`…\` AST).`,
+            `--       Add the WHERE manually before applying:`,
+            `-- CREATE ${kind} IF NOT EXISTS ${quoteIdent(idx.name)} ON ${quoteIdent(tableName)} (${colList}) WHERE <your-condition>;`,
+        ].join("\n");
+    }
+    const where = idx.whereSql !== undefined ? ` WHERE ${idx.whereSql}` : "";
+    return `CREATE ${kind} IF NOT EXISTS ${quoteIdent(idx.name)} ON ${quoteIdent(tableName)} (${colList})${where};`;
+}
+export function renderTableDdl(meta) {
+    const colLines = meta.columns.map(renderColumn);
+    const lines = [...colLines];
+    if (meta.compositePrimaryKey !== undefined) {
+        const pkCols = meta.compositePrimaryKey.columns.map(quoteIdent).join(",");
+        lines.push(`CONSTRAINT ${quoteIdent(meta.compositePrimaryKey.name)} PRIMARY KEY(${pkCols})`);
+    }
+    const create = `CREATE TABLE IF NOT EXISTS ${quoteIdent(meta.tableName)} (\n  ${lines.join(",\n  ")}\n);`;
+    const indexes = meta.indexes.map((idx) => renderIndex(meta.tableName, idx));
+    return [create, ...indexes];
+}
+export function renderTablesDdl(metas) {
+    const stmts = [];
+    for (const m of metas)
+        stmts.push(...renderTableDdl(m));
+    return stmts;
+}

package/dist/db/replay-migration-sql.d.ts

@@ -0,0 +1,12 @@
+import type { Snapshot } from "./migrate-generator.js";
+export type ReplayedTable = {
+    readonly columns: ReadonlySet<string>;
+};
+export type ReplayedSchema = ReadonlyMap<string, ReplayedTable>;
+export declare function replayMigrationsDir(migrationsDir: string): ReplayedSchema;
+export type ReplayMismatch = {
+    readonly tableName: string;
+    readonly kind: "missing-table" | "unexpected-table" | "column-drift";
+    readonly detail: string;
+};
+export declare function diffReplayAgainstSnapshot(replayed: ReplayedSchema, snapshot: Snapshot): readonly ReplayMismatch[];