@cosmicdrift/kumiko-framework 0.46.0 → 0.48.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.46.0",
+  "version": "0.48.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -181,7 +181,7 @@
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.40.1",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.45.0",
     "@types/uuid": "^11.0.0",
     "bun-types": "^1.3.13",
     "pino-pretty": "^13.1.3"

package/src/db/__tests__/managed-recreate.integration.test.ts

@@ -0,0 +1,78 @@
+// #356: a generated migration that adds a NOT NULL ride-along column to a
+// MANAGED projection must APPLY on a populated table — where the old additive
+// `ALTER TABLE ADD COLUMN ... NOT NULL` dies on existing rows. The generator
+// emits DROP+CREATE for that case; the rebuild (covered by
+// pending-rebuilds.integration.test.ts) refills from events afterwards.
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createTestDb, type TestDb } from "../../stack";
+import type { ColumnMeta, EntityTableMeta } from "../entity-table-meta";
+import { diffSnapshots, renderMigrationSql, snapshotFromMetas } from "../migrate-generator";
+import { runMigrationsFromDir } from "../migrate-runner";
+import { asRawClient } from "../query";
+
+const ID_COL: ColumnMeta = { name: "id", pgType: "uuid", notNull: true, primaryKey: true };
+const NAME_COL: ColumnMeta = { name: "name", pgType: "text", notNull: true };
+// The studio#58/publicstatus#116 blocker shape: NOT NULL without a default.
+const ENVELOPE_COL: ColumnMeta = { name: "envelope", pgType: "jsonb", notNull: true };
+
+function managedMeta(tableName: string, columns: readonly ColumnMeta[]): EntityTableMeta {
+  return { tableName, source: "managed", indexes: [], columns };
+}
+
+let testDb: TestDb;
+let dir: string;
+
+beforeAll(async () => {
+  testDb = await createTestDb();
+  dir = mkdtempSync(join(tmpdir(), "managed-recreate-"));
+});
+
+afterAll(async () => {
+  rmSync(dir, { recursive: true, force: true });
+  await testDb.cleanup();
+});
+
+describe("managed projection recreate applies on a populated table (#356)", () => {
+  test("generated DROP+CREATE applies where an in-place ALTER NOT NULL dies", async () => {
+    const raw = asRawClient(testDb.db);
+
+    // A populated managed projection in its OLD shape (id, name).
+    await raw.unsafe(`DROP TABLE IF EXISTS "read_proof"`);
+    await raw.unsafe(`CREATE TABLE "read_proof" ("id" uuid PRIMARY KEY, "name" text NOT NULL)`);
+    await raw.unsafe(
+      `INSERT INTO "read_proof" (id, name) VALUES (gen_random_uuid(), 'a'), (gen_random_uuid(), 'b')`,
+    );
+
+    // Generate the migration: managed table gains `envelope NOT NULL` → DROP+CREATE.
+    // (An in-place ALTER ADD COLUMN ... NOT NULL would die on the two existing
+    // rows — that failure mode is exactly what #356 removes for projections.)
+    const prev = snapshotFromMetas([managedMeta("read_proof", [ID_COL, NAME_COL])]);
+    const next = snapshotFromMetas([managedMeta("read_proof", [ID_COL, NAME_COL, ENVELOPE_COL])]);
+    const sql = renderMigrationSql(diffSnapshots(prev, next), {
+      name: "add_envelope",
+      sequenceNumber: 1,
+    });
+    expect(sql).toContain('DROP TABLE IF EXISTS "read_proof";');
+    expect(sql).not.toContain("ADD COLUMN");
+
+    writeFileSync(join(dir, "0001_add_envelope.sql"), sql);
+    const result = await runMigrationsFromDir(testDb.db, dir);
+    expect(result.applied).toContain("0001_add_envelope");
+
+    // Table is back at the new shape (envelope present) and emptied — the DROP
+    // ran; the queued rebuild refills it from events (proven elsewhere).
+    // SQL-result boundary: shape is known from the SELECT projection.
+    const colRows = (await raw.unsafe(
+      `SELECT column_name FROM information_schema.columns WHERE table_name = 'read_proof'`,
+    )) as readonly { column_name: string }[];
+    expect(colRows.map((r) => r.column_name)).toContain("envelope");
+    const countRows = (await raw.unsafe(
+      `SELECT count(*)::int AS count FROM "read_proof"`,
+    )) as readonly { count: number }[];
+    expect(countRows[0]?.count).toBe(0);
+  });
+});

package/src/db/__tests__/migrate-generator.test.ts

@@ -10,10 +10,11 @@ import {
 function meta(
   tableName: string,
   extraColumn?: EntityTableMeta["columns"][number],
+  source: EntityTableMeta["source"] = "unmanaged",
 ): EntityTableMeta {
   return {
     tableName,
-    source: "unmanaged",
+    source,
     indexes: [],
     columns: [
       { name: "id", pgType: "uuid", notNull: true, primaryKey: true },
@@ -69,3 +70,58 @@ describe("renderMigrationSql / generateMigration", () => {
     expect(out.filename).toBe("0001_init.sql");
   });
 });
+
+describe("renderMigrationSql — managed recreate vs unmanaged in-place", () => {
+  test("managed: NOT NULL column without default → DROP+CREATE, no in-place ADD", () => {
+    const prev = snapshotFromMetas([meta("read_secrets", undefined, "managed")]);
+    const next = snapshotFromMetas([
+      meta("read_secrets", { name: "envelope", pgType: "jsonb", notNull: true }, "managed"),
+    ]);
+    const sql = renderMigrationSql(diffSnapshots(prev, next), {
+      name: "secrets",
+      sequenceNumber: 2,
+    });
+    expect(sql).toContain('DROP TABLE IF EXISTS "read_secrets";');
+    expect(sql).toContain('CREATE TABLE IF NOT EXISTS "read_secrets"');
+    expect(sql).not.toContain("ADD COLUMN");
+  });
+
+  test("managed: column rename (drop + add NOT NULL) → DROP+CREATE with new shape", () => {
+    const prev = snapshotFromMetas([
+      meta("read_a", { name: "old_name", pgType: "text", notNull: true }, "managed"),
+    ]);
+    const next = snapshotFromMetas([
+      meta("read_a", { name: "new_name", pgType: "text", notNull: true }, "managed"),
+    ]);
+    const sql = renderMigrationSql(diffSnapshots(prev, next), {
+      name: "rename",
+      sequenceNumber: 3,
+    });
+    expect(sql).toContain('DROP TABLE IF EXISTS "read_a";');
+    expect(sql).toContain('"new_name"');
+    expect(sql).not.toContain("DROP COLUMN");
+  });
+
+  test("managed: additive nullable column → in-place ADD COLUMN, no recreate", () => {
+    const prev = snapshotFromMetas([meta("read_a", undefined, "managed")]);
+    const next = snapshotFromMetas([
+      meta("read_a", { name: "note", pgType: "text", notNull: false }, "managed"),
+    ]);
+    const sql = renderMigrationSql(diffSnapshots(prev, next), { name: "note", sequenceNumber: 4 });
+    expect(sql).toContain('ALTER TABLE "read_a" ADD COLUMN "note"');
+    expect(sql).not.toContain("DROP TABLE");
+  });
+
+  test("unmanaged: NOT NULL column without default → in-place ADD (real data, never recreated)", () => {
+    const prev = snapshotFromMetas([meta("app_data")]);
+    const next = snapshotFromMetas([
+      meta("app_data", { name: "envelope", pgType: "jsonb", notNull: true }),
+    ]);
+    const sql = renderMigrationSql(diffSnapshots(prev, next), {
+      name: "appdata",
+      sequenceNumber: 5,
+    });
+    expect(sql).toContain('ALTER TABLE "app_data" ADD COLUMN "envelope"');
+    expect(sql).not.toContain("DROP TABLE");
+  });
+});

package/src/db/__tests__/rebuild-marker.test.ts

@@ -10,10 +10,11 @@ function meta(
   tableName: string,
   extraColumn?: EntityTableMeta["columns"][number],
   indexes: EntityTableMeta["indexes"] = [],
+  source: EntityTableMeta["source"] = "managed",
 ): EntityTableMeta {
   return {
     tableName,
-    source: "unmanaged",
+    source,
     indexes,
     columns: [
       { name: "id", pgType: "uuid", notNull: true, primaryKey: true },
@@ -27,10 +28,10 @@ function tmpDir(): string {
 }
 
 describe("rebuildTablesFromDiff", () => {
-  test("includes new + changed tables, excludes dropped, sorted + unique", () => {
+  test("managed: new + changed tables included, dropped excluded, sorted + unique", () => {
     const prev = snapshotFromMetas([meta("read_a"), meta("read_c")]);
     const next = snapshotFromMetas([
-      meta("read_a", { name: "title", pgType: "text", notNull: true }),
+      meta("read_a", { name: "title", pgType: "text", notNull: false }),
       meta("read_b"),
     ]);
     const diff = diffSnapshots(prev, next);
@@ -42,7 +43,7 @@ describe("rebuildTablesFromDiff", () => {
     expect(rebuildTablesFromDiff(diffSnapshots(snap, snap))).toEqual([]);
   });
 
-  test("index-only change → no rebuild (ALTER bringt Tabelle alleine in Soll)", () => {
+  test("managed non-unique index-only change → no rebuild (ALTER bringt Tabelle alleine in Soll)", () => {
     const prev = snapshotFromMetas([meta("read_a")]);
     const next = snapshotFromMetas([
       meta("read_a", undefined, [{ name: "read_a_id_idx", columns: ["id"] }]),
@@ -50,21 +51,48 @@ describe("rebuildTablesFromDiff", () => {
     expect(rebuildTablesFromDiff(diffSnapshots(prev, next))).toEqual([]);
   });
 
-  test("dropped-column-only change → no rebuild", () => {
+  test("managed new nullable column → rebuild (Backfill aus Events nötig)", () => {
+    const prev = snapshotFromMetas([meta("read_a")]);
+    const next = snapshotFromMetas([
+      meta("read_a", { name: "title", pgType: "text", notNull: false }),
+    ]);
+    expect(rebuildTablesFromDiff(diffSnapshots(prev, next))).toEqual(["read_a"]);
+  });
+
+  test("managed dropped column → rebuild (Tabelle wird DROP+CREATE'd, neu füllen)", () => {
     const prev = snapshotFromMetas([
       meta("read_a", { name: "old", pgType: "text", notNull: false }),
     ]);
     const next = snapshotFromMetas([meta("read_a")]);
-    expect(rebuildTablesFromDiff(diffSnapshots(prev, next))).toEqual([]);
+    expect(rebuildTablesFromDiff(diffSnapshots(prev, next))).toEqual(["read_a"]);
+  });
+
+  test("managed SET NOT NULL → rebuild (recreate)", () => {
+    const prev = snapshotFromMetas([
+      meta("read_a", { name: "title", pgType: "text", notNull: false }),
+    ]);
+    const next = snapshotFromMetas([
+      meta("read_a", { name: "title", pgType: "text", notNull: true }),
+    ]);
+    expect(rebuildTablesFromDiff(diffSnapshots(prev, next))).toEqual(["read_a"]);
   });
 
-  test("new-column change → rebuild (Backfill aus Events nötig)", () => {
+  test("managed new UNIQUE index → rebuild (recreate, könnte an Dups scheitern)", () => {
     const prev = snapshotFromMetas([meta("read_a")]);
     const next = snapshotFromMetas([
-      meta("read_a", { name: "title", pgType: "text", notNull: false }),
+      meta("read_a", undefined, [{ name: "read_a_id_uq", columns: ["id"], unique: true }]),
     ]);
     expect(rebuildTablesFromDiff(diffSnapshots(prev, next))).toEqual(["read_a"]);
   });
+
+  test("unmanaged tables are never rebuilt (echte Daten, keine Projektion)", () => {
+    const prev = snapshotFromMetas([meta("u_a", undefined, [], "unmanaged")]);
+    const next = snapshotFromMetas([
+      meta("u_a", { name: "title", pgType: "text", notNull: true }, [], "unmanaged"),
+      meta("u_new", undefined, [], "unmanaged"),
+    ]);
+    expect(rebuildTablesFromDiff(diffSnapshots(prev, next))).toEqual([]);
+  });
 });
 
 describe("write/read marker", () => {

package/src/db/migrate-generator.ts

@@ -42,6 +42,10 @@ export type TableDiff = {
   readonly changedColumns: readonly ColumnChange[];
   readonly newIndexes: readonly IndexMeta[];
   readonly droppedIndexes: readonly string[];
+  // Full target meta — carried so the renderer can emit DROP+CREATE for a
+  // managed projection whose change cannot apply in-place (see
+  // managedChangeRequiresRecreate). Source-discriminator reached via nextMeta.source.
+  readonly nextMeta: EntityTableMeta;
 };
 
 export type SchemaDiff = {
@@ -161,6 +165,7 @@ function diffOneTable(prev: EntityTableMeta, next: EntityTableMeta): TableDiff |
     changedColumns,
     newIndexes,
     droppedIndexes,
+    nextMeta: next,
   };
 }
 
@@ -192,6 +197,23 @@ export function diffSnapshots(prev: Snapshot | null, next: Snapshot): SchemaDiff
   return { newTables, droppedTables, changedTables };
 }
 
+// A managed projection is a disposable derivative of the event stream. When a
+// schema change cannot apply in-place against existing rows — NOT NULL without
+// default, a UNIQUE index (may hit duplicates), SET NOT NULL, a type change, or
+// a dropped column (incl. the drop-half of a rename) — the additive ALTER would
+// die on the very rows the queued rebuild discards anyway. Such a change is
+// rendered as DROP+CREATE and refilled from events instead. Purely additive,
+// in-place-safe changes (nullable/defaulted ADD, non-unique index, DROP NOT
+// NULL, default-only) stay as cheap ALTERs with no forced replay.
+export function managedChangeRequiresRecreate(td: TableDiff): boolean {
+  if (td.droppedColumns.length > 0) return true;
+  if (td.newColumns.some((c) => c.notNull && c.defaultSql === undefined)) return true;
+  if (td.newIndexes.some((idx) => idx.unique === true)) return true;
+  return td.changedColumns.some(
+    (c) => c.nullabilityChanged?.to === true || c.typeChanged !== undefined,
+  );
+}
+
 // --- SQL-Render ---------------------------------------------------------
 
 function quoteIdent(name: string): string {
@@ -275,6 +297,13 @@ export function renderMigrationSql(
     lines.push("-- === Changed tables ===");
     for (const td of diff.changedTables) {
       lines.push(`-- ${td.tableName}`);
+      if (td.nextMeta.source === "managed" && managedChangeRequiresRecreate(td)) {
+        lines.push("-- managed projection — recreated + rebuilt from events (see .rebuild.json)");
+        lines.push(`DROP TABLE IF EXISTS ${quoteIdent(td.tableName)};`);
+        lines.push(...renderTableDdl(td.nextMeta));
+        lines.push("");
+        continue;
+      }
       for (const col of td.newColumns) {
         lines.push(renderAddColumn(td.tableName, col));
       }

package/src/db/rebuild-marker.ts

@@ -19,7 +19,7 @@
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
-import type { SchemaDiff } from "./migrate-generator";
+import { managedChangeRequiresRecreate, type SchemaDiff } from "./migrate-generator";
 
 const MARKER_VERSION = 1 as const;
 
@@ -32,18 +32,22 @@ function markerPathFor(migrationsDir: string, migrationId: string): string {
   return join(migrationsDir, `${migrationId}.rebuild.json`);
 }
 
-// Tabellen die nach dieser Migration einen Projection-Rebuild brauchen können:
-// neu angelegte + Tabellen mit neuer Spalte. Index-/Nullability-/Default-/
-// Drop-only-Änderungen brauchen KEINEN Rebuild — das generierte ALTER-SQL
-// bringt die Tabelle alleine in den Soll-Zustand. Über-Rebuild ist safe-but-
-// wasteful (Tabellen-Lock + Full-Replay auf grossen Streams), deshalb hier
-// konservativ einschränken. Sortiert + dedupliziert für stabilen PR-Diff.
+// Nur MANAGED Projektionen (Derivate des Event-Streams) brauchen/erlauben einen
+// Rebuild — unmanaged Tabellen tragen echte Daten und werden nie aus Events
+// rekonstruiert, dürfen also nie in den Marker. Eine managed Tabelle kommt rein,
+// wenn sie eine neue Spalte bekommt (Backfill) oder DROP+CREATE'd wird
+// (managedChangeRequiresRecreate → Tabelle geleert, muss neu gefüllt werden).
+// Reine non-unique-Index-/Default-/DROP-NOT-NULL-Änderungen brauchen keinen
+// Rebuild. Sortiert + dedupliziert für stabilen PR-Diff.
 export function rebuildTablesFromDiff(diff: SchemaDiff): readonly string[] {
   const names = new Set<string>();
   for (const t of diff.changedTables) {
-    if (t.newColumns.length > 0) names.add(t.tableName);
+    if (t.nextMeta.source !== "managed") continue;
+    if (t.newColumns.length > 0 || managedChangeRequiresRecreate(t)) names.add(t.tableName);
+  }
+  for (const t of diff.newTables) {
+    if (t.source === "managed") names.add(t.tableName);
   }
-  for (const t of diff.newTables) names.add(t.tableName);
   return [...names].sort();
 }