@cosmicdrift/kumiko-framework 0.156.2 → 0.157.0

package/src/engine/registry-ingest.ts

@@ -23,7 +23,7 @@ export function populateFeatureCore(state: RegistryState, feature: FeatureDefini
     if (clash?.kind === "raw") {
       throw new Error(
         `Entity "${name}" (feature "${feature.name}") has physical table "${physical}" which ` +
-          `collides with r.rawTable("${physical}") (feature "${clash.featureName}"). ` +
+          `collides with r.storeTable("${physical}") (feature "${clash.featureName}"). ` +
           `Pick a different tableName — both would emit CREATE TABLE "${physical}".`,
       );
     }
@@ -249,7 +249,7 @@ export function populateMetricsAndSecrets(state: RegistryState, feature: Feature
   }
 }
 
-// Explicit + multi-stream projections (source-entity indexed) + raw tables +
+// Explicit + multi-stream projections (source-entity indexed) + store tables +
 // unmanaged tables (both cross-feature-uniqueness-by-physical-name guarded).
 export function populateProjectionsAndTables(
   state: RegistryState,
@@ -300,15 +300,15 @@ export function populateProjectionsAndTables(
     state.multiStreamProjectionFeatureMap.set(qualified, feature.name);
   }
 
-  // Raw tables: aggregated by feature-local short name (unprefixed —
+  // Store tables: aggregated by feature-local short name (unprefixed —
   // these bypass the qualified-name namespace because they have no
   // event-stream binding to disambiguate). Reject cross-feature
   // duplicates at boot so the dev-server doesn't race two CREATE TABLE
   // statements that target the same physical table name. Two features
   // registering the same physical tableName would also race two CREATE
   // TABLE statements via migrate-runner.
-  for (const [rawName, rawDef] of Object.entries(feature.rawTables ?? {})) {
-    const existing = state.rawTableMap.get(rawName);
+  for (const [rawName, rawDef] of Object.entries(feature.storeTables ?? {})) {
+    const existing = state.storeTableMap.get(rawName);
     if (existing) {
       throw new Error(
         `Raw-table "${rawName}" registered by both feature "${existing.featureName}" and ` +
@@ -337,7 +337,7 @@ export function populateProjectionsAndTables(
       owner: rawName,
       featureName: feature.name,
     });
-    state.rawTableMap.set(rawName, { ...rawDef, featureName: feature.name });
+    state.storeTableMap.set(rawName, { ...rawDef, featureName: feature.name });
   }
 }
 

package/src/engine/registry-state.ts

@@ -29,7 +29,6 @@ import type {
   PreSaveHookFn,
   ProjectionDefinition,
   QueryHandlerDef,
-  RawTableDef,
   ReferenceDataDef,
   RegistrarExtensionDef,
   RegistrarExtensionRegistration,
@@ -37,6 +36,7 @@ import type {
   ScreenDefinition,
   SearchPayloadContributorFn,
   SecretKeyDefinition,
+  StoreTableDef,
   TreeActionDef,
   WorkspaceDefinition,
   WriteHandlerDef,
@@ -199,7 +199,7 @@ export type RegistryState = {
   projectionsBySource: Map<string, ProjectionDefinition[]>;
   multiStreamProjectionMap: Map<string, MultiStreamProjectionDefinition>;
   multiStreamProjectionFeatureMap: Map<string, string>;
-  rawTableMap: Map<string, RawTableDef>;
+  storeTableMap: Map<string, StoreTableDef>;
   physicalTableOwners: Map<string, { kind: "entity" | "raw"; owner: string; featureName: string }>;
   authClaimsHooks: AuthClaimsHookDef[];
   claimKeyMap: Map<string, ClaimKeyDefinition>;
@@ -260,7 +260,7 @@ export function createInitialState(): RegistryState {
     projectionsBySource: new Map(),
     multiStreamProjectionMap: new Map(),
     multiStreamProjectionFeatureMap: new Map(),
-    rawTableMap: new Map(),
+    storeTableMap: new Map(),
     physicalTableOwners: new Map(),
     authClaimsHooks: [],
     claimKeyMap: new Map(),

package/src/engine/types/feature.ts

@@ -125,18 +125,18 @@ export type SecretKeyHandle = {
   readonly name: string;
 };
 
-// --- Raw tables (declared by features via r.rawTable()) ---
+// --- Store tables (declared by features via r.storeTable()) ---
 // Post-drizzle-cut: unified with the former r.unmanagedTable() — both
 // carried the same reason/audit contract, differing only in whether the
-// table value was a legacy Drizzle PgTable (rawTable) or the framework-
+// table value was a legacy Drizzle PgTable (storeTable) or the framework-
 // native EntityTableMeta (unmanagedTable, consumed by migrate-runner).
-// EntityTableMeta was the forward-compatible shape, so rawTable adopted it.
+// EntityTableMeta was the forward-compatible shape, so storeTable adopted it.
 
-/** Options accepted by `r.rawTable()`. The `reason` is required so the
+/** Options accepted by `r.storeTable()`. The `reason` is required so the
  *  bypass leaves an audit trail at the registration site — reviewers can
  *  judge legitimacy without spelunking into history, and a future cleanup
  *  pass can find candidates for migration to `r.entity()`. */
-export type RawTableOptions = {
+export type StoreTableOptions = {
   /** Why this table needs to bypass the event-sourcing system. Examples:
    *  "imported from pre-ES system, read-only", "external Stripe webhook
    *  payload cache, write-only by webhook handler", "denormalised
@@ -149,22 +149,22 @@ export type RawTableOptions = {
   readonly piiEncryptedOnWrite?: true;
 };
 
-/** Per-feature raw-table registration. `meta` is the `EntityTableMeta`
+/** Per-feature store-table registration. `meta` is the `EntityTableMeta`
  *  (framework-native shape used by `migrate-runner`). Carries the
  *  bypass-justification reason but knows nothing about the owning
  *  feature — that's added when the registry aggregates entries
- *  cross-feature into `RawTableDef`. */
-export type RawTableEntry = {
+ *  cross-feature into `StoreTableDef`. */
+export type StoreTableEntry = {
   readonly name: string;
   readonly meta: EntityTableMeta;
   readonly reason: string;
   readonly piiEncryptedOnWrite?: true;
 };
 
-/** Registry-aggregated raw-table — the per-feature `RawTableEntry` plus
- *  the owning feature name. This is what `Registry.getAllRawTables()`
+/** Registry-aggregated store-table — the per-feature `StoreTableEntry` plus
+ *  the owning feature name. This is what `Registry.getAllStoreTables()`
  *  exposes to readers (dev-server, ops UIs). */
-export type RawTableDef = RawTableEntry & {
+export type StoreTableDef = StoreTableEntry & {
   readonly featureName: string;
 };
 
@@ -341,13 +341,13 @@ export type FeatureDefinition = {
   // den Hono-app (außerhalb /api/*). Pattern symmetrisch zu queryHandlers/
   // writeHandlers — Routes leben mit dem Feature, nicht im Bootstrap.
   readonly httpRoutes: Readonly<Record<string, HttpRouteDefinition>>;
-  // Raw tables declared via r.rawTable() — bypass the event-sourcing
+  // Store tables declared via r.storeTable() — bypass the event-sourcing
   // system. Keyed by feature-local short name (derived from
   // meta.tableName). The registry attaches featureName on aggregation,
-  // lifting RawTableEntry → RawTableDef. `kumiko schema generate`
+  // lifting StoreTableEntry → StoreTableDef. `kumiko schema generate`
   // aggregates these alongside r.entity()-derived metas to build the
   // full schema.
-  readonly rawTables: Readonly<Record<string, RawTableEntry>>;
+  readonly storeTables: Readonly<Record<string, StoreTableEntry>>;
   // Optional Zod-schema for env-vars this feature reads at runtime.
   // Declared via `r.envSchema(z.object({...}))`. `composeEnvSchema` reads
   // this to build one app-wide schema for boot-validation + dry-run
@@ -735,7 +735,7 @@ export type FeatureRegistrar<TFeature extends string = string> = {
   // The required `reason` string is the marker that justifies the bypass —
   // a non-empty string is the contract. If you can't write a reason,
   // declare data via `r.entity()` instead.
-  rawTable(meta: EntityTableMeta, options: RawTableOptions): void;
+  storeTable(meta: EntityTableMeta, options: StoreTableOptions): void;
 
   // Register the tree-actions schema for this feature — a map of
   // action-name → action-definition (with optional typed args). At-most-
@@ -894,12 +894,12 @@ export type Registry = {
   getProjectionsForSource(entityName: string): readonly ProjectionDefinition[];
   getAllProjections(): ReadonlyMap<string, ProjectionDefinition>;
 
-  // All r.rawTable() registrations across all features, keyed by
+  // All r.storeTable() registrations across all features, keyed by
   // feature-local short name. The dev-server iterates this alongside
   // implicit projections at boot. Cross-feature uniqueness is enforced
   // at registry-build — duplicate names from different features fail
   // the boot, so callers can rely on a flat keyspace.
-  getAllRawTables(): ReadonlyMap<string, RawTableDef>;
+  getAllStoreTables(): ReadonlyMap<string, StoreTableDef>;
 
   // Multi-stream projections registered via r.multiStreamProjection().
   // Keyed by qualified name. The server wires each into the event-dispatcher

package/src/engine/types/index.ts

@@ -67,13 +67,13 @@ export type {
   FeatureMetricType,
   FeatureRegistrar,
   MetricOptions,
-  RawTableDef,
-  RawTableEntry,
-  RawTableOptions,
   Registry,
   SecretKeyDefinition,
   SecretKeyHandle,
   SecretOptions,
+  StoreTableDef,
+  StoreTableEntry,
+  StoreTableOptions,
   UiHintOption,
   UiHints,
 } from "./feature";

package/src/files/__tests__/file-ref-entity.test.ts

@@ -32,3 +32,11 @@ describe("fileRefEntity base-column drift", () => {
     expect(insertedById[0]?.notNull).toBe(false);
   });
 });
+
+describe("fileRefEntity — DDL (#1205 regression)", () => {
+  test("size column is bigint, not double precision", () => {
+    const size = col("size");
+    expect(size).toHaveLength(1);
+    expect(size[0]?.pgType).toBe("bigint");
+  });
+});

package/src/files/file-ref-entity.ts

@@ -1,4 +1,4 @@
-import { createEntity, createNumberField, createTextField } from "../engine";
+import { createBigIntField, createEntity, createTextField } from "../engine";
 
 // fileRef — das File-Metadata-Entity. Ganz normales ES-Entity: Upload/Delete
 // laufen über den Standard-Executor (file-routes.ts), die Tabelle `file_refs`
@@ -25,7 +25,7 @@ export const fileRefEntity = createEntity({
     storageKey: createTextField({ required: true }),
     fileName: createTextField({ required: true, pii: true }),
     mimeType: createTextField({ required: true }),
-    size: createNumberField({ required: true }),
+    size: createBigIntField({ required: true }),
     entityType: createTextField(),
     entityId: createTextField(),
     fieldName: createTextField(),

package/src/migrations/pending-rebuilds.ts

@@ -200,7 +200,7 @@ export type EnqueueProjectionRebuildDeps = {
   readonly db: DbConnection;
   readonly registry: Registry;
   // Present + projection-rebuild job registered (jobs composed) → tracked,
-  // retryable job (read_job_runs + read_job_run_logs). Absent → inline rebuild.
+  // retryable job (read_job_runs + store_job_run_logs). Absent → inline rebuild.
   readonly jobRunner?: JobRunner;
 };
 

package/src/search/__tests__/reindex-entity.integration.test.ts

@@ -0,0 +1,121 @@
+// reindexEntity — Integration Test
+// Proves: rows written before a search consumer ever indexed them (the
+// #1206 scenario — searchable:true added retroactively) become findable
+// after a backfill run, soft-deleted rows stay excluded, and dryRun writes
+// nothing.
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import {
+  buildEntityTable,
+  createEventStoreExecutor,
+  createTenantDb,
+} from "@cosmicdrift/kumiko-framework/db";
+import { createEntity, createTextField, defineFeature } from "@cosmicdrift/kumiko-framework/engine";
+import { createEventsTable } from "@cosmicdrift/kumiko-framework/event-store";
+import { reindexEntity } from "@cosmicdrift/kumiko-framework/search";
+import {
+  setupTestStack,
+  type TestStack,
+  TestUsers,
+  unsafeCreateEntityTable,
+} from "@cosmicdrift/kumiko-framework/stack";
+
+const widgetEntity = createEntity({
+  table: "read_reindex_widgets",
+  softDelete: true,
+  fields: {
+    name: createTextField({ required: true, searchable: true }),
+  },
+});
+
+const widgetTable = buildEntityTable("widget", widgetEntity);
+
+const widgetFeature = defineFeature("reindex-test", (r) => {
+  r.entity("widget", widgetEntity);
+});
+
+let stack: TestStack;
+const admin = TestUsers.admin;
+
+beforeAll(async () => {
+  stack = await setupTestStack({ features: [widgetFeature] });
+  await unsafeCreateEntityTable(stack.db, widgetEntity);
+  await createEventsTable(stack.db);
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+function seedExecutor() {
+  return createEventStoreExecutor(widgetTable, widgetEntity, { entityName: "widget" });
+}
+
+function tenantDb() {
+  return createTenantDb(stack.db, admin.tenantId, "system");
+}
+
+describe("reindexEntity", () => {
+  test("indexes rows that were never drained through the search consumer", async () => {
+    const executor = seedExecutor();
+    const created = await executor.create({ name: "Backfillable Widget" }, admin, tenantDb());
+    if (!created.isSuccess) throw new Error("seed failed");
+
+    // No stack.eventDispatcher.runOnce() call — simulates rows that existed
+    // before search indexing ever ran for this entity.
+    const preResults = await stack.search.search(admin.tenantId, "backfillable", {
+      filterType: "widget",
+    });
+    expect(preResults).toHaveLength(0);
+
+    const result = await reindexEntity(
+      stack.db,
+      stack.registry,
+      stack.search,
+      "widget",
+      admin.tenantId,
+    );
+    expect(result.indexedRows).toBe(1);
+    expect(result.failures).toHaveLength(0);
+
+    const postResults = await stack.search.search(admin.tenantId, "backfillable", {
+      filterType: "widget",
+    });
+    expect(postResults.some((r) => r.entityId === created.data.id)).toBe(true);
+  });
+
+  test("skips soft-deleted rows", async () => {
+    const executor = seedExecutor();
+    const created = await executor.create({ name: "Erased Widget" }, admin, tenantDb());
+    if (!created.isSuccess) throw new Error("seed failed");
+    const deleted = await executor.delete({ id: created.data.id }, admin, tenantDb());
+    if (!deleted.isSuccess) throw new Error("delete failed");
+
+    await reindexEntity(stack.db, stack.registry, stack.search, "widget", admin.tenantId);
+
+    const postResults = await stack.search.search(admin.tenantId, "erased", {
+      filterType: "widget",
+    });
+    expect(postResults.some((r) => r.entityId === created.data.id)).toBe(false);
+  });
+
+  test("dryRun scans without writing to the index", async () => {
+    const executor = seedExecutor();
+    await executor.create({ name: "DryRun Widget" }, admin, tenantDb());
+
+    const result = await reindexEntity(
+      stack.db,
+      stack.registry,
+      stack.search,
+      "widget",
+      admin.tenantId,
+      { dryRun: true },
+    );
+    expect(result.indexedRows).toBeGreaterThan(0);
+
+    const postResults = await stack.search.search(admin.tenantId, "dryrun", {
+      filterType: "widget",
+    });
+    expect(postResults).toHaveLength(0);
+  });
+});

package/src/search/index.ts

@@ -3,6 +3,12 @@
 // von SearchAdapter-Types. Apps die Meilisearch nicht nutzen, ziehen den
 // Client-Code nicht mit rein.
 export { createInMemorySearchAdapter } from "./in-memory-adapter";
+export type {
+  ReindexEntityFailure,
+  ReindexEntityOptions,
+  ReindexEntityResult,
+} from "./reindex-entity";
+export { reindexEntity } from "./reindex-entity";
 export type {
   SearchAdapter,
   SearchAdapterConfig,

package/src/search/reindex-entity.ts

@@ -0,0 +1,148 @@
+// One-time backfill for entities that only got `searchable: true` after rows
+// already existed (#1206). buildSearchDocument() only runs on the write path
+// (createSearchEventConsumer, system-hooks.ts) — pre-existing rows never
+// pass through it, so they stay unfindable until their next write. This
+// re-derives a SearchDocument straight from the read-table row for every
+// existing row and indexes it, same as a live write would.
+
+import type { DbRunner } from "../db/connection";
+import { resolveTableName } from "../db/entity-table-meta";
+import { executeRawQuery } from "../db/queries/raw-sql";
+import type { Registry, TenantId } from "../engine/types";
+import { buildSearchDocument } from "../pipeline/system-hooks";
+import { toSnakeCase } from "../utils/case";
+import type { SearchAdapter, SearchDocument } from "./types";
+
+export type ReindexEntityFailure = {
+  readonly entityId: string;
+  readonly reason: string;
+};
+
+export type ReindexEntityResult = {
+  readonly scannedRows: number;
+  readonly indexedRows: number;
+  readonly failures: readonly ReindexEntityFailure[];
+};
+
+export type ReindexEntityOptions = {
+  readonly batchSize?: number;
+  // Scan + build docs, write nothing.
+  readonly dryRun?: boolean;
+};
+
+function quoteIdent(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+// Read-table state, forward-mapped from entity.fields (fieldName →
+// toSnakeCase(fieldName)). SELECT * + forward-map instead of building an
+// explicit column list: some field types (files/images) emit no column at
+// all, others (locatedTimestamp, money) emit companion columns under a
+// different name — an explicit alias list breaks on those. Missing columns
+// are just skipped, matching what a `.created` event payload would carry.
+function rowToState(
+  row: Record<string, unknown>,
+  fieldNames: readonly string[],
+): Record<string, unknown> {
+  const state: Record<string, unknown> = {};
+  for (const fieldName of fieldNames) {
+    const column = toSnakeCase(fieldName);
+    if (Object.hasOwn(row, column)) state[fieldName] = row[column];
+  }
+  return state;
+}
+
+export async function reindexEntity(
+  db: DbRunner,
+  registry: Registry,
+  searchAdapter: SearchAdapter,
+  entityName: string,
+  tenantId: TenantId,
+  options: ReindexEntityOptions = {},
+): Promise<ReindexEntityResult> {
+  const entity = registry.getEntity(entityName);
+  if (!entity) {
+    throw new Error(`reindexEntity: unknown entity "${entityName}"`);
+  }
+  const searchableFields = registry.getSearchableFields(entityName);
+  const extensions = registry.getSearchPayloadExtensions(entityName);
+  if (searchableFields.length === 0 && extensions.length === 0) {
+    throw new Error(
+      `reindexEntity: entity "${entityName}" has no searchable fields and no search-payload ` +
+        `extensions — nothing to index.`,
+    );
+  }
+
+  const batchSize = options.batchSize ?? 500;
+  const tableName = resolveTableName(entityName, entity, undefined);
+  const fieldNames = Object.keys(entity.fields);
+  // Soft-deleted rows already left the live index (the .deleted consumer
+  // calls searchAdapter.remove()) — resurrecting them here would make
+  // erased entities findable again.
+  const deletedFilter =
+    entity.softDelete === true ? `AND ${quoteIdent("is_deleted")} IS NOT TRUE` : "";
+
+  const result = { scannedRows: 0, indexedRows: 0, failures: [] as ReindexEntityFailure[] };
+
+  let offset = 0;
+  for (;;) {
+    // ponytail: LIMIT/OFFSET, not a keyset cursor — id can be uuid or
+    // serial depending on the entity, and a uniform cursor comparison
+    // across both types needs a text cast that breaks integer ordering.
+    // This is a one-time backfill over existing rows, not a live hot path;
+    // switch to keyset if it ever needs to run against a churning table.
+    const rows = await executeRawQuery<Record<string, unknown>>(
+      db,
+      `SELECT * FROM ${quoteIdent(tableName)}
+        WHERE ${quoteIdent("tenant_id")} = $1 ${deletedFilter}
+        ORDER BY ${quoteIdent("id")} ASC
+        LIMIT $2 OFFSET $3`,
+      [tenantId, batchSize, offset],
+    );
+    if (rows.length === 0) break;
+
+    const docs: Array<{ entityId: string; doc: SearchDocument }> = [];
+    for (const row of rows) {
+      result.scannedRows++;
+      const entityId = String(row["id"]);
+      try {
+        const state = rowToState(row, fieldNames);
+        const doc = await buildSearchDocument(entityName, entityId, state, registry);
+        if (doc) docs.push({ entityId, doc });
+      } catch (e) {
+        result.failures.push({ entityId, reason: e instanceof Error ? e.message : String(e) });
+      }
+    }
+
+    if (!options.dryRun && docs.length > 0) {
+      if (searchAdapter.indexBatch) {
+        try {
+          await searchAdapter.indexBatch(
+            tenantId,
+            docs.map((d) => d.doc),
+          );
+          result.indexedRows += docs.length;
+        } catch (e) {
+          const reason = e instanceof Error ? e.message : String(e);
+          for (const { entityId } of docs) result.failures.push({ entityId, reason });
+        }
+      } else {
+        for (const { entityId, doc } of docs) {
+          try {
+            await searchAdapter.index(tenantId, doc);
+            result.indexedRows++;
+          } catch (e) {
+            result.failures.push({ entityId, reason: e instanceof Error ? e.message : String(e) });
+          }
+        }
+      }
+    } else if (options.dryRun) {
+      result.indexedRows += docs.length;
+    }
+
+    offset += rows.length;
+    if (rows.length < batchSize) break;
+  }
+
+  return result;
+}

package/src/stack/push-entity-projection-tables.ts

@@ -9,7 +9,7 @@ const logInfo = (msg: string): void => console.log(msg);
 /**
  * Push all implicit-projection tables — one per `r.entity()` — that the
  * registry knows about. setupTestStack already handles explicit
- * projections, MSPs, and `r.rawTable()` declarations in its own loop;
+ * projections, MSPs, and `r.storeTable()` declarations in its own loop;
  * implicit projections are the missing piece for a fresh boot.
  *
  * Idempotent via `tableExists` so a persistent dev DB
@@ -20,7 +20,7 @@ const logInfo = (msg: string): void => console.log(msg);
  * Lives next to `setupTestStack` because both are stack-bootstrap
  * helpers that legitimately speak the `unsafe*`-DDL layer; the
  * Table-DDL Guard's stack/** allowlist is the single shared exemption
- * site. Apps still declare data via `r.entity()` / `r.rawTable()` and
+ * site. Apps still declare data via `r.entity()` / `r.storeTable()` and
  * never call this directly.
  */
 export async function pushEntityProjectionTables(