@cosmicdrift/kumiko-framework 0.326.1 → 0.328.0

package/src/engine/feature-builder-state.ts

@@ -1,3 +1,4 @@
+import type { WebSocketRouteDefinition } from "@cosmicdrift/kumiko-types/websocket-route";
 import type * as z from "zod";
 import type { ZodType } from "zod";
 import { LifecycleHookTypes } from "./constants";
@@ -99,6 +100,7 @@ export type FeatureBuilderState = {
   contentCollections: Record<string, ContentCollectionDefinition>;
   workspaces: Record<string, WorkspaceDefinition>;
   httpRoutes: Record<string, HttpRouteDefinition>;
+  webSocketRoutes: Record<string, WebSocketRouteDefinition>;
   translations: TranslationKeys;
   isSystemScoped: boolean;
   toggleableDefault: boolean | undefined;
@@ -160,6 +162,7 @@ export function createInitialFeatureBuilderState(): FeatureBuilderState {
     contentCollections: {},
     workspaces: {},
     httpRoutes: {},
+    webSocketRoutes: {},
     translations: {},
     isSystemScoped: false,
     toggleableDefault: undefined,

package/src/engine/feature-ui-extensions.ts

@@ -1,3 +1,9 @@
+import {
+  WEBSOCKET_MAX_CONNECTIONS_PER_USER_LIMIT,
+  WEBSOCKET_MAX_PAYLOAD_BYTES,
+  WEBSOCKET_ROUTE_PATH_PREFIX,
+  type WebSocketRouteDefinition,
+} from "@cosmicdrift/kumiko-types/websocket-route";
 import type { EntityTableMeta } from "../db/entity-table-meta";
 import { bindHookEscapeHatchGrant } from "../pipeline/system-identity-switch";
 import { LifecycleHookTypes } from "./constants";
@@ -514,6 +520,52 @@ export function buildUiExtensionsMethods<TName extends string>(
       }
       state.httpRoutes[key] = definition;
     },
+    webSocketRoute(definition: WebSocketRouteDefinition): void {
+      if (!definition.path.startsWith(WEBSOCKET_ROUTE_PATH_PREFIX)) {
+        throw new Error(
+          `[Feature ${name}] webSocketRoute path "${definition.path}" must start with ` +
+            `"${WEBSOCKET_ROUTE_PATH_PREFIX}" — only that namespace rides the /api/* auth chain ` +
+            "the upgrade relies on.",
+        );
+      }
+      if (definition.path.includes("*")) {
+        throw new Error(
+          `[Feature ${name}] webSocketRoute path "${definition.path}" must not contain "*" — ` +
+            "wildcards would let one route swallow unrelated /api/ws/* paths.",
+        );
+      }
+      const { maxMessageBytes } = definition;
+      if (
+        maxMessageBytes !== undefined &&
+        (!Number.isInteger(maxMessageBytes) ||
+          maxMessageBytes < 1 ||
+          maxMessageBytes > WEBSOCKET_MAX_PAYLOAD_BYTES)
+      ) {
+        throw new Error(
+          `[Feature ${name}] webSocketRoute "${definition.path}" maxMessageBytes must be an ` +
+            `integer between 1 and ${WEBSOCKET_MAX_PAYLOAD_BYTES}, got ${maxMessageBytes}.`,
+        );
+      }
+      const { maxConnectionsPerUser } = definition;
+      if (
+        maxConnectionsPerUser !== undefined &&
+        (!Number.isInteger(maxConnectionsPerUser) ||
+          maxConnectionsPerUser < 1 ||
+          maxConnectionsPerUser > WEBSOCKET_MAX_CONNECTIONS_PER_USER_LIMIT)
+      ) {
+        throw new Error(
+          `[Feature ${name}] webSocketRoute "${definition.path}" maxConnectionsPerUser must be an ` +
+            `integer between 1 and ${WEBSOCKET_MAX_CONNECTIONS_PER_USER_LIMIT}, got ${maxConnectionsPerUser}.`,
+        );
+      }
+      if (state.webSocketRoutes[definition.path]) {
+        throw new Error(
+          `[Feature ${name}] WebSocket route "${definition.path}" already registered. ` +
+            "path must be unique per feature.",
+        );
+      }
+      state.webSocketRoutes[definition.path] = definition;
+    },
     storeTable(meta: EntityTableMeta, options: StoreTableOptions): void {
       // Name comes from the meta itself — apps already give the table a
       // name when calling defineUnmanagedTable, no need to repeat it.

package/src/engine/index.ts

@@ -298,6 +298,10 @@ export type { TransitionGraph } from "./state-machine";
 export { defineTransitions, guardTransition } from "./state-machine";
 export { evaluateEventMatch } from "./steps/_event-match";
 export {
+  STEP_DISPATCH_AGGREGATE_TYPE,
+  STEP_DISPATCH_FAILED_TYPE,
+  STEP_DISPATCH_REQUESTED_TYPE,
+  STEP_DISPATCHED_TYPE,
   SUSPEND_SENTINEL,
   WORKFLOW_AGGREGATE_TYPE,
   WORKFLOW_RESUMED_TYPE,
@@ -308,6 +312,7 @@ export {
   WORKFLOW_WAITING_FOR_EVENT_TYPE,
   WORKFLOW_WAITING_TYPE,
 } from "./steps/_step-dispatch-constants";
+export { describeWorkflowStepError } from "./steps/describe-workflow-step-error";
 export {
   ANONYMOUS_ROLE,
   ANONYMOUS_USER_ID,

package/src/engine/steps/describe-workflow-step-error.ts

@@ -0,0 +1,6 @@
+// Workflow events (run-failed, retry.scheduled) outlive an Art. 17 erase, and a
+// raw error message can echo recipients or payload fragments. Events carry only
+// this generic text; the raw error goes to the log.
+export function describeWorkflowStepError(error: unknown): string {
+  return `workflow step failed (${error instanceof Error ? error.name : "unknown error"})`;
+}

package/src/engine/steps/mail-send.ts

@@ -33,7 +33,11 @@ defineStep<MailSendArgs, void>({
       type: STEP_DISPATCH_REQUESTED_TYPE,
       payload: {
         stepKind: "mail.send",
-        spec: { to, subject, body, ...(from && { from }) },
+        // Flat string fields: event PII encryption only handles top-level strings.
+        to: JSON.stringify(to),
+        subject,
+        body,
+        ...(from && { from }),
       },
     });
   },

package/src/engine/steps/retry.ts

@@ -7,6 +7,7 @@
 // after the backoff duration and re-enters the step at the same index.
 // After all attempts exhausted, the original error propagates.
 
+import { createFallbackLogger } from "../../logging";
 import { defineStep } from "../define-step";
 import { runStepList } from "../run-pipeline";
 import type { PipelineCtx, StepInstance } from "../types/step";
@@ -15,6 +16,9 @@ import {
   WORKFLOW_AGGREGATE_TYPE,
   WORKFLOW_RETRY_SCHEDULED_TYPE,
 } from "./_step-dispatch-constants";
+import { describeWorkflowStepError } from "./describe-workflow-step-error";
+
+const log = createFallbackLogger("workflow-retry");
 
 type RetryStepArgs = {
   readonly times: number;
@@ -47,6 +51,14 @@ defineStep<RetryStepArgs, undefined | typeof SUSPEND_SENTINEL>({
         throw error;
       }
 
+      log.warn("workflow step failed, retry scheduled", {
+        runId: ctx.workflow.runId,
+        workflowName: ctx.workflow.workflowName,
+        stepIndex: ctx.workflow.stepIndex,
+        attempt,
+        error: String(error),
+      });
+
       const backoffMs = calculateBackoff(attempt, args.backoff);
       const wakeAt = Temporal.Now.instant().add({ milliseconds: backoffMs }).toString();
 
@@ -60,7 +72,7 @@ defineStep<RetryStepArgs, undefined | typeof SUSPEND_SENTINEL>({
           maxAttempts,
           wakeAt,
           workflowName: ctx.workflow.workflowName,
-          error: String(error),
+          error: describeWorkflowStepError(error),
           triggerEventType: ctx.event.type,
           triggerPayload: ctx.event.payload,
           ...(ctx.workflow.definitionFingerprint && {

package/src/engine/steps/webhook-send.ts

@@ -14,15 +14,6 @@ import {
   STEP_DISPATCH_REQUESTED_TYPE,
 } from "./_step-dispatch-constants";
 
-// Re-export for back-compat callers (bundled step-dispatcher imports
-// these). The canonical home is _step-dispatch-constants.ts.
-export {
-  STEP_DISPATCH_AGGREGATE_TYPE,
-  STEP_DISPATCH_FAILED_TYPE,
-  STEP_DISPATCH_REQUESTED_TYPE,
-  STEP_DISPATCHED_TYPE,
-} from "./_step-dispatch-constants";
-
 type WebhookHttpMethod = "POST" | "PUT" | "PATCH";
 
 // Name inside the tenant-owned secrets namespace
@@ -55,13 +46,12 @@ defineStep<WebhookSendArgs, void>({
       type: STEP_DISPATCH_REQUESTED_TYPE,
       payload: {
         stepKind: "webhook.send",
-        spec: {
-          url,
-          method: args.method ?? "POST",
-          headers,
-          body,
-          ...(args.auth && { auth: args.auth }),
-        },
+        // Flat string fields: event PII encryption only handles top-level strings.
+        url,
+        method: args.method ?? "POST",
+        headersJson: JSON.stringify(headers),
+        ...(body !== undefined && { bodyJson: JSON.stringify(body) }),
+        ...(args.auth && { auth: args.auth }),
         retry: args.retry ?? { times: 3, backoff: "exponential" },
       },
     });

package/src/engine/types/index.ts

@@ -143,6 +143,7 @@ export type {
   ResolvedPiiFlags,
   RetentionDef,
   SelectFieldDef,
+  SelectOptionTone,
   TextFieldDef,
   TimestampFieldDef,
   TransitionMap,
@@ -367,6 +368,13 @@ export type {
   TreeNode,
   TreeNodeState,
 } from "@cosmicdrift/kumiko-types/tree-node";
+export type {
+  WebSocketConnection,
+  WebSocketMessageData,
+  WebSocketRouteConnectDeps,
+  WebSocketRouteDefinition,
+  WebSocketSessionHandlers,
+} from "@cosmicdrift/kumiko-types/websocket-route";
 export type { WorkspaceDefinition } from "@cosmicdrift/kumiko-types/workspace";
 // Re-export types that were duplicated in types.ts but are canonical in constants.ts
 export type {

package/src/event-store/index.ts

@@ -1,8 +1,12 @@
 export {
   backfillEventPiiEncryption,
+  backfillEventPiiEncryptionBatch,
+  type PiiBackfillBatchOptions,
+  type PiiBackfillBatchResult,
   type PiiBackfillFailure,
   type PiiBackfillOptions,
   type PiiBackfillResult,
+  type PiiBackfillScanCache,
 } from "../db/queries/backfill-pii";
 export {
   type ArchiveStreamArgs,

package/src/event-store/provenance-append.ts

@@ -1,8 +1,8 @@
+import { SYSTEM_EVENT_PREFIX } from "../crypto/system-event-pii";
 import { runInSavepointIfSupported } from "../db/query";
 import { type TenantDb, unsafeRawForDeclaredStep, withUnsafeRawGrant } from "../db/tenant-db";
 import type { TenantId } from "../engine/types";
 import { AccessDeniedError, InternalError } from "../errors";
-import { SYSTEM_EVENT_PREFIX } from "../pipeline/append-event-core";
 import { append, type EventMetadata, getStreamVersion } from "./event-store";
 
 // Fixed by the framework, not the caller — the point of this entry point is

package/src/event-store/transfer.ts

@@ -8,6 +8,7 @@
 // commits it together with the root row's ownership write — see
 // move-entity-graph.ts's file header for why that matters).
 
+import { AGGREGATE_TRANSFERRED_EVENT_TYPE } from "../crypto/system-event-pii";
 import type { DbRunner } from "../db";
 import {
   deleteSnapshotsForAggregates,
@@ -15,12 +16,11 @@ import {
   transferEventRows,
 } from "../db/queries/event-store-transfer";
 import type { TenantId } from "../engine/types";
-import { SYSTEM_EVENT_PREFIX } from "../pipeline/append-event-core";
 import { generateId } from "../utils/ids";
 import { append } from "./event-store";
 
 export const AGGREGATE_TRANSFER_STREAM_TYPE = "aggregate-transfer";
-export const AGGREGATE_TRANSFERRED_EVENT_TYPE = `${SYSTEM_EVENT_PREFIX}aggregate.transferred`;
+export { AGGREGATE_TRANSFERRED_EVENT_TYPE };
 
 export type TransferAggregateStreamsArgs = {
   readonly sourceTenantId: TenantId;

package/src/i18n/required-surface-keys.ts

@@ -166,6 +166,8 @@ export function requiredKeysFromScreen(
           out.add(fieldLabelKey(featureName, list.entity, normalized.field));
         }
       }
+      pushKey(out, list.createLabel);
+      pushKey(out, list.searchPlaceholder);
       for (const action of list.rowActions ?? []) pushRowActionKeys(out, action);
       for (const action of list.toolbarActions ?? []) pushToolbarActionKeys(out, action);
       break;
@@ -177,6 +179,8 @@ export function requiredKeysFromScreen(
         const normalized = normalizeListColumn(col);
         if (normalized.label !== undefined) pushKey(out, normalized.label);
       }
+      pushKey(out, list.createLabel);
+      pushKey(out, list.searchPlaceholder);
       for (const action of list.rowActions ?? []) pushRowActionKeys(out, action);
       for (const action of list.toolbarActions ?? []) pushToolbarActionKeys(out, action);
       break;
@@ -207,6 +211,8 @@ export function requiredKeysFromScreen(
     case "actionForm": {
       const form = screen as ActionFormScreenDefinition;
       pushKey(out, form.submitLabel);
+      pushKey(out, form.summary?.title);
+      pushKey(out, form.summary?.subtitle);
       for (const fieldName of Object.keys(form.fields)) {
         const override = form.fieldLabels?.[fieldName];
         if (override !== undefined) pushKey(out, override);
@@ -318,6 +324,8 @@ export function requiredKeysFromScreen(
         }
         if (section.kind === "relatedList") {
           pushKey(out, section.title);
+          pushKey(out, section.description);
+          pushKey(out, section.itemNoun);
           for (const col of section.columns) {
             const normalized = normalizeListColumn(col);
             if (normalized.label !== undefined) pushKey(out, normalized.label);

package/src/jobs/__tests__/jobs.integration.test.ts

@@ -1530,3 +1530,34 @@ describe("boot gates", () => {
     expect(() => createRegistry([validStringForm])).not.toThrow();
   });
 });
+
+describe("runner lifecycle: immediate stop (fw#1805)", () => {
+  test("stop() right after construction leaves no unhandled rejection", async () => {
+    const unhandled: unknown[] = [];
+    const onUnhandled = (reason: unknown) => {
+      unhandled.push(reason);
+    };
+    process.on("unhandledRejection", onUnhandled);
+    try {
+      // @timeout-exception: #1805 the 0-4ms delay is the variable under test (stop during connect), nothing to poll for
+      for (let i = 0; i < 20; i++) {
+        const runner = createJobRunner({
+          registry: createRegistry([testFeature]),
+          context: {},
+          redisUrl,
+          consumerLane: "worker",
+          queueNamePrefix: `kumiko-test-stop-${Date.now()}-${i}-${Math.random().toString(36).slice(2, 8)}`,
+        });
+        // Deterministic 0-4ms delays hit the window where the Queue
+        // connection is still 'initializing'.
+        await sleep(i % 5);
+        await runner.stop();
+      }
+      // Rejections surface on a later tick.
+      await sleep(200);
+    } finally {
+      process.off("unhandledRejection", onUnhandled);
+    }
+    expect(unhandled).toEqual([]);
+  });
+});

package/src/jobs/job-runner.ts

@@ -1181,6 +1181,24 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
         await worker.close();
         worker = null;
       }
+      // BullMQ close() while a Queue's connection is still 'initializing'
+      // rejects the in-flight INFO with "Connection is closed." on an emitter
+      // without listeners -> unhandled rejection (fw#1805). Wait for readiness
+      // first; a failed/timed-out wait must not block the close itself.
+      const readinessTimeout = timeoutReject(
+        bootRedisTimeoutMs,
+        `job-runner: Queue not ready within ${bootRedisTimeoutMs}ms during stop()`,
+      );
+      try {
+        await Promise.race([
+          Promise.all([queues.api.waitUntilReady(), queues.worker.waitUntilReady()]),
+          readinessTimeout.promise,
+        ]);
+      } catch {
+        // close() below still has to run
+      } finally {
+        readinessTimeout.cancel();
+      }
       await Promise.all([queues.api.close(), queues.worker.close()]);
       if (lockRedis) {
         // quit() drains in-flight commands; disconnect() would cancel them

package/src/migrations/__tests__/pending-rebuilds.integration.test.ts

@@ -6,7 +6,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:tes
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { integer, table as pgTable, uuid } from "../../db/dialect";
+import { integer, table as pgTable, text, uuid } from "../../db/dialect";
 import { createEventStoreExecutor } from "../../db/event-store-executor";
 import { asRawClient, selectMany } from "../../db/query";
 import { writeRebuildMarker } from "../../db/rebuild-marker";
@@ -21,7 +21,7 @@ import {
   type ProjectionDefinition,
 } from "../../engine";
 import type { JobRunner } from "../../jobs/job-runner";
-import { createProjectionStateTable } from "../../pipeline";
+import { createEventConsumerStateTable, createProjectionStateTable } from "../../pipeline";
 import {
   createTestDb,
   type TestDb,
@@ -54,6 +54,12 @@ const countsTable = pgTable("read_pending_counts", {
   itemCount: integer("item_count").notNull().default(0),
 });
 
+const mspItemNamesTable = pgTable("read_pending_msp_names", {
+  id: uuid("id").primaryKey(),
+  tenantId: uuid("tenant_id").notNull(),
+  name: text("name"),
+});
+
 // Exists in the DB but has no registered projection — models an owning
 // feature missing from the composition (distinct from a deliberately DROPped
 // table, which is never pushed at all).
@@ -83,6 +89,20 @@ const countsProjection: ProjectionDefinition = {
 const feature = defineFeature("pendingtest", (r) => {
   r.entity("pending-item", itemEntity);
   r.projection(countsProjection);
+  r.multiStreamProjection({
+    name: "pending-item-names",
+    table: mspItemNamesTable,
+    apply: {
+      "pending-item.created": async (event, tx) => {
+        const payload = event.payload as { name: string };
+        await asRawClient(tx).unsafe(
+          `INSERT INTO "read_pending_msp_names" (id, tenant_id, name) VALUES ($1::uuid, $2::uuid, $3)
+           ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name`,
+          [event.aggregateId, event.tenantId, payload.name],
+        );
+      },
+    },
+  });
 });
 
 const admin = TestUsers.admin;
@@ -97,9 +117,11 @@ beforeAll(async () => {
   testDb = await createTestDb();
   await unsafeCreateEntityTable(testDb.db, itemEntity, "pending-item");
   await createProjectionStateTable(testDb.db);
+  await createEventConsumerStateTable(testDb.db);
   await unsafePushTables(testDb.db, {
     readPendingCounts: countsTable,
     readOrphanProjection: orphanProjectionTable,
+    readPendingMspNames: mspItemNamesTable,
   });
   tdb = createTenantDb(testDb.db, admin.tenantId);
   markerDir = mkdtempSync(join(tmpdir(), "pending-rebuilds-"));
@@ -113,7 +135,7 @@ afterAll(async () => {
 beforeEach(async () => {
   failApply = false;
   await asRawClient(testDb.db).unsafe(
-    `TRUNCATE kumiko_events, read_pending_items, read_pending_counts, read_orphan_projection, kumiko_projections RESTART IDENTITY CASCADE`,
+    `TRUNCATE kumiko_events, read_pending_items, read_pending_counts, read_orphan_projection, read_pending_msp_names, kumiko_projections RESTART IDENTITY CASCADE`,
   );
   await asRawClient(testDb.db).unsafe(`DROP TABLE IF EXISTS kumiko_pending_rebuilds`);
 });
@@ -160,6 +182,26 @@ describe("pending-rebuilds queue", () => {
     expect(await getCount()).toBe(2);
   });
 
+  test("a queued multi-stream projection table is rebuilt through the multi-stream path", async () => {
+    await executor.create({ groupId: GROUP, name: "a" }, admin, tdb);
+    await executor.create({ groupId: GROUP, name: "b" }, admin, tdb);
+    writeRebuildMarker(markerDir, "0003_msp.sql", ["read_pending_msp_names"]);
+    await queueRebuildsFromMarkers(testDb.db, {
+      migrationsDir: markerDir,
+      appliedIds: ["0003_msp"],
+    });
+
+    const run = await runPendingRebuilds(testDb.db, registry);
+
+    expect(run.failed).toEqual([]);
+    expect(run.rebuilt).toEqual([
+      { projection: "pendingtest:projection:pending-item-names", eventsProcessed: 2 },
+    ]);
+    expect(await listPendingRebuilds(testDb.db)).toEqual([]);
+    const rows = await selectMany(testDb.db, mspItemNamesTable);
+    expect(rows.map((r) => r.name).sort()).toEqual(["a", "b"]);
+  });
+
   test("tables without a registered projection are drained, not stuck forever", async () => {
     writeRebuildMarker(markerDir, "0002_unmapped.sql", ["read_some_plain_table"]);
     await queueRebuildsFromMarkers(testDb.db, {