@cosmicdrift/kumiko-framework 0.326.1 → 0.328.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/package.json +4 -4
  2. package/src/api/__tests__/auth-middleware-transport.test.ts +23 -1
  3. package/src/api/__tests__/client-ip.test.ts +20 -0
  4. package/src/api/__tests__/origin-middleware.test.ts +65 -0
  5. package/src/api/__tests__/websocket-route.test.ts +552 -0
  6. package/src/api/auth-middleware.ts +29 -7
  7. package/src/api/client-ip.ts +15 -2
  8. package/src/api/index.ts +24 -1
  9. package/src/api/jwt.ts +5 -0
  10. package/src/api/origin-middleware.ts +27 -1
  11. package/src/api/server.ts +161 -2
  12. package/src/api/websocket-route.ts +327 -0
  13. package/src/bun-db/__tests__/closed-connection-retry.integration.test.ts +94 -1
  14. package/src/bun-db/query.ts +160 -23
  15. package/src/changes.json +54 -0
  16. package/src/crypto/__tests__/system-event-pii.test.ts +77 -0
  17. package/src/crypto/event-pii.ts +20 -4
  18. package/src/crypto/system-event-pii.ts +57 -0
  19. package/src/db/__tests__/apply-entity-event-removed-column.integration.test.ts +109 -0
  20. package/src/db/queries/backfill-pii.ts +184 -85
  21. package/src/db/queries/pii-backfill-state.ts +33 -0
  22. package/src/engine/__tests__/build-app-schema.test.ts +26 -0
  23. package/src/engine/__tests__/describe-workflow-step-error.test.ts +17 -0
  24. package/src/engine/__tests__/steps-mail-send.test.ts +6 -5
  25. package/src/engine/__tests__/steps-webhook-send.test.ts +16 -8
  26. package/src/engine/build-app-schema.ts +2 -0
  27. package/src/engine/define-feature.ts +1 -0
  28. package/src/engine/feature-builder-state.ts +3 -0
  29. package/src/engine/feature-ui-extensions.ts +52 -0
  30. package/src/engine/index.ts +5 -0
  31. package/src/engine/steps/describe-workflow-step-error.ts +6 -0
  32. package/src/engine/steps/mail-send.ts +5 -1
  33. package/src/engine/steps/retry.ts +13 -1
  34. package/src/engine/steps/webhook-send.ts +6 -16
  35. package/src/engine/types/index.ts +8 -0
  36. package/src/event-store/index.ts +4 -0
  37. package/src/event-store/provenance-append.ts +1 -1
  38. package/src/event-store/transfer.ts +2 -2
  39. package/src/i18n/required-surface-keys.ts +8 -0
  40. package/src/jobs/__tests__/jobs.integration.test.ts +31 -0
  41. package/src/jobs/job-runner.ts +18 -0
  42. package/src/migrations/__tests__/pending-rebuilds.integration.test.ts +45 -3
  43. package/src/migrations/__tests__/pii-event-backfill.integration.test.ts +457 -0
  44. package/src/migrations/index.ts +11 -0
  45. package/src/migrations/pending-rebuilds.ts +29 -3
  46. package/src/migrations/pii-event-backfill.ts +422 -0
  47. package/src/pipeline/append-event-core.ts +1 -8
  48. package/src/pipeline/irreversible-operation-gate.ts +4 -3
  49. package/src/pipeline/msp-rebuild.ts +7 -1
  50. package/src/search/__tests__/meilisearch-adapter.integration.test.ts +65 -0
  51. package/src/search/__tests__/search-adapter.test.ts +15 -0
  52. package/src/search/in-memory-adapter.ts +6 -0
  53. package/src/search/meilisearch-adapter.ts +24 -0
  54. package/src/stack/test-stack.ts +26 -2
  55. package/src/ui-types/index.ts +1 -0
@@ -6,7 +6,11 @@ import postgres from "postgres";
6
6
  import { constraintOf, extractPgError, isUniqueViolation } from "../../db/pg-error";
7
7
  import { testDatabaseUrl } from "../../testing/closed-connection-error";
8
8
  import { waitFor } from "../../testing/wait-for";
9
- import { isClosedConnectionError, unsafeReadRetrying } from "../query";
9
+ import {
10
+ describeClosedConnectionFailure,
11
+ isClosedConnectionError,
12
+ unsafeReadRetrying,
13
+ } from "../query";
10
14
 
11
15
  const DATABASE_URL = testDatabaseUrl();
12
16
 
@@ -157,3 +161,92 @@ describe.each(drivers)("extractPgError — %s unique-violation SQLSTATE", (kind)
157
161
  }
158
162
  });
159
163
  });
164
+
165
+ describe("describeClosedConnectionFailure", () => {
166
+ test("describes a real postgres-js pool error without SQL text or params", async () => {
167
+ const applicationName = randomAppName();
168
+ const pool = postgres(DATABASE_URL, {
169
+ max: 2,
170
+ connection: { application_name: applicationName },
171
+ });
172
+ try {
173
+ await Promise.all([pool.unsafe("select 1"), pool.unsafe("select 1")]);
174
+ const secretParam = `secret-${crypto.randomUUID()}`;
175
+ const pending = pool
176
+ .unsafe("select $1::text as x from pg_sleep(0.3)", [secretParam])
177
+ .catch((e: unknown) => e);
178
+ const pid = await pollForActivePid(applicationName, "%pg_sleep(0.3)%");
179
+ await terminateBackendPid(pid);
180
+ const caught = await pending;
181
+
182
+ const described = describeClosedConnectionFailure(pool, caught, 3);
183
+ expect(described).toMatchObject({
184
+ driver: "postgres-js",
185
+ pooled: true,
186
+ attempts: 3,
187
+ poolMax: 2,
188
+ });
189
+ expect(described?.["errorCode"]).toBe(extractPgError(caught)?.code);
190
+ const serialized = JSON.stringify(described);
191
+ expect(serialized).not.toContain(secretParam);
192
+ expect(serialized).not.toContain("pg_sleep");
193
+ } finally {
194
+ await pool.end({ timeout: 0 });
195
+ }
196
+ });
197
+
198
+ test("describes a real postgres-js transaction handle as not pooled", async () => {
199
+ const applicationName = randomAppName();
200
+ const pool = postgres(DATABASE_URL, {
201
+ max: 2,
202
+ connection: { application_name: applicationName },
203
+ });
204
+ try {
205
+ let described: Record<string, unknown> | undefined;
206
+ await pool
207
+ .begin(async (tx) => {
208
+ const pending = tx.unsafe("select 1 from pg_sleep(0.3)").catch((e: unknown) => e);
209
+ const pid = await pollForActivePid(applicationName, "%pg_sleep(0.3)%");
210
+ await terminateBackendPid(pid);
211
+ described = describeClosedConnectionFailure(tx, await pending, 1);
212
+ })
213
+ .catch(() => undefined);
214
+ expect(described).toMatchObject({ driver: "postgres-js", pooled: false, attempts: 1 });
215
+ } finally {
216
+ await pool.end({ timeout: 0 });
217
+ }
218
+ });
219
+
220
+ test("describes a real Bun.SQL closed-pool error", async () => {
221
+ const pool = new Bun.SQL({ url: DATABASE_URL, max: 1, idleTimeout: 5, maxLifetime: 10 });
222
+ await pool.unsafe("select 1");
223
+ await pool.close();
224
+ const caught = await pool.unsafe("select 1").catch((e: unknown) => e);
225
+
226
+ expect(describeClosedConnectionFailure(pool, caught, 1)).toMatchObject({
227
+ driver: "bun-sql",
228
+ pooled: true,
229
+ poolMax: 1,
230
+ idleTimeoutMs: 5000,
231
+ maxLifetimeMs: 10000,
232
+ errorCode: "ERR_POSTGRES_CONNECTION_CLOSED",
233
+ });
234
+ });
235
+
236
+ test("returns undefined for an unrelated error", () => {
237
+ expect(describeClosedConnectionFailure({}, new Error("boom"), 1)).toBeUndefined();
238
+ expect(describeClosedConnectionFailure({}, "not an error", 1)).toBeUndefined();
239
+ });
240
+
241
+ test("matches the driver's AbortError for a closed connection", () => {
242
+ const abort = new DOMException("The connection was closed.", "AbortError");
243
+ expect(describeClosedConnectionFailure({}, abort, 2)).toMatchObject({
244
+ driver: "unknown",
245
+ attempts: 2,
246
+ errorName: "AbortError",
247
+ });
248
+ expect(
249
+ describeClosedConnectionFailure({}, new DOMException("other", "AbortError"), 1),
250
+ ).toBeUndefined();
251
+ });
252
+ });
@@ -28,6 +28,7 @@ import type {
28
28
  // Static polyfill import (not getTemporal()): coercion results stay on one
29
29
  // Temporal implementation repo-wide, even when Bun exposes a global (#1480).
30
30
  import { Temporal } from "temporal-polyfill";
31
+ import { requestContext } from "../api/request-context";
31
32
  import { computeBlindIndex, configuredBlindIndexKey } from "../crypto/blind-index";
32
33
  import { SQL_EXPR_BRAND } from "../db/dialect";
33
34
  import type { EntityTableMeta } from "../db/entity-table-meta";
@@ -35,6 +36,7 @@ import { extractPgError } from "../db/pg-error";
35
36
  import { type NotExecutorOnly, toSnakeCase } from "../db/table-builder";
36
37
  import { camelCase as envCamelCase } from "../env";
37
38
  import { InternalError } from "../errors";
39
+ import { createFallbackLogger } from "../logging";
38
40
  import { parseJsonSafe } from "../utils/safe-json";
39
41
 
40
42
  // Idempotent snake_case → camelCase. `env.camelCase` always lowercases first
@@ -755,6 +757,120 @@ function isPooledClient(raw: unknown): boolean {
755
757
  );
756
758
  }
757
759
 
760
+ const closedConnectionLog = createFallbackLogger("bun-db");
761
+
762
+ // The driver rejects in-flight queries with this AbortError (no code) when the
763
+ // pool closes under them, so isClosedConnectionError can't see it.
764
+ const DRIVER_CLOSED_ABORT_MESSAGE = "The connection was closed.";
765
+
766
+ function isDriverClosedAbort(err: unknown): boolean {
767
+ return (
768
+ err instanceof Error && err.name === "AbortError" && err.message === DRIVER_CLOSED_ABORT_MESSAGE
769
+ );
770
+ }
771
+
772
+ function isHandleLike(raw: unknown): raw is Record<string, unknown> {
773
+ return raw !== null && (typeof raw === "object" || typeof raw === "function");
774
+ }
775
+
776
+ type ClosedConnectionDriver = "bun-sql" | "postgres-js" | "unknown";
777
+
778
+ // Bun.SQL is not instanceof-checkable (Bun.SQL.prototype is invalid) and its tx
779
+ // handles are plain function objects, so discriminate by members only one driver
780
+ // has: `flush` on Bun.SQL (pool and tx), `typed` on postgres-js (pool and tx).
781
+ function driverOf(raw: unknown): ClosedConnectionDriver {
782
+ if (!isHandleLike(raw)) return "unknown";
783
+ if (typeof raw["flush"] === "function") return "bun-sql";
784
+ if (typeof raw["typed"] === "function") return "postgres-js";
785
+ return "unknown";
786
+ }
787
+
788
+ function positiveNumber(v: unknown): number | undefined {
789
+ return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : undefined;
790
+ }
791
+
792
+ // postgres-js configures timeouts in seconds, Bun.SQL in milliseconds.
793
+ function timeoutOptionMs(
794
+ options: Record<string, unknown>,
795
+ driver: ClosedConnectionDriver,
796
+ bunName: string,
797
+ postgresJsName: string,
798
+ ): number | undefined {
799
+ if (driver === "postgres-js") {
800
+ const seconds = positiveNumber(options[postgresJsName]);
801
+ return seconds === undefined ? undefined : seconds * 1000;
802
+ }
803
+ return positiveNumber(options[bunName]);
804
+ }
805
+
806
+ function errorProp(err: unknown, name: string): unknown {
807
+ return isHandleLike(err) ? err[name] : undefined;
808
+ }
809
+
810
+ // Deliberately excludes SQL text and parameters: app SQL can carry PII.
811
+ export function describeClosedConnectionFailure(
812
+ raw: unknown,
813
+ err: unknown,
814
+ attempts: number,
815
+ ): Record<string, unknown> | undefined {
816
+ if (!isClosedConnectionError(err) && !isDriverClosedAbort(err)) return undefined;
817
+ const driver = driverOf(raw);
818
+ const rawOptions = isHandleLike(raw) ? raw["options"] : undefined;
819
+ const options = isHandleLike(rawOptions) ? rawOptions : {};
820
+ return {
821
+ driver,
822
+ pooled: isPooledClient(raw),
823
+ attempts,
824
+ poolMax: positiveNumber(options["max"]),
825
+ idleTimeoutMs: timeoutOptionMs(options, driver, "idleTimeout", "idle_timeout"),
826
+ maxLifetimeMs: timeoutOptionMs(options, driver, "maxLifetime", "max_lifetime"),
827
+ connectionTimeoutMs: timeoutOptionMs(options, driver, "connectionTimeout", "connect_timeout"),
828
+ errorName: errorProp(err, "name"),
829
+ errorCode: extractPgError(err)?.code ?? errorProp(err, "code"),
830
+ errorErrno: errorProp(err, "errno"),
831
+ errorMessage: errorProp(err, "message"),
832
+ };
833
+ }
834
+
835
+ type ClosedConnectionContext = { readonly operation: string; readonly table?: string };
836
+
837
+ function logClosedConnection(
838
+ raw: unknown,
839
+ err: unknown,
840
+ attempts: number,
841
+ context?: ClosedConnectionContext,
842
+ ): void {
843
+ const failure = describeClosedConnectionFailure(raw, err, attempts);
844
+ // skip: not a closed-connection error, nothing to diagnose
845
+ if (!failure) return;
846
+ // Request identity lets this line be joined to the route's `handler failed` log.
847
+ const request = requestContext.get();
848
+ closedConnectionLog.warn("closed connection surfaced", {
849
+ ...failure,
850
+ ...context,
851
+ requestId: request?.requestId,
852
+ correlationId: request?.correlationId,
853
+ handler: request?.handler,
854
+ });
855
+ }
856
+
857
+ // Write and other non-retried paths: log when a closed-connection error leaves
858
+ // bun-db, rethrow unchanged.
859
+ async function unsafeLoggingClosedConnection(
860
+ db: AnyDb,
861
+ sqlText: string,
862
+ params: readonly unknown[],
863
+ context: ClosedConnectionContext,
864
+ ): Promise<readonly unknown[]> {
865
+ const raw = asRawClient(db);
866
+ try {
867
+ return await raw.unsafe(sqlText, params);
868
+ } catch (err) {
869
+ logClosedConnection(raw, err, 1, context);
870
+ throw err;
871
+ }
872
+ }
873
+
758
874
  function poolMaxOf(raw: unknown): number {
759
875
  // @cast-boundary driver pool options — both postgres-js and Bun.SQL expose options.max
760
876
  const max = (raw as { options?: { max?: unknown } }).options?.max;
@@ -774,7 +890,10 @@ export async function unsafeReadRetrying<TRow>(
774
890
  try {
775
891
  return (await raw.unsafe(sqlText, params)) as readonly TRow[];
776
892
  } catch (err) {
777
- if (!isPooledClient(raw) || !isClosedConnectionError(err)) throw err;
893
+ if (!isPooledClient(raw) || !isClosedConnectionError(err)) {
894
+ logClosedConnection(raw, err, 1);
895
+ throw err;
896
+ }
778
897
  const maxAttempts = poolMaxOf(raw) + 1;
779
898
  let lastErr: unknown = err;
780
899
  for (let attempt = 0; attempt < maxAttempts; attempt++) {
@@ -782,9 +901,13 @@ export async function unsafeReadRetrying<TRow>(
782
901
  return (await raw.unsafe(sqlText, params)) as readonly TRow[];
783
902
  } catch (retryErr) {
784
903
  lastErr = retryErr;
785
- if (!isClosedConnectionError(retryErr)) throw retryErr;
904
+ if (!isClosedConnectionError(retryErr)) {
905
+ logClosedConnection(raw, retryErr, attempt + 2);
906
+ throw retryErr;
907
+ }
786
908
  }
787
909
  }
910
+ logClosedConnection(raw, lastErr, maxAttempts + 1);
788
911
  throw lastErr;
789
912
  }
790
913
  }
@@ -876,7 +999,10 @@ export async function insertMany<TRow = any>(
876
999
  valuesClauses.push(`(${placeholders.join(", ")})`);
877
1000
  }
878
1001
  const sqlText = `INSERT INTO ${quoteIdent(info.name)} (${cols}) VALUES ${valuesClauses.join(", ")} RETURNING *`;
879
- const raw = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<string, unknown>[];
1002
+ const raw = (await unsafeLoggingClosedConnection(db, sqlText, params, {
1003
+ operation: "insertMany",
1004
+ table: info.name,
1005
+ })) as readonly Record<string, unknown>[];
880
1006
  return coerceRows(raw, info) as readonly TRow[];
881
1007
  }
882
1008
 
@@ -909,10 +1035,10 @@ export async function insertOne<TRow = any>(
909
1035
  })
910
1036
  .join(", ");
911
1037
  const sqlText = `INSERT INTO ${quoteIdent(info.name)} (${cols}) VALUES (${placeholders}) RETURNING *`;
912
- const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<
913
- string,
914
- unknown
915
- >[];
1038
+ const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
1039
+ operation: "insertOne",
1040
+ table: info.name,
1041
+ })) as readonly Record<string, unknown>[];
916
1042
  const first = rows[0];
917
1043
  if (!first) return undefined;
918
1044
  return coerceRow(first, info) as TRow;
@@ -930,11 +1056,13 @@ export async function updateMany<TRow = any>(
930
1056
  return scoped.updateMany<TRow>(table, set, where);
931
1057
  }
932
1058
  const info = extractTableInfo(table);
933
- const setEntries = Object.entries(set).map(([k, v]) => {
934
- const col = info.columnOf(k);
935
- const pgType = info.pgTypeOf(col);
936
- return { col, prepared: prepareValue(v, pgType) };
937
- });
1059
+ const setEntries = Object.entries(set)
1060
+ .filter(([k]) => info.hasColumn(k))
1061
+ .map(([k, v]) => {
1062
+ const col = info.columnOf(k);
1063
+ const pgType = info.pgTypeOf(col);
1064
+ return { col, prepared: prepareValue(v, pgType) };
1065
+ });
938
1066
  if (setEntries.length === 0) throw new Error("updateMany: empty set object");
939
1067
  const values: unknown[] = [];
940
1068
  let idx = 1;
@@ -954,7 +1082,10 @@ export async function updateMany<TRow = any>(
954
1082
  for (const v of w.values) values.push(v);
955
1083
  }
956
1084
  sqlText += " RETURNING *";
957
- const raw = (await asRawClient(db).unsafe(sqlText, values)) as readonly Record<string, unknown>[];
1085
+ const raw = (await unsafeLoggingClosedConnection(db, sqlText, values, {
1086
+ operation: "updateMany",
1087
+ table: info.name,
1088
+ })) as readonly Record<string, unknown>[];
958
1089
  return coerceRows(raw, info) as readonly TRow[];
959
1090
  }
960
1091
 
@@ -971,7 +1102,10 @@ export async function deleteMany(
971
1102
  const w = buildWhereClause(info, where, 1);
972
1103
  let sqlText = `DELETE FROM ${quoteIdent(info.name)}`;
973
1104
  if (w.sqlText) sqlText += ` WHERE ${w.sqlText}`;
974
- await asRawClient(db).unsafe(sqlText, w.values);
1105
+ await unsafeLoggingClosedConnection(db, sqlText, w.values, {
1106
+ operation: "deleteMany",
1107
+ table: info.name,
1108
+ });
975
1109
  }
976
1110
 
977
1111
  type InsertEntry = {
@@ -1104,10 +1238,10 @@ export async function upsertOnConflict<TRow = any>(
1104
1238
 
1105
1239
  const conflictList = conflictCols.map((c) => quoteIdent(c)).join(", ");
1106
1240
  const sqlText = `${sqlPrefix} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateParts.join(", ")} RETURNING *`;
1107
- const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<
1108
- string,
1109
- unknown
1110
- >[];
1241
+ const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
1242
+ operation: "upsertOnConflict",
1243
+ table: info.name,
1244
+ })) as readonly Record<string, unknown>[];
1111
1245
  const first = rows[0];
1112
1246
  if (!first) return undefined;
1113
1247
  return coerceRow(first, info) as TRow;
@@ -1186,10 +1320,10 @@ export async function incrementCounter<TRow = any>(
1186
1320
 
1187
1321
  const conflictList = conflictCols.map((c) => quoteIdent(c)).join(", ");
1188
1322
  const sqlText = `${sqlPrefix} ON CONFLICT (${conflictList}) DO UPDATE SET ${updateParts.join(", ")} RETURNING *`;
1189
- const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly Record<
1190
- string,
1191
- unknown
1192
- >[];
1323
+ const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
1324
+ operation: "incrementCounter",
1325
+ table: info.name,
1326
+ })) as readonly Record<string, unknown>[];
1193
1327
  const first = rows[0];
1194
1328
  if (!first) return undefined;
1195
1329
  return coerceRow(first, info) as TRow;
@@ -1225,7 +1359,10 @@ export async function deleteManyBatched(
1225
1359
  const sqlText = `DELETE FROM ${tableQ} WHERE ${idQ} IN (
1226
1360
  SELECT ${idQ} FROM ${tableQ} WHERE ${w.sqlText} LIMIT $${limitIdx}
1227
1361
  ) RETURNING ${idQ}`;
1228
- const rows = (await asRawClient(db).unsafe(sqlText, params)) as readonly unknown[];
1362
+ const rows = (await unsafeLoggingClosedConnection(db, sqlText, params, {
1363
+ operation: "deleteManyBatched",
1364
+ table: info.name,
1365
+ })) as readonly unknown[];
1229
1366
  batches++;
1230
1367
  deleted += rows.length;
1231
1368
  if (rows.length < limit) break;
package/src/changes.json CHANGED
@@ -1,4 +1,58 @@
1
1
  [
2
+ {
3
+ "version": "0.328.0",
4
+ "type": "improvement",
5
+ "title": "bun-db logs a diagnostic warning when a closed-connection error surfaces",
6
+ "detail": "When a closed-connection error leaves bun-db (retries exhausted, a transaction/reserved handle, or a write path), a `[bun-db] closed connection surfaced` warning records driver, pooled vs. transaction handle, attempts, pool size, timeouts and the error code. It never includes SQL text or parameters. Behaviour, retries and timeouts are unchanged."
7
+ },
8
+ {
9
+ "version": "0.328.0",
10
+ "type": "improvement",
11
+ "title": "runProdApp and runWorkerApp re-encrypt plaintext events after a field gains a PII annotation",
12
+ "detail": "When an event field is annotated as PII after events were already stored, the stored payloads stay plaintext because a projection rebuild copies them unchanged. After start, `runProdApp` and `runWorkerApp` now run `runPiiEventBackfill` in the background: it encrypts those events in place once per annotation state, queues the affected projections (including multi-stream projections with their own table) in `kumiko_pending_rebuilds` and rebuilds them, so ciphertext and blind-index columns get materialized. Later boots only run a cheap catch-up over the newest events. Replicas are serialized by an advisory lock, each batch commits on its own, and failed events are logged and retried on the next boot. Progress lives in the new table `kumiko_pii_backfill_state`, created on first run. Behavior change: the first boot after the bump rewrites `kumiko_events` payloads for annotated fields; set `KUMIKO_SKIP_PII_BACKFILL=1` to opt out. Calling `backfillEventPiiEncryption` by hand stays necessary for `eraseUnresolvableSubjects`, which the boot run never sets. `backfillEventPiiEncryptionBatch` is the new single-batch primitive both build on. `runPendingRebuilds` (the schema-apply queue drain) now also rebuilds queued multi-stream projection tables instead of failing on them; a failed multi-stream rebuild in either path stays queued and leaves the live consumer running."
13
+ },
14
+ {
15
+ "version": "0.328.0",
16
+ "type": "fix",
17
+ "title": "The projection-rebuild job also rebuilds multi-stream projections",
18
+ "detail": "`enqueueProjectionRebuild` and the `jobs:job:projection-rebuild` worker used to call the single-stream rebuild only, so a multi-stream projection name failed with \"not registered\". Both now go through `rebuildProjectionOrMultiStream` (exported from `@cosmicdrift/kumiko-framework/migrations`), the same helper the pending-rebuild queue uses. For multi-stream projections the skip mode comes from the projection's declared `errorMode`; the job payload's `skipApplyErrors` only applies to single-stream projections."
19
+ },
20
+ {
21
+ "version": "0.328.0",
22
+ "type": "improvement",
23
+ "title": "SearchAdapter gains an optional dropAllIndexes() for rebuilds on a persistent index",
24
+ "detail": "`dropAllIndexes()` deletes every tenant index the adapter owns and returns the count. The in-memory adapter clears all tenants; the Meilisearch adapter deletes all indexes under its prefix (and refuses an empty prefix) and resets its configured-tenant state so the next write recreates a filterable index."
25
+ },
26
+ {
27
+ "version": "0.328.0",
28
+ "type": "improvement",
29
+ "title": "setupTestStack can wire job run-logger callbacks, and setupTestStackFromFeatures does so by default",
30
+ "detail": "`TestStackOptions.jobs.runLogger` receives `{ registry, db }` and returns `onJobStart`/`onJobComplete`/`onJobFailed`. The stack awaits them before drainJobs() observes the outcome, so a log row is always written before drainJobs() resolves or rejects. `setupTestStackFromFeatures` (and with it `setupAppTestStack`) defaults to the same `jobRunLoggerCallbacks` prod uses whenever `jobs` is set and the jobs feature is mounted, so `tenantVisibleFailure` rows and `jobs:query:failures` work in tests without a hand-built harness. Such suites now write `job_runs` and tenant failure rows for every job run; a suite that counts those rows, or that wants the old behavior, passes its own `jobs.runLogger` (for example one returning `undefined`)."
31
+ },
32
+ {
33
+ "version": "0.328.0",
34
+ "type": "improvement",
35
+ "title": "build-app-schema carries optionTones and collects the new screen i18n keys",
36
+ "detail": "Select `optionTones` reach the client schema. `createLabel`, `searchPlaceholder`, `summary`, related list `description` and `itemNoun` are collected as i18n keys of the screen, so they are translated like other screen text."
37
+ },
38
+ {
39
+ "version": "0.327.0",
40
+ "type": "improvement",
41
+ "title": "r.webSocketRoute for authenticated WebSocket routes under /api/ws/",
42
+ "detail": "renderer-web gains `FloatingPanel` (movable, resizable, non-modal panel with persisted geometry and a full-screen sheet on narrow viewports) and exports `useIsNarrowViewport`. Features can declare `r.webSocketRoute` under `/api/ws/` with session auth, an Origin check (allowlist, or same host without one), a per-route message cap, backpressure protection (4 MiB, the socket is closed beyond it) and a per-user connection cap (`maxConnectionsPerUser`, default 5, per server process; over it the upgrade gets 429). Handlers run one after another in arrival order per socket; on close `onClose` runs immediately (not queued behind a hung handler), queued messages never start, and `connection.signal` aborts. A 25 s heartbeat revalidates session, roles, tenant lifecycle and the token's own expiry (close 1008); a session store that keeps failing closes the socket with 1013 after three failed checks in a row. A server without upgrade wiring answers 501 `websocket_upgrade_not_wired` and logs the fix. `buildBunServeOptions` takes an optional `{ upgradeFetch, heartbeatIntervalMs? }` object as 4th argument and `runProdApp` handles expose `webSocketUpgradeFetch`; the dev server wires it. Upgrade rejections carry the same security headers as other responses. Additive, no migration."
43
+ },
44
+ {
45
+ "version": "0.327.0",
46
+ "type": "fix",
47
+ "title": "JobRunner.stop() waits for queue readiness before closing",
48
+ "detail": "JobRunner.stop() waits (bounded by the boot Redis timeout) for its queues to be ready before closing them. Stopping a runner right after construction no longer leaves an unhandled \"Connection is closed.\" rejection behind."
49
+ },
50
+ {
51
+ "version": "0.327.0",
52
+ "type": "fix",
53
+ "title": "updateMany drops unknown columns, so replaying an update event for a removed field no longer fails the rebuild",
54
+ "detail": "`updateMany` now ignores keys that have no column on the target table, matching `insertOne`/`insertMany`. A historical `<entity>.updated` event that still carries a field removed from the entity no longer aborts a projection rebuild with \"column ... does not exist\"."
55
+ },
2
56
  {
3
57
  "version": "0.325.0",
4
58
  "type": "improvement",
@@ -0,0 +1,77 @@
1
+ // System events bypass the defineEvent catalog, so their PII stance lives in
2
+ // a static map. An undeclared system type must fail closed (also without a
3
+ // KMS), and every known system type must have a declared stance.
4
+
5
+ import { afterEach, describe, expect, test } from "bun:test";
6
+ import * as stepDispatchConstants from "../../engine/steps/_step-dispatch-constants";
7
+ import type { TenantId } from "../../engine/types/identifiers";
8
+ import { encryptEventPayloadPii } from "../event-pii";
9
+ import { InMemoryKmsAdapter } from "../in-memory-kms-adapter";
10
+ import {
11
+ configurePiiSubjectKms,
12
+ isPiiCiphertext,
13
+ resetPiiSubjectKmsForTests,
14
+ } from "../pii-field-encryption";
15
+ import type { EventSubjectEnvelope } from "../subject-resolver";
16
+ import {
17
+ AGGREGATE_TRANSFERRED_EVENT_TYPE,
18
+ SYSTEM_EVENT_PII_STANCES,
19
+ SYSTEM_EVENT_PREFIX,
20
+ } from "../system-event-pii";
21
+
22
+ const ENVELOPE: EventSubjectEnvelope = {
23
+ tenantId: "6b2f4a0e-1c9d-4f3a-9d2e-0000000000e1" as TenantId,
24
+ aggregateType: "step-dispatch",
25
+ aggregateId: "6b2f4a0e-1c9d-4f3a-9d2e-0000000000e2",
26
+ };
27
+
28
+ afterEach(() => {
29
+ resetPiiSubjectKmsForTests();
30
+ });
31
+
32
+ describe("system event PII stances", () => {
33
+ test("every system event type constant has a declared stance", () => {
34
+ const declaredTypes = [
35
+ ...Object.entries(stepDispatchConstants)
36
+ .filter(([name]) => name.endsWith("_TYPE"))
37
+ .map(([, value]: [string, unknown]) => value)
38
+ .filter(
39
+ (value): value is string =>
40
+ typeof value === "string" && value.startsWith(SYSTEM_EVENT_PREFIX),
41
+ ),
42
+ AGGREGATE_TRANSFERRED_EVENT_TYPE,
43
+ ];
44
+ expect(declaredTypes.length).toBeGreaterThanOrEqual(11);
45
+ for (const type of declaredTypes) {
46
+ expect(SYSTEM_EVENT_PII_STANCES.has(type)).toBe(true);
47
+ }
48
+ });
49
+
50
+ test("an undeclared system type fails closed, even without a configured KMS", async () => {
51
+ await expect(
52
+ encryptEventPayloadPii(`${SYSTEM_EVENT_PREFIX}made.up`, { secret: "x" }, ENVELOPE),
53
+ ).rejects.toThrow(/System event "kumiko:system:made\.up" has no PII stance/);
54
+ });
55
+
56
+ test('a "none" stance passes the payload through untouched', async () => {
57
+ configurePiiSubjectKms(new InMemoryKmsAdapter());
58
+ const payload = { stepKind: "mail.send", status: 202 };
59
+ expect(
60
+ await encryptEventPayloadPii(stepDispatchConstants.STEP_DISPATCHED_TYPE, payload, ENVELOPE),
61
+ ).toBe(payload);
62
+ });
63
+
64
+ test("dispatch-requested encrypts its string fields under the per-dispatch record key", async () => {
65
+ configurePiiSubjectKms(new InMemoryKmsAdapter());
66
+ const out = await encryptEventPayloadPii(
67
+ stepDispatchConstants.STEP_DISPATCH_REQUESTED_TYPE,
68
+ { stepKind: "mail.send", to: '"a@example.com"', subject: "Hi", body: "Body" },
69
+ ENVELOPE,
70
+ );
71
+ expect(out["stepKind"]).toBe("mail.send");
72
+ for (const field of ["to", "subject", "body"]) {
73
+ expect(isPiiCiphertext(out[field])).toBe(true);
74
+ expect(String(out[field])).toContain(`record:step-dispatch:${ENVELOPE.aggregateId}`);
75
+ }
76
+ });
77
+ });
@@ -4,7 +4,9 @@
4
4
  // path at all. `r.defineEvent(name, schema, { piiFields })` declares which
5
5
  // payload fields are PII and which payload field names the owning user;
6
6
  // createRegistry publishes the catalog and append() — the single write funnel
7
- // into kumiko_events — encrypts every catalogued field.
7
+ // into kumiko_events — encrypts every catalogued field. `kumiko:system:*`
8
+ // events skip the registry and take their stance from system-event-pii.ts
9
+ // instead (fail-closed when missing).
8
10
  //
9
11
  // `piiFields` is a mandatory, explicit stance (fw#2558): a payload can only
10
12
  // go uncatalogued through a declared `piiFields: "none"`, never by omission —
@@ -12,10 +14,11 @@
12
14
  // silent no-op (forgetting the option) is now either an explicit "none" or a
13
15
  // missing subject KMS at encrypt-time, both visible states, not a gap.
14
16
 
15
- import type { EventPiiFields } from "@cosmicdrift/kumiko-types/handlers";
17
+ import type { EventPiiFields, EventPiiStance } from "@cosmicdrift/kumiko-types/handlers";
16
18
  import { requestContext } from "../api/request-context";
17
19
  import { configuredPiiSubjectKms, encryptPiiValueForSubject } from "./pii-field-encryption";
18
20
  import { type EventSubjectEnvelope, resolveEventSubject } from "./subject-resolver";
21
+ import { SYSTEM_EVENT_PII_STANCES, SYSTEM_EVENT_PREFIX } from "./system-event-pii";
19
22
 
20
23
  export type EventPiiCatalog = ReadonlyMap<string, EventPiiFields>;
21
24
 
@@ -36,6 +39,19 @@ export function resetEventPiiCatalogForTests(): void {
36
39
  catalog = new Map();
37
40
  }
38
41
 
42
+ // System types are checked before the KMS gate: an undeclared one throws
43
+ // even in plaintext rollout mode.
44
+ function resolvePiiStance(eventType: string): EventPiiStance | undefined {
45
+ if (!eventType.startsWith(SYSTEM_EVENT_PREFIX)) return catalog.get(eventType);
46
+ const stance = SYSTEM_EVENT_PII_STANCES.get(eventType);
47
+ if (stance === undefined) {
48
+ throw new Error(
49
+ `System event "${eventType}" has no PII stance — declare one in SYSTEM_EVENT_PII_STANCES (crypto/system-event-pii.ts)`,
50
+ );
51
+ }
52
+ return stance;
53
+ }
54
+
39
55
  // Encrypts catalogued payload fields under the declared subject's DEK
40
56
  // (user/tenant/self — resolveEventSubject, fw#2801). No-op when the event
41
57
  // type is uncatalogued or no subject KMS is configured (plaintext rollout
@@ -51,8 +67,8 @@ export async function encryptEventPayloadPii(
51
67
  payload: Record<string, unknown>,
52
68
  envelope: EventSubjectEnvelope,
53
69
  ): Promise<Record<string, unknown>> {
54
- const piiFields = catalog.get(eventType);
55
- if (!piiFields) return payload;
70
+ const piiFields = resolvePiiStance(eventType);
71
+ if (!piiFields || piiFields === "none") return payload;
56
72
  const kms = configuredPiiSubjectKms();
57
73
  if (!kms) return payload;
58
74
 
@@ -0,0 +1,57 @@
1
+ // PII stance of every `kumiko:system:*` event. System events bypass the
2
+ // registry (append-event-core), so they never reach the defineEvent catalog;
3
+ // this static map is their catalog. encryptEventPayloadPii fails closed on a
4
+ // system type with no entry here, so a new system event cannot ship its
5
+ // payload as plaintext by omission.
6
+
7
+ import type { EventPiiStance } from "@cosmicdrift/kumiko-types/handlers";
8
+ import {
9
+ STEP_DISPATCH_FAILED_TYPE,
10
+ STEP_DISPATCH_REQUESTED_TYPE,
11
+ STEP_DISPATCHED_TYPE,
12
+ WORKFLOW_RESUMED_TYPE,
13
+ WORKFLOW_RETRY_SCHEDULED_TYPE,
14
+ WORKFLOW_RUN_COMPLETED_TYPE,
15
+ WORKFLOW_RUN_FAILED_TYPE,
16
+ WORKFLOW_RUN_STARTED_TYPE,
17
+ WORKFLOW_WAITING_FOR_EVENT_TYPE,
18
+ WORKFLOW_WAITING_TYPE,
19
+ } from "../engine/steps/_step-dispatch-constants";
20
+
21
+ export const SYSTEM_EVENT_PREFIX = "kumiko:system:";
22
+
23
+ // Lives here (not in event-store/transfer.ts) so this module stays free of
24
+ // an event-store import cycle; transfer.ts re-exports it.
25
+ export const AGGREGATE_TRANSFERRED_EVENT_TYPE = `${SYSTEM_EVENT_PREFIX}aggregate.transferred`;
26
+
27
+ const SELF = { personal: "self" } as const;
28
+
29
+ // Workflow payloads carry `triggerPayload`, the stored payload of the trigger
30
+ // event: catalogued fields there are already ciphertext, so "none" leaks nothing.
31
+ export const SYSTEM_EVENT_PII_STANCES: ReadonlyMap<string, EventPiiStance> = new Map<
32
+ string,
33
+ EventPiiStance
34
+ >([
35
+ [
36
+ STEP_DISPATCH_REQUESTED_TYPE,
37
+ {
38
+ to: SELF,
39
+ subject: SELF,
40
+ body: SELF,
41
+ from: SELF,
42
+ url: SELF,
43
+ headersJson: SELF,
44
+ bodyJson: SELF,
45
+ },
46
+ ],
47
+ [STEP_DISPATCHED_TYPE, "none"],
48
+ [STEP_DISPATCH_FAILED_TYPE, "none"],
49
+ [AGGREGATE_TRANSFERRED_EVENT_TYPE, "none"],
50
+ [WORKFLOW_WAITING_TYPE, "none"],
51
+ [WORKFLOW_WAITING_FOR_EVENT_TYPE, "none"],
52
+ [WORKFLOW_RESUMED_TYPE, "none"],
53
+ [WORKFLOW_RUN_STARTED_TYPE, "none"],
54
+ [WORKFLOW_RUN_COMPLETED_TYPE, "none"],
55
+ [WORKFLOW_RUN_FAILED_TYPE, "none"],
56
+ [WORKFLOW_RETRY_SCHEDULED_TYPE, "none"],
57
+ ]);