@cosmicdrift/kumiko-framework 0.269.2 → 0.273.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +5 -1
  2. package/package.json +3 -3
  3. package/src/__tests__/anonymous-access.integration.test.ts +1 -1
  4. package/src/__tests__/error-contract.integration.test.ts +1 -1
  5. package/src/__tests__/fixtures/migrate-cross-tenant/manual-only.expected.ts +44 -0
  6. package/src/__tests__/fixtures/migrate-cross-tenant/manual-only.input.ts +44 -0
  7. package/src/__tests__/fixtures/migrate-cross-tenant/rewrite.expected.ts +41 -0
  8. package/src/__tests__/fixtures/migrate-cross-tenant/rewrite.input.ts +41 -0
  9. package/src/__tests__/fixtures/migrate-open-to-all/manual-only.expected.ts +16 -0
  10. package/src/__tests__/fixtures/migrate-open-to-all/manual-only.input.ts +16 -0
  11. package/src/__tests__/fixtures/migrate-open-to-all/test-nested-shorthand.expected.ts +12 -0
  12. package/src/__tests__/fixtures/migrate-open-to-all/test-nested-shorthand.input.ts +12 -0
  13. package/src/__tests__/fixtures/migrate-open-to-all/test-simple.expected.ts +7 -0
  14. package/src/__tests__/fixtures/migrate-open-to-all/test-simple.input.ts +7 -0
  15. package/src/__tests__/full-stack.integration.test.ts +2 -2
  16. package/src/__tests__/migrate-cross-tenant-codemod.test.ts +106 -0
  17. package/src/__tests__/migrate-open-to-all-codemod.test.ts +84 -0
  18. package/src/__tests__/related-list-parent-filter.integration.test.ts +103 -0
  19. package/src/__tests__/transition-guard.integration.test.ts +7 -7
  20. package/src/api/__tests__/api.test.ts +2 -2
  21. package/src/api/__tests__/body-limit.test.ts +1 -1
  22. package/src/api/__tests__/nested-write-ownership.integration.test.ts +268 -0
  23. package/src/api/__tests__/pii-leak-guard.integration.test.ts +2 -2
  24. package/src/api/__tests__/request-locale.integration.test.ts +1 -1
  25. package/src/api/__tests__/server-error-logging.test.ts +1 -1
  26. package/src/api/server.ts +7 -20
  27. package/src/bun-db/__tests__/query-guards.test.ts +60 -3
  28. package/src/bun-db/query.ts +24 -21
  29. package/src/changes.json +99 -0
  30. package/src/db/__tests__/event-store-executor-write-verbs.integration.test.ts +77 -0
  31. package/src/db/__tests__/event-store-executor.integration.test.ts +1 -1
  32. package/src/db/__tests__/list-filter-field-access.integration.test.ts +1 -1
  33. package/src/db/__tests__/replay-migration-sql.test.ts +17 -0
  34. package/src/db/__tests__/tenant-db-convention-cross-tenant.test.ts +72 -0
  35. package/src/db/__tests__/tenant-db-declared-unsafe-raw.test.ts +104 -0
  36. package/src/db/__tests__/tenant-db-global.test.ts +92 -0
  37. package/src/db/__tests__/tenant-db-no-raw.test.ts +75 -0
  38. package/src/db/__tests__/tenant-db.integration.test.ts +3 -1
  39. package/src/db/__tests__/unchecked-system-db.test.ts +76 -0
  40. package/src/db/apply-entity-event.ts +1 -1
  41. package/src/db/assert-exists-in.ts +1 -1
  42. package/src/db/event-store-executor-context.ts +5 -2
  43. package/src/db/event-store-executor-read.ts +6 -4
  44. package/src/db/event-store-executor-write.ts +24 -18
  45. package/src/db/queries/shadow-swap.ts +36 -0
  46. package/src/db/replay-migration-sql.ts +5 -5
  47. package/src/db/tenant-db-runner.ts +23 -0
  48. package/src/db/tenant-db.ts +97 -24
  49. package/src/derivatives/__tests__/derivatives-context.integration.test.ts +2 -2
  50. package/src/derivatives/__tests__/derivatives-context.test.ts +0 -1
  51. package/src/engine/__tests__/boot-validator-dashboard.test.ts +5 -5
  52. package/src/engine/__tests__/boot-validator-nav-area.test.ts +116 -0
  53. package/src/engine/__tests__/boot-validator-pii-retention.test.ts +1 -1
  54. package/src/engine/__tests__/boot-validator-projection-list.test.ts +108 -17
  55. package/src/engine/__tests__/boot-validator-query-output-schema.test.ts +161 -18
  56. package/src/engine/__tests__/boot-validator-query-refs.test.ts +3 -3
  57. package/src/engine/__tests__/boot-validator-s0-integration.test.ts +2 -2
  58. package/src/engine/__tests__/boot-validator.test.ts +69 -18
  59. package/src/engine/__tests__/build-app-schema.test.ts +10 -6
  60. package/src/engine/__tests__/build-config-feature-schema-secrets.test.ts +48 -0
  61. package/src/engine/__tests__/build-config-feature-schema.test.ts +34 -5
  62. package/src/engine/__tests__/engine.test.ts +39 -25
  63. package/src/engine/__tests__/entity-create-caller-id.integration.test.ts +87 -0
  64. package/src/engine/__tests__/entity-handlers-escape-hatch.test.ts +91 -0
  65. package/src/engine/__tests__/entity-handlers.test.ts +47 -8
  66. package/src/engine/__tests__/entity-presave-wiring.integration.test.ts +2 -2
  67. package/src/engine/__tests__/entity-restore-tenant-isolation.integration.test.ts +30 -5
  68. package/src/engine/__tests__/entity-write-crosstenant.integration.test.ts +206 -47
  69. package/src/engine/__tests__/extends-registrar.test.ts +3 -3
  70. package/src/engine/__tests__/feature-crud-shorthand.test.ts +9 -3
  71. package/src/engine/__tests__/feature-crud-verb-access.integration.test.ts +1 -1
  72. package/src/engine/__tests__/hook-phases.test.ts +5 -5
  73. package/src/engine/__tests__/lifecycle-hooks.test.ts +9 -3
  74. package/src/engine/__tests__/multiselect-filter.integration.test.ts +1 -1
  75. package/src/engine/__tests__/post-query-hook.test.ts +6 -2
  76. package/src/engine/__tests__/projection-detail-actions.test.ts +2 -2
  77. package/src/engine/__tests__/projection-detail-singleton.test.ts +3 -3
  78. package/src/engine/__tests__/projection-detail-tabs.test.ts +3 -3
  79. package/src/engine/__tests__/projection-list-filter.integration.test.ts +1 -1
  80. package/src/engine/__tests__/read-steps-tenant-filter.integration.test.ts +215 -0
  81. package/src/engine/__tests__/registry.test.ts +19 -13
  82. package/src/engine/__tests__/required-surface-keys.test.ts +16 -0
  83. package/src/engine/__tests__/screen.test.ts +176 -20
  84. package/src/engine/__tests__/soft-delete-cleanup.test.ts +0 -1
  85. package/src/engine/__tests__/steps-read.test.ts +204 -8
  86. package/src/engine/__tests__/steps-unsafe-projection-delete.test.ts +7 -4
  87. package/src/engine/__tests__/steps-unsafe-projection-upsert.test.ts +6 -3
  88. package/src/engine/boot-validator/__tests__/access-declarations.test.ts +5 -4
  89. package/src/engine/boot-validator/__tests__/global-tenancy.test.ts +16 -1
  90. package/src/engine/boot-validator/__tests__/parent-ref.test.ts +18 -6
  91. package/src/engine/boot-validator/__tests__/rate-limit-disabled-reason.test.ts +45 -0
  92. package/src/engine/boot-validator/access-declarations.ts +3 -5
  93. package/src/engine/boot-validator/entity-handler.ts +21 -0
  94. package/src/engine/boot-validator/global-tenancy.ts +13 -1
  95. package/src/engine/boot-validator/nav.ts +4 -3
  96. package/src/engine/boot-validator/parent-ref.ts +1 -1
  97. package/src/engine/boot-validator/projection-list-screens.ts +67 -2
  98. package/src/engine/boot-validator/query-output-columns.ts +33 -0
  99. package/src/engine/boot-validator/screens.ts +154 -31
  100. package/src/engine/build-config-feature-schema.ts +12 -2
  101. package/src/engine/entity-handlers.ts +72 -20
  102. package/src/engine/feature-ast/__tests__/canonical-form.test.ts +3 -1
  103. package/src/engine/feature-ast/__tests__/parse-happy-path.test.ts +2 -2
  104. package/src/engine/feature-ast/__tests__/parse.test.ts +106 -4
  105. package/src/engine/feature-ast/__tests__/patcher.test.ts +3 -3
  106. package/src/engine/feature-ast/__tests__/read-optional-access-rule.test.ts +4 -4
  107. package/src/engine/feature-ast/__tests__/render-roundtrip.test.ts +5 -1
  108. package/src/engine/feature-ast/extractors/handlers.ts +16 -2
  109. package/src/engine/feature-ast/extractors/hooks.ts +25 -4
  110. package/src/engine/feature-ast/patcher.ts +15 -3
  111. package/src/engine/feature-ast/patterns.ts +4 -0
  112. package/src/engine/feature-ast/render.ts +3 -0
  113. package/src/engine/feature-entity-handlers.ts +4 -4
  114. package/src/engine/index.ts +5 -0
  115. package/src/engine/pattern-library/shared-fields.ts +7 -3
  116. package/src/engine/registry-validate.ts +12 -4
  117. package/src/engine/schema-builder.ts +6 -0
  118. package/src/engine/screen-helpers.ts +52 -0
  119. package/src/engine/steps/_read-source.ts +12 -0
  120. package/src/engine/steps/read-find-many.ts +8 -2
  121. package/src/engine/steps/read-find-one.ts +11 -8
  122. package/src/engine/steps/unsafe-projection-delete.ts +2 -1
  123. package/src/engine/steps/unsafe-projection-upsert.ts +2 -1
  124. package/src/engine/types/index.ts +8 -1
  125. package/src/entrypoint/__tests__/entrypoint-attach-dispatcher.integration.test.ts +1 -1
  126. package/src/entrypoint/__tests__/entrypoint-job-wiring.integration.test.ts +3 -3
  127. package/src/entrypoint/__tests__/split-deploy.integration.test.ts +1 -1
  128. package/src/errors/i18n/de.yaml +14 -0
  129. package/src/errors/i18n/en.yaml +13 -0
  130. package/src/errors/reasons.ts +4 -0
  131. package/src/es-ops/__tests__/runner.integration.test.ts +2 -2
  132. package/src/i18n/required-surface-keys.ts +6 -2
  133. package/src/jobs/__tests__/job-event-trigger.integration.test.ts +2 -2
  134. package/src/jobs/__tests__/job-multi-trigger.integration.test.ts +2 -2
  135. package/src/jobs/__tests__/job-systemdb.integration.test.ts +5 -4
  136. package/src/jobs/__tests__/jobs.integration.test.ts +42 -1
  137. package/src/jobs/job-runner.ts +35 -5
  138. package/src/observability/__tests__/observability.integration.test.ts +7 -7
  139. package/src/pipeline/__tests__/archive-stream.integration.test.ts +2 -2
  140. package/src/pipeline/__tests__/dispatcher.test.ts +15 -15
  141. package/src/pipeline/__tests__/escape-hatch-audit.integration.test.ts +254 -0
  142. package/src/pipeline/__tests__/escape-hatch-report.test.ts +149 -0
  143. package/src/pipeline/__tests__/lifecycle-pipeline.test.ts +10 -10
  144. package/src/pipeline/__tests__/load-aggregate-query.integration.test.ts +1 -1
  145. package/src/pipeline/__tests__/msp-rebuild.integration.test.ts +28 -0
  146. package/src/pipeline/__tests__/post-query-hook.integration.test.ts +3 -3
  147. package/src/pipeline/__tests__/projection-rebuild.integration.test.ts +94 -0
  148. package/src/pipeline/__tests__/projections-runner-systemscope.integration.test.ts +101 -0
  149. package/src/pipeline/__tests__/query-projection.integration.test.ts +172 -17
  150. package/src/pipeline/__tests__/rate-limit-default.integration.test.ts +118 -0
  151. package/src/pipeline/__tests__/rate-limit-default.test.ts +117 -0
  152. package/src/pipeline/__tests__/search-consumer-named-events.integration.test.ts +5 -2
  153. package/src/pipeline/__tests__/system-identity-switch.test.ts +240 -1
  154. package/src/pipeline/dispatch-query.ts +27 -8
  155. package/src/pipeline/dispatch-shared.ts +141 -57
  156. package/src/pipeline/dispatch-stream.ts +1 -3
  157. package/src/pipeline/dispatch-write.ts +62 -14
  158. package/src/pipeline/dispatcher.ts +20 -0
  159. package/src/pipeline/escape-hatch-report.ts +127 -0
  160. package/src/pipeline/index.ts +8 -0
  161. package/src/pipeline/member-read-only-transaction.ts +75 -0
  162. package/src/pipeline/msp-rebuild.ts +2 -0
  163. package/src/pipeline/projection-rebuild.ts +2 -0
  164. package/src/pipeline/projections-runner.ts +15 -13
  165. package/src/pipeline/system-identity-switch.ts +121 -9
  166. package/src/scripts/codemod/migrate-cross-tenant.ts +335 -0
  167. package/src/scripts/codemod/migrate-db-raw.ts +19 -6
  168. package/src/scripts/codemod/migrate-open-to-all.ts +249 -0
  169. package/src/secrets/__tests__/contains-secret.test.ts +5 -5
  170. package/src/stack/__tests__/request-helper.integration.test.ts +3 -3
  171. package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +1 -1
  172. package/src/testing/__tests__/e2e-generator.test.ts +19 -5
  173. package/src/ui-types/index.ts +3 -0
  174. package/src/utils/__tests__/ids.test.ts +29 -1
  175. package/src/utils/ids.ts +10 -1
  176. package/src/utils/index.ts +1 -1
@@ -27,13 +27,13 @@ const orderFeature = defineFeature("multi", (r) => {
27
27
  "order:open",
28
28
  z.object({ id: z.string() }),
29
29
  async (event) => ({ isSuccess: true, data: { id: event.payload.id } }),
30
- { access: { openToAll: true } },
30
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
31
31
  );
32
32
  r.writeHandler(
33
33
  "order:cancel",
34
34
  z.object({ id: z.string() }),
35
35
  async (event) => ({ isSuccess: true, data: { id: event.payload.id } }),
36
- { access: { openToAll: true } },
36
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
37
37
  );
38
38
  r.job(
39
39
  "fanout",
@@ -1,5 +1,6 @@
1
1
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2
2
  import { type BunTestDb, createTestDb } from "../../bun-db/__tests__/bun-test-db";
3
+ import { tenantDbRunner } from "../../db/tenant-db-runner";
3
4
  import { createRegistry, defineFeature, type Registry } from "../../engine";
4
5
  import { createTestRedis, type TestRedis, testTenantId } from "../../stack";
5
6
  import { waitFor } from "../../testing";
@@ -12,9 +13,9 @@ import { createJobRunner, type JobRunner } from "../job-runner";
12
13
  type JobRunResult = {
13
14
  readonly name: "system" | "tenant" | "system-per-tenant";
14
15
  readonly present: boolean;
15
- // assertTenantMatch() must return a TenantDb whose `.raw` is the SAME
16
- // underlying DbConnection ctx.db carries — proves systemDb is bound to
17
- // the job's own tenant-scoped db, not a separate instance.
16
+ // assertTenantMatch() must return a TenantDb bound (via tenantDbRunner) to
17
+ // the SAME underlying DbConnection ctx.db carries — proves systemDb is
18
+ // bound to the job's own tenant-scoped db, not a separate instance.
18
19
  readonly boundToRawDb: boolean | undefined;
19
20
  // A foreign tenantId must throw fail-closed (AccessDeniedError), same as
20
21
  // the HandlerContext.systemDb self-check.
@@ -48,7 +49,7 @@ const systemScopedFeature = defineFeature("jobsystemdb-system", (r) => {
48
49
  results.push({
49
50
  name: "system",
50
51
  present: true,
51
- boundToRawDb: checked.raw === ctx.db,
52
+ boundToRawDb: tenantDbRunner(checked) === ctx.db,
52
53
  foreignTenantThrew,
53
54
  });
54
55
  });
@@ -31,6 +31,10 @@ let redisUrl: string;
31
31
  // Track which jobs ran and when
32
32
  const jobLog: Array<{ name: string; payload: Record<string, unknown>; timestamp: number }> = [];
33
33
 
34
+ // Counts eventRetrySucceeds' own attempts — throws once on attempt 1, then
35
+ // succeeds, so a real retry must have happened for the job to ever reach 2.
36
+ let eventRetryAttemptCount = 0;
37
+
34
38
  function clearLog() {
35
39
  jobLog.length = 0;
36
40
  }
@@ -119,7 +123,7 @@ const testFeature = defineFeature("test", (r) => {
119
123
  "capped-event",
120
124
  z.object({ n: z.number() }),
121
125
  async (event) => ({ isSuccess: true, data: { n: event.payload.n } }),
122
- { access: { openToAll: true } },
126
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
123
127
  );
124
128
  // Event-triggered twin — exercises handleEvent's maxPerTenant guard
125
129
  // (dispatch and handleEvent share isOverPerTenantLimit).
@@ -140,6 +144,25 @@ const testFeature = defineFeature("test", (r) => {
140
144
  },
141
145
  );
142
146
 
147
+ r.writeHandler(
148
+ "retry-event-trigger",
149
+ z.object({ n: z.number() }),
150
+ async (event) => ({ isSuccess: true, data: { n: event.payload.n } }),
151
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
152
+ );
153
+ // Event-triggered job with retries — proves handleEvent passes
154
+ // retries/backoff to the queue the same way dispatch() does (fw
155
+ // job-runner retries regression, solon).
156
+ r.job(
157
+ "eventRetrySucceeds",
158
+ { trigger: { on: "test:write:retry-event-trigger" }, retries: 2 },
159
+ async (payload) => {
160
+ eventRetryAttemptCount += 1;
161
+ jobLog.push({ name: "test:job:event-retry-succeeds", payload, timestamp: Date.now() });
162
+ if (eventRetryAttemptCount === 1) throw new Error("fails on first attempt");
163
+ },
164
+ );
165
+
143
166
  // Job that fails
144
167
  r.job("failingJob", { trigger: { manual: true }, retries: 1 }, async () => {
145
168
  throw new Error("intentional failure");
@@ -1197,6 +1220,24 @@ describe("handleEvent maxPerTenant", () => {
1197
1220
  });
1198
1221
  });
1199
1222
 
1223
+ describe("handleEvent retries", () => {
1224
+ test("an event-triggered job with retries retries on failure and eventually succeeds", async () => {
1225
+ clearLog();
1226
+ eventRetryAttemptCount = 0;
1227
+ await withRunner(async (runner) => {
1228
+ const user = { ...TestUsers.admin, tenantId: "retry-tenant-evt" };
1229
+ await runner.handleEvent("test:write:retry-event-trigger", { n: 1 }, user);
1230
+ // Without retries/backoff reaching the queue, the job fails on
1231
+ // attempt 1 and BullMQ never retries — this would time out.
1232
+ await waitFor(() => {
1233
+ const entries = jobLog.filter((e) => e.name === "test:job:event-retry-succeeds");
1234
+ expect(entries.length).toBe(2);
1235
+ });
1236
+ });
1237
+ expect(eventRetryAttemptCount).toBe(2);
1238
+ });
1239
+ });
1240
+
1200
1241
  // --- Registry ---
1201
1242
 
1202
1243
  describe("job registry", () => {
@@ -9,6 +9,7 @@ import {
9
9
  type AppContext,
10
10
  type DispatchWriteRef,
11
11
  type JobContext,
12
+ type JobDefinition,
12
13
  type JobRunIn,
13
14
  type MemberReader,
14
15
  type Registry,
@@ -27,6 +28,7 @@ import {
27
28
  type Tracer,
28
29
  } from "../observability";
29
30
  import { createDistributedLock, type DistributedLock } from "../pipeline/distributed-lock";
31
+ import { createEscapeHatchReporter } from "../pipeline/escape-hatch-report";
30
32
  import { RedisKeys } from "../pipeline/redis-keys";
31
33
  import { bridgeStub } from "../testing/handler-context";
32
34
 
@@ -251,6 +253,16 @@ function timeoutReject(
251
253
  };
252
254
  }
253
255
 
256
+ // Shared by dispatch() and handleEvent() — an event-triggered job must retry
257
+ // on failure the same way a directly-dispatched one does; a duplicated
258
+ // inline computation in handleEvent previously dropped both options.
259
+ function buildRetryBullOpts(jobDef: JobDefinition): Record<string, unknown> {
260
+ const opts: Record<string, unknown> = {};
261
+ if (jobDef.retries !== undefined) opts["attempts"] = jobDef.retries + 1;
262
+ if (jobDef.backoff) opts["backoff"] = { type: jobDef.backoff };
263
+ return opts;
264
+ }
265
+
254
266
  export function createJobRunner(options: JobRunnerOptions): JobRunner {
255
267
  const { registry, context, redisUrl, consumerLane } = options;
256
268
  const queueNamePrefix = options.queueNamePrefix ?? DEFAULT_QUEUE_NAME_PREFIX;
@@ -282,12 +294,19 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
282
294
  return async () => {
283
295
  const systemUser = createSystemUser(SYSTEM_TENANT_ID);
284
296
  const systemModeDb = createTenantDb(db, SYSTEM_TENANT_ID, "system");
297
+ const reportEscapeHatch = createEscapeHatchReporter({
298
+ handler: ACTIVE_TENANT_IDS_QUERY_NAME,
299
+ tenantId: SYSTEM_TENANT_ID,
300
+ actor: systemUser.id,
301
+ sink: context._escapeHatchAuditSink,
302
+ log: context.log,
303
+ });
285
304
  const result = await handler.handler(
286
305
  { type: ACTIVE_TENANT_IDS_QUERY_NAME, payload: {}, user: systemUser },
287
306
  {
288
307
  db: systemModeDb,
289
308
  dbOutsideTransaction: systemModeDb,
290
- systemDb: createUncheckedSystemDb(systemModeDb),
309
+ systemDb: createUncheckedSystemDb(systemModeDb, undefined, reportEscapeHatch),
291
310
  registry,
292
311
  ...bridgeStub(),
293
312
  },
@@ -533,7 +552,19 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
533
552
  const tenantScopedDb = configDb ? createTenantDb(configDb, tenantId, "system") : undefined;
534
553
  const isSystemJob = registry.isJobSystemScoped(jobName);
535
554
  const systemDb =
536
- isSystemJob && tenantScopedDb ? createUncheckedSystemDb(tenantScopedDb) : undefined;
555
+ isSystemJob && tenantScopedDb
556
+ ? createUncheckedSystemDb(
557
+ tenantScopedDb,
558
+ undefined,
559
+ createEscapeHatchReporter({
560
+ handler: jobName,
561
+ tenantId,
562
+ actor: jobSystemUser.id,
563
+ sink: context._escapeHatchAuditSink,
564
+ log: context.log,
565
+ }),
566
+ )
567
+ : undefined;
537
568
  const config =
538
569
  context._configAccessorFactory && tenantScopedDb
539
570
  ? context._configAccessorFactory({
@@ -883,8 +914,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
883
914
  break;
884
915
  }
885
916
 
886
- if (jobDef.retries !== undefined) bullOpts["attempts"] = jobDef.retries + 1;
887
- if (jobDef.backoff) bullOpts["backoff"] = { type: jobDef.backoff };
917
+ Object.assign(bullOpts, buildRetryBullOpts(jobDef));
888
918
  if (jobDef.timeout) bullOpts["timeout"] = jobDef.timeout;
889
919
  if (meta?.priority !== undefined) bullOpts["priority"] = meta.priority;
890
920
 
@@ -948,7 +978,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
948
978
  }
949
979
  // Route to the job's declared lane, not a fixed queue — that's
950
980
  // the whole reason both queues are held.
951
- await queues[laneForJob(jobDef)].add(name, data);
981
+ await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
952
982
  }
953
983
  },
954
984
  attachDispatcher(ref: DispatchWriteRef): void {
@@ -33,7 +33,7 @@ const productFeature = defineFeature("product", (r) => {
33
33
  ctx.metrics.inc("tracked_total", { kind: event.payload.kind });
34
34
  return { isSuccess: true, data: { ok: true } };
35
35
  },
36
- { access: { openToAll: true } },
36
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
37
37
  );
38
38
  });
39
39
 
@@ -44,7 +44,7 @@ const errorFeature = defineFeature("err", (r) => {
44
44
  async () => {
45
45
  throw new Error("boom from handler");
46
46
  },
47
- { access: { openToAll: true } },
47
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
48
48
  );
49
49
  });
50
50
 
@@ -90,7 +90,7 @@ const todoFeature = defineFeature("todo", (r) => {
90
90
  },
91
91
  };
92
92
  },
93
- { access: { openToAll: true } },
93
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
94
94
  );
95
95
 
96
96
  r.hook("postSave", "create", async (_save: SaveContext) => {
@@ -288,7 +288,7 @@ describe("Observability (integration) — Redis wrapper", () => {
288
288
  await ctx.redis.get("session:abc123:token");
289
289
  return { isSuccess: true, data: { ok: true } };
290
290
  },
291
- { access: { openToAll: true } },
291
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
292
292
  );
293
293
  });
294
294
 
@@ -352,7 +352,7 @@ describe("Observability (integration) — Pino trace bridge", () => {
352
352
  ctx.log?.info(event.payload.msg, { custom: "field" });
353
353
  return { isSuccess: true, data: { ok: true } };
354
354
  },
355
- { access: { openToAll: true } },
355
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
356
356
  );
357
357
  });
358
358
 
@@ -581,7 +581,7 @@ const consumerAFeature = defineFeature("consumer-a", (r) => {
581
581
  recordSharedCall(ctx);
582
582
  return { isSuccess: true, data: { ok: true } };
583
583
  },
584
- { access: { openToAll: true } },
584
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
585
585
  );
586
586
  });
587
587
 
@@ -594,7 +594,7 @@ const consumerBFeature = defineFeature("consumer-b", (r) => {
594
594
  ctx.metricsFor("unregistered-lib").inc("never_total");
595
595
  return { isSuccess: true, data: { ok: true } };
596
596
  },
597
- { access: { openToAll: true } },
597
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
598
598
  );
599
599
  });
600
600
 
@@ -109,14 +109,14 @@ const archFeature = defineFeature("archtest", (r) => {
109
109
  "item:events",
110
110
  z.object({ id: z.uuid() }),
111
111
  async (query, ctx) => ctx.loadAggregate(query.payload.id),
112
- { access: { openToAll: true } },
112
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
113
113
  );
114
114
 
115
115
  r.queryHandler(
116
116
  "item:is-archived",
117
117
  z.object({ id: z.uuid() }),
118
118
  async (query, ctx) => ctx.isStreamArchived(query.payload.id),
119
- { access: { openToAll: true } },
119
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
120
120
  );
121
121
  });
122
122
 
@@ -33,7 +33,7 @@ const echoFeature = defineFeature("echo", (r) => {
33
33
  items: [],
34
34
  search: query.payload.search,
35
35
  }),
36
- { access: { openToAll: true } },
36
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
37
37
  );
38
38
 
39
39
  r.streamHandler(
@@ -137,7 +137,7 @@ describe("dispatcher.write", () => {
137
137
  z.object({ name: z.string() }),
138
138
  async (event) => ({ isSuccess: true, data: { name: event.payload.name } }),
139
139
  {
140
- access: { openToAll: true },
140
+ access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
141
141
  rateLimit: { per: "ip+handler", limit: 3, windowSeconds: 60 },
142
142
  },
143
143
  );
@@ -168,7 +168,7 @@ describe("dispatcher.write", () => {
168
168
  z.object({ name: z.string() }),
169
169
  async (event) => ({ isSuccess: true, data: { name: event.payload.name } }),
170
170
  {
171
- access: { openToAll: true },
171
+ access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
172
172
  rateLimit: { per: "user", limit: 3, windowSeconds: 60 },
173
173
  },
174
174
  );
@@ -428,7 +428,7 @@ describe("dispatcher.stream", () => {
428
428
  yield { i };
429
429
  }
430
430
  },
431
- { access: { openToAll: true } },
431
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
432
432
  );
433
433
  });
434
434
  const dispatcher = createDispatcher(createRegistry([nestedFeature]), {
@@ -466,7 +466,7 @@ describe("dispatcher.stream", () => {
466
466
  throw new Error("close-boom");
467
467
  }
468
468
  },
469
- { access: { openToAll: true } },
469
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
470
470
  );
471
471
  });
472
472
  const dispatcher = createDispatcher(createRegistry([closeFailFeature]), {
@@ -500,7 +500,7 @@ describe("dispatcher.stream", () => {
500
500
  yield { i: 0 };
501
501
  yield { i: 1 };
502
502
  },
503
- { access: { openToAll: true } },
503
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
504
504
  );
505
505
  });
506
506
  const dispatcher = createDispatcher(createRegistry([feature]), {
@@ -536,7 +536,7 @@ describe("dispatcher.stream", () => {
536
536
  yield { i: 1 };
537
537
  yield { i: 2 };
538
538
  },
539
- { access: { openToAll: true } },
539
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
540
540
  );
541
541
  });
542
542
  const dispatcher = createDispatcher(createRegistry([feature]), {
@@ -571,7 +571,7 @@ describe("dispatcher.query postQuery hooks", () => {
571
571
  // pass on entity-existence makes such a hook register silently + never run.
572
572
  const feature = defineFeature("dash", (r) => {
573
573
  r.queryHandler("dashboard", z.object({}), async () => ({ count: 1 }), {
574
- access: { openToAll: true },
574
+ access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
575
575
  });
576
576
  r.hook("postQuery", "dashboard", async ({ entityName, rows }) => {
577
577
  expect(entityName).toBeUndefined();
@@ -590,7 +590,7 @@ describe("dispatcher.query postQuery hooks", () => {
590
590
  // [...null]). The hook here returns the row unchanged so the shape survives.
591
591
  const feature = defineFeature("nullrows", (r) => {
592
592
  r.queryHandler("dashboard", z.object({}), async () => ({ rows: null, nextCursor: null }), {
593
- access: { openToAll: true },
593
+ access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
594
594
  });
595
595
  r.hook("postQuery", "dashboard", async ({ rows }) => ({ rows }));
596
596
  });
@@ -603,7 +603,7 @@ describe("dispatcher.query postQuery hooks", () => {
603
603
  test("single-object postQuery hook returning ≠1 row throws", async () => {
604
604
  const feature = defineFeature("multi", (r) => {
605
605
  r.queryHandler("dashboard", z.object({}), async () => ({ count: 1 }), {
606
- access: { openToAll: true },
606
+ access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
607
607
  });
608
608
  r.hook("postQuery", "dashboard", async ({ rows }) => ({ rows: [...rows, ...rows] }));
609
609
  });
@@ -708,7 +708,7 @@ describe("dispatcher feature-gate", () => {
708
708
  }),
709
709
  );
710
710
  r.queryHandler("widget:list", z.object({}).passthrough(), async () => ({ items: [] }), {
711
- access: { openToAll: true },
711
+ access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
712
712
  });
713
713
  r.writeHandler(
714
714
  "widget:create",
@@ -728,7 +728,7 @@ describe("dispatcher feature-gate", () => {
728
728
  i++;
729
729
  }
730
730
  },
731
- { access: { openToAll: true } },
731
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
732
732
  );
733
733
  });
734
734
  }
@@ -900,7 +900,7 @@ describe("dispatcher feature-gate", () => {
900
900
  "check",
901
901
  z.object({}).passthrough(),
902
902
  async (_event, ctx) => ({ enabled: await ctx.hasFeature("toggled") }),
903
- { access: { openToAll: true } },
903
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
904
904
  );
905
905
  });
906
906
  const registry = createRegistry([toggled(), probe]);
@@ -940,7 +940,7 @@ describe("dispatcher feature-gate", () => {
940
940
  "check",
941
941
  z.object({}).passthrough(),
942
942
  async (_event, ctx) => ({ enabled: await ctx.hasFeature("premium-companion") }),
943
- { access: { openToAll: true } },
943
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
944
944
  );
945
945
  });
946
946
  const registry = createRegistry([probe]);
@@ -1027,7 +1027,7 @@ describe("dispatcher context.geoTzProvider (680/1)", () => {
1027
1027
  longitude: query.payload.longitude,
1028
1028
  }),
1029
1029
  }),
1030
- { access: { openToAll: true } },
1030
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
1031
1031
  );
1032
1032
  });
1033
1033
  const registry = createRegistry([geoFeature]);
@@ -0,0 +1,254 @@
1
+ // fw#2861 — every escape-hatch use reports through AppContext._escapeHatchAuditSink
2
+ // (or warn-logs when no sink is wired). Real HTTP calls + setupTestStack — never
3
+ // createTestDispatcher. Modelled on unsafe-raw-escape-hatch.integration.test.ts and
4
+ // system-identity-switch.integration.test.ts.
5
+
6
+ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7
+ import { z } from "zod";
8
+ import { defineUnmanagedTable } from "../../db/entity-table-meta";
9
+ import { executeRawQuery } from "../../db/queries/raw-sql";
10
+ import { createSystemUser, defineFeature } from "../../engine";
11
+ import type { EscapeHatchUseEvent } from "../../engine/types";
12
+ import type { Logger } from "../../logging/types";
13
+ import { createTestUser, setupTestStack, type TestStack } from "../../stack";
14
+ import { ESCAPE_HATCH_USED_SIGNAL } from "../escape-hatch-report";
15
+
16
+ const globalStoreTable = defineUnmanagedTable({
17
+ tableName: "store_fw2861_escape_hatch_audit_items",
18
+ tenancy: "global",
19
+ columns: [
20
+ {
21
+ name: "id",
22
+ pgType: "uuid",
23
+ notNull: true,
24
+ primaryKey: true,
25
+ defaultSql: "gen_random_uuid()",
26
+ },
27
+ { name: "note", pgType: "text", notNull: true },
28
+ ],
29
+ });
30
+
31
+ const UNSAFE_RAW_REASON = "fw#2861 integration test — declared unsafeRaw write";
32
+ const GLOBAL_WRITE_REASON = "fw#2861 integration test — declared cross-tenant write";
33
+ const IDENTITY_SWITCH_REASON = "fw#2861 integration test — needs SYSTEM to look itself up";
34
+ const ACKNOWLEDGE_REASON = "fw#2861 integration test — system-wide scan";
35
+
36
+ const probeFeature = defineFeature("escape-hatch-audit-probe", (r) => {
37
+ r.storeTable(globalStoreTable, {
38
+ reason: "fw#2861 integration test — cross-tenant store table for db.global()",
39
+ });
40
+
41
+ r.queryHandler(
42
+ "whoami",
43
+ z.object({}),
44
+ async (query) => ({ id: query.user.id, tenantId: query.user.tenantId }),
45
+ { access: { roles: ["system", "User"] } },
46
+ );
47
+
48
+ r.writeHandler(
49
+ "unsafe-raw-write",
50
+ z.object({}),
51
+ async (_event, ctx) => {
52
+ const runner = ctx.db.unsafeRaw(UNSAFE_RAW_REASON);
53
+ const rows = await executeRawQuery<{ one: number }>(runner, "SELECT 1 AS one");
54
+ return { isSuccess: true as const, data: { one: rows[0]?.one } };
55
+ },
56
+ { access: { roles: ["User"] }, escapeHatch: { reason: UNSAFE_RAW_REASON } },
57
+ );
58
+
59
+ r.writeHandler(
60
+ "global-write",
61
+ z.object({}),
62
+ async (_event, ctx) => {
63
+ const row = await ctx.db.global(globalStoreTable).insertOne<{ id: string }>({
64
+ note: "hi",
65
+ });
66
+ return { isSuccess: true as const, data: row };
67
+ },
68
+ { access: { roles: ["User"] }, escapeHatch: { reason: GLOBAL_WRITE_REASON } },
69
+ );
70
+
71
+ r.writeHandler(
72
+ "identity-switch-write",
73
+ z.object({}),
74
+ async (event, ctx) =>
75
+ ctx
76
+ .queryAs(createSystemUser(event.user.tenantId), "escape-hatch-audit-probe:query:whoami", {})
77
+ .then((data) => ({ isSuccess: true as const, data })),
78
+ { access: { roles: ["User"] }, escapeHatch: { reason: IDENTITY_SWITCH_REASON } },
79
+ );
80
+
81
+ r.writeHandler(
82
+ "identity-switch-self-write",
83
+ z.object({}),
84
+ async (event, ctx) =>
85
+ ctx
86
+ .queryAs(event.user, "escape-hatch-audit-probe:query:whoami", {})
87
+ .then((data) => ({ isSuccess: true as const, data })),
88
+ { access: { roles: ["User"] }, escapeHatch: { reason: IDENTITY_SWITCH_REASON } },
89
+ );
90
+ });
91
+
92
+ const systemScopeProbeFeature = defineFeature("escape-hatch-audit-probe-system", (r) => {
93
+ r.systemScope();
94
+
95
+ r.queryHandler(
96
+ "acknowledge-cross-tenant-query",
97
+ z.object({}),
98
+ async (_query, ctx) => {
99
+ if (!ctx.systemDb) throw new Error("expected ctx.systemDb on a systemScope() handler");
100
+ const db = ctx.systemDb.acknowledgeCrossTenant(ACKNOWLEDGE_REASON);
101
+ return { tenantId: db.tenantId };
102
+ },
103
+ { access: { roles: ["User"] } },
104
+ );
105
+ });
106
+
107
+ const user = createTestUser({
108
+ id: 93,
109
+ tenantId: "11111111-0000-4000-8000-000000000093",
110
+ roles: ["User"],
111
+ });
112
+
113
+ function recordingSink(events: EscapeHatchUseEvent[]) {
114
+ return async (event: EscapeHatchUseEvent) => {
115
+ events.push(event);
116
+ };
117
+ }
118
+
119
+ describe("escape-hatch uses report through AppContext._escapeHatchAuditSink", () => {
120
+ let stack: TestStack;
121
+ const events: EscapeHatchUseEvent[] = [];
122
+
123
+ beforeAll(async () => {
124
+ stack = await setupTestStack({
125
+ features: [probeFeature, systemScopeProbeFeature],
126
+ extraContext: { _escapeHatchAuditSink: recordingSink(events) },
127
+ });
128
+ });
129
+
130
+ afterAll(async () => {
131
+ await stack.cleanup();
132
+ });
133
+
134
+ test("unsafeRaw reports exactly one unsafe-raw event with handler/kind/reason/tenantId/actor, deduped on a repeat call within the window", async () => {
135
+ events.length = 0;
136
+ await stack.http.writeOk("escape-hatch-audit-probe:write:unsafe-raw-write", {}, user);
137
+ // Identical (handler, kind, reason, tenantId, actor) — the 60s dedup window
138
+ // must suppress this second report.
139
+ await stack.http.writeOk("escape-hatch-audit-probe:write:unsafe-raw-write", {}, user);
140
+
141
+ const matches = events.filter((e) => e.kind === "unsafe-raw");
142
+ expect(matches).toEqual([
143
+ {
144
+ handler: "escape-hatch-audit-probe:write:unsafe-raw-write",
145
+ kind: "unsafe-raw",
146
+ reason: UNSAFE_RAW_REASON,
147
+ tenantId: user.tenantId,
148
+ actor: user.id,
149
+ },
150
+ ]);
151
+ });
152
+
153
+ test("db.global() write reports exactly one global-write event with the escapeHatch's reason", async () => {
154
+ events.length = 0;
155
+ await stack.http.writeOk("escape-hatch-audit-probe:write:global-write", {}, user);
156
+
157
+ const matches = events.filter((e) => e.kind === "global-write");
158
+ expect(matches).toEqual([
159
+ {
160
+ handler: "escape-hatch-audit-probe:write:global-write",
161
+ kind: "global-write",
162
+ reason: GLOBAL_WRITE_REASON,
163
+ tenantId: user.tenantId,
164
+ actor: user.id,
165
+ },
166
+ ]);
167
+ });
168
+
169
+ test("ctx.systemDb.acknowledgeCrossTenant reports exactly one acknowledge-cross-tenant event", async () => {
170
+ events.length = 0;
171
+ await stack.http.queryOk(
172
+ "escape-hatch-audit-probe-system:query:acknowledge-cross-tenant-query",
173
+ {},
174
+ user,
175
+ );
176
+
177
+ const matches = events.filter((e) => e.kind === "acknowledge-cross-tenant");
178
+ expect(matches).toEqual([
179
+ {
180
+ handler: "escape-hatch-audit-probe-system:query:acknowledge-cross-tenant-query",
181
+ kind: "acknowledge-cross-tenant",
182
+ reason: ACKNOWLEDGE_REASON,
183
+ tenantId: user.tenantId,
184
+ actor: user.id,
185
+ },
186
+ ]);
187
+ });
188
+
189
+ test("a granted SYSTEM identity switch reports exactly one identity-switch event with the target", async () => {
190
+ events.length = 0;
191
+ await stack.http.writeOk("escape-hatch-audit-probe:write:identity-switch-write", {}, user);
192
+
193
+ const matches = events.filter((e) => e.kind === "identity-switch");
194
+ expect(matches).toEqual([
195
+ {
196
+ handler: "escape-hatch-audit-probe:write:identity-switch-write",
197
+ kind: "identity-switch",
198
+ reason: IDENTITY_SWITCH_REASON,
199
+ tenantId: user.tenantId,
200
+ actor: user.id,
201
+ target: { id: createSystemUser(user.tenantId).id, tenantId: user.tenantId },
202
+ },
203
+ ]);
204
+ });
205
+
206
+ test("self-delegation with a grant reports no identity-switch event", async () => {
207
+ events.length = 0;
208
+ await stack.http.writeOk("escape-hatch-audit-probe:write:identity-switch-self-write", {}, user);
209
+
210
+ expect(events.filter((e) => e.kind === "identity-switch")).toEqual([]);
211
+ });
212
+ });
213
+
214
+ describe("escape-hatch uses without a sink fall back to a structured warn log", () => {
215
+ let stack: TestStack;
216
+ const warnCalls: Array<{ msg: string; data?: Record<string, unknown> }> = [];
217
+
218
+ beforeAll(async () => {
219
+ const log: Logger = {
220
+ info() {},
221
+ debug() {},
222
+ warn(msg, data) {
223
+ warnCalls.push({ msg, data });
224
+ },
225
+ error() {},
226
+ child() {
227
+ return log;
228
+ },
229
+ };
230
+ stack = await setupTestStack({
231
+ features: [probeFeature, systemScopeProbeFeature],
232
+ extraContext: { log },
233
+ });
234
+ });
235
+
236
+ afterAll(async () => {
237
+ await stack.cleanup();
238
+ });
239
+
240
+ test("unsafeRaw without a sink warn-logs security:escape-hatch-used", async () => {
241
+ warnCalls.length = 0;
242
+ await stack.http.writeOk("escape-hatch-audit-probe:write:unsafe-raw-write", {}, user);
243
+
244
+ const matches = warnCalls.filter((c) => c.msg === ESCAPE_HATCH_USED_SIGNAL);
245
+ expect(matches).toHaveLength(1);
246
+ expect(matches[0]?.data).toEqual({
247
+ handler: "escape-hatch-audit-probe:write:unsafe-raw-write",
248
+ kind: "unsafe-raw",
249
+ reason: UNSAFE_RAW_REASON,
250
+ tenantId: user.tenantId,
251
+ actor: user.id,
252
+ });
253
+ });
254
+ });