@cosmicdrift/kumiko-framework 0.323.0 → 0.325.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cosmicdrift/kumiko-framework",
3
- "version": "0.323.0",
3
+ "version": "0.325.0",
4
4
  "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
5
5
  "license": "BUSL-1.1",
6
6
  "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -202,8 +202,8 @@
202
202
  "./package.json": "./package.json"
203
203
  },
204
204
  "dependencies": {
205
- "@cosmicdrift/kumiko-http": "0.323.0",
206
- "@cosmicdrift/kumiko-types": "0.323.0",
205
+ "@cosmicdrift/kumiko-http": "0.325.0",
206
+ "@cosmicdrift/kumiko-types": "0.325.0",
207
207
  "bullmq": "^5.76.7",
208
208
  "bun-types": "^1.3.13",
209
209
  "hono": "^4.13.1",
@@ -219,7 +219,7 @@
219
219
  "zod": "^4.4.3"
220
220
  },
221
221
  "devDependencies": {
222
- "@cosmicdrift/kumiko-dispatcher-live": "0.323.0",
222
+ "@cosmicdrift/kumiko-dispatcher-live": "0.325.0",
223
223
  "bun-types": "^1.3.13",
224
224
  "pino-pretty": "^13.1.3"
225
225
  },
package/src/changes.json CHANGED
@@ -1,4 +1,22 @@
1
1
  [
2
+ {
3
+ "version": "0.325.0",
4
+ "type": "improvement",
5
+ "title": "JobContext exposes ctx.attempt and ctx.finalAttempt",
6
+ "detail": "Job handlers can read the 1-based run number and whether the current run is\nthe last one BullMQ will attempt, so a handler can defer writing a\nbusiness-facing failure state until the final attempt instead of writing\nit on every retry.\n\nMigration note: hand-built `JobContext` objects (test doubles, boot\nseeders) must add `attempt` and `finalAttempt` (e.g. `attempt: 1,\nfinalAttempt: true`)."
7
+ },
8
+ {
9
+ "version": "0.324.0",
10
+ "type": "breaking",
11
+ "title": "Writes on readAsInstruction fields now require agent.risk \"high\" on the directly-dispatched handler",
12
+ "migration": "New field flag `readAsInstruction: true` on `text`, `longText`, and `jsonb`\nfields (`@cosmicdrift/kumiko-types`) marks a field whose value a later run\nreads as an instruction — a prompt, rule, or template. Any create/update\nwhose payload writes such a field must resolve `agent.risk: \"high\"` on the\ndirectly-dispatched entry handler, or the executor gate denies it with\n`access_denied` / `instruction_field_write_requires_high_risk` before any\nDB write. This is opt-in per field, but touches every write path once a\nfield carries the flag — consumer checklist:\n\n1. Mark fields that a later run reads as an instruction (prompts, rules,\n templates) with `readAsInstruction: true`.\n2. Update your own handlers that write such a field via\n `executor.create`/`executor.update`, or delegate into one via\n `ctx.write`/`writeAs`, to `agent: { risk: \"high\" }` — the directly\n called handler must carry it, delegation does not inherit it.\n3. Standard entity-convention create/update handlers on an entity with such\n a field now default to \"high\" automatically; declaring a lower\n `agent.risk` on one is a define-time error, same as the delete floor\n from #3345.\n4. An update whose payload doesn't include the flagged field stays allowed\n unchanged — the gate checks payload presence after `preSave`, not\n whether the value actually changed.\n5. Not gated: `delete`/`restore` (they don't write the field; the existing\n irreversible-delete floor still applies), and your own domain events\n applied via `ctx.appendEvent` whose projection writes a flagged field —\n route such writes through the executor primitives (`create`/`update`)\n to get the gate.\n6. On `create`, a flagged field with a `default` counts as written even if\n the payload omits it — `applyDefaults` runs before the gate. A custom\n mid-risk create handler whose payload leaves the field out is denied\n just the same, and a standard create with `excludeFields` on such a\n field still resolves to \"high\"."
13
+ },
14
+ {
15
+ "version": "0.324.0",
16
+ "type": "improvement",
17
+ "title": "TestStack.drainJobs() waits deterministically for jobs and their follow-up consumers",
18
+ "detail": "Migration note: replace `waitFor` polling on a job's side effect with `await\nstack.drainJobs()`; it rejects with the job's name and error on final\nfailure. New `JobRunner.countPendingJobs()` — hand-written `JobRunner`\nstubs (test doubles) must add it."
19
+ },
2
20
  {
3
21
  "version": "0.323.0",
4
22
  "type": "improvement",
@@ -1,4 +1,5 @@
1
1
  import { checkWriteFieldOwnership } from "../engine/field-access";
2
+ import { instructionFieldNames } from "../engine/instruction-fields";
2
3
  import { userCanCreateFieldRow, userCanWriteFieldRow } from "../engine/ownership";
3
4
  import { SYSTEM_ROLE, SYSTEM_USER_ID } from "../engine/system-user";
4
5
  import type { EntityId, SessionUser } from "../engine/types";
@@ -19,6 +20,7 @@ import {
19
20
  getStreamVersion,
20
21
  } from "../event-store";
21
22
  import {
23
+ assertInstructionFieldWriteAllowed,
22
24
  assertIrreversibleOperationAllowed,
23
25
  isIrreversibleEntityVerb,
24
26
  } from "../pipeline/irreversible-operation-gate";
@@ -122,6 +124,7 @@ export function createWriteVerbs(
122
124
  assertStreamWritable,
123
125
  loadExpectSnapshot,
124
126
  } = ctx;
127
+ const entityInstructionFieldNames = instructionFieldNames(entity);
125
128
 
126
129
  return {
127
130
  async create(payload, user, db, options) {
@@ -152,6 +155,12 @@ export function createWriteVerbs(
152
155
 
153
156
  // After preSave so derived fields count, before the event append so nothing persists.
154
157
  assertPersonalDataWrite(db, tableNameOf(table), Object.keys(data), entity);
158
+ assertInstructionFieldWriteAllowed(
159
+ entityName,
160
+ "create",
161
+ Object.keys(data),
162
+ entityInstructionFieldNames,
163
+ );
155
164
 
156
165
  // H.2 — entity-level write-ownership on create. No oldRow exists, so
157
166
  // only the new row is checked. No Straddle concern for creates.
@@ -344,6 +353,12 @@ export function createWriteVerbs(
344
353
 
345
354
  // After preSave so derived fields count, before the event append so nothing persists.
346
355
  assertPersonalDataWrite(db, tableNameOf(table), Object.keys(changes), entity);
356
+ assertInstructionFieldWriteAllowed(
357
+ entityName,
358
+ "update",
359
+ Object.keys(changes),
360
+ entityInstructionFieldNames,
361
+ );
347
362
 
348
363
  // H.2 — entity-level write-ownership on update. Load old row (already
349
364
  // done above), build post-change row via shallow merge. Straddle-safe
@@ -13,7 +13,7 @@ import {
13
13
  type EntityCrudRegistrar,
14
14
  registerEntityCrud,
15
15
  } from "../entity-handlers";
16
- import { createEntity, createTextField } from "../factories";
16
+ import { createEntity, createLongTextField, createTextField } from "../factories";
17
17
  // Barrel import, not "../entity-handlers": covers that entityListSchema is
18
18
  // actually re-exported through engine/index.ts.
19
19
  import { entityListSchema, resolveAgentExposure } from "../index";
@@ -37,6 +37,31 @@ const noteEntitySoftDelete = createEntity({
37
37
  softDelete: true,
38
38
  });
39
39
 
40
+ const promptEntity = createEntity({
41
+ table: "prompts",
42
+ fields: {
43
+ label: createTextField({ required: true, personal: false, reason: "test_fixture" }),
44
+ body: createLongTextField({
45
+ personal: false,
46
+ reason: "test_fixture",
47
+ readAsInstruction: true,
48
+ }),
49
+ },
50
+ });
51
+
52
+ const promptEntityDefaultedBody = createEntity({
53
+ table: "prompts_defaulted",
54
+ fields: {
55
+ label: createTextField({ required: true, personal: false, reason: "test_fixture" }),
56
+ body: createLongTextField({
57
+ personal: false,
58
+ reason: "test_fixture",
59
+ readAsInstruction: true,
60
+ default: "system default prompt",
61
+ }),
62
+ },
63
+ });
64
+
40
65
  const adminAccess = { access: { roles: ["Admin"] } } as const;
41
66
 
42
67
  describe("defineEntityWriteHandler", () => {
@@ -138,6 +163,42 @@ describe("defineEntityWriteHandler", () => {
138
163
  expect(def.agent).toBeUndefined();
139
164
  expect(resolveAgentExposure(def, "write").risk).toBe("mid");
140
165
  });
166
+
167
+ test("standard create on an entity with a readAsInstruction field defaults agent.risk to high", () => {
168
+ const def = defineEntityCreateHandler("prompt", promptEntity, adminAccess);
169
+ expect(resolveAgentExposure(def, "write").risk).toBe("high");
170
+ });
171
+
172
+ test("standard update on an entity with a readAsInstruction field defaults agent.risk to high", () => {
173
+ const def = defineEntityUpdateHandler("prompt", promptEntity, adminAccess);
174
+ expect(resolveAgentExposure(def, "write").risk).toBe("high");
175
+ });
176
+
177
+ test("declaring mid on a readAsInstruction-writing standard handler throws", () => {
178
+ expect(() =>
179
+ defineEntityUpdateHandler("prompt", promptEntity, {
180
+ access: { roles: ["Admin"] },
181
+ agent: { risk: "mid" },
182
+ }),
183
+ ).toThrow(/agent\.risk must be "high"/);
184
+ });
185
+
186
+ test("update with excludeFields excluding all readAsInstruction fields is not forced to high", () => {
187
+ const def = defineEntityUpdateHandler("prompt", promptEntity, {
188
+ access: { roles: ["Admin"] },
189
+ excludeFields: ["body"],
190
+ });
191
+ expect(def.agent).toBeUndefined();
192
+ expect(resolveAgentExposure(def, "write").risk).toBe("mid");
193
+ });
194
+
195
+ test("create excluding a readAsInstruction field that has a default is still forced to high", () => {
196
+ const def = defineEntityCreateHandler("prompt", promptEntityDefaultedBody, {
197
+ access: { roles: ["Admin"] },
198
+ excludeFields: ["body"],
199
+ });
200
+ expect(resolveAgentExposure(def, "write").risk).toBe("high");
201
+ });
141
202
  });
142
203
 
143
204
  describe("defineEntityQueryHandler", () => {
@@ -13,6 +13,7 @@ import { acknowledgeConventionCrossTenant, type TenantDb } from "../db/tenant-db
13
13
  import { isSystemIdentity } from "../pipeline/system-identity-switch";
14
14
  import { assertUnreachable } from "../utils";
15
15
  import { PAGED_QUERY_HANDLER_BRAND } from "./define-handler";
16
+ import { instructionFieldNames } from "./instruction-fields";
16
17
  import { buildInsertSchema, buildUpdateSchema } from "./schema-builder";
17
18
  import type {
18
19
  AccessRule,
@@ -214,19 +215,45 @@ function isInTotalsMatchPair(entity: EntityDefinition, fieldName: string): boole
214
215
  );
215
216
  }
216
217
 
217
- // delete on an entity without softDelete hard-purges the row — the agent
218
- // risk gate only protects it if this handler resolves to "high".
218
+ // A create/update handler can write a field flagged readAsInstruction: true if:
219
+ // update — the field isn't in excludeFields; create — the field isn't excluded,
220
+ // or is excluded but has a default (applyDefaults writes it regardless, so the
221
+ // define-time floor must match what the executor gate will see at runtime).
222
+ function writableInstructionFields(
223
+ verb: "create" | "update",
224
+ entity: EntityDefinition,
225
+ excludedFields: readonly string[],
226
+ ): readonly string[] {
227
+ return instructionFieldNames(entity).filter((fieldName) => {
228
+ if (!excludedFields.includes(fieldName)) return true;
229
+ if (verb !== "create") return false;
230
+ const field = entity.fields[fieldName];
231
+ return field !== undefined && "default" in field && field.default !== undefined;
232
+ });
233
+ }
234
+
235
+ // delete on an entity without softDelete hard-purges the row, and create/update
236
+ // handlers writing a readAsInstruction field are irreversible in effect (a
237
+ // revert doesn't undo a run that already read the value) — the agent risk gate
238
+ // only protects either case if this handler resolves to "high".
219
239
  function resolveEntityWriteAgentHints(
220
240
  name: string,
221
241
  verb: (typeof WRITE_VERBS)[number],
222
242
  entity: EntityDefinition,
223
243
  agent: AgentHandlerHints | undefined,
244
+ excludedFields: readonly string[],
224
245
  ): AgentHandlerHints | undefined {
225
- if (verb !== "delete" || entity.softDelete) return agent;
246
+ const isIrreversibleDelete = verb === "delete" && !entity.softDelete;
247
+ const instructionFields =
248
+ verb === "create" || verb === "update"
249
+ ? writableInstructionFields(verb, entity, excludedFields)
250
+ : [];
251
+ if (!isIrreversibleDelete && instructionFields.length === 0) return agent;
226
252
  if (agent?.risk !== undefined && agent.risk !== "high") {
227
- throw new Error(
228
- `"${name}": delete on an entity without softDelete is irreversible — agent.risk must be "high" (got "${agent.risk}").`,
229
- );
253
+ const reason = isIrreversibleDelete
254
+ ? "delete on an entity without softDelete is irreversible"
255
+ : `it writes field(s) ${instructionFields.join(", ")} flagged readAsInstruction: true`;
256
+ throw new Error(`"${name}": ${reason} — agent.risk must be "high" (got "${agent.risk}").`);
230
257
  }
231
258
  return { ...agent, risk: "high" };
232
259
  }
@@ -271,9 +298,15 @@ export function defineEntityWriteHandler(
271
298
  `"${name}": restore is only valid for entities declared with softDelete: true.`,
272
299
  );
273
300
  }
274
- const agentHints = resolveEntityWriteAgentHints(name, verb, entity, options.agent);
275
301
  const excludedFields = options.excludeFields ?? [];
276
302
  assertExcludableFields(name, verb, entity, excludedFields);
303
+ const agentHints = resolveEntityWriteAgentHints(
304
+ name,
305
+ verb,
306
+ entity,
307
+ options.agent,
308
+ excludedFields,
309
+ );
277
310
 
278
311
  const table = buildEntityTable(entityName, entity);
279
312
  const executor = createEventStoreExecutor(table, entity, { entityName });
@@ -0,0 +1,11 @@
1
+ import type { EntityDefinition } from "./types";
2
+
3
+ // Shared between the executor gate (payload-presence check on create/update)
4
+ // and the define-time floor (resolveEntityWriteAgentHints) so both read the
5
+ // exact same field set — a divergence here would let one enforce a flag the
6
+ // other doesn't know about.
7
+ export function instructionFieldNames(entity: EntityDefinition): readonly string[] {
8
+ return Object.entries(entity.fields)
9
+ .filter(([, field]) => "readAsInstruction" in field && field.readAsInstruction === true)
10
+ .map(([name]) => name);
11
+ }
@@ -173,6 +173,20 @@ irreversible_operation_requires_high_risk:
173
173
 
174
174
  `agent: { risk: "high" }` am genannten Handler deklarieren.
175
175
 
176
+ instruction_field_write_requires_high_risk:
177
+ endUser: |
178
+ Diese Aktion ist nicht erlaubt.
179
+ Wende dich an einen Administrator wenn du glaubst, dass das ein Fehler ist.
180
+ developer: |
181
+ `AccessDeniedError`: der direkt aufgerufene Einstiegs-Handler schreibt ein
182
+ Feld mit `readAsInstruction: true` (`details.fields`), aber sein
183
+ aufgelöstes Agent-Risiko (`details.resolvedRisk`) liegt unter `"high"`.
184
+ `details.handler` nennt den Einstiegs-Handler — eine Delegation über
185
+ `ctx.write`/`writeAs` aus einem Handler mit niedrigerem Risiko erbt das
186
+ deklarierte Risiko des inneren Handlers nicht.
187
+
188
+ `agent: { risk: "high" }` am genannten Handler deklarieren.
189
+
176
190
  agent.tool_not_allowed:
177
191
  endUser: |
178
192
  Dieses Tool ist im aktuellen Modus nicht verfügbar.
@@ -166,6 +166,20 @@ irreversible_operation_requires_high_risk:
166
166
 
167
167
  Declare `agent: { risk: "high" }` on the named handler.
168
168
 
169
+ instruction_field_write_requires_high_risk:
170
+ endUser: |
171
+ This action isn't allowed.
172
+ Please contact an administrator if you believe this is a mistake.
173
+ developer: |
174
+ `AccessDeniedError`: the directly-dispatched entry handler writes a field
175
+ flagged `readAsInstruction: true` (`details.fields`) but its resolved
176
+ agent risk (`details.resolvedRisk`) is below `"high"`. `details.handler`
177
+ names the entry handler — delegating the call via `ctx.write`/`writeAs`
178
+ from a lower-risk handler does not inherit an inner handler's declared
179
+ risk.
180
+
181
+ Declare `agent: { risk: "high" }` on the named handler.
182
+
169
183
  agent.tool_not_allowed:
170
184
  endUser: |
171
185
  This tool is not available in the current mode.
@@ -68,6 +68,12 @@ export const FrameworkReasons = {
68
68
  // delete/forget but its agent.risk resolves below "high" — delegation via
69
69
  // ctx.write/writeAs from a lower-risk handler does not inherit the gate.
70
70
  irreversibleOperationRequiresHighRisk: "irreversible_operation_requires_high_risk",
71
+
72
+ // AccessDeniedError: the directly-dispatched entry handler writes a field
73
+ // flagged readAsInstruction: true but its agent.risk resolves below "high" —
74
+ // delegation via ctx.write/writeAs from a lower-risk handler does not
75
+ // inherit the gate.
76
+ instructionFieldWriteRequiresHighRisk: "instruction_field_write_requires_high_risk",
71
77
  } as const;
72
78
 
73
79
  export type FrameworkReason = (typeof FrameworkReasons)[keyof typeof FrameworkReasons];
@@ -0,0 +1,76 @@
1
+ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
2
+ import * as z from "zod";
3
+ import { defineFeature } from "../../engine";
4
+ import { setupTestStack, type TestStack, TestUsers } from "../../stack";
5
+
6
+ const retryRuns: [number, boolean][] = [];
7
+ const singleRuns: [number, boolean][] = [];
8
+
9
+ const attemptFixtureFeature = defineFeature("attemptfixture", (r) => {
10
+ r.writeHandler(
11
+ "trigger-retry",
12
+ z.object({}),
13
+ async () => ({ isSuccess: true as const, data: {} }),
14
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
15
+ );
16
+ r.job(
17
+ "retry",
18
+ { trigger: { on: "attemptfixture:write:trigger-retry" }, retries: 2 },
19
+ async (_payload, ctx) => {
20
+ retryRuns.push([ctx.attempt, ctx.finalAttempt]);
21
+ if (ctx.attempt < 3) throw new Error(`fails on attempt ${ctx.attempt}`);
22
+ },
23
+ );
24
+
25
+ r.writeHandler(
26
+ "trigger-single",
27
+ z.object({}),
28
+ async () => ({ isSuccess: true as const, data: {} }),
29
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
30
+ );
31
+ r.job(
32
+ "single",
33
+ { trigger: { on: "attemptfixture:write:trigger-single" } },
34
+ async (_payload, ctx) => {
35
+ singleRuns.push([ctx.attempt, ctx.finalAttempt]);
36
+ },
37
+ );
38
+ });
39
+
40
+ let stack: TestStack;
41
+
42
+ beforeAll(async () => {
43
+ stack = await setupTestStack({
44
+ features: [attemptFixtureFeature],
45
+ jobs: { consumerLane: "worker" },
46
+ });
47
+ });
48
+
49
+ afterAll(async () => {
50
+ await stack.cleanup();
51
+ });
52
+
53
+ beforeEach(() => {
54
+ retryRuns.length = 0;
55
+ singleRuns.length = 0;
56
+ });
57
+
58
+ describe("JobContext.attempt / finalAttempt", () => {
59
+ test("increments per retry and marks the last configured attempt final", async () => {
60
+ await stack.http.writeOk("attemptfixture:write:trigger-retry", {}, TestUsers.admin);
61
+
62
+ await stack.drainJobs();
63
+ expect(retryRuns).toEqual([
64
+ [1, false],
65
+ [2, false],
66
+ [3, true],
67
+ ]);
68
+ });
69
+
70
+ test("a job without retries starts at attempt 1 and is already final", async () => {
71
+ await stack.http.writeOk("attemptfixture:write:trigger-single", {}, TestUsers.admin);
72
+
73
+ await stack.drainJobs();
74
+ expect(singleRuns).toEqual([[1, true]]);
75
+ });
76
+ });
@@ -7,13 +7,12 @@
7
7
  //
8
8
  // Only lower bounds are asserted (never upper) — BullMQ schedules the next
9
9
  // attempt at failureTime + delay using the same wall clock this test reads,
10
- // so a >= assertion is flake-free without inflating waitFor's budget.
10
+ // so a >= assertion is flake-free without inflating the wait budget.
11
11
 
12
12
  import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
13
13
  import * as z from "zod";
14
14
  import { defineFeature } from "../../engine";
15
15
  import { setupTestStack, type TestStack, TestUsers } from "../../stack";
16
- import { waitFor } from "../../testing";
17
16
 
18
17
  const exponentialDefaultStarts: number[] = [];
19
18
  const exponentialCustomStarts: number[] = [];
@@ -113,9 +112,8 @@ describe("job backoff waits between retries (fw#3167)", () => {
113
112
  TestUsers.admin,
114
113
  );
115
114
 
116
- await waitFor(() => {
117
- expect(exponentialDefaultStarts).toHaveLength(2);
118
- });
115
+ await stack.drainJobs();
116
+ expect(exponentialDefaultStarts).toHaveLength(2);
119
117
 
120
118
  const [gap = 0] = gapsBetween(exponentialDefaultStarts);
121
119
  expect(gap).toBeGreaterThanOrEqual(1000);
@@ -128,9 +126,8 @@ describe("job backoff waits between retries (fw#3167)", () => {
128
126
  TestUsers.admin,
129
127
  );
130
128
 
131
- await waitFor(() => {
132
- expect(exponentialCustomStarts).toHaveLength(4);
133
- });
129
+ await stack.drainJobs();
130
+ expect(exponentialCustomStarts).toHaveLength(4);
134
131
 
135
132
  const [gap1 = 0, gap2 = 0, gap3 = 0] = gapsBetween(exponentialCustomStarts);
136
133
  expect(gap1).toBeGreaterThanOrEqual(50);
@@ -141,9 +138,8 @@ describe("job backoff waits between retries (fw#3167)", () => {
141
138
  test('backoff: { type: "fixed", delayMs } waits the configured constant delay', async () => {
142
139
  await stack.http.writeOk("backofffixture:write:trigger-fixed-custom", {}, TestUsers.admin);
143
140
 
144
- await waitFor(() => {
145
- expect(fixedCustomStarts).toHaveLength(2);
146
- });
141
+ await stack.drainJobs();
142
+ expect(fixedCustomStarts).toHaveLength(2);
147
143
 
148
144
  const [gap = 0] = gapsBetween(fixedCustomStarts);
149
145
  expect(gap).toBeGreaterThanOrEqual(300);
@@ -239,6 +239,11 @@ export type JobRunner = {
239
239
  // once at boot, after the dispatcher exists (job-runner construction happens
240
240
  // before it). Before this runs, JobContext.write/writeAs/queryAs throw.
241
241
  attachDispatcher(ref: DispatchWriteRef): void;
242
+ // Read-only snapshot across both lane queues for test-stack draining
243
+ // (stack.drainJobs()). Excludes delayed jobs whose BullMQ id starts with
244
+ // "repeat:" — those are future cron ticks, not work in flight, and would
245
+ // make drainJobs wait forever on a stack with a scheduled job.
246
+ countPendingJobs(): Promise<number>;
242
247
  };
243
248
 
244
249
  export type JobRunnerOptions = {
@@ -740,10 +745,11 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
740
745
  // retry number, so audit queries distinguish fresh from retry runs
741
746
  // without peeking at BullMQ internals.
742
747
  const rawData = bullJob.data as DbRow;
748
+ const attempt = bullJob.attemptsMade + 1;
743
749
  const meta: JobMeta = {
744
750
  triggeredById: rawData["_triggeredById"] as string | undefined, // @cast-boundary dynamic-key
745
751
  payload: rawData["_payload"] as string | undefined, // @cast-boundary dynamic-key
746
- attempt: bullJob.attemptsMade + 1,
752
+ attempt,
747
753
  };
748
754
 
749
755
  // Build handler payload (without internal meta fields)
@@ -768,7 +774,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
768
774
  : null;
769
775
  // BullMQ stops retrying once attemptsMade reaches the configured attempts
770
776
  // (`retries + 1`), so this is the attempt whose failure is final.
771
- const finalAttempt = bullJob.attemptsMade + 1 >= (jobDef.retries ?? 0) + 1;
777
+ const finalAttempt = attempt >= (jobDef.retries ?? 0) + 1;
772
778
  const outcomeMeta = (messageKey: string | null): JobOutcomeMeta => ({
773
779
  tenantId,
774
780
  finalAttempt,
@@ -893,6 +899,8 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
893
899
  ...(selfRunner !== undefined && { jobRunner: selfRunner }),
894
900
  systemUser: jobSystemUser,
895
901
  triggeredBy: triggeredById !== null ? { id: triggeredById, tenantId } : null,
902
+ attempt,
903
+ finalAttempt,
896
904
  log: createJobLogger(logs),
897
905
  ...(triggerName !== undefined && { triggerName }),
898
906
  ...(triggerEvent !== undefined && { triggerEvent }),
@@ -998,7 +1006,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
998
1006
  attributes: {
999
1007
  "job.name": jobName,
1000
1008
  "job.id": jobId,
1001
- "job.attempt": bullJob.attemptsMade + 1,
1009
+ "job.attempt": attempt,
1002
1010
  "kumiko.tenant_id": tenantId,
1003
1011
  // Lane-routing attributes (Welle 2.6). `run_in` is the job's
1004
1012
  // declared lane (explicit or default-"worker"); `consumer_lane`
@@ -1335,6 +1343,28 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
1335
1343
  attachDispatcher(ref: DispatchWriteRef): void {
1336
1344
  dispatchWriteRef = ref;
1337
1345
  },
1346
+ async countPendingJobs(): Promise<number> {
1347
+ let total = 0;
1348
+ for (const queue of Object.values(queues)) {
1349
+ const counts = await queue.getJobCounts(
1350
+ "waiting",
1351
+ "active",
1352
+ "prioritized",
1353
+ "waiting-children",
1354
+ );
1355
+ total +=
1356
+ (counts["waiting"] ?? 0) +
1357
+ (counts["active"] ?? 0) +
1358
+ (counts["prioritized"] ?? 0) +
1359
+ (counts["waiting-children"] ?? 0);
1360
+ // getJobCounts has no id-prefix filter for "delayed" — getJobs(["delayed"])
1361
+ // loads the full Job objects (payload included) just to read .id; fine for
1362
+ // test-draining's low volume, not a pattern for a prod-path hot loop.
1363
+ const delayed = await queue.getJobs(["delayed"]);
1364
+ total += delayed.filter((job) => !(job.id ?? "").startsWith("repeat:")).length;
1365
+ }
1366
+ return total;
1367
+ },
1338
1368
  };
1339
1369
 
1340
1370
  selfRunner = runnerApi;
@@ -351,6 +351,7 @@ describe("enqueueProjectionRebuild — inline fallback (jobRunner without the jo
351
351
  return "should-not-happen";
352
352
  },
353
353
  attachDispatcher: () => {},
354
+ countPendingJobs: async () => 0,
354
355
  };
355
356
 
356
357
  const outcome = await enqueueProjectionRebuild("pendingtest:projection:pending-counts", {
@@ -0,0 +1,213 @@
1
+ // Agent-risk floor for writes on fields flagged readAsInstruction: true — a
2
+ // later run reads such a field's value as an instruction (prompt, rule,
3
+ // template), so the directly-dispatched entry handler must resolve agent.risk
4
+ // "high", or the executor gate refuses before any DB write. Real HTTP calls +
5
+ // setupTestStack — never createTestDispatcher. See kumiko-framework#3358.
6
+
7
+ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
8
+ import * as z from "zod";
9
+ import { createEventStoreExecutor } from "../../db/event-store-executor";
10
+ import { buildEntityTable } from "../../db/table-builder";
11
+ import { createEntity, createLongTextField, createTextField, defineFeature } from "../../engine";
12
+ import type { Registry } from "../../engine/types";
13
+ import { resolveAgentExposure } from "../../engine/types";
14
+ import { resetEventStore, setupTestStack, type TestStack, TestUsers } from "../../stack";
15
+
16
+ const admin = TestUsers.admin;
17
+
18
+ const promptEntity = createEntity({
19
+ table: "gate_instruction_prompts",
20
+ fields: {
21
+ label: createTextField({ personal: false, reason: "test_fixture", required: true }),
22
+ body: createLongTextField({
23
+ personal: false,
24
+ reason: "test_fixture",
25
+ readAsInstruction: true,
26
+ }),
27
+ },
28
+ });
29
+ const promptTable = buildEntityTable("prompt", promptEntity);
30
+ const promptExecutor = createEventStoreExecutor(promptTable, promptEntity, {
31
+ entityName: "prompt",
32
+ });
33
+
34
+ const gateFeature = defineFeature("instructiongate", (r) => {
35
+ r.entity("prompt", promptEntity);
36
+
37
+ r.crud("prompt", promptEntity, {
38
+ write: { access: { roles: ["Admin"] } },
39
+ read: { access: { roles: ["Admin"] } },
40
+ });
41
+
42
+ r.writeHandler(
43
+ "create-prompt-mid",
44
+ z.object({ label: z.string(), body: z.string().optional() }),
45
+ async (event, ctx) => promptExecutor.create(event.payload, event.user, ctx.db),
46
+ { access: { roles: ["Admin"] } },
47
+ );
48
+
49
+ r.writeHandler(
50
+ "create-prompt-high",
51
+ z.object({ label: z.string(), body: z.string().optional() }),
52
+ async (event, ctx) => promptExecutor.create(event.payload, event.user, ctx.db),
53
+ { access: { roles: ["Admin"] }, agent: { risk: "high" } },
54
+ );
55
+
56
+ r.writeHandler(
57
+ "update-prompt-mid",
58
+ z.object({ id: z.uuid(), label: z.string().optional(), body: z.string().optional() }),
59
+ async (event, ctx) => {
60
+ const { id, ...changes } = event.payload;
61
+ return promptExecutor.update({ id, changes }, event.user, ctx.db, {
62
+ skipOptimisticLock: true,
63
+ });
64
+ },
65
+ { access: { roles: ["Admin"] } },
66
+ );
67
+
68
+ r.writeHandler(
69
+ "delegate-update-to-standard",
70
+ z.object({ id: z.uuid(), body: z.string() }),
71
+ async (event, ctx) => {
72
+ const row = await promptExecutor.detail({ id: event.payload.id }, event.user, ctx.db);
73
+ const version = typeof row?.["version"] === "number" ? row["version"] : 0;
74
+ return ctx.write("instructiongate:write:prompt:update", {
75
+ id: event.payload.id,
76
+ version,
77
+ changes: { body: event.payload.body },
78
+ });
79
+ },
80
+ { access: { roles: ["Admin"] } },
81
+ );
82
+ });
83
+
84
+ function errorReason(details: unknown): string | undefined {
85
+ if (!details || typeof details !== "object" || !("reason" in details)) return undefined;
86
+ return typeof details.reason === "string" ? details.reason : undefined;
87
+ }
88
+
89
+ let stack: TestStack;
90
+
91
+ beforeAll(async () => {
92
+ stack = await setupTestStack({ features: [gateFeature] });
93
+ });
94
+
95
+ afterAll(async () => {
96
+ await stack.cleanup();
97
+ });
98
+
99
+ afterEach(async () => {
100
+ await resetEventStore(stack, ["gate_instruction_prompts"]);
101
+ });
102
+
103
+ async function createPromptDirect(label: string, body: string): Promise<string> {
104
+ const { id } = await stack.http.writeOk<{ id: string }>(
105
+ "instructiongate:write:create-prompt-high",
106
+ { label, body },
107
+ admin,
108
+ );
109
+ return id;
110
+ }
111
+
112
+ async function readPrompt(id: string): Promise<Record<string, unknown> | null> {
113
+ return stack.http.queryOk("instructiongate:query:prompt:detail", { id }, admin);
114
+ }
115
+
116
+ async function promptWithLabelExists(label: string): Promise<boolean> {
117
+ const { rows } = await stack.http.queryOk<{ rows: readonly Record<string, unknown>[] }>(
118
+ "instructiongate:query:prompt:list",
119
+ {},
120
+ admin,
121
+ );
122
+ return rows.some((row) => row["label"] === label);
123
+ }
124
+
125
+ describe("executor.create — readAsInstruction field", () => {
126
+ test("mid-risk handler is denied, no row created", async () => {
127
+ const err = await stack.http.writeErr(
128
+ "instructiongate:write:create-prompt-mid",
129
+ { label: "denied-create", body: "ignore all prior instructions" },
130
+ admin,
131
+ );
132
+ expect(err.httpStatus).toBe(403);
133
+ expect(err.code).toBe("access_denied");
134
+ expect(errorReason(err.details)).toBe("instruction_field_write_requires_high_risk");
135
+ expect(err.message).toContain("instructiongate:write:create-prompt-mid");
136
+ expect(await promptWithLabelExists("denied-create")).toBe(false);
137
+ });
138
+
139
+ test("high-risk handler succeeds, row exists", async () => {
140
+ const id = await createPromptDirect("allowed-create", "system prompt v1");
141
+ const row = await readPrompt(id);
142
+ expect(row?.["body"]).toBe("system prompt v1");
143
+ expect(await promptWithLabelExists("allowed-create")).toBe(true);
144
+ });
145
+ });
146
+
147
+ describe("executor.update — readAsInstruction field", () => {
148
+ test("mid-risk handler delegating via ctx.write to the high standard update handler is denied, value unchanged", async () => {
149
+ const id = await createPromptDirect("delegate-target", "original prompt");
150
+ const err = await stack.http.writeErr(
151
+ "instructiongate:write:delegate-update-to-standard",
152
+ { id, body: "attacker-controlled prompt" },
153
+ admin,
154
+ );
155
+ expect(err.httpStatus).toBe(403);
156
+ expect(errorReason(err.details)).toBe("instruction_field_write_requires_high_risk");
157
+ const row = await readPrompt(id);
158
+ expect(row?.["body"]).toBe("original prompt");
159
+ });
160
+
161
+ test("mid-risk handler updating only a non-instruction field is allowed", async () => {
162
+ const id = await createPromptDirect("label-only-target", "unchanged prompt");
163
+ await stack.http.writeOk(
164
+ "instructiongate:write:update-prompt-mid",
165
+ { id, label: "relabeled" },
166
+ admin,
167
+ );
168
+ const row = await readPrompt(id);
169
+ expect(row?.["label"]).toBe("relabeled");
170
+ expect(row?.["body"]).toBe("unchanged prompt");
171
+ });
172
+
173
+ test("mid-risk handler with the instruction field in the payload is denied, value unchanged", async () => {
174
+ const id = await createPromptDirect("body-payload-target", "kept prompt");
175
+ const err = await stack.http.writeErr(
176
+ "instructiongate:write:update-prompt-mid",
177
+ { id, body: "smuggled prompt" },
178
+ admin,
179
+ );
180
+ expect(err.httpStatus).toBe(403);
181
+ expect(errorReason(err.details)).toBe("instruction_field_write_requires_high_risk");
182
+ const row = await readPrompt(id);
183
+ expect(row?.["body"]).toBe("kept prompt");
184
+ });
185
+ });
186
+
187
+ describe("standard entity-convention create/update handlers", () => {
188
+ test("resolve agent.risk to high because the entity has a readAsInstruction field", () => {
189
+ const registry = stack.registry as Registry;
190
+ const createHandler = registry.getWriteHandler("instructiongate:write:prompt:create");
191
+ const updateHandler = registry.getWriteHandler("instructiongate:write:prompt:update");
192
+ if (!createHandler || !updateHandler) throw new Error("standard handlers missing");
193
+ expect(resolveAgentExposure(createHandler, "write").risk).toBe("high");
194
+ expect(resolveAgentExposure(updateHandler, "write").risk).toBe("high");
195
+ });
196
+
197
+ test("standard create and update succeed over HTTP", async () => {
198
+ const { id } = await stack.http.writeOk<{ id: string }>(
199
+ "instructiongate:write:prompt:create",
200
+ { label: "standard-create", body: "initial prompt" },
201
+ admin,
202
+ );
203
+ const created = await readPrompt(id);
204
+ const version = typeof created?.["version"] === "number" ? created["version"] : 0;
205
+ await stack.http.writeOk(
206
+ "instructiongate:write:prompt:update",
207
+ { id, version, changes: { body: "revised prompt" } },
208
+ admin,
209
+ );
210
+ const row = await readPrompt(id);
211
+ expect(row?.["body"]).toBe("revised prompt");
212
+ });
213
+ });
@@ -51,6 +51,7 @@ export { ConsumerLagError, pruneEvents } from "./event-retention";
51
51
  export type { IdempotencyGuard } from "./idempotency";
52
52
  export { createIdempotencyGuard } from "./idempotency";
53
53
  export {
54
+ assertInstructionFieldWriteAllowed,
54
55
  assertIrreversibleOperationAllowed,
55
56
  isIrreversibleEntityVerb,
56
57
  } from "./irreversible-operation-gate";
@@ -1,6 +1,8 @@
1
- import { requestContext } from "../api/request-context";
1
+ import { type RequestContextData, requestContext } from "../api/request-context";
2
2
  import type { EntityDefinition } from "../engine/types";
3
- import { AccessDeniedError, FrameworkReasons } from "../errors";
3
+ import { AccessDeniedError, type FrameworkReason, FrameworkReasons } from "../errors";
4
+
5
+ type EntryHandler = NonNullable<RequestContextData["entryHandler"]>;
4
6
 
5
7
  // forget hard-purges regardless of softDelete; delete is only irreversible
6
8
  // when the entity has no soft-delete recovery path.
@@ -16,22 +18,58 @@ export function isIrreversibleEntityVerb(
16
18
  // executor/KMS calls without one stay ungated in exactly two cases: deleting derived data
17
19
  // rebuildable from a reversible source (document-ingest-foundation's forget-extract-with-file-ref
18
20
  // consumer, re-created on fileRef.restored), and pipelines started by a risk "high" handler or a
19
- // cron scan (forget cleanup, retention, tenant destroy). A deferred context can't return
20
- // irreversibleOperationRequiresHighRisk to the agent synchronously, so the gate sits at the primitives.
21
- export function assertIrreversibleOperationAllowed(operation: string): void {
21
+ // cron scan (forget cleanup, retention, tenant destroy). A deferred context can't return the
22
+ // reason to the agent synchronously, so the gate sits at the primitives.
23
+ function assertEntryHandlerHighRisk(args: {
24
+ reason: FrameworkReason;
25
+ buildMessage: (entryHandler: EntryHandler) => string;
26
+ details: Record<string, unknown>;
27
+ }): void {
22
28
  const entryHandler = requestContext.get()?.entryHandler;
23
29
  // skip: no entry handler (see above) or the entry handler already resolves "high".
24
30
  if (!entryHandler || entryHandler.risk === "high") return;
25
31
  throw new AccessDeniedError({
26
- message:
27
- `Handler "${entryHandler.qn}" performs an irreversible ${operation} but resolves agent risk ` +
28
- `"${entryHandler.risk}" — declare agent: { risk: "high" } on "${entryHandler.qn}". The ` +
29
- "directly called handler must carry it; delegating via ctx.write/writeAs does not.",
32
+ message: args.buildMessage(entryHandler),
30
33
  details: {
31
- reason: FrameworkReasons.irreversibleOperationRequiresHighRisk,
34
+ reason: args.reason,
32
35
  handler: entryHandler.qn,
33
- operation,
34
36
  resolvedRisk: entryHandler.risk,
37
+ ...args.details,
35
38
  },
36
39
  });
37
40
  }
41
+
42
+ export function assertIrreversibleOperationAllowed(operation: string): void {
43
+ assertEntryHandlerHighRisk({
44
+ reason: FrameworkReasons.irreversibleOperationRequiresHighRisk,
45
+ buildMessage: (entryHandler) =>
46
+ `Handler "${entryHandler.qn}" performs an irreversible ${operation} but resolves agent risk ` +
47
+ `"${entryHandler.risk}" — declare agent: { risk: "high" } on "${entryHandler.qn}". The ` +
48
+ "directly called handler must carry it; delegating via ctx.write/writeAs does not.",
49
+ details: { operation },
50
+ });
51
+ }
52
+
53
+ // Fields flagged readAsInstruction: true have their write payload presence checked here — the
54
+ // same reasoning as assertIrreversibleOperationAllowed, but the irreversible property belongs to
55
+ // the target field, not the verb. Presence in the payload (not a value comparison against the
56
+ // loaded row) is the deliberate fail-closed choice — see kumiko-framework#3358.
57
+ export function assertInstructionFieldWriteAllowed(
58
+ entityName: string,
59
+ verb: "create" | "update",
60
+ writtenFieldNames: readonly string[],
61
+ instructionFieldNames: readonly string[],
62
+ ): void {
63
+ const fields = writtenFieldNames.filter((name) => instructionFieldNames.includes(name));
64
+ // skip: the write touches no readAsInstruction field.
65
+ if (fields.length === 0) return;
66
+ assertEntryHandlerHighRisk({
67
+ reason: FrameworkReasons.instructionFieldWriteRequiresHighRisk,
68
+ buildMessage: (entryHandler) =>
69
+ `Handler "${entryHandler.qn}" ${verb}s "${entityName}" field(s) ${fields.join(", ")} ` +
70
+ `flagged readAsInstruction: true but resolves agent risk "${entryHandler.risk}" — declare ` +
71
+ `agent: { risk: "high" } on "${entryHandler.qn}". The directly called handler must carry ` +
72
+ "it; delegating via ctx.write/writeAs does not.",
73
+ details: { entityName, verb, fields },
74
+ });
75
+ }
@@ -0,0 +1,159 @@
1
+ // fw#3360: stack.drainJobs() waits deterministically for every in-flight
2
+ // job AND its follow-up event-consumer / job-trigger cascade — replaces
3
+ // `waitFor` polling on a specific job's side effect.
4
+
5
+ import { afterEach, describe, expect, test } from "bun:test";
6
+ import * as z from "zod";
7
+ import { defineFeature } from "../../engine";
8
+ import { SYSTEM_ROLE } from "../../engine/system-user";
9
+ import { setupTestStack, type TestStack } from "../test-stack";
10
+ import { TestUsers } from "../test-users";
11
+
12
+ const chainResults: Array<{ value: string }> = [];
13
+
14
+ // Job A → write handler that appends an r.defineEvent event → the async
15
+ // job-trigger event-consumer → Job B. Exercises exactly the gap drainJobs
16
+ // exists for: a job-completion that only cascades into a follow-up job via
17
+ // the event-dispatcher, not the synchronous write-handler dispatch path.
18
+ const chainFeature = defineFeature("draintestchain", (r) => {
19
+ const relayed = r.defineEvent("relayed", z.object({ value: z.string() }), {
20
+ piiFields: "none",
21
+ });
22
+
23
+ r.writeHandler(
24
+ "start",
25
+ z.object({ value: z.string() }),
26
+ async () => ({ isSuccess: true as const, data: {} }),
27
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
28
+ );
29
+
30
+ r.writeHandler(
31
+ "emit-relayed",
32
+ z.object({ value: z.string() }),
33
+ async (event, ctx) => {
34
+ await ctx.unsafeAppendEvent({
35
+ aggregateId: crypto.randomUUID(),
36
+ aggregateType: "draintestchain",
37
+ type: relayed.name,
38
+ payload: { value: event.payload.value },
39
+ });
40
+ return { isSuccess: true as const, data: {} };
41
+ },
42
+ { access: { roles: [SYSTEM_ROLE] } },
43
+ );
44
+
45
+ r.job(
46
+ "job-a",
47
+ { trigger: { on: "draintestchain:write:start" }, retries: 0 },
48
+ async (payload, ctx) => {
49
+ await ctx.write("draintestchain:write:emit-relayed", { value: payload["value"] });
50
+ },
51
+ );
52
+
53
+ r.job("job-b", { trigger: { on: relayed.name }, retries: 0 }, async (payload) => {
54
+ chainResults.push({ value: payload["value"] as string }); // @cast-boundary test-fixture
55
+ });
56
+ });
57
+
58
+ const alwaysFailsRuns: string[] = [];
59
+ const retrySucceedsAttempts: number[] = [];
60
+
61
+ const failureFeature = defineFeature("draintestfailure", (r) => {
62
+ r.job("always-fails", { trigger: { manual: true }, retries: 0 }, async () => {
63
+ alwaysFailsRuns.push("ran");
64
+ throw new Error("job-c always fails");
65
+ });
66
+
67
+ r.job(
68
+ "retry-succeeds",
69
+ { trigger: { manual: true }, retries: 1, backoff: { type: "fixed", delayMs: 10 } },
70
+ async () => {
71
+ retrySucceedsAttempts.push(Date.now());
72
+ if (retrySucceedsAttempts.length === 1) throw new Error("first attempt fails on purpose");
73
+ },
74
+ );
75
+ });
76
+
77
+ let stack: TestStack | undefined;
78
+
79
+ afterEach(async () => {
80
+ if (stack) await stack.cleanup();
81
+ stack = undefined;
82
+ });
83
+
84
+ describe("stack.drainJobs() waits for a job's async event-trigger cascade", () => {
85
+ test("job A → event → job B settles before drainJobs() resolves", async () => {
86
+ chainResults.length = 0;
87
+ stack = await setupTestStack({
88
+ features: [chainFeature],
89
+ jobs: { consumerLane: "worker" },
90
+ });
91
+
92
+ await stack.http.writeOk("draintestchain:write:start", { value: "hi" }, TestUsers.admin);
93
+ await stack.drainJobs();
94
+
95
+ expect(chainResults).toEqual([{ value: "hi" }]);
96
+ });
97
+
98
+ test("a huge event-dispatcher poll interval doesn't stop drainJobs() from driving the cascade itself", async () => {
99
+ chainResults.length = 0;
100
+ stack = await setupTestStack({
101
+ features: [chainFeature],
102
+ jobs: { consumerLane: "worker" },
103
+ });
104
+ // stop() tears down both the poll timer AND the LISTEN/NOTIFY
105
+ // subscription (idempotent — cleanup()'s own stop() later is a no-op).
106
+ // runOnce() still works afterward (event-dispatcher.ts's own contract),
107
+ // so this leaves drainJobs() as the ONLY thing that can advance the
108
+ // job-trigger consumer past job A's emitted event — without this, the
109
+ // always-on pgClient wake-up races the assertion regardless of
110
+ // eventDispatcherPollIntervalMs.
111
+ await stack.eventDispatcher?.stop();
112
+
113
+ await stack.http.writeOk("draintestchain:write:start", { value: "race" }, TestUsers.admin);
114
+ await stack.drainJobs();
115
+
116
+ expect(chainResults).toEqual([{ value: "race" }]);
117
+ });
118
+ });
119
+
120
+ describe("stack.drainJobs() surfaces final job failures", () => {
121
+ test("a job that always throws rejects drainJobs() with its name and error", async () => {
122
+ alwaysFailsRuns.length = 0;
123
+ stack = await setupTestStack({
124
+ features: [failureFeature],
125
+ jobs: { consumerLane: "worker" },
126
+ });
127
+
128
+ await stack.jobRunner?.dispatch("draintestfailure:job:always-fails");
129
+
130
+ await expect(stack.drainJobs()).rejects.toThrow(
131
+ /draintestfailure:job:always-fails.*job-c always fails/s,
132
+ );
133
+
134
+ // The tracker is cleared after a rejection — a later drainJobs() call
135
+ // (e.g. the next test's setup) must not still see this failure.
136
+ await expect(stack.drainJobs()).resolves.toBeUndefined();
137
+ });
138
+
139
+ test("a job with retries:1 that fails once then succeeds resolves drainJobs()", async () => {
140
+ retrySucceedsAttempts.length = 0;
141
+ stack = await setupTestStack({
142
+ features: [failureFeature],
143
+ jobs: { consumerLane: "worker" },
144
+ });
145
+
146
+ await stack.jobRunner?.dispatch("draintestfailure:job:retry-succeeds");
147
+
148
+ await expect(stack.drainJobs()).resolves.toBeUndefined();
149
+ expect(retrySucceedsAttempts).toHaveLength(2);
150
+ });
151
+ });
152
+
153
+ describe("stack.drainJobs() without a jobRunner", () => {
154
+ test("rejects with a clear message when the stack has no `jobs` opt-in", async () => {
155
+ stack = await setupTestStack({ features: [chainFeature] });
156
+
157
+ await expect(stack.drainJobs()).rejects.toThrow(/jobRunner is undefined/);
158
+ });
159
+ });
@@ -0,0 +1,129 @@
1
+ import type { DbConnection } from "../db";
2
+ import { getEventsHighWaterMark } from "../event-store";
3
+ import type { JobRunner } from "../jobs";
4
+ import { type EventDispatcher, getConsumerState } from "../pipeline";
5
+
6
+ // Gap between idle-checks while waiting for a job hook to fire. Short enough
7
+ // that a test never feels it, long enough that consecutive idle passes don't
8
+ // hammer Postgres/Redis with pointless polling between hook-driven wakeups.
9
+ const DRAIN_IDLE_POLL_MS = 25;
10
+ // Two consecutive idle passes (not one) before declaring the stack drained —
11
+ // a single "pending===0 and consumers caught up" pass can be a race: a job
12
+ // hook fires and enqueues a follow-up between the countPendingJobs() read
13
+ // and the consumer check of the SAME pass. A DRAIN_IDLE_POLL_MS wait sits
14
+ // between every pass, including the first and second idle one, so an
15
+ // enqueue that lands just after the first idle read still gets picked up
16
+ // by the second pass instead of racing straight past it.
17
+ const REQUIRED_IDLE_PASSES = 2;
18
+
19
+ function sleep(ms: number): Promise<void> {
20
+ return new Promise((resolve) => setTimeout(resolve, ms));
21
+ }
22
+
23
+ export type JobFailureTracker = {
24
+ readonly onJobComplete: (jobName: string, jobId: string) => void;
25
+ readonly onJobFailed: (jobName: string, jobId: string, error: string) => void;
26
+ waitForActivity(): Promise<void>;
27
+ readonly failures: ReadonlyMap<string, { readonly jobName: string; readonly error: string }>;
28
+ clear(): void;
29
+ };
30
+
31
+ export function createJobFailureTracker(): JobFailureTracker {
32
+ const failures = new Map<string, { jobName: string; error: string }>();
33
+ let wake: (() => void) | undefined;
34
+ let armed: Promise<void> | undefined;
35
+
36
+ function fire(): void {
37
+ wake?.();
38
+ armed = undefined;
39
+ wake = undefined;
40
+ }
41
+
42
+ return {
43
+ onJobComplete(_jobName, jobId) {
44
+ // A retry that eventually succeeds must not leave a stale failure
45
+ // behind from its earlier attempt(s).
46
+ failures.delete(jobId);
47
+ fire();
48
+ },
49
+ onJobFailed(jobName, jobId, error) {
50
+ failures.set(jobId, { jobName, error });
51
+ fire();
52
+ },
53
+ waitForActivity() {
54
+ armed ??= new Promise<void>((resolve) => {
55
+ wake = resolve;
56
+ });
57
+ return armed;
58
+ },
59
+ failures,
60
+ clear() {
61
+ failures.clear();
62
+ },
63
+ };
64
+ }
65
+
66
+ export type DrainJobsStack = {
67
+ readonly db: DbConnection;
68
+ readonly eventDispatcher?: EventDispatcher;
69
+ readonly jobRunner?: JobRunner;
70
+ };
71
+
72
+ async function sharedConsumersCaughtUp(
73
+ db: DbConnection,
74
+ consumerNames: readonly string[],
75
+ ): Promise<boolean> {
76
+ // skip: no shared consumers wired — nothing to catch up on.
77
+ if (consumerNames.length === 0) return true;
78
+ const target = await getEventsHighWaterMark(db);
79
+ let allCaughtUp = true;
80
+ for (const name of consumerNames) {
81
+ const state = await getConsumerState(db, name);
82
+ if (!state) {
83
+ throw new Error(`drainJobs: consumer "${name}" not registered or per-instance`);
84
+ }
85
+ if (state.status === "dead") {
86
+ throw new Error(
87
+ `drainJobs: consumer "${state.name}" is dead — lastError: ${state.lastError ?? "unknown"}`,
88
+ );
89
+ }
90
+ if (state.lastProcessedEventId < target) allCaughtUp = false;
91
+ }
92
+ return allCaughtUp;
93
+ }
94
+
95
+ export async function drainJobs(stack: DrainJobsStack, tracker: JobFailureTracker): Promise<void> {
96
+ const { db, eventDispatcher, jobRunner } = stack;
97
+ if (!jobRunner) {
98
+ throw new Error(
99
+ "drainJobs: stack.jobRunner is undefined — pass `jobs: {...}` to setupTestStack and " +
100
+ "register at least one r.job(...) so a runner is built.",
101
+ );
102
+ }
103
+
104
+ const sharedConsumerNames = (eventDispatcher?.consumers ?? [])
105
+ .filter((consumer) => (consumer.delivery ?? "shared") === "shared")
106
+ .map((consumer) => consumer.name);
107
+
108
+ try {
109
+ let consecutiveIdlePasses = 0;
110
+ while (consecutiveIdlePasses < REQUIRED_IDLE_PASSES) {
111
+ await eventDispatcher?.runOnce();
112
+ const pending = await jobRunner.countPendingJobs();
113
+ const caughtUp = await sharedConsumersCaughtUp(db, sharedConsumerNames);
114
+ consecutiveIdlePasses = pending === 0 && caughtUp ? consecutiveIdlePasses + 1 : 0;
115
+ if (consecutiveIdlePasses < REQUIRED_IDLE_PASSES) {
116
+ await Promise.race([tracker.waitForActivity(), sleep(DRAIN_IDLE_POLL_MS)]);
117
+ }
118
+ }
119
+
120
+ if (tracker.failures.size > 0) {
121
+ const details = [...tracker.failures.values()]
122
+ .map((failure) => `${failure.jobName}: ${failure.error}`)
123
+ .join("; ");
124
+ throw new Error(`drainJobs: job(s) failed: ${details}`);
125
+ }
126
+ } finally {
127
+ tracker.clear();
128
+ }
129
+ }
@@ -22,6 +22,7 @@ import { createRateLimitResolver } from "../rate-limit";
22
22
  import { createInMemorySearchAdapter } from "../search";
23
23
  import type { SearchAdapter } from "../search/types";
24
24
  import { createTestDb } from "./db";
25
+ import { createJobFailureTracker, drainJobs } from "./drain-jobs";
25
26
  import { createEventCollector, type EventCollector } from "./event-collector";
26
27
  import { createTestRedis, type TestRedis } from "./redis";
27
28
  import { createRequestHelper, type RequestHelper } from "./request-helper";
@@ -64,6 +65,16 @@ export type TestStack = {
64
65
  // registered in the mounted features. Lets integration tests call
65
66
  // `stack.jobRunner.dispatch(...)` directly, mirroring `ctx.jobRunner`.
66
67
  jobRunner?: JobRunner;
68
+ // Waits for every in-flight job and its follow-up event-consumer /
69
+ // job-trigger cascade to settle — instead of `waitFor` polling on a
70
+ // specific job's side effect. Rejects with the failing job's name and
71
+ // error once a job exhausts its retries, or with a clear error when this
72
+ // stack has no jobRunner at all. Tracked failures accumulate since the
73
+ // PREVIOUS drainJobs() call (or setup, if none yet) — on a shared stack
74
+ // (e.g. beforeAll), a test that deliberately fails a job and never calls
75
+ // drainJobs() itself leaves that failure for the next test's drainJobs()
76
+ // call to reject on.
77
+ drainJobs: () => Promise<void>;
67
78
  cleanup: () => Promise<void>;
68
79
  };
69
80
 
@@ -361,6 +372,11 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
361
372
  // context = appContext + tracer/meter, mirroring the prod entrypoint's
362
373
  // `contextWithObservability(options.context, observability)`.
363
374
  let jobRunner: JobRunner | undefined;
375
+ // Fed to createJobRunner's onJobComplete/onJobFailed below so drainJobs()
376
+ // can tell a job's final outcome apart from an attempt that will retry —
377
+ // built before createJobRunner regardless of whether a runner ends up
378
+ // existing, so the returned drainJobs() closure always has one to read.
379
+ const jobFailureTracker = createJobFailureTracker();
364
380
  if (options.jobs && registry.getAllJobs().size > 0) {
365
381
  jobRunner = createJobRunner({
366
382
  registry,
@@ -374,6 +390,8 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
374
390
  ...(options.jobs.getActiveTenantIds !== undefined && {
375
391
  getActiveTenantIds: options.jobs.getActiveTenantIds,
376
392
  }),
393
+ onJobComplete: (jobName, jobId) => jobFailureTracker.onJobComplete(jobName, jobId),
394
+ onJobFailed: (jobName, jobId, error) => jobFailureTracker.onJobFailed(jobName, jobId, error),
377
395
  });
378
396
  }
379
397
 
@@ -540,6 +558,7 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
540
558
  ...(eventDispatcher ? { eventDispatcher } : {}),
541
559
  ...(server.lifecycle ? { lifecycle: server.lifecycle } : {}),
542
560
  ...(jobRunner ? { jobRunner } : {}),
561
+ drainJobs: () => drainJobs({ db: testDb.db, eventDispatcher, jobRunner }, jobFailureTracker),
543
562
  cleanup: async () => {
544
563
  if (jobRunner) await jobRunner.stop();
545
564
  if (eventDispatcher) await eventDispatcher.stop();