@cosmicdrift/kumiko-framework 0.299.0 → 0.305.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/package.json +4 -4
  2. package/src/api/__tests__/api.test.ts +3 -2
  3. package/src/api/__tests__/extra-route-rejection.test.ts +38 -0
  4. package/src/api/__tests__/extra-routes.integration.test.ts +30 -0
  5. package/src/api/__tests__/http-route-entry.integration.test.ts +114 -0
  6. package/src/api/__tests__/server-error-logging.test.ts +33 -0
  7. package/src/api/api-constants.ts +13 -0
  8. package/src/api/auth-routes.ts +53 -24
  9. package/src/api/extra-route.ts +33 -4
  10. package/src/api/index.ts +1 -0
  11. package/src/api/server.ts +79 -34
  12. package/src/changes.json +68 -0
  13. package/src/db/event-store-executor-write.ts +7 -0
  14. package/src/db/tenant-db.ts +50 -3
  15. package/src/engine/__tests__/http-route-anonymous-required.test.ts +43 -0
  16. package/src/engine/__tests__/membership-roles.test.ts +13 -4
  17. package/src/engine/boot-validator/__tests__/access-declarations.test.ts +127 -0
  18. package/src/engine/boot-validator/__tests__/no-all-role-in-handler-access.test.ts +77 -0
  19. package/src/engine/boot-validator/access-declarations.ts +58 -67
  20. package/src/engine/boot-validator/entity-handler.ts +20 -0
  21. package/src/engine/feature-ast/__tests__/patch.test.ts +1 -0
  22. package/src/engine/feature-ast/__tests__/patcher.test.ts +1 -0
  23. package/src/engine/feature-ast/__tests__/read-optional-access-rule.test.ts +14 -0
  24. package/src/engine/feature-ast/extractors/hooks.ts +3 -1
  25. package/src/engine/feature-ast/extractors/jobs-routes.ts +5 -2
  26. package/src/engine/feature-ast/patcher.ts +2 -2
  27. package/src/engine/feature-ast/patterns.ts +1 -1
  28. package/src/engine/feature-ast/render.ts +1 -1
  29. package/src/engine/feature-ui-extensions.ts +6 -0
  30. package/src/engine/index.ts +4 -0
  31. package/src/engine/membership-roles.ts +20 -4
  32. package/src/engine/pattern-library/__tests__/library.test.ts +1 -0
  33. package/src/engine/pattern-library/mixed-schemas.ts +1 -0
  34. package/src/engine/personal-data-fields.ts +66 -0
  35. package/src/engine/registry-validate.ts +15 -0
  36. package/src/engine/registry.ts +2 -0
  37. package/src/engine/types/index.ts +4 -0
  38. package/src/env/__tests__/dry-run.test.ts +43 -3
  39. package/src/env/dry-run.ts +28 -15
  40. package/src/errors/__tests__/write-failures.test.ts +47 -4
  41. package/src/errors/i18n/de.yaml +12 -0
  42. package/src/errors/i18n/en.yaml +12 -0
  43. package/src/errors/reasons.ts +4 -0
  44. package/src/errors/write-error-info.ts +12 -3
  45. package/src/jobs/__tests__/job-backoff.integration.test.ts +155 -0
  46. package/src/jobs/__tests__/jobs.integration.test.ts +35 -0
  47. package/src/jobs/job-runner.ts +19 -5
  48. package/src/observability/__tests__/metrics-wiring.test.ts +61 -0
  49. package/src/observability/index.ts +5 -0
  50. package/src/observability/metrics-wiring.ts +32 -0
  51. package/src/pipeline/__tests__/public-intake-runtime-gate.integration.test.ts +425 -0
  52. package/src/pipeline/active-membership.ts +5 -1
  53. package/src/pipeline/dispatch-batch.ts +3 -0
  54. package/src/pipeline/dispatch-query.ts +16 -5
  55. package/src/pipeline/dispatch-shared.ts +12 -5
  56. package/src/pipeline/dispatch-stream.ts +7 -2
  57. package/src/pipeline/dispatch-write.ts +22 -5
  58. package/src/pipeline/dispatcher.ts +9 -2
  59. package/src/pipeline/member-reader.ts +3 -1
  60. package/src/pipeline/write-origin.ts +107 -0
  61. package/src/rate-limit/__tests__/middleware.integration.test.ts +40 -0
  62. package/src/rate-limit/middleware.ts +3 -0
  63. package/src/stack/__tests__/setup-test-stack-metrics.integration.test.ts +79 -0
  64. package/src/stack/test-stack.ts +5 -0
  65. package/src/testing/handler-context.ts +3 -1
  66. package/src/ui-types/index.ts +2 -0
@@ -1,6 +1,7 @@
1
1
  import { describe, expect, it } from "bun:test";
2
2
  import { z } from "zod";
3
3
  import { defineFeature } from "../../engine/define-feature";
4
+ import { prometheusMetricsEnvSchema } from "../../observability/metrics-wiring";
4
5
  import { renderDryRun } from "../dry-run";
5
6
  import { composeEnvSchema } from "../index";
6
7
 
@@ -90,7 +91,7 @@ describe("renderDryRun", () => {
90
91
  expect(ver?.default).toBe("1");
91
92
  });
92
93
 
93
- it("pulumi mode emits `pulumi config set` lines, omitting optional+defaulted", () => {
94
+ it("pulumi mode emits required lines, optional ones commented out, omits defaulted", () => {
94
95
  const composed = buildComposed();
95
96
  const out = renderDryRun(composed, "pulumi", { pulumiPrefix: "studio" });
96
97
  expect(out).toContain(
@@ -100,9 +101,36 @@ describe("renderDryRun", () => {
100
101
  'pulumi config set --secret studioSecretsMasterKey "$(openssl rand -base64 32)"',
101
102
  );
102
103
  expect(out).toContain('pulumi config set studioStudioAdminEmail "<set-me>"');
103
- // Optional + default skipped:
104
- expect(out).not.toContain("SMTP_HOST");
104
+ expect(out).toContain(
105
+ '# pulumi config set studioSmtpHost "<set-me>" # SMTP_HOST (channel-email-smtp): Outbound SMTP host',
106
+ );
107
+ expect(out).not.toMatch(/^pulumi config set.*SMTP_HOST/m);
105
108
  expect(out).not.toContain("CURRENT_VERSION");
109
+ const requiredLineIndex = out.lastIndexOf("pulumi config set studioStudioAdminEmail");
110
+ const optionalHeaderIndex = out.indexOf("# Optional (uncomment and set to enable):");
111
+ expect(requiredLineIndex).toBeGreaterThanOrEqual(0);
112
+ expect(optionalHeaderIndex).toBeGreaterThan(requiredLineIndex);
113
+ });
114
+
115
+ it("pulumi mode lists an optional secret field (PROMETHEUS_METRICS_TOKEN) commented out with generator", () => {
116
+ const composed = composeEnvSchema({
117
+ features: [],
118
+ extend: z.object({ FOO: z.string() }).extend(prometheusMetricsEnvSchema.shape),
119
+ });
120
+ const out = renderDryRun(composed, "pulumi");
121
+ expect(out).toContain(
122
+ '# pulumi config set --secret prometheusMetricsToken "$(openssl rand -base64 32)" # PROMETHEUS_METRICS_TOKEN (app): Bearer token for /metrics; unset keeps the endpoint off.',
123
+ );
124
+ expect(out).not.toMatch(/^pulumi config set.*PROMETHEUS_METRICS_TOKEN/m);
125
+ });
126
+
127
+ it("pulumi mode omits the Optional header when there are no optional fields", () => {
128
+ const composed = composeEnvSchema({
129
+ features: [],
130
+ extend: z.object({ FOO: z.string() }),
131
+ });
132
+ const out = renderDryRun(composed, "pulumi");
133
+ expect(out).not.toContain("# Optional");
106
134
  });
107
135
 
108
136
  it("k8s mode emits a Secret manifest", () => {
@@ -117,5 +145,17 @@ describe("renderDryRun", () => {
117
145
  expect(out).toContain("namespace: studio");
118
146
  expect(out).toContain('JWT_SECRET: "<set-me>"');
119
147
  expect(out).toContain('KUMIKO_SECRETS_MASTER_KEY_V1: "<set-me>"');
148
+ expect(out).toContain(' # SMTP_HOST: "<set-me>" # (channel-email-smtp): Outbound SMTP host');
149
+ expect(out).not.toMatch(/^ {2}SMTP_HOST:/m);
150
+ expect(out).not.toContain("CURRENT_VERSION");
151
+ });
152
+
153
+ it("k8s mode omits the Optional header when there are no optional fields", () => {
154
+ const composed = composeEnvSchema({
155
+ features: [],
156
+ extend: z.object({ FOO: z.string() }),
157
+ });
158
+ const out = renderDryRun(composed, "k8s");
159
+ expect(out).not.toContain("# Optional");
120
160
  });
121
161
  });
@@ -152,20 +152,27 @@ function renderJson(fields: readonly EnvField[], options: DryRunOptions): string
152
152
  )}\n`;
153
153
  }
154
154
 
155
+ function pulumiConfigSetLine(f: EnvField, options: DryRunOptions): string {
156
+ const meta = readKumikoMeta(f.field);
157
+ const key = pulumiConfigKey(f.name, f.field, options.pulumiPrefix);
158
+ const secretFlag = meta.pulumi?.secret ? " --secret" : "";
159
+ const value = meta.pulumi?.generator ? `"$(${meta.pulumi.generator})"` : `"<set-me>"`;
160
+ const comment = f.description
161
+ ? ` # ${f.name} (${f.source}): ${f.description}`
162
+ : ` # ${f.name} (${f.source})`;
163
+ return `pulumi config set${secretFlag} ${key} ${value}${comment}`;
164
+ }
165
+
155
166
  function renderPulumi(fields: readonly EnvField[], options: DryRunOptions): string {
156
167
  // Defaulted vars are skipped — the framework provides them, ops doesn't.
157
- const lines: string[] = [];
158
- for (const f of fields) {
159
- if (f.klass === "withDefault") continue;
160
- if (f.klass === "optional") continue;
161
- const meta = readKumikoMeta(f.field);
162
- const key = pulumiConfigKey(f.name, f.field, options.pulumiPrefix);
163
- const secretFlag = meta.pulumi?.secret ? " --secret" : "";
164
- const value = meta.pulumi?.generator ? `"$(${meta.pulumi.generator})"` : `"<set-me>"`;
165
- const comment = f.description
166
- ? ` # ${f.name} (${f.source}): ${f.description}`
167
- : ` # ${f.name} (${f.source})`;
168
- lines.push(`pulumi config set${secretFlag} ${key} ${value}${comment}`);
168
+ // Optional vars stay commented out: setting one turns its feature on.
169
+ const required = fields.filter((f) => f.klass === "required");
170
+ const optional = fields.filter((f) => f.klass === "optional");
171
+ const lines: string[] = required.map((f) => pulumiConfigSetLine(f, options));
172
+ if (optional.length > 0) {
173
+ if (lines.length > 0) lines.push("");
174
+ lines.push("# Optional (uncomment and set to enable):");
175
+ for (const f of optional) lines.push(`# ${pulumiConfigSetLine(f, options)}`);
169
176
  }
170
177
  return `${lines.join("\n")}\n`;
171
178
  }
@@ -183,9 +190,15 @@ function renderK8s(fields: readonly EnvField[], options: DryRunOptions): string
183
190
  "stringData:",
184
191
  ];
185
192
  for (const f of fields) {
186
- if (f.klass === "withDefault") continue;
187
- if (f.klass === "optional") continue;
188
- lines.push(` ${f.name}: "<set-me>"`);
193
+ if (f.klass === "required") lines.push(` ${f.name}: "<set-me>"`);
194
+ }
195
+ const optional = fields.filter((f) => f.klass === "optional");
196
+ if (optional.length > 0) {
197
+ lines.push(" # Optional (uncomment and set to enable):");
198
+ for (const f of optional) {
199
+ const comment = f.description ? ` # (${f.source}): ${f.description}` : ` # (${f.source})`;
200
+ lines.push(` # ${f.name}: "<set-me>"${comment}`);
201
+ }
189
202
  }
190
203
  return `${lines.join("\n")}\n`;
191
204
  }
@@ -1,5 +1,12 @@
1
1
  import { describe, expect, test } from "bun:test";
2
- import { failNotFound, failTransition, failUnprocessable } from "../write-error-info";
2
+ import { InternalError, NotFoundError } from "../classes";
3
+ import {
4
+ failNotFound,
5
+ failTransition,
6
+ failUnprocessable,
7
+ reraiseAsKumikoError,
8
+ toWriteErrorInfo,
9
+ } from "../write-error-info";
3
10
 
4
11
  describe("failNotFound", () => {
5
12
  test("baut WriteFailure mit reason=not_found + entity-id-details", () => {
@@ -73,7 +80,6 @@ describe("failTransition", () => {
73
80
  describe("toWriteErrorInfo — dev cause-snapshot", () => {
74
81
  test("InternalError mit cause exposed cause-Snapshot in details (dev)", async () => {
75
82
  const { toWriteErrorInfo } = await import("../write-error-info");
76
- const { InternalError } = await import("../classes");
77
83
  const previous = process.env["NODE_ENV"];
78
84
  process.env["NODE_ENV"] = "development";
79
85
  try {
@@ -93,7 +99,6 @@ describe("toWriteErrorInfo — dev cause-snapshot", () => {
93
99
 
94
100
  test("Production: InternalError lässt details undefined (kein Stack-Leak)", async () => {
95
101
  const { toWriteErrorInfo } = await import("../write-error-info");
96
- const { InternalError } = await import("../classes");
97
102
  const previous = process.env["NODE_ENV"];
98
103
  process.env["NODE_ENV"] = "production";
99
104
  try {
@@ -107,7 +112,6 @@ describe("toWriteErrorInfo — dev cause-snapshot", () => {
107
112
 
108
113
  test("InternalError MIT bereits gesetztem details → Author-details gewinnt (kein Overwrite)", async () => {
109
114
  const { toWriteErrorInfo } = await import("../write-error-info");
110
- const { InternalError } = await import("../classes");
111
115
  const previous = process.env["NODE_ENV"];
112
116
  process.env["NODE_ENV"] = "development";
113
117
  try {
@@ -122,3 +126,42 @@ describe("toWriteErrorInfo — dev cause-snapshot", () => {
122
126
  }
123
127
  });
124
128
  });
129
+
130
+ // The cause must reach routes.ts's logServerFault via reraiseAsKumikoError,
131
+ // but never the wire body or the idempotency cache (both serialize the info).
132
+ describe("toWriteErrorInfo / reraiseAsKumikoError — cause round-trip", () => {
133
+ test("cause survives toWriteErrorInfo → reraiseAsKumikoError without appearing on the info object", () => {
134
+ const boom = new Error("connection was closed");
135
+ const info = toWriteErrorInfo(new InternalError({ cause: boom }));
136
+ expect(reraiseAsKumikoError(info).cause).toBe(boom);
137
+ expect(Object.keys(info)).not.toContain("cause");
138
+ });
139
+
140
+ test("production: cause still reaches reraise even though details/message stay sanitized", () => {
141
+ const previous = process.env["NODE_ENV"];
142
+ process.env["NODE_ENV"] = "production";
143
+ try {
144
+ const boom = new Error("connection was closed");
145
+ const info = toWriteErrorInfo(new InternalError({ cause: boom }));
146
+ expect(info.details).toBeUndefined();
147
+ const serialized = JSON.stringify(info);
148
+ expect(serialized).not.toContain("connection was closed");
149
+ expect(serialized).not.toContain("cause");
150
+ expect(reraiseAsKumikoError(info).cause).toBe(boom);
151
+ } finally {
152
+ if (previous === undefined) delete process.env["NODE_ENV"];
153
+ else process.env["NODE_ENV"] = previous;
154
+ }
155
+ });
156
+
157
+ test("KumikoError without a cause → reraised error has no cause", () => {
158
+ const info = toWriteErrorInfo(new NotFoundError("invoice", "inv-1"));
159
+ expect(reraiseAsKumikoError(info).cause).toBeUndefined();
160
+ });
161
+
162
+ test("a plain object with the same shape (simulated idempotency-cache replay) carries no cause", () => {
163
+ const info = toWriteErrorInfo(new InternalError({ cause: new Error("connection was closed") }));
164
+ const replayed = { ...info };
165
+ expect(reraiseAsKumikoError(replayed).cause).toBeUndefined();
166
+ });
167
+ });
@@ -147,6 +147,18 @@ member_resolution_read_only:
147
147
  `jobRunner`/...). Aufgelöste Member-Principals sind nur lesend — es gibt
148
148
  kein `writeAsMember`.
149
149
 
150
+ public_intake_required:
151
+ endUser: |
152
+ Diese Aktion ist nicht erlaubt.
153
+ Wende dich an einen Administrator wenn du glaubst, dass das ein Fehler ist.
154
+ developer: |
155
+ `AccessDeniedError`: ein Write unter einem anonymen Wurzel-Handler hat ein
156
+ Personendaten-Feld (`pii` / `userOwned` / `recordOwned`) geschrieben, direkt
157
+ oder über `ctx.write`/`writeAs`/`queryAs` oder einen Hook. `details` nennen
158
+ `rootHandler`, `target` und `fields`. Ist der anonyme Intake gewollt, am
159
+ Wurzel-Handler `access: { roles: [..., "anonymous"], personalData: "public-intake" }`
160
+ deklarieren.
161
+
150
162
  agent.tool_not_allowed:
151
163
  endUser: |
152
164
  Dieses Tool ist im aktuellen Modus nicht verfügbar.
@@ -141,6 +141,18 @@ member_resolution_read_only:
141
141
  to write (`ctx.write`/`writeAs`/`appendEvent`/`jobRunner`/...). Resolved
142
142
  member principals are read-only — there is no `writeAsMember`.
143
143
 
144
+ public_intake_required:
145
+ endUser: |
146
+ This action isn't allowed.
147
+ Please contact an administrator if you believe this is a mistake.
148
+ developer: |
149
+ `AccessDeniedError`: a write under an anonymous root handler touched a
150
+ personal-data field (`pii` / `userOwned` / `recordOwned`), directly or via
151
+ `ctx.write`/`writeAs`/`queryAs` or a hook. `details` name `rootHandler`,
152
+ `target` and `fields`. Declare
153
+ `access: { roles: [..., "anonymous"], personalData: "public-intake" }` on
154
+ the root handler if the anonymous intake is intended.
155
+
144
156
  agent.tool_not_allowed:
145
157
  endUser: |
146
158
  This tool is not available in the current mode.
@@ -59,6 +59,10 @@ export const FrameworkReasons = {
59
59
  // AccessDeniedError: a query handler invoked via ctx.queryAsMember tried to
60
60
  // write — a resolved member principal is read-only by construction.
61
61
  memberResolutionReadOnly: "member_resolution_read_only",
62
+
63
+ // AccessDeniedError: a write under an anonymous root touched a personal-data field
64
+ // without the root handler declaring access.personalData: "public-intake".
65
+ publicIntakeRequired: "public_intake_required",
62
66
  } as const;
63
67
 
64
68
  export type FrameworkReason = (typeof FrameworkReasons)[keyof typeof FrameworkReasons];
@@ -58,6 +58,12 @@ export function failTransition(from: string, to: string, allowed: readonly strin
58
58
  );
59
59
  }
60
60
 
61
+ // The cause travels out-of-band, keyed on the info object's identity: a field
62
+ // on WriteErrorInfo would leak the underlying Error into the /batch failure
63
+ // body and the idempotency cache, both of which serialize the info to JSON.
64
+ // A cache replay yields a fresh object without an entry, so no cause there.
65
+ const causeByWriteErrorInfo = new WeakMap<WriteErrorInfo, Error>();
66
+
61
67
  export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
62
68
  // In dev/test surface the cause-snapshot through `details` so the
63
69
  // HTTP response carries something useful. Without this, internal_error
@@ -79,7 +85,7 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
79
85
  : undefined
80
86
  : undefined;
81
87
  const effectiveDetails = err.details ?? causeDetails;
82
- return {
88
+ const info: WriteErrorInfo = {
83
89
  code: err.code,
84
90
  httpStatus: err.httpStatus,
85
91
  i18nKey: err.i18nKey,
@@ -87,6 +93,8 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
87
93
  ...(err.i18nParams && { i18nParams: err.i18nParams }),
88
94
  ...(effectiveDetails !== undefined && { details: effectiveDetails }),
89
95
  };
96
+ if (err.cause instanceof Error) causeByWriteErrorInfo.set(info, err.cause);
97
+ return info;
90
98
  }
91
99
 
92
100
  // Reconstitutes an error from WriteErrorInfo so command() (throw-based) can
@@ -95,19 +103,20 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
95
103
  // httpStatus / details but `instanceof NotFoundError` won't work. That's OK:
96
104
  // the HTTP layer keys off code + httpStatus, not class identity.
97
105
  export function reraiseAsKumikoError(info: WriteErrorInfo): KumikoError {
98
- return new ReraisedError(info);
106
+ return new ReraisedError(info, causeByWriteErrorInfo.get(info));
99
107
  }
100
108
 
101
109
  class ReraisedError extends KumikoError {
102
110
  readonly code: string;
103
111
  readonly httpStatus: number;
104
112
 
105
- constructor(info: WriteErrorInfo) {
113
+ constructor(info: WriteErrorInfo, cause?: Error) {
106
114
  super({
107
115
  message: info.message,
108
116
  i18nKey: info.i18nKey,
109
117
  ...(info.i18nParams && { i18nParams: info.i18nParams }),
110
118
  ...(info.details !== undefined && { details: info.details }),
119
+ ...(cause && { cause }),
111
120
  });
112
121
  this.code = info.code;
113
122
  this.httpStatus = info.httpStatus;
@@ -0,0 +1,155 @@
1
+ // fw#3167: buildRetryBullOpts only passed `{ type }` to BullMQ, never
2
+ // `delay` — BullMQ's fixed/exponential backoff strategies compute
3
+ // NaN/undefined without it (falsy), so a job with `backoff` set retried
4
+ // immediately instead of waiting. These tests assert real inter-attempt
5
+ // gaps via a real HTTP write that triggers the job through `trigger.on`,
6
+ // same delivery path as dispatch-write.ts's afterCommitHooks.
7
+ //
8
+ // Only lower bounds are asserted (never upper) — BullMQ schedules the next
9
+ // attempt at failureTime + delay using the same wall clock this test reads,
10
+ // so a >= assertion is flake-free without inflating waitFor's budget.
11
+
12
+ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
13
+ import { z } from "zod";
14
+ import { defineFeature } from "../../engine";
15
+ import { setupTestStack, type TestStack, TestUsers } from "../../stack";
16
+ import { waitFor } from "../../testing";
17
+ import { generateId } from "../../utils";
18
+
19
+ const exponentialDefaultStarts: number[] = [];
20
+ const exponentialCustomStarts: number[] = [];
21
+ const fixedCustomStarts: number[] = [];
22
+
23
+ function gapsBetween(starts: readonly number[]): number[] {
24
+ return starts.slice(1).map((start, index) => start - (starts[index] ?? start));
25
+ }
26
+
27
+ const backoffFixtureFeature = defineFeature("backofffixture", (r) => {
28
+ r.writeHandler(
29
+ "trigger-exponential-default",
30
+ z.object({}),
31
+ async () => ({ isSuccess: true as const, data: {} }),
32
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
33
+ );
34
+ // AC: retries: 3, backoff: "exponential" — fails once, succeeds on retry 2.
35
+ // Default base delay is 1000ms, so the single retry gap must be >= 1000ms.
36
+ r.job(
37
+ "exponential-default",
38
+ {
39
+ trigger: { on: "backofffixture:write:trigger-exponential-default" },
40
+ retries: 3,
41
+ backoff: "exponential",
42
+ },
43
+ async () => {
44
+ exponentialDefaultStarts.push(Date.now());
45
+ if (exponentialDefaultStarts.length === 1) throw new Error("fails on attempt 1");
46
+ },
47
+ );
48
+
49
+ r.writeHandler(
50
+ "trigger-exponential-custom",
51
+ z.object({}),
52
+ async () => ({ isSuccess: true as const, data: {} }),
53
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
54
+ );
55
+ // Per-job configurable + growing: delayMs 50 → gaps 50, 100, 200ms.
56
+ r.job(
57
+ "exponential-custom",
58
+ {
59
+ trigger: { on: "backofffixture:write:trigger-exponential-custom" },
60
+ retries: 3,
61
+ backoff: { type: "exponential", delayMs: 50 },
62
+ },
63
+ async () => {
64
+ exponentialCustomStarts.push(Date.now());
65
+ if (exponentialCustomStarts.length < 4)
66
+ throw new Error(`fails on attempt ${exponentialCustomStarts.length}`);
67
+ },
68
+ );
69
+
70
+ r.writeHandler(
71
+ "trigger-fixed-custom",
72
+ z.object({}),
73
+ async () => ({ isSuccess: true as const, data: {} }),
74
+ { access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
75
+ );
76
+ r.job(
77
+ "fixed-custom",
78
+ {
79
+ trigger: { on: "backofffixture:write:trigger-fixed-custom" },
80
+ retries: 1,
81
+ backoff: { type: "fixed", delayMs: 300 },
82
+ },
83
+ async () => {
84
+ fixedCustomStarts.push(Date.now());
85
+ if (fixedCustomStarts.length === 1) throw new Error("fails on attempt 1");
86
+ },
87
+ );
88
+ });
89
+
90
+ let stack: TestStack;
91
+
92
+ beforeAll(async () => {
93
+ stack = await setupTestStack({
94
+ features: [backoffFixtureFeature],
95
+ // Own queue namespace: the default "kumiko-jobs" queue on the shared test
96
+ // Redis is consumed by every parallel test process, and a foreign worker
97
+ // grabbing a promoted delayed retry fails it as "Unknown job".
98
+ jobs: { consumerLane: "worker", queueNamePrefix: `backoff-test-${generateId()}` },
99
+ });
100
+ });
101
+
102
+ afterAll(async () => {
103
+ await stack.cleanup();
104
+ });
105
+
106
+ beforeEach(() => {
107
+ exponentialDefaultStarts.length = 0;
108
+ exponentialCustomStarts.length = 0;
109
+ fixedCustomStarts.length = 0;
110
+ });
111
+
112
+ describe("job backoff waits between retries (fw#3167)", () => {
113
+ test('backoff: "exponential" without delayMs waits the default 1000ms base delay', async () => {
114
+ await stack.http.writeOk(
115
+ "backofffixture:write:trigger-exponential-default",
116
+ {},
117
+ TestUsers.admin,
118
+ );
119
+
120
+ await waitFor(() => {
121
+ expect(exponentialDefaultStarts).toHaveLength(2);
122
+ });
123
+
124
+ const [gap = 0] = gapsBetween(exponentialDefaultStarts);
125
+ expect(gap).toBeGreaterThanOrEqual(1000);
126
+ });
127
+
128
+ test("object-form backoff.delayMs is configurable per job and grows exponentially", async () => {
129
+ await stack.http.writeOk(
130
+ "backofffixture:write:trigger-exponential-custom",
131
+ {},
132
+ TestUsers.admin,
133
+ );
134
+
135
+ await waitFor(() => {
136
+ expect(exponentialCustomStarts).toHaveLength(4);
137
+ });
138
+
139
+ const [gap1 = 0, gap2 = 0, gap3 = 0] = gapsBetween(exponentialCustomStarts);
140
+ expect(gap1).toBeGreaterThanOrEqual(50);
141
+ expect(gap2).toBeGreaterThanOrEqual(100);
142
+ expect(gap3).toBeGreaterThanOrEqual(200);
143
+ });
144
+
145
+ test('backoff: { type: "fixed", delayMs } waits the configured constant delay', async () => {
146
+ await stack.http.writeOk("backofffixture:write:trigger-fixed-custom", {}, TestUsers.admin);
147
+
148
+ await waitFor(() => {
149
+ expect(fixedCustomStarts).toHaveLength(2);
150
+ });
151
+
152
+ const [gap = 0] = gapsBetween(fixedCustomStarts);
153
+ expect(gap).toBeGreaterThanOrEqual(300);
154
+ });
155
+ });
@@ -1390,4 +1390,39 @@ describe("boot gates", () => {
1390
1390
  });
1391
1391
  expect(() => createRegistry([sequentialGate])).toThrow(/bootGate with concurrency/);
1392
1392
  });
1393
+
1394
+ test("object-form backoff.delayMs must be a positive integer", () => {
1395
+ for (const badDelayMs of [0, -5, 1.5]) {
1396
+ const feature = defineFeature("badbackoff", (r) => {
1397
+ r.job(
1398
+ "check",
1399
+ { trigger: { manual: true }, backoff: { type: "fixed", delayMs: badDelayMs } },
1400
+ async () => {
1401
+ gateLog.push("never");
1402
+ },
1403
+ );
1404
+ });
1405
+ expect(() => createRegistry([feature])).toThrow(
1406
+ /backoff\.delayMs must be a positive integer/,
1407
+ );
1408
+ }
1409
+
1410
+ const validObjectForm = defineFeature("goodbackoffobject", (r) => {
1411
+ r.job(
1412
+ "check",
1413
+ { trigger: { manual: true }, backoff: { type: "exponential", delayMs: 100 } },
1414
+ async () => {
1415
+ gateLog.push("never");
1416
+ },
1417
+ );
1418
+ });
1419
+ expect(() => createRegistry([validObjectForm])).not.toThrow();
1420
+
1421
+ const validStringForm = defineFeature("goodbackoffstring", (r) => {
1422
+ r.job("check", { trigger: { manual: true }, backoff: "fixed" }, async () => {
1423
+ gateLog.push("never");
1424
+ });
1425
+ });
1426
+ expect(() => createRegistry([validStringForm])).not.toThrow();
1427
+ });
1393
1428
  });
@@ -1,4 +1,4 @@
1
- import { Queue, Worker } from "bullmq";
1
+ import { type JobsOptions, Queue, Worker } from "bullmq";
2
2
  import { Redis } from "ioredis";
3
3
  import { requestContext } from "../api/request-context";
4
4
  import type { DbConnection, DbRow } from "../db/connection";
@@ -317,13 +317,27 @@ function timeoutReject(
317
317
  };
318
318
  }
319
319
 
320
+ // Default base delay when a job opts into backoff without an explicit
321
+ // delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
322
+ // NaN/undefined, which is falsy — the job retries immediately instead of
323
+ // waiting.
324
+ const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
325
+
320
326
  // Shared by dispatch() and handleEvent() — an event-triggered job must retry
321
327
  // on failure the same way a directly-dispatched one does; a duplicated
322
328
  // inline computation in handleEvent previously dropped both options.
323
- function buildRetryBullOpts(jobDef: JobDefinition): Record<string, unknown> {
324
- const opts: Record<string, unknown> = {};
325
- if (jobDef.retries !== undefined) opts["attempts"] = jobDef.retries + 1;
326
- if (jobDef.backoff) opts["backoff"] = { type: jobDef.backoff };
329
+ function buildRetryBullOpts(jobDef: JobDefinition): Pick<JobsOptions, "attempts" | "backoff"> {
330
+ const opts: Pick<JobsOptions, "attempts" | "backoff"> = {};
331
+ if (jobDef.retries !== undefined) opts.attempts = jobDef.retries + 1;
332
+ if (jobDef.backoff) {
333
+ opts.backoff =
334
+ typeof jobDef.backoff === "string"
335
+ ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
336
+ : {
337
+ type: jobDef.backoff.type,
338
+ delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
339
+ };
340
+ }
327
341
  return opts;
328
342
  }
329
343
 
@@ -0,0 +1,61 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import { z } from "zod";
3
+ import { composeEnvSchema, readKumikoMeta } from "../../env";
4
+ import { prometheusMetricsEnvSchema, resolveObservabilityWiring } from "../metrics-wiring";
5
+
6
+ describe("resolveObservabilityWiring", () => {
7
+ it("returns {} without a token", () => {
8
+ expect(resolveObservabilityWiring(undefined)).toEqual({});
9
+ });
10
+
11
+ it("returns {} for an empty token", () => {
12
+ expect(resolveObservabilityWiring("")).toEqual({});
13
+ });
14
+
15
+ it("wires a prometheus provider and metrics route when a token is set", () => {
16
+ const token = "a".repeat(32);
17
+ const wiring = resolveObservabilityWiring(token);
18
+
19
+ expect("metrics" in wiring).toBe(true);
20
+ if (!("metrics" in wiring)) throw new Error("expected wiring to include metrics");
21
+
22
+ expect(wiring.metrics).toEqual({ path: "/metrics", token });
23
+ expect(wiring.observability.name).toBe("prometheus");
24
+ expect(wiring.observability.meter.snapshot()).toEqual(new Map());
25
+ });
26
+ });
27
+
28
+ describe("prometheusMetricsEnvSchema", () => {
29
+ it("rejects tokens shorter than 32 characters", () => {
30
+ const result = prometheusMetricsEnvSchema.safeParse({
31
+ PROMETHEUS_METRICS_TOKEN: "a".repeat(31),
32
+ });
33
+ expect(result.success).toBe(false);
34
+ });
35
+
36
+ it("accepts a 32-character token", () => {
37
+ const result = prometheusMetricsEnvSchema.safeParse({
38
+ PROMETHEUS_METRICS_TOKEN: "a".repeat(32),
39
+ });
40
+ expect(result.success).toBe(true);
41
+ });
42
+
43
+ it("accepts a missing token", () => {
44
+ const result = prometheusMetricsEnvSchema.safeParse({});
45
+ expect(result.success).toBe(true);
46
+ });
47
+
48
+ it("exposes pulumi secret metadata for consumer env-schemas", () => {
49
+ const { schema } = composeEnvSchema({
50
+ features: [],
51
+ extend: z.object({ FOO: z.string() }).extend(prometheusMetricsEnvSchema.shape),
52
+ });
53
+
54
+ const field = schema.shape["PROMETHEUS_METRICS_TOKEN"];
55
+ if (!(field instanceof z.ZodType))
56
+ throw new Error("expected PROMETHEUS_METRICS_TOKEN in composed schema");
57
+ const meta = readKumikoMeta(field);
58
+ expect(meta.pulumi?.secret).toBe(true);
59
+ expect(meta.pulumi?.generator).toBe("openssl rand -base64 32");
60
+ });
61
+ });
@@ -16,6 +16,11 @@ export {
16
16
  createSafeMetricsHandle,
17
17
  createUnboundMetricsHandle,
18
18
  } from "./metrics-handle";
19
+ export {
20
+ type ObservabilityWiring,
21
+ prometheusMetricsEnvSchema,
22
+ resolveObservabilityWiring,
23
+ } from "./metrics-wiring";
19
24
  export { createNoopProvider } from "./noop-provider";
20
25
  export {
21
26
  createPrometheusMeter,
@@ -0,0 +1,32 @@
1
+ import { z } from "zod";
2
+ import { createNoopProvider } from "./noop-provider";
3
+ import { createPrometheusMeter, type PrometheusMeter } from "./prometheus-meter";
4
+ import type { ObservabilityProvider } from "./types";
5
+
6
+ export const prometheusMetricsEnvSchema = z.object({
7
+ PROMETHEUS_METRICS_TOKEN: z
8
+ .string()
9
+ .min(32)
10
+ .optional()
11
+ .describe("Bearer token for /metrics; unset keeps the endpoint off.")
12
+ .meta({ kumiko: { pulumi: { secret: true, generator: "openssl rand -base64 32" } } }),
13
+ });
14
+
15
+ type PrometheusObservabilityProvider = ObservabilityProvider & { readonly meter: PrometheusMeter };
16
+
17
+ export type ObservabilityWiring =
18
+ | {
19
+ readonly observability: PrometheusObservabilityProvider;
20
+ readonly metrics: { readonly path: string; readonly token: string };
21
+ }
22
+ | Record<string, never>;
23
+
24
+ // Fail-closed: public tenant hosts share the /metrics port, so no token means no endpoint.
25
+ export function resolveObservabilityWiring(metricsToken: string | undefined): ObservabilityWiring {
26
+ if (!metricsToken) return {};
27
+ return {
28
+ // Overrides the spread's name "noop", which would otherwise show up in diagnostics/logs.
29
+ observability: { ...createNoopProvider(), name: "prometheus", meter: createPrometheusMeter() },
30
+ metrics: { path: "/metrics", token: metricsToken },
31
+ };
32
+ }