@cosmicdrift/kumiko-framework 0.299.0 → 0.305.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/__tests__/api.test.ts +3 -2
- package/src/api/__tests__/extra-route-rejection.test.ts +38 -0
- package/src/api/__tests__/extra-routes.integration.test.ts +30 -0
- package/src/api/__tests__/http-route-entry.integration.test.ts +114 -0
- package/src/api/__tests__/server-error-logging.test.ts +33 -0
- package/src/api/api-constants.ts +13 -0
- package/src/api/auth-routes.ts +53 -24
- package/src/api/extra-route.ts +33 -4
- package/src/api/index.ts +1 -0
- package/src/api/server.ts +79 -34
- package/src/changes.json +68 -0
- package/src/db/event-store-executor-write.ts +7 -0
- package/src/db/tenant-db.ts +50 -3
- package/src/engine/__tests__/http-route-anonymous-required.test.ts +43 -0
- package/src/engine/__tests__/membership-roles.test.ts +13 -4
- package/src/engine/boot-validator/__tests__/access-declarations.test.ts +127 -0
- package/src/engine/boot-validator/__tests__/no-all-role-in-handler-access.test.ts +77 -0
- package/src/engine/boot-validator/access-declarations.ts +58 -67
- package/src/engine/boot-validator/entity-handler.ts +20 -0
- package/src/engine/feature-ast/__tests__/patch.test.ts +1 -0
- package/src/engine/feature-ast/__tests__/patcher.test.ts +1 -0
- package/src/engine/feature-ast/__tests__/read-optional-access-rule.test.ts +14 -0
- package/src/engine/feature-ast/extractors/hooks.ts +3 -1
- package/src/engine/feature-ast/extractors/jobs-routes.ts +5 -2
- package/src/engine/feature-ast/patcher.ts +2 -2
- package/src/engine/feature-ast/patterns.ts +1 -1
- package/src/engine/feature-ast/render.ts +1 -1
- package/src/engine/feature-ui-extensions.ts +6 -0
- package/src/engine/index.ts +4 -0
- package/src/engine/membership-roles.ts +20 -4
- package/src/engine/pattern-library/__tests__/library.test.ts +1 -0
- package/src/engine/pattern-library/mixed-schemas.ts +1 -0
- package/src/engine/personal-data-fields.ts +66 -0
- package/src/engine/registry-validate.ts +15 -0
- package/src/engine/registry.ts +2 -0
- package/src/engine/types/index.ts +4 -0
- package/src/env/__tests__/dry-run.test.ts +43 -3
- package/src/env/dry-run.ts +28 -15
- package/src/errors/__tests__/write-failures.test.ts +47 -4
- package/src/errors/i18n/de.yaml +12 -0
- package/src/errors/i18n/en.yaml +12 -0
- package/src/errors/reasons.ts +4 -0
- package/src/errors/write-error-info.ts +12 -3
- package/src/jobs/__tests__/job-backoff.integration.test.ts +155 -0
- package/src/jobs/__tests__/jobs.integration.test.ts +35 -0
- package/src/jobs/job-runner.ts +19 -5
- package/src/observability/__tests__/metrics-wiring.test.ts +61 -0
- package/src/observability/index.ts +5 -0
- package/src/observability/metrics-wiring.ts +32 -0
- package/src/pipeline/__tests__/public-intake-runtime-gate.integration.test.ts +425 -0
- package/src/pipeline/active-membership.ts +5 -1
- package/src/pipeline/dispatch-batch.ts +3 -0
- package/src/pipeline/dispatch-query.ts +16 -5
- package/src/pipeline/dispatch-shared.ts +12 -5
- package/src/pipeline/dispatch-stream.ts +7 -2
- package/src/pipeline/dispatch-write.ts +22 -5
- package/src/pipeline/dispatcher.ts +9 -2
- package/src/pipeline/member-reader.ts +3 -1
- package/src/pipeline/write-origin.ts +107 -0
- package/src/rate-limit/__tests__/middleware.integration.test.ts +40 -0
- package/src/rate-limit/middleware.ts +3 -0
- package/src/stack/__tests__/setup-test-stack-metrics.integration.test.ts +79 -0
- package/src/stack/test-stack.ts +5 -0
- package/src/testing/handler-context.ts +3 -1
- package/src/ui-types/index.ts +2 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "bun:test";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { defineFeature } from "../../engine/define-feature";
|
|
4
|
+
import { prometheusMetricsEnvSchema } from "../../observability/metrics-wiring";
|
|
4
5
|
import { renderDryRun } from "../dry-run";
|
|
5
6
|
import { composeEnvSchema } from "../index";
|
|
6
7
|
|
|
@@ -90,7 +91,7 @@ describe("renderDryRun", () => {
|
|
|
90
91
|
expect(ver?.default).toBe("1");
|
|
91
92
|
});
|
|
92
93
|
|
|
93
|
-
it("pulumi mode emits
|
|
94
|
+
it("pulumi mode emits required lines, optional ones commented out, omits defaulted", () => {
|
|
94
95
|
const composed = buildComposed();
|
|
95
96
|
const out = renderDryRun(composed, "pulumi", { pulumiPrefix: "studio" });
|
|
96
97
|
expect(out).toContain(
|
|
@@ -100,9 +101,36 @@ describe("renderDryRun", () => {
|
|
|
100
101
|
'pulumi config set --secret studioSecretsMasterKey "$(openssl rand -base64 32)"',
|
|
101
102
|
);
|
|
102
103
|
expect(out).toContain('pulumi config set studioStudioAdminEmail "<set-me>"');
|
|
103
|
-
|
|
104
|
-
|
|
104
|
+
expect(out).toContain(
|
|
105
|
+
'# pulumi config set studioSmtpHost "<set-me>" # SMTP_HOST (channel-email-smtp): Outbound SMTP host',
|
|
106
|
+
);
|
|
107
|
+
expect(out).not.toMatch(/^pulumi config set.*SMTP_HOST/m);
|
|
105
108
|
expect(out).not.toContain("CURRENT_VERSION");
|
|
109
|
+
const requiredLineIndex = out.lastIndexOf("pulumi config set studioStudioAdminEmail");
|
|
110
|
+
const optionalHeaderIndex = out.indexOf("# Optional (uncomment and set to enable):");
|
|
111
|
+
expect(requiredLineIndex).toBeGreaterThanOrEqual(0);
|
|
112
|
+
expect(optionalHeaderIndex).toBeGreaterThan(requiredLineIndex);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("pulumi mode lists an optional secret field (PROMETHEUS_METRICS_TOKEN) commented out with generator", () => {
|
|
116
|
+
const composed = composeEnvSchema({
|
|
117
|
+
features: [],
|
|
118
|
+
extend: z.object({ FOO: z.string() }).extend(prometheusMetricsEnvSchema.shape),
|
|
119
|
+
});
|
|
120
|
+
const out = renderDryRun(composed, "pulumi");
|
|
121
|
+
expect(out).toContain(
|
|
122
|
+
'# pulumi config set --secret prometheusMetricsToken "$(openssl rand -base64 32)" # PROMETHEUS_METRICS_TOKEN (app): Bearer token for /metrics; unset keeps the endpoint off.',
|
|
123
|
+
);
|
|
124
|
+
expect(out).not.toMatch(/^pulumi config set.*PROMETHEUS_METRICS_TOKEN/m);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("pulumi mode omits the Optional header when there are no optional fields", () => {
|
|
128
|
+
const composed = composeEnvSchema({
|
|
129
|
+
features: [],
|
|
130
|
+
extend: z.object({ FOO: z.string() }),
|
|
131
|
+
});
|
|
132
|
+
const out = renderDryRun(composed, "pulumi");
|
|
133
|
+
expect(out).not.toContain("# Optional");
|
|
106
134
|
});
|
|
107
135
|
|
|
108
136
|
it("k8s mode emits a Secret manifest", () => {
|
|
@@ -117,5 +145,17 @@ describe("renderDryRun", () => {
|
|
|
117
145
|
expect(out).toContain("namespace: studio");
|
|
118
146
|
expect(out).toContain('JWT_SECRET: "<set-me>"');
|
|
119
147
|
expect(out).toContain('KUMIKO_SECRETS_MASTER_KEY_V1: "<set-me>"');
|
|
148
|
+
expect(out).toContain(' # SMTP_HOST: "<set-me>" # (channel-email-smtp): Outbound SMTP host');
|
|
149
|
+
expect(out).not.toMatch(/^ {2}SMTP_HOST:/m);
|
|
150
|
+
expect(out).not.toContain("CURRENT_VERSION");
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it("k8s mode omits the Optional header when there are no optional fields", () => {
|
|
154
|
+
const composed = composeEnvSchema({
|
|
155
|
+
features: [],
|
|
156
|
+
extend: z.object({ FOO: z.string() }),
|
|
157
|
+
});
|
|
158
|
+
const out = renderDryRun(composed, "k8s");
|
|
159
|
+
expect(out).not.toContain("# Optional");
|
|
120
160
|
});
|
|
121
161
|
});
|
package/src/env/dry-run.ts
CHANGED
|
@@ -152,20 +152,27 @@ function renderJson(fields: readonly EnvField[], options: DryRunOptions): string
|
|
|
152
152
|
)}\n`;
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
function pulumiConfigSetLine(f: EnvField, options: DryRunOptions): string {
|
|
156
|
+
const meta = readKumikoMeta(f.field);
|
|
157
|
+
const key = pulumiConfigKey(f.name, f.field, options.pulumiPrefix);
|
|
158
|
+
const secretFlag = meta.pulumi?.secret ? " --secret" : "";
|
|
159
|
+
const value = meta.pulumi?.generator ? `"$(${meta.pulumi.generator})"` : `"<set-me>"`;
|
|
160
|
+
const comment = f.description
|
|
161
|
+
? ` # ${f.name} (${f.source}): ${f.description}`
|
|
162
|
+
: ` # ${f.name} (${f.source})`;
|
|
163
|
+
return `pulumi config set${secretFlag} ${key} ${value}${comment}`;
|
|
164
|
+
}
|
|
165
|
+
|
|
155
166
|
function renderPulumi(fields: readonly EnvField[], options: DryRunOptions): string {
|
|
156
167
|
// Defaulted vars are skipped — the framework provides them, ops doesn't.
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
const
|
|
165
|
-
const comment = f.description
|
|
166
|
-
? ` # ${f.name} (${f.source}): ${f.description}`
|
|
167
|
-
: ` # ${f.name} (${f.source})`;
|
|
168
|
-
lines.push(`pulumi config set${secretFlag} ${key} ${value}${comment}`);
|
|
168
|
+
// Optional vars stay commented out: setting one turns its feature on.
|
|
169
|
+
const required = fields.filter((f) => f.klass === "required");
|
|
170
|
+
const optional = fields.filter((f) => f.klass === "optional");
|
|
171
|
+
const lines: string[] = required.map((f) => pulumiConfigSetLine(f, options));
|
|
172
|
+
if (optional.length > 0) {
|
|
173
|
+
if (lines.length > 0) lines.push("");
|
|
174
|
+
lines.push("# Optional (uncomment and set to enable):");
|
|
175
|
+
for (const f of optional) lines.push(`# ${pulumiConfigSetLine(f, options)}`);
|
|
169
176
|
}
|
|
170
177
|
return `${lines.join("\n")}\n`;
|
|
171
178
|
}
|
|
@@ -183,9 +190,15 @@ function renderK8s(fields: readonly EnvField[], options: DryRunOptions): string
|
|
|
183
190
|
"stringData:",
|
|
184
191
|
];
|
|
185
192
|
for (const f of fields) {
|
|
186
|
-
if (f.klass === "
|
|
187
|
-
|
|
188
|
-
|
|
193
|
+
if (f.klass === "required") lines.push(` ${f.name}: "<set-me>"`);
|
|
194
|
+
}
|
|
195
|
+
const optional = fields.filter((f) => f.klass === "optional");
|
|
196
|
+
if (optional.length > 0) {
|
|
197
|
+
lines.push(" # Optional (uncomment and set to enable):");
|
|
198
|
+
for (const f of optional) {
|
|
199
|
+
const comment = f.description ? ` # (${f.source}): ${f.description}` : ` # (${f.source})`;
|
|
200
|
+
lines.push(` # ${f.name}: "<set-me>"${comment}`);
|
|
201
|
+
}
|
|
189
202
|
}
|
|
190
203
|
return `${lines.join("\n")}\n`;
|
|
191
204
|
}
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
-
import {
|
|
2
|
+
import { InternalError, NotFoundError } from "../classes";
|
|
3
|
+
import {
|
|
4
|
+
failNotFound,
|
|
5
|
+
failTransition,
|
|
6
|
+
failUnprocessable,
|
|
7
|
+
reraiseAsKumikoError,
|
|
8
|
+
toWriteErrorInfo,
|
|
9
|
+
} from "../write-error-info";
|
|
3
10
|
|
|
4
11
|
describe("failNotFound", () => {
|
|
5
12
|
test("baut WriteFailure mit reason=not_found + entity-id-details", () => {
|
|
@@ -73,7 +80,6 @@ describe("failTransition", () => {
|
|
|
73
80
|
describe("toWriteErrorInfo — dev cause-snapshot", () => {
|
|
74
81
|
test("InternalError mit cause exposed cause-Snapshot in details (dev)", async () => {
|
|
75
82
|
const { toWriteErrorInfo } = await import("../write-error-info");
|
|
76
|
-
const { InternalError } = await import("../classes");
|
|
77
83
|
const previous = process.env["NODE_ENV"];
|
|
78
84
|
process.env["NODE_ENV"] = "development";
|
|
79
85
|
try {
|
|
@@ -93,7 +99,6 @@ describe("toWriteErrorInfo — dev cause-snapshot", () => {
|
|
|
93
99
|
|
|
94
100
|
test("Production: InternalError lässt details undefined (kein Stack-Leak)", async () => {
|
|
95
101
|
const { toWriteErrorInfo } = await import("../write-error-info");
|
|
96
|
-
const { InternalError } = await import("../classes");
|
|
97
102
|
const previous = process.env["NODE_ENV"];
|
|
98
103
|
process.env["NODE_ENV"] = "production";
|
|
99
104
|
try {
|
|
@@ -107,7 +112,6 @@ describe("toWriteErrorInfo — dev cause-snapshot", () => {
|
|
|
107
112
|
|
|
108
113
|
test("InternalError MIT bereits gesetztem details → Author-details gewinnt (kein Overwrite)", async () => {
|
|
109
114
|
const { toWriteErrorInfo } = await import("../write-error-info");
|
|
110
|
-
const { InternalError } = await import("../classes");
|
|
111
115
|
const previous = process.env["NODE_ENV"];
|
|
112
116
|
process.env["NODE_ENV"] = "development";
|
|
113
117
|
try {
|
|
@@ -122,3 +126,42 @@ describe("toWriteErrorInfo — dev cause-snapshot", () => {
|
|
|
122
126
|
}
|
|
123
127
|
});
|
|
124
128
|
});
|
|
129
|
+
|
|
130
|
+
// The cause must reach routes.ts's logServerFault via reraiseAsKumikoError,
|
|
131
|
+
// but never the wire body or the idempotency cache (both serialize the info).
|
|
132
|
+
describe("toWriteErrorInfo / reraiseAsKumikoError — cause round-trip", () => {
|
|
133
|
+
test("cause survives toWriteErrorInfo → reraiseAsKumikoError without appearing on the info object", () => {
|
|
134
|
+
const boom = new Error("connection was closed");
|
|
135
|
+
const info = toWriteErrorInfo(new InternalError({ cause: boom }));
|
|
136
|
+
expect(reraiseAsKumikoError(info).cause).toBe(boom);
|
|
137
|
+
expect(Object.keys(info)).not.toContain("cause");
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("production: cause still reaches reraise even though details/message stay sanitized", () => {
|
|
141
|
+
const previous = process.env["NODE_ENV"];
|
|
142
|
+
process.env["NODE_ENV"] = "production";
|
|
143
|
+
try {
|
|
144
|
+
const boom = new Error("connection was closed");
|
|
145
|
+
const info = toWriteErrorInfo(new InternalError({ cause: boom }));
|
|
146
|
+
expect(info.details).toBeUndefined();
|
|
147
|
+
const serialized = JSON.stringify(info);
|
|
148
|
+
expect(serialized).not.toContain("connection was closed");
|
|
149
|
+
expect(serialized).not.toContain("cause");
|
|
150
|
+
expect(reraiseAsKumikoError(info).cause).toBe(boom);
|
|
151
|
+
} finally {
|
|
152
|
+
if (previous === undefined) delete process.env["NODE_ENV"];
|
|
153
|
+
else process.env["NODE_ENV"] = previous;
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test("KumikoError without a cause → reraised error has no cause", () => {
|
|
158
|
+
const info = toWriteErrorInfo(new NotFoundError("invoice", "inv-1"));
|
|
159
|
+
expect(reraiseAsKumikoError(info).cause).toBeUndefined();
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
test("a plain object with the same shape (simulated idempotency-cache replay) carries no cause", () => {
|
|
163
|
+
const info = toWriteErrorInfo(new InternalError({ cause: new Error("connection was closed") }));
|
|
164
|
+
const replayed = { ...info };
|
|
165
|
+
expect(reraiseAsKumikoError(replayed).cause).toBeUndefined();
|
|
166
|
+
});
|
|
167
|
+
});
|
package/src/errors/i18n/de.yaml
CHANGED
|
@@ -147,6 +147,18 @@ member_resolution_read_only:
|
|
|
147
147
|
`jobRunner`/...). Aufgelöste Member-Principals sind nur lesend — es gibt
|
|
148
148
|
kein `writeAsMember`.
|
|
149
149
|
|
|
150
|
+
public_intake_required:
|
|
151
|
+
endUser: |
|
|
152
|
+
Diese Aktion ist nicht erlaubt.
|
|
153
|
+
Wende dich an einen Administrator wenn du glaubst, dass das ein Fehler ist.
|
|
154
|
+
developer: |
|
|
155
|
+
`AccessDeniedError`: ein Write unter einem anonymen Wurzel-Handler hat ein
|
|
156
|
+
Personendaten-Feld (`pii` / `userOwned` / `recordOwned`) geschrieben, direkt
|
|
157
|
+
oder über `ctx.write`/`writeAs`/`queryAs` oder einen Hook. `details` nennen
|
|
158
|
+
`rootHandler`, `target` und `fields`. Ist der anonyme Intake gewollt, am
|
|
159
|
+
Wurzel-Handler `access: { roles: [..., "anonymous"], personalData: "public-intake" }`
|
|
160
|
+
deklarieren.
|
|
161
|
+
|
|
150
162
|
agent.tool_not_allowed:
|
|
151
163
|
endUser: |
|
|
152
164
|
Dieses Tool ist im aktuellen Modus nicht verfügbar.
|
package/src/errors/i18n/en.yaml
CHANGED
|
@@ -141,6 +141,18 @@ member_resolution_read_only:
|
|
|
141
141
|
to write (`ctx.write`/`writeAs`/`appendEvent`/`jobRunner`/...). Resolved
|
|
142
142
|
member principals are read-only — there is no `writeAsMember`.
|
|
143
143
|
|
|
144
|
+
public_intake_required:
|
|
145
|
+
endUser: |
|
|
146
|
+
This action isn't allowed.
|
|
147
|
+
Please contact an administrator if you believe this is a mistake.
|
|
148
|
+
developer: |
|
|
149
|
+
`AccessDeniedError`: a write under an anonymous root handler touched a
|
|
150
|
+
personal-data field (`pii` / `userOwned` / `recordOwned`), directly or via
|
|
151
|
+
`ctx.write`/`writeAs`/`queryAs` or a hook. `details` name `rootHandler`,
|
|
152
|
+
`target` and `fields`. Declare
|
|
153
|
+
`access: { roles: [..., "anonymous"], personalData: "public-intake" }` on
|
|
154
|
+
the root handler if the anonymous intake is intended.
|
|
155
|
+
|
|
144
156
|
agent.tool_not_allowed:
|
|
145
157
|
endUser: |
|
|
146
158
|
This tool is not available in the current mode.
|
package/src/errors/reasons.ts
CHANGED
|
@@ -59,6 +59,10 @@ export const FrameworkReasons = {
|
|
|
59
59
|
// AccessDeniedError: a query handler invoked via ctx.queryAsMember tried to
|
|
60
60
|
// write — a resolved member principal is read-only by construction.
|
|
61
61
|
memberResolutionReadOnly: "member_resolution_read_only",
|
|
62
|
+
|
|
63
|
+
// AccessDeniedError: a write under an anonymous root touched a personal-data field
|
|
64
|
+
// without the root handler declaring access.personalData: "public-intake".
|
|
65
|
+
publicIntakeRequired: "public_intake_required",
|
|
62
66
|
} as const;
|
|
63
67
|
|
|
64
68
|
export type FrameworkReason = (typeof FrameworkReasons)[keyof typeof FrameworkReasons];
|
|
@@ -58,6 +58,12 @@ export function failTransition(from: string, to: string, allowed: readonly strin
|
|
|
58
58
|
);
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
// The cause travels out-of-band, keyed on the info object's identity: a field
|
|
62
|
+
// on WriteErrorInfo would leak the underlying Error into the /batch failure
|
|
63
|
+
// body and the idempotency cache, both of which serialize the info to JSON.
|
|
64
|
+
// A cache replay yields a fresh object without an entry, so no cause there.
|
|
65
|
+
const causeByWriteErrorInfo = new WeakMap<WriteErrorInfo, Error>();
|
|
66
|
+
|
|
61
67
|
export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
|
|
62
68
|
// In dev/test surface the cause-snapshot through `details` so the
|
|
63
69
|
// HTTP response carries something useful. Without this, internal_error
|
|
@@ -79,7 +85,7 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
|
|
|
79
85
|
: undefined
|
|
80
86
|
: undefined;
|
|
81
87
|
const effectiveDetails = err.details ?? causeDetails;
|
|
82
|
-
|
|
88
|
+
const info: WriteErrorInfo = {
|
|
83
89
|
code: err.code,
|
|
84
90
|
httpStatus: err.httpStatus,
|
|
85
91
|
i18nKey: err.i18nKey,
|
|
@@ -87,6 +93,8 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
|
|
|
87
93
|
...(err.i18nParams && { i18nParams: err.i18nParams }),
|
|
88
94
|
...(effectiveDetails !== undefined && { details: effectiveDetails }),
|
|
89
95
|
};
|
|
96
|
+
if (err.cause instanceof Error) causeByWriteErrorInfo.set(info, err.cause);
|
|
97
|
+
return info;
|
|
90
98
|
}
|
|
91
99
|
|
|
92
100
|
// Reconstitutes an error from WriteErrorInfo so command() (throw-based) can
|
|
@@ -95,19 +103,20 @@ export function toWriteErrorInfo(err: KumikoError): WriteErrorInfo {
|
|
|
95
103
|
// httpStatus / details but `instanceof NotFoundError` won't work. That's OK:
|
|
96
104
|
// the HTTP layer keys off code + httpStatus, not class identity.
|
|
97
105
|
export function reraiseAsKumikoError(info: WriteErrorInfo): KumikoError {
|
|
98
|
-
return new ReraisedError(info);
|
|
106
|
+
return new ReraisedError(info, causeByWriteErrorInfo.get(info));
|
|
99
107
|
}
|
|
100
108
|
|
|
101
109
|
class ReraisedError extends KumikoError {
|
|
102
110
|
readonly code: string;
|
|
103
111
|
readonly httpStatus: number;
|
|
104
112
|
|
|
105
|
-
constructor(info: WriteErrorInfo) {
|
|
113
|
+
constructor(info: WriteErrorInfo, cause?: Error) {
|
|
106
114
|
super({
|
|
107
115
|
message: info.message,
|
|
108
116
|
i18nKey: info.i18nKey,
|
|
109
117
|
...(info.i18nParams && { i18nParams: info.i18nParams }),
|
|
110
118
|
...(info.details !== undefined && { details: info.details }),
|
|
119
|
+
...(cause && { cause }),
|
|
111
120
|
});
|
|
112
121
|
this.code = info.code;
|
|
113
122
|
this.httpStatus = info.httpStatus;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// fw#3167: buildRetryBullOpts only passed `{ type }` to BullMQ, never
|
|
2
|
+
// `delay` — BullMQ's fixed/exponential backoff strategies compute
|
|
3
|
+
// NaN/undefined without it (falsy), so a job with `backoff` set retried
|
|
4
|
+
// immediately instead of waiting. These tests assert real inter-attempt
|
|
5
|
+
// gaps via a real HTTP write that triggers the job through `trigger.on`,
|
|
6
|
+
// same delivery path as dispatch-write.ts's afterCommitHooks.
|
|
7
|
+
//
|
|
8
|
+
// Only lower bounds are asserted (never upper) — BullMQ schedules the next
|
|
9
|
+
// attempt at failureTime + delay using the same wall clock this test reads,
|
|
10
|
+
// so a >= assertion is flake-free without inflating waitFor's budget.
|
|
11
|
+
|
|
12
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
13
|
+
import { z } from "zod";
|
|
14
|
+
import { defineFeature } from "../../engine";
|
|
15
|
+
import { setupTestStack, type TestStack, TestUsers } from "../../stack";
|
|
16
|
+
import { waitFor } from "../../testing";
|
|
17
|
+
import { generateId } from "../../utils";
|
|
18
|
+
|
|
19
|
+
const exponentialDefaultStarts: number[] = [];
|
|
20
|
+
const exponentialCustomStarts: number[] = [];
|
|
21
|
+
const fixedCustomStarts: number[] = [];
|
|
22
|
+
|
|
23
|
+
function gapsBetween(starts: readonly number[]): number[] {
|
|
24
|
+
return starts.slice(1).map((start, index) => start - (starts[index] ?? start));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const backoffFixtureFeature = defineFeature("backofffixture", (r) => {
|
|
28
|
+
r.writeHandler(
|
|
29
|
+
"trigger-exponential-default",
|
|
30
|
+
z.object({}),
|
|
31
|
+
async () => ({ isSuccess: true as const, data: {} }),
|
|
32
|
+
{ access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
|
|
33
|
+
);
|
|
34
|
+
// AC: retries: 3, backoff: "exponential" — fails once, succeeds on retry 2.
|
|
35
|
+
// Default base delay is 1000ms, so the single retry gap must be >= 1000ms.
|
|
36
|
+
r.job(
|
|
37
|
+
"exponential-default",
|
|
38
|
+
{
|
|
39
|
+
trigger: { on: "backofffixture:write:trigger-exponential-default" },
|
|
40
|
+
retries: 3,
|
|
41
|
+
backoff: "exponential",
|
|
42
|
+
},
|
|
43
|
+
async () => {
|
|
44
|
+
exponentialDefaultStarts.push(Date.now());
|
|
45
|
+
if (exponentialDefaultStarts.length === 1) throw new Error("fails on attempt 1");
|
|
46
|
+
},
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
r.writeHandler(
|
|
50
|
+
"trigger-exponential-custom",
|
|
51
|
+
z.object({}),
|
|
52
|
+
async () => ({ isSuccess: true as const, data: {} }),
|
|
53
|
+
{ access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
|
|
54
|
+
);
|
|
55
|
+
// Per-job configurable + growing: delayMs 50 → gaps 50, 100, 200ms.
|
|
56
|
+
r.job(
|
|
57
|
+
"exponential-custom",
|
|
58
|
+
{
|
|
59
|
+
trigger: { on: "backofffixture:write:trigger-exponential-custom" },
|
|
60
|
+
retries: 3,
|
|
61
|
+
backoff: { type: "exponential", delayMs: 50 },
|
|
62
|
+
},
|
|
63
|
+
async () => {
|
|
64
|
+
exponentialCustomStarts.push(Date.now());
|
|
65
|
+
if (exponentialCustomStarts.length < 4)
|
|
66
|
+
throw new Error(`fails on attempt ${exponentialCustomStarts.length}`);
|
|
67
|
+
},
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
r.writeHandler(
|
|
71
|
+
"trigger-fixed-custom",
|
|
72
|
+
z.object({}),
|
|
73
|
+
async () => ({ isSuccess: true as const, data: {} }),
|
|
74
|
+
{ access: { openToAll: { reason: "test handler callable by any signed-in test user" } } },
|
|
75
|
+
);
|
|
76
|
+
r.job(
|
|
77
|
+
"fixed-custom",
|
|
78
|
+
{
|
|
79
|
+
trigger: { on: "backofffixture:write:trigger-fixed-custom" },
|
|
80
|
+
retries: 1,
|
|
81
|
+
backoff: { type: "fixed", delayMs: 300 },
|
|
82
|
+
},
|
|
83
|
+
async () => {
|
|
84
|
+
fixedCustomStarts.push(Date.now());
|
|
85
|
+
if (fixedCustomStarts.length === 1) throw new Error("fails on attempt 1");
|
|
86
|
+
},
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
let stack: TestStack;
|
|
91
|
+
|
|
92
|
+
beforeAll(async () => {
|
|
93
|
+
stack = await setupTestStack({
|
|
94
|
+
features: [backoffFixtureFeature],
|
|
95
|
+
// Own queue namespace: the default "kumiko-jobs" queue on the shared test
|
|
96
|
+
// Redis is consumed by every parallel test process, and a foreign worker
|
|
97
|
+
// grabbing a promoted delayed retry fails it as "Unknown job".
|
|
98
|
+
jobs: { consumerLane: "worker", queueNamePrefix: `backoff-test-${generateId()}` },
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
afterAll(async () => {
|
|
103
|
+
await stack.cleanup();
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
beforeEach(() => {
|
|
107
|
+
exponentialDefaultStarts.length = 0;
|
|
108
|
+
exponentialCustomStarts.length = 0;
|
|
109
|
+
fixedCustomStarts.length = 0;
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
describe("job backoff waits between retries (fw#3167)", () => {
|
|
113
|
+
test('backoff: "exponential" without delayMs waits the default 1000ms base delay', async () => {
|
|
114
|
+
await stack.http.writeOk(
|
|
115
|
+
"backofffixture:write:trigger-exponential-default",
|
|
116
|
+
{},
|
|
117
|
+
TestUsers.admin,
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
await waitFor(() => {
|
|
121
|
+
expect(exponentialDefaultStarts).toHaveLength(2);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
const [gap = 0] = gapsBetween(exponentialDefaultStarts);
|
|
125
|
+
expect(gap).toBeGreaterThanOrEqual(1000);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test("object-form backoff.delayMs is configurable per job and grows exponentially", async () => {
|
|
129
|
+
await stack.http.writeOk(
|
|
130
|
+
"backofffixture:write:trigger-exponential-custom",
|
|
131
|
+
{},
|
|
132
|
+
TestUsers.admin,
|
|
133
|
+
);
|
|
134
|
+
|
|
135
|
+
await waitFor(() => {
|
|
136
|
+
expect(exponentialCustomStarts).toHaveLength(4);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
const [gap1 = 0, gap2 = 0, gap3 = 0] = gapsBetween(exponentialCustomStarts);
|
|
140
|
+
expect(gap1).toBeGreaterThanOrEqual(50);
|
|
141
|
+
expect(gap2).toBeGreaterThanOrEqual(100);
|
|
142
|
+
expect(gap3).toBeGreaterThanOrEqual(200);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test('backoff: { type: "fixed", delayMs } waits the configured constant delay', async () => {
|
|
146
|
+
await stack.http.writeOk("backofffixture:write:trigger-fixed-custom", {}, TestUsers.admin);
|
|
147
|
+
|
|
148
|
+
await waitFor(() => {
|
|
149
|
+
expect(fixedCustomStarts).toHaveLength(2);
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
const [gap = 0] = gapsBetween(fixedCustomStarts);
|
|
153
|
+
expect(gap).toBeGreaterThanOrEqual(300);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
@@ -1390,4 +1390,39 @@ describe("boot gates", () => {
|
|
|
1390
1390
|
});
|
|
1391
1391
|
expect(() => createRegistry([sequentialGate])).toThrow(/bootGate with concurrency/);
|
|
1392
1392
|
});
|
|
1393
|
+
|
|
1394
|
+
test("object-form backoff.delayMs must be a positive integer", () => {
|
|
1395
|
+
for (const badDelayMs of [0, -5, 1.5]) {
|
|
1396
|
+
const feature = defineFeature("badbackoff", (r) => {
|
|
1397
|
+
r.job(
|
|
1398
|
+
"check",
|
|
1399
|
+
{ trigger: { manual: true }, backoff: { type: "fixed", delayMs: badDelayMs } },
|
|
1400
|
+
async () => {
|
|
1401
|
+
gateLog.push("never");
|
|
1402
|
+
},
|
|
1403
|
+
);
|
|
1404
|
+
});
|
|
1405
|
+
expect(() => createRegistry([feature])).toThrow(
|
|
1406
|
+
/backoff\.delayMs must be a positive integer/,
|
|
1407
|
+
);
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
const validObjectForm = defineFeature("goodbackoffobject", (r) => {
|
|
1411
|
+
r.job(
|
|
1412
|
+
"check",
|
|
1413
|
+
{ trigger: { manual: true }, backoff: { type: "exponential", delayMs: 100 } },
|
|
1414
|
+
async () => {
|
|
1415
|
+
gateLog.push("never");
|
|
1416
|
+
},
|
|
1417
|
+
);
|
|
1418
|
+
});
|
|
1419
|
+
expect(() => createRegistry([validObjectForm])).not.toThrow();
|
|
1420
|
+
|
|
1421
|
+
const validStringForm = defineFeature("goodbackoffstring", (r) => {
|
|
1422
|
+
r.job("check", { trigger: { manual: true }, backoff: "fixed" }, async () => {
|
|
1423
|
+
gateLog.push("never");
|
|
1424
|
+
});
|
|
1425
|
+
});
|
|
1426
|
+
expect(() => createRegistry([validStringForm])).not.toThrow();
|
|
1427
|
+
});
|
|
1393
1428
|
});
|
package/src/jobs/job-runner.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Queue, Worker } from "bullmq";
|
|
1
|
+
import { type JobsOptions, Queue, Worker } from "bullmq";
|
|
2
2
|
import { Redis } from "ioredis";
|
|
3
3
|
import { requestContext } from "../api/request-context";
|
|
4
4
|
import type { DbConnection, DbRow } from "../db/connection";
|
|
@@ -317,13 +317,27 @@ function timeoutReject(
|
|
|
317
317
|
};
|
|
318
318
|
}
|
|
319
319
|
|
|
320
|
+
// Default base delay when a job opts into backoff without an explicit
|
|
321
|
+
// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
|
|
322
|
+
// NaN/undefined, which is falsy — the job retries immediately instead of
|
|
323
|
+
// waiting.
|
|
324
|
+
const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
|
|
325
|
+
|
|
320
326
|
// Shared by dispatch() and handleEvent() — an event-triggered job must retry
|
|
321
327
|
// on failure the same way a directly-dispatched one does; a duplicated
|
|
322
328
|
// inline computation in handleEvent previously dropped both options.
|
|
323
|
-
function buildRetryBullOpts(jobDef: JobDefinition):
|
|
324
|
-
const opts:
|
|
325
|
-
if (jobDef.retries !== undefined) opts
|
|
326
|
-
if (jobDef.backoff)
|
|
329
|
+
function buildRetryBullOpts(jobDef: JobDefinition): Pick<JobsOptions, "attempts" | "backoff"> {
|
|
330
|
+
const opts: Pick<JobsOptions, "attempts" | "backoff"> = {};
|
|
331
|
+
if (jobDef.retries !== undefined) opts.attempts = jobDef.retries + 1;
|
|
332
|
+
if (jobDef.backoff) {
|
|
333
|
+
opts.backoff =
|
|
334
|
+
typeof jobDef.backoff === "string"
|
|
335
|
+
? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
|
|
336
|
+
: {
|
|
337
|
+
type: jobDef.backoff.type,
|
|
338
|
+
delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
327
341
|
return opts;
|
|
328
342
|
}
|
|
329
343
|
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { composeEnvSchema, readKumikoMeta } from "../../env";
|
|
4
|
+
import { prometheusMetricsEnvSchema, resolveObservabilityWiring } from "../metrics-wiring";
|
|
5
|
+
|
|
6
|
+
describe("resolveObservabilityWiring", () => {
|
|
7
|
+
it("returns {} without a token", () => {
|
|
8
|
+
expect(resolveObservabilityWiring(undefined)).toEqual({});
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it("returns {} for an empty token", () => {
|
|
12
|
+
expect(resolveObservabilityWiring("")).toEqual({});
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it("wires a prometheus provider and metrics route when a token is set", () => {
|
|
16
|
+
const token = "a".repeat(32);
|
|
17
|
+
const wiring = resolveObservabilityWiring(token);
|
|
18
|
+
|
|
19
|
+
expect("metrics" in wiring).toBe(true);
|
|
20
|
+
if (!("metrics" in wiring)) throw new Error("expected wiring to include metrics");
|
|
21
|
+
|
|
22
|
+
expect(wiring.metrics).toEqual({ path: "/metrics", token });
|
|
23
|
+
expect(wiring.observability.name).toBe("prometheus");
|
|
24
|
+
expect(wiring.observability.meter.snapshot()).toEqual(new Map());
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
describe("prometheusMetricsEnvSchema", () => {
|
|
29
|
+
it("rejects tokens shorter than 32 characters", () => {
|
|
30
|
+
const result = prometheusMetricsEnvSchema.safeParse({
|
|
31
|
+
PROMETHEUS_METRICS_TOKEN: "a".repeat(31),
|
|
32
|
+
});
|
|
33
|
+
expect(result.success).toBe(false);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it("accepts a 32-character token", () => {
|
|
37
|
+
const result = prometheusMetricsEnvSchema.safeParse({
|
|
38
|
+
PROMETHEUS_METRICS_TOKEN: "a".repeat(32),
|
|
39
|
+
});
|
|
40
|
+
expect(result.success).toBe(true);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("accepts a missing token", () => {
|
|
44
|
+
const result = prometheusMetricsEnvSchema.safeParse({});
|
|
45
|
+
expect(result.success).toBe(true);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("exposes pulumi secret metadata for consumer env-schemas", () => {
|
|
49
|
+
const { schema } = composeEnvSchema({
|
|
50
|
+
features: [],
|
|
51
|
+
extend: z.object({ FOO: z.string() }).extend(prometheusMetricsEnvSchema.shape),
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
const field = schema.shape["PROMETHEUS_METRICS_TOKEN"];
|
|
55
|
+
if (!(field instanceof z.ZodType))
|
|
56
|
+
throw new Error("expected PROMETHEUS_METRICS_TOKEN in composed schema");
|
|
57
|
+
const meta = readKumikoMeta(field);
|
|
58
|
+
expect(meta.pulumi?.secret).toBe(true);
|
|
59
|
+
expect(meta.pulumi?.generator).toBe("openssl rand -base64 32");
|
|
60
|
+
});
|
|
61
|
+
});
|
|
@@ -16,6 +16,11 @@ export {
|
|
|
16
16
|
createSafeMetricsHandle,
|
|
17
17
|
createUnboundMetricsHandle,
|
|
18
18
|
} from "./metrics-handle";
|
|
19
|
+
export {
|
|
20
|
+
type ObservabilityWiring,
|
|
21
|
+
prometheusMetricsEnvSchema,
|
|
22
|
+
resolveObservabilityWiring,
|
|
23
|
+
} from "./metrics-wiring";
|
|
19
24
|
export { createNoopProvider } from "./noop-provider";
|
|
20
25
|
export {
|
|
21
26
|
createPrometheusMeter,
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { createNoopProvider } from "./noop-provider";
|
|
3
|
+
import { createPrometheusMeter, type PrometheusMeter } from "./prometheus-meter";
|
|
4
|
+
import type { ObservabilityProvider } from "./types";
|
|
5
|
+
|
|
6
|
+
export const prometheusMetricsEnvSchema = z.object({
|
|
7
|
+
PROMETHEUS_METRICS_TOKEN: z
|
|
8
|
+
.string()
|
|
9
|
+
.min(32)
|
|
10
|
+
.optional()
|
|
11
|
+
.describe("Bearer token for /metrics; unset keeps the endpoint off.")
|
|
12
|
+
.meta({ kumiko: { pulumi: { secret: true, generator: "openssl rand -base64 32" } } }),
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
type PrometheusObservabilityProvider = ObservabilityProvider & { readonly meter: PrometheusMeter };
|
|
16
|
+
|
|
17
|
+
export type ObservabilityWiring =
|
|
18
|
+
| {
|
|
19
|
+
readonly observability: PrometheusObservabilityProvider;
|
|
20
|
+
readonly metrics: { readonly path: string; readonly token: string };
|
|
21
|
+
}
|
|
22
|
+
| Record<string, never>;
|
|
23
|
+
|
|
24
|
+
// Fail-closed: public tenant hosts share the /metrics port, so no token means no endpoint.
|
|
25
|
+
export function resolveObservabilityWiring(metricsToken: string | undefined): ObservabilityWiring {
|
|
26
|
+
if (!metricsToken) return {};
|
|
27
|
+
return {
|
|
28
|
+
// Overrides the spread's name "noop", which would otherwise show up in diagnostics/logs.
|
|
29
|
+
observability: { ...createNoopProvider(), name: "prometheus", meter: createPrometheusMeter() },
|
|
30
|
+
metrics: { path: "/metrics", token: metricsToken },
|
|
31
|
+
};
|
|
32
|
+
}
|