@cosmicdrift/kumiko-framework 0.304.0 → 0.306.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/__tests__/extra-route-rejection.test.ts +38 -0
- package/src/api/__tests__/extra-routes.integration.test.ts +30 -0
- package/src/api/__tests__/server-boot-guards.test.ts +1 -0
- package/src/api/__tests__/server-error-logging.test.ts +104 -0
- package/src/api/api-constants.ts +13 -0
- package/src/api/extra-route.ts +33 -4
- package/src/api/index.ts +1 -0
- package/src/api/request-context.ts +5 -4
- package/src/api/routes.ts +26 -1
- package/src/api/server.ts +8 -2
- package/src/bun-db/__tests__/closed-connection-retry.integration.test.ts +159 -0
- package/src/bun-db/__tests__/select-many-retry.integration.test.ts +138 -0
- package/src/bun-db/query.ts +42 -18
- package/src/changes.json +108 -0
- package/src/db/__tests__/pg-error.test.ts +14 -0
- package/src/db/__tests__/system-db-view-export.test.ts +107 -0
- package/src/db/__tests__/tenant-db-no-raw.test.ts +10 -0
- package/src/db/__tests__/with-systemdb-unsafe-raw-grant.test.ts +71 -0
- package/src/db/event-store-executor-write.ts +7 -0
- package/src/db/index.ts +1 -1
- package/src/db/pg-error.ts +13 -0
- package/src/db/queries/__tests__/{unsafe-read-retrying.test.ts → unsafe-read-retrying.integration.test.ts} +28 -28
- package/src/db/tenant-db.ts +140 -16
- package/src/engine/__tests__/boot-validator-gdpr-storage.test.ts +6 -1
- package/src/engine/__tests__/boot-validator-s0-integration.test.ts +2 -2
- package/src/engine/__tests__/boot-validator.test.ts +1 -1
- package/src/engine/__tests__/tier-resolver-extension.test.ts +1 -1
- package/src/engine/boot-validator/access-declarations.ts +5 -66
- package/src/engine/extension-names.ts +55 -25
- package/src/engine/extensions/storage-provider.ts +14 -41
- package/src/engine/extensions/tenant-data.ts +4 -0
- package/src/engine/extensions/tenant-resource.ts +40 -0
- package/src/engine/extensions/user-data.ts +8 -7
- package/src/engine/feature-ast/__tests__/handler-header-roundtrip.test.ts +669 -0
- package/src/engine/feature-ast/__tests__/parse.test.ts +5 -3
- package/src/engine/feature-ast/__tests__/patch-update.test.ts +718 -0
- package/src/engine/feature-ast/__tests__/pattern-change-schema.test.ts +819 -0
- package/src/engine/feature-ast/entity-field-types.ts +41 -0
- package/src/engine/feature-ast/extractors/handlers.ts +217 -84
- package/src/engine/feature-ast/extractors/hooks.ts +72 -15
- package/src/engine/feature-ast/extractors/round2.ts +21 -0
- package/src/engine/feature-ast/extractors/shared.ts +9 -0
- package/src/engine/feature-ast/index.ts +11 -1
- package/src/engine/feature-ast/patch.ts +338 -5
- package/src/engine/feature-ast/patcher.ts +2 -2
- package/src/engine/feature-ast/pattern-change-schema.ts +1411 -0
- package/src/engine/feature-ast/patterns.ts +22 -15
- package/src/engine/feature-ast/render.ts +1 -0
- package/src/engine/feature-ui-extensions.ts +8 -7
- package/src/engine/index.ts +23 -5
- package/src/engine/personal-data-fields.ts +66 -0
- package/src/engine/registry-validate.ts +15 -0
- package/src/engine/registry.ts +2 -0
- package/src/engine/types/extension-options-map.ts +1 -0
- package/src/engine/types/index.ts +8 -0
- package/src/env/__tests__/dry-run.test.ts +43 -3
- package/src/env/dry-run.ts +28 -15
- package/src/errors/__tests__/write-failures.test.ts +47 -4
- package/src/errors/i18n/de.yaml +12 -0
- package/src/errors/i18n/en.yaml +12 -0
- package/src/errors/reasons.ts +4 -0
- package/src/errors/write-error-info.ts +12 -3
- package/src/jobs/__tests__/job-backoff.integration.test.ts +155 -0
- package/src/jobs/__tests__/job-retention.integration.test.ts +316 -0
- package/src/jobs/__tests__/job-retry-enqueue-paths.integration.test.ts +309 -0
- package/src/jobs/__tests__/jobs.integration.test.ts +38 -3
- package/src/jobs/job-runner.ts +170 -19
- package/src/pipeline/__tests__/ctx-bridge.integration.test.ts +113 -26
- package/src/pipeline/__tests__/dispatcher.test.ts +5 -0
- package/src/pipeline/__tests__/hook-systemdb-escape-hatch.integration.test.ts +246 -0
- package/src/pipeline/__tests__/idempotency-transient-failure.integration.test.ts +198 -0
- package/src/pipeline/__tests__/public-intake-runtime-gate.integration.test.ts +425 -0
- package/src/pipeline/__tests__/redis-pipeline.integration.test.ts +59 -0
- package/src/pipeline/active-membership.ts +5 -1
- package/src/pipeline/dispatch-batch.ts +59 -13
- package/src/pipeline/dispatch-query.ts +16 -5
- package/src/pipeline/dispatch-shared.ts +12 -5
- package/src/pipeline/dispatch-stream.ts +7 -2
- package/src/pipeline/dispatch-write.ts +22 -5
- package/src/pipeline/dispatcher.ts +9 -2
- package/src/pipeline/idempotency.ts +16 -0
- package/src/pipeline/member-reader.ts +3 -1
- package/src/pipeline/system-identity-switch.ts +22 -4
- package/src/pipeline/write-origin.ts +107 -0
- package/src/rate-limit/__tests__/middleware.integration.test.ts +40 -0
- package/src/rate-limit/middleware.ts +3 -0
- package/src/stack/__tests__/setup-test-stack-metrics.integration.test.ts +79 -0
- package/src/stack/test-stack.ts +5 -0
- package/src/testing/closed-connection-error.ts +62 -0
- package/src/testing/index.ts +1 -0
- package/src/bun-db/__tests__/select-many-retry.test.ts +0 -79
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.306.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -198,8 +198,8 @@
|
|
|
198
198
|
"./package.json": "./package.json"
|
|
199
199
|
},
|
|
200
200
|
"dependencies": {
|
|
201
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
202
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
201
|
+
"@cosmicdrift/kumiko-http": "0.306.0",
|
|
202
|
+
"@cosmicdrift/kumiko-types": "0.306.0",
|
|
203
203
|
"bullmq": "^5.76.7",
|
|
204
204
|
"bun-types": "^1.3.13",
|
|
205
205
|
"hono": "^4.13.1",
|
|
@@ -215,7 +215,7 @@
|
|
|
215
215
|
"zod": "^4.4.3"
|
|
216
216
|
},
|
|
217
217
|
"devDependencies": {
|
|
218
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
218
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.306.0",
|
|
219
219
|
"bun-types": "^1.3.13",
|
|
220
220
|
"pino-pretty": "^13.1.3"
|
|
221
221
|
},
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Constructor invariants for ExtraRouteRejection's `retryAfterSeconds` option
|
|
2
|
+
// (kumiko-framework#3168): it only makes sense together with 503 and must
|
|
3
|
+
// render as a valid Retry-After delta-seconds header.
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
import { ExtraRouteRejection } from "../extra-route";
|
|
7
|
+
|
|
8
|
+
describe("ExtraRouteRejection retryAfterSeconds invariants", () => {
|
|
9
|
+
test("retryAfterSeconds with a non-503 status throws RangeError", () => {
|
|
10
|
+
expect(
|
|
11
|
+
() => new ExtraRouteRejection(404, { error: "x" }, undefined, { retryAfterSeconds: 30 }),
|
|
12
|
+
).toThrow(RangeError);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test("negative retryAfterSeconds throws RangeError", () => {
|
|
16
|
+
expect(
|
|
17
|
+
() => new ExtraRouteRejection(503, { error: "x" }, undefined, { retryAfterSeconds: -1 }),
|
|
18
|
+
).toThrow(RangeError);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("non-integer retryAfterSeconds throws RangeError", () => {
|
|
22
|
+
expect(
|
|
23
|
+
() => new ExtraRouteRejection(503, { error: "x" }, undefined, { retryAfterSeconds: 1.5 }),
|
|
24
|
+
).toThrow(RangeError);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("retryAfterSeconds: 0 with status 503 is accepted", () => {
|
|
28
|
+
const rejection = new ExtraRouteRejection(503, { error: "x" }, undefined, {
|
|
29
|
+
retryAfterSeconds: 0,
|
|
30
|
+
});
|
|
31
|
+
expect(rejection.retryAfterSeconds).toBe(0);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("no options leaves retryAfterSeconds undefined", () => {
|
|
35
|
+
const rejection = new ExtraRouteRejection(503, { error: "x" });
|
|
36
|
+
expect(rejection.retryAfterSeconds).toBeUndefined();
|
|
37
|
+
});
|
|
38
|
+
});
|
|
@@ -461,6 +461,14 @@ describe("extraRoutes: entry:signature", () => {
|
|
|
461
461
|
if (req.headers["x-force-404"] === "1") {
|
|
462
462
|
throw new ExtraRouteRejection(404, { error: "unknown-provider" });
|
|
463
463
|
}
|
|
464
|
+
if (req.headers["x-force-503"] === "retry") {
|
|
465
|
+
throw new ExtraRouteRejection(503, { error: "not-ready" }, undefined, {
|
|
466
|
+
retryAfterSeconds: 30,
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
if (req.headers["x-force-503"] === "plain") {
|
|
470
|
+
throw new ExtraRouteRejection(503, { error: "not-ready" });
|
|
471
|
+
}
|
|
464
472
|
if (req.headers["x-hmac"] !== signHmac(req.rawBody)) {
|
|
465
473
|
throw new Error("signature mismatch");
|
|
466
474
|
}
|
|
@@ -536,6 +544,28 @@ describe("extraRoutes: entry:signature", () => {
|
|
|
536
544
|
expect(await res.json()).toEqual({ error: "unknown-provider" });
|
|
537
545
|
});
|
|
538
546
|
|
|
547
|
+
test("verify() throwing ExtraRouteRejection(503, body, options) surfaces the body and Retry-After header", async () => {
|
|
548
|
+
const res = await stack.app.request("/webhooks/probe", {
|
|
549
|
+
method: "POST",
|
|
550
|
+
headers: { "x-force-503": "retry", "content-type": "application/json" },
|
|
551
|
+
body: JSON.stringify({ note: "x" }),
|
|
552
|
+
});
|
|
553
|
+
expect(res.status).toBe(503);
|
|
554
|
+
expect(await res.json()).toEqual({ error: "not-ready" });
|
|
555
|
+
expect(res.headers.get("retry-after")).toBe("30");
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
test("verify() throwing ExtraRouteRejection(503, body) without options omits the Retry-After header", async () => {
|
|
559
|
+
const res = await stack.app.request("/webhooks/probe", {
|
|
560
|
+
method: "POST",
|
|
561
|
+
headers: { "x-force-503": "plain", "content-type": "application/json" },
|
|
562
|
+
body: JSON.stringify({ note: "x" }),
|
|
563
|
+
});
|
|
564
|
+
expect(res.status).toBe(503);
|
|
565
|
+
expect(await res.json()).toEqual({ error: "not-ready" });
|
|
566
|
+
expect(res.headers.get("retry-after")).toBeNull();
|
|
567
|
+
});
|
|
568
|
+
|
|
539
569
|
test("mounted under /api/:provider — no session needed, honoPathToRegex matches the :param, rawBody arrives intact through /api/*", async () => {
|
|
540
570
|
const rawBody = JSON.stringify({ event: "payment.succeeded" });
|
|
541
571
|
const res = await stack.app.request("/api/webhooks/stripe", {
|
|
@@ -228,6 +228,7 @@ describe("buildServer — auth membershipQuery requires a principalStatus provid
|
|
|
228
228
|
r.extendsRegistrar(EXT_PRINCIPAL_STATUS, {});
|
|
229
229
|
r.useExtension(EXT_PRINCIPAL_STATUS, "has-principal-status", {
|
|
230
230
|
resolveStatus: async () => "active" as const,
|
|
231
|
+
resolveProfile: async () => ({ globalRoles: [] }),
|
|
231
232
|
});
|
|
232
233
|
});
|
|
233
234
|
|
|
@@ -27,6 +27,14 @@ const boomFeature = defineFeature("boom", (r) => {
|
|
|
27
27
|
},
|
|
28
28
|
openToAll,
|
|
29
29
|
);
|
|
30
|
+
r.writeHandler(
|
|
31
|
+
"explode",
|
|
32
|
+
z.object({}),
|
|
33
|
+
async () => {
|
|
34
|
+
throw new Error("disk on fire during write");
|
|
35
|
+
},
|
|
36
|
+
openToAll,
|
|
37
|
+
);
|
|
30
38
|
r.queryHandler(
|
|
31
39
|
"decode",
|
|
32
40
|
z.object({}),
|
|
@@ -58,6 +66,16 @@ const boomFeature = defineFeature("boom", (r) => {
|
|
|
58
66
|
async () => ({ ok: true }),
|
|
59
67
|
openToAll,
|
|
60
68
|
);
|
|
69
|
+
// Same abort path TenantDb uses (signal.throwIfAborted()), without a DB.
|
|
70
|
+
r.queryHandler(
|
|
71
|
+
"abort-probe",
|
|
72
|
+
z.object({}),
|
|
73
|
+
async (_query, ctx) => {
|
|
74
|
+
ctx.signal?.throwIfAborted();
|
|
75
|
+
return { ok: true };
|
|
76
|
+
},
|
|
77
|
+
openToAll,
|
|
78
|
+
);
|
|
61
79
|
});
|
|
62
80
|
|
|
63
81
|
const { app, jwt } = buildServer({
|
|
@@ -140,11 +158,97 @@ describe("HTTP layer logs unexpected 5xx faults", () => {
|
|
|
140
158
|
}
|
|
141
159
|
});
|
|
142
160
|
|
|
161
|
+
test("a throwing write 500s AND its cause reaches the log (reraise keeps the cause)", async () => {
|
|
162
|
+
const calls: unknown[][] = [];
|
|
163
|
+
const spy = spyOn(console, "error").mockImplementation((...args) => {
|
|
164
|
+
calls.push(args);
|
|
165
|
+
});
|
|
166
|
+
try {
|
|
167
|
+
const res = await app.request("/api/write", {
|
|
168
|
+
method: "POST",
|
|
169
|
+
headers: await auth(),
|
|
170
|
+
body: JSON.stringify({ type: "boom:write:explode", payload: {} }),
|
|
171
|
+
});
|
|
172
|
+
expect(res.status).toBe(500);
|
|
173
|
+
const hit = calls.find(
|
|
174
|
+
(args) => typeof args[0] === "string" && args[0].includes("[api] handler failed"),
|
|
175
|
+
);
|
|
176
|
+
const data = hit?.[1];
|
|
177
|
+
expect(isRecord(data)).toBe(true);
|
|
178
|
+
if (!isRecord(data)) return;
|
|
179
|
+
expect(data["type"]).toBe("boom:write:explode");
|
|
180
|
+
expect(data["cause"]).toBe("disk on fire during write");
|
|
181
|
+
} finally {
|
|
182
|
+
spy.mockRestore();
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
|
|
143
186
|
test("a 404 stays off the error level (it is a client outcome, not a server fault)", async () => {
|
|
144
187
|
const { status, errors } = await queryWithCapturedWarnings("nope:query:nothing", {});
|
|
145
188
|
expect(status).toBe(404);
|
|
146
189
|
expect(apiFaultLog(errors)).toBeUndefined();
|
|
147
190
|
});
|
|
191
|
+
|
|
192
|
+
test("a pre-aborted client signal 499s and logs a warn, not '[api] handler failed'", async () => {
|
|
193
|
+
const controller = new AbortController();
|
|
194
|
+
controller.abort();
|
|
195
|
+
const warnings: unknown[][] = [];
|
|
196
|
+
const errors: unknown[][] = [];
|
|
197
|
+
const warnSpy = spyOn(console, "warn").mockImplementation((...args) => {
|
|
198
|
+
warnings.push(args);
|
|
199
|
+
});
|
|
200
|
+
const errorSpy = spyOn(console, "error").mockImplementation((...args) => {
|
|
201
|
+
errors.push(args);
|
|
202
|
+
});
|
|
203
|
+
try {
|
|
204
|
+
const res = await app.request(
|
|
205
|
+
new Request("http://test.local/api/query", {
|
|
206
|
+
method: "POST",
|
|
207
|
+
headers: await auth(),
|
|
208
|
+
body: JSON.stringify({ type: "boom:query:abort-probe", payload: {} }),
|
|
209
|
+
signal: controller.signal,
|
|
210
|
+
}),
|
|
211
|
+
);
|
|
212
|
+
expect(res.status).toBe(499);
|
|
213
|
+
expect(apiFaultLog(errors)).toBeUndefined();
|
|
214
|
+
const hit = warnings.find(
|
|
215
|
+
(args) =>
|
|
216
|
+
typeof args[0] === "string" && args[0].includes("[api] request aborted by client"),
|
|
217
|
+
);
|
|
218
|
+
expect(hit).toBeDefined();
|
|
219
|
+
const data = hit?.[1];
|
|
220
|
+
expect(isRecord(data)).toBe(true);
|
|
221
|
+
if (!isRecord(data)) return;
|
|
222
|
+
expect(data["status"]).toBe(499);
|
|
223
|
+
expect(data["type"]).toBe("boom:query:abort-probe");
|
|
224
|
+
} finally {
|
|
225
|
+
warnSpy.mockRestore();
|
|
226
|
+
errorSpy.mockRestore();
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("a pre-aborted client signal still 500s + logs when the handler fails for an unrelated reason", async () => {
|
|
231
|
+
const controller = new AbortController();
|
|
232
|
+
controller.abort();
|
|
233
|
+
const calls: unknown[][] = [];
|
|
234
|
+
const spy = spyOn(console, "error").mockImplementation((...args) => {
|
|
235
|
+
calls.push(args);
|
|
236
|
+
});
|
|
237
|
+
try {
|
|
238
|
+
const res = await app.request(
|
|
239
|
+
new Request("http://test.local/api/query", {
|
|
240
|
+
method: "POST",
|
|
241
|
+
headers: await auth(),
|
|
242
|
+
body: JSON.stringify({ type: "boom:query:explode", payload: {} }),
|
|
243
|
+
signal: controller.signal,
|
|
244
|
+
}),
|
|
245
|
+
);
|
|
246
|
+
expect(res.status).toBe(500);
|
|
247
|
+
expect(apiFaultLog(calls)).toBeDefined();
|
|
248
|
+
} finally {
|
|
249
|
+
spy.mockRestore();
|
|
250
|
+
}
|
|
251
|
+
});
|
|
148
252
|
});
|
|
149
253
|
|
|
150
254
|
describe("HTTP layer logs 4xx client faults on warn (#3077)", () => {
|
package/src/api/api-constants.ts
CHANGED
|
@@ -101,6 +101,19 @@ export const BODY_LIMIT_OPT_OUT_PATHS: ReadonlySet<string> = new Set([
|
|
|
101
101
|
`/api${Routes.files}`,
|
|
102
102
|
]);
|
|
103
103
|
|
|
104
|
+
// authEndpointRateLimit (L2) exceptions, exact method+path match, no prefix/glob.
|
|
105
|
+
// A forgotten entry here is safe (over-limited, not unlimited).
|
|
106
|
+
export const AUTH_RATE_LIMIT_EXEMPT_ROUTES: ReadonlyArray<{
|
|
107
|
+
readonly method: string;
|
|
108
|
+
readonly path: string;
|
|
109
|
+
}> = [{ method: "GET", path: `/api${Routes.authTenants}` }];
|
|
110
|
+
|
|
111
|
+
export function isAuthRateLimitExempt(method: string, path: string): boolean {
|
|
112
|
+
return AUTH_RATE_LIMIT_EXEMPT_ROUTES.some(
|
|
113
|
+
(route) => route.method === method && route.path === path,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
104
117
|
// Methods that can mutate server state. GET/HEAD/OPTIONS are safe under
|
|
105
118
|
// CORS + SameSite-cookie semantics and skip the CSRF / Origin guards entirely.
|
|
106
119
|
export const STATE_CHANGING_METHODS: ReadonlySet<string> = new Set([
|
package/src/api/extra-route.ts
CHANGED
|
@@ -128,19 +128,48 @@ export function signatureRoute<T>(def: SignatureExtraRoute<T>): ExtraRouteDefini
|
|
|
128
128
|
return def as unknown as SignatureExtraRoute<unknown>;
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
-
export type ExtraRouteRejectionStatus = 400 | 401 | 403 | 404 | 500;
|
|
131
|
+
export type ExtraRouteRejectionStatus = 400 | 401 | 403 | 404 | 500 | 503;
|
|
132
|
+
|
|
133
|
+
export type ExtraRouteRejectionOptions = { readonly retryAfterSeconds?: number };
|
|
132
134
|
|
|
133
135
|
/** Thrown by `verify()` to reject a signature route with a specific status +
|
|
134
136
|
* JSON body. Any other throw from `verify()` is mapped to 401
|
|
135
|
-
* `extra_route_signature_invalid` by the buildServer wrapper.
|
|
137
|
+
* `extra_route_signature_invalid` by the buildServer wrapper.
|
|
138
|
+
*
|
|
139
|
+
* 503 signals "temporarily not ready" (e.g. a dependency the verify step
|
|
140
|
+
* needs is down) — webhook providers like Stripe retry on 503.
|
|
141
|
+
* `options.retryAfterSeconds` renders as the `Retry-After` header. */
|
|
136
142
|
export class ExtraRouteRejection extends Error {
|
|
137
143
|
readonly status: ExtraRouteRejectionStatus;
|
|
138
144
|
readonly body: unknown;
|
|
139
|
-
|
|
140
|
-
|
|
145
|
+
readonly retryAfterSeconds: number | undefined;
|
|
146
|
+
|
|
147
|
+
constructor(
|
|
148
|
+
status: ExtraRouteRejectionStatus,
|
|
149
|
+
body: unknown,
|
|
150
|
+
message?: string,
|
|
151
|
+
options?: ExtraRouteRejectionOptions,
|
|
152
|
+
) {
|
|
141
153
|
super(message ?? `extra route rejected with status ${status}`);
|
|
142
154
|
this.name = "ExtraRouteRejection";
|
|
143
155
|
this.status = status;
|
|
144
156
|
this.body = body;
|
|
157
|
+
const retryAfterSeconds = options?.retryAfterSeconds;
|
|
158
|
+
if (retryAfterSeconds !== undefined) {
|
|
159
|
+
// Retry-After is only meaningful for 503 in this union (RFC 9110) and
|
|
160
|
+
// is rendered as a delta-seconds header — reject anything that could
|
|
161
|
+
// not survive that round-trip.
|
|
162
|
+
if (status !== 503) {
|
|
163
|
+
throw new RangeError(
|
|
164
|
+
"ExtraRouteRejection: retryAfterSeconds is only valid with status 503",
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
if (!Number.isInteger(retryAfterSeconds) || retryAfterSeconds < 0) {
|
|
168
|
+
throw new RangeError(
|
|
169
|
+
"ExtraRouteRejection: retryAfterSeconds must be a non-negative integer",
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
this.retryAfterSeconds = retryAfterSeconds;
|
|
145
174
|
}
|
|
146
175
|
}
|
package/src/api/index.ts
CHANGED
|
@@ -17,10 +17,11 @@ import { generateId } from "../utils";
|
|
|
17
17
|
// with correlationId, forms a causal DAG across streams.
|
|
18
18
|
// signal — AbortSignal from the underlying HTTP request. Aborts
|
|
19
19
|
// when the client disconnects (mobile back-press, tab
|
|
20
|
-
// close).
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
// Undefined for non-HTTP
|
|
20
|
+
// close). Query/stream handlers check signal.aborted at
|
|
21
|
+
// chunk or query boundaries; runBatch (write dispatch)
|
|
22
|
+
// strips it before executing so a disconnect can't abort
|
|
23
|
+
// a transaction mid-commit. Undefined for non-HTTP
|
|
24
|
+
// entry-points (jobs, MSP-applies) and inside write batches.
|
|
24
25
|
export type RequestContextData = {
|
|
25
26
|
readonly requestId: string;
|
|
26
27
|
readonly correlationId: string;
|
package/src/api/routes.ts
CHANGED
|
@@ -29,6 +29,10 @@ export const StreamFrame = {
|
|
|
29
29
|
error: "error",
|
|
30
30
|
} as const;
|
|
31
31
|
|
|
32
|
+
// Non-standard but widely used for "client hung up" — distinct from a real
|
|
33
|
+
// 5xx so a disconnect never gets logged as a server fault.
|
|
34
|
+
const CLIENT_CLOSED_REQUEST_STATUS = 499;
|
|
35
|
+
|
|
32
36
|
export type ApiRoutesOptions = {
|
|
33
37
|
// Override the SSE heartbeat interval (ms). Default SSE_HEARTBEAT_INTERVAL_MS.
|
|
34
38
|
// Deployment-tunable for proxies with different idle timeouts — also used
|
|
@@ -203,7 +207,7 @@ export function createApiRoutes(dispatcher: Dispatcher, options: ApiRoutesOption
|
|
|
203
207
|
// in-flight .next(), so awaiting here would block the response until
|
|
204
208
|
// that pending pull resolves (which may be never for an idle stream).
|
|
205
209
|
void generator.return(undefined).catch(() => {});
|
|
206
|
-
return c.body(null,
|
|
210
|
+
return c.body(null, CLIENT_CLOSED_REQUEST_STATUS as ContentfulStatusCode); // @cast-boundary non-standard client-closed-request status, Hono's union doesn't include it
|
|
207
211
|
}
|
|
208
212
|
|
|
209
213
|
return streamSSE(c, async (stream) => {
|
|
@@ -364,6 +368,13 @@ function logClientFault(err: KumikoError, requestId: string | undefined, type?:
|
|
|
364
368
|
});
|
|
365
369
|
}
|
|
366
370
|
|
|
371
|
+
// Identity-checked against THIS request's signal — signal.aborted alone
|
|
372
|
+
// would also match a real 5xx that happens to race a disconnect.
|
|
373
|
+
function isClientAbort(err: KumikoError): boolean {
|
|
374
|
+
const signal = requestContext.get()?.signal;
|
|
375
|
+
return signal?.aborted === true && err.cause !== undefined && err.cause === signal.reason;
|
|
376
|
+
}
|
|
377
|
+
|
|
367
378
|
// Unexpected server faults (5xx) carry their diagnostic stack only on the
|
|
368
379
|
// in-process error — serializeError strips cause/details from the wire body.
|
|
369
380
|
// Without this a wrapped throw (InternalError{cause}) returns a 500 with zero
|
|
@@ -401,6 +412,20 @@ function writeErrorResponse(c: Context, err: KumikoError, type?: string) {
|
|
|
401
412
|
// keep the same lean shape on failure — only the `error` key.
|
|
402
413
|
function queryErrorResponse(c: Context, err: KumikoError, type?: string) {
|
|
403
414
|
const requestId = requestContext.get()?.requestId;
|
|
415
|
+
if (isClientAbort(err)) {
|
|
416
|
+
if (clientFaultLoggingEnabled()) {
|
|
417
|
+
const startedAt = requestContext.get()?.startedAt;
|
|
418
|
+
createFallbackLogger("api").warn("request aborted by client", {
|
|
419
|
+
requestId,
|
|
420
|
+
type: type?.slice(0, MAX_LOGGED_TYPE_LENGTH),
|
|
421
|
+
status: CLIENT_CLOSED_REQUEST_STATUS,
|
|
422
|
+
...(startedAt === undefined
|
|
423
|
+
? {}
|
|
424
|
+
: { durationMs: Math.round(performance.now() - startedAt) }),
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
return c.body(null, CLIENT_CLOSED_REQUEST_STATUS as ContentfulStatusCode); // @cast-boundary non-standard client-closed-request status, Hono's union doesn't include it
|
|
428
|
+
}
|
|
404
429
|
logServerFault(err, requestId, type);
|
|
405
430
|
const body = serializeError(err, requestId);
|
|
406
431
|
return c.json(body, err.httpStatus as ContentfulStatusCode); // @cast-boundary engine-payload
|
package/src/api/server.ts
CHANGED
|
@@ -160,14 +160,17 @@ export type ServerOptions = {
|
|
|
160
160
|
// of the app process.
|
|
161
161
|
// - `auth`: gates a single path-pattern (default `/api/auth/*`)
|
|
162
162
|
// with tighter limits. Typically `limit: 5, windowSeconds: 60`
|
|
163
|
-
// to slow brute-force without breaking real users.
|
|
163
|
+
// to slow brute-force without breaking real users. GET /api/auth/tenants
|
|
164
|
+
// (session read, called on every page load) is always exempt — see
|
|
165
|
+
// AUTH_RATE_LIMIT_EXEMPT_ROUTES in api-constants.ts.
|
|
164
166
|
// Both omitted → no L1/L2 wired and no resolver auto-built unless an
|
|
165
167
|
// L3 handler declared `rateLimit:`. This keeps zero-cost when unused.
|
|
166
168
|
rateLimit?: {
|
|
167
169
|
readonly global?: Omit<GlobalIpRateLimitOptions, "resolver">;
|
|
168
170
|
readonly auth?: Omit<AuthEndpointRateLimitOptions, "resolver"> & {
|
|
169
171
|
// Path-pattern the L2 middleware applies to. Default `/api/auth/*`.
|
|
170
|
-
// Override for apps with a different auth route layout.
|
|
172
|
+
// Override for apps with a different auth route layout. The
|
|
173
|
+
// GET /api/auth/tenants exemption above applies regardless of `path`.
|
|
171
174
|
readonly path?: string;
|
|
172
175
|
};
|
|
173
176
|
};
|
|
@@ -1178,6 +1181,9 @@ function buildExtraRouteHonoHandler(
|
|
|
1178
1181
|
);
|
|
1179
1182
|
} catch (e) {
|
|
1180
1183
|
if (e instanceof ExtraRouteRejection) {
|
|
1184
|
+
if (e.retryAfterSeconds !== undefined) {
|
|
1185
|
+
return c.json(e.body, e.status, { "Retry-After": String(e.retryAfterSeconds) });
|
|
1186
|
+
}
|
|
1181
1187
|
return c.json(e.body, e.status);
|
|
1182
1188
|
}
|
|
1183
1189
|
return c.json(
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// Proves the closed-connection retry against a real postgres-js driver error,
|
|
2
|
+
// plus the Bun.SQL matcher and both drivers' extractPgError — see query.ts.
|
|
3
|
+
|
|
4
|
+
import { afterAll, describe, expect, test } from "bun:test";
|
|
5
|
+
import postgres from "postgres";
|
|
6
|
+
import { constraintOf, extractPgError, isUniqueViolation } from "../../db/pg-error";
|
|
7
|
+
import { testDatabaseUrl } from "../../testing/closed-connection-error";
|
|
8
|
+
import { waitFor } from "../../testing/wait-for";
|
|
9
|
+
import { isClosedConnectionError, unsafeReadRetrying } from "../query";
|
|
10
|
+
|
|
11
|
+
const DATABASE_URL = testDatabaseUrl();
|
|
12
|
+
|
|
13
|
+
// Terminate targets only the backend running the slow query so the retry
|
|
14
|
+
// lands on the pool's idle connection instead of reconnecting — a postgres-js
|
|
15
|
+
// reconnect under Bun can hang until connect_timeout (30s).
|
|
16
|
+
const adminClient = postgres(DATABASE_URL, { max: 1 });
|
|
17
|
+
afterAll(async () => {
|
|
18
|
+
await adminClient.end({ timeout: 0 });
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
type UnsafeFn = (sql: string, params?: readonly unknown[]) => Promise<readonly unknown[]>;
|
|
22
|
+
type Pool = { unsafe: UnsafeFn };
|
|
23
|
+
|
|
24
|
+
// Counts unsafe() calls while forwarding everything (including begin/
|
|
25
|
+
// savepoint/options) to the real handle — no mocked behavior, only counting.
|
|
26
|
+
type CountingClient<T> = T & { readonly calls: number };
|
|
27
|
+
function countingClient<T extends Pool>(real: T): CountingClient<T> {
|
|
28
|
+
let calls = 0;
|
|
29
|
+
return new Proxy(real, {
|
|
30
|
+
get(target, prop, receiver) {
|
|
31
|
+
if (prop === "calls") return calls;
|
|
32
|
+
if (prop === "unsafe") {
|
|
33
|
+
return async (...args: Parameters<UnsafeFn>) => {
|
|
34
|
+
calls++;
|
|
35
|
+
return target.unsafe(...args);
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
const value = Reflect.get(target, prop, receiver);
|
|
39
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
40
|
+
},
|
|
41
|
+
}) as CountingClient<T>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function randomAppName(): string {
|
|
45
|
+
return `kumiko-cc-retry-${crypto.randomUUID()}`;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Waits for the slow query's backend to show up as the single active
|
|
49
|
+
// session before terminating exactly that one — the pg_sleep(0.3) window
|
|
50
|
+
// bounds this, so short fixed delays instead of waitFor's default backoff.
|
|
51
|
+
async function pollForActivePid(applicationName: string, queryLike: string): Promise<number> {
|
|
52
|
+
let pid: number | undefined;
|
|
53
|
+
await waitFor(
|
|
54
|
+
async () => {
|
|
55
|
+
const rows = (await adminClient.unsafe(
|
|
56
|
+
"select pid from pg_stat_activity where application_name = $1 and state = 'active' and query like $2",
|
|
57
|
+
[applicationName, queryLike],
|
|
58
|
+
)) as readonly { pid: number }[];
|
|
59
|
+
if (rows.length === 1 && rows[0]?.pid !== undefined) {
|
|
60
|
+
pid = rows[0].pid;
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
return false;
|
|
64
|
+
},
|
|
65
|
+
{ delays: Array(100).fill(10) },
|
|
66
|
+
);
|
|
67
|
+
if (pid === undefined) {
|
|
68
|
+
throw new Error(`pollForActivePid: no single active backend found for ${applicationName}`);
|
|
69
|
+
}
|
|
70
|
+
return pid;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async function terminateBackendPid(pid: number): Promise<void> {
|
|
74
|
+
await adminClient.unsafe("select pg_terminate_backend($1)", [pid]);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
type DriverKind = "postgres-js" | "bun-sql";
|
|
78
|
+
|
|
79
|
+
function makePool(kind: DriverKind, max: number, applicationName: string): Pool {
|
|
80
|
+
if (kind === "postgres-js") {
|
|
81
|
+
return postgres(DATABASE_URL, {
|
|
82
|
+
max,
|
|
83
|
+
connection: { application_name: applicationName },
|
|
84
|
+
}) as unknown as Pool;
|
|
85
|
+
}
|
|
86
|
+
return new Bun.SQL({
|
|
87
|
+
url: DATABASE_URL,
|
|
88
|
+
max,
|
|
89
|
+
connection: { application_name: applicationName },
|
|
90
|
+
}) as unknown as Pool;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function closePool(kind: DriverKind, pool: unknown): Promise<void> {
|
|
94
|
+
if (kind === "postgres-js") {
|
|
95
|
+
await (pool as { end: (opts?: { timeout?: number }) => Promise<void> }).end({ timeout: 0 });
|
|
96
|
+
} else {
|
|
97
|
+
await (pool as { close: () => Promise<void> }).close();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const drivers: DriverKind[] = ["postgres-js", "bun-sql"];
|
|
102
|
+
|
|
103
|
+
// postgres-js only: Bun.SQL sporadically emits an unhandled reject from its
|
|
104
|
+
// internal handleClose in this terminate window, and isn't the prod driver.
|
|
105
|
+
// Multi-retry (≥3 calls) is covered deterministically by the fake-client
|
|
106
|
+
// test in select-many-retry.integration.test.ts instead of a second live test.
|
|
107
|
+
test("retries through a real server-side connection close", async () => {
|
|
108
|
+
const applicationName = randomAppName();
|
|
109
|
+
const pool = postgres(DATABASE_URL, {
|
|
110
|
+
max: 2,
|
|
111
|
+
connection: { application_name: applicationName },
|
|
112
|
+
});
|
|
113
|
+
try {
|
|
114
|
+
await Promise.all([pool.unsafe("select 1"), pool.unsafe("select 1")]);
|
|
115
|
+
const counted = countingClient(pool as unknown as Pool);
|
|
116
|
+
const pending = unsafeReadRetrying(counted as never, "select 1 as x from pg_sleep(0.3)", []);
|
|
117
|
+
const pid = await pollForActivePid(applicationName, "%pg_sleep(0.3)%");
|
|
118
|
+
await terminateBackendPid(pid);
|
|
119
|
+
const rows = await pending;
|
|
120
|
+
expect([...rows]).toEqual([{ x: 1 }]);
|
|
121
|
+
expect(counted.calls).toBe(2);
|
|
122
|
+
} finally {
|
|
123
|
+
await pool.end({ timeout: 0 });
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("Bun.SQL closed-connection error matches isClosedConnectionError", async () => {
|
|
128
|
+
const pool = new Bun.SQL({ url: DATABASE_URL, max: 1 });
|
|
129
|
+
await pool.unsafe("select 1");
|
|
130
|
+
await pool.close();
|
|
131
|
+
const caught = await pool.unsafe("select 1").catch((e: unknown) => e);
|
|
132
|
+
// @cast-boundary error-details — asserting the real driver error shape
|
|
133
|
+
expect((caught as { code?: string }).code).toBe("ERR_POSTGRES_CONNECTION_CLOSED");
|
|
134
|
+
expect(isClosedConnectionError(caught)).toBe(true);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe.each(drivers)("extractPgError — %s unique-violation SQLSTATE", (kind) => {
|
|
138
|
+
test("isUniqueViolation and constraintOf resolve the real driver error", async () => {
|
|
139
|
+
const applicationName = randomAppName();
|
|
140
|
+
const pool = makePool(kind, 1, applicationName);
|
|
141
|
+
const tableName = `cc_retry_uniq_${crypto.randomUUID().replace(/-/g, "").slice(0, 12)}`;
|
|
142
|
+
await pool.unsafe(
|
|
143
|
+
`create table "${tableName}" (id uuid not null, constraint "${tableName}_pk" primary key (id))`,
|
|
144
|
+
);
|
|
145
|
+
try {
|
|
146
|
+
const fixedId = crypto.randomUUID();
|
|
147
|
+
await pool.unsafe(`insert into "${tableName}" (id) values ($1)`, [fixedId]);
|
|
148
|
+
const caught = await pool
|
|
149
|
+
.unsafe(`insert into "${tableName}" (id) values ($1)`, [fixedId])
|
|
150
|
+
.catch((e: unknown) => e);
|
|
151
|
+
expect(isUniqueViolation(caught)).toBe(true);
|
|
152
|
+
expect(extractPgError(caught)?.code).toBe("23505");
|
|
153
|
+
expect(constraintOf(caught)).toBeDefined();
|
|
154
|
+
} finally {
|
|
155
|
+
await pool.unsafe(`drop table if exists "${tableName}"`);
|
|
156
|
+
await closePool(kind, pool);
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
});
|