@cosmicdrift/kumiko-framework 0.290.0 → 0.292.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/__tests__/pii-personal-migration-report-codemod.test.ts +160 -0
- package/src/api/__tests__/server-error-logging.test.ts +168 -17
- package/src/api/request-context.ts +3 -0
- package/src/api/request-id-middleware.ts +2 -1
- package/src/api/routes.ts +35 -3
- package/src/changes.json +63 -0
- package/src/crypto/__tests__/event-pii.test.ts +110 -9
- package/src/crypto/__tests__/subject-resolver.test.ts +23 -2
- package/src/crypto/subject-resolver.ts +25 -8
- package/src/db/queries/shadow-swap.ts +35 -0
- package/src/engine/__tests__/boot-validator-pii-retention.test.ts +122 -0
- package/src/engine/__tests__/boot-validator-projection-list.test.ts +93 -0
- package/src/engine/__tests__/boot-validator.test.ts +226 -0
- package/src/engine/__tests__/build-app-schema.test.ts +18 -0
- package/src/engine/__tests__/engine.test.ts +87 -0
- package/src/engine/__tests__/form-money-currency-types.test.ts +90 -0
- package/src/engine/boot-validator/entity-handler.ts +44 -0
- package/src/engine/boot-validator/index.ts +10 -4
- package/src/engine/boot-validator/pii-retention.ts +8 -0
- package/src/engine/boot-validator/projection-list-screens.ts +52 -2
- package/src/engine/boot-validator/screens.ts +50 -0
- package/src/engine/create-app.ts +54 -0
- package/src/engine/extension-names.ts +10 -0
- package/src/engine/feature-config-events-jobs.ts +19 -0
- package/src/engine/index.ts +3 -0
- package/src/engine/screen-helpers.ts +1 -0
- package/src/engine/system-user.ts +3 -5
- package/src/event-store/__tests__/backfill-pii.integration.test.ts +30 -6
- package/src/files/provider-resolver.ts +9 -2
- package/src/i18n/required-surface-keys.ts +1 -0
- package/src/jobs/__tests__/job-last-success.integration.test.ts +135 -0
- package/src/jobs/index.ts +7 -1
- package/src/jobs/job-runner.ts +94 -4
- package/src/logging/utils.ts +14 -1
- package/src/observability/index.ts +1 -0
- package/src/observability/standard-metrics.ts +20 -0
- package/src/pipeline/__tests__/blind-index-rebuild-guard.integration.test.ts +96 -0
- package/src/pipeline/projection-rebuild.ts +7 -0
- package/src/schema-cli.ts +21 -0
- package/src/scripts/codemod/pii-personal-migration.ts +242 -2
- package/src/ui-types/list-row-meta.ts +4 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.292.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -198,8 +198,8 @@
|
|
|
198
198
|
"./package.json": "./package.json"
|
|
199
199
|
},
|
|
200
200
|
"dependencies": {
|
|
201
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
202
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
201
|
+
"@cosmicdrift/kumiko-http": "0.292.0",
|
|
202
|
+
"@cosmicdrift/kumiko-types": "0.292.0",
|
|
203
203
|
"bullmq": "^5.76.7",
|
|
204
204
|
"bun-types": "^1.3.13",
|
|
205
205
|
"hono": "^4.13.1",
|
|
@@ -215,7 +215,7 @@
|
|
|
215
215
|
"zod": "^4.4.3"
|
|
216
216
|
},
|
|
217
217
|
"devDependencies": {
|
|
218
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
218
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.292.0",
|
|
219
219
|
"bun-types": "^1.3.13",
|
|
220
220
|
"pino-pretty": "^13.1.3"
|
|
221
221
|
},
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { reportStanceForSource } from "../scripts/codemod/pii-personal-migration";
|
|
3
|
+
|
|
4
|
+
function wrapField(fieldSrc: string): string {
|
|
5
|
+
return `
|
|
6
|
+
const entity = createEntity({
|
|
7
|
+
table: "entity_table",
|
|
8
|
+
fields: {
|
|
9
|
+
${fieldSrc}
|
|
10
|
+
},
|
|
11
|
+
});
|
|
12
|
+
`;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
describe("reportStanceForSource", () => {
|
|
16
|
+
it("classifies an exact PII_DIRECT_NAME_HINTS match", () => {
|
|
17
|
+
const [site] = reportStanceForSource(wrapField("email: createTextField({}),"), "t.ts");
|
|
18
|
+
expect(site?.stance).toBe("direct");
|
|
19
|
+
expect(site?.hint).toBe("email");
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it("classifies an exact PII_USER_OWNED_NAME_HINTS match", () => {
|
|
23
|
+
const [site] = reportStanceForSource(wrapField("note: createTextField({}),"), "t.ts");
|
|
24
|
+
expect(site?.stance).toBe("user-owned");
|
|
25
|
+
expect(site?.hint).toBe("note");
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it("classifies an exact PII_USER_REFERENCE_NAME_HINTS match", () => {
|
|
29
|
+
const [site] = reportStanceForSource(wrapField("authorId: createTextField({}),"), "t.ts");
|
|
30
|
+
expect(site?.stance).toBe("user-reference");
|
|
31
|
+
expect(site?.hint).toBe("authorid");
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("classifies a near-miss when a hint occurs at a segment boundary", () => {
|
|
35
|
+
const [site] = reportStanceForSource(
|
|
36
|
+
wrapField("advisorDisplayName: createTextField({}),"),
|
|
37
|
+
"t.ts",
|
|
38
|
+
);
|
|
39
|
+
expect(site?.stance).toBe("near-miss");
|
|
40
|
+
expect(site?.hint).toBe("displayname");
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("picks the longest hint on multiple boundary matches", () => {
|
|
44
|
+
const [site] = reportStanceForSource(
|
|
45
|
+
wrapField("customerEmailAddress: createTextField({}),"),
|
|
46
|
+
"t.ts",
|
|
47
|
+
);
|
|
48
|
+
expect(site?.stance).toBe("near-miss");
|
|
49
|
+
expect(site?.hint).toBe("address");
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// No hint entry is a substring of "performedbyuserid" itself, but its
|
|
53
|
+
// segment-aligned suffix "userid" (>= 5 chars) is a substring of the
|
|
54
|
+
// full-name hints "createdbyuserid"/"updatedbyuserid"/"assigneeuserid" —
|
|
55
|
+
// the shortest of the three, "assigneeuserid", is reported.
|
|
56
|
+
it.each([
|
|
57
|
+
["performedByUserId", "assigneeuserid"],
|
|
58
|
+
["portalUserId", "assigneeuserid"],
|
|
59
|
+
["ownerUserId", "assigneeuserid"],
|
|
60
|
+
])("classifies %s as near-miss via its segment-aligned suffix", (field, expectedHint) => {
|
|
61
|
+
const [site] = reportStanceForSource(wrapField(`${field}: createTextField({}),`), "t.ts");
|
|
62
|
+
expect(site?.stance).toBe("near-miss");
|
|
63
|
+
expect(site?.hint).toBe(expectedHint);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it.each([
|
|
67
|
+
"contextId",
|
|
68
|
+
"stepKey",
|
|
69
|
+
"runId",
|
|
70
|
+
"scheduleId",
|
|
71
|
+
"costCategoryId",
|
|
72
|
+
"toolCallId",
|
|
73
|
+
"conversationId",
|
|
74
|
+
"handlerQn",
|
|
75
|
+
])("classifies %s as unclassified — no hint containment or suffix match", (field) => {
|
|
76
|
+
const [site] = reportStanceForSource(wrapField(`${field}: createTextField({}),`), "t.ts");
|
|
77
|
+
expect(site?.stance).toBe("unclassified");
|
|
78
|
+
expect(site?.hint).toBeUndefined();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("does not report a call already annotated with a personal stance", () => {
|
|
82
|
+
const sites = reportStanceForSource(
|
|
83
|
+
wrapField('email: createTextField({ personal: false, reason: "x" }),'),
|
|
84
|
+
"t.ts",
|
|
85
|
+
);
|
|
86
|
+
expect(sites).toHaveLength(0);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it("reports a call whose personal value is undefined", () => {
|
|
90
|
+
const sites = reportStanceForSource(
|
|
91
|
+
wrapField("email: createTextField({ personal: undefined }),"),
|
|
92
|
+
"t.ts",
|
|
93
|
+
);
|
|
94
|
+
expect(sites).toHaveLength(1);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("does not report a call whose options literal contains a spread", () => {
|
|
98
|
+
const sites = reportStanceForSource(wrapField("email: createTextField({ ...base }),"), "t.ts");
|
|
99
|
+
expect(sites).toHaveLength(0);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it("does not report a call with a non-literal options argument", () => {
|
|
103
|
+
const sites = reportStanceForSource(wrapField("email: createTextField(options),"), "t.ts");
|
|
104
|
+
expect(sites).toHaveLength(0);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it("reports a bare createTextField() call with no arguments", () => {
|
|
108
|
+
const sites = reportStanceForSource(wrapField("stepKey: createTextField(),"), "t.ts");
|
|
109
|
+
expect(sites).toHaveLength(1);
|
|
110
|
+
expect(sites[0]?.stance).toBe("unclassified");
|
|
111
|
+
expect(sites[0]?.hint).toBeUndefined();
|
|
112
|
+
expect(sites[0]?.field).toBe("stepKey");
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("falls back to a synthetic field name and stays unclassified when there is no enclosing PropertyAssignment", () => {
|
|
116
|
+
const source = `
|
|
117
|
+
const x = [createTextField({})];
|
|
118
|
+
`;
|
|
119
|
+
const sites = reportStanceForSource(source, "t.ts");
|
|
120
|
+
expect(sites).toHaveLength(1);
|
|
121
|
+
expect(sites[0]?.field).toBe("createTextField(...)");
|
|
122
|
+
expect(sites[0]?.stance).toBe("unclassified");
|
|
123
|
+
expect(sites[0]?.hint).toBeUndefined();
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("resolves the entity from createEntity's table property", () => {
|
|
127
|
+
const [site] = reportStanceForSource(wrapField("email: createTextField({}),"), "t.ts");
|
|
128
|
+
expect(site?.entity).toBe("entity_table");
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it("falls back to the enclosing variable name when createEntity has no table property", () => {
|
|
132
|
+
const source = `
|
|
133
|
+
const fields = createEntity({
|
|
134
|
+
fields: {
|
|
135
|
+
email: createTextField({}),
|
|
136
|
+
},
|
|
137
|
+
});
|
|
138
|
+
`;
|
|
139
|
+
const [site] = reportStanceForSource(source, "t.ts");
|
|
140
|
+
expect(site?.entity).toBe("fields");
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("resolves entity to null when the call is not inside a createEntity call", () => {
|
|
144
|
+
const source = `
|
|
145
|
+
const standalone = { email: createTextField({}) };
|
|
146
|
+
`;
|
|
147
|
+
const [site] = reportStanceForSource(source, "t.ts");
|
|
148
|
+
expect(site?.entity).toBeNull();
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it("captures createLongTextField the same way as createTextField", () => {
|
|
152
|
+
const [site] = reportStanceForSource(
|
|
153
|
+
wrapField("description: createLongTextField({}),"),
|
|
154
|
+
"t.ts",
|
|
155
|
+
);
|
|
156
|
+
expect(site?.callee).toBe("createLongTextField");
|
|
157
|
+
expect(site?.stance).toBe("user-owned");
|
|
158
|
+
expect(site?.hint).toBe("description");
|
|
159
|
+
});
|
|
160
|
+
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { describe, expect, spyOn, test } from "bun:test";
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { createRegistry, defineFeature } from "../../engine";
|
|
4
|
+
import { RateLimitError, UnprocessableError } from "../../errors";
|
|
4
5
|
import { TestUsers } from "../../stack";
|
|
5
6
|
import { ensureTemporalPolyfill } from "../../time";
|
|
6
7
|
import { buildServer } from "../server";
|
|
@@ -13,6 +14,10 @@ await ensureTemporalPolyfill();
|
|
|
13
14
|
|
|
14
15
|
const JWT_SECRET = "test-secret-at-least-32-chars-long!!";
|
|
15
16
|
|
|
17
|
+
const openToAll = {
|
|
18
|
+
access: { openToAll: { reason: "test handler callable by any signed-in test user" } },
|
|
19
|
+
} as const;
|
|
20
|
+
|
|
16
21
|
const boomFeature = defineFeature("boom", (r) => {
|
|
17
22
|
r.queryHandler(
|
|
18
23
|
"explode",
|
|
@@ -20,7 +25,38 @@ const boomFeature = defineFeature("boom", (r) => {
|
|
|
20
25
|
async () => {
|
|
21
26
|
throw new Error("disk on fire");
|
|
22
27
|
},
|
|
23
|
-
|
|
28
|
+
openToAll,
|
|
29
|
+
);
|
|
30
|
+
r.queryHandler(
|
|
31
|
+
"decode",
|
|
32
|
+
z.object({}),
|
|
33
|
+
async () => {
|
|
34
|
+
throw new UnprocessableError("vin_not_decodable", {
|
|
35
|
+
details: { vin: "WDB0000000SECRET" },
|
|
36
|
+
});
|
|
37
|
+
},
|
|
38
|
+
openToAll,
|
|
39
|
+
);
|
|
40
|
+
r.queryHandler(
|
|
41
|
+
"throttled",
|
|
42
|
+
z.object({}),
|
|
43
|
+
async () => {
|
|
44
|
+
throw new RateLimitError({
|
|
45
|
+
bucket: "ip:203.0.113.7",
|
|
46
|
+
limit: 1,
|
|
47
|
+
windowSeconds: 60,
|
|
48
|
+
remaining: 0,
|
|
49
|
+
retryAfterSeconds: 60,
|
|
50
|
+
resetAt: "2026-01-01T00:00:00.000Z",
|
|
51
|
+
});
|
|
52
|
+
},
|
|
53
|
+
openToAll,
|
|
54
|
+
);
|
|
55
|
+
r.queryHandler(
|
|
56
|
+
"login",
|
|
57
|
+
z.object({ email: z.email(), password: z.string().min(8) }),
|
|
58
|
+
async () => ({ ok: true }),
|
|
59
|
+
openToAll,
|
|
24
60
|
);
|
|
25
61
|
});
|
|
26
62
|
|
|
@@ -45,6 +81,43 @@ function apiFaultLog(calls: unknown[][]): string | undefined {
|
|
|
45
81
|
return hit ? JSON.stringify(hit) : undefined;
|
|
46
82
|
}
|
|
47
83
|
|
|
84
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
85
|
+
return typeof value === "object" && value !== null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// The `[api] handler rejected` warn line logServerFault emits for 4xx.
|
|
89
|
+
function apiRejectionLog(calls: unknown[][]): Record<string, unknown> | undefined {
|
|
90
|
+
const hit = calls.find(
|
|
91
|
+
(args) => typeof args[0] === "string" && args[0].includes("[api] handler rejected"),
|
|
92
|
+
);
|
|
93
|
+
return isRecord(hit?.[1]) ? hit[1] : undefined;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function queryWithCapturedWarnings(
|
|
97
|
+
type: string,
|
|
98
|
+
payload: unknown,
|
|
99
|
+
): Promise<{ status: number; warnings: unknown[][]; errors: unknown[][] }> {
|
|
100
|
+
const warnings: unknown[][] = [];
|
|
101
|
+
const errors: unknown[][] = [];
|
|
102
|
+
const warnSpy = spyOn(console, "warn").mockImplementation((...args) => {
|
|
103
|
+
warnings.push(args);
|
|
104
|
+
});
|
|
105
|
+
const errorSpy = spyOn(console, "error").mockImplementation((...args) => {
|
|
106
|
+
errors.push(args);
|
|
107
|
+
});
|
|
108
|
+
try {
|
|
109
|
+
const res = await app.request("/api/query", {
|
|
110
|
+
method: "POST",
|
|
111
|
+
headers: await auth(),
|
|
112
|
+
body: JSON.stringify({ type, payload }),
|
|
113
|
+
});
|
|
114
|
+
return { status: res.status, warnings, errors };
|
|
115
|
+
} finally {
|
|
116
|
+
warnSpy.mockRestore();
|
|
117
|
+
errorSpy.mockRestore();
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
48
121
|
describe("HTTP layer logs unexpected 5xx faults", () => {
|
|
49
122
|
test("a throwing query 500s AND the cause stack reaches the log", async () => {
|
|
50
123
|
const calls: unknown[][] = [];
|
|
@@ -67,21 +140,99 @@ describe("HTTP layer logs unexpected 5xx faults", () => {
|
|
|
67
140
|
}
|
|
68
141
|
});
|
|
69
142
|
|
|
70
|
-
test("
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
143
|
+
test("a 404 stays off the error level (it is a client outcome, not a server fault)", async () => {
|
|
144
|
+
const { status, errors } = await queryWithCapturedWarnings("nope:query:nothing", {});
|
|
145
|
+
expect(status).toBe(404);
|
|
146
|
+
expect(apiFaultLog(errors)).toBeUndefined();
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
describe("HTTP layer logs 4xx client faults on warn (#3077)", () => {
|
|
151
|
+
let previousLogLevel: string | undefined;
|
|
152
|
+
|
|
153
|
+
beforeEach(() => {
|
|
154
|
+
previousLogLevel = process.env["LOG_LEVEL"];
|
|
155
|
+
process.env["LOG_LEVEL"] = "info";
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
afterEach(() => {
|
|
159
|
+
if (previousLogLevel === undefined) delete process.env["LOG_LEVEL"];
|
|
160
|
+
else process.env["LOG_LEVEL"] = previousLogLevel;
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test("a 422 leaves a log line with status, code and duration", async () => {
|
|
164
|
+
const { status, warnings } = await queryWithCapturedWarnings("boom:query:decode", {});
|
|
165
|
+
expect(status).toBe(422);
|
|
166
|
+
const logged = apiRejectionLog(warnings);
|
|
167
|
+
expect(logged).toBeDefined();
|
|
168
|
+
expect(logged?.["status"]).toBe(422);
|
|
169
|
+
expect(logged?.["code"]).toBe("unprocessable");
|
|
170
|
+
expect(logged?.["type"]).toBe("boom:query:decode");
|
|
171
|
+
expect(typeof logged?.["durationMs"]).toBe("number");
|
|
172
|
+
expect(logged?.["durationMs"]).toBeGreaterThanOrEqual(0);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test("a 429 leaves a log line", async () => {
|
|
176
|
+
const { status, warnings } = await queryWithCapturedWarnings("boom:query:throttled", {});
|
|
177
|
+
expect(status).toBe(429);
|
|
178
|
+
expect(apiRejectionLog(warnings)?.["status"]).toBe(429);
|
|
179
|
+
expect(apiRejectionLog(warnings)?.["code"]).toBe("rate_limited");
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
test("a 400 validation failure leaves a log line", async () => {
|
|
183
|
+
const { status, warnings } = await queryWithCapturedWarnings("boom:query:login", {
|
|
184
|
+
email: "nope",
|
|
185
|
+
password: "short",
|
|
74
186
|
});
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
187
|
+
expect(status).toBe(400);
|
|
188
|
+
expect(apiRejectionLog(warnings)?.["status"]).toBe(400);
|
|
189
|
+
expect(apiRejectionLog(warnings)?.["code"]).toBe("validation_error");
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
test("a 404 leaves a log line and truncates the client-supplied type", async () => {
|
|
193
|
+
const longType = `ghost:query:${"x".repeat(500)}`;
|
|
194
|
+
const { status, warnings } = await queryWithCapturedWarnings(longType, {});
|
|
195
|
+
expect(status).toBe(404);
|
|
196
|
+
const loggedType = apiRejectionLog(warnings)?.["type"];
|
|
197
|
+
expect(typeof loggedType).toBe("string");
|
|
198
|
+
expect(String(loggedType).length).toBeLessThanOrEqual(120);
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
test("the 4xx line carries no submitted values, no message, no details, no stack", async () => {
|
|
202
|
+
const { warnings } = await queryWithCapturedWarnings("boom:query:login", {
|
|
203
|
+
email: "victim-at-example.com",
|
|
204
|
+
password: "hunter2-super-secret",
|
|
205
|
+
apiKey: "sk-live-0000000000",
|
|
206
|
+
});
|
|
207
|
+
const logged = apiRejectionLog(warnings);
|
|
208
|
+
expect(logged).toBeDefined();
|
|
209
|
+
expect(Object.keys(logged ?? {}).sort()).toEqual([
|
|
210
|
+
"code",
|
|
211
|
+
"durationMs",
|
|
212
|
+
"requestId",
|
|
213
|
+
"status",
|
|
214
|
+
"type",
|
|
215
|
+
]);
|
|
216
|
+
const serialized = JSON.stringify(logged);
|
|
217
|
+
expect(serialized).not.toContain("victim-at-example.com");
|
|
218
|
+
expect(serialized).not.toContain("hunter2-super-secret");
|
|
219
|
+
expect(serialized).not.toContain("sk-live-0000000000");
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test("a 422 does NOT reach the error level (5xx contract unchanged)", async () => {
|
|
223
|
+
const { errors } = await queryWithCapturedWarnings("boom:query:decode", {});
|
|
224
|
+
expect(apiFaultLog(errors)).toBeUndefined();
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
test("LOG_LEVEL=error silences the 4xx lines but keeps 5xx", async () => {
|
|
228
|
+
process.env["LOG_LEVEL"] = "error";
|
|
229
|
+
|
|
230
|
+
const rejected = await queryWithCapturedWarnings("boom:query:decode", {});
|
|
231
|
+
expect(rejected.status).toBe(422);
|
|
232
|
+
expect(apiRejectionLog(rejected.warnings)).toBeUndefined();
|
|
233
|
+
|
|
234
|
+
const exploded = await queryWithCapturedWarnings("boom:query:explode", {});
|
|
235
|
+
expect(exploded.status).toBe(500);
|
|
236
|
+
expect(apiFaultLog(exploded.errors)).toBeDefined();
|
|
86
237
|
});
|
|
87
238
|
});
|
|
@@ -44,6 +44,9 @@ export type RequestContextData = {
|
|
|
44
44
|
// event written under this scope.
|
|
45
45
|
readonly feature?: string;
|
|
46
46
|
readonly handler?: string;
|
|
47
|
+
// performance.now() at request entry, so a failing request can report how
|
|
48
|
+
// long it ran. Monotonic — a wall-clock step cannot make it negative.
|
|
49
|
+
readonly startedAt?: number;
|
|
47
50
|
};
|
|
48
51
|
|
|
49
52
|
const storage = new AsyncLocalStorage<RequestContextData>();
|
|
@@ -62,6 +62,7 @@ export function buildRequestContextDataFromRequest(req: Request): RequestContext
|
|
|
62
62
|
return {
|
|
63
63
|
requestId,
|
|
64
64
|
correlationId,
|
|
65
|
+
startedAt: performance.now(),
|
|
65
66
|
...(signal ? { signal } : {}),
|
|
66
67
|
...(ip && ip.length > 0 ? { ip } : {}),
|
|
67
68
|
...(userAgent !== undefined ? { userAgent } : {}),
|
|
@@ -81,7 +82,7 @@ export function buildRequestContextData(c: Context): RequestContextData {
|
|
|
81
82
|
// instead of letting req.headers.get() throw on every request.
|
|
82
83
|
if (!c.req.raw) {
|
|
83
84
|
const requestId = requestContext.generateId();
|
|
84
|
-
return { requestId, correlationId: requestId };
|
|
85
|
+
return { requestId, correlationId: requestId, startedAt: performance.now() };
|
|
85
86
|
}
|
|
86
87
|
return buildRequestContextDataFromRequest(c.req.raw);
|
|
87
88
|
}
|
package/src/api/routes.ts
CHANGED
|
@@ -333,15 +333,47 @@ function assertPatAllowed(user: SessionUser, type: string): void {
|
|
|
333
333
|
}
|
|
334
334
|
}
|
|
335
335
|
|
|
336
|
+
// Log levels that silence the 4xx tier. Checked here and not in the logger
|
|
337
|
+
// because this module uses the console fallback, which pino's level never
|
|
338
|
+
// reaches — LOG_LEVEL is the volume knob for client faults.
|
|
339
|
+
const FAULT_LOG_SILENCED_LEVELS = new Set(["error", "fatal", "silent"]);
|
|
340
|
+
|
|
341
|
+
// `type` is client-supplied on an unknown-handler 404, so cap it before it
|
|
342
|
+
// reaches the log — an unbounded field would let a caller flood the sink.
|
|
343
|
+
const MAX_LOGGED_TYPE_LENGTH = 120;
|
|
344
|
+
|
|
345
|
+
function clientFaultLoggingEnabled(): boolean {
|
|
346
|
+
return !FAULT_LOG_SILENCED_LEVELS.has(process.env["LOG_LEVEL"] ?? "");
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// A failing request must leave a trace even when it ends in 4xx — a paid
|
|
350
|
+
// external call that 422s was invisible before (offlot#117). Status, error
|
|
351
|
+
// code and duration only: message/details/stack can carry submitted values.
|
|
352
|
+
function logClientFault(err: KumikoError, requestId: string | undefined, type?: string): void {
|
|
353
|
+
if (!clientFaultLoggingEnabled()) {
|
|
354
|
+
// skip: LOG_LEVEL silences the 4xx tier — the deployment opted out of client-fault volume
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
const startedAt = requestContext.get()?.startedAt;
|
|
358
|
+
createFallbackLogger("api").warn("handler rejected", {
|
|
359
|
+
requestId,
|
|
360
|
+
type: type?.slice(0, MAX_LOGGED_TYPE_LENGTH),
|
|
361
|
+
status: err.httpStatus,
|
|
362
|
+
code: err.code,
|
|
363
|
+
...(startedAt === undefined ? {} : { durationMs: Math.round(performance.now() - startedAt) }),
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
|
|
336
367
|
// Unexpected server faults (5xx) carry their diagnostic stack only on the
|
|
337
368
|
// in-process error — serializeError strips cause/details from the wire body.
|
|
338
369
|
// Without this a wrapped throw (InternalError{cause}) returns a 500 with zero
|
|
339
|
-
// log lines, leaving ops nothing to debug (the bug this guards). 4xx
|
|
340
|
-
//
|
|
370
|
+
// log lines, leaving ops nothing to debug (the bug this guards). 4xx take the
|
|
371
|
+
// redacted `warn` line above instead. `type` is the only handler
|
|
341
372
|
// discriminator — every request hits the same /api/{query,command} path.
|
|
342
373
|
function logServerFault(err: KumikoError, requestId: string | undefined, type?: string): void {
|
|
343
374
|
if (err.httpStatus < 500) {
|
|
344
|
-
|
|
375
|
+
logClientFault(err, requestId, type);
|
|
376
|
+
// skip: 4xx already logged on warn by logClientFault — the error level stays 5xx-only
|
|
345
377
|
return;
|
|
346
378
|
}
|
|
347
379
|
const cause = err.cause;
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,67 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.292.0",
|
|
4
|
+
"type": "fix",
|
|
5
|
+
"title": "Validate changeset folding in PR CI",
|
|
6
|
+
"detail": "The release-time `changes fold` now also runs as a dry run on every PR, so a changeset with an unresolvable feature fails its own PR instead of the next release."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.292.0",
|
|
10
|
+
"type": "improvement",
|
|
11
|
+
"title": "A projectionList can declare a time-range filter",
|
|
12
|
+
"detail": "A list bound to a query that already accepts time bounds had no way to expose them: `ListFacetSpec` knew `select`, `boolean` and `reference`, so every list with a timestamp — which, through `createdAt`, is practically every list — could be searched but not narrowed to \"the week the incident happened\". The audit log shipped a `description` promising date filters that no control backed.\n`{ type: \"dateRange\", field, label, params: { from, to } }` closes that. The renderer maps it to two native `<input type=\"date\">` next to the facet dropdowns (no date dependency; the browser supplies the calendar, the locale and the keyboard handling) and sends the picked bounds as the two query params the facet names — explicit rather than a `from`/`to` convention, since a query is free to call them anything, and checked against the handler's Zod schema at boot. Filtering stays server-side; either bound alone is a valid open interval; changing the range resets the page like every other facet; an inverted range is clamped in the UI instead of reaching the handler's `from <= to` refine.\nA calendar date covers a whole day in the viewer's time zone: \"to the 14th\" includes everything through the last instant of the 14th, computed across DST boundaries rather than by adding 24 hours. `audit:screen:audit-log` now declares the facet on `createdAt`, so its description holds."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"version": "0.291.0",
|
|
16
|
+
"type": "breaking",
|
|
17
|
+
"title": "projection-rebuild aborts instead of silently NULLing populated blind-index columns (fw#3091)",
|
|
18
|
+
"detail": "`kumiko schema apply` rebuilds a projection through a fresh shadow table replay, and the shadow always recomputes every `<field>_bidx` column with whatever blind-index key is configured in the running process. A projection rebuilt in a process without `KUMIKO_BLIND_INDEX_KEY` set — most commonly the `migrate-db` init container that runs `kumiko schema apply` on deploy — silently swapped the live table for one where every bidx column had gone NULL, breaking equality lookups (login, password reset) with no error anywhere. `rebuildProjection` now checks, right before the swap, whether the live table already has populated bidx columns while no key is configured in this process; if so it throws and leaves the live table untouched instead of completing the swap.",
|
|
19
|
+
"migration": "Plaintext installations and the fw#1610 case (KMS configured, no blind-index\nkey) are unaffected — their bidx columns are NULL already, so there is\nnothing for the rebuild to lose. This only blocks a rebuild that would\notherwise destroy already-populated bidx columns: any process running\n`kumiko schema apply` (or another projection rebuild) against a table with\nlive blind-index data must have `KUMIKO_BLIND_INDEX_KEY` set. Wire that env\nvar into the `migrate-db` init container (or wherever schema apply runs in\ndeploy) alongside the app's own KUMIKO_BLIND_INDEX_KEY, or the rebuild aborts\ninstead of quietly breaking equality lookups."
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"version": "0.291.0",
|
|
23
|
+
"type": "breaking",
|
|
24
|
+
"title": "Declared event PII fails closed without a subject KMS (fw#2776)",
|
|
25
|
+
"detail": "`defineEvent` has required an explicit PII stance since fw#2558, but a declared stance still did not guarantee ciphertext in `kumiko_events`. Two paths leaked silently and now fail closed.\nBoot: `assertPiiBootInvariants` only looked at entity annotations, so an app whose PII lives exclusively in catalogued events booted without a `kms` adapter and wrote plaintext. It now collects events with a non-`\"none\"` stance alongside the PII entities — prod aborts, dev warns, `allowPlaintextPii: \"<reason>\"` acknowledges, same as for entities.\nAppend: `{ personal: { of: \"<ownerField>\" } }` skipped encryption whenever the owner field carried no id, so the same event type was ciphertext for user-triggered writes and plaintext for system-triggered ones with no signal. The stance now carries `whenAbsent`: `\"tenant\"` encrypts under the envelope tenant key, `\"plaintext\"` is an explicit acknowledgement that the value cannot be crypto-shredded. Registration rejects a nullable owner field without one, and an owner that is empty at append time with no declared fallback aborts the write instead of storing the value in the clear.\n`delivery:event:attempt` declares `whenAbsent: \"tenant\"` — a send whose `recipientId` is null now stores the recipient address under the tenant key instead of in plaintext.",
|
|
26
|
+
"migration": "Three things can newly fail. (1) Boot aborts with `BOOT ABORTED — ... events\n[...]` when a mounted feature declares a non-`\"none\"` `piiFields` stance and\n`runProdApp`/`runWorkerApp` gets no `kms`. Pass\n`kms: createPgKmsAdapter({ databaseUrl, platformKek })`, or acknowledge the\nplaintext with `allowPlaintextPii: \"<reason>\"` until the KMS is provisioned;\n`runDevApp` only warns. (2) Registration aborts when a\n`{ personal: { of: \"<ownerField>\" } }` stance names an owner field the payload\nschema allows to be null or undefined. Add `whenAbsent: \"tenant\"` to encrypt\nthose writes under the envelope tenant key, or `whenAbsent: \"plaintext\"` to\ndeclare that the value ships unencrypted and is not crypto-shreddable. The\ndeprecated `{ subjectField: \"<ownerField>\" }` form cannot express `whenAbsent`\n— move it to the canonical `{ personal: { of: ... } }` form. (3) `append()`\nthrows `SubjectResolutionError` when the owner field is empty at write time\nand the event declared no `whenAbsent`. Registration catches this for\nZodObject payload schemas; a non-object schema surfaces it here. An owner\nvalue that is not a non-empty string — a numeric id, an empty string — counts\nas absent, so it takes the same path and needs the same declaration.\nSeparately, `delivery:event:attempt` rows written with a null `recipientId`\nused to hold a plaintext recipient address in `kumiko_events` and in\n`store_delivery_attempts`. New rows are tenant-subject ciphertext.\n`delivery:query:log` decrypts either form, so the admin log view is unchanged;\ntooling that reads `store_delivery_attempts.recipient_address` directly must\ngo through `decryptStoredPii`. Existing plaintext rows stay readable and are\nre-encrypted by `backfillEventPiiEncryption`."
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"version": "0.291.0",
|
|
30
|
+
"type": "improvement",
|
|
31
|
+
"title": "Boot warns for text/longText fields without a personal stance (fw#2918).",
|
|
32
|
+
"detail": "`validatePiiAndRetention` so far only warned when an unannotated field name hit one of the PII name heuristics. It now warns for every `text`/`longText` field that declares no stance at all — naming feature, entity, field and all valid stances verbatim — so consumers can work off their own baseline before fw#2810 turns the missing stance into a compile error and a throw. Annotated fields (including `personal: false` with a reason and `personal: \"ref\"`) stay silent, other field types are untouched.\nA clean boot does not mean \"ready for fw#2810\": `validatePiiAndRetention` only walks `feature.entities[*].fields`, so embedded sub-schemas and call sites that never boot (fixtures, helper modules) produce no warning while still breaking later. The completeness instrument is and stays `guard-text-field-stance` — ready means a guard count of 0. The reverse does not hold either: the guard counts `createTextField()` call sites, the boot validator walks resolved field defs, so a raw field-def object literal warns at boot without ever showing up in the guard's baseline."
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"version": "0.291.0",
|
|
36
|
+
"type": "fix",
|
|
37
|
+
"title": "API handler rejections (4xx) now leave a log line instead of being silently dropped",
|
|
38
|
+
"detail": "`logServerFault` returned early for every `httpStatus < 500`, so a failing request (validation, unprocessable, rate-limited) left no log trace at all — a paid external call that 422'd was invisible end to end (offlot#117). 4xx now log on `warn` via the same fallback logger 5xx already used, with status, error code and duration only — no message, details, stack or cause, so submitted values never reach the log line. 5xx behavior on the `error` level is unchanged.\nConsumers that set `LOG_LEVEL=error`, `fatal` or `silent` suppress the new 4xx lines; anything else (including the default) now logs them. Expect more log volume on routes with frequent client-side validation failures."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"version": "0.291.0",
|
|
42
|
+
"type": "improvement",
|
|
43
|
+
"title": "pii-personal-migration codemod gets a --report-stance mode (fw#2919)",
|
|
44
|
+
"detail": "`bun scripts/codemod/pii-personal-migration.ts <dir> --report-stance` scans every `createTextField`/`createLongTextField` call without a `personal` stance and classifies its field name against the `entity-handler.ts` PII name hints (direct/user-owned/user-reference/near-miss/unclassified) — no files are written."
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"version": "0.291.0",
|
|
48
|
+
"type": "improvement",
|
|
49
|
+
"title": "Generic job-liveness metric kumiko_job_last_success_timestamp_seconds{job} (fw#3052).",
|
|
50
|
+
"detail": "The job-runner now stamps a standard gauge with the Unix timestamp of the last successful run of every job registered via `r.job`, so any consumer gets a real dead-man for all its crons with `time() - kumiko_job_last_success_timestamp_seconds{job=\"…\"} > <interval + buffer>`. A failed run leaves the value untouched. The k8s CronJob alerts never covered these jobs — they run in-process in a long-lived pod and create no CronJob object. Note that the series is absent until the first success after a restart; `docs/reference/job-liveness-metric.md` explains the `for:` that implies for the alert side."
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"version": "0.291.0",
|
|
54
|
+
"type": "breaking",
|
|
55
|
+
"title": "money fields on actionForm/secretMint screens declare their currency source (fw#2839)",
|
|
56
|
+
"detail": "`actionForm` and `secretMint` have no entity, so their money fields never received `entity.defaultCurrency`: an untouched one seeded a bare `0` that the handler's zod schema then rejected on submit. fw#2763 closed the prefill half of this; the default half stayed open. `MoneyCurrencySource` gains `{ kind: \"literal\", code }` next to the existing `{ kind: \"tenant\" }`, and the field maps of `actionForm`, `secretMint` and its `confirm` step are narrowed so a money field there requires `currency` — enforced by the compiler at bump time and by the boot validator for untyped callers. A literal code is checked against the app's `currencies` list, the same rule `entity.defaultCurrency` already follows. Entity fields, embedded-list money cells and `configEdit`'s plain-number contract are unchanged.",
|
|
57
|
+
"migration": "Only affects entity-less form screens — `actionForm`, `secretMint` and a\nsecretMint's `confirm` step — that hold a `money` field. Entity money fields\nare unchanged (`entity.defaultCurrency` is already boot-enforced for them),\nas are embedded-list money cells (currency at the head, fw#2764) and\n`configEdit`, which keeps its plain-number contract.\nAdd a `currency` to each money field in such a screen's `fields` map:\n`currency: { kind: \"literal\", code: \"EUR\" }` for one fixed currency, or\n`currency: { kind: \"tenant\" }` for the tenant's own currency.\nA `literal` code must be in the app's `currencies` list (`createApp({ currencies })`,\nwhich already includes the defaults). A `tenant`-declared field resolves through\nthe tenant-settings bundle and holds the form until the value has landed, so that\nbundle has to be mounted. Missing declarations fail at compile time; an untyped\ncaller fails at boot with the screen and field name in the message."
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"version": "0.291.0",
|
|
61
|
+
"type": "improvement",
|
|
62
|
+
"title": "Reference fields can source their picker from a query handler",
|
|
63
|
+
"detail": "`labelField` names one column of the referenced entity, so an entity whose identity is composed from joined rows — a lease identified by its tenant and unit, not by any column on the lease row — has no right answer, only a least-wrong one, and its picker lists raw dates or UUIDs. `ReferenceFieldDef.optionsQuery` (also on a reference sub-field of an embedded field) names a query handler that returns `{ rows: { id, label }[] }` and receives `{ limit, search? }` like the default list handler, so the app composes the label itself. The picker, the read-only display of a reference value and an embedded-list reference cell all read it; the QN is pinned at boot against the registered handlers, the same treatment `DashboardFilterDefinition.optionsQuery` gets.\nIt is additive, not a replacement: `labelField` keeps serving the paths a query handler cannot back, since list cells, `searchable` and `sortable` all resolve to an SQL column on the referenced table. A field without `optionsQuery` behaves exactly as before."
|
|
64
|
+
},
|
|
2
65
|
{
|
|
3
66
|
"version": "0.290.0",
|
|
4
67
|
"type": "improvement",
|