@cosmicdrift/kumiko-framework 0.290.0 → 0.292.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/__tests__/pii-personal-migration-report-codemod.test.ts +160 -0
- package/src/api/__tests__/server-error-logging.test.ts +168 -17
- package/src/api/request-context.ts +3 -0
- package/src/api/request-id-middleware.ts +2 -1
- package/src/api/routes.ts +35 -3
- package/src/changes.json +63 -0
- package/src/crypto/__tests__/event-pii.test.ts +110 -9
- package/src/crypto/__tests__/subject-resolver.test.ts +23 -2
- package/src/crypto/subject-resolver.ts +25 -8
- package/src/db/queries/shadow-swap.ts +35 -0
- package/src/engine/__tests__/boot-validator-pii-retention.test.ts +122 -0
- package/src/engine/__tests__/boot-validator-projection-list.test.ts +93 -0
- package/src/engine/__tests__/boot-validator.test.ts +226 -0
- package/src/engine/__tests__/build-app-schema.test.ts +18 -0
- package/src/engine/__tests__/engine.test.ts +87 -0
- package/src/engine/__tests__/form-money-currency-types.test.ts +90 -0
- package/src/engine/boot-validator/entity-handler.ts +44 -0
- package/src/engine/boot-validator/index.ts +10 -4
- package/src/engine/boot-validator/pii-retention.ts +8 -0
- package/src/engine/boot-validator/projection-list-screens.ts +52 -2
- package/src/engine/boot-validator/screens.ts +50 -0
- package/src/engine/create-app.ts +54 -0
- package/src/engine/extension-names.ts +10 -0
- package/src/engine/feature-config-events-jobs.ts +19 -0
- package/src/engine/index.ts +3 -0
- package/src/engine/screen-helpers.ts +1 -0
- package/src/engine/system-user.ts +3 -5
- package/src/event-store/__tests__/backfill-pii.integration.test.ts +30 -6
- package/src/files/provider-resolver.ts +9 -2
- package/src/i18n/required-surface-keys.ts +1 -0
- package/src/jobs/__tests__/job-last-success.integration.test.ts +135 -0
- package/src/jobs/index.ts +7 -1
- package/src/jobs/job-runner.ts +94 -4
- package/src/logging/utils.ts +14 -1
- package/src/observability/index.ts +1 -0
- package/src/observability/standard-metrics.ts +20 -0
- package/src/pipeline/__tests__/blind-index-rebuild-guard.integration.test.ts +96 -0
- package/src/pipeline/projection-rebuild.ts +7 -0
- package/src/schema-cli.ts +21 -0
- package/src/scripts/codemod/pii-personal-migration.ts +242 -2
- package/src/ui-types/list-row-meta.ts +4 -1
package/src/jobs/job-runner.ts
CHANGED
|
@@ -18,10 +18,12 @@ import {
|
|
|
18
18
|
SYSTEM_TENANT_ID,
|
|
19
19
|
type TenantId,
|
|
20
20
|
} from "../engine/types";
|
|
21
|
+
import { isKumikoError } from "../errors/kumiko-error";
|
|
21
22
|
import { createFileContext } from "../files/file-handle";
|
|
22
23
|
import { createFallbackLogger } from "../logging";
|
|
23
24
|
import type { Logger } from "../logging/types";
|
|
24
25
|
import {
|
|
26
|
+
emitJobLastSuccess,
|
|
25
27
|
emitJobQueueDepth,
|
|
26
28
|
getFallbackTracer,
|
|
27
29
|
type Meter,
|
|
@@ -139,6 +141,55 @@ export type JobMeta = {
|
|
|
139
141
|
priority?: number | undefined;
|
|
140
142
|
};
|
|
141
143
|
|
|
144
|
+
// What a finished run tells the run-logger about the tenant-visible failure
|
|
145
|
+
// record (`JobDefinition.tenantVisibleFailure`). `tenantVisible` is set only
|
|
146
|
+
// when the job opted in; `messageKey` is null on the success path, where the
|
|
147
|
+
// record is cleared rather than written.
|
|
148
|
+
export type JobOutcomeMeta = {
|
|
149
|
+
readonly tenantId: string;
|
|
150
|
+
readonly finalAttempt: boolean;
|
|
151
|
+
readonly tenantVisible?:
|
|
152
|
+
| { readonly subject: string | null; readonly messageKey: string | null }
|
|
153
|
+
| undefined;
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
// Stable identity for the declared payload fields: sorted keys, so two runs
|
|
157
|
+
// with the same subject values produce the same string and the later one
|
|
158
|
+
// replaces the earlier record. Non-primitives would serialize into something
|
|
159
|
+
// no caller can reconstruct for a lookup, so they fail the run loudly.
|
|
160
|
+
function jobSubjectKey(
|
|
161
|
+
jobName: string,
|
|
162
|
+
payload: Record<string, unknown>,
|
|
163
|
+
fields: readonly string[] | undefined,
|
|
164
|
+
): string | null {
|
|
165
|
+
if (fields === undefined || fields.length === 0) return null;
|
|
166
|
+
const entries: [string, string | number | boolean | null][] = [];
|
|
167
|
+
for (const field of [...fields].sort()) {
|
|
168
|
+
const value = payload[field] ?? null;
|
|
169
|
+
if (value !== null && typeof value === "object") {
|
|
170
|
+
throw new Error(
|
|
171
|
+
`Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`,
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
entries.push([field, value as string | number | boolean | null]);
|
|
175
|
+
}
|
|
176
|
+
return JSON.stringify(entries);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Only a translation key ever travels to the tenant: the thrown error's own
|
|
180
|
+
// i18nKey when it carries one, otherwise the key declared at the job. The
|
|
181
|
+
// error message itself can echo provider or payload content and stays on the
|
|
182
|
+
// run row and in the run log.
|
|
183
|
+
function tenantFailureMessageKey(
|
|
184
|
+
err: unknown,
|
|
185
|
+
declaration: JobDefinition["tenantVisibleFailure"],
|
|
186
|
+
): string | null {
|
|
187
|
+
// skip: job did not opt in — nothing tenant-visible to record
|
|
188
|
+
if (!declaration) return null;
|
|
189
|
+
if (isKumikoError(err) && err.i18nKey.length > 0) return err.i18nKey;
|
|
190
|
+
return declaration.messageKey;
|
|
191
|
+
}
|
|
192
|
+
|
|
142
193
|
export type JobRunner = {
|
|
143
194
|
start(): Promise<void>;
|
|
144
195
|
stop(): Promise<void>;
|
|
@@ -176,8 +227,20 @@ export type JobRunnerOptions = {
|
|
|
176
227
|
bootRedisTimeoutMs?: number | undefined;
|
|
177
228
|
getActiveTenantIds?: () => Promise<TenantId[]>;
|
|
178
229
|
onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
|
|
179
|
-
onJobComplete?: (
|
|
180
|
-
|
|
230
|
+
onJobComplete?: (
|
|
231
|
+
jobName: string,
|
|
232
|
+
jobId: string,
|
|
233
|
+
duration: number,
|
|
234
|
+
logs: JobLogEntry[],
|
|
235
|
+
outcome?: JobOutcomeMeta,
|
|
236
|
+
) => void;
|
|
237
|
+
onJobFailed?: (
|
|
238
|
+
jobName: string,
|
|
239
|
+
jobId: string,
|
|
240
|
+
error: string,
|
|
241
|
+
logs: JobLogEntry[],
|
|
242
|
+
outcome?: JobOutcomeMeta,
|
|
243
|
+
) => void;
|
|
181
244
|
};
|
|
182
245
|
|
|
183
246
|
// Serialized trace context lives under this key in the BullMQ job data.
|
|
@@ -527,6 +590,24 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
527
590
|
SYSTEM_TENANT_ID;
|
|
528
591
|
const triggeredById = (rawData["_triggeredById"] as string | undefined) ?? null; // @cast-boundary dynamic-key
|
|
529
592
|
|
|
593
|
+
// Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
|
|
594
|
+
// subject is read from the handler payload, so it is resolved here where
|
|
595
|
+
// the payload is built, not in the callbacks.
|
|
596
|
+
const tenantVisibleDecl = jobDef.tenantVisibleFailure;
|
|
597
|
+
const tenantVisibleSubject = tenantVisibleDecl
|
|
598
|
+
? jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields)
|
|
599
|
+
: null;
|
|
600
|
+
// BullMQ stops retrying once attemptsMade reaches the configured attempts
|
|
601
|
+
// (`retries + 1`), so this is the attempt whose failure is final.
|
|
602
|
+
const finalAttempt = bullJob.attemptsMade + 1 >= (jobDef.retries ?? 0) + 1;
|
|
603
|
+
const outcomeMeta = (messageKey: string | null): JobOutcomeMeta => ({
|
|
604
|
+
tenantId,
|
|
605
|
+
finalAttempt,
|
|
606
|
+
...(tenantVisibleDecl && {
|
|
607
|
+
tenantVisible: { subject: tenantVisibleSubject, messageKey },
|
|
608
|
+
}),
|
|
609
|
+
});
|
|
610
|
+
|
|
530
611
|
// Carry `_triggerName` from rawData when set — handleEvent injects it on
|
|
531
612
|
// multi-trigger dispatch; exposed as jobContext.triggerName so handlers
|
|
532
613
|
// don't dig through the raw payload themselves.
|
|
@@ -676,12 +757,21 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
676
757
|
},
|
|
677
758
|
() => jobDef.handler(payload, jobContext),
|
|
678
759
|
);
|
|
760
|
+
// Stamped before the observer hook: a throwing onJobComplete must not
|
|
761
|
+
// make a run that actually succeeded look dead to the liveness alert.
|
|
762
|
+
if (context.meter) emitJobLastSuccess(context.meter, jobName);
|
|
679
763
|
const duration = Date.now() - startTime;
|
|
680
|
-
await options.onJobComplete?.(jobName, jobId, duration, logs);
|
|
764
|
+
await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
|
|
681
765
|
} catch (err) {
|
|
682
766
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
683
767
|
logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
|
|
684
|
-
await options.onJobFailed?.(
|
|
768
|
+
await options.onJobFailed?.(
|
|
769
|
+
jobName,
|
|
770
|
+
jobId,
|
|
771
|
+
errorMsg,
|
|
772
|
+
logs,
|
|
773
|
+
outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl)),
|
|
774
|
+
);
|
|
685
775
|
throw err;
|
|
686
776
|
}
|
|
687
777
|
};
|
package/src/logging/utils.ts
CHANGED
|
@@ -2,18 +2,27 @@ import type { Logger } from "./types";
|
|
|
2
2
|
|
|
3
3
|
type FallbackLogger = {
|
|
4
4
|
error(msg: string, data?: Record<string, unknown>): void;
|
|
5
|
+
warn(msg: string, data?: Record<string, unknown>): void;
|
|
5
6
|
debug(msg: string, data?: Record<string, unknown>): void;
|
|
6
7
|
};
|
|
7
8
|
|
|
8
9
|
export function createFallbackLogger(
|
|
9
10
|
namespace: string,
|
|
10
|
-
logger?: (Pick<Logger, "error"> & Partial<Pick<Logger, "debug">>) | undefined,
|
|
11
|
+
logger?: (Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>) | undefined,
|
|
11
12
|
): FallbackLogger {
|
|
12
13
|
if (logger) {
|
|
13
14
|
return {
|
|
14
15
|
error(msg, data) {
|
|
15
16
|
logger.error(`[${namespace}] ${msg}`, data);
|
|
16
17
|
},
|
|
18
|
+
warn(msg, data) {
|
|
19
|
+
if (logger.warn) {
|
|
20
|
+
logger.warn(`[${namespace}] ${msg}`, data);
|
|
21
|
+
} else {
|
|
22
|
+
// biome-ignore lint/suspicious/noConsole: ops-visible fallback when the wrapped logger has no warn method
|
|
23
|
+
console.warn(`[${namespace}] ${msg}`, data);
|
|
24
|
+
}
|
|
25
|
+
},
|
|
17
26
|
debug(msg, data) {
|
|
18
27
|
if (logger.debug) {
|
|
19
28
|
logger.debug(`[${namespace}] ${msg}`, data);
|
|
@@ -29,6 +38,10 @@ export function createFallbackLogger(
|
|
|
29
38
|
// biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
|
|
30
39
|
console.error(`[${namespace}] ${msg}`, data);
|
|
31
40
|
},
|
|
41
|
+
warn(msg, data) {
|
|
42
|
+
// biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
|
|
43
|
+
console.warn(`[${namespace}] ${msg}`, data);
|
|
44
|
+
},
|
|
32
45
|
debug(msg, data) {
|
|
33
46
|
// biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
|
|
34
47
|
console.debug(`[${namespace}] ${msg}`, data);
|
|
@@ -123,6 +123,19 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
|
|
|
123
123
|
description: "BullMQ job counts per lane and state.",
|
|
124
124
|
labels: ["lane", "state"],
|
|
125
125
|
},
|
|
126
|
+
// Dead-man for in-process jobs: the k8s CronJob alerts never fire for them
|
|
127
|
+
// (no CronJob object exists), and a gauge the job itself sets freezes on its
|
|
128
|
+
// last value when the job dies, so neither `absent()` nor a threshold
|
|
129
|
+
// catches a silently stopped cron. This one is stamped by the runner, so
|
|
130
|
+
// `time() - kumiko_job_last_success_timestamp_seconds{job="…"} > interval`
|
|
131
|
+
// is a true liveness check. Absent until the first success after a restart —
|
|
132
|
+
// see docs/reference/job-liveness-metric.md for the `for:` that implies.
|
|
133
|
+
{
|
|
134
|
+
name: "kumiko_job_last_success_timestamp_seconds",
|
|
135
|
+
type: "gauge",
|
|
136
|
+
description: "Unix timestamp of the last successful run, per registered job.",
|
|
137
|
+
labels: ["job"],
|
|
138
|
+
},
|
|
126
139
|
] as const;
|
|
127
140
|
|
|
128
141
|
export function registerStandardMetrics(meter: Meter): void {
|
|
@@ -273,3 +286,10 @@ export function emitJobQueueDepth(
|
|
|
273
286
|
meter.gauge("kumiko_job_queue_depth").set(count, { lane, state });
|
|
274
287
|
}
|
|
275
288
|
}
|
|
289
|
+
|
|
290
|
+
// `job` is the registry-declared job name — handleJob rejects an unknown name
|
|
291
|
+
// before it can reach here, so the label set is bounded by the app's r.job
|
|
292
|
+
// registrations and carries no tenant or user input.
|
|
293
|
+
export function emitJobLastSuccess(meter: Meter, job: string): void {
|
|
294
|
+
meter.gauge("kumiko_job_last_success_timestamp_seconds").set(Date.now() / 1000, { job });
|
|
295
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// projection-rebuild aborts instead of silently NULLing an already-populated
|
|
2
|
+
// blind-index column when this process has no blind-index key configured
|
|
3
|
+
// (fw#3091) — see assertNoBlindIndexLoss in db/queries/shadow-swap.ts.
|
|
4
|
+
|
|
5
|
+
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
6
|
+
import { resetBlindIndexKeyForTests } from "@cosmicdrift/kumiko-framework/testing";
|
|
7
|
+
import { computeBlindIndex, configureBlindIndexKey, decodeBlindIndexKey } from "../../crypto";
|
|
8
|
+
import { createEventStoreExecutor } from "../../db/event-store-executor";
|
|
9
|
+
import { asRawClient } from "../../db/query";
|
|
10
|
+
import { buildEntityTable } from "../../db/table-builder";
|
|
11
|
+
import { createTenantDb, type TenantDb } from "../../db/tenant-db";
|
|
12
|
+
import { createEntity, createRegistry, createTextField, defineFeature } from "../../engine";
|
|
13
|
+
import { createEventsTable } from "../../event-store";
|
|
14
|
+
import { createProjectionStateTable, rebuildProjection } from "../../pipeline";
|
|
15
|
+
import { createTestDb, type TestDb, TestUsers, unsafeCreateEntityTable } from "../../stack";
|
|
16
|
+
|
|
17
|
+
const TEST_KEY_B64 = Buffer.alloc(32, 9).toString("base64");
|
|
18
|
+
const TEST_KEY = decodeBlindIndexKey(TEST_KEY_B64);
|
|
19
|
+
|
|
20
|
+
const personEntity = createEntity({
|
|
21
|
+
table: "read_bidx_guard_persons",
|
|
22
|
+
fields: {
|
|
23
|
+
email: createTextField({ required: true, personal: "self", find: "exact" }),
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
const personFeature = defineFeature("bidxguardtest", (r) => {
|
|
27
|
+
r.entity("person", personEntity);
|
|
28
|
+
});
|
|
29
|
+
const personTable = buildEntityTable("person", personEntity);
|
|
30
|
+
const implicitName = "bidxguardtest:projection:person-entity";
|
|
31
|
+
|
|
32
|
+
const admin = TestUsers.admin;
|
|
33
|
+
let testDb: TestDb;
|
|
34
|
+
let tdb: TenantDb;
|
|
35
|
+
const registry = createRegistry([personFeature]);
|
|
36
|
+
const crud = createEventStoreExecutor(personTable, personEntity, { entityName: "person" });
|
|
37
|
+
|
|
38
|
+
beforeAll(async () => {
|
|
39
|
+
testDb = await createTestDb();
|
|
40
|
+
await unsafeCreateEntityTable(testDb.db, personEntity, "person");
|
|
41
|
+
await createEventsTable(testDb.db);
|
|
42
|
+
await createProjectionStateTable(testDb.db);
|
|
43
|
+
tdb = createTenantDb(testDb.db, admin.tenantId);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
afterAll(async () => {
|
|
47
|
+
await testDb.cleanup();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
beforeEach(async () => {
|
|
51
|
+
await asRawClient(testDb.db).unsafe(
|
|
52
|
+
`TRUNCATE kumiko_events, read_bidx_guard_persons, kumiko_projections RESTART IDENTITY CASCADE`,
|
|
53
|
+
);
|
|
54
|
+
configureBlindIndexKey(TEST_KEY_B64);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
afterEach(() => {
|
|
58
|
+
resetBlindIndexKeyForTests();
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
async function rawRow(id: string): Promise<Record<string, unknown>> {
|
|
62
|
+
const rows = await asRawClient(testDb.db).unsafe<Record<string, unknown>>(
|
|
63
|
+
`SELECT * FROM read_bidx_guard_persons WHERE id = $1`,
|
|
64
|
+
[id],
|
|
65
|
+
);
|
|
66
|
+
const row = rows[0];
|
|
67
|
+
if (!row) throw new Error(`no row for ${id}`);
|
|
68
|
+
return row;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
describe("projection-rebuild — blind-index loss guard (fw#3091)", () => {
|
|
72
|
+
test("populated bidx column + no key in this process → rebuild throws, live table untouched", async () => {
|
|
73
|
+
const created = await crud.create({ email: "marc@example.com" }, admin, tdb);
|
|
74
|
+
if (!created.isSuccess) throw new Error("create failed");
|
|
75
|
+
const before = await rawRow(String(created.data.id));
|
|
76
|
+
expect(before["email_bidx"]).toBe(computeBlindIndex(TEST_KEY, "marc@example.com"));
|
|
77
|
+
|
|
78
|
+
resetBlindIndexKeyForTests();
|
|
79
|
+
await expect(rebuildProjection(implicitName, { db: testDb.db, registry })).rejects.toThrow(
|
|
80
|
+
/KUMIKO_BLIND_INDEX_KEY/,
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
const after = await rawRow(String(created.data.id));
|
|
84
|
+
expect(after).toEqual(before);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("populated bidx column + key configured → rebuild succeeds, bidx recomputed", async () => {
|
|
88
|
+
const created = await crud.create({ email: "marc@example.com" }, admin, tdb);
|
|
89
|
+
if (!created.isSuccess) throw new Error("create failed");
|
|
90
|
+
|
|
91
|
+
await rebuildProjection(implicitName, { db: testDb.db, registry });
|
|
92
|
+
|
|
93
|
+
const after = await rawRow(String(created.data.id));
|
|
94
|
+
expect(after["email_bidx"]).toBe(computeBlindIndex(TEST_KEY, "marc@example.com"));
|
|
95
|
+
});
|
|
96
|
+
});
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
import {
|
|
10
10
|
assertLiveColumnsMatchMeta,
|
|
11
11
|
assertLiveTableHasNoRowLevelSecurity,
|
|
12
|
+
assertNoBlindIndexLoss,
|
|
12
13
|
assertNoUnreachableLiveRows,
|
|
13
14
|
buildShadowTable,
|
|
14
15
|
type ColumnDriftResult,
|
|
@@ -358,6 +359,12 @@ export async function rebuildProjection(
|
|
|
358
359
|
if (skipped.length > 0) {
|
|
359
360
|
await recordRebuildDeadLetters(tx, projectionName, skipped);
|
|
360
361
|
}
|
|
362
|
+
// Guard the swap: abort if the process has no blind-index key configured
|
|
363
|
+
// but the live table already has populated bidx columns — the rebuild
|
|
364
|
+
// would otherwise silently null them out (fw#3091). Applies to every
|
|
365
|
+
// projection, not just implicit ones: the shadow always gets a fresh
|
|
366
|
+
// computeBlindIndexValues() pass regardless of projection kind.
|
|
367
|
+
await assertNoBlindIndexLoss(tx, meta.tableName, meta, projectionName);
|
|
361
368
|
// Guard the swap: abort if the live table holds a row no event can
|
|
362
369
|
// reconstruct (#498 ghost — direct-inserted without a .created event),
|
|
363
370
|
// which the swap would silently drop. Implicit projections only.
|
package/src/schema-cli.ts
CHANGED
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
|
|
11
11
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
12
12
|
import { join, resolve as resolvePath } from "node:path";
|
|
13
|
+
import {
|
|
14
|
+
configureBlindIndexKey,
|
|
15
|
+
configurePiiSubjectKms,
|
|
16
|
+
type KmsWiring,
|
|
17
|
+
resolveKmsWiringAsync,
|
|
18
|
+
} from "./crypto";
|
|
13
19
|
import {
|
|
14
20
|
assertValidMigrationName,
|
|
15
21
|
baselineMigrations,
|
|
@@ -316,6 +322,7 @@ export async function runSchemaCli(
|
|
|
316
322
|
return 1;
|
|
317
323
|
}
|
|
318
324
|
const { db, close } = createDbConnection(dbUrl);
|
|
325
|
+
let wiring: KmsWiring | undefined;
|
|
319
326
|
try {
|
|
320
327
|
const result = await runMigrationsFromDir(db, migrationsDir);
|
|
321
328
|
// Framework-Infra-Tabellen (event-store + pipeline-state) — die erfasst
|
|
@@ -346,6 +353,19 @@ export async function runSchemaCli(
|
|
|
346
353
|
// retry it — otherwise a failed rebuild is silently never retried,
|
|
347
354
|
// since the migration itself is already tracked applied (#2464).
|
|
348
355
|
if (options.features) {
|
|
356
|
+
// A rebuild replays applyEntityEvent, which computes bidx columns from
|
|
357
|
+
// configureBlindIndexKey/configurePiiSubjectKms — both boot-injected by
|
|
358
|
+
// run{Prod,Dev}App but never by this standalone CLI (the migrate-db
|
|
359
|
+
// initContainer). Without wiring them here, a rebuild would silently
|
|
360
|
+
// null out bidx columns and, for ciphertext, throw only once it hits
|
|
361
|
+
// the row (fw#3091).
|
|
362
|
+
wiring = await resolveKmsWiringAsync(process.env, {
|
|
363
|
+
logPrefix: "[kumiko schema apply]",
|
|
364
|
+
});
|
|
365
|
+
if ("kms" in wiring) {
|
|
366
|
+
configurePiiSubjectKms(wiring.kms);
|
|
367
|
+
configureBlindIndexKey(wiring.blindIndexKey);
|
|
368
|
+
}
|
|
349
369
|
const thisRunTables = await queueRebuildsFromMarkers(db, {
|
|
350
370
|
migrationsDir,
|
|
351
371
|
appliedIds: result.applied,
|
|
@@ -376,6 +396,7 @@ export async function runSchemaCli(
|
|
|
376
396
|
out.err("");
|
|
377
397
|
return 1;
|
|
378
398
|
} finally {
|
|
399
|
+
await wiring?.close();
|
|
379
400
|
await close();
|
|
380
401
|
}
|
|
381
402
|
}
|
|
@@ -23,9 +23,11 @@
|
|
|
23
23
|
//
|
|
24
24
|
// Usage: bun scripts/codemod/pii-personal-migration.ts <targetDir> [--dry-run]
|
|
25
25
|
|
|
26
|
+
import { readFileSync } from "node:fs";
|
|
26
27
|
import { relative, resolve } from "node:path";
|
|
27
28
|
import { Glob } from "bun";
|
|
28
29
|
import {
|
|
30
|
+
type CallExpression,
|
|
29
31
|
Node,
|
|
30
32
|
type ObjectLiteralExpression,
|
|
31
33
|
Project,
|
|
@@ -33,6 +35,11 @@ import {
|
|
|
33
35
|
type SourceFile,
|
|
34
36
|
SyntaxKind,
|
|
35
37
|
} from "ts-morph";
|
|
38
|
+
import {
|
|
39
|
+
PII_DIRECT_NAME_HINTS,
|
|
40
|
+
PII_USER_OWNED_NAME_HINTS,
|
|
41
|
+
PII_USER_REFERENCE_NAME_HINTS,
|
|
42
|
+
} from "../../engine/boot-validator/entity-handler";
|
|
36
43
|
|
|
37
44
|
const SUBJECT_FLAG_NAMES = [
|
|
38
45
|
"pii",
|
|
@@ -431,11 +438,242 @@ function findTargetFiles(rootDir: string): string[] {
|
|
|
431
438
|
return files.sort();
|
|
432
439
|
}
|
|
433
440
|
|
|
441
|
+
export type StanceClass = "direct" | "user-owned" | "user-reference" | "near-miss" | "unclassified";
|
|
442
|
+
|
|
443
|
+
export type StanceSite = {
|
|
444
|
+
readonly line: number;
|
|
445
|
+
readonly field: string;
|
|
446
|
+
readonly entity: string | null;
|
|
447
|
+
readonly callee: string;
|
|
448
|
+
readonly stance: StanceClass;
|
|
449
|
+
readonly hint: string | undefined;
|
|
450
|
+
};
|
|
451
|
+
|
|
452
|
+
const ALL_PII_NAME_HINTS: readonly string[] = [
|
|
453
|
+
...PII_DIRECT_NAME_HINTS,
|
|
454
|
+
...PII_USER_OWNED_NAME_HINTS,
|
|
455
|
+
...PII_USER_REFERENCE_NAME_HINTS,
|
|
456
|
+
];
|
|
457
|
+
|
|
458
|
+
const REPORT_STANCE_CALLEES = new Set(["createTextField", "createLongTextField"]);
|
|
459
|
+
|
|
460
|
+
// Mirrors guard-text-field-stance.ts's hasPersonalStance exactly.
|
|
461
|
+
function reportStanceHasPersonalStance(obj: ObjectLiteralExpression): boolean {
|
|
462
|
+
const prop = obj.getProperty("personal");
|
|
463
|
+
if (!prop || !Node.isPropertyAssignment(prop)) return false;
|
|
464
|
+
const init = prop.getInitializer();
|
|
465
|
+
return (
|
|
466
|
+
init !== undefined &&
|
|
467
|
+
init.getKind() !== SyntaxKind.UndefinedKeyword &&
|
|
468
|
+
!(Node.isIdentifier(init) && init.getText() === "undefined") &&
|
|
469
|
+
init.getKind() !== SyntaxKind.NullKeyword
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
function reportStanceEnclosingFieldName(call: CallExpression): string | undefined {
|
|
474
|
+
let node = call.getParent();
|
|
475
|
+
while (node) {
|
|
476
|
+
if (Node.isPropertyAssignment(node)) return node.getName();
|
|
477
|
+
node = node.getParent();
|
|
478
|
+
}
|
|
479
|
+
return undefined;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
function reportStanceResolveEntity(call: CallExpression): string | null {
|
|
483
|
+
let node: Node | undefined = call.getParent();
|
|
484
|
+
let entityCall: CallExpression | undefined;
|
|
485
|
+
while (node) {
|
|
486
|
+
if (Node.isCallExpression(node)) {
|
|
487
|
+
const expr = node.getExpression();
|
|
488
|
+
if (Node.isIdentifier(expr) && expr.getText() === "createEntity") {
|
|
489
|
+
entityCall = node;
|
|
490
|
+
break;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
node = node.getParent();
|
|
494
|
+
}
|
|
495
|
+
if (!entityCall) return null;
|
|
496
|
+
|
|
497
|
+
const firstArg = entityCall.getArguments()[0];
|
|
498
|
+
if (firstArg && Node.isObjectLiteralExpression(firstArg)) {
|
|
499
|
+
const tableProp = firstArg.getProperty("table");
|
|
500
|
+
if (tableProp && Node.isPropertyAssignment(tableProp)) {
|
|
501
|
+
const init = tableProp.getInitializer();
|
|
502
|
+
if (init && Node.isStringLiteral(init)) return init.getLiteralText();
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
const varDecl = entityCall.getParentIfKind(SyntaxKind.VariableDeclaration);
|
|
506
|
+
return varDecl ? varDecl.getName() : null;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// A hint only counts at a segment boundary (index 0, an uppercase letter in
|
|
510
|
+
// the original, or preceded by `_`) — otherwise a coincidental substring
|
|
511
|
+
// like "text" inside "contextId" would false-positive.
|
|
512
|
+
function hintOccursAtBoundary(fieldLower: string, fieldOriginal: string, hint: string): boolean {
|
|
513
|
+
let searchFrom = 0;
|
|
514
|
+
for (;;) {
|
|
515
|
+
const index = fieldLower.indexOf(hint, searchFrom);
|
|
516
|
+
if (index === -1) return false;
|
|
517
|
+
const atBoundary =
|
|
518
|
+
index === 0 || /[A-Z]/.test(fieldOriginal[index] ?? "") || fieldOriginal[index - 1] === "_";
|
|
519
|
+
if (atBoundary) return true;
|
|
520
|
+
searchFrom = index + 1;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
function findLongestBoundaryHint(fieldLower: string, fieldOriginal: string): string | undefined {
|
|
525
|
+
let best: string | undefined;
|
|
526
|
+
for (const hint of ALL_PII_NAME_HINTS) {
|
|
527
|
+
if (best && hint.length <= best.length) continue;
|
|
528
|
+
if (hintOccursAtBoundary(fieldLower, fieldOriginal, hint)) best = hint;
|
|
529
|
+
}
|
|
530
|
+
return best;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function segmentAlignedSuffixes(fieldOriginal: string): string[] {
|
|
534
|
+
const fieldLower = fieldOriginal.toLowerCase();
|
|
535
|
+
const suffixes: string[] = [];
|
|
536
|
+
for (let index = 0; index < fieldOriginal.length; index++) {
|
|
537
|
+
const atBoundary =
|
|
538
|
+
index === 0 || /[A-Z]/.test(fieldOriginal[index] ?? "") || fieldOriginal[index - 1] === "_";
|
|
539
|
+
if (atBoundary) suffixes.push(fieldLower.slice(index));
|
|
540
|
+
}
|
|
541
|
+
return suffixes;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// The hint sets only carry exact full names, so a suffix variant of one
|
|
545
|
+
// (e.g. "...UserId" of "assigneeUserId") otherwise slips through undetected.
|
|
546
|
+
function findShortestHintContainingSuffix(fieldOriginal: string): string | undefined {
|
|
547
|
+
let best: string | undefined;
|
|
548
|
+
for (const suffix of segmentAlignedSuffixes(fieldOriginal)) {
|
|
549
|
+
if (suffix.length < 5) continue;
|
|
550
|
+
for (const hint of ALL_PII_NAME_HINTS) {
|
|
551
|
+
if (!hint.includes(suffix)) continue;
|
|
552
|
+
if (!best || hint.length < best.length) best = hint;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return best;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function classifyFieldStance(field: string): { stance: StanceClass; hint: string | undefined } {
|
|
559
|
+
const fieldLower = field.toLowerCase();
|
|
560
|
+
if (PII_DIRECT_NAME_HINTS.has(fieldLower)) return { stance: "direct", hint: fieldLower };
|
|
561
|
+
if (PII_USER_OWNED_NAME_HINTS.has(fieldLower)) return { stance: "user-owned", hint: fieldLower };
|
|
562
|
+
if (PII_USER_REFERENCE_NAME_HINTS.has(fieldLower))
|
|
563
|
+
return { stance: "user-reference", hint: fieldLower };
|
|
564
|
+
const containmentHint = findLongestBoundaryHint(fieldLower, field);
|
|
565
|
+
if (containmentHint) return { stance: "near-miss", hint: containmentHint };
|
|
566
|
+
const suffixHint = findShortestHintContainingSuffix(field);
|
|
567
|
+
if (suffixHint) return { stance: "near-miss", hint: suffixHint };
|
|
568
|
+
return { stance: "unclassified", hint: undefined };
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
export function reportStanceForSource(source: string, filePath: string): StanceSite[] {
|
|
572
|
+
const project = new Project({ useInMemoryFileSystem: true, skipFileDependencyResolution: true });
|
|
573
|
+
const sourceFile = project.createSourceFile(filePath, source);
|
|
574
|
+
|
|
575
|
+
const sites: StanceSite[] = [];
|
|
576
|
+
for (const call of sourceFile.getDescendantsOfKind(SyntaxKind.CallExpression)) {
|
|
577
|
+
const exprNode = call.getExpression();
|
|
578
|
+
if (!Node.isIdentifier(exprNode)) continue;
|
|
579
|
+
const callee = exprNode.getText();
|
|
580
|
+
if (!REPORT_STANCE_CALLEES.has(callee)) continue;
|
|
581
|
+
|
|
582
|
+
const args = call.getArguments();
|
|
583
|
+
if (args.length > 0) {
|
|
584
|
+
const options = args[0];
|
|
585
|
+
if (!options || !Node.isObjectLiteralExpression(options)) continue;
|
|
586
|
+
if (options.getProperties().some((p) => p.isKind(SyntaxKind.SpreadAssignment))) continue;
|
|
587
|
+
if (reportStanceHasPersonalStance(options)) continue;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
const enclosingField = reportStanceEnclosingFieldName(call);
|
|
591
|
+
// A call with no enclosing field (e.g. a bare createTextField() at top
|
|
592
|
+
// level) has no real name to classify — `createTextField(...)` would
|
|
593
|
+
// otherwise false-positive as a near-miss on its own "Text".
|
|
594
|
+
const { stance, hint } = enclosingField
|
|
595
|
+
? classifyFieldStance(enclosingField)
|
|
596
|
+
: { stance: "unclassified" as const, hint: undefined };
|
|
597
|
+
|
|
598
|
+
sites.push({
|
|
599
|
+
line: call.getStartLineNumber(),
|
|
600
|
+
field: enclosingField ?? `${callee}(...)`,
|
|
601
|
+
entity: reportStanceResolveEntity(call),
|
|
602
|
+
callee,
|
|
603
|
+
stance,
|
|
604
|
+
hint,
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
return sites;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
function findReportStanceFiles(rootDir: string): string[] {
|
|
611
|
+
const glob = new Glob("**/*.{ts,tsx}");
|
|
612
|
+
const EXCLUDE = ["/node_modules/", "/dist/", "/build/"];
|
|
613
|
+
const files: string[] = [];
|
|
614
|
+
for (const file of glob.scanSync({ cwd: rootDir, dot: false })) {
|
|
615
|
+
if (file.endsWith(".d.ts")) continue;
|
|
616
|
+
const abs = resolve(rootDir, file);
|
|
617
|
+
if (EXCLUDE.some((p) => abs.includes(p))) continue;
|
|
618
|
+
files.push(abs);
|
|
619
|
+
}
|
|
620
|
+
return files.sort();
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function reportStance(rootDir: string): void {
|
|
624
|
+
console.log(
|
|
625
|
+
`Scanning every .ts/.tsx under ${rootDir} except node_modules/dist/build/*.d.ts — guard-text-field-stance scans packages/*/src/** only, so counts can differ outside that scope.`,
|
|
626
|
+
);
|
|
627
|
+
|
|
628
|
+
const totals: Record<StanceClass, number> = {
|
|
629
|
+
direct: 0,
|
|
630
|
+
"user-owned": 0,
|
|
631
|
+
"user-reference": 0,
|
|
632
|
+
"near-miss": 0,
|
|
633
|
+
unclassified: 0,
|
|
634
|
+
};
|
|
635
|
+
let total = 0;
|
|
636
|
+
|
|
637
|
+
for (const file of findReportStanceFiles(rootDir)) {
|
|
638
|
+
const sites = reportStanceForSource(readFileSync(file, "utf8"), file);
|
|
639
|
+
if (sites.length === 0) continue;
|
|
640
|
+
|
|
641
|
+
console.log(`\n${relative(rootDir, file)}`);
|
|
642
|
+
for (const site of sites) {
|
|
643
|
+
totals[site.stance]++;
|
|
644
|
+
total++;
|
|
645
|
+
const entity = site.entity ?? "<unresolved>";
|
|
646
|
+
const hintSuffix = site.hint ? ` (hint: ${site.hint})` : "";
|
|
647
|
+
console.log(
|
|
648
|
+
` ${site.line} ${site.field} entity=${entity} ${site.callee} ${site.stance}${hintSuffix}`,
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
console.log("\nBy stance:");
|
|
654
|
+
for (const [stance, count] of Object.entries(totals)) {
|
|
655
|
+
if (count > 0) console.log(` ${stance}: ${count}`);
|
|
656
|
+
}
|
|
657
|
+
console.log(`Total: ${total}`);
|
|
658
|
+
|
|
659
|
+
if (totals["near-miss"] > 0) {
|
|
660
|
+
console.log(
|
|
661
|
+
"\nHint sets in entity-handler.ts are exact name matches — near-miss field names slip past the boot heuristic.",
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
434
666
|
async function main(): Promise<void> {
|
|
435
667
|
const positional = process.argv.slice(2).filter((a) => !a.startsWith("--"));
|
|
436
|
-
const dryRun = process.argv.includes("--dry-run");
|
|
437
668
|
const rootDir = resolve(positional[0] ?? process.cwd());
|
|
438
669
|
|
|
670
|
+
if (process.argv.includes("--report-stance")) {
|
|
671
|
+
reportStance(rootDir);
|
|
672
|
+
// skip: report mode never rewrites, so the transform path below must not run
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
const dryRun = process.argv.includes("--dry-run");
|
|
439
677
|
const files = findTargetFiles(rootDir);
|
|
440
678
|
const project = new Project({
|
|
441
679
|
skipAddingFilesFromTsConfig: true,
|
|
@@ -477,4 +715,6 @@ async function main(): Promise<void> {
|
|
|
477
715
|
}
|
|
478
716
|
}
|
|
479
717
|
|
|
480
|
-
|
|
718
|
+
if (import.meta.main) {
|
|
719
|
+
await main();
|
|
720
|
+
}
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// deletedById) — this is also the SAME set the boot-validator's entityList
|
|
13
13
|
// column checks accept, so a softDelete column stays a boot-time error
|
|
14
14
|
// instead of a renderer-side throw (see screens.ts / entity-list-screens.ts).
|
|
15
|
-
import { SYSTEM_TENANT_ID } from "../engine/types/identifiers";
|
|
15
|
+
import { SYSTEM_TENANT_ID, SYSTEM_USER_ID } from "../engine/types/identifiers";
|
|
16
16
|
|
|
17
17
|
export type ListRowMetaColumnType = "text" | "number" | "timestamp";
|
|
18
18
|
|
|
@@ -56,4 +56,7 @@ export type SystemReferenceLabel = {
|
|
|
56
56
|
// entity, not just delivery-log (fw#2662).
|
|
57
57
|
export const SYSTEM_REFERENCE_LABELS: Readonly<Record<string, SystemReferenceLabel>> = {
|
|
58
58
|
"tenant:tenant": { id: SYSTEM_TENANT_ID, labelKey: "kumiko.reference.system-tenant" },
|
|
59
|
+
// createdBy on a system write (fw#3103) — SYSTEM_USER_ID is an alias for
|
|
60
|
+
// "no human caller", so read_users never holds a matching row.
|
|
61
|
+
"user:user": { id: SYSTEM_USER_ID, labelKey: "kumiko.reference.system-user" },
|
|
59
62
|
};
|