@cosmicdrift/kumiko-framework 0.341.0 → 0.342.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crypto/kek-source.d.ts +3 -4
- package/dist/crypto/kek-source.js +2 -5
- package/dist/crypto/kms-wiring.d.ts +2 -2
- package/dist/crypto/kms-wiring.js +2 -2
- package/dist/db/index.d.ts +2 -1
- package/dist/db/index.js +2 -1
- package/dist/db/migrate-generator.d.ts +6 -0
- package/dist/db/migrate-generator.js +15 -1
- package/dist/db/migrate-runner.d.ts +1 -0
- package/dist/db/migrate-runner.js +11 -1
- package/dist/db/retired-framework-tables.d.ts +10 -0
- package/dist/db/retired-framework-tables.js +72 -0
- package/dist/engine/soft-delete-cleanup.d.ts +2 -0
- package/dist/engine/soft-delete-cleanup.js +28 -9
- package/dist/jobs/job-runner.js +7 -1
- package/dist/observability/escape-hatch-report.d.ts +10 -1
- package/dist/observability/escape-hatch-report.js +19 -1
- package/dist/observability/index.d.ts +2 -2
- package/dist/observability/index.js +2 -2
- package/dist/observability/standard-metrics.d.ts +1 -0
- package/dist/observability/standard-metrics.js +14 -0
- package/dist/schema-cli.d.ts +8 -6
- package/dist/schema-cli.js +37 -4
- package/package.json +4 -4
- package/src/changes.json +28 -0
|
@@ -16,8 +16,7 @@ export type KekSourceOptions = {
|
|
|
16
16
|
readonly logPrefix?: string;
|
|
17
17
|
/** Where the boot line naming the KEK source goes. Defaults to `console.info`. */
|
|
18
18
|
readonly log?: (message: string) => void;
|
|
19
|
-
/** Env names to resolve, e.g. `kmsSlotsOf(schema)`.
|
|
20
|
-
|
|
21
|
-
readonly slots?: readonly string[];
|
|
19
|
+
/** Env names to resolve, e.g. `kmsSlotsOf(schema)`. */
|
|
20
|
+
readonly slots: readonly string[];
|
|
22
21
|
};
|
|
23
|
-
export declare function resolvePlatformKeks(env: KekSourceEnv, options
|
|
22
|
+
export declare function resolvePlatformKeks(env: KekSourceEnv, options: KekSourceOptions): Promise<KekSourceEnv>;
|
|
@@ -7,9 +7,6 @@ const SCALEWAY_KEY_MANAGER_API_VERSION = "v1alpha1";
|
|
|
7
7
|
const DEFAULT_REGION = "fr-par";
|
|
8
8
|
const DECRYPT_TIMEOUT_MS = 5_000;
|
|
9
9
|
const RETRY_DELAYS_MS = [200, 800];
|
|
10
|
-
// Used when the caller names no slots. Apps that predate `kms` schema meta
|
|
11
|
-
// rely on it; drop once every consumer declares its slots in the env schema.
|
|
12
|
-
const LEGACY_SLOTS = ["PLATFORM_KEK", "PLATFORM_KEK_PREVIOUS", "KUMIKO_BLIND_INDEX_KEY"];
|
|
13
10
|
function isRetryableStatus(status) {
|
|
14
11
|
return status >= 500;
|
|
15
12
|
}
|
|
@@ -97,10 +94,10 @@ async function resolveSlot(name, env, options, fetchImpl) {
|
|
|
97
94
|
// another slot's fallback path — the trio check downstream still applies.
|
|
98
95
|
// Slots resolve in parallel: sequentially, a slow Key Manager would cost up to
|
|
99
96
|
// ~16 s per slot (3 x 5 s timeout + backoff) and could trip startup probes.
|
|
100
|
-
export async function resolvePlatformKeks(env, options
|
|
97
|
+
export async function resolvePlatformKeks(env, options) {
|
|
101
98
|
const fetchImpl = options.fetch ?? globalThis.fetch;
|
|
102
99
|
const prefix = options.logPrefix ? `${options.logPrefix} ` : "";
|
|
103
|
-
const slots = options
|
|
100
|
+
const { slots } = options;
|
|
104
101
|
// Misconfiguration must not cost a Key Manager round trip.
|
|
105
102
|
if (slots.includes("PLATFORM_KEK_PREVIOUS") &&
|
|
106
103
|
(env.PLATFORM_KEK_PREVIOUS || env.PLATFORM_KEK_PREVIOUS_CIPHERTEXT) &&
|
|
@@ -62,6 +62,6 @@ export declare function requireKmsWiring(env: KmsWiringEnv, options?: KmsWiringO
|
|
|
62
62
|
* from a Key Manager ciphertext first when no plaintext is set — see
|
|
63
63
|
* `resolvePlatformKeks`. The validation stays in the sync function; this only
|
|
64
64
|
* adds the KEK-fetching step in front of it. */
|
|
65
|
-
export declare function resolveKmsWiringAsync(env: KmsWiringEnv, options
|
|
65
|
+
export declare function resolveKmsWiringAsync(env: KmsWiringEnv, options: KmsWiringOptions & KekSourceOptions): Promise<KmsWiring>;
|
|
66
66
|
/** Async counterpart to `requireKmsWiring`, KEK-resolving like `resolveKmsWiringAsync`. */
|
|
67
|
-
export declare function requireKmsWiringAsync(env: KmsWiringEnv, options
|
|
67
|
+
export declare function requireKmsWiringAsync(env: KmsWiringEnv, options: KmsWiringOptions & KekSourceOptions): Promise<ActiveKmsWiring>;
|
|
@@ -116,10 +116,10 @@ export function requireKmsWiring(env, options = {}) {
|
|
|
116
116
|
* from a Key Manager ciphertext first when no plaintext is set — see
|
|
117
117
|
* `resolvePlatformKeks`. The validation stays in the sync function; this only
|
|
118
118
|
* adds the KEK-fetching step in front of it. */
|
|
119
|
-
export async function resolveKmsWiringAsync(env, options
|
|
119
|
+
export async function resolveKmsWiringAsync(env, options) {
|
|
120
120
|
return resolveKmsWiring(await resolvePlatformKeks(env, options), options);
|
|
121
121
|
}
|
|
122
122
|
/** Async counterpart to `requireKmsWiring`, KEK-resolving like `resolveKmsWiringAsync`. */
|
|
123
|
-
export async function requireKmsWiringAsync(env, options
|
|
123
|
+
export async function requireKmsWiringAsync(env, options) {
|
|
124
124
|
return requireKmsWiring(await resolvePlatformKeks(env, options), options);
|
|
125
125
|
}
|
package/dist/db/index.d.ts
CHANGED
|
@@ -20,7 +20,7 @@ export { createEventStoreExecutor, entityEventName } from "./event-store-executo
|
|
|
20
20
|
export { enumerateFeatureTableSources, type FeatureTableSource, } from "./feature-table-sources.js";
|
|
21
21
|
export { flattenLocatedTimestamp, rehydrateLocatedTimestamp } from "./located-timestamp.js";
|
|
22
22
|
export { assertValidMigrationName, diffSnapshots, type GenerateMigrationInput, type GenerateMigrationOutput, generateMigration, loadSnapshotJson, renderMigrationSql, type SchemaDiff, type Snapshot, snapshotFromMetas, writeSnapshotJson, } from "./migrate-generator.js";
|
|
23
|
-
export { type AppliedMigration, type ApplyResult, type BaselineResult, baselineMigrations, fetchAppliedMigrations, loadMigrationsFromDir, type Migration, MigrationChecksumMismatchError, runMigrations, runMigrationsFromDir, splitSqlStatements, } from "./migrate-runner.js";
|
|
23
|
+
export { type AppliedMigration, type ApplyResult, type BaselineResult, baselineMigrations, fetchAppliedMigrations, loadMigrationsFromDir, type Migration, MigrationChecksumMismatchError, readMigrationSqlTexts, runMigrations, runMigrationsFromDir, splitSqlStatements, } from "./migrate-runner.js";
|
|
24
24
|
export { flattenMoney, type MoneyRead, moneyPayloadToMinorUnits, rehydrateMoney } from "./money.js";
|
|
25
25
|
export { constraintOf, extractPgError, isTableAlreadyExists, isUniqueViolation, type PgErrorInfo, } from "./pg-error.js";
|
|
26
26
|
export { acquireNamespacedAdvisoryLock } from "./queries/advisory-lock.js";
|
|
@@ -31,6 +31,7 @@ export { readRebuildMarker, rebuildTablesFromDiff, writeRebuildMarker, } from ".
|
|
|
31
31
|
export { seedReferenceData } from "./reference-data.js";
|
|
32
32
|
export { renderTableDdl, renderTablesDdl } from "./render-ddl.js";
|
|
33
33
|
export { diffReplayAgainstSnapshot, type ReplayedSchema, type ReplayedTable, type ReplayMismatch, replayMigrationsDir, } from "./replay-migration-sql.js";
|
|
34
|
+
export { findCommentedDropTables, isRetiredFrameworkTable, type RetiredFrameworkTable, retiredFrameworkTables, } from "./retired-framework-tables.js";
|
|
34
35
|
export { tableExists } from "./schema-inspection.js";
|
|
35
36
|
export { buildBaseColumns, buildEntityTable, declareGlobalTenancy, type EntityTable, physicalColumnName, toSnakeCase, toTableName, } from "./table-builder.js";
|
|
36
37
|
export type { TenantDb, TenantDbGrants, TenantDbMode, UncheckedSystemDb } from "./tenant-db.js";
|
package/dist/db/index.js
CHANGED
|
@@ -14,7 +14,7 @@ export { createEventStoreExecutor, entityEventName } from "./event-store-executo
|
|
|
14
14
|
export { enumerateFeatureTableSources, } from "./feature-table-sources.js";
|
|
15
15
|
export { flattenLocatedTimestamp, rehydrateLocatedTimestamp } from "./located-timestamp.js";
|
|
16
16
|
export { assertValidMigrationName, diffSnapshots, generateMigration, loadSnapshotJson, renderMigrationSql, snapshotFromMetas, writeSnapshotJson, } from "./migrate-generator.js";
|
|
17
|
-
export { baselineMigrations, fetchAppliedMigrations, loadMigrationsFromDir, MigrationChecksumMismatchError, runMigrations, runMigrationsFromDir, splitSqlStatements, } from "./migrate-runner.js";
|
|
17
|
+
export { baselineMigrations, fetchAppliedMigrations, loadMigrationsFromDir, MigrationChecksumMismatchError, readMigrationSqlTexts, runMigrations, runMigrationsFromDir, splitSqlStatements, } from "./migrate-runner.js";
|
|
18
18
|
export { flattenMoney, moneyPayloadToMinorUnits, rehydrateMoney } from "./money.js";
|
|
19
19
|
export { constraintOf, extractPgError, isTableAlreadyExists, isUniqueViolation, } from "./pg-error.js";
|
|
20
20
|
export { acquireNamespacedAdvisoryLock } from "./queries/advisory-lock.js";
|
|
@@ -24,6 +24,7 @@ export { readRebuildMarker, rebuildTablesFromDiff, writeRebuildMarker, } from ".
|
|
|
24
24
|
export { seedReferenceData } from "./reference-data.js";
|
|
25
25
|
export { renderTableDdl, renderTablesDdl } from "./render-ddl.js";
|
|
26
26
|
export { diffReplayAgainstSnapshot, replayMigrationsDir, } from "./replay-migration-sql.js";
|
|
27
|
+
export { findCommentedDropTables, isRetiredFrameworkTable, retiredFrameworkTables, } from "./retired-framework-tables.js";
|
|
27
28
|
export { tableExists } from "./schema-inspection.js";
|
|
28
29
|
export { buildBaseColumns, buildEntityTable, declareGlobalTenancy, physicalColumnName, toSnakeCase, toTableName, } from "./table-builder.js";
|
|
29
30
|
export { castTenantRows, createSystemDbView, createTenantDb, runInOwnTransaction, SYSTEM_SCOPE_CHECK_BRAND, } from "./tenant-db.js";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ColumnMeta, EntityTableMeta, IndexMeta } from "./entity-table-meta.js";
|
|
2
|
+
import { type RetiredFrameworkTable } from "./retired-framework-tables.js";
|
|
2
3
|
declare const SNAPSHOT_VERSION: 1;
|
|
3
4
|
export type Snapshot = {
|
|
4
5
|
readonly version: typeof SNAPSHOT_VERSION;
|
|
@@ -62,18 +63,23 @@ export declare function managedChangeRequiresRecreate(td: TableDiff): boolean;
|
|
|
62
63
|
export declare function renderMigrationSql(diff: SchemaDiff, options: {
|
|
63
64
|
readonly name: string;
|
|
64
65
|
readonly sequenceNumber: number;
|
|
66
|
+
readonly retiredDrops?: readonly RetiredFrameworkTable[];
|
|
65
67
|
}): string;
|
|
66
68
|
export type GenerateMigrationInput = {
|
|
67
69
|
readonly metas: readonly EntityTableMeta[];
|
|
68
70
|
readonly prevSnapshot: Snapshot | null;
|
|
69
71
|
readonly name: string;
|
|
70
72
|
readonly sequenceNumber: number;
|
|
73
|
+
/** Raw SQL of all earlier migrations, in order — lets the generator decide
|
|
74
|
+
* whether a retired framework table still needs its executable drop. */
|
|
75
|
+
readonly priorMigrationsSql?: readonly string[];
|
|
71
76
|
};
|
|
72
77
|
export type GenerateMigrationOutput = {
|
|
73
78
|
readonly filename: string;
|
|
74
79
|
readonly sqlContent: string;
|
|
75
80
|
readonly snapshot: Snapshot;
|
|
76
81
|
readonly diff: SchemaDiff;
|
|
82
|
+
readonly retiredDrops: readonly RetiredFrameworkTable[];
|
|
77
83
|
};
|
|
78
84
|
export declare const MIGRATION_NAME_RE: RegExp;
|
|
79
85
|
export declare function assertValidMigrationName(name: string): void;
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
18
18
|
import { compareByCodepoint } from "../utils/index.js";
|
|
19
19
|
import { renderIndex, renderTableDdl } from "./render-ddl.js";
|
|
20
|
+
import { findRetiredTableDrops } from "./retired-framework-tables.js";
|
|
20
21
|
const SNAPSHOT_VERSION = 1;
|
|
21
22
|
export function snapshotFromMetas(metas) {
|
|
22
23
|
// Stable ordering by tableName so the snapshot.json diff in PRs is
|
|
@@ -321,6 +322,7 @@ function describeIndexChange(change) {
|
|
|
321
322
|
// the index serves performance) are emitted as commented-out statements
|
|
322
323
|
// for explicit reviewer-action.
|
|
323
324
|
export function renderMigrationSql(diff, options) {
|
|
325
|
+
const retiredDrops = options.retiredDrops ?? [];
|
|
324
326
|
const lines = [];
|
|
325
327
|
const seq = options.sequenceNumber.toString().padStart(4, "0");
|
|
326
328
|
lines.push(`-- Migration ${seq}_${options.name}`);
|
|
@@ -393,9 +395,18 @@ export function renderMigrationSql(diff, options) {
|
|
|
393
395
|
}
|
|
394
396
|
lines.push("");
|
|
395
397
|
}
|
|
398
|
+
if (retiredDrops.length > 0) {
|
|
399
|
+
lines.push("-- === Retired framework tables ===");
|
|
400
|
+
for (const retired of retiredDrops) {
|
|
401
|
+
lines.push(`-- ${retired.tableName}: retired since framework ${retired.sinceVersion} — ${retired.explanation}`);
|
|
402
|
+
lines.push(`DROP TABLE IF EXISTS ${quoteIdent(retired.tableName)};`);
|
|
403
|
+
}
|
|
404
|
+
lines.push("");
|
|
405
|
+
}
|
|
396
406
|
if (diff.newTables.length === 0 &&
|
|
397
407
|
diff.changedTables.length === 0 &&
|
|
398
|
-
diff.droppedTables.length === 0
|
|
408
|
+
diff.droppedTables.length === 0 &&
|
|
409
|
+
retiredDrops.length === 0) {
|
|
399
410
|
lines.push("-- No schema changes detected.");
|
|
400
411
|
}
|
|
401
412
|
return lines.join("\n");
|
|
@@ -413,9 +424,11 @@ export function generateMigration(input) {
|
|
|
413
424
|
assertValidMigrationName(input.name);
|
|
414
425
|
const nextSnapshot = snapshotFromMetas(input.metas);
|
|
415
426
|
const diff = diffSnapshots(input.prevSnapshot, nextSnapshot);
|
|
427
|
+
const retiredDrops = findRetiredTableDrops(input.priorMigrationsSql ?? [], nextSnapshot);
|
|
416
428
|
const sqlContent = renderMigrationSql(diff, {
|
|
417
429
|
name: input.name,
|
|
418
430
|
sequenceNumber: input.sequenceNumber,
|
|
431
|
+
retiredDrops,
|
|
419
432
|
});
|
|
420
433
|
const seq = input.sequenceNumber.toString().padStart(4, "0");
|
|
421
434
|
return {
|
|
@@ -423,5 +436,6 @@ export function generateMigration(input) {
|
|
|
423
436
|
sqlContent,
|
|
424
437
|
snapshot: nextSnapshot,
|
|
425
438
|
diff,
|
|
439
|
+
retiredDrops,
|
|
426
440
|
};
|
|
427
441
|
}
|
|
@@ -14,6 +14,7 @@ export type ApplyResult = {
|
|
|
14
14
|
};
|
|
15
15
|
export declare function splitSqlStatements(sqlText: string): readonly string[];
|
|
16
16
|
export declare function loadMigrationsFromDir(dir: string, preprocess?: (sql: string) => string): readonly Migration[];
|
|
17
|
+
export declare function readMigrationSqlTexts(dir: string): readonly string[];
|
|
17
18
|
export declare function fetchAppliedMigrations(db: DbConnection): Promise<readonly AppliedMigration[]>;
|
|
18
19
|
export declare class MigrationChecksumMismatchError extends Error {
|
|
19
20
|
readonly migrationId: string;
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// fehl mit klarer Fehlermeldung (production-DBs haben das alte SQL drin —
|
|
15
15
|
// edit nachträglich = production-state inkonsistent mit committed-state).
|
|
16
16
|
import { createHash } from "node:crypto";
|
|
17
|
-
import { readdirSync, readFileSync } from "node:fs";
|
|
17
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
18
18
|
import { join } from "node:path";
|
|
19
19
|
// Adapter: extract raw postgres-js client from drizzle DbConnection,
|
|
20
20
|
// or use Bun.sql instance directly. Either way `.unsafe()` is the
|
|
@@ -171,6 +171,16 @@ export function loadMigrationsFromDir(dir, preprocess) {
|
|
|
171
171
|
};
|
|
172
172
|
});
|
|
173
173
|
}
|
|
174
|
+
// Raw file texts in migration order, comments intact (loadMigrationsFromDir
|
|
175
|
+
// strips them) — the retired-table scan reads them for DESTRUCTIVE markers.
|
|
176
|
+
export function readMigrationSqlTexts(dir) {
|
|
177
|
+
if (!existsSync(dir))
|
|
178
|
+
return [];
|
|
179
|
+
return readdirSync(dir)
|
|
180
|
+
.filter((f) => f.endsWith(".sql"))
|
|
181
|
+
.sort()
|
|
182
|
+
.map((file) => readFileSync(join(dir, file), "utf8"));
|
|
183
|
+
}
|
|
174
184
|
// Raw-SQL via postgres-js or Bun.sql .unsafe() — same shape for both.
|
|
175
185
|
async function executeRaw(db, sqlText) {
|
|
176
186
|
await rawClient(db).unsafe(sqlText);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Snapshot } from "./migrate-generator.js";
|
|
2
|
+
export type RetiredFrameworkTable = {
|
|
3
|
+
readonly tableName: string;
|
|
4
|
+
readonly sinceVersion: string;
|
|
5
|
+
readonly explanation: string;
|
|
6
|
+
};
|
|
7
|
+
export declare const retiredFrameworkTables: readonly RetiredFrameworkTable[];
|
|
8
|
+
export declare function findRetiredTableDrops(priorMigrationsSql: readonly string[], nextSnapshot: Snapshot): readonly RetiredFrameworkTable[];
|
|
9
|
+
export declare function findCommentedDropTables(migrationsSql: readonly string[], snapshot: Snapshot | null): readonly string[];
|
|
10
|
+
export declare function isRetiredFrameworkTable(tableName: string): boolean;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Framework tables that a framework release stopped creating and whose data
|
|
2
|
+
// was explicitly declared dispensable. `kumiko schema generate` drops them
|
|
3
|
+
// for real (instead of the commented-out DESTRUCTIVE marker app tables get),
|
|
4
|
+
// because the framework — not the app — owns the decision that the data is
|
|
5
|
+
// gone. Without this, every app carried a stale table forever behind a
|
|
6
|
+
// commented drop nobody uncomments.
|
|
7
|
+
//
|
|
8
|
+
// Add an entry only when the release notes state the data is not preserved.
|
|
9
|
+
// A rename is NOT a retirement: the data lives on under the new name.
|
|
10
|
+
export const retiredFrameworkTables = [
|
|
11
|
+
{
|
|
12
|
+
tableName: "read_job_runs",
|
|
13
|
+
sinceVersion: "0.210.0",
|
|
14
|
+
explanation: "replaced by store_job_runs; the job run history was not migrated (operational data, not a system of record)",
|
|
15
|
+
},
|
|
16
|
+
];
|
|
17
|
+
const IDENT = `"?([^"\\s(;,.]+)"?`;
|
|
18
|
+
const CREATE_TABLE_RE = new RegExp(`^CREATE TABLE\\s+(?:IF NOT EXISTS\\s+)?${IDENT}`, "i");
|
|
19
|
+
const DROP_TABLE_RE = new RegExp(`^DROP TABLE\\s+(?:IF EXISTS\\s+)?${IDENT}`, "i");
|
|
20
|
+
const COMMENTED_DROP_TABLE_RE = new RegExp(`^--\\s*DESTRUCTIVE:\\s*DROP TABLE\\s+(?:IF EXISTS\\s+)?${IDENT}`, "i");
|
|
21
|
+
// Last known state per table across all migrations, in order — so a
|
|
22
|
+
// DROP followed by a re-CREATE (managed-projection recreate) counts as
|
|
23
|
+
// "created", and a commented drop followed by an executable one as "dropped".
|
|
24
|
+
function scanTableLifecycle(sqlTexts) {
|
|
25
|
+
const states = new Map();
|
|
26
|
+
for (const text of sqlTexts) {
|
|
27
|
+
for (const rawLine of text.split("\n")) {
|
|
28
|
+
const line = rawLine.trim();
|
|
29
|
+
const commentedDrop = COMMENTED_DROP_TABLE_RE.exec(line)?.[1];
|
|
30
|
+
if (commentedDrop !== undefined) {
|
|
31
|
+
states.set(commentedDrop, "commented-drop");
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (line.startsWith("--"))
|
|
35
|
+
continue;
|
|
36
|
+
const created = CREATE_TABLE_RE.exec(line)?.[1];
|
|
37
|
+
if (created !== undefined) {
|
|
38
|
+
states.set(created, "created");
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
const dropped = DROP_TABLE_RE.exec(line)?.[1];
|
|
42
|
+
if (dropped !== undefined)
|
|
43
|
+
states.set(dropped, "dropped");
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return states;
|
|
47
|
+
}
|
|
48
|
+
function snapshotHasTable(snapshot, tableName) {
|
|
49
|
+
return snapshot?.tables.some((t) => t.tableName === tableName) ?? false;
|
|
50
|
+
}
|
|
51
|
+
// Retired tables the next migration must drop: a migration created them (or
|
|
52
|
+
// carries a commented drop), none dropped them for real, and the schema no
|
|
53
|
+
// longer declares them.
|
|
54
|
+
export function findRetiredTableDrops(priorMigrationsSql, nextSnapshot) {
|
|
55
|
+
const states = scanTableLifecycle(priorMigrationsSql);
|
|
56
|
+
return retiredFrameworkTables.filter((retired) => {
|
|
57
|
+
const state = states.get(retired.tableName);
|
|
58
|
+
return ((state === "created" || state === "commented-drop") &&
|
|
59
|
+
!snapshotHasTable(nextSnapshot, retired.tableName));
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
// Tables a migration only carries as a commented DESTRUCTIVE drop although
|
|
63
|
+
// the snapshot no longer has them: if they still exist in the live DB, the
|
|
64
|
+
// drop was never made.
|
|
65
|
+
export function findCommentedDropTables(migrationsSql, snapshot) {
|
|
66
|
+
return [...scanTableLifecycle(migrationsSql)]
|
|
67
|
+
.filter(([tableName, state]) => state === "commented-drop" && !snapshotHasTable(snapshot, tableName))
|
|
68
|
+
.map(([tableName]) => tableName);
|
|
69
|
+
}
|
|
70
|
+
export function isRetiredFrameworkTable(tableName) {
|
|
71
|
+
return retiredFrameworkTables.some((retired) => retired.tableName === tableName);
|
|
72
|
+
}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { ConfigKeyDefinition, JobDefinition, JobHandlerFn } from "./types/config.js";
|
|
2
|
+
import type { EntityDefinition } from "./types/index.js";
|
|
2
3
|
export declare const SOFT_DELETE_CLEANUP_JOB = "soft-delete:job:cleanup";
|
|
3
4
|
export declare const SOFT_DELETE_CLEANUP_SYSTEM_JOB = "soft-delete:job:cleanup-system";
|
|
4
5
|
export declare const SOFT_DELETE_GRACE_DAYS_KEY = "soft-delete:config:grace-days";
|
|
5
6
|
export declare const DEFAULT_GRACE_DAYS = 30;
|
|
6
7
|
export declare const softDeleteGraceDaysConfig: ConfigKeyDefinition;
|
|
8
|
+
export declare function isSystemScopedSoftDeleteTable(table: unknown, entity: Pick<EntityDefinition, "tenancy">): boolean;
|
|
7
9
|
export declare const softDeleteCleanupJob: JobHandlerFn;
|
|
8
10
|
export declare const softDeleteCleanupSystemJob: JobHandlerFn;
|
|
9
11
|
export declare function buildSoftDeleteCleanupJob(): JobDefinition;
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// resurrect the row as isDeleted=true. That's acceptable: cleanup bounds LIVE
|
|
11
11
|
// table growth; irreversible event-log purging is data-retention's job
|
|
12
12
|
// (pruneEvents), a separate, consumer-lag-guarded path.
|
|
13
|
-
import { deleteMany } from "../db/query.js";
|
|
13
|
+
import { asEntityTableMeta, deleteMany } from "../db/query.js";
|
|
14
14
|
import { SYSTEM_USER_ID } from "./system-user.js";
|
|
15
15
|
// qualifyEntityName convention (feature:type:kebab-name) with a reserved
|
|
16
16
|
// "soft-delete" owner — no real feature owns these; the framework synthesizes
|
|
@@ -31,6 +31,18 @@ export const softDeleteGraceDaysConfig = {
|
|
|
31
31
|
// existing silent-below-bound / audited-crossing behavior.
|
|
32
32
|
bounds: { min: 1 },
|
|
33
33
|
};
|
|
34
|
+
function hasTenantIdColumn(table) {
|
|
35
|
+
return table["tenantId"] !== undefined;
|
|
36
|
+
}
|
|
37
|
+
// The entity declaration decides, not the column: a `tenancy: "global"` table
|
|
38
|
+
// (e.g. user) still carries a tenantId column (SYSTEM_TENANT_ID), yet no
|
|
39
|
+
// tenant-mode delete can ever match its rows.
|
|
40
|
+
export function isSystemScopedSoftDeleteTable(table, entity) {
|
|
41
|
+
return entity.tenancy === "global" || !hasTenantIdColumn(table);
|
|
42
|
+
}
|
|
43
|
+
function isGlobalTenancyTable(table) {
|
|
44
|
+
return asEntityTableMeta(table)?.tenancy === "global";
|
|
45
|
+
}
|
|
34
46
|
export const softDeleteCleanupJob = async (_payload, ctx) => {
|
|
35
47
|
const { db, registry } = ctx;
|
|
36
48
|
if (!db || !registry) {
|
|
@@ -57,21 +69,18 @@ export const softDeleteCleanupJob = async (_payload, ctx) => {
|
|
|
57
69
|
const entity = registry.getEntity(proj.source);
|
|
58
70
|
if (!entity?.softDelete)
|
|
59
71
|
continue;
|
|
60
|
-
//
|
|
61
|
-
// list() does on table["tenantId"] to decide tenant-scoping.
|
|
62
|
-
const hasTenantColumn = proj.table["tenantId"] !== undefined;
|
|
63
|
-
// System-global entities (no tenantId column, e.g. `user`) are NOT swept
|
|
72
|
+
// System-global entities (tenancy "global" or no tenantId column, e.g. `user`) are NOT swept
|
|
64
73
|
// here: this handler runs once PER TENANT, so a tenant-scoped grace value
|
|
65
74
|
// would purge every OTHER tenant's still-within-grace rows too (effective
|
|
66
75
|
// grace = min() across all tenants). softDeleteCleanupSystemJob handles
|
|
67
76
|
// these separately, once, with a fixed grace period.
|
|
68
|
-
if (
|
|
77
|
+
if (isSystemScopedSoftDeleteTable(proj.table, entity))
|
|
69
78
|
continue;
|
|
70
79
|
const where = { isDeleted: true, deletedAt: { lt: cutoff }, tenantId };
|
|
71
80
|
await deleteMany(db, proj.table, where);
|
|
72
81
|
}
|
|
73
82
|
};
|
|
74
|
-
// System-global soft-deleted entities (no tenantId column) can't take a
|
|
83
|
+
// System-global soft-deleted entities (tenancy "global" or no tenantId column) can't take a
|
|
75
84
|
// per-tenant grace period — there's no "this tenant's view" of a row that
|
|
76
85
|
// isn't scoped to any tenant. Runs once (not perTenant) with a fixed grace
|
|
77
86
|
// period; a future system-scope config key could make this configurable
|
|
@@ -88,9 +97,15 @@ export const softDeleteCleanupSystemJob = async (_payload, ctx) => {
|
|
|
88
97
|
const entity = registry.getEntity(proj.source);
|
|
89
98
|
if (!entity?.softDelete)
|
|
90
99
|
continue;
|
|
91
|
-
if (proj.table
|
|
100
|
+
if (!isSystemScopedSoftDeleteTable(proj.table, entity))
|
|
92
101
|
continue;
|
|
93
|
-
|
|
102
|
+
const where = { isDeleted: true, deletedAt: { lt: cutoff } };
|
|
103
|
+
if (isGlobalTenancyTable(proj.table)) {
|
|
104
|
+
await db.global(proj.table).deleteMany(where);
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
await deleteMany(db, proj.table, where);
|
|
108
|
+
}
|
|
94
109
|
}
|
|
95
110
|
};
|
|
96
111
|
export function buildSoftDeleteCleanupJob() {
|
|
@@ -110,5 +125,9 @@ export function buildSoftDeleteCleanupSystemJob() {
|
|
|
110
125
|
trigger: { cron: "15 3 * * *" },
|
|
111
126
|
concurrency: "skip",
|
|
112
127
|
runIn: "worker",
|
|
128
|
+
escapeHatch: {
|
|
129
|
+
reason: "hard-deletes expired soft-deleted rows of global entities (e.g. user) once, system-wide",
|
|
130
|
+
grants: ["globalWrites"],
|
|
131
|
+
},
|
|
113
132
|
};
|
|
114
133
|
}
|
package/dist/jobs/job-runner.js
CHANGED
|
@@ -11,7 +11,7 @@ import { InternalError } from "../errors/index.js";
|
|
|
11
11
|
import { isKumikoError } from "../errors/kumiko-error.js";
|
|
12
12
|
import { createFileContext } from "../files/file-handle.js";
|
|
13
13
|
import { createFallbackLogger } from "../logging/index.js";
|
|
14
|
-
import { createEscapeHatchReporter, emitJobLastSuccess, emitJobQueueDepth, getFallbackTracer, registerStandardMetrics, } from "../observability/index.js";
|
|
14
|
+
import { createEscapeHatchProcessDedup, createEscapeHatchReporter, emitJobLastSuccess, emitJobQueueDepth, getFallbackTracer, registerStandardMetrics, } from "../observability/index.js";
|
|
15
15
|
import { createDistributedLock } from "../pipeline/distributed-lock.js";
|
|
16
16
|
import { RedisKeys } from "../pipeline/redis-keys.js";
|
|
17
17
|
import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
|
|
@@ -316,6 +316,8 @@ export function createJobRunner(options) {
|
|
|
316
316
|
// must not depend on buildServer having done it first. Idempotent.
|
|
317
317
|
if (context.meter)
|
|
318
318
|
registerStandardMetrics(context.meter);
|
|
319
|
+
// Instance-scoped, not module-level: each runner (and each test) audits a system cron's declared hatch once.
|
|
320
|
+
const systemCronEscapeHatchDedup = createEscapeHatchProcessDedup(["unsafe-raw", "global-write"]);
|
|
319
321
|
// Set at the top of stop() — a graceful shutdown closes the redis/BullMQ
|
|
320
322
|
// clients itself, which fires the exact same 'error' listeners below with
|
|
321
323
|
// an expected "Connection is closed." Downgrading to debug once stopping
|
|
@@ -692,12 +694,15 @@ export function createJobRunner(options) {
|
|
|
692
694
|
: undefined;
|
|
693
695
|
const isSystemJob = registry.isJobSystemScoped(jobName);
|
|
694
696
|
// One reporter for ctx.systemDb and ctx.db.unsafeRaw() so both dedupe in the same window.
|
|
697
|
+
const isSystemCronRun = triggeredById === null && jobDef.escapeHatch !== undefined && "cron" in jobDef.trigger;
|
|
695
698
|
const reportEscapeHatch = createEscapeHatchReporter({
|
|
696
699
|
handler: jobName,
|
|
697
700
|
tenantId,
|
|
698
701
|
actor: jobSystemUser.id,
|
|
699
702
|
sink: context._escapeHatchAuditSink,
|
|
700
703
|
log: context.log,
|
|
704
|
+
...(isSystemCronRun && { processDedup: systemCronEscapeHatchDedup }),
|
|
705
|
+
...(context.meter && { meter: context.meter }),
|
|
701
706
|
});
|
|
702
707
|
const systemDb = isSystemJob && tenantScopedDb
|
|
703
708
|
? createUncheckedSystemDb(tenantScopedDb, undefined, reportEscapeHatch)
|
|
@@ -705,6 +710,7 @@ export function createJobRunner(options) {
|
|
|
705
710
|
const jobDb = configDb
|
|
706
711
|
? createTenantDb(configDb, tenantId, "tenant", context.tracer, context.meter, undefined, {
|
|
707
712
|
unsafeRaw: escapeHatchFor(jobDef.escapeHatch, "unsafeRaw"),
|
|
713
|
+
globalWrites: escapeHatchFor(jobDef.escapeHatch, "globalWrites"),
|
|
708
714
|
report: reportEscapeHatch,
|
|
709
715
|
...(jobPersonalDataGate && { personalDataGate: jobPersonalDataGate }),
|
|
710
716
|
})
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { EscapeHatchAuditSink, EscapeHatchReporter, EscapeHatchUseEvent, TenantId } from "../engine/types/index.js";
|
|
1
|
+
import type { EscapeHatchAuditSink, EscapeHatchKind, EscapeHatchReporter, EscapeHatchUseEvent, TenantId } from "../engine/types/index.js";
|
|
2
2
|
import type { Logger } from "../logging/types.js";
|
|
3
|
+
import type { Meter } from "./types/index.js";
|
|
3
4
|
export declare const ESCAPE_HATCH_USED_SIGNAL = "security:escape-hatch-used";
|
|
4
5
|
export type EscapeHatchReportWindow = {
|
|
5
6
|
shouldReport(key: string, nowMs: number): boolean;
|
|
@@ -8,6 +9,11 @@ export declare function createEscapeHatchReportWindow(opts?: {
|
|
|
8
9
|
readonly windowMs?: number;
|
|
9
10
|
readonly maxEntries?: number;
|
|
10
11
|
}): EscapeHatchReportWindow;
|
|
12
|
+
export type EscapeHatchProcessDedup = {
|
|
13
|
+
readonly kinds: ReadonlySet<EscapeHatchKind>;
|
|
14
|
+
readonly seen: Set<string>;
|
|
15
|
+
};
|
|
16
|
+
export declare function createEscapeHatchProcessDedup(kinds: readonly EscapeHatchKind[]): EscapeHatchProcessDedup;
|
|
11
17
|
export declare function reportEscapeHatchUse(event: EscapeHatchUseEvent, deps: {
|
|
12
18
|
readonly sink?: EscapeHatchAuditSink;
|
|
13
19
|
readonly log?: Logger;
|
|
@@ -19,6 +25,9 @@ export declare function createEscapeHatchReporter(opts: {
|
|
|
19
25
|
readonly sink?: EscapeHatchAuditSink;
|
|
20
26
|
readonly log?: Logger;
|
|
21
27
|
readonly window?: EscapeHatchReportWindow;
|
|
28
|
+
readonly processDedup?: EscapeHatchProcessDedup;
|
|
29
|
+
readonly meter?: Meter;
|
|
30
|
+
readonly now?: () => number;
|
|
22
31
|
}): EscapeHatchReporter;
|
|
23
32
|
export declare const UNATTRIBUTED_ACTOR = "system";
|
|
24
33
|
export declare function fallbackEscapeHatchReporter(tenantId: TenantId): EscapeHatchReporter;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { emitEscapeHatchUse } from "./standard-metrics.js";
|
|
1
2
|
export const ESCAPE_HATCH_USED_SIGNAL = "security:escape-hatch-used";
|
|
2
3
|
const DEFAULT_WINDOW_MS = 60_000;
|
|
3
4
|
const DEFAULT_MAX_ENTRIES = 1_000;
|
|
@@ -24,6 +25,9 @@ export function createEscapeHatchReportWindow(opts) {
|
|
|
24
25
|
},
|
|
25
26
|
};
|
|
26
27
|
}
|
|
28
|
+
export function createEscapeHatchProcessDedup(kinds) {
|
|
29
|
+
return { kinds: new Set(kinds), seen: new Set() };
|
|
30
|
+
}
|
|
27
31
|
// Default window for createEscapeHatchReporter callers that don't pass their own.
|
|
28
32
|
const fallbackReportWindow = createEscapeHatchReportWindow();
|
|
29
33
|
const consoleLogger = {
|
|
@@ -67,7 +71,21 @@ export function reportEscapeHatchUse(event, deps) {
|
|
|
67
71
|
}
|
|
68
72
|
export function createEscapeHatchReporter(opts) {
|
|
69
73
|
const window = opts.window ?? fallbackReportWindow;
|
|
74
|
+
const now = opts.now ?? Date.now;
|
|
70
75
|
return (kind, reason, target) => {
|
|
76
|
+
if (opts.meter)
|
|
77
|
+
emitEscapeHatchUse(opts.meter, opts.handler, kind);
|
|
78
|
+
const { processDedup } = opts;
|
|
79
|
+
if (processDedup?.kinds.has(kind)) {
|
|
80
|
+
const processKey = JSON.stringify([opts.handler, kind, opts.tenantId]);
|
|
81
|
+
// skip: this (handler, kind, tenant) was already audited by this process
|
|
82
|
+
if (processDedup.seen.has(processKey))
|
|
83
|
+
return;
|
|
84
|
+
processDedup.seen.add(processKey);
|
|
85
|
+
reportEscapeHatchUse({ handler: opts.handler, kind, reason, tenantId: opts.tenantId, actor: opts.actor, target }, { sink: opts.sink, log: opts.log });
|
|
86
|
+
// skip: audited above, the window dedup below only serves the other kinds
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
71
89
|
const key = JSON.stringify([
|
|
72
90
|
opts.handler,
|
|
73
91
|
kind,
|
|
@@ -78,7 +96,7 @@ export function createEscapeHatchReporter(opts) {
|
|
|
78
96
|
target?.tenantId ?? null,
|
|
79
97
|
]);
|
|
80
98
|
// skip: same (handler, kind, reason, target) already reported within the window — dedup
|
|
81
|
-
if (!window.shouldReport(key,
|
|
99
|
+
if (!window.shouldReport(key, now()))
|
|
82
100
|
return;
|
|
83
101
|
reportEscapeHatchUse({ handler: opts.handler, kind, reason, tenantId: opts.tenantId, actor: opts.actor, target }, { sink: opts.sink, log: opts.log });
|
|
84
102
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { type ConsoleProviderOptions, createConsoleProvider } from "./console-provider.js";
|
|
2
2
|
export { observabilityContext } from "./context.js";
|
|
3
|
-
export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, type EscapeHatchReportWindow, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "./escape-hatch-report.js";
|
|
3
|
+
export { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, type EscapeHatchProcessDedup, type EscapeHatchReportWindow, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "./escape-hatch-report.js";
|
|
4
4
|
export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
|
|
5
5
|
export { generateSpanId, generateTraceId } from "./ids.js";
|
|
6
6
|
export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
|
|
@@ -12,5 +12,5 @@ export { type MetricEvent, type MetricEventHandler, RecordingMeter, } from "./re
|
|
|
12
12
|
export { type RecordedSpan, RecordingTracer, type RecordingTracerOptions, serializeSpanContext, } from "./recording-tracer.js";
|
|
13
13
|
export { wrapRedisClient } from "./redis-wrapper.js";
|
|
14
14
|
export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
|
|
15
|
-
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
15
|
+
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
16
16
|
export type { Counter, Gauge, Histogram, Meter, MetricDefinition, MetricLabels, MetricsHandle, MetricType, ObservabilityOptions, ObservabilityProvider, SamplingConfig, SensitiveFilterConfig, SerializedTraceContext, Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatus, StartSpanOptions, Tracer, } from "./types/index.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Public surface of the observability module.
|
|
2
2
|
export { createConsoleProvider } from "./console-provider.js";
|
|
3
3
|
export { observabilityContext } from "./context.js";
|
|
4
|
-
export { createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "./escape-hatch-report.js";
|
|
4
|
+
export { createEscapeHatchProcessDedup, createEscapeHatchReporter, createEscapeHatchReportWindow, ESCAPE_HATCH_USED_SIGNAL, fallbackEscapeHatchReporter, reportEscapeHatchUse, UNATTRIBUTED_ACTOR, } from "./escape-hatch-report.js";
|
|
5
5
|
export { getFallbackMeter, getFallbackProvider, getFallbackTracer } from "./fallback.js";
|
|
6
6
|
export { generateSpanId, generateTraceId } from "./ids.js";
|
|
7
7
|
export { buildMetricName, validateLabelKey, validateMetricName, } from "./metric-validator.js";
|
|
@@ -13,4 +13,4 @@ export { RecordingMeter, } from "./recording-meter.js";
|
|
|
13
13
|
export { RecordingTracer, serializeSpanContext, } from "./recording-tracer.js";
|
|
14
14
|
export { wrapRedisClient } from "./redis-wrapper.js";
|
|
15
15
|
export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
|
|
16
|
-
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
16
|
+
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEscapeHatchUse, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
@@ -43,3 +43,4 @@ export declare function emitEventConsumerPassOutcome(meter: Meter, labels: {
|
|
|
43
43
|
}, processed: number, failed: number): void;
|
|
44
44
|
export declare function emitJobQueueDepth(meter: Meter, lane: string, counts: Readonly<Record<string, number>>): void;
|
|
45
45
|
export declare function emitJobLastSuccess(meter: Meter, job: string): void;
|
|
46
|
+
export declare function emitEscapeHatchUse(meter: Meter, handler: string, kind: string): void;
|
|
@@ -140,6 +140,15 @@ export const STANDARD_METRIC_DEFS = [
|
|
|
140
140
|
description: "Unix timestamp of the last successful run, per registered job.",
|
|
141
141
|
labels: ["job"],
|
|
142
142
|
},
|
|
143
|
+
// Every escape-hatch use counts here, including the ones the audit sink
|
|
144
|
+
// dedups away: a system cron firing every minute writes one audit event per
|
|
145
|
+
// process, so this counter is the only record of how often it ran.
|
|
146
|
+
{
|
|
147
|
+
name: "kumiko_escape_hatch_uses_total",
|
|
148
|
+
type: "counter",
|
|
149
|
+
description: "Escape-hatch uses by handler and kind, whether or not an audit event was written.",
|
|
150
|
+
labels: ["handler", "kind"],
|
|
151
|
+
},
|
|
143
152
|
];
|
|
144
153
|
export function registerStandardMetrics(meter) {
|
|
145
154
|
for (const def of STANDARD_METRIC_DEFS) {
|
|
@@ -251,3 +260,8 @@ export function emitJobQueueDepth(meter, lane, counts) {
|
|
|
251
260
|
export function emitJobLastSuccess(meter, job) {
|
|
252
261
|
meter.gauge("kumiko_job_last_success_timestamp_seconds").set(Date.now() / 1000, { job });
|
|
253
262
|
}
|
|
263
|
+
// `handler` is a registry-declared handler/job name, `kind` a closed union —
|
|
264
|
+
// no tenant or user input reaches the label set.
|
|
265
|
+
export function emitEscapeHatchUse(meter, handler, kind) {
|
|
266
|
+
meter.counter("kumiko_escape_hatch_uses_total").inc(1, { handler, kind });
|
|
267
|
+
}
|
package/dist/schema-cli.d.ts
CHANGED
|
@@ -4,15 +4,17 @@ export type SchemaCliOut = {
|
|
|
4
4
|
readonly err: (line: string) => void;
|
|
5
5
|
};
|
|
6
6
|
export type RunSchemaCliOptions = {
|
|
7
|
+
readonly features?: undefined;
|
|
8
|
+
readonly kmsSlots?: undefined;
|
|
9
|
+
} | {
|
|
7
10
|
/** Composed app features. When given, `apply` rebuilds the projections whose
|
|
8
11
|
* tables a freshly applied migration changed (via its `.rebuild.json`
|
|
9
12
|
* marker). Omitted (dev `kumiko schema`) → no rebuild, migrations only. */
|
|
10
|
-
readonly features
|
|
11
|
-
/** Schema-declared Key Manager slots,
|
|
12
|
-
* This CLI parses no env schema, so
|
|
13
|
-
*
|
|
14
|
-
|
|
15
|
-
readonly kmsSlots?: readonly string[];
|
|
13
|
+
readonly features: readonly FeatureDefinition[];
|
|
14
|
+
/** Schema-declared Key Manager slots, `kmsSlotsOf(composedEnv.schema)`, or
|
|
15
|
+
* `[]` when the app wires none. This CLI parses no env schema, so the
|
|
16
|
+
* rebuild's KMS wiring cannot derive them itself. */
|
|
17
|
+
readonly kmsSlots: readonly string[];
|
|
16
18
|
};
|
|
17
19
|
/**
|
|
18
20
|
* Runs a schema-CLI subcommand. `appCwd` is the app workspace root (where
|
package/dist/schema-cli.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
11
11
|
import { join, resolve as resolvePath } from "node:path";
|
|
12
12
|
import { configureBlindIndexKey, configurePiiSubjectKms, resolveKmsWiringAsync, } from "./crypto/index.js";
|
|
13
|
-
import { assertValidMigrationName, baselineMigrations, createDbConnection, diffReplayAgainstSnapshot, fetchAppliedMigrations, generateMigration, loadMigrationsFromDir, loadSnapshotJson, rebuildTablesFromDiff, replayMigrationsDir, runMigrationsFromDir, tableExists, writeRebuildMarker, writeSnapshotJson, } from "./db/index.js";
|
|
13
|
+
import { assertValidMigrationName, baselineMigrations, createDbConnection, diffReplayAgainstSnapshot, fetchAppliedMigrations, findCommentedDropTables, generateMigration, isRetiredFrameworkTable, loadMigrationsFromDir, loadSnapshotJson, readMigrationSqlTexts, rebuildTablesFromDiff, replayMigrationsDir, runMigrationsFromDir, tableExists, writeRebuildMarker, writeSnapshotJson, } from "./db/index.js";
|
|
14
14
|
import { validateBoot } from "./engine/boot-validator.js";
|
|
15
15
|
import { createRegistry } from "./engine/registry.js";
|
|
16
16
|
import { createEventsTable } from "./event-store/index.js";
|
|
@@ -49,12 +49,40 @@ function nextSequenceNumber(migrationsDir) {
|
|
|
49
49
|
}
|
|
50
50
|
return max + 1;
|
|
51
51
|
}
|
|
52
|
+
// A table that a migration only drops as a commented DESTRUCTIVE marker and
|
|
53
|
+
// that the snapshot no longer has is gone on paper but still in the live DB.
|
|
54
|
+
async function reportLeftoverDroppedTables(db, migrationsDir, out) {
|
|
55
|
+
const snapshotPath = join(migrationsDir, SNAPSHOT_FILENAME);
|
|
56
|
+
const snapshot = existsSync(snapshotPath) ? loadSnapshotJson(snapshotPath) : null;
|
|
57
|
+
const leftovers = [];
|
|
58
|
+
for (const tableName of findCommentedDropTables(readMigrationSqlTexts(migrationsDir), snapshot)) {
|
|
59
|
+
if (await tableExists(db, tableName))
|
|
60
|
+
leftovers.push(tableName);
|
|
61
|
+
}
|
|
62
|
+
// skip: nothing left over, nothing to report
|
|
63
|
+
if (leftovers.length === 0)
|
|
64
|
+
return;
|
|
65
|
+
out.log(" Tables dropped only as a commented DESTRUCTIVE marker but still in the database:");
|
|
66
|
+
for (const tableName of leftovers) {
|
|
67
|
+
out.log(isRetiredFrameworkTable(tableName)
|
|
68
|
+
? ` ! ${tableName} — retired framework table: the next \`schema generate <name>\` creates the drop`
|
|
69
|
+
: ` ! ${tableName} — app table: after a backup, write a new migration with the drop`);
|
|
70
|
+
}
|
|
71
|
+
out.log("");
|
|
72
|
+
}
|
|
52
73
|
const SCHEMA_SUBCOMMANDS = ["generate", "validate", "apply", "baseline", "status"];
|
|
53
74
|
/**
|
|
54
75
|
* Runs a schema-CLI subcommand. `appCwd` is the app workspace root (where
|
|
55
76
|
* `kumiko/schema.ts` + `kumiko/migrations/` live). Returns a process exit code.
|
|
56
77
|
*/
|
|
57
78
|
export async function runSchemaCli(argv, appCwd, out, options = {}) {
|
|
79
|
+
// Runtime guard for untyped callers: the rebuild after `apply` needs the
|
|
80
|
+
// slots, and failing there would leave migrations applied but unrebuilt.
|
|
81
|
+
if (options.features && !options.kmsSlots) {
|
|
82
|
+
out.err("runSchemaCli: `features` requires `kmsSlots` — pass kmsSlots: kmsSlotsOf(<composedEnv>.schema), " +
|
|
83
|
+
"or [] when the app wires no Key Manager slots.");
|
|
84
|
+
return 1;
|
|
85
|
+
}
|
|
58
86
|
// runProdApp/runDevApp install this at boot; the standalone CLI (the
|
|
59
87
|
// migrate-db initContainer, `bun kumiko.js schema apply`) never goes through
|
|
60
88
|
// that boot path, so a projection rebuild's tz/timestamp coercion throws
|
|
@@ -114,10 +142,12 @@ export async function runSchemaCli(argv, appCwd, out, options = {}) {
|
|
|
114
142
|
prevSnapshot,
|
|
115
143
|
name,
|
|
116
144
|
sequenceNumber: nextSequenceNumber(migrationsDir),
|
|
145
|
+
priorMigrationsSql: readMigrationSqlTexts(migrationsDir),
|
|
117
146
|
});
|
|
118
147
|
const isEmpty = result.diff.newTables.length === 0 &&
|
|
119
148
|
result.diff.changedTables.length === 0 &&
|
|
120
|
-
result.diff.droppedTables.length === 0
|
|
149
|
+
result.diff.droppedTables.length === 0 &&
|
|
150
|
+
result.retiredDrops.length === 0;
|
|
121
151
|
if (isEmpty) {
|
|
122
152
|
out.log(" No schema changes detected — kein neues Migration-File geschrieben.");
|
|
123
153
|
return 0;
|
|
@@ -133,7 +163,7 @@ export async function runSchemaCli(argv, appCwd, out, options = {}) {
|
|
|
133
163
|
writeRebuildMarker(migrationsDir, result.filename, rebuildTables);
|
|
134
164
|
out.log("");
|
|
135
165
|
out.log(` ✓ ${result.filename}`);
|
|
136
|
-
out.log(` new tables: ${result.diff.newTables.length}, changed: ${result.diff.changedTables.length}, dropped: ${result.diff.droppedTables.length}`);
|
|
166
|
+
out.log(` new tables: ${result.diff.newTables.length}, changed: ${result.diff.changedTables.length}, dropped: ${result.diff.droppedTables.length}, retired framework tables dropped: ${result.retiredDrops.length}`);
|
|
137
167
|
if (rebuildTables.length > 0) {
|
|
138
168
|
out.log(` rebuild-marker: ${result.filename.replace(/\.sql$/, ".rebuild.json")} (${rebuildTables.length} table(s))`);
|
|
139
169
|
}
|
|
@@ -182,11 +212,13 @@ export async function runSchemaCli(argv, appCwd, out, options = {}) {
|
|
|
182
212
|
prevSnapshot,
|
|
183
213
|
name: "validate",
|
|
184
214
|
sequenceNumber: nextSequenceNumber(migrationsDir),
|
|
215
|
+
priorMigrationsSql: readMigrationSqlTexts(migrationsDir),
|
|
185
216
|
});
|
|
186
217
|
const pendingTables = [
|
|
187
218
|
...drift.diff.newTables.map((t) => t.tableName),
|
|
188
219
|
...drift.diff.changedTables.map((t) => t.tableName),
|
|
189
220
|
...drift.diff.droppedTables,
|
|
221
|
+
...drift.retiredDrops.map((t) => t.tableName),
|
|
190
222
|
];
|
|
191
223
|
if (pendingTables.length === 0) {
|
|
192
224
|
out.log(" ✓ schema: migrations match the entity definitions");
|
|
@@ -308,7 +340,7 @@ export async function runSchemaCli(argv, appCwd, out, options = {}) {
|
|
|
308
340
|
wiring = await resolveKmsWiringAsync(process.env, {
|
|
309
341
|
logPrefix: "[kumiko schema apply]",
|
|
310
342
|
log: out.log,
|
|
311
|
-
|
|
343
|
+
slots: options.kmsSlots,
|
|
312
344
|
});
|
|
313
345
|
if ("kms" in wiring) {
|
|
314
346
|
configurePiiSubjectKms(wiring.kms);
|
|
@@ -398,6 +430,7 @@ export async function runSchemaCli(argv, appCwd, out, options = {}) {
|
|
|
398
430
|
out.log("");
|
|
399
431
|
out.log(` ${applied.size} applied, ${pending} pending.`);
|
|
400
432
|
out.log("");
|
|
433
|
+
await reportLeftoverDroppedTables(db, migrationsDir, out);
|
|
401
434
|
return pending === 0 ? 0 : 1;
|
|
402
435
|
}
|
|
403
436
|
finally {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.342.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -206,8 +206,8 @@
|
|
|
206
206
|
"./package.json": "./package.json"
|
|
207
207
|
},
|
|
208
208
|
"dependencies": {
|
|
209
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
210
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
209
|
+
"@cosmicdrift/kumiko-http": "0.342.0",
|
|
210
|
+
"@cosmicdrift/kumiko-types": "0.342.0",
|
|
211
211
|
"bullmq": "^5.76.7",
|
|
212
212
|
"bun-types": "^1.3.13",
|
|
213
213
|
"hono": "^4.13.1",
|
|
@@ -223,7 +223,7 @@
|
|
|
223
223
|
"zod": "^4.4.3"
|
|
224
224
|
},
|
|
225
225
|
"devDependencies": {
|
|
226
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
226
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.342.0",
|
|
227
227
|
"bun-types": "^1.3.13",
|
|
228
228
|
"pino-pretty": "^13.1.3"
|
|
229
229
|
},
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,32 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.342.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "System cron escape hatches are audited once per process, with a use counter and audit retention",
|
|
6
|
+
"detail": "System crons audit their declared escape hatch once per process; audit events get a retention\nA cron job with a declared `escapeHatch` that uses `unsafeRaw()` or `db.global()` writes one `audit:event:escape-hatch-used` per process, handler and tenant instead of one per run. Manual runs by a user, identity switches, `acknowledge-cross-tenant` and `unsafe-all-tenants` keep the 60-second dedup window. Every use, audited or not, counts in the new metric `kumiko_escape_hatch_uses_total{handler,kind}`; `createEscapeHatchReporter` takes optional `processDedup`, `meter` and `now` for this. The audit feature gains a daily job `audit:job:escape-hatch-retention` that removes `escapeHatchUse` events through `pruneEvents`, with the period in the system config key `audit:config:escape-hatch-retention-days` (default 90). A new runbook, `docs/runbooks/kumiko-events-bloat.md`, shows how to measure `kumiko_events` bloat and reclaim the space.",
|
|
7
|
+
"migration": "The first run of `audit:job:escape-hatch-retention` deletes `escapeHatchUse` events older than 90 days. To keep them longer, set the system config key `audit:config:escape-hatch-retention-days` (for example through an app override) before the bump rolls out. The audit feature now also requires the `config` feature, which every app with `tenant` already mounts."
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"version": "0.342.0",
|
|
11
|
+
"type": "breaking",
|
|
12
|
+
"title": "Key Manager slots are required, LEGACY_SLOTS default removed",
|
|
13
|
+
"detail": "Key Manager slots are required: no more LEGACY_SLOTS default\n`resolvePlatformKeks`, `resolveKmsWiringAsync` and `requireKmsWiringAsync` no longer fall back to the three platform slots; `slots` is a required option. `runSchemaCli` called with `features` now requires `kmsSlots` and exits 1 with a hint before any DB connect or migration apply when it is missing. `kmsSlots: []` stays valid for apps that wire no Key Manager slots. The scaffolded `bin/kumiko.ts` passes `kmsSlots` derived from the composed env schema.",
|
|
14
|
+
"migration": "Pass `slots` (for example `kmsSlotsOf(<app>ComposedEnv.schema)`) to `resolvePlatformKeks`, `resolveKmsWiringAsync` and `requireKmsWiringAsync`. In `bin/kumiko.ts`, `runSchemaCli(..., { features })` needs `kmsSlots: kmsSlotsOf(<app>ComposedEnv.schema)` (or `[]` when the app wires no Key Manager slots. With `[]` every `*_CIPHERTEXT` env var is ignored, so an app that ships a ciphertext must pass its slots); without it the CLI exits 1 before applying migrations.\nWorkspace state at release: show-pony `bin/kumiko.ts` and its docs copy `kumiko-platform/apps/docs/_samples/show-pony/bin/kumiko.ts` still call `runSchemaCli` with `features` only. publicstatus, solon, offlot-app, money-horse, phronexsis and kumiko-studio already pass `kmsSlots` / `slots`."
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"version": "0.342.0",
|
|
18
|
+
"type": "fix",
|
|
19
|
+
"title": "Retired framework tables (read_job_runs) are dropped by the next schema generate",
|
|
20
|
+
"detail": "`kumiko schema generate` now writes an executable `DROP TABLE IF EXISTS \"read_job_runs\";` for retired framework tables (new `retiredFrameworkTables` list next to the migration code). Until now the drop stayed a commented `-- DESTRUCTIVE:` marker in every app, so the table lived on forever. A retired table is dropped only if an earlier migration created it or carries the commented drop, no earlier migration dropped it for real, and the schema no longer declares it. App tables stay commented out. `generate` writes a migration even when the retired drop is its only content, and `schema validate` reports it as pending. `kumiko schema status` now lists tables that a migration only drops as a commented marker but that still exist in the database, with a hint per case.",
|
|
21
|
+
"migration": "Run `kumiko schema generate <name>` in publicstatus, solon, money-horse, phronexsis, show-pony and kumiko-studio, review and commit the drop of read_job_runs (old job run history is not preserved), then apply."
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"version": "0.342.0",
|
|
25
|
+
"type": "fix",
|
|
26
|
+
"title": "Soft-delete cleanup works for global tables such as user",
|
|
27
|
+
"detail": "Soft-delete cleanup no longer aborts on `tenancy: \"global\"` tables\n`soft-delete:job:cleanup` aborted for a tenant as soon as an entity with `tenancy: \"global\"` and a `tenantId` column (such as `user`) had softDelete enabled, so the tenant's other entities were not cleaned either. Both cleanup jobs now decide by the entity's tenancy declaration. The per-tenant job skips global tables; `soft-delete:job:cleanup-system` hard-deletes their expired rows once, system-wide, through `db.global()`. Jobs can now receive the `globalWrites` grant from their `escapeHatch` declaration.\nThe first run of `soft-delete:job:cleanup-system` after the update hard-deletes soft-deleted users older than the fixed 30-day grace period in every app with the user feature. Earlier versions never removed them.",
|
|
28
|
+
"migration": "None required. The first nightly run of `soft-delete:job:cleanup-system` after the bump hard-deletes soft-deleted users older than 30 days; restore any you still need before it runs."
|
|
29
|
+
},
|
|
2
30
|
{
|
|
3
31
|
"version": "0.341.0",
|
|
4
32
|
"type": "improvement",
|