@cosmicdrift/kumiko-framework 0.312.0 → 0.313.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/auth-routes.ts +27 -2
- package/src/changes.json +18 -0
- package/src/crypto/pg-kms-adapter.ts +2 -1
- package/src/db/__tests__/connection-close-outage.integration.test.ts +110 -0
- package/src/db/api.ts +6 -0
- package/src/db/bun-provider.ts +8 -3
- package/src/db/connection.ts +3 -3
- package/src/db/eagerload.ts +2 -9
- package/src/db/index.ts +1 -1
- package/src/db/postgres-provider.ts +7 -4
- package/src/db/queries/event-consumer.ts +4 -3
- package/src/engine/boot-validator/__tests__/transfer-graph.test.ts +12 -0
- package/src/engine/boot-validator/transfer-graph.ts +2 -1
- package/src/engine/index.ts +3 -0
- package/src/engine/parse-ref-target.ts +6 -0
- package/src/engine/registry-validate.ts +3 -12
- package/src/engine/types/index.ts +1 -0
- package/src/files/__tests__/storage-tracking.test.ts +46 -0
- package/src/files/index.ts +6 -1
- package/src/files/storage-tracking.ts +160 -15
- package/src/jobs/job-runner.ts +27 -0
- package/src/pipeline/__tests__/event-dispatcher-db-outage.integration.test.ts +27 -0
- package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +34 -6
- package/src/pipeline/event-dispatcher-delivery.ts +22 -1
- package/src/pipeline/event-dispatcher.ts +25 -2
- package/src/pipeline/index.ts +4 -1
- package/src/pipeline/system-hooks.ts +4 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.313.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -198,8 +198,8 @@
|
|
|
198
198
|
"./package.json": "./package.json"
|
|
199
199
|
},
|
|
200
200
|
"dependencies": {
|
|
201
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
202
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
201
|
+
"@cosmicdrift/kumiko-http": "0.313.0",
|
|
202
|
+
"@cosmicdrift/kumiko-types": "0.313.0",
|
|
203
203
|
"bullmq": "^5.76.7",
|
|
204
204
|
"bun-types": "^1.3.13",
|
|
205
205
|
"hono": "^4.13.1",
|
|
@@ -215,7 +215,7 @@
|
|
|
215
215
|
"zod": "^4.4.3"
|
|
216
216
|
},
|
|
217
217
|
"devDependencies": {
|
|
218
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
218
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.313.0",
|
|
219
219
|
"bun-types": "^1.3.13",
|
|
220
220
|
"pino-pretty": "^13.1.3"
|
|
221
221
|
},
|
package/src/api/auth-routes.ts
CHANGED
|
@@ -148,6 +148,20 @@ const SignupConfirmBody = z.object({
|
|
|
148
148
|
password: z.string().min(8).max(200),
|
|
149
149
|
});
|
|
150
150
|
|
|
151
|
+
// Cross-device try-first handover (kumiko-framework#3035 follow-up,
|
|
152
|
+
// offlot-app#454): an optional tenant-handover grant the signup-request
|
|
153
|
+
// route forwards to the handler for server-side verification and binding —
|
|
154
|
+
// the grant itself never appears anywhere else (URL, mail, log, response).
|
|
155
|
+
const SignupRequestBody = z.object({
|
|
156
|
+
email: z.email(),
|
|
157
|
+
handover: z
|
|
158
|
+
.object({
|
|
159
|
+
entityType: z.string().min(1).max(200),
|
|
160
|
+
token: z.string().min(1).max(4096),
|
|
161
|
+
})
|
|
162
|
+
.optional(),
|
|
163
|
+
});
|
|
164
|
+
|
|
151
165
|
const InviteAcceptBody = z.object({
|
|
152
166
|
token: z.string().min(1),
|
|
153
167
|
});
|
|
@@ -1122,6 +1136,7 @@ export function createAuthRoutes(
|
|
|
1122
1136
|
dispatcher,
|
|
1123
1137
|
path: Routes.authSignupRequest,
|
|
1124
1138
|
requestHandler: sg.requestHandler,
|
|
1139
|
+
schema: SignupRequestBody,
|
|
1125
1140
|
});
|
|
1126
1141
|
|
|
1127
1142
|
api.post(Routes.authSignupConfirm, async (c) => {
|
|
@@ -1149,6 +1164,7 @@ export function createAuthRoutes(
|
|
|
1149
1164
|
kind: "auth-session";
|
|
1150
1165
|
session: SessionUser;
|
|
1151
1166
|
tenantKey: string;
|
|
1167
|
+
handover?: { entityType: string; id: string };
|
|
1152
1168
|
};
|
|
1153
1169
|
|
|
1154
1170
|
// Session creation + JWT sign + cookies — see mintSessionAndRespond.
|
|
@@ -1168,6 +1184,11 @@ export function createAuthRoutes(
|
|
|
1168
1184
|
// membership — die Frontend-UI nimmt das direkt als Redirect-
|
|
1169
1185
|
// Target.
|
|
1170
1186
|
tenantKey: data.tenantKey,
|
|
1187
|
+
// Cross-device try-first handover (kumiko-framework#3035 follow-up):
|
|
1188
|
+
// present only when signup-confirm actually claimed a bound grant —
|
|
1189
|
+
// absent, not null, so a client that doesn't know the field sees
|
|
1190
|
+
// nothing unusual.
|
|
1191
|
+
...(data.handover !== undefined && { handover: data.handover }),
|
|
1171
1192
|
});
|
|
1172
1193
|
});
|
|
1173
1194
|
}
|
|
@@ -1462,8 +1483,12 @@ function registerTokenRequestRoute(opts: {
|
|
|
1462
1483
|
dispatcher: Dispatcher;
|
|
1463
1484
|
path: string;
|
|
1464
1485
|
requestHandler: string;
|
|
1486
|
+
// reset/verify/unlock keep the plain RequestTokenBody; signup passes its
|
|
1487
|
+
// own schema (adds an optional `handover`, see SignupRequestBody above)
|
|
1488
|
+
// so the extra field reaches the handler instead of being parsed away.
|
|
1489
|
+
schema?: typeof RequestTokenBody | typeof SignupRequestBody;
|
|
1465
1490
|
}): void {
|
|
1466
|
-
const body = RequestTokenBody;
|
|
1491
|
+
const body = opts.schema ?? RequestTokenBody;
|
|
1467
1492
|
opts.api.post(opts.path, async (c) => {
|
|
1468
1493
|
const raw = await c.req.json().catch(() => null);
|
|
1469
1494
|
const parsed = body.safeParse(raw);
|
|
@@ -1478,7 +1503,7 @@ function registerTokenRequestRoute(opts: {
|
|
|
1478
1503
|
// the log line (PII).
|
|
1479
1504
|
const result = await opts.dispatcher.write(
|
|
1480
1505
|
opts.requestHandler,
|
|
1481
|
-
|
|
1506
|
+
parsed.data,
|
|
1482
1507
|
createAnonymousUser(SYSTEM_TENANT_ID),
|
|
1483
1508
|
);
|
|
1484
1509
|
if (!result.isSuccess) {
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,22 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.313.0",
|
|
4
|
+
"type": "fix",
|
|
5
|
+
"title": "DB pool close() no longer hangs forever during an outage",
|
|
6
|
+
"detail": "createDbConnection/createConnection's close() and PgKmsAdapter.close()\ncalled client.end() with no timeout, which postgres.js waits on\nindefinitely if a query is still referenced on a dead connection — a DB\noutage during process shutdown could hang forever. close() now bounds\nthe wait via a new closeTimeoutSeconds option on DbConnectionOptions\n(default 5s, DEFAULT_DB_CLOSE_TIMEOUT_SECONDS). No migration needed."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.313.0",
|
|
10
|
+
"type": "fix",
|
|
11
|
+
"title": "event-dispatcher stop() waits for a pass still in its idle pre-check",
|
|
12
|
+
"detail": "Since 0.308.0 a pass started by the timer or a NOTIFY wake-up shortly\nbefore stop() could still be inside its idle pre-check, which stop()\nnever waited for. That pass kept running after stop() returned and\nqueried the DB pool the caller was closing, which could make\npostgres.js' end() hang. stop() now also drains in-flight passes.\nNo migration needed."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"version": "0.313.0",
|
|
16
|
+
"type": "improvement",
|
|
17
|
+
"title": "Event-triggered jobs get the triggering event's id and headers via ctx.triggerEvent",
|
|
18
|
+
"detail": "r.job's `trigger.on` reached via an r.defineEvent QN (async, at-least-once\ndelivery through the job-trigger event consumer) had no way to key an\nidempotency check on the stored event that fired it. JobContext now\ncarries `triggerEvent?: { id, headers }` — the stored event's id and\nmetadata.headers — when the job was reached this way. Undefined for\ncron/manual jobs and for jobs triggered synchronously off a write/query\nhandler (no stored event to hand over). No migration needed."
|
|
19
|
+
},
|
|
2
20
|
{
|
|
3
21
|
"version": "0.312.0",
|
|
4
22
|
"type": "improvement",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
2
2
|
import postgres from "postgres";
|
|
3
|
+
import { DEFAULT_DB_CLOSE_TIMEOUT_SECONDS } from "../db/api";
|
|
3
4
|
import {
|
|
4
5
|
KeyAlreadyExistsError,
|
|
5
6
|
KeyErasedError,
|
|
@@ -172,7 +173,7 @@ export class PgKmsAdapter implements LocalKeyKmsAdapter {
|
|
|
172
173
|
}
|
|
173
174
|
|
|
174
175
|
async close(): Promise<void> {
|
|
175
|
-
await this.sql.end();
|
|
176
|
+
await this.sql.end({ timeout: DEFAULT_DB_CLOSE_TIMEOUT_SECONDS });
|
|
176
177
|
}
|
|
177
178
|
|
|
178
179
|
private ensureSchema(): Promise<void> {
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// A DB outage must not turn process shutdown into a hang: postgres.js keeps
|
|
2
|
+
// an in-flight query referenced on a dead socket, so a plain end() (no
|
|
3
|
+
// timeout = wait for queries to finish) never resolves once the DB is gone.
|
|
4
|
+
// close() has to bound that wait (db/api.ts's closeTimeoutSeconds). Routes
|
|
5
|
+
// createDbConnection through a TCP proxy that stops forwarding bytes after
|
|
6
|
+
// connect ("blackhole") — same simulated-outage approach as
|
|
7
|
+
// pipeline/__tests__/event-dispatcher-db-outage.integration.test.ts, but
|
|
8
|
+
// without touching the shared test DB's actual availability.
|
|
9
|
+
|
|
10
|
+
import { afterAll, describe, expect, test } from "bun:test";
|
|
11
|
+
import net from "node:net";
|
|
12
|
+
import { waitFor } from "../../testing";
|
|
13
|
+
import { testDatabaseUrl } from "../../testing/closed-connection-error";
|
|
14
|
+
import { createDbConnection } from "../connection";
|
|
15
|
+
|
|
16
|
+
function startBlackholeProxy(dbUrl: URL): {
|
|
17
|
+
readonly server: net.Server;
|
|
18
|
+
blackhole: () => void;
|
|
19
|
+
swallowedBytes: () => number;
|
|
20
|
+
destroy: () => void;
|
|
21
|
+
} {
|
|
22
|
+
let blackholed = false;
|
|
23
|
+
let swallowed = 0;
|
|
24
|
+
const sockets = new Set<net.Socket>();
|
|
25
|
+
const server = net.createServer((downstream) => {
|
|
26
|
+
sockets.add(downstream);
|
|
27
|
+
const upstream = net.connect(Number(dbUrl.port), dbUrl.hostname);
|
|
28
|
+
sockets.add(upstream);
|
|
29
|
+
downstream.on("data", (chunk: Buffer) => {
|
|
30
|
+
if (blackholed) {
|
|
31
|
+
swallowed += chunk.length;
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
upstream.write(chunk);
|
|
35
|
+
});
|
|
36
|
+
upstream.on("data", (chunk: Buffer) => {
|
|
37
|
+
if (!blackholed) downstream.write(chunk);
|
|
38
|
+
});
|
|
39
|
+
downstream.on("error", () => {});
|
|
40
|
+
upstream.on("error", () => {});
|
|
41
|
+
downstream.on("close", () => sockets.delete(downstream));
|
|
42
|
+
upstream.on("close", () => sockets.delete(upstream));
|
|
43
|
+
});
|
|
44
|
+
return {
|
|
45
|
+
server,
|
|
46
|
+
blackhole: () => {
|
|
47
|
+
blackholed = true;
|
|
48
|
+
},
|
|
49
|
+
swallowedBytes: () => swallowed,
|
|
50
|
+
destroy: () => {
|
|
51
|
+
for (const socket of sockets) socket.destroy();
|
|
52
|
+
server.close();
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function listenOn(server: net.Server, port: number): Promise<void> {
|
|
58
|
+
await new Promise<void>((resolve, reject) => {
|
|
59
|
+
server.once("error", reject);
|
|
60
|
+
server.listen(port, resolve);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
let proxy: ReturnType<typeof startBlackholeProxy>;
|
|
65
|
+
|
|
66
|
+
afterAll(() => {
|
|
67
|
+
proxy?.destroy();
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
describe("createDbConnection close() bounds shutdown during a DB outage", () => {
|
|
71
|
+
test("close() resolves within the bounded timeout even with in-flight queries on a dead socket", async () => {
|
|
72
|
+
const dbUrl = new URL(testDatabaseUrl());
|
|
73
|
+
proxy = startBlackholeProxy(dbUrl);
|
|
74
|
+
await listenOn(proxy.server, 0);
|
|
75
|
+
const address = proxy.server.address();
|
|
76
|
+
if (address === null || typeof address === "string") {
|
|
77
|
+
throw new Error("proxy did not bind to a TCP port");
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const proxiedUrl = new URL(dbUrl.toString());
|
|
81
|
+
proxiedUrl.hostname = "127.0.0.1";
|
|
82
|
+
proxiedUrl.port = String(address.port);
|
|
83
|
+
|
|
84
|
+
const conn = createDbConnection(proxiedUrl.toString(), {
|
|
85
|
+
maxConnections: 1,
|
|
86
|
+
closeTimeoutSeconds: 1,
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
// Establish the connection while the proxy still forwards normally —
|
|
90
|
+
// isolates the assertion to close()'s own timeout, not connect_timeout.
|
|
91
|
+
await conn.client.unsafe("select 1");
|
|
92
|
+
|
|
93
|
+
// Fired without awaiting: once the socket goes dead, end() has to
|
|
94
|
+
// reject these instead of waiting on them forever. .catch prevents an
|
|
95
|
+
// unhandled rejection when end() tears the connection down.
|
|
96
|
+
const inFlightBeforeBlackhole = conn.client.unsafe("select 1").catch(() => undefined);
|
|
97
|
+
proxy.blackhole();
|
|
98
|
+
const inFlightAfterBlackhole = conn.client.unsafe("select 1").catch(() => undefined);
|
|
99
|
+
|
|
100
|
+
await waitFor(() => proxy.swallowedBytes() > 0);
|
|
101
|
+
|
|
102
|
+
const start = Date.now();
|
|
103
|
+
await conn.close();
|
|
104
|
+
const elapsed = Date.now() - start;
|
|
105
|
+
|
|
106
|
+
expect(elapsed).toBeLessThan(3000);
|
|
107
|
+
|
|
108
|
+
await Promise.all([inFlightBeforeBlackhole, inFlightAfterBlackhole]);
|
|
109
|
+
});
|
|
110
|
+
});
|
package/src/db/api.ts
CHANGED
|
@@ -11,8 +11,14 @@ export type DbConnectionOptions = {
|
|
|
11
11
|
readonly maxConnections?: number;
|
|
12
12
|
readonly idleTimeoutSeconds?: number;
|
|
13
13
|
readonly connectTimeoutSeconds?: number;
|
|
14
|
+
readonly closeTimeoutSeconds?: number;
|
|
14
15
|
};
|
|
15
16
|
|
|
17
|
+
// A DB outage can leave an in-flight query referenced on a dead connection,
|
|
18
|
+
// so a plain end() (no timeout = wait for queries to finish) hangs forever —
|
|
19
|
+
// close() must always bound how long shutdown waits.
|
|
20
|
+
export const DEFAULT_DB_CLOSE_TIMEOUT_SECONDS = 5;
|
|
21
|
+
|
|
16
22
|
// Connection-Handle: db für Queries, client für Legacy-Zugriff (LISTEN/NOTIFY-Peer
|
|
17
23
|
// bei Bun.SQL), close für Pool-Shutdown.
|
|
18
24
|
export type DbPoolHandle = {
|
package/src/db/bun-provider.ts
CHANGED
|
@@ -8,7 +8,11 @@
|
|
|
8
8
|
// Bun.SQL hat kein LISTEN — postgres-js-Peer für event-dispatcher.
|
|
9
9
|
|
|
10
10
|
import postgres from "postgres";
|
|
11
|
-
import
|
|
11
|
+
import {
|
|
12
|
+
type DbConnectionOptions,
|
|
13
|
+
type DbPoolHandle,
|
|
14
|
+
DEFAULT_DB_CLOSE_TIMEOUT_SECONDS,
|
|
15
|
+
} from "./api";
|
|
12
16
|
|
|
13
17
|
export function createBunConnection(url: string, options: DbConnectionOptions = {}): DbPoolHandle {
|
|
14
18
|
const bunOpts: { max?: number; idleTimeout?: number; connectionTimeout?: number } = {};
|
|
@@ -25,13 +29,14 @@ export function createBunConnection(url: string, options: DbConnectionOptions =
|
|
|
25
29
|
pgOpts.connect_timeout = options.connectTimeoutSeconds;
|
|
26
30
|
const listenClient = postgres(url, pgOpts);
|
|
27
31
|
|
|
32
|
+
const closeTimeout = options.closeTimeoutSeconds ?? DEFAULT_DB_CLOSE_TIMEOUT_SECONDS;
|
|
28
33
|
return {
|
|
29
34
|
db,
|
|
30
35
|
client: listenClient,
|
|
31
36
|
listenClient,
|
|
32
37
|
close: async () => {
|
|
33
|
-
await db.end();
|
|
34
|
-
await listenClient.end();
|
|
38
|
+
await db.end({ timeout: closeTimeout });
|
|
39
|
+
await listenClient.end({ timeout: closeTimeout });
|
|
35
40
|
},
|
|
36
41
|
};
|
|
37
42
|
}
|
package/src/db/connection.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import type { DbConnection, PgClient } from "@cosmicdrift/kumiko-types/db-connection";
|
|
4
4
|
import postgres from "postgres";
|
|
5
5
|
import { readPositiveIntEnv } from "../utils/env-parse";
|
|
6
|
+
import { DEFAULT_DB_CLOSE_TIMEOUT_SECONDS } from "./api";
|
|
6
7
|
|
|
7
8
|
// Raw client types (postgres-js | Bun.SQL) — the name used across query/
|
|
8
9
|
// event-store/pipeline call sites. The structural pool handle from ./api is
|
|
@@ -33,9 +34,8 @@ export function createDbConnection(
|
|
|
33
34
|
return {
|
|
34
35
|
db: client,
|
|
35
36
|
client,
|
|
36
|
-
close:
|
|
37
|
-
|
|
38
|
-
},
|
|
37
|
+
close: () =>
|
|
38
|
+
client.end({ timeout: options.closeTimeoutSeconds ?? DEFAULT_DB_CLOSE_TIMEOUT_SECONDS }),
|
|
39
39
|
};
|
|
40
40
|
}
|
|
41
41
|
|
package/src/db/eagerload.ts
CHANGED
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
import { requestContext } from "../api/request-context";
|
|
27
27
|
import { collectPiiSubjectFields, configuredPiiSubjectKms, decryptPiiFieldValues } from "../crypto";
|
|
28
28
|
import { selectMany } from "../db/query";
|
|
29
|
+
import { parseRefTargetEntityName } from "../engine/parse-ref-target";
|
|
29
30
|
import type { EntityDefinition, FieldDefinition, ReferenceFieldDef } from "../engine/types";
|
|
30
31
|
import {
|
|
31
32
|
collectEncryptedFieldNames,
|
|
@@ -67,21 +68,13 @@ function isReferenceField(field: FieldDefinition): field is ReferenceFieldDef {
|
|
|
67
68
|
return field.type === "reference";
|
|
68
69
|
}
|
|
69
70
|
|
|
70
|
-
function parseRefEntity(raw: string): string {
|
|
71
|
-
// Same-feature ("user") oder cross-feature ("users:user") — wir
|
|
72
|
-
// brauchen nur den entity-name (Names sind global eindeutig in
|
|
73
|
-
// entityMap). Der feature-prefix dient nur der Author-Klarheit.
|
|
74
|
-
const idx = raw.indexOf(":");
|
|
75
|
-
return idx < 0 ? raw : raw.slice(idx + 1);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
71
|
export function collectReferenceFields(entity: EntityDefinition): readonly ReferenceFieldEntry[] {
|
|
79
72
|
const out: ReferenceFieldEntry[] = [];
|
|
80
73
|
for (const [fieldName, fieldDef] of Object.entries(entity.fields)) {
|
|
81
74
|
if (!isReferenceField(fieldDef)) continue;
|
|
82
75
|
out.push({
|
|
83
76
|
fieldName,
|
|
84
|
-
refEntityName:
|
|
77
|
+
refEntityName: parseRefTargetEntityName(fieldDef.entity),
|
|
85
78
|
multiple: fieldDef.multiple === true,
|
|
86
79
|
});
|
|
87
80
|
}
|
package/src/db/index.ts
CHANGED
|
@@ -114,7 +114,7 @@ export {
|
|
|
114
114
|
type PgErrorInfo,
|
|
115
115
|
} from "./pg-error";
|
|
116
116
|
export { acquireNamespacedAdvisoryLock } from "./queries/advisory-lock";
|
|
117
|
-
export { executeRawQuery } from "./queries/raw-sql";
|
|
117
|
+
export { executeRawQuery, executeRawQueryRead } from "./queries/raw-sql";
|
|
118
118
|
export type { SelectOptions, WhereObject, WhereValue } from "./query-api";
|
|
119
119
|
export {
|
|
120
120
|
asRawClient,
|
|
@@ -3,7 +3,11 @@
|
|
|
3
3
|
// `asRawClient(db)` wrappt .unsafe und .begin transparent.
|
|
4
4
|
|
|
5
5
|
import postgres from "postgres";
|
|
6
|
-
import
|
|
6
|
+
import {
|
|
7
|
+
type DbConnectionOptions,
|
|
8
|
+
type DbPoolHandle,
|
|
9
|
+
DEFAULT_DB_CLOSE_TIMEOUT_SECONDS,
|
|
10
|
+
} from "./api";
|
|
7
11
|
|
|
8
12
|
export function createPgConnection(url: string, options: DbConnectionOptions = {}): DbPoolHandle {
|
|
9
13
|
const pgOptions: Parameters<typeof postgres>[1] = {};
|
|
@@ -18,8 +22,7 @@ export function createPgConnection(url: string, options: DbConnectionOptions = {
|
|
|
18
22
|
return {
|
|
19
23
|
db: client,
|
|
20
24
|
client,
|
|
21
|
-
close:
|
|
22
|
-
|
|
23
|
-
},
|
|
25
|
+
close: () =>
|
|
26
|
+
client.end({ timeout: options.closeTimeoutSeconds ?? DEFAULT_DB_CLOSE_TIMEOUT_SECONDS }),
|
|
24
27
|
};
|
|
25
28
|
}
|
|
@@ -326,11 +326,12 @@ export async function selectConsumerForUpdate(
|
|
|
326
326
|
db: AnyDb,
|
|
327
327
|
name: string,
|
|
328
328
|
instanceId: string,
|
|
329
|
-
): Promise<
|
|
330
|
-
await asRawClient(db).unsafe(
|
|
329
|
+
): Promise<Record<string, unknown> | undefined> {
|
|
330
|
+
const rows = (await asRawClient(db).unsafe(
|
|
331
331
|
`SELECT * FROM "kumiko_event_consumers" WHERE "name" = $1 AND "instance_id" = $2 FOR UPDATE`,
|
|
332
332
|
[name, instanceId],
|
|
333
|
-
)
|
|
333
|
+
)) as ReadonlyArray<Record<string, unknown>>;
|
|
334
|
+
return rows[0];
|
|
334
335
|
}
|
|
335
336
|
|
|
336
337
|
export async function updateConsumerRebuildCursor(
|
|
@@ -115,6 +115,18 @@ describe("validateTransferGraph", () => {
|
|
|
115
115
|
expect(() => validate(chain)).not.toThrow();
|
|
116
116
|
});
|
|
117
117
|
|
|
118
|
+
// A feature-prefixed reference target ("<feature>:<entity>") must resolve
|
|
119
|
+
// to the same entity name as an unprefixed one, or a chain built entirely
|
|
120
|
+
// out of prefixed refs would silently never trip the depth limit.
|
|
121
|
+
test("counts a chain of feature-prefixed reference targets into the depth limit", () => {
|
|
122
|
+
const chain: Record<string, EntityDefinition> = { e0: entity({}) };
|
|
123
|
+
for (let i = 1; i <= MAX_TRANSFER_DEPTH + 1; i++) {
|
|
124
|
+
chain[`e${i}`] = entity({ references: { parentId: { entity: `graph:e${i - 1}` } } });
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
expect(() => validate(chain)).toThrow(/deeper than the 5-level limit.*e0 -> e1/s);
|
|
128
|
+
});
|
|
129
|
+
|
|
118
130
|
test("terminates on a reference cycle instead of reporting false depth", () => {
|
|
119
131
|
expect(() =>
|
|
120
132
|
validate({
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { parseRefTargetEntityName } from "../parse-ref-target";
|
|
1
2
|
import type { EntityDefinition, FeatureDefinition } from "../types";
|
|
2
3
|
|
|
3
4
|
// --- Transfer-graph boot validation (fw#3088) ---
|
|
@@ -39,7 +40,7 @@ function referenceTargets(entity: EntityDefinition): readonly string[] {
|
|
|
39
40
|
for (const field of Object.values(entity.fields)) {
|
|
40
41
|
if (field.type !== "reference") continue;
|
|
41
42
|
if (field.multiple === true) continue;
|
|
42
|
-
targets.push(field.entity);
|
|
43
|
+
targets.push(parseRefTargetEntityName(field.entity));
|
|
43
44
|
}
|
|
44
45
|
return targets;
|
|
45
46
|
}
|
package/src/engine/index.ts
CHANGED
|
@@ -264,6 +264,8 @@ export {
|
|
|
264
264
|
userCanReadFieldRow,
|
|
265
265
|
userCanWriteFieldRow,
|
|
266
266
|
} from "./ownership";
|
|
267
|
+
export type { ParsedRefTarget } from "./parse-ref-target";
|
|
268
|
+
export { parseRefTarget, parseRefTargetEntityName } from "./parse-ref-target";
|
|
267
269
|
export { buildPipelineSteps, stepsPipeline } from "./pipeline";
|
|
268
270
|
export { defineApply, defineMspApply, setFields } from "./projection-helpers";
|
|
269
271
|
export type { BuiltinQnType, ParsedQn, QnType } from "./qualified-name";
|
|
@@ -431,6 +433,7 @@ export type {
|
|
|
431
433
|
JobDefinition,
|
|
432
434
|
JobHandlerFn,
|
|
433
435
|
JobTrigger,
|
|
436
|
+
JobTriggerEvent,
|
|
434
437
|
JsonbFieldDef,
|
|
435
438
|
KumikoEntityTypeMap,
|
|
436
439
|
KumikoEventTypeMap,
|
|
@@ -20,3 +20,9 @@ export function parseRefTarget(raw: string, currentFeature: string): ParsedRefTa
|
|
|
20
20
|
}
|
|
21
21
|
return { featureName: raw.slice(0, idx), entityName: raw.slice(idx + 1) };
|
|
22
22
|
}
|
|
23
|
+
|
|
24
|
+
// Entity names are globally unique in the registry, so callers that only
|
|
25
|
+
// need the entity (not its owning feature) can drop the currentFeature arg.
|
|
26
|
+
export function parseRefTargetEntityName(raw: string): string {
|
|
27
|
+
return parseRefTarget(raw, "").entityName;
|
|
28
|
+
}
|
|
@@ -2,6 +2,7 @@ import { configureEventPiiCatalog } from "../crypto/event-pii";
|
|
|
2
2
|
import { hasSearchablePlaintext, isSensitiveLabelField } from "../db/entity-field-encryption";
|
|
3
3
|
import { bindHookEscapeHatchGrant } from "../pipeline/system-identity-switch";
|
|
4
4
|
import { resolveName } from "./handler-helpers";
|
|
5
|
+
import { parseRefTargetEntityName } from "./parse-ref-target";
|
|
5
6
|
import type {
|
|
6
7
|
RegistryState,
|
|
7
8
|
SearchableReferenceField,
|
|
@@ -192,16 +193,6 @@ export function applyExtensionUsages(state: RegistryState): void {
|
|
|
192
193
|
}
|
|
193
194
|
}
|
|
194
195
|
|
|
195
|
-
// "customer" (same-feature) or "users:customer" (cross-feature,
|
|
196
|
-
// "<featureName>:<entityName>") — only the entity name matters here, names
|
|
197
|
-
// are globally unique in entityMap. Local duplicate of the same parse done
|
|
198
|
-
// in db/eagerload.ts's parseRefEntity — this loop has no feature context to
|
|
199
|
-
// hand to engine/parse-ref-target.ts's parseRefTarget.
|
|
200
|
-
function parseReferenceTargetEntityName(raw: string): string {
|
|
201
|
-
const idx = raw.indexOf(":");
|
|
202
|
-
return idx < 0 ? raw : raw.slice(idx + 1);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
196
|
// fw#2660: labelField defaults to "id" (a UUID column) elsewhere, but a
|
|
206
197
|
// searchable reference ILIKEs the label column — an implicit or explicit
|
|
207
198
|
// "id" default would 500 on every search request. Fail at boot.
|
|
@@ -224,7 +215,7 @@ function buildSearchableReferenceField(
|
|
|
224
215
|
`(array) reference fields — remove "multiple" or "searchable".`,
|
|
225
216
|
);
|
|
226
217
|
}
|
|
227
|
-
const targetEntityName =
|
|
218
|
+
const targetEntityName = parseRefTargetEntityName(field.entity);
|
|
228
219
|
const targetEntity = entityMap.get(targetEntityName);
|
|
229
220
|
if (targetEntity !== undefined) {
|
|
230
221
|
if (
|
|
@@ -273,7 +264,7 @@ function buildSortableReferenceField(
|
|
|
273
264
|
}
|
|
274
265
|
return {
|
|
275
266
|
fieldName,
|
|
276
|
-
targetEntityName:
|
|
267
|
+
targetEntityName: parseRefTargetEntityName(field.entity),
|
|
277
268
|
labelField: field.labelField,
|
|
278
269
|
};
|
|
279
270
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// fileRefStorageDelta — the single source of truth the tenant-storage-usage
|
|
2
|
+
// MSP's apply handlers and transferTenantStorageUsage's replay both use to
|
|
3
|
+
// turn one fileRef event into a signed (bytes, files) delta.
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
import { entityEventName } from "../../db";
|
|
7
|
+
import { fileRefStorageDelta } from "../storage-tracking";
|
|
8
|
+
|
|
9
|
+
const CREATED = entityEventName("fileRef", "created");
|
|
10
|
+
const DELETED = entityEventName("fileRef", "deleted");
|
|
11
|
+
const RESTORED = entityEventName("fileRef", "restored");
|
|
12
|
+
|
|
13
|
+
describe("fileRefStorageDelta", () => {
|
|
14
|
+
test("created: positive delta from payload.size", () => {
|
|
15
|
+
expect(fileRefStorageDelta({ type: CREATED, payload: { size: 42 } })).toEqual({
|
|
16
|
+
bytes: 42,
|
|
17
|
+
files: 1,
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("created: missing/non-numeric size reads as 0 bytes, still counts a file", () => {
|
|
22
|
+
expect(fileRefStorageDelta({ type: CREATED, payload: {} })).toEqual({ bytes: 0, files: 1 });
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("deleted: negative delta from payload.previous.size", () => {
|
|
26
|
+
expect(fileRefStorageDelta({ type: DELETED, payload: { previous: { size: 42 } } })).toEqual({
|
|
27
|
+
bytes: -42,
|
|
28
|
+
files: -1,
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("deleted: missing previous reads as 0 bytes, still counts -1 file", () => {
|
|
33
|
+
expect(fileRefStorageDelta({ type: DELETED, payload: {} })).toEqual({ bytes: 0, files: -1 });
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test("restored: positive delta from payload.previous.size, symmetric to delete", () => {
|
|
37
|
+
expect(fileRefStorageDelta({ type: RESTORED, payload: { previous: { size: 42 } } })).toEqual({
|
|
38
|
+
bytes: 42,
|
|
39
|
+
files: 1,
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("an unrelated event type returns null", () => {
|
|
44
|
+
expect(fileRefStorageDelta({ type: "fileRef.renamed", payload: { size: 42 } })).toBeNull();
|
|
45
|
+
});
|
|
46
|
+
});
|
package/src/files/index.ts
CHANGED
|
@@ -26,7 +26,12 @@ export {
|
|
|
26
26
|
isFileProviderPlugin,
|
|
27
27
|
makeFileProviderResolver,
|
|
28
28
|
} from "./provider-resolver";
|
|
29
|
-
export {
|
|
29
|
+
export {
|
|
30
|
+
fileRefStorageDelta,
|
|
31
|
+
filesStorageTrackingFeature,
|
|
32
|
+
tenantStorageUsageTable,
|
|
33
|
+
transferTenantStorageUsage,
|
|
34
|
+
} from "./storage-tracking";
|
|
30
35
|
export type {
|
|
31
36
|
FileMetadata,
|
|
32
37
|
FileStorageProvider,
|
|
@@ -11,18 +11,28 @@
|
|
|
11
11
|
// consumer-cursor row. Apps that want it pass filesStorageTrackingFeature
|
|
12
12
|
// into createApp / setupTestStack alongside their domain features.
|
|
13
13
|
|
|
14
|
-
import { entityEventName } from "../db";
|
|
14
|
+
import { type DbRunner, entityEventName, executeRawQueryRead } from "../db";
|
|
15
15
|
import { bigint, instant, integer, table as pgTable, sql, uuid } from "../db/dialect";
|
|
16
16
|
import { incrementCounter } from "../db/query";
|
|
17
|
-
import { defineFeature } from "../engine";
|
|
17
|
+
import { defineFeature, qn, type Registry, toKebab } from "../engine";
|
|
18
|
+
import {
|
|
19
|
+
type PendingGapEntry,
|
|
20
|
+
SHARED_INSTANCE_SENTINEL,
|
|
21
|
+
selectConsumerCursorForUpdate,
|
|
22
|
+
} from "../pipeline";
|
|
23
|
+
import { parseJsonOrThrow } from "../utils";
|
|
24
|
+
|
|
25
|
+
const FILE_REF_AGGREGATE_TYPE = "fileRef";
|
|
26
|
+
const STORAGE_TRACKING_FEATURE_NAME = "files-storage-tracking";
|
|
27
|
+
const TENANT_STORAGE_USAGE_MSP_NAME = "tenant-storage-usage";
|
|
18
28
|
|
|
19
29
|
// fileRef is a standard ES entity, so usage tracking subscribes to its
|
|
20
30
|
// auto-verb events. `fileRef.created` payload carries the entity fields
|
|
21
31
|
// (incl. size); `fileRef.deleted` / `fileRef.restored` carry `{ previous }`
|
|
22
32
|
// (the pre-event row), so the byte count to apply lives at previous.size.
|
|
23
|
-
const FILE_REF_CREATED = entityEventName(
|
|
24
|
-
const FILE_REF_DELETED = entityEventName(
|
|
25
|
-
const FILE_REF_RESTORED = entityEventName(
|
|
33
|
+
const FILE_REF_CREATED = entityEventName(FILE_REF_AGGREGATE_TYPE, "created");
|
|
34
|
+
const FILE_REF_DELETED = entityEventName(FILE_REF_AGGREGATE_TYPE, "deleted");
|
|
35
|
+
const FILE_REF_RESTORED = entityEventName(FILE_REF_AGGREGATE_TYPE, "restored");
|
|
26
36
|
|
|
27
37
|
function readNumber(value: unknown): number {
|
|
28
38
|
return typeof value === "number" ? value : 0;
|
|
@@ -36,6 +46,28 @@ function sizeFromPrevious(payload: Record<string, unknown>): number {
|
|
|
36
46
|
return 0;
|
|
37
47
|
}
|
|
38
48
|
|
|
49
|
+
// Single source of truth for "how much does this fileRef event move the
|
|
50
|
+
// tenant's storage counters" — shared by the MSP's own apply handlers and by
|
|
51
|
+
// transferTenantStorageUsage's replay of already-applied events during a
|
|
52
|
+
// tenant handover.
|
|
53
|
+
export function fileRefStorageDelta(event: {
|
|
54
|
+
readonly type: string;
|
|
55
|
+
readonly payload: Record<string, unknown>;
|
|
56
|
+
}): { readonly bytes: number; readonly files: number } | null {
|
|
57
|
+
switch (event.type) {
|
|
58
|
+
case FILE_REF_CREATED:
|
|
59
|
+
return { bytes: readNumber(event.payload["size"]), files: 1 };
|
|
60
|
+
case FILE_REF_DELETED:
|
|
61
|
+
// `0 - x`, not `-x`: a zero-byte previous size must stay `0`, not `-0`
|
|
62
|
+
// (toEqual/JSON round-trips treat -0 as distinct from 0).
|
|
63
|
+
return { bytes: 0 - sizeFromPrevious(event.payload), files: -1 };
|
|
64
|
+
case FILE_REF_RESTORED:
|
|
65
|
+
return { bytes: sizeFromPrevious(event.payload), files: 1 };
|
|
66
|
+
default:
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
39
71
|
// bigint in `mode: "number"` returns a JS number (safe up to 2^53 ≈ 9e15
|
|
40
72
|
// bytes ≈ 8 petabytes per tenant — large enough for any practical storage
|
|
41
73
|
// quota). Default "bigint" mode would hand back a bigint value, which
|
|
@@ -48,13 +80,15 @@ export const tenantStorageUsageTable = pgTable("read_tenant_storage_usage", {
|
|
|
48
80
|
lastUpdatedAt: instant("last_updated_at").default(sql`now()`).notNull(),
|
|
49
81
|
});
|
|
50
82
|
|
|
51
|
-
export const filesStorageTrackingFeature = defineFeature(
|
|
83
|
+
export const filesStorageTrackingFeature = defineFeature(STORAGE_TRACKING_FEATURE_NAME, (r) => {
|
|
52
84
|
r.multiStreamProjection({
|
|
53
|
-
name:
|
|
85
|
+
name: TENANT_STORAGE_USAGE_MSP_NAME,
|
|
54
86
|
table: tenantStorageUsageTable,
|
|
55
87
|
apply: {
|
|
56
88
|
[FILE_REF_CREATED]: async (event, tx) => {
|
|
57
|
-
const
|
|
89
|
+
const delta = fileRefStorageDelta(event);
|
|
90
|
+
// skip: not a tracked fileRef storage change
|
|
91
|
+
if (!delta) return;
|
|
58
92
|
|
|
59
93
|
// UPSERT: INSERT on first upload per tenant, otherwise atomic increment.
|
|
60
94
|
// The SQL increment guarantees correctness under concurrent dispatcher
|
|
@@ -63,13 +97,15 @@ export const filesStorageTrackingFeature = defineFeature("files-storage-tracking
|
|
|
63
97
|
await incrementCounter(
|
|
64
98
|
tx,
|
|
65
99
|
tenantStorageUsageTable,
|
|
66
|
-
{ tenantId: event.tenantId, totalBytes:
|
|
67
|
-
{ totalBytes:
|
|
100
|
+
{ tenantId: event.tenantId, totalBytes: delta.bytes, fileCount: delta.files },
|
|
101
|
+
{ totalBytes: delta.bytes, fileCount: delta.files },
|
|
68
102
|
{ set: { lastUpdatedAt: sql`now()` } },
|
|
69
103
|
);
|
|
70
104
|
},
|
|
71
105
|
[FILE_REF_DELETED]: async (event, tx) => {
|
|
72
|
-
const
|
|
106
|
+
const delta = fileRefStorageDelta(event);
|
|
107
|
+
// skip: not a tracked fileRef storage change
|
|
108
|
+
if (!delta) return;
|
|
73
109
|
// Decrement on delete. INSERT values are 0/0 so a delete that somehow
|
|
74
110
|
// precedes any upload can't create negative usage; the on-conflict
|
|
75
111
|
// path applies the real negative delta. Async (dispatcher) —
|
|
@@ -78,7 +114,7 @@ export const filesStorageTrackingFeature = defineFeature("files-storage-tracking
|
|
|
78
114
|
tx,
|
|
79
115
|
tenantStorageUsageTable,
|
|
80
116
|
{ tenantId: event.tenantId, totalBytes: 0, fileCount: 0 },
|
|
81
|
-
{ totalBytes:
|
|
117
|
+
{ totalBytes: delta.bytes, fileCount: delta.files },
|
|
82
118
|
{ set: { lastUpdatedAt: sql`now()` } },
|
|
83
119
|
);
|
|
84
120
|
},
|
|
@@ -86,15 +122,124 @@ export const filesStorageTrackingFeature = defineFeature("files-storage-tracking
|
|
|
86
122
|
// Restore re-increments by the soft-deleted row's size — symmetric
|
|
87
123
|
// to delete. Without this handler totalBytes/fileCount drift low
|
|
88
124
|
// after every delete→restore round-trip.
|
|
89
|
-
const
|
|
125
|
+
const delta = fileRefStorageDelta(event);
|
|
126
|
+
// skip: not a tracked fileRef storage change
|
|
127
|
+
if (!delta) return;
|
|
90
128
|
await incrementCounter(
|
|
91
129
|
tx,
|
|
92
130
|
tenantStorageUsageTable,
|
|
93
|
-
{ tenantId: event.tenantId, totalBytes:
|
|
94
|
-
{ totalBytes:
|
|
131
|
+
{ tenantId: event.tenantId, totalBytes: delta.bytes, fileCount: delta.files },
|
|
132
|
+
{ totalBytes: delta.bytes, fileCount: delta.files },
|
|
95
133
|
{ set: { lastUpdatedAt: sql`now()` } },
|
|
96
134
|
);
|
|
97
135
|
},
|
|
98
136
|
},
|
|
99
137
|
});
|
|
100
138
|
});
|
|
139
|
+
|
|
140
|
+
// Qualified name of the tenant-storage-usage MSP consumer row in
|
|
141
|
+
// kumiko_event_consumers — the same qualification registry-ingest.ts applies
|
|
142
|
+
// to every multiStreamProjection (qualify(featureName, "projection", name)).
|
|
143
|
+
const TENANT_STORAGE_USAGE_MSP_QN = qn(
|
|
144
|
+
toKebab(STORAGE_TRACKING_FEATURE_NAME),
|
|
145
|
+
"projection",
|
|
146
|
+
toKebab(TENANT_STORAGE_USAGE_MSP_NAME),
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
const TRACKED_FILE_REF_EVENT_TYPES = [FILE_REF_CREATED, FILE_REF_DELETED, FILE_REF_RESTORED];
|
|
150
|
+
|
|
151
|
+
function isWithinPendingGap(eventId: bigint, gaps: readonly PendingGapEntry[]): boolean {
|
|
152
|
+
return gaps.some((gap) => eventId >= BigInt(gap.from) && eventId <= BigInt(gap.to));
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Moves the already-applied share of the tenant-storage-usage counters from
|
|
156
|
+
// source to destination tenant when a tenant-handover claim moves fileRef
|
|
157
|
+
// rows. Must run BEFORE moveEventHistory rewrites the events' tenant_id —
|
|
158
|
+
// the query below finds "already applied" events by sourceTenantId, and once
|
|
159
|
+
// the rewrite runs there is no way to tell an already-applied event from a
|
|
160
|
+
// not-yet-applied one by tenant_id alone.
|
|
161
|
+
//
|
|
162
|
+
// Events with id above the cursor (or inside a pending gap) are NOT summed
|
|
163
|
+
// here: the dispatcher hasn't applied them yet, they will carry the
|
|
164
|
+
// destination tenant_id once moveEventHistory runs, and the dispatcher will
|
|
165
|
+
// apply them against the destination counter on its own next pass.
|
|
166
|
+
export async function transferTenantStorageUsage(args: {
|
|
167
|
+
readonly db: DbRunner;
|
|
168
|
+
readonly registry: Registry;
|
|
169
|
+
readonly fileRefIds: readonly string[];
|
|
170
|
+
readonly sourceTenantId: string;
|
|
171
|
+
readonly destinationTenantId: string;
|
|
172
|
+
}): Promise<void> {
|
|
173
|
+
const { db, registry, fileRefIds, sourceTenantId, destinationTenantId } = args;
|
|
174
|
+
// skip: no fileRefs moved, nothing to transfer
|
|
175
|
+
if (fileRefIds.length === 0) return;
|
|
176
|
+
// skip: storage tracking is not mounted in this app
|
|
177
|
+
if (!registry.getAllMultiStreamProjections().has(TENANT_STORAGE_USAGE_MSP_QN)) return;
|
|
178
|
+
|
|
179
|
+
// Locks the same consumer row the dispatcher's own FOR UPDATE SKIP LOCKED
|
|
180
|
+
// pass takes before applying a batch — serializing against it here means no
|
|
181
|
+
// fileRef event for these ids can be applied between reading the cursor
|
|
182
|
+
// below and the tenant move that follows in the caller.
|
|
183
|
+
const cursor = await selectConsumerCursorForUpdate(
|
|
184
|
+
db,
|
|
185
|
+
TENANT_STORAGE_USAGE_MSP_QN,
|
|
186
|
+
SHARED_INSTANCE_SENTINEL,
|
|
187
|
+
);
|
|
188
|
+
// skip: no consumer row means the MSP never applied anything yet
|
|
189
|
+
if (!cursor) return;
|
|
190
|
+
|
|
191
|
+
const rows = await executeRawQueryRead<{ id: string; type: string; payload: string }>(
|
|
192
|
+
db,
|
|
193
|
+
`SELECT "id"::text AS id, "type", "payload"::text AS payload FROM kumiko_events
|
|
194
|
+
WHERE "tenant_id" = $1 AND "aggregate_type" = $2 AND "aggregate_id" = ANY($3)
|
|
195
|
+
AND "type" = ANY($4) AND "id" <= $5`,
|
|
196
|
+
[
|
|
197
|
+
sourceTenantId,
|
|
198
|
+
FILE_REF_AGGREGATE_TYPE,
|
|
199
|
+
fileRefIds,
|
|
200
|
+
TRACKED_FILE_REF_EVENT_TYPES,
|
|
201
|
+
cursor.lastProcessedEventId,
|
|
202
|
+
],
|
|
203
|
+
);
|
|
204
|
+
|
|
205
|
+
let bytes = 0;
|
|
206
|
+
let files = 0;
|
|
207
|
+
for (const row of rows) {
|
|
208
|
+
if (isWithinPendingGap(BigInt(row.id), cursor.pendingGaps)) continue;
|
|
209
|
+
const delta = fileRefStorageDelta({
|
|
210
|
+
type: row.type,
|
|
211
|
+
payload: parseJsonOrThrow<Record<string, unknown>>(
|
|
212
|
+
row.payload,
|
|
213
|
+
"transferTenantStorageUsage: kumiko_events.payload",
|
|
214
|
+
),
|
|
215
|
+
});
|
|
216
|
+
if (!delta) continue;
|
|
217
|
+
bytes += delta.bytes;
|
|
218
|
+
files += delta.files;
|
|
219
|
+
}
|
|
220
|
+
// skip: the moved fileRefs carry no applied storage share
|
|
221
|
+
if (bytes === 0 && files === 0) return;
|
|
222
|
+
|
|
223
|
+
// Source: insert-values stay 0/0 (same guard as the DELETED handler above)
|
|
224
|
+
// — the row this subtracts from is expected to already exist (the MSP
|
|
225
|
+
// already applied the create against it), and if it somehow doesn't, this
|
|
226
|
+
// must not conjure a negative balance out of thin air.
|
|
227
|
+
await incrementCounter(
|
|
228
|
+
db,
|
|
229
|
+
tenantStorageUsageTable,
|
|
230
|
+
{ tenantId: sourceTenantId, totalBytes: 0, fileCount: 0 },
|
|
231
|
+
{ totalBytes: -bytes, fileCount: -files },
|
|
232
|
+
{ set: { lastUpdatedAt: sql`now()` } },
|
|
233
|
+
);
|
|
234
|
+
// Destination: insert-values ARE the transferred amount (same as CREATED/
|
|
235
|
+
// RESTORED above) — a tenant claiming its first-ever fileRef must start
|
|
236
|
+
// its counter row at the transferred total, not at 0 with an increment
|
|
237
|
+
// that only fires once a row already exists.
|
|
238
|
+
await incrementCounter(
|
|
239
|
+
db,
|
|
240
|
+
tenantStorageUsageTable,
|
|
241
|
+
{ tenantId: destinationTenantId, totalBytes: bytes, fileCount: files },
|
|
242
|
+
{ totalBytes: bytes, fileCount: files },
|
|
243
|
+
{ set: { lastUpdatedAt: sql`now()` } },
|
|
244
|
+
);
|
|
245
|
+
}
|
package/src/jobs/job-runner.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
type JobContext,
|
|
14
14
|
type JobDefinition,
|
|
15
15
|
type JobRunIn,
|
|
16
|
+
type JobTriggerEvent,
|
|
16
17
|
type MemberReader,
|
|
17
18
|
type Registry,
|
|
18
19
|
type SessionUser,
|
|
@@ -42,6 +43,22 @@ import {
|
|
|
42
43
|
} from "../pipeline/write-origin";
|
|
43
44
|
import { bridgeStub } from "../testing/handler-context";
|
|
44
45
|
|
|
46
|
+
// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
|
|
47
|
+
// (a security-relevant escape hatch that must fail the run closed), a
|
|
48
|
+
// malformed `_triggerEvent` only costs an idempotency-key convenience — the
|
|
49
|
+
// run proceeds without it rather than failing over what would be a framework
|
|
50
|
+
// bug, not untrusted input.
|
|
51
|
+
function isJobTriggerEvent(value: unknown): value is JobTriggerEvent {
|
|
52
|
+
if (typeof value !== "object" || value === null) return false;
|
|
53
|
+
const candidate = value as Record<string, unknown>;
|
|
54
|
+
if (typeof candidate["id"] !== "string") return false;
|
|
55
|
+
const headers = candidate["headers"];
|
|
56
|
+
if (typeof headers !== "object" || headers === null) return false;
|
|
57
|
+
return Object.values(headers as Record<string, unknown>).every(
|
|
58
|
+
(v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean",
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
45
62
|
// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
|
|
46
63
|
function stampGatedWriteOrigin(data: Record<string, unknown>): void {
|
|
47
64
|
delete data["_writeOrigin"];
|
|
@@ -216,6 +233,7 @@ export type JobRunner = {
|
|
|
216
233
|
eventName: string,
|
|
217
234
|
payload: Record<string, unknown>,
|
|
218
235
|
user?: SessionUser,
|
|
236
|
+
triggerEvent?: JobTriggerEvent,
|
|
219
237
|
): Promise<void>;
|
|
220
238
|
// Wires JobContext.write/writeAs/queryAs to the real dispatcher — called
|
|
221
239
|
// once at boot, after the dispatcher exists (job-runner construction happens
|
|
@@ -759,6 +777,12 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
759
777
|
// don't dig through the raw payload themselves.
|
|
760
778
|
const triggerName = rawData["_triggerName"] as string | undefined; // @cast-boundary dynamic-key
|
|
761
779
|
|
|
780
|
+
// Same carry-through for `_triggerEvent` — handleEvent injects it when
|
|
781
|
+
// the job-trigger event consumer supplied the stored event that fired
|
|
782
|
+
// this run (see createJobTriggerEventConsumer).
|
|
783
|
+
const rawTriggerEvent = rawData["_triggerEvent"];
|
|
784
|
+
const triggerEvent = isJobTriggerEvent(rawTriggerEvent) ? rawTriggerEvent : undefined;
|
|
785
|
+
|
|
762
786
|
// Absent = legacy or ungated root; present but invalid fails the run closed.
|
|
763
787
|
const rawWriteOrigin = rawData["_writeOrigin"]; // @cast-boundary dynamic-key
|
|
764
788
|
let jobOrigin: WriteOrigin | undefined;
|
|
@@ -866,6 +890,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
866
890
|
triggeredBy: triggeredById !== null ? { id: triggeredById, tenantId } : null,
|
|
867
891
|
log: createJobLogger(logs),
|
|
868
892
|
...(triggerName !== undefined && { triggerName }),
|
|
893
|
+
...(triggerEvent !== undefined && { triggerEvent }),
|
|
869
894
|
write: (qn: string, payload: unknown) => {
|
|
870
895
|
if (!dispatchWriteRef) {
|
|
871
896
|
throw new Error(
|
|
@@ -1258,6 +1283,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1258
1283
|
eventName: string,
|
|
1259
1284
|
payload: Record<string, unknown>,
|
|
1260
1285
|
user?: SessionUser,
|
|
1286
|
+
triggerEvent?: JobTriggerEvent,
|
|
1261
1287
|
): Promise<void> {
|
|
1262
1288
|
const traceCtx = captureTraceContext(tracer);
|
|
1263
1289
|
// Same correlation propagation as dispatch(): events triggered from
|
|
@@ -1285,6 +1311,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1285
1311
|
// setzen wir es auch — kostet nichts und vereinfacht Handler-Code
|
|
1286
1312
|
// (kein "ist es Multi?"-Branch nötig).
|
|
1287
1313
|
data["_triggerName"] = eventName;
|
|
1314
|
+
if (triggerEvent) data["_triggerEvent"] = triggerEvent;
|
|
1288
1315
|
if (traceCtx) data[TRACE_CONTEXT_KEY] = traceCtx;
|
|
1289
1316
|
if (reqCtx?.correlationId) data["_correlationId"] = reqCtx.correlationId;
|
|
1290
1317
|
// Same maxPerTenant guard as dispatch — events that fan into many
|
|
@@ -195,3 +195,30 @@ describe("E: dispatcher survives a DB outage and logs the recovery", () => {
|
|
|
195
195
|
}
|
|
196
196
|
});
|
|
197
197
|
});
|
|
198
|
+
|
|
199
|
+
describe("F: stop() waits for a pass still mid-flight", () => {
|
|
200
|
+
test("stop() waits for a pass still in its idle pre-check", async () => {
|
|
201
|
+
const consumer: EventConsumer = {
|
|
202
|
+
name: "dboutage:consumer-stop-race",
|
|
203
|
+
handler: async () => {},
|
|
204
|
+
};
|
|
205
|
+
const dispatcher: EventDispatcher = createEventDispatcher({
|
|
206
|
+
db: stack.db,
|
|
207
|
+
consumers: [consumer],
|
|
208
|
+
context: { db: stack.db },
|
|
209
|
+
pollIntervalMs: 60_000,
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
await dispatcher.start();
|
|
213
|
+
let passSettled = false;
|
|
214
|
+
// A pass woken by runOnce() has not registered any turn yet while it is
|
|
215
|
+
// still awaiting the idle pre-check (selectIdleConsumerKeys) — stop()
|
|
216
|
+
// must wait for it too, or it queries the pool stop() is closing.
|
|
217
|
+
const pass = dispatcher.runOnce().finally(() => {
|
|
218
|
+
passSettled = true;
|
|
219
|
+
});
|
|
220
|
+
await dispatcher.stop();
|
|
221
|
+
expect(passSettled).toBe(true);
|
|
222
|
+
await pass;
|
|
223
|
+
});
|
|
224
|
+
});
|
|
@@ -19,7 +19,8 @@
|
|
|
19
19
|
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
20
20
|
import * as z from "zod";
|
|
21
21
|
import { entityEventName } from "../../db";
|
|
22
|
-
import { defineFeature } from "../../engine";
|
|
22
|
+
import { defineFeature, type JobContext } from "../../engine";
|
|
23
|
+
import { loadAggregate } from "../../event-store";
|
|
23
24
|
import { createInMemoryFileProvider, type InMemoryFileProvider } from "../../files";
|
|
24
25
|
import { setupTestStack, type TestStack, TestUsers } from "../../stack";
|
|
25
26
|
import { waitFor } from "../../testing";
|
|
@@ -27,8 +28,13 @@ import { generateId } from "../../utils";
|
|
|
27
28
|
|
|
28
29
|
const ITEM_REQUESTED_EVENT_QN = "job-trigger-fixture:event:item-requested";
|
|
29
30
|
const FILE_REF_CREATED = entityEventName("fileRef", "created");
|
|
31
|
+
const PROVIDER_EVENT_ID_HEADER = "providerEventId";
|
|
30
32
|
|
|
31
|
-
const processedItems: Array<{
|
|
33
|
+
const processedItems: Array<{
|
|
34
|
+
readonly fileRefId: string;
|
|
35
|
+
readonly triggerEvent: JobContext["triggerEvent"];
|
|
36
|
+
readonly rawDataKeys: readonly string[];
|
|
37
|
+
}> = [];
|
|
32
38
|
|
|
33
39
|
const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
|
|
34
40
|
r.defineEvent("item-requested", z.object({ fileRefId: z.string().min(1) }), {
|
|
@@ -37,7 +43,8 @@ const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
|
|
|
37
43
|
|
|
38
44
|
// Mirrors document-ingest-foundation's request-ingest MSP exactly: reacts
|
|
39
45
|
// to fileRef.created, appends a NEW event via unsafeAppendEvent — no
|
|
40
|
-
// write-handler behind the appended event itself.
|
|
46
|
+
// write-handler behind the appended event itself. headers carries a
|
|
47
|
+
// provider-style idempotency key so the job side can assert it round-trips.
|
|
41
48
|
r.multiStreamProjection({
|
|
42
49
|
name: "request-item",
|
|
43
50
|
apply: {
|
|
@@ -47,6 +54,7 @@ const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
|
|
|
47
54
|
aggregateType: "job-trigger-fixture-request",
|
|
48
55
|
type: ITEM_REQUESTED_EVENT_QN,
|
|
49
56
|
payload: { fileRefId: event.aggregateId },
|
|
57
|
+
headers: { [PROVIDER_EVENT_ID_HEADER]: "provider-evt-42" },
|
|
50
58
|
});
|
|
51
59
|
},
|
|
52
60
|
},
|
|
@@ -57,8 +65,12 @@ const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
|
|
|
57
65
|
r.job(
|
|
58
66
|
"process-item",
|
|
59
67
|
{ trigger: { on: ITEM_REQUESTED_EVENT_QN }, runIn: "worker" },
|
|
60
|
-
async (payload) => {
|
|
61
|
-
processedItems.push({
|
|
68
|
+
async (payload, ctx) => {
|
|
69
|
+
processedItems.push({
|
|
70
|
+
fileRefId: payload["fileRefId"] as string,
|
|
71
|
+
triggerEvent: ctx.triggerEvent,
|
|
72
|
+
rawDataKeys: Object.keys(payload),
|
|
73
|
+
});
|
|
62
74
|
},
|
|
63
75
|
);
|
|
64
76
|
});
|
|
@@ -169,7 +181,23 @@ describe("job-trigger event consumer", () => {
|
|
|
169
181
|
expect(processedItems).toHaveLength(1);
|
|
170
182
|
});
|
|
171
183
|
|
|
172
|
-
|
|
184
|
+
const processed = processedItems[0];
|
|
185
|
+
expect(processed?.fileRefId).toBeTruthy();
|
|
186
|
+
|
|
187
|
+
// aggregateId of the appended item-requested event is the fileRef's id
|
|
188
|
+
// (the MSP apply carries event.aggregateId straight through).
|
|
189
|
+
const fileRefId = processed?.fileRefId as string;
|
|
190
|
+
const storedEvents = await loadAggregate(stack.db, fileRefId, TestUsers.admin.tenantId);
|
|
191
|
+
const itemRequested = storedEvents.find((e) => e.type === ITEM_REQUESTED_EVENT_QN);
|
|
192
|
+
expect(itemRequested).toBeDefined();
|
|
193
|
+
|
|
194
|
+
// triggerEvent carries the stored event's own id and headers through to
|
|
195
|
+
// the job — the idempotency-key surface at-least-once delivery needs.
|
|
196
|
+
expect(processed?.triggerEvent?.id).toBe(itemRequested?.id);
|
|
197
|
+
expect(processed?.triggerEvent?.headers[PROVIDER_EVENT_ID_HEADER]).toBe("provider-evt-42");
|
|
198
|
+
|
|
199
|
+
// Reserved keys never leak into the payload the handler receives.
|
|
200
|
+
expect(processed?.rawDataKeys).not.toContain("_triggerEvent");
|
|
173
201
|
});
|
|
174
202
|
});
|
|
175
203
|
|
|
@@ -3,12 +3,13 @@
|
|
|
3
3
|
// resolve against (same #1438 dual-package-hazard pattern as event-store.ts).
|
|
4
4
|
import { Temporal as TemporalPolyfill } from "temporal-polyfill";
|
|
5
5
|
import { type RequestContextData, requestContext } from "../api/request-context";
|
|
6
|
-
import type { DbConnection, DbTx } from "../db/connection";
|
|
6
|
+
import type { DbConnection, DbRunner, DbTx } from "../db/connection";
|
|
7
7
|
import {
|
|
8
8
|
insertConsumerIfAbsent,
|
|
9
9
|
markConsumerProcessing,
|
|
10
10
|
rearmDeadConsumer,
|
|
11
11
|
recordConsumerPassFailure,
|
|
12
|
+
selectConsumerForUpdate,
|
|
12
13
|
selectConsumerForUpdateSkipLocked,
|
|
13
14
|
selectProvablyIdleConsumerPairs,
|
|
14
15
|
updateConsumerDeliveryOutcome,
|
|
@@ -160,6 +161,26 @@ export async function acquireConsumerState(
|
|
|
160
161
|
return { state, skip: null };
|
|
161
162
|
}
|
|
162
163
|
|
|
164
|
+
export type ConsumerCursor = {
|
|
165
|
+
readonly lastProcessedEventId: bigint;
|
|
166
|
+
readonly pendingGaps: readonly PendingGapEntry[];
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
// Blocking FOR UPDATE (not SKIP LOCKED): callers of this one need to actually
|
|
170
|
+
// wait for the dispatcher's own turn to finish, not skip past it, so their
|
|
171
|
+
// read of the already-applied cursor is guaranteed to be up to date rather
|
|
172
|
+
// than possibly stale.
|
|
173
|
+
export async function selectConsumerCursorForUpdate(
|
|
174
|
+
db: DbRunner,
|
|
175
|
+
name: string,
|
|
176
|
+
instanceId: string,
|
|
177
|
+
): Promise<ConsumerCursor | undefined> {
|
|
178
|
+
const rawState = await selectConsumerForUpdate(db, name, instanceId);
|
|
179
|
+
if (!rawState) return undefined;
|
|
180
|
+
const state = coerceRow(rawState, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow;
|
|
181
|
+
return { lastProcessedEventId: state.lastProcessedEventId, pendingGaps: state.pendingGaps };
|
|
182
|
+
}
|
|
183
|
+
|
|
163
184
|
// Read-only pre-check run once per doPass, before any consumer's turn opens
|
|
164
185
|
// a transaction (event-dispatcher.ts's doPass/runConsumerTurn). Keyed like
|
|
165
186
|
// consumerBackoff/inFlightTurns: `${name}:${instanceId}`. Consumers not
|
|
@@ -385,13 +385,34 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
|
|
|
385
385
|
);
|
|
386
386
|
}
|
|
387
387
|
|
|
388
|
+
// A pass still in doPass's idle pre-check (selectIdleConsumerKeys) hasn't
|
|
389
|
+
// registered any consumer turn in inFlightTurns yet — stop() must wait for
|
|
390
|
+
// it too, or it keeps querying the pool stop() is about to close.
|
|
391
|
+
const inFlightPasses = new Set<Promise<DispatcherPassResult>>();
|
|
392
|
+
|
|
393
|
+
async function drainInFlightPasses(): Promise<void> {
|
|
394
|
+
await Promise.all(
|
|
395
|
+
[...inFlightPasses].map((pass) =>
|
|
396
|
+
pass.catch(() => {
|
|
397
|
+
// skip: errors already recorded per-consumer inside the pass
|
|
398
|
+
}),
|
|
399
|
+
),
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
|
|
388
403
|
async function runOnce(): Promise<DispatcherPassResult> {
|
|
389
404
|
if (!preRegistered) {
|
|
390
405
|
throw new Error(
|
|
391
406
|
"EventDispatcher.runOnce() called before start() — consumer state rows are not registered. Call start() first (production) or ensureRegistered() (tests after truncating kumiko_event_consumers).",
|
|
392
407
|
);
|
|
393
408
|
}
|
|
394
|
-
|
|
409
|
+
const pass = doPass();
|
|
410
|
+
inFlightPasses.add(pass);
|
|
411
|
+
try {
|
|
412
|
+
return await pass;
|
|
413
|
+
} finally {
|
|
414
|
+
inFlightPasses.delete(pass);
|
|
415
|
+
}
|
|
395
416
|
}
|
|
396
417
|
|
|
397
418
|
async function doPass(): Promise<DispatcherPassResult> {
|
|
@@ -764,7 +785,9 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
|
|
|
764
785
|
emitEventDispatcherListenConnected(meter, false);
|
|
765
786
|
}
|
|
766
787
|
|
|
767
|
-
// Drain any in-flight pass so shutdown observes consistent state
|
|
788
|
+
// Drain any in-flight pass so shutdown observes consistent state —
|
|
789
|
+
// including one still stuck in the idle pre-check above inFlightTurns.
|
|
790
|
+
await drainInFlightPasses();
|
|
768
791
|
await drainInFlightTurns();
|
|
769
792
|
// preRegistered stays true — the rows survive stop(). runOnce()
|
|
770
793
|
// after a stop() still works (tests stop the timer and then drain
|
package/src/pipeline/index.ts
CHANGED
|
@@ -15,11 +15,12 @@ export type { DistributedLock } from "./distributed-lock";
|
|
|
15
15
|
export { createDistributedLock } from "./distributed-lock";
|
|
16
16
|
export type { EntityCache, EntityCacheOptions } from "./entity-cache";
|
|
17
17
|
export { createEntityCache } from "./entity-cache";
|
|
18
|
-
export type { ConsumerStatus } from "./event-consumer-state";
|
|
18
|
+
export type { ConsumerStatus, PendingGapEntry } from "./event-consumer-state";
|
|
19
19
|
export {
|
|
20
20
|
ConsumerStatuses,
|
|
21
21
|
createEventConsumerStateTable,
|
|
22
22
|
eventConsumerStateTable,
|
|
23
|
+
SHARED_INSTANCE_SENTINEL,
|
|
23
24
|
} from "./event-consumer-state";
|
|
24
25
|
export type { EventDedup } from "./event-dedup";
|
|
25
26
|
export { createEventDedup } from "./event-dedup";
|
|
@@ -43,6 +44,8 @@ export {
|
|
|
43
44
|
restartConsumer,
|
|
44
45
|
skipPoisonEvent,
|
|
45
46
|
} from "./event-dispatcher";
|
|
47
|
+
export type { ConsumerCursor } from "./event-dispatcher-delivery";
|
|
48
|
+
export { selectConsumerCursorForUpdate } from "./event-dispatcher-delivery";
|
|
46
49
|
export type { PruneEventsOptions, PruneEventsResult } from "./event-retention";
|
|
47
50
|
export { ConsumerLagError, pruneEvents } from "./event-retention";
|
|
48
51
|
export type { IdempotencyGuard } from "./idempotency";
|
|
@@ -542,7 +542,10 @@ export function createJobTriggerEventConsumer(
|
|
|
542
542
|
tenantId: event.tenantId,
|
|
543
543
|
roles: [],
|
|
544
544
|
};
|
|
545
|
-
await jobRunner.handleEvent(event.type, event.payload, user
|
|
545
|
+
await jobRunner.handleEvent(event.type, event.payload, user, {
|
|
546
|
+
id: event.id,
|
|
547
|
+
headers: event.metadata.headers ?? {},
|
|
548
|
+
});
|
|
546
549
|
},
|
|
547
550
|
};
|
|
548
551
|
}
|