@cosmicdrift/kumiko-framework 0.357.0 → 0.359.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crypto/pii-field-encryption.d.ts +1 -0
- package/dist/crypto/pii-field-encryption.js +1 -0
- package/dist/crypto/subject-resolver.d.ts +2 -0
- package/dist/crypto/subject-resolver.js +9 -6
- package/dist/db/connection.d.ts +2 -0
- package/dist/db/connection.js +10 -0
- package/dist/db/event-store-executor-context.d.ts +1 -0
- package/dist/db/event-store-executor-context.js +1 -0
- package/dist/db/event-store-executor-write.js +3 -0
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/queries/backfill-pii.js +58 -34
- package/dist/engine/boot-validator/pii-retention.js +7 -0
- package/dist/engine/boot-validator/screens.js +2 -1
- package/dist/engine/build-app-schema.js +10 -1
- package/dist/engine/factories.js +23 -19
- package/dist/engine/feature-ast/extractors/resolve-import.d.ts +3 -1
- package/dist/engine/feature-ast/extractors/resolve-import.js +9 -8
- package/dist/engine/feature-ast/extractors/round3.js +3 -2
- package/dist/engine/feature-ast/extractors/shared.d.ts +12 -2
- package/dist/engine/feature-ast/extractors/shared.js +31 -11
- package/dist/jobs/job-runner.d.ts +1 -0
- package/dist/jobs/job-runner.js +51 -13
- package/dist/pipeline/dispatch-shared.d.ts +1 -0
- package/dist/pipeline/dispatch-shared.js +1 -1
- package/dist/pipeline/dispatcher.d.ts +2 -0
- package/dist/pipeline/dispatcher.js +1 -0
- package/dist/stack/db.d.ts +3 -0
- package/dist/stack/db.js +7 -1
- package/dist/stack/test-stack.d.ts +4 -0
- package/dist/stack/test-stack.js +5 -0
- package/package.json +4 -4
- package/src/changes.json +34 -0
|
@@ -11,6 +11,7 @@ export interface EncryptPiiOptions {
|
|
|
11
11
|
readonly tenantId?: TenantId;
|
|
12
12
|
readonly subjectSource?: Record<string, unknown>;
|
|
13
13
|
readonly entityName: string;
|
|
14
|
+
readonly ownerFallback?: Record<string, unknown>;
|
|
14
15
|
}
|
|
15
16
|
export declare function encryptPiiFieldValues(row: Record<string, unknown>, entity: EntityDefinition, piiFields: readonly string[], kms: LocalKeyKmsAdapter, kmsCtx: KmsContext, opts: EncryptPiiOptions): Promise<Record<string, unknown>>;
|
|
16
17
|
export declare function serializeJsonbValue(field: string, value: unknown): string;
|
|
@@ -151,6 +151,7 @@ export async function encryptPiiFieldValues(row, entity, piiFields, kms, kmsCtx,
|
|
|
151
151
|
const subject = resolveSubjectForField(entity, name, subjectSource, {
|
|
152
152
|
entityName: opts.entityName,
|
|
153
153
|
...(opts.tenantId !== undefined && { tenantId: opts.tenantId }),
|
|
154
|
+
...(opts.ownerFallback !== undefined && { ownerFallback: opts.ownerFallback }),
|
|
154
155
|
});
|
|
155
156
|
// skip: collectPiiSubjectFields only yields annotated fields — null is unreachable, kept as a type guard
|
|
156
157
|
if (subject === null)
|
|
@@ -9,6 +9,8 @@ export declare class SubjectResolutionError extends Error {
|
|
|
9
9
|
export interface ResolveSubjectOptions {
|
|
10
10
|
readonly tenantId?: TenantId;
|
|
11
11
|
readonly entityName: string;
|
|
12
|
+
readonly ownerFallback?: Record<string, unknown>;
|
|
13
|
+
readonly ignoreWhenAbsent?: boolean;
|
|
12
14
|
}
|
|
13
15
|
/**
|
|
14
16
|
* Maps a pii-annotated field to the subject whose key encrypts it.
|
|
@@ -29,12 +29,15 @@ function resolveRecordSubject(fieldName, row, opts) {
|
|
|
29
29
|
}
|
|
30
30
|
return { kind: "record", entity: entityName, id: recordId };
|
|
31
31
|
}
|
|
32
|
-
function resolveUserOwnedSubject(
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
function resolveUserOwnedSubject(userOwned, fieldName, row, opts) {
|
|
33
|
+
const { ownerField } = userOwned;
|
|
34
|
+
const userId = nonEmptyString(row[ownerField]) ?? nonEmptyString(opts.ownerFallback?.[ownerField]);
|
|
35
|
+
if (userId !== null)
|
|
36
|
+
return { kind: "user", userId };
|
|
37
|
+
if (userOwned.whenAbsent === "tenant" && opts.ignoreWhenAbsent !== true) {
|
|
38
|
+
return resolveTenantSubject(fieldName, row, opts);
|
|
36
39
|
}
|
|
37
|
-
|
|
40
|
+
throw new SubjectResolutionError(fieldName, `owner field "${ownerField}" is empty on the row and the field declares no whenAbsent — declare { personal: { of: "${ownerField}", whenAbsent: "tenant" } } (fw#3654)`);
|
|
38
41
|
}
|
|
39
42
|
function resolveTenantSubject(fieldName, row, opts) {
|
|
40
43
|
const tenantId = nonEmptyString(row["tenantId"]) ?? opts.tenantId;
|
|
@@ -68,7 +71,7 @@ export function resolveSubjectForField(entity, fieldName, row, opts) {
|
|
|
68
71
|
return resolveRecordSubject(fieldName, row, opts);
|
|
69
72
|
}
|
|
70
73
|
if ("userOwned" in field && field.userOwned !== undefined) {
|
|
71
|
-
return resolveUserOwnedSubject(field.userOwned
|
|
74
|
+
return resolveUserOwnedSubject(field.userOwned, fieldName, row, opts);
|
|
72
75
|
}
|
|
73
76
|
if ("tenantOwned" in field && field.tenantOwned === true) {
|
|
74
77
|
return resolveTenantSubject(fieldName, row, opts);
|
package/dist/db/connection.d.ts
CHANGED
|
@@ -8,3 +8,5 @@ export declare function createDbConnection(url: string, options?: import("./api.
|
|
|
8
8
|
close: () => Promise<void>;
|
|
9
9
|
};
|
|
10
10
|
export declare function dbConnectionOptionsFromEnv(env?: Readonly<Record<string, string | undefined>>): import("./api.js").DbConnectionOptions;
|
|
11
|
+
export declare const DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX = 3;
|
|
12
|
+
export declare function outsideTransactionPoolOptionsFromEnv(env?: Readonly<Record<string, string | undefined>>): import("./api.js").DbConnectionOptions;
|
package/dist/db/connection.js
CHANGED
|
@@ -33,3 +33,13 @@ export function dbConnectionOptionsFromEnv(env = process.env) {
|
|
|
33
33
|
opts.connectTimeoutSeconds = connect;
|
|
34
34
|
return opts;
|
|
35
35
|
}
|
|
36
|
+
// Outside-tx access runs on its own pool: sharing the write pool deadlocks once
|
|
37
|
+
// pool-max concurrent writes each hold a tx and then wait for a second connection.
|
|
38
|
+
export const DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX = 3;
|
|
39
|
+
export function outsideTransactionPoolOptionsFromEnv(env = process.env) {
|
|
40
|
+
return {
|
|
41
|
+
...dbConnectionOptionsFromEnv(env),
|
|
42
|
+
maxConnections: readPositiveIntEnv(env, "DATABASE_OUTSIDE_TX_POOL_MAX") ??
|
|
43
|
+
DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -62,6 +62,7 @@ export type ExecutorContext = {
|
|
|
62
62
|
onlyKeys?: Iterable<string>;
|
|
63
63
|
subjectSource?: Record<string, unknown>;
|
|
64
64
|
keyTenantId?: TenantId | undefined;
|
|
65
|
+
ownerFallback?: Record<string, unknown>;
|
|
65
66
|
}) => Promise<Record<string, unknown>>;
|
|
66
67
|
readonly decryptForRead: (row: Record<string, unknown>, opts?: DecryptForReadOptions) => Promise<Record<string, unknown>>;
|
|
67
68
|
readonly applyDefaults: (payload: Record<string, unknown>) => Record<string, unknown>;
|
|
@@ -208,6 +208,7 @@ export function buildExecutorContext(table, entity, options) {
|
|
|
208
208
|
entityName,
|
|
209
209
|
...(opts?.onlyKeys !== undefined && { onlyKeys: opts.onlyKeys }),
|
|
210
210
|
...(opts?.subjectSource !== undefined && { subjectSource: opts.subjectSource }),
|
|
211
|
+
...(opts?.ownerFallback !== undefined && { ownerFallback: opts.ownerFallback }),
|
|
211
212
|
});
|
|
212
213
|
}
|
|
213
214
|
catch (e) {
|
|
@@ -464,8 +464,11 @@ export function createWriteVerbs(ctx) {
|
|
|
464
464
|
// re-encrypt it before it's persisted so plaintext of pii/encrypted
|
|
465
465
|
// fields doesn't land in the immutable log (flatChanges is already
|
|
466
466
|
// ciphertext from encryptForStorage above).
|
|
467
|
+
// Owner chain for the ownerless-then-assigned row: previous owner, then the
|
|
468
|
+
// owner this event sets, then whenAbsent (fw#3654).
|
|
467
469
|
const encryptedPrevious = await encryptForStorage(previous, user, {
|
|
468
470
|
keyTenantId: streamTenantId,
|
|
471
|
+
ownerFallback: submittedChanges,
|
|
469
472
|
});
|
|
470
473
|
// Savepoint-scoped — see the create() append() above for why:
|
|
471
474
|
// confines a losing writer's unique-violation to a nested scope
|
package/dist/db/index.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ export { collectTableMetas } from "./collect-table-metas.js";
|
|
|
4
4
|
export { flattenCompoundTypes, rehydrateCompoundTypes } from "./compound-types.js";
|
|
5
5
|
export { seedConfigValues } from "./config-seed.js";
|
|
6
6
|
export type { DbConnection, DbConnectionOptions, DbPoolHandle, DbRow, DbRunner, DbTx, } from "./connection.js";
|
|
7
|
-
export { createDbConnection, dbConnectionOptionsFromEnv } from "./connection.js";
|
|
7
|
+
export { createDbConnection, DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX, dbConnectionOptionsFromEnv, outsideTransactionPoolOptionsFromEnv, } from "./connection.js";
|
|
8
8
|
export type { CursorQueryOptions, CursorResult, DecodedKeysetCursor } from "./cursor.js";
|
|
9
9
|
export { decodeCursor, decodeKeysetCursor, encodeCursor, encodeKeysetCursor } from "./cursor.js";
|
|
10
10
|
export type { SchemaTable, SelectQuery, TableColumns } from "./dialect.js";
|
package/dist/db/index.js
CHANGED
|
@@ -3,7 +3,7 @@ export { nullBlindIndexesForSubject, recordEventsOwnedExclusivelyByTenant, recor
|
|
|
3
3
|
export { collectTableMetas } from "./collect-table-metas.js";
|
|
4
4
|
export { flattenCompoundTypes, rehydrateCompoundTypes } from "./compound-types.js";
|
|
5
5
|
export { seedConfigValues } from "./config-seed.js";
|
|
6
|
-
export { createDbConnection, dbConnectionOptionsFromEnv } from "./connection.js";
|
|
6
|
+
export { createDbConnection, DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX, dbConnectionOptionsFromEnv, outsideTransactionPoolOptionsFromEnv, } from "./connection.js";
|
|
7
7
|
export { decodeCursor, decodeKeysetCursor, encodeCursor, encodeKeysetCursor } from "./cursor.js";
|
|
8
8
|
export { bigint, bigserial, boolean, extractTableName, index, instant, instantToDriver, integer, jsonb, moneyAmount, numeric, primaryKey, serial, sql, table, text, timestamp, uniqueIndex, uuid, } from "./dialect.js";
|
|
9
9
|
export { collectReferenceFields, enrichRowWithReferences, enrichWithReferences, } from "./eagerload.js";
|
|
@@ -25,17 +25,22 @@
|
|
|
25
25
|
// second run reports 0 updates. One failing event does not abort the run;
|
|
26
26
|
// failures are collected and reported (fail-loud at the caller).
|
|
27
27
|
//
|
|
28
|
-
// Owner resolution for entity-lifecycle events
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
28
|
+
// Owner resolution for entity-lifecycle events. The live write path
|
|
29
|
+
// (event-store-executor-write.ts) resolves the same chain via
|
|
30
|
+
// subject-resolver.ts: row owner, then the owner the same event sets, then
|
|
31
|
+
// the field's whenAbsent (fw#3654). Backfill adds two opt-in stages and
|
|
32
|
+
// delays whenAbsent behind the projection so a known owner is never shadowed:
|
|
33
|
+
// 1. payload — the event section (plus the owner the same event sets) names
|
|
34
|
+
// the owner field; whenAbsent is not applied yet.
|
|
32
35
|
// 2. projection ({@link PiiBackfillOptions.resolveOwnerFromProjection}) —
|
|
33
36
|
// pre-owner-field-addition events don't carry it; read it from the
|
|
34
37
|
// entity's current projection row instead.
|
|
38
|
+
// 2b. whenAbsent — the payload again, now falling back to the tenant key for
|
|
39
|
+
// fields that declare it.
|
|
35
40
|
// 3. erase ({@link PiiBackfillOptions.eraseUnresolvableSubjects}) — still
|
|
36
41
|
// unresolvable (hard-deleted aggregate, no projection) → PII_ERASED_SENTINEL.
|
|
37
|
-
//
|
|
38
|
-
// subject still fails loud into `failures`, as before.
|
|
42
|
+
// Stages 2 and 3 are opt-in and off by default; without them an
|
|
43
|
+
// unresolvable subject still fails loud into `failures`, as before.
|
|
39
44
|
//
|
|
40
45
|
// Snapshots of touched aggregates are dropped per batch (they may cache
|
|
41
46
|
// plaintext); the next snapshotting load recreates them. AFTER a run, rebuild the
|
|
@@ -293,14 +298,7 @@ export async function backfillEventPiiEncryptionBatch(db, registry, options = {}
|
|
|
293
298
|
async function applyEntityLifecycleFields(target, projectionOwnerRow) {
|
|
294
299
|
const sections = lifecycleSections(payload);
|
|
295
300
|
for (const section of sections) {
|
|
296
|
-
|
|
297
|
-
// resolve subjects from the merged view; aggregate_id backs the
|
|
298
|
-
// self-subject when a section lacks the id column.
|
|
299
|
-
const subjectSource = {
|
|
300
|
-
id: row.aggregate_id,
|
|
301
|
-
...Object.assign({}, ...sections),
|
|
302
|
-
...section,
|
|
303
|
-
};
|
|
301
|
+
const { subjectSource, ownerFallback } = lifecycleSubjectView(section, payload, row.aggregate_id);
|
|
304
302
|
for (const field of target.piiFields) {
|
|
305
303
|
const value = section[field];
|
|
306
304
|
if (value === null || value === undefined)
|
|
@@ -316,7 +314,7 @@ export async function backfillEventPiiEncryptionBatch(db, registry, options = {}
|
|
|
316
314
|
// must stay untouched, not get erased by stage 3.
|
|
317
315
|
if (isPiiCiphertext(value) || value === PII_ERASED_SENTINEL)
|
|
318
316
|
continue;
|
|
319
|
-
const resolution = resolveOwnerSubject(target.entity, field, subjectSource, projectionOwnerRow,
|
|
317
|
+
const resolution = resolveOwnerSubject(target.entity, field, subjectSource, ownerFallback, projectionOwnerRow,
|
|
320
318
|
// @cast-boundary db-read — tenant_id column is the branded TenantId
|
|
321
319
|
row.tenant_id, row.aggregate_type);
|
|
322
320
|
if (resolution.kind === "unannotated")
|
|
@@ -401,32 +399,41 @@ export async function backfillEventPiiEncryptionBatch(db, registry, options = {}
|
|
|
401
399
|
return scanCache.destroyedTenants;
|
|
402
400
|
}
|
|
403
401
|
}
|
|
404
|
-
|
|
405
|
-
// (erase) is the caller's call, made from the "unresolved" outcome.
|
|
406
|
-
function resolveOwnerSubject(entity, field, subjectSource, projectionOwnerRow, tenantId, entityName) {
|
|
402
|
+
function attemptResolve(entity, field, source, opts) {
|
|
407
403
|
try {
|
|
408
|
-
|
|
409
|
-
return subject === null
|
|
410
|
-
? { kind: "unannotated" }
|
|
411
|
-
: { kind: "resolved", subject, viaProjection: false };
|
|
404
|
+
return { ok: true, subject: resolveSubjectForField(entity, field, source, opts) };
|
|
412
405
|
}
|
|
413
406
|
catch (e) {
|
|
414
407
|
if (!(e instanceof SubjectResolutionError))
|
|
415
408
|
throw e;
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
409
|
+
return { ok: false, error: e };
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
// Stage 1 (payload) and stage 2 (projection row, when supplied) resolve the
|
|
413
|
+
// owner strictly, ignoring whenAbsent — an ownerless payload must not shadow
|
|
414
|
+
// an owner the projection knows. Stage 2b then applies whenAbsent to the
|
|
415
|
+
// payload. Stage 3 (erase) is the caller's call, made from "unresolved".
|
|
416
|
+
function resolveOwnerSubject(entity, field, subjectSource, ownerFallback, projectionOwnerRow, tenantId, entityName) {
|
|
417
|
+
const baseOpts = { tenantId, entityName, ...(ownerFallback && { ownerFallback }) };
|
|
418
|
+
const strictOpts = { ...baseOpts, ignoreWhenAbsent: true };
|
|
419
|
+
const payloadAttempt = attemptResolve(entity, field, subjectSource, strictOpts);
|
|
420
|
+
if (payloadAttempt.ok) {
|
|
421
|
+
return payloadAttempt.subject === null
|
|
422
|
+
? { kind: "unannotated" }
|
|
423
|
+
: { kind: "resolved", subject: payloadAttempt.subject, viaProjection: false };
|
|
424
|
+
}
|
|
425
|
+
if (projectionOwnerRow) {
|
|
426
|
+
const augmented = { ...subjectSource, ...projectionOwnerRow };
|
|
427
|
+
const projectionAttempt = attemptResolve(entity, field, augmented, strictOpts);
|
|
428
|
+
if (projectionAttempt.ok && projectionAttempt.subject !== null) {
|
|
429
|
+
return { kind: "resolved", subject: projectionAttempt.subject, viaProjection: true };
|
|
427
430
|
}
|
|
428
|
-
return { kind: "unresolved", error: e };
|
|
429
431
|
}
|
|
432
|
+
const whenAbsentAttempt = attemptResolve(entity, field, subjectSource, baseOpts);
|
|
433
|
+
if (whenAbsentAttempt.ok && whenAbsentAttempt.subject !== null) {
|
|
434
|
+
return { kind: "resolved", subject: whenAbsentAttempt.subject, viaProjection: false };
|
|
435
|
+
}
|
|
436
|
+
return { kind: "unresolved", error: payloadAttempt.error };
|
|
430
437
|
}
|
|
431
438
|
// Projection columns are snake_case (author_id); event-payload/subject
|
|
432
439
|
// fields are camelCase (authorId) — forward-map from entity.fields like
|
|
@@ -460,6 +467,23 @@ function lifecycleSections(payload) {
|
|
|
460
467
|
sections.push(payload);
|
|
461
468
|
return sections;
|
|
462
469
|
}
|
|
470
|
+
// Same view the live write path resolves from: changes see the merged row,
|
|
471
|
+
// previous sees the owner the same event sets as fallback. aggregate_id backs
|
|
472
|
+
// the self-subject when a section lacks the id column.
|
|
473
|
+
function lifecycleSubjectView(section, payload, aggregateId) {
|
|
474
|
+
const changes = isRecord(payload["changes"]) ? payload["changes"] : undefined;
|
|
475
|
+
const previous = isRecord(payload["previous"]) ? payload["previous"] : undefined;
|
|
476
|
+
if (section === changes) {
|
|
477
|
+
return {
|
|
478
|
+
subjectSource: { id: aggregateId, ...previous, ...changes },
|
|
479
|
+
ownerFallback: undefined,
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
return {
|
|
483
|
+
subjectSource: { id: aggregateId, ...section },
|
|
484
|
+
ownerFallback: section === previous ? changes : undefined,
|
|
485
|
+
};
|
|
486
|
+
}
|
|
463
487
|
function isRecord(value) {
|
|
464
488
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
465
489
|
}
|
|
@@ -203,6 +203,13 @@ export function validatePiiAndRetention(feature) {
|
|
|
203
203
|
if (ownerField.type !== "reference" && ownerField.type !== "text") {
|
|
204
204
|
throw new Error(`[Feature ${feature.name}] personal.of "${ownerName}" on entity "${entityName}" must be a reference or text (userId) field, got type "${ownerField.type}"`);
|
|
205
205
|
}
|
|
206
|
+
const { whenAbsent } = annot.userOwned;
|
|
207
|
+
if (whenAbsent !== undefined && whenAbsent !== "tenant") {
|
|
208
|
+
throw new Error(`[Feature ${feature.name}] Field "${fieldName}" on entity "${entityName}" declares personal.whenAbsent "${String(whenAbsent)}" — entity fields only support whenAbsent: "tenant" (no plaintext fallback for stored PII).`);
|
|
209
|
+
}
|
|
210
|
+
if (ownerField.required !== true && whenAbsent === undefined) {
|
|
211
|
+
throw new Error(`[Feature ${feature.name}] Field "${fieldName}" on entity "${entityName}" is personal: { of: "${ownerName}" }, but "${ownerName}" is not required — rows without an owner would fail to encrypt (SubjectResolutionError, HTTP 500) on update, delete and forget. Declare { personal: { of: "${ownerName}", whenAbsent: "tenant" } } to encrypt those rows under the tenant key, or make "${ownerName}" required: true. Entity fields have no plaintext fallback (fw#3654).`);
|
|
212
|
+
}
|
|
206
213
|
// Soft-Warning wenn das reference-target nicht offensichtlich user
|
|
207
214
|
// ist — custom subject-entities (HR-Mitarbeiter, Patient) sind
|
|
208
215
|
// erlaubt, müssen aber bewusste Wahl sein.
|
|
@@ -1874,7 +1874,7 @@ function validateDashboardChartDisplay(featureName, screenId, panel) {
|
|
|
1874
1874
|
"from-today",
|
|
1875
1875
|
]);
|
|
1876
1876
|
validateStackedAreaEnumProp(where, "markerLegend", panel.markerLegend, ["list", "legend"]);
|
|
1877
|
-
const nonBooleanProp = ["brush", "legendTotals"].find((prop) => panel[prop] !== undefined && typeof panel[prop] !== "boolean");
|
|
1877
|
+
const nonBooleanProp = ["brush", "legendTotals", "markerLabels"].find((prop) => panel[prop] !== undefined && typeof panel[prop] !== "boolean");
|
|
1878
1878
|
if (nonBooleanProp !== undefined) {
|
|
1879
1879
|
throw new Error(`${where} has a non-boolean ${nonBooleanProp}.`);
|
|
1880
1880
|
}
|
|
@@ -1909,6 +1909,7 @@ function validateStackedAreaOnlyProps(where, panel) {
|
|
|
1909
1909
|
"dateFormat",
|
|
1910
1910
|
"initialWindow",
|
|
1911
1911
|
"markerLegend",
|
|
1912
|
+
"markerLabels",
|
|
1912
1913
|
].filter((prop) => panel[prop] !== undefined);
|
|
1913
1914
|
if (stackedAreaOnly.length > 0) {
|
|
1914
1915
|
throw new Error(`${where} sets ${stackedAreaOnly.join(", ")} on chart "${panel.chart}" — only "stacked-area" supports it.`);
|
|
@@ -281,7 +281,16 @@ function projectScreens(featureName, screens, registry, urlPrefillFieldsByScreen
|
|
|
281
281
|
...entityEditWriteExcludedFields(featureName, screen.entity, registry),
|
|
282
282
|
};
|
|
283
283
|
}
|
|
284
|
-
if (screen.type === "actionForm"
|
|
284
|
+
if (screen.type === "actionForm") {
|
|
285
|
+
const { writeExcludedFields: _authoredExclusions, ...authoredScreen } = screen;
|
|
286
|
+
const excluded = registry.getWriteHandler(screen.handler)?.excludedFields;
|
|
287
|
+
return {
|
|
288
|
+
...authoredScreen,
|
|
289
|
+
urlPrefillFields: sortedUrlPrefillFields(urlPrefillFieldsByScreenQn.get(qualifyEntityName(featureName, "screen", shortId))),
|
|
290
|
+
...(excluded !== undefined && { writeExcludedFields: excluded }),
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
if (screen.type === "secretMint") {
|
|
285
294
|
return {
|
|
286
295
|
...screen,
|
|
287
296
|
urlPrefillFields: sortedUrlPrefillFields(urlPrefillFieldsByScreenQn.get(qualifyEntityName(featureName, "screen", shortId))),
|
package/dist/engine/factories.js
CHANGED
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
// `personal: { of: "id" }` means the row itself is the subject, not a
|
|
2
2
|
// referenced user — resolves to `recordOwned` instead of `userOwned`.
|
|
3
3
|
const RECORD_OWNER_FIELD = "id";
|
|
4
|
+
function resolvePersonalFlags(personal, reason) {
|
|
5
|
+
if (personal === "self")
|
|
6
|
+
return { pii: true };
|
|
7
|
+
if (personal === "tenant")
|
|
8
|
+
return { tenantOwned: true };
|
|
9
|
+
if (personal === "ref")
|
|
10
|
+
return { subjectRef: true };
|
|
11
|
+
if (personal === false)
|
|
12
|
+
return { allowPlaintext: reason };
|
|
13
|
+
if (!personal)
|
|
14
|
+
return {};
|
|
15
|
+
if (personal.of === RECORD_OWNER_FIELD) {
|
|
16
|
+
if (personal.whenAbsent !== undefined) {
|
|
17
|
+
throw new Error(`personal: { of: "${RECORD_OWNER_FIELD}", whenAbsent } is meaningless: a row always has an id, so there is never an absent owner (fw#3654)`);
|
|
18
|
+
}
|
|
19
|
+
return { recordOwned: true };
|
|
20
|
+
}
|
|
21
|
+
const { whenAbsent } = personal;
|
|
22
|
+
return {
|
|
23
|
+
userOwned: { ownerField: personal.of, ...(whenAbsent !== undefined && { whenAbsent }) },
|
|
24
|
+
};
|
|
25
|
+
}
|
|
4
26
|
// Resolves the author-facing `personal`/`find` annotations (kumiko-framework#2250)
|
|
5
27
|
// into the internal ResolvedPiiFlags every factory below merges into its
|
|
6
28
|
// return value. `personal`, `find`, `reason` never survive into the field def.
|
|
@@ -9,25 +31,7 @@ function expandPersonalAnnotations(overrides) {
|
|
|
9
31
|
return {};
|
|
10
32
|
}
|
|
11
33
|
const { personal, find, reason, anonymize, ...rest } = overrides;
|
|
12
|
-
|
|
13
|
-
if (personal === "self") {
|
|
14
|
-
personalFlags = { pii: true };
|
|
15
|
-
}
|
|
16
|
-
else if (personal === "tenant") {
|
|
17
|
-
personalFlags = { tenantOwned: true };
|
|
18
|
-
}
|
|
19
|
-
else if (personal === "ref") {
|
|
20
|
-
personalFlags = { subjectRef: true };
|
|
21
|
-
}
|
|
22
|
-
else if (personal === false) {
|
|
23
|
-
personalFlags = { allowPlaintext: reason };
|
|
24
|
-
}
|
|
25
|
-
else if (personal && personal.of === RECORD_OWNER_FIELD) {
|
|
26
|
-
personalFlags = { recordOwned: true };
|
|
27
|
-
}
|
|
28
|
-
else if (personal) {
|
|
29
|
-
personalFlags = { userOwned: { ownerField: personal.of } };
|
|
30
|
-
}
|
|
34
|
+
const personalFlags = resolvePersonalFlags(personal, reason);
|
|
31
35
|
// `searchable`/`sensitive` live on the individual FieldDef, not on
|
|
32
36
|
// ResolvedPiiFlags — "fuzzy"/"secret" resolve into those too.
|
|
33
37
|
let findFlags = {};
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { SourceFile, VariableDeclaration } from "ts-morph";
|
|
2
2
|
import { Node } from "ts-morph";
|
|
3
|
-
export declare function resolveModuleFile(fromFile: SourceFile, specifier: string
|
|
3
|
+
export declare function resolveModuleFile(fromFile: SourceFile, specifier: string, { allowDeclarationFiles }?: {
|
|
4
|
+
readonly allowDeclarationFiles?: boolean;
|
|
5
|
+
}): SourceFile | undefined;
|
|
4
6
|
export declare function findImportBindingForLocalName(sourceFile: SourceFile, localName: string): {
|
|
5
7
|
readonly moduleSpecifier: string;
|
|
6
8
|
readonly importedName: string;
|
|
@@ -5,7 +5,8 @@ const MODULE_RESOLUTION_OPTIONS = {
|
|
|
5
5
|
target: ts.ScriptTarget.ESNext,
|
|
6
6
|
};
|
|
7
7
|
const MAX_RESOLUTION_DEPTH = 24;
|
|
8
|
-
|
|
8
|
+
const DECLARATION_FILES_OK = { allowDeclarationFiles: true };
|
|
9
|
+
export function resolveModuleFile(fromFile, specifier, { allowDeclarationFiles = false } = {}) {
|
|
9
10
|
const project = fromFile.getProject();
|
|
10
11
|
// The project's own host, not ts.sys: it honours an in-memory file system
|
|
11
12
|
// (Designer) and exists outside Node.
|
|
@@ -13,10 +14,10 @@ export function resolveModuleFile(fromFile, specifier) {
|
|
|
13
14
|
const resolvedFileName = resolved.resolvedModule?.resolvedFileName;
|
|
14
15
|
if (!resolvedFileName)
|
|
15
16
|
return undefined;
|
|
16
|
-
// A declaration file has no initializers to read.
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
if (resolvedFileName.endsWith(".d.ts"))
|
|
17
|
+
// A declaration file has no initializers to read. Callers that only need
|
|
18
|
+
// `export declare const X: "literal"` types opt in (installed packages
|
|
19
|
+
// resolve to dist/*.d.ts through their publishConfig exports).
|
|
20
|
+
if (!allowDeclarationFiles && resolvedFileName.endsWith(".d.ts"))
|
|
20
21
|
return undefined;
|
|
21
22
|
return fromFile.getProject().addSourceFileAtPathIfExists(resolvedFileName);
|
|
22
23
|
}
|
|
@@ -71,7 +72,7 @@ function resolveViaNamedExportSpecifier(file, exportedName, visited, depth) {
|
|
|
71
72
|
const moduleSpecifierValue = exportDecl.getModuleSpecifierValue();
|
|
72
73
|
if (!moduleSpecifierValue)
|
|
73
74
|
continue;
|
|
74
|
-
const target = resolveModuleFile(file, moduleSpecifierValue);
|
|
75
|
+
const target = resolveModuleFile(file, moduleSpecifierValue, DECLARATION_FILES_OK);
|
|
75
76
|
if (!target)
|
|
76
77
|
continue;
|
|
77
78
|
const resolved = resolveExportedVariable(target, originalName, visited, depth + 1);
|
|
@@ -87,7 +88,7 @@ function resolveLocalExportBinding(file, originalName, visited, depth) {
|
|
|
87
88
|
const binding = findImportBindingForLocalName(file, originalName);
|
|
88
89
|
if (!binding)
|
|
89
90
|
return undefined;
|
|
90
|
-
const target = resolveModuleFile(file, binding.moduleSpecifier);
|
|
91
|
+
const target = resolveModuleFile(file, binding.moduleSpecifier, DECLARATION_FILES_OK);
|
|
91
92
|
if (!target)
|
|
92
93
|
return undefined;
|
|
93
94
|
return resolveExportedVariable(target, binding.importedName, visited, depth + 1);
|
|
@@ -103,7 +104,7 @@ function resolveViaStarExport(file, exportedName, visited, depth) {
|
|
|
103
104
|
const moduleSpecifierValue = exportDecl.getModuleSpecifierValue();
|
|
104
105
|
if (!moduleSpecifierValue)
|
|
105
106
|
continue;
|
|
106
|
-
const target = resolveModuleFile(file, moduleSpecifierValue);
|
|
107
|
+
const target = resolveModuleFile(file, moduleSpecifierValue, DECLARATION_FILES_OK);
|
|
107
108
|
if (!target)
|
|
108
109
|
continue;
|
|
109
110
|
const resolved = resolveExportedVariable(target, exportedName, visited, depth + 1);
|
|
@@ -212,6 +212,7 @@ export function extractReferenceData(call, sourceFile) {
|
|
|
212
212
|
...(upsertKey !== undefined && { upsertKey }),
|
|
213
213
|
});
|
|
214
214
|
}
|
|
215
|
+
const USE_EXTENSION_READ_OPTIONS = { functionValuesAsRaw: true };
|
|
215
216
|
export function extractUseExtension(call, sourceFile) {
|
|
216
217
|
const args = call.getArguments();
|
|
217
218
|
const first = args[0];
|
|
@@ -245,7 +246,7 @@ export function extractUseExtension(call, sourceFile) {
|
|
|
245
246
|
?.asKind(SyntaxKind.PropertyAssignment)
|
|
246
247
|
?.getInitializer();
|
|
247
248
|
if (optionsInit) {
|
|
248
|
-
const parsed = readDataLiteralNode(optionsInit);
|
|
249
|
+
const parsed = readDataLiteralNode(optionsInit, USE_EXTENSION_READ_OPTIONS);
|
|
249
250
|
if (!isPlainObject(parsed)) {
|
|
250
251
|
return fail("useExtension", sourceLocationFromNode(call, sourceFile), "options could not be read as a plain object");
|
|
251
252
|
}
|
|
@@ -274,7 +275,7 @@ export function extractUseExtension(call, sourceFile) {
|
|
|
274
275
|
const optionsArg = args[2];
|
|
275
276
|
let options;
|
|
276
277
|
if (optionsArg) {
|
|
277
|
-
const parsed = readDataLiteralNode(optionsArg);
|
|
278
|
+
const parsed = readDataLiteralNode(optionsArg, USE_EXTENSION_READ_OPTIONS);
|
|
278
279
|
if (!isPlainObject(parsed)) {
|
|
279
280
|
return fail("useExtension", sourceLocationFromNode(call, sourceFile), "options could not be read as a plain object");
|
|
280
281
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { CallExpression,
|
|
1
|
+
import type { CallExpression, ObjectLiteralExpression } from "ts-morph";
|
|
2
|
+
import { Node } from "ts-morph";
|
|
2
3
|
import type { ParseError } from "../parse.js";
|
|
3
4
|
export type ExtractOutput<TPattern> = {
|
|
4
5
|
readonly kind: "pattern";
|
|
@@ -26,7 +27,16 @@ export type RawRefSentinel = {
|
|
|
26
27
|
};
|
|
27
28
|
export declare function isRawRefSentinel(value: unknown): value is RawRefSentinel;
|
|
28
29
|
export declare function containsRawRefSentinel(value: unknown): boolean;
|
|
29
|
-
export
|
|
30
|
+
export type ReadDataLiteralOptions = {
|
|
31
|
+
/**
|
|
32
|
+
* Keep arrow/function-expression values as `__raw` sentinels instead of
|
|
33
|
+
* failing the whole read. Only for pattern options that are code by
|
|
34
|
+
* nature (useExtension); data-only definitions (entity, nav) must still
|
|
35
|
+
* reject functions.
|
|
36
|
+
*/
|
|
37
|
+
readonly functionValuesAsRaw?: boolean;
|
|
38
|
+
};
|
|
39
|
+
export declare function readDataLiteralNode(node: Node, readOptions?: ReadDataLiteralOptions): unknown;
|
|
30
40
|
/**
|
|
31
41
|
* A node's literal string value, or the raw-ref sentinel when it resolves to
|
|
32
42
|
* an unresolvable identifier/factory-call/member-access (see
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SyntaxKind } from "ts-morph";
|
|
1
|
+
import { Node, SyntaxKind } from "ts-morph";
|
|
2
2
|
import { isPlainObject } from "../../../utils/is-plain-object.js";
|
|
3
3
|
import { findImportBindingForLocalName, findScopedVariableDeclaration, resolveExportedVariable, resolveModuleFile, } from "./resolve-import.js";
|
|
4
4
|
export function ok(pattern) {
|
|
@@ -55,7 +55,7 @@ export function containsRawRefSentinel(value) {
|
|
|
55
55
|
return Object.values(value).some(containsRawRefSentinel);
|
|
56
56
|
return false;
|
|
57
57
|
}
|
|
58
|
-
export function readDataLiteralNode(node) {
|
|
58
|
+
export function readDataLiteralNode(node, readOptions = {}) {
|
|
59
59
|
const kind = node.getKind();
|
|
60
60
|
switch (kind) {
|
|
61
61
|
case SyntaxKind.StringLiteral:
|
|
@@ -74,7 +74,7 @@ export function readDataLiteralNode(node) {
|
|
|
74
74
|
const expr = node.asKindOrThrow(SyntaxKind.PrefixUnaryExpression);
|
|
75
75
|
if (expr.getOperatorToken() !== SyntaxKind.MinusToken)
|
|
76
76
|
return undefined;
|
|
77
|
-
const inner = readDataLiteralNode(expr.getOperand());
|
|
77
|
+
const inner = readDataLiteralNode(expr.getOperand(), readOptions);
|
|
78
78
|
if (typeof inner !== "number")
|
|
79
79
|
return undefined;
|
|
80
80
|
return -inner;
|
|
@@ -83,7 +83,7 @@ export function readDataLiteralNode(node) {
|
|
|
83
83
|
const arr = node.asKindOrThrow(SyntaxKind.ArrayLiteralExpression);
|
|
84
84
|
const out = [];
|
|
85
85
|
for (const el of arr.getElements()) {
|
|
86
|
-
const value = readDataLiteralNode(el);
|
|
86
|
+
const value = readDataLiteralNode(el, readOptions);
|
|
87
87
|
if (value === undefined)
|
|
88
88
|
return undefined;
|
|
89
89
|
out.push(value);
|
|
@@ -100,7 +100,7 @@ export function readDataLiteralNode(node) {
|
|
|
100
100
|
const initializer = propAssign.getInitializer();
|
|
101
101
|
if (!initializer)
|
|
102
102
|
return undefined;
|
|
103
|
-
const value = readDataLiteralNode(initializer);
|
|
103
|
+
const value = readDataLiteralNode(initializer, readOptions);
|
|
104
104
|
if (value === undefined)
|
|
105
105
|
return undefined;
|
|
106
106
|
out[readPropertyKey(propAssign)] = value;
|
|
@@ -108,11 +108,11 @@ export function readDataLiteralNode(node) {
|
|
|
108
108
|
return out;
|
|
109
109
|
}
|
|
110
110
|
case SyntaxKind.AsExpression:
|
|
111
|
-
return readDataLiteralNode(node.asKindOrThrow(SyntaxKind.AsExpression).getExpression());
|
|
111
|
+
return readDataLiteralNode(node.asKindOrThrow(SyntaxKind.AsExpression).getExpression(), readOptions);
|
|
112
112
|
case SyntaxKind.SatisfiesExpression:
|
|
113
|
-
return readDataLiteralNode(node.asKindOrThrow(SyntaxKind.SatisfiesExpression).getExpression());
|
|
113
|
+
return readDataLiteralNode(node.asKindOrThrow(SyntaxKind.SatisfiesExpression).getExpression(), readOptions);
|
|
114
114
|
case SyntaxKind.ParenthesizedExpression:
|
|
115
|
-
return readDataLiteralNode(node.asKindOrThrow(SyntaxKind.ParenthesizedExpression).getExpression());
|
|
115
|
+
return readDataLiteralNode(node.asKindOrThrow(SyntaxKind.ParenthesizedExpression).getExpression(), readOptions);
|
|
116
116
|
case SyntaxKind.Identifier:
|
|
117
117
|
case SyntaxKind.CallExpression:
|
|
118
118
|
case SyntaxKind.PropertyAccessExpression:
|
|
@@ -121,6 +121,11 @@ export function readDataLiteralNode(node) {
|
|
|
121
121
|
// resolving/inlining it would silently rewrite the source on the next
|
|
122
122
|
// parse->render roundtrip (see readDataLiteralNode doc above).
|
|
123
123
|
return { __raw: node.getText() };
|
|
124
|
+
case SyntaxKind.ArrowFunction:
|
|
125
|
+
case SyntaxKind.FunctionExpression:
|
|
126
|
+
return readOptions.functionValuesAsRaw
|
|
127
|
+
? { __raw: node.getText() }
|
|
128
|
+
: undefined;
|
|
124
129
|
default:
|
|
125
130
|
return undefined;
|
|
126
131
|
}
|
|
@@ -178,6 +183,15 @@ function unwrapLiteralInitializer(node) {
|
|
|
178
183
|
return unwrapLiteralInitializer(paren.getExpression());
|
|
179
184
|
return undefined;
|
|
180
185
|
}
|
|
186
|
+
// `export declare const NAME: "value"` in a .d.ts has no initializer, only a
|
|
187
|
+
// string-literal type.
|
|
188
|
+
function readDeclaredStringLiteralType(decl) {
|
|
189
|
+
const typeNode = decl.getTypeNode();
|
|
190
|
+
if (!typeNode || !Node.isLiteralTypeNode(typeNode))
|
|
191
|
+
return undefined;
|
|
192
|
+
const literal = typeNode.getLiteral();
|
|
193
|
+
return Node.isStringLiteral(literal) ? literal.getLiteralValue() : undefined;
|
|
194
|
+
}
|
|
181
195
|
/**
|
|
182
196
|
* Resolves a bare Identifier to the string value of its declaration's
|
|
183
197
|
* initializer (`export const EXT_TENANT_DATA = "tenant-data" as const`),
|
|
@@ -200,12 +214,18 @@ function resolveIdentifierToStringLiteral(identifier) {
|
|
|
200
214
|
const binding = findImportBindingForLocalName(id.getSourceFile(), name);
|
|
201
215
|
if (!binding)
|
|
202
216
|
return undefined;
|
|
203
|
-
const targetFile = resolveModuleFile(id.getSourceFile(), binding.moduleSpecifier
|
|
217
|
+
const targetFile = resolveModuleFile(id.getSourceFile(), binding.moduleSpecifier, {
|
|
218
|
+
allowDeclarationFiles: true,
|
|
219
|
+
});
|
|
204
220
|
if (!targetFile)
|
|
205
221
|
return undefined;
|
|
206
222
|
const decl = resolveExportedVariable(targetFile, binding.importedName);
|
|
207
|
-
|
|
208
|
-
|
|
223
|
+
if (!decl)
|
|
224
|
+
return undefined;
|
|
225
|
+
const init = decl.getInitializer();
|
|
226
|
+
if (init)
|
|
227
|
+
return unwrapLiteralInitializer(init);
|
|
228
|
+
return readDeclaredStringLiteralType(decl);
|
|
209
229
|
}
|
|
210
230
|
/**
|
|
211
231
|
* Like `readNameLiteral`, but for callers that must preserve an
|
|
@@ -9,6 +9,7 @@ import { type AppContext, type DispatchWriteRef, type JobRunIn, type JobTriggerE
|
|
|
9
9
|
*/
|
|
10
10
|
export declare function schedulerIdForJobName(jobName: string): string;
|
|
11
11
|
export declare function bootJobIdForJobName(jobName: string): string;
|
|
12
|
+
export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
|
|
12
13
|
export type JobLogEntry = {
|
|
13
14
|
level: "info" | "warn" | "error";
|
|
14
15
|
message: string;
|
package/dist/jobs/job-runner.js
CHANGED
|
@@ -73,9 +73,48 @@ export function schedulerIdForJobName(jobName) {
|
|
|
73
73
|
export function bootJobIdForJobName(jobName) {
|
|
74
74
|
return `boot-${jobName.replace(/[.:]/g, "-")}`;
|
|
75
75
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
const SCHEDULER_PAGE_SIZE = 100;
|
|
77
|
+
// Rolling deploys run old and new pods against the same Redis for a while; an
|
|
78
|
+
// old runner restarting in that window must not delete the scheduler of a
|
|
79
|
+
// cron job the new version just added.
|
|
80
|
+
export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
|
|
81
|
+
function schedulerSeenKey(queue) {
|
|
82
|
+
return queue.toKey("kumiko-scheduler-seen");
|
|
83
|
+
}
|
|
84
|
+
// A cron job dropped from the app (or an older scheduler-id format) leaves its
|
|
85
|
+
// scheduler in Redis, where it keeps firing jobs the worker no longer knows
|
|
86
|
+
// ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
|
|
87
|
+
async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
|
|
88
|
+
try {
|
|
89
|
+
// Collect first: removing while paging would shift the indices.
|
|
90
|
+
const existingKeys = [];
|
|
91
|
+
for (let start = 0;; start += SCHEDULER_PAGE_SIZE) {
|
|
92
|
+
const page = await queue.getJobSchedulers(start, start + SCHEDULER_PAGE_SIZE - 1);
|
|
93
|
+
for (const scheduler of page)
|
|
94
|
+
existingKeys.push(scheduler.key);
|
|
95
|
+
if (page.length < SCHEDULER_PAGE_SIZE)
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
const client = await queue.client;
|
|
99
|
+
const seenAt = await client.hgetall(schedulerSeenKey(queue));
|
|
100
|
+
const now = Date.now();
|
|
101
|
+
for (const key of existingKeys) {
|
|
102
|
+
if (expectedSchedulerIds.has(key))
|
|
103
|
+
continue;
|
|
104
|
+
const lastSeen = Number(seenAt[key]);
|
|
105
|
+
if (key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS)
|
|
106
|
+
continue;
|
|
107
|
+
await queue.removeJobScheduler(key);
|
|
108
|
+
await client.hdel(schedulerSeenKey(queue), key);
|
|
109
|
+
log.info("removed orphan job scheduler", { schedulerId: key, lane });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
log.warn("orphan job scheduler cleanup failed", {
|
|
114
|
+
lane,
|
|
115
|
+
error: err instanceof Error ? err.message : String(err),
|
|
116
|
+
});
|
|
117
|
+
}
|
|
79
118
|
}
|
|
80
119
|
// Deterministic per-tenant child job id, derived from the perTenant
|
|
81
120
|
// wrapper's own BullMQ job id. A cron tick's wrapper id is
|
|
@@ -1108,21 +1147,13 @@ export function createJobRunner(options) {
|
|
|
1108
1147
|
// Only schedule cron + boot for jobs that belong to this lane. Jobs
|
|
1109
1148
|
// assigned to the other lane get their cron/boot wiring from the
|
|
1110
1149
|
// runner running on that lane. Running both here would double-fire.
|
|
1150
|
+
const expectedSchedulerIds = new Set();
|
|
1111
1151
|
for (const [name, jobDef] of allJobs) {
|
|
1112
1152
|
if (laneForJob(jobDef) !== consumerLane)
|
|
1113
1153
|
continue;
|
|
1114
1154
|
if ("cron" in jobDef.trigger) {
|
|
1115
1155
|
const schedulerId = schedulerIdForJobName(name);
|
|
1116
|
-
|
|
1117
|
-
// Drop pre-sanitize scheduler ids so colon-form ghosts stop firing.
|
|
1118
|
-
if (legacyId !== schedulerId) {
|
|
1119
|
-
try {
|
|
1120
|
-
await consumerQueue.removeJobScheduler(legacyId);
|
|
1121
|
-
}
|
|
1122
|
-
catch {
|
|
1123
|
-
// skip: legacy scheduler absent (fresh install / already purged)
|
|
1124
|
-
}
|
|
1125
|
-
}
|
|
1156
|
+
expectedSchedulerIds.add(schedulerId);
|
|
1126
1157
|
await consumerQueue.upsertJobScheduler(schedulerId, { pattern: jobDef.trigger.cron }, {
|
|
1127
1158
|
name: jobDef.perTenant ? `_perTenant:${name}` : name,
|
|
1128
1159
|
data: {},
|
|
@@ -1133,8 +1164,15 @@ export function createJobRunner(options) {
|
|
|
1133
1164
|
...buildRetryBullOpts(jobDef),
|
|
1134
1165
|
},
|
|
1135
1166
|
});
|
|
1167
|
+
await (await consumerQueue.client).hset(schedulerSeenKey(consumerQueue), {
|
|
1168
|
+
[schedulerId]: String(Date.now()),
|
|
1169
|
+
});
|
|
1136
1170
|
}
|
|
1137
1171
|
}
|
|
1172
|
+
await removeOrphanJobSchedulers(consumerQueue, expectedSchedulerIds, consumerLane, {
|
|
1173
|
+
info: (msg, data) => context.log ? context.log.info(msg, data) : errorLogger.debug(msg, data),
|
|
1174
|
+
warn: errorLogger.warn,
|
|
1175
|
+
});
|
|
1138
1176
|
// Persistent marker outside the swept completed/failed sets: a plain
|
|
1139
1177
|
// Redis hash at one key per consumer queue, fields = boot job ids
|
|
1140
1178
|
// already enqueued (BullMQ's IRedisClient has no set commands, only
|
|
@@ -47,6 +47,7 @@ export type DispatchContext = {
|
|
|
47
47
|
meter: ReturnType<typeof getFallbackMeter>;
|
|
48
48
|
membershipQuery: string;
|
|
49
49
|
memberReadTimeoutMs: number;
|
|
50
|
+
outsideTransactionDb: DbConnection | undefined;
|
|
50
51
|
};
|
|
51
52
|
export declare const DEFAULT_MEMBER_READ_TIMEOUT_MS = 10000;
|
|
52
53
|
export declare function resolveDbSource(ctx: DispatchContext, tx: DbTx | undefined): DbConnection | DbTx | undefined;
|
|
@@ -248,7 +248,7 @@ export async function buildHandlerContext(ctx, type, user, origin, tx, afterComm
|
|
|
248
248
|
// that is meant to outlive the request. Computed before `systemDb` below
|
|
249
249
|
// so its guarded escape hatch (ctx.systemDb.outsideTransaction) can wrap
|
|
250
250
|
// this same TenantDb instead of a second, independently-built one.
|
|
251
|
-
const outsideTxSource = resolveDbSource(ctx, undefined);
|
|
251
|
+
const outsideTxSource = ctx.outsideTransactionDb ?? resolveDbSource(ctx, undefined);
|
|
252
252
|
const rawDbOutsideTransaction = outsideTxSource
|
|
253
253
|
? buildTenantScopedDb(outsideTxSource, undefined)
|
|
254
254
|
: undefined;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { CacheSyncBus } from "@cosmicdrift/kumiko-types/cache-sync-types";
|
|
2
2
|
import type { SseBroker } from "../api/sse-broker.js";
|
|
3
|
+
import type { DbConnection } from "../db/connection.js";
|
|
3
4
|
import type { EffectiveFeaturesResolver } from "../engine/tier-resolver-extension.js";
|
|
4
5
|
import type { TenantId } from "../engine/types/identifiers.js";
|
|
5
6
|
import type { ActiveMembershipResult, AppContext, DispatchWriteRef, JobRunnerRef, MemberReader, Registry, SessionUser, WriteResult } from "../engine/types/index.js";
|
|
@@ -16,6 +17,7 @@ export type DispatcherOptions = {
|
|
|
16
17
|
jobRunner?: JobRunnerRef;
|
|
17
18
|
membershipQuery?: string;
|
|
18
19
|
memberReadTimeoutMs?: number;
|
|
20
|
+
outsideTransactionDb?: DbConnection;
|
|
19
21
|
effectiveFeatures?: EffectiveFeaturesResolver;
|
|
20
22
|
sseBroker?: SseBroker;
|
|
21
23
|
cacheSync?: CacheSyncBus | null;
|
|
@@ -97,6 +97,7 @@ export function createDispatcher(registry, context, options = {}) {
|
|
|
97
97
|
meter: dispatcherMeter,
|
|
98
98
|
membershipQuery,
|
|
99
99
|
memberReadTimeoutMs,
|
|
100
|
+
outsideTransactionDb: options.outsideTransactionDb,
|
|
100
101
|
};
|
|
101
102
|
const dispatcher = {
|
|
102
103
|
async write(typeOrRef, payload, user, requestId) {
|
package/dist/stack/db.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { DbConnection } from "@cosmicdrift/kumiko-types/db-connection";
|
|
|
2
2
|
declare function requireEnv(name: string): string;
|
|
3
3
|
export type TestDb = {
|
|
4
4
|
db: DbConnection;
|
|
5
|
+
outsideTransactionDb: DbConnection;
|
|
5
6
|
client: unknown;
|
|
6
7
|
dbName: string;
|
|
7
8
|
cleanup: () => Promise<void>;
|
|
@@ -10,6 +11,8 @@ export type CreateTestDbOptions = {
|
|
|
10
11
|
readonly baseUrl?: string;
|
|
11
12
|
readonly dbName?: string;
|
|
12
13
|
readonly persistent?: boolean;
|
|
14
|
+
readonly poolMax?: number;
|
|
15
|
+
readonly outsideTransactionPoolMax?: number;
|
|
13
16
|
};
|
|
14
17
|
/**
|
|
15
18
|
* Provider-agnostische Test-DB. createConnection liest DB_PROVIDER.
|
package/dist/stack/db.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// Provider-agnostic via createConnection (DB_PROVIDER env).
|
|
3
3
|
// postgres-js = default. DB_PROVIDER=bun = Bun.SQL (experimentell).
|
|
4
4
|
import { createConnection } from "../db/api.js";
|
|
5
|
+
import { DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX } from "../db/connection.js";
|
|
5
6
|
import { createDatabase, databaseExists, dropDatabaseIfExists } from "../db/queries/test-stack.js";
|
|
6
7
|
import { ensureTemporalPolyfill } from "../time/polyfill.js";
|
|
7
8
|
import { generateId } from "../utils/index.js";
|
|
@@ -37,15 +38,20 @@ export async function createTestDb(arg) {
|
|
|
37
38
|
await admin.close();
|
|
38
39
|
}
|
|
39
40
|
const testUrl = url.replace(/\/[^/]+$/, `/${dbName}`);
|
|
40
|
-
const conn = await createConnection(testUrl);
|
|
41
|
+
const conn = await createConnection(testUrl, opts.poolMax !== undefined ? { maxConnections: opts.poolMax } : {});
|
|
42
|
+
const outsideConn = await createConnection(testUrl, {
|
|
43
|
+
maxConnections: opts.outsideTransactionPoolMax ?? DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX,
|
|
44
|
+
});
|
|
41
45
|
const { createEventsTable } = await import("../event-store/index.js");
|
|
42
46
|
await createEventsTable(conn.db);
|
|
43
47
|
return {
|
|
44
48
|
db: conn.db,
|
|
49
|
+
outsideTransactionDb: outsideConn.db,
|
|
45
50
|
client: conn.client,
|
|
46
51
|
dbName,
|
|
47
52
|
cleanup: async () => {
|
|
48
53
|
await conn.close();
|
|
54
|
+
await outsideConn.close();
|
|
49
55
|
if (!opts.persistent) {
|
|
50
56
|
const admin2 = await createConnection(adminUrl, { maxConnections: 1 });
|
|
51
57
|
try {
|
|
@@ -101,6 +101,10 @@ export type TestStackOptions = {
|
|
|
101
101
|
* owns its lifecycle. Default false (test contract). Used by
|
|
102
102
|
* dev-server wiring to survive hot-reloads. */
|
|
103
103
|
persistentDb?: boolean;
|
|
104
|
+
/** Cap on the main app pool (default: driver default). Lets tests saturate it deliberately. */
|
|
105
|
+
dbPoolMax?: number;
|
|
106
|
+
/** Size of the dedicated pool behind ctx.dbOutsideTransaction / ctx.outsideTransaction. */
|
|
107
|
+
outsideTransactionPoolMax?: number;
|
|
104
108
|
/** Forwarded to buildServer — when set, requests without a JWT pass
|
|
105
109
|
* through as anonymous instead of 401. See AnonymousAccessConfig.
|
|
106
110
|
* Akzeptiert entweder einen statischen Config-Object ODER eine Factory
|
package/dist/stack/test-stack.js
CHANGED
|
@@ -44,6 +44,10 @@ export async function setupTestStack(options) {
|
|
|
44
44
|
createTestDb({
|
|
45
45
|
...(options.dbName !== undefined && { dbName: options.dbName }),
|
|
46
46
|
...(options.persistentDb !== undefined && { persistent: options.persistentDb }),
|
|
47
|
+
...(options.dbPoolMax !== undefined && { poolMax: options.dbPoolMax }),
|
|
48
|
+
...(options.outsideTransactionPoolMax !== undefined && {
|
|
49
|
+
outsideTransactionPoolMax: options.outsideTransactionPoolMax,
|
|
50
|
+
}),
|
|
47
51
|
}),
|
|
48
52
|
createTestRedis(options.sharedRedisWith && { borrowKeyPrefix: options.sharedRedisWith.redis.keyPrefix }),
|
|
49
53
|
]);
|
|
@@ -286,6 +290,7 @@ export async function setupTestStack(options) {
|
|
|
286
290
|
dispatcherOptions: {
|
|
287
291
|
idempotency,
|
|
288
292
|
cacheSync,
|
|
293
|
+
outsideTransactionDb: testDb.outsideTransactionDb,
|
|
289
294
|
...(options.effectiveFeatures && { effectiveFeatures: options.effectiveFeatures }),
|
|
290
295
|
...(jobRunner && { jobRunner }),
|
|
291
296
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.359.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -210,8 +210,8 @@
|
|
|
210
210
|
"./package.json": "./package.json"
|
|
211
211
|
},
|
|
212
212
|
"dependencies": {
|
|
213
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
214
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
213
|
+
"@cosmicdrift/kumiko-http": "0.359.0",
|
|
214
|
+
"@cosmicdrift/kumiko-types": "0.359.0",
|
|
215
215
|
"bullmq": "^5.76.7",
|
|
216
216
|
"bun-types": "^1.3.13",
|
|
217
217
|
"hono": "^4.13.1",
|
|
@@ -227,7 +227,7 @@
|
|
|
227
227
|
"zod": "^4.4.3"
|
|
228
228
|
},
|
|
229
229
|
"devDependencies": {
|
|
230
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
230
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.359.0",
|
|
231
231
|
"bun-types": "^1.3.13",
|
|
232
232
|
"pino-pretty": "^13.1.3"
|
|
233
233
|
},
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,38 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.359.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "actionForm screens honour the called handler's excludedFields",
|
|
6
|
+
"detail": "buildAppSchema projects the handler's excludedFields onto actionForm screens as writeExcludedFields (a flat list, unlike entityEdit's per-verb object). The form shows these fields read-only, not required and never in the payload.",
|
|
7
|
+
"migration": "Drop a hand-written readOnly lock on an actionForm field once the handler it calls declares excludedFields for it."
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"version": "0.358.0",
|
|
11
|
+
"type": "fix",
|
|
12
|
+
"title": "useExtension options with function values and package-imported extension names no longer fail to parse",
|
|
13
|
+
"detail": "r.useExtension(...) options that contain arrow or function expressions (for example isPublic: ({ entityId }) => ...) were reported as \"options could not be read as a plain object\", which turned the whole feature into a legacy corpus entry. The functions are now kept verbatim as raw references, option keys stay visible and parse to render round-trips unchanged. An extension name imported from a package whose types resolve to dist/*.d.ts (an installed @cosmicdrift/* package in a consumer repo) is now read from its `export declare const NAME: \"value\"` type instead of failing. Function values elsewhere (entity, nav, metric, job) are still rejected on purpose.",
|
|
14
|
+
"migration": "Consumers that ship a checked-in few-shot corpus (Enterprise) should regenerate it on the next framework bump; its drift test enforces that. The new ratchet test in dev-server pins the framework's own canonical count and parse-error ceiling."
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"version": "0.358.0",
|
|
18
|
+
"type": "fix",
|
|
19
|
+
"title": "Job runner removes job schedulers of cron jobs that no longer exist",
|
|
20
|
+
"detail": "When an app drops a cron job, its BullMQ job scheduler stayed in Redis and kept firing, so the worker logged `Unknown job` on every tick. On start, each lane now removes every scheduler in its own queue that does not belong to a registered cron job of that lane (logged per removal). Each runner records when it last upserted a scheduler; a scheduler outside the runner's own set is only removed if it was not upserted within the last hour, so an old pod restarting during a rolling deploy does not delete the scheduler of a cron job the new version just added. Schedulers with no record, such as the old colon-form ids, are removed right away. A failing cleanup is logged as a warning and does not stop the start. This replaces the dedicated removal of the old colon-form scheduler ids."
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"version": "0.358.0",
|
|
24
|
+
"type": "fix",
|
|
25
|
+
"title": "Outside-transaction access no longer deadlocks against the write pool",
|
|
26
|
+
"detail": "ctx.dbOutsideTransaction, ctx.systemDb.outsideTransaction and ctx.outsideTransaction used the same pool as the handler's own transaction. With as many concurrent writes as pool slots, each write held a connection and waited for a second one, so every request hung forever. They now use a dedicated pool (DispatcherOptions.outsideTransactionDb) sized by the new DATABASE_OUTSIDE_TX_POOL_MAX (default 3). runProdApp and the worker boot also read DATABASE_POOL_MAX, DATABASE_POOL_IDLE_TIMEOUT and DATABASE_POOL_CONNECT_TIMEOUT now; before, production ignored them and used the postgres.js default of 10.",
|
|
27
|
+
"migration": "Each pod now opens up to DATABASE_POOL_MAX + DATABASE_OUTSIDE_TX_POOL_MAX connections (defaults 10 + 3) plus the LISTEN connection; check the total against Postgres max_connections, including rolling-update surge. A set but invalid DATABASE_POOL_MAX, DATABASE_POOL_IDLE_TIMEOUT, DATABASE_POOL_CONNECT_TIMEOUT or DATABASE_OUTSIDE_TX_POOL_MAX now aborts the boot. Hand-built dispatchers can pass outsideTransactionDb in DispatcherOptions; without it the app pool is shared as before."
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"version": "0.358.0",
|
|
31
|
+
"type": "breaking",
|
|
32
|
+
"title": "Entity PII fields need whenAbsent: \"tenant\" when the owner field is nullable",
|
|
33
|
+
"detail": "personal: { of: \"<owner>\", whenAbsent: \"tenant\" } makes rows without an owner editable, deletable and forgettable with an active KMS, without plaintext PII. The subject is resolved in this order: the owner on the previous row, the owner the same event sets, then the tenant key (tenant:<tenantId>). Before, an update, delete or forget of an ownerless row failed with SubjectResolutionError (HTTP 500) once a KMS was configured. backfill-pii follows the same chain and checks the projection row for the owner before it falls back to the tenant key; an updated event whose previous.owner is null but whose changes.owner is set no longer loses the owner. Entity fields have no plaintext fallback, so \"tenant\" is the only value. bundled jobs: cron and system runs (triggeredById null) now store payload, error and log messages as ciphertext under the system tenant key instead of plaintext.",
|
|
34
|
+
"migration": "Boot now aborts for a personal: { of: \"<owner>\" } field whose owner field is not required and that declares no whenAbsent: [Feature X] Field \"f\" on entity \"e\" is personal: { of: \"ownerId\" }, but \"ownerId\" is not required — rows without an owner would fail to encrypt (SubjectResolutionError, HTTP 500) on update, delete and forget. Declare { personal: { of: \"ownerId\", whenAbsent: \"tenant\" } } to encrypt those rows under the tenant key, or make \"ownerId\" required: true. Entity fields have no plaintext fallback (fw#3654). Fix: add whenAbsent: \"tenant\", or make the owner field required: true if every row has one. whenAbsent on personal: { of: \"id\" } throws at declaration. Known consumer sites: kumiko-enterprise kumiko-credit entity.ts and bauspar-entity.ts (ownerUserId), ai-agent entity.ts (authorId, ownerId) and attachment-store.ts (authorId), ai-conversation entity.ts (authorId), pattern-storage entity.ts (authorId), offlot-app src/features/lead/schema/entity.ts (enquiryId). Existing plaintext job-run rows of cron runs stay readable; new ones are ciphertext."
|
|
35
|
+
},
|
|
2
36
|
{
|
|
3
37
|
"version": "0.357.0",
|
|
4
38
|
"type": "improvement",
|