@cosmicdrift/kumiko-bundled-features 0.405.0 → 0.406.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tenant-lifecycle/constants.d.ts +1 -0
- package/dist/tenant-lifecycle/constants.js +5 -0
- package/dist/tenant-lifecycle/events.d.ts +2 -0
- package/dist/tenant-lifecycle/events.js +2 -0
- package/dist/tenant-lifecycle/feature.js +1 -1
- package/dist/tenant-lifecycle/index.d.ts +1 -1
- package/dist/tenant-lifecycle/index.js +1 -1
- package/dist/tenant-lifecycle/run-tenant-destroy.d.ts +2 -0
- package/dist/tenant-lifecycle/run-tenant-destroy.js +42 -7
- package/dist/tenant-lifecycle/stages.d.ts +1 -0
- package/dist/tenant-lifecycle/stages.js +4 -1
- package/package.json +11 -11
- package/src/tenant-lifecycle/changes.json +12 -0
|
@@ -28,3 +28,4 @@ export declare const TENANT_DESTRUCTION_STAGES: readonly ["external-resources",
|
|
|
28
28
|
export type TenantDestructionStageName = (typeof TENANT_DESTRUCTION_STAGES)[number];
|
|
29
29
|
export declare const DESTRUCTION_STAGE_TICK_BUDGET_MS = 30000;
|
|
30
30
|
export declare const DESTRUCTION_STAGE_MIN_SLICE_MS = 2000;
|
|
31
|
+
export declare const DESTRUCTION_STAGE_MAX_PENDING_MS: number;
|
|
@@ -42,3 +42,8 @@ export const DESTRUCTION_STAGE_TICK_BUDGET_MS = 30_000;
|
|
|
42
42
|
// without a guaranteed first batch processes nothing, and a no-progress stage
|
|
43
43
|
// counts as a failed attempt, which would abandon the pipeline under load.
|
|
44
44
|
export const DESTRUCTION_STAGE_MIN_SLICE_MS = 2_000;
|
|
45
|
+
// How long one stage attempt may report pending (a hook waiting on an external
|
|
46
|
+
// operation) after its stage-started event before the wait counts as a failed attempt.
|
|
47
|
+
// Sized for a queued app teardown (pod drain up to 330 s, database drop up to 180 s,
|
|
48
|
+
// two teardowns in parallel) with headroom for several apps per tenant.
|
|
49
|
+
export const DESTRUCTION_STAGE_MAX_PENDING_MS = 60 * 60_000;
|
|
@@ -12,6 +12,7 @@ export declare const tenantDestructionStartedSchema: z.ZodObject<{
|
|
|
12
12
|
export declare const tenantDestructionStageStartedSchema: z.ZodObject<{
|
|
13
13
|
stage: z.ZodString;
|
|
14
14
|
attempts: z.ZodNumber;
|
|
15
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
15
16
|
}, z.core.$strip>;
|
|
16
17
|
export declare const tenantDestructionStageSucceededSchema: z.ZodObject<{
|
|
17
18
|
stage: z.ZodString;
|
|
@@ -21,6 +22,7 @@ export declare const tenantDestructionStageProgressedSchema: z.ZodObject<{
|
|
|
21
22
|
stage: z.ZodString;
|
|
22
23
|
attempts: z.ZodNumber;
|
|
23
24
|
processed: z.ZodNumber;
|
|
25
|
+
pending: z.ZodOptional<z.ZodBoolean>;
|
|
24
26
|
}, z.core.$strip>;
|
|
25
27
|
export declare const tenantDestructionStageFailedSchema: z.ZodObject<{
|
|
26
28
|
stage: z.ZodString;
|
|
@@ -12,6 +12,7 @@ export const tenantDestructionStartedSchema = z.object({
|
|
|
12
12
|
export const tenantDestructionStageStartedSchema = z.object({
|
|
13
13
|
stage: z.string().min(1),
|
|
14
14
|
attempts: z.number().int().positive(),
|
|
15
|
+
startedAt: z.string().datetime().optional(),
|
|
15
16
|
});
|
|
16
17
|
export const tenantDestructionStageSucceededSchema = z.object({
|
|
17
18
|
stage: z.string().min(1),
|
|
@@ -21,6 +22,7 @@ export const tenantDestructionStageProgressedSchema = z.object({
|
|
|
21
22
|
stage: z.string().min(1),
|
|
22
23
|
attempts: z.number().int().positive(),
|
|
23
24
|
processed: z.number().int().nonnegative(),
|
|
25
|
+
pending: z.boolean().optional(),
|
|
24
26
|
});
|
|
25
27
|
export const tenantDestructionStageFailedSchema = z.object({
|
|
26
28
|
stage: z.string().min(1),
|
|
@@ -67,7 +67,7 @@ export function createTenantLifecycleFeature() {
|
|
|
67
67
|
const RUN_TENANT_DESTRUCTION_REASON = "sweeps tenant destruction across every tenant";
|
|
68
68
|
r.job({
|
|
69
69
|
name: "run-tenant-destruction",
|
|
70
|
-
trigger: { cron: "0 * * * * *" },
|
|
70
|
+
trigger: { cron: "0 * * * * *", wakesSleepingApp: false },
|
|
71
71
|
concurrency: "skip",
|
|
72
72
|
escapeHatch: { reason: RUN_TENANT_DESTRUCTION_REASON },
|
|
73
73
|
handler: async (_payload, ctx) => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { archiveAndDeleteTenantRows, TENANT_DESTROY_ARCHIVE_REASON, TENANT_DESTROY_ARCHIVED_BY, } from "./archive-and-delete-tenant-rows.ts";
|
|
2
|
-
export { TENANT_DESTRUCTION_STAGES, TENANT_LIFECYCLE_FEATURE, TenantLifecycleHandlers, } from "./constants.ts";
|
|
2
|
+
export { TENANT_DESTRUCTION_STAGE_FAILED_EVENT_QN, TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_QN, TENANT_DESTRUCTION_STAGES, TENANT_LIFECYCLE_FEATURE, TenantLifecycleHandlers, } from "./constants.ts";
|
|
3
3
|
export { createTenantLifecycleFeature } from "./feature.ts";
|
|
4
4
|
export { resolveTenantLifecycleGate } from "./run-tenant-destroy.ts";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { archiveAndDeleteTenantRows, TENANT_DESTROY_ARCHIVE_REASON, TENANT_DESTROY_ARCHIVED_BY, } from "./archive-and-delete-tenant-rows.js";
|
|
2
|
-
export { TENANT_DESTRUCTION_STAGES, TENANT_LIFECYCLE_FEATURE, TenantLifecycleHandlers, } from "./constants.js";
|
|
2
|
+
export { TENANT_DESTRUCTION_STAGE_FAILED_EVENT_QN, TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_QN, TENANT_DESTRUCTION_STAGES, TENANT_LIFECYCLE_FEATURE, TenantLifecycleHandlers, } from "./constants.js";
|
|
3
3
|
export { createTenantLifecycleFeature } from "./feature.js";
|
|
4
4
|
export { resolveTenantLifecycleGate } from "./run-tenant-destroy.js";
|
|
@@ -9,6 +9,7 @@ type StageTickResult = {
|
|
|
9
9
|
readonly error?: string;
|
|
10
10
|
readonly halted?: boolean;
|
|
11
11
|
readonly progressed?: boolean;
|
|
12
|
+
readonly pending?: boolean;
|
|
12
13
|
};
|
|
13
14
|
export declare function runNextDestructionStage(args: {
|
|
14
15
|
readonly db: DbRunner;
|
|
@@ -21,6 +22,7 @@ export declare function runNextDestructionStage(args: {
|
|
|
21
22
|
readonly escapeHatchAuditLog?: Logger;
|
|
22
23
|
readonly actor?: string;
|
|
23
24
|
readonly deadlineAt?: number;
|
|
25
|
+
readonly now?: Temporal.Instant;
|
|
24
26
|
}): Promise<StageTickResult>;
|
|
25
27
|
export declare function hasTenantDestructionWork(reads: Pick<TenantDb, "count">, now: Temporal.Instant): Promise<boolean>;
|
|
26
28
|
/** Cron entry: start destruction for tenants past grace, then advance stages. */
|
|
@@ -3,7 +3,7 @@ import { createSystemUser, SYSTEM_TENANT_ID, } from "@cosmicdrift/kumiko-framewo
|
|
|
3
3
|
import { append, loadAggregate, VersionConflictError, } from "@cosmicdrift/kumiko-framework/event-store";
|
|
4
4
|
import { getTemporal } from "@cosmicdrift/kumiko-framework/time";
|
|
5
5
|
import { tenantEntity, tenantTable } from "../tenant/index.js";
|
|
6
|
-
import { DESTRUCTION_STAGE_MIN_SLICE_MS, DESTRUCTION_STAGE_TICK_BUDGET_MS, TENANT_AGGREGATE_TYPE, TENANT_DESTRUCTION_COMPLETED_EVENT_QN, TENANT_DESTRUCTION_FAILED_EVENT_QN, TENANT_DESTRUCTION_STAGE_ABANDONED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT, TENANT_DESTRUCTION_STARTED_EVENT_SHORT, } from "./constants.js";
|
|
6
|
+
import { DESTRUCTION_STAGE_MAX_PENDING_MS, DESTRUCTION_STAGE_MIN_SLICE_MS, DESTRUCTION_STAGE_TICK_BUDGET_MS, TENANT_AGGREGATE_TYPE, TENANT_DESTRUCTION_COMPLETED_EVENT_QN, TENANT_DESTRUCTION_FAILED_EVENT_QN, TENANT_DESTRUCTION_STAGE_ABANDONED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT, TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT, TENANT_DESTRUCTION_STARTED_EVENT_SHORT, } from "./constants.js";
|
|
7
7
|
import { invalidateTenantLifecycleGate, TEARDOWN_GATE_SETTLE_MS } from "./lifecycle-gate.js";
|
|
8
8
|
import { isDestructionPipelineComplete, pickNextStage, } from "./stages.js";
|
|
9
9
|
const tenantCrud = createEventStoreExecutor(tenantTable, tenantEntity, { entityName: "tenant" });
|
|
@@ -44,6 +44,20 @@ function isStageContinuation(events, stageName) {
|
|
|
44
44
|
const lastStageEvent = events.findLast((event) => stageEventTypes.has(event.type) && event.payload["stage"] === stageName);
|
|
45
45
|
return lastStageEvent?.type === qualified(TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT);
|
|
46
46
|
}
|
|
47
|
+
// Stage-started events written before startedAt existed fall back to their append time.
|
|
48
|
+
function currentAttemptStartedAt(events, stageName) {
|
|
49
|
+
const started = events.findLast((event) => event.type === qualified(TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT) &&
|
|
50
|
+
event.payload["stage"] === stageName);
|
|
51
|
+
if (!started)
|
|
52
|
+
return undefined;
|
|
53
|
+
const startedAt = started.payload["startedAt"];
|
|
54
|
+
return typeof startedAt === "string" ? getTemporal().Instant.from(startedAt) : started.createdAt;
|
|
55
|
+
}
|
|
56
|
+
function isPendingWindowOpen(events, stageName, now) {
|
|
57
|
+
const startedAt = currentAttemptStartedAt(events, stageName);
|
|
58
|
+
return (startedAt !== undefined &&
|
|
59
|
+
now.epochMilliseconds - startedAt.epochMilliseconds <= DESTRUCTION_STAGE_MAX_PENDING_MS);
|
|
60
|
+
}
|
|
47
61
|
function lastEventVersion(events) {
|
|
48
62
|
const last = events.at(-1);
|
|
49
63
|
return last?.version ?? 0;
|
|
@@ -144,7 +158,28 @@ async function recordFailedStageAttempt(args) {
|
|
|
144
158
|
await appendTenantStageEvent(args.db, args.tenantId, qualified(TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT), { stage: args.stage.name, attempts: args.attempt, error: args.error }, args.version);
|
|
145
159
|
return { done: false, error: args.error };
|
|
146
160
|
}
|
|
161
|
+
// A tick that neither finishes nor processes anything would otherwise loop forever,
|
|
162
|
+
// because progressed events do not count as attempts. A pending tick still appends a
|
|
163
|
+
// progressed event: without it the next tick would open a new stage-started and the
|
|
164
|
+
// pending window would never close.
|
|
165
|
+
async function recordIdleStageTick(args) {
|
|
166
|
+
if (args.pending && isPendingWindowOpen(args.events, args.stage.name, args.now)) {
|
|
167
|
+
await appendTenantStageEvent(args.db, args.tenantId, qualified(TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT), { stage: args.stage.name, attempts: args.attempt, processed: 0, pending: true }, lastEventVersion(args.events));
|
|
168
|
+
return { done: false, pending: true };
|
|
169
|
+
}
|
|
170
|
+
return await recordFailedStageAttempt({
|
|
171
|
+
db: args.db,
|
|
172
|
+
tenantId: args.tenantId,
|
|
173
|
+
stage: args.stage,
|
|
174
|
+
attempt: args.attempt,
|
|
175
|
+
error: args.pending
|
|
176
|
+
? `stage "${args.stage.name}" still pending after ${DESTRUCTION_STAGE_MAX_PENDING_MS / 60_000} min`
|
|
177
|
+
: `stage "${args.stage.name}" made no progress`,
|
|
178
|
+
version: lastEventVersion(args.events),
|
|
179
|
+
});
|
|
180
|
+
}
|
|
147
181
|
export async function runNextDestructionStage(args) {
|
|
182
|
+
const now = args.now ?? getTemporal().Now.instant();
|
|
148
183
|
const events = await loadAggregate(args.db, args.tenantId, SYSTEM_TENANT_ID);
|
|
149
184
|
const { completed, abandoned, attemptsByStage } = replayStageState(events);
|
|
150
185
|
if (abandoned.size > 0) {
|
|
@@ -171,7 +206,7 @@ export async function runNextDestructionStage(args) {
|
|
|
171
206
|
};
|
|
172
207
|
let version = lastEventVersion(events);
|
|
173
208
|
if (!isStageContinuation(events, next.name)) {
|
|
174
|
-
version = await appendTenantStageEventIdempotent(args.db, args.tenantId, qualified(TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT), { stage: next.name, attempts: attempt }, version, stageOutcomeRecorded(events, next.name, TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT));
|
|
209
|
+
version = await appendTenantStageEventIdempotent(args.db, args.tenantId, qualified(TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT), { stage: next.name, attempts: attempt, startedAt: now.toString() }, version, stageOutcomeRecorded(events, next.name, TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT));
|
|
175
210
|
}
|
|
176
211
|
try {
|
|
177
212
|
const outcome = await next.run(ctx);
|
|
@@ -180,16 +215,15 @@ export async function runNextDestructionStage(args) {
|
|
|
180
215
|
// the version fresh instead of trusting the pre-run `version`, or this
|
|
181
216
|
// append throws VersionConflictError on every stage whose run() writes.
|
|
182
217
|
const eventsAfterRun = await loadAggregate(args.db, args.tenantId, SYSTEM_TENANT_ID);
|
|
183
|
-
// A tick that neither finishes nor processes anything would otherwise loop
|
|
184
|
-
// forever, because progressed events do not count as attempts.
|
|
185
218
|
if (!outcome.done && outcome.processed === 0) {
|
|
186
|
-
return await
|
|
219
|
+
return await recordIdleStageTick({
|
|
187
220
|
db: args.db,
|
|
188
221
|
tenantId: args.tenantId,
|
|
189
222
|
stage: next,
|
|
190
223
|
attempt,
|
|
191
|
-
|
|
192
|
-
|
|
224
|
+
pending: outcome.pending === true,
|
|
225
|
+
events: eventsAfterRun,
|
|
226
|
+
now,
|
|
193
227
|
});
|
|
194
228
|
}
|
|
195
229
|
if (!outcome.done) {
|
|
@@ -267,6 +301,7 @@ export async function runTenantDestructionSweep(args) {
|
|
|
267
301
|
try {
|
|
268
302
|
const result = await runNextDestructionStage({
|
|
269
303
|
deadlineAt: tenantDeadlineAt,
|
|
304
|
+
now,
|
|
270
305
|
db: args.db,
|
|
271
306
|
registry: args.registry,
|
|
272
307
|
tenantId: row.id,
|
|
@@ -17,14 +17,17 @@ const tenantMembershipCrud = createEventStoreExecutor(tenantMembershipsTable, te
|
|
|
17
17
|
// hooks time out with nothing processed until it finishes.
|
|
18
18
|
function newOutcomeAccumulator() {
|
|
19
19
|
let allDone = true;
|
|
20
|
+
let pending = false;
|
|
20
21
|
let processed = 0;
|
|
21
22
|
return {
|
|
22
23
|
add: (result) => {
|
|
23
24
|
if (result?.done === false)
|
|
24
25
|
allDone = false;
|
|
26
|
+
if (result?.done === false && result.pending === true)
|
|
27
|
+
pending = true;
|
|
25
28
|
processed += result?.processed ?? 0;
|
|
26
29
|
},
|
|
27
|
-
outcome: () => ({ done: allDone, processed }),
|
|
30
|
+
outcome: () => ({ done: allDone, processed, pending }),
|
|
28
31
|
};
|
|
29
32
|
}
|
|
30
33
|
async function runExtensionDestroyHooks(registry, extensionName, ctx) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-bundled-features",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.406.0",
|
|
4
4
|
"description": "Built-in features — tenant, user, auth, delivery. The stuff you'd rewrite anyway, already typed.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -570,9 +570,9 @@
|
|
|
570
570
|
}
|
|
571
571
|
},
|
|
572
572
|
"dependencies": {
|
|
573
|
-
"@cosmicdrift/kumiko-framework": "0.
|
|
574
|
-
"@cosmicdrift/kumiko-renderer": "0.
|
|
575
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
573
|
+
"@cosmicdrift/kumiko-framework": "0.406.0",
|
|
574
|
+
"@cosmicdrift/kumiko-renderer": "0.406.0",
|
|
575
|
+
"@cosmicdrift/kumiko-types": "0.406.0",
|
|
576
576
|
"@mollie/api-client": "^4.6.0",
|
|
577
577
|
"@node-rs/argon2": "^2.2.2",
|
|
578
578
|
"@tiptap/core": "^3.32.2",
|
|
@@ -1166,17 +1166,17 @@
|
|
|
1166
1166
|
],
|
|
1167
1167
|
"devDependencies": {
|
|
1168
1168
|
"@testing-library/user-event": "^14.6.8",
|
|
1169
|
-
"@cosmicdrift/kumiko-locale-de": "0.
|
|
1170
|
-
"@cosmicdrift/kumiko-locale-es": "0.
|
|
1171
|
-
"@cosmicdrift/kumiko-locale-fr": "0.
|
|
1172
|
-
"@cosmicdrift/kumiko-locale-it": "0.
|
|
1173
|
-
"@cosmicdrift/kumiko-renderer-mail-html": "0.
|
|
1169
|
+
"@cosmicdrift/kumiko-locale-de": "0.406.0",
|
|
1170
|
+
"@cosmicdrift/kumiko-locale-es": "0.406.0",
|
|
1171
|
+
"@cosmicdrift/kumiko-locale-fr": "0.406.0",
|
|
1172
|
+
"@cosmicdrift/kumiko-locale-it": "0.406.0",
|
|
1173
|
+
"@cosmicdrift/kumiko-renderer-mail-html": "0.406.0",
|
|
1174
1174
|
"jsqr": "^1.4.0",
|
|
1175
1175
|
"unpdf": "^1.9.1",
|
|
1176
|
-
"@cosmicdrift/kumiko-testing": "0.
|
|
1176
|
+
"@cosmicdrift/kumiko-testing": "0.406.0"
|
|
1177
1177
|
},
|
|
1178
1178
|
"peerDependencies": {
|
|
1179
|
-
"@cosmicdrift/kumiko-testing": "0.
|
|
1179
|
+
"@cosmicdrift/kumiko-testing": "0.406.0"
|
|
1180
1180
|
},
|
|
1181
1181
|
"peerDependenciesMeta": {
|
|
1182
1182
|
"@cosmicdrift/kumiko-testing": {
|
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.406.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "Destroy stages can wait on external operations",
|
|
6
|
+
"detail": "A tick in which a hook reports pending is recorded as a progressed event with pending true and does not count as a failed attempt. The wait is bounded: once 60 minutes (DESTRUCTION_STAGE_MAX_PENDING_MS) have passed since the attempt's stage-started event, a pending tick fails the attempt with \"still pending\", and the stage is abandoned after its usual maxAttempts, so a stage waits at most maxAttempts x 60 minutes. Ticks without progress and without pending still count as failed attempts. stage-started events now carry startedAt from the sweep clock."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.406.0",
|
|
10
|
+
"type": "improvement",
|
|
11
|
+
"title": "Destruction sweep no longer wakes a sleeping app",
|
|
12
|
+
"detail": "The run-tenant-destruction cron declares wakesSleepingApp false, so its per-minute schedule does not keep a sleeping shared app awake. A due destruction runs on the next wake, for example the daily maintenance window. Its scheduler id changes to scheduler-nowake-tenant-lifecycle-job-run-tenant-destruction; the old scheduler is removed on the first start."
|
|
13
|
+
},
|
|
2
14
|
{
|
|
3
15
|
"version": "0.405.0",
|
|
4
16
|
"type": "improvement",
|