@cosmicdrift/kumiko-samples 0.405.0 → 0.406.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/bundled-features/package.json +1 -1
- package/packages/bundled-features/src/tenant-lifecycle/changes.json +12 -0
- package/packages/bundled-features/src/tenant-lifecycle/constants.ts +6 -0
- package/packages/bundled-features/src/tenant-lifecycle/events.ts +2 -0
- package/packages/bundled-features/src/tenant-lifecycle/feature.ts +1 -1
- package/packages/bundled-features/src/tenant-lifecycle/index.ts +2 -0
- package/packages/bundled-features/src/tenant-lifecycle/run-tenant-destroy.ts +73 -6
- package/packages/bundled-features/src/tenant-lifecycle/stages.ts +8 -2
- package/samples/recipes/lane-routing/README.md +1 -0
- package/samples/recipes/lane-routing/src/feature.ts +8 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-samples",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.406.0",
|
|
4
4
|
"description": "Source trees of the Kumiko sample recipes, sample apps and bundled features in repo layout, for tooling such as few-shot corpus builds.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-bundled-features",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.406.0",
|
|
4
4
|
"description": "Built-in features — tenant, user, auth, delivery. The stuff you'd rewrite anyway, already typed.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.406.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "Destroy stages can wait on external operations",
|
|
6
|
+
"detail": "A tick in which a hook reports pending is recorded as a progressed event with pending true and does not count as a failed attempt. The wait is bounded: once 60 minutes (DESTRUCTION_STAGE_MAX_PENDING_MS) have passed since the attempt's stage-started event, a pending tick fails the attempt with \"still pending\", and the stage is abandoned after its usual maxAttempts, so a stage waits at most maxAttempts x 60 minutes. Ticks without progress and without pending still count as failed attempts. stage-started events now carry startedAt from the sweep clock."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.406.0",
|
|
10
|
+
"type": "improvement",
|
|
11
|
+
"title": "Destruction sweep no longer wakes a sleeping app",
|
|
12
|
+
"detail": "The run-tenant-destruction cron declares wakesSleepingApp false, so its per-minute schedule does not keep a sleeping shared app awake. A due destruction runs on the next wake, for example the daily maintenance window. Its scheduler id changes to scheduler-nowake-tenant-lifecycle-job-run-tenant-destruction; the old scheduler is removed on the first start."
|
|
13
|
+
},
|
|
2
14
|
{
|
|
3
15
|
"version": "0.405.0",
|
|
4
16
|
"type": "improvement",
|
|
@@ -67,3 +67,9 @@ export const DESTRUCTION_STAGE_TICK_BUDGET_MS = 30_000;
|
|
|
67
67
|
// without a guaranteed first batch processes nothing, and a no-progress stage
|
|
68
68
|
// counts as a failed attempt, which would abandon the pipeline under load.
|
|
69
69
|
export const DESTRUCTION_STAGE_MIN_SLICE_MS = 2_000;
|
|
70
|
+
|
|
71
|
+
// How long one stage attempt may report pending (a hook waiting on an external
|
|
72
|
+
// operation) after its stage-started event before the wait counts as a failed attempt.
|
|
73
|
+
// Sized for a queued app teardown (pod drain up to 330 s, database drop up to 180 s,
|
|
74
|
+
// two teardowns in parallel) with headroom for several apps per tenant.
|
|
75
|
+
export const DESTRUCTION_STAGE_MAX_PENDING_MS = 60 * 60_000;
|
|
@@ -16,6 +16,7 @@ export const tenantDestructionStartedSchema = z.object({
|
|
|
16
16
|
export const tenantDestructionStageStartedSchema = z.object({
|
|
17
17
|
stage: z.string().min(1),
|
|
18
18
|
attempts: z.number().int().positive(),
|
|
19
|
+
startedAt: z.string().datetime().optional(),
|
|
19
20
|
});
|
|
20
21
|
|
|
21
22
|
export const tenantDestructionStageSucceededSchema = z.object({
|
|
@@ -27,6 +28,7 @@ export const tenantDestructionStageProgressedSchema = z.object({
|
|
|
27
28
|
stage: z.string().min(1),
|
|
28
29
|
attempts: z.number().int().positive(),
|
|
29
30
|
processed: z.number().int().nonnegative(),
|
|
31
|
+
pending: z.boolean().optional(),
|
|
30
32
|
});
|
|
31
33
|
|
|
32
34
|
export const tenantDestructionStageFailedSchema = z.object({
|
|
@@ -133,7 +133,7 @@ export function createTenantLifecycleFeature(): FeatureDefinition {
|
|
|
133
133
|
const RUN_TENANT_DESTRUCTION_REASON = "sweeps tenant destruction across every tenant";
|
|
134
134
|
r.job({
|
|
135
135
|
name: "run-tenant-destruction",
|
|
136
|
-
trigger: { cron: "0 * * * * *" },
|
|
136
|
+
trigger: { cron: "0 * * * * *", wakesSleepingApp: false },
|
|
137
137
|
concurrency: "skip",
|
|
138
138
|
escapeHatch: { reason: RUN_TENANT_DESTRUCTION_REASON },
|
|
139
139
|
handler: async (_payload, ctx) => {
|
|
@@ -4,6 +4,8 @@ export {
|
|
|
4
4
|
TENANT_DESTROY_ARCHIVED_BY,
|
|
5
5
|
} from "./archive-and-delete-tenant-rows.ts";
|
|
6
6
|
export {
|
|
7
|
+
TENANT_DESTRUCTION_STAGE_FAILED_EVENT_QN,
|
|
8
|
+
TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_QN,
|
|
7
9
|
TENANT_DESTRUCTION_STAGES,
|
|
8
10
|
TENANT_LIFECYCLE_FEATURE,
|
|
9
11
|
TenantLifecycleHandlers,
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
import {
|
|
17
17
|
append,
|
|
18
18
|
loadAggregate,
|
|
19
|
+
type StoredEvent,
|
|
19
20
|
VersionConflictError,
|
|
20
21
|
} from "@cosmicdrift/kumiko-framework/event-store";
|
|
21
22
|
import type { FileProviderResolver } from "@cosmicdrift/kumiko-framework/files";
|
|
@@ -24,6 +25,7 @@ import type { SearchAdapter } from "@cosmicdrift/kumiko-framework/search";
|
|
|
24
25
|
import { getTemporal } from "@cosmicdrift/kumiko-framework/time";
|
|
25
26
|
import { tenantEntity, tenantTable } from "../tenant/index.ts";
|
|
26
27
|
import {
|
|
28
|
+
DESTRUCTION_STAGE_MAX_PENDING_MS,
|
|
27
29
|
DESTRUCTION_STAGE_MIN_SLICE_MS,
|
|
28
30
|
DESTRUCTION_STAGE_TICK_BUDGET_MS,
|
|
29
31
|
TENANT_AGGREGATE_TYPE,
|
|
@@ -106,6 +108,33 @@ function isStageContinuation(
|
|
|
106
108
|
return lastStageEvent?.type === qualified(TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT);
|
|
107
109
|
}
|
|
108
110
|
|
|
111
|
+
// Stage-started events written before startedAt existed fall back to their append time.
|
|
112
|
+
function currentAttemptStartedAt(
|
|
113
|
+
events: readonly StoredEvent[],
|
|
114
|
+
stageName: string,
|
|
115
|
+
): Temporal.Instant | undefined {
|
|
116
|
+
const started = events.findLast(
|
|
117
|
+
(event) =>
|
|
118
|
+
event.type === qualified(TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT) &&
|
|
119
|
+
event.payload["stage"] === stageName,
|
|
120
|
+
);
|
|
121
|
+
if (!started) return undefined;
|
|
122
|
+
const startedAt = started.payload["startedAt"];
|
|
123
|
+
return typeof startedAt === "string" ? getTemporal().Instant.from(startedAt) : started.createdAt;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function isPendingWindowOpen(
|
|
127
|
+
events: readonly StoredEvent[],
|
|
128
|
+
stageName: string,
|
|
129
|
+
now: Temporal.Instant,
|
|
130
|
+
): boolean {
|
|
131
|
+
const startedAt = currentAttemptStartedAt(events, stageName);
|
|
132
|
+
return (
|
|
133
|
+
startedAt !== undefined &&
|
|
134
|
+
now.epochMilliseconds - startedAt.epochMilliseconds <= DESTRUCTION_STAGE_MAX_PENDING_MS
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
109
138
|
function lastEventVersion(events: ReadonlyArray<{ version: number }>): number {
|
|
110
139
|
const last = events.at(-1);
|
|
111
140
|
return last?.version ?? 0;
|
|
@@ -255,6 +284,7 @@ type StageTickResult = {
|
|
|
255
284
|
readonly error?: string;
|
|
256
285
|
readonly halted?: boolean;
|
|
257
286
|
readonly progressed?: boolean;
|
|
287
|
+
readonly pending?: boolean;
|
|
258
288
|
};
|
|
259
289
|
|
|
260
290
|
// `version` differs per caller: the catch path appends against the pre-run
|
|
@@ -287,6 +317,41 @@ async function recordFailedStageAttempt(args: {
|
|
|
287
317
|
return { done: false, error: args.error };
|
|
288
318
|
}
|
|
289
319
|
|
|
320
|
+
// A tick that neither finishes nor processes anything would otherwise loop forever,
|
|
321
|
+
// because progressed events do not count as attempts. A pending tick still appends a
|
|
322
|
+
// progressed event: without it the next tick would open a new stage-started and the
|
|
323
|
+
// pending window would never close.
|
|
324
|
+
async function recordIdleStageTick(args: {
|
|
325
|
+
readonly db: DbRunner;
|
|
326
|
+
readonly tenantId: TenantId;
|
|
327
|
+
readonly stage: DestructionStage;
|
|
328
|
+
readonly attempt: number;
|
|
329
|
+
readonly pending: boolean;
|
|
330
|
+
readonly events: readonly StoredEvent[];
|
|
331
|
+
readonly now: Temporal.Instant;
|
|
332
|
+
}): Promise<StageTickResult> {
|
|
333
|
+
if (args.pending && isPendingWindowOpen(args.events, args.stage.name, args.now)) {
|
|
334
|
+
await appendTenantStageEvent(
|
|
335
|
+
args.db,
|
|
336
|
+
args.tenantId,
|
|
337
|
+
qualified(TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT),
|
|
338
|
+
{ stage: args.stage.name, attempts: args.attempt, processed: 0, pending: true },
|
|
339
|
+
lastEventVersion(args.events),
|
|
340
|
+
);
|
|
341
|
+
return { done: false, pending: true };
|
|
342
|
+
}
|
|
343
|
+
return await recordFailedStageAttempt({
|
|
344
|
+
db: args.db,
|
|
345
|
+
tenantId: args.tenantId,
|
|
346
|
+
stage: args.stage,
|
|
347
|
+
attempt: args.attempt,
|
|
348
|
+
error: args.pending
|
|
349
|
+
? `stage "${args.stage.name}" still pending after ${DESTRUCTION_STAGE_MAX_PENDING_MS / 60_000} min`
|
|
350
|
+
: `stage "${args.stage.name}" made no progress`,
|
|
351
|
+
version: lastEventVersion(args.events),
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
290
355
|
export async function runNextDestructionStage(args: {
|
|
291
356
|
readonly db: DbRunner;
|
|
292
357
|
readonly registry: Registry;
|
|
@@ -298,7 +363,9 @@ export async function runNextDestructionStage(args: {
|
|
|
298
363
|
readonly escapeHatchAuditLog?: Logger;
|
|
299
364
|
readonly actor?: string;
|
|
300
365
|
readonly deadlineAt?: number;
|
|
366
|
+
readonly now?: Temporal.Instant;
|
|
301
367
|
}): Promise<StageTickResult> {
|
|
368
|
+
const now = args.now ?? getTemporal().Now.instant();
|
|
302
369
|
const events = await loadAggregate(args.db, args.tenantId, SYSTEM_TENANT_ID);
|
|
303
370
|
const { completed, abandoned, attemptsByStage } = replayStageState(events);
|
|
304
371
|
|
|
@@ -333,7 +400,7 @@ export async function runNextDestructionStage(args: {
|
|
|
333
400
|
args.db,
|
|
334
401
|
args.tenantId,
|
|
335
402
|
qualified(TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT),
|
|
336
|
-
{ stage: next.name, attempts: attempt },
|
|
403
|
+
{ stage: next.name, attempts: attempt, startedAt: now.toString() },
|
|
337
404
|
version,
|
|
338
405
|
stageOutcomeRecorded(events, next.name, TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT),
|
|
339
406
|
);
|
|
@@ -346,16 +413,15 @@ export async function runNextDestructionStage(args: {
|
|
|
346
413
|
// the version fresh instead of trusting the pre-run `version`, or this
|
|
347
414
|
// append throws VersionConflictError on every stage whose run() writes.
|
|
348
415
|
const eventsAfterRun = await loadAggregate(args.db, args.tenantId, SYSTEM_TENANT_ID);
|
|
349
|
-
// A tick that neither finishes nor processes anything would otherwise loop
|
|
350
|
-
// forever, because progressed events do not count as attempts.
|
|
351
416
|
if (!outcome.done && outcome.processed === 0) {
|
|
352
|
-
return await
|
|
417
|
+
return await recordIdleStageTick({
|
|
353
418
|
db: args.db,
|
|
354
419
|
tenantId: args.tenantId,
|
|
355
420
|
stage: next,
|
|
356
421
|
attempt,
|
|
357
|
-
|
|
358
|
-
|
|
422
|
+
pending: outcome.pending === true,
|
|
423
|
+
events: eventsAfterRun,
|
|
424
|
+
now,
|
|
359
425
|
});
|
|
360
426
|
}
|
|
361
427
|
if (!outcome.done) {
|
|
@@ -482,6 +548,7 @@ export async function runTenantDestructionSweep(args: {
|
|
|
482
548
|
try {
|
|
483
549
|
const result = await runNextDestructionStage({
|
|
484
550
|
deadlineAt: tenantDeadlineAt,
|
|
551
|
+
now,
|
|
485
552
|
db: args.db,
|
|
486
553
|
registry: args.registry,
|
|
487
554
|
tenantId: row.id,
|
|
@@ -69,7 +69,11 @@ export type DestructionStageCtx = {
|
|
|
69
69
|
readonly deadlineAt: number;
|
|
70
70
|
};
|
|
71
71
|
|
|
72
|
-
export type StageRunOutcome = {
|
|
72
|
+
export type StageRunOutcome = {
|
|
73
|
+
readonly done: boolean;
|
|
74
|
+
readonly processed: number;
|
|
75
|
+
readonly pending?: boolean;
|
|
76
|
+
};
|
|
73
77
|
|
|
74
78
|
const STAGE_DONE: StageRunOutcome = { done: true, processed: 0 };
|
|
75
79
|
|
|
@@ -97,13 +101,15 @@ function newOutcomeAccumulator(): {
|
|
|
97
101
|
outcome: () => StageRunOutcome;
|
|
98
102
|
} {
|
|
99
103
|
let allDone = true;
|
|
104
|
+
let pending = false;
|
|
100
105
|
let processed = 0;
|
|
101
106
|
return {
|
|
102
107
|
add: (result) => {
|
|
103
108
|
if (result?.done === false) allDone = false;
|
|
109
|
+
if (result?.done === false && result.pending === true) pending = true;
|
|
104
110
|
processed += result?.processed ?? 0;
|
|
105
111
|
},
|
|
106
|
-
outcome: () => ({ done: allDone, processed }),
|
|
112
|
+
outcome: () => ({ done: allDone, processed, pending }),
|
|
107
113
|
};
|
|
108
114
|
}
|
|
109
115
|
|
|
@@ -6,6 +6,7 @@ Pin jobs to deploy lanes (`api` | `worker`) with event-triggered fan-out.
|
|
|
6
6
|
|
|
7
7
|
- `r.job({ runIn: "worker" })` after an HTTP write
|
|
8
8
|
- `trigger: { maintenance: true }` for a daily housekeeping job in the app's maintenance window (`KUMIKO_MAINTENANCE_WINDOW`)
|
|
9
|
+
- `wakesSleepingApp: false` on a cron trigger for upkeep that must not wake a sleeping shared app
|
|
9
10
|
- End-to-end verification via `createAllInOneEntrypoint`
|
|
10
11
|
|
|
11
12
|
## Source
|
|
@@ -108,5 +108,13 @@ export function createLaneRoutingFeature(): FeatureDefinition {
|
|
|
108
108
|
r.job("prune-old-receipts", { trigger: { maintenance: true }, runIn: "worker" }, async () => {
|
|
109
109
|
pruneRuns.push(Date.now());
|
|
110
110
|
});
|
|
111
|
+
|
|
112
|
+
// Optional upkeep: `wakesSleepingApp: false` keeps a sleeping shared app asleep when a run is
|
|
113
|
+
// due; the run happens on the next wake instead.
|
|
114
|
+
r.job(
|
|
115
|
+
"warm-receipt-cache",
|
|
116
|
+
{ trigger: { cron: "*/15 * * * *", wakesSleepingApp: false }, runIn: "worker" },
|
|
117
|
+
async () => {},
|
|
118
|
+
);
|
|
111
119
|
});
|
|
112
120
|
}
|