@timurproko/a1 0.1.8-dev.224 → 0.1.8-dev.226
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/foundation/release/cohort-state.d.ts +25 -0
- package/dist/foundation/release/cohort-state.js +90 -3
- package/dist/foundation/release/release-gc.d.ts +12 -1
- package/dist/foundation/release/release-gc.js +252 -56
- package/dist/native/darwin-arm64/manifest.json +1 -1
- package/dist/native/linux-x64/manifest.json +1 -1
- package/dist/native/win32-x64/manifest.json +2 -2
- package/dist/native/win32-x64/process-guardian.exe +0 -0
- package/package.json +1 -1
|
@@ -43,9 +43,30 @@ export interface ReleaseCleanupDiagnostic {
|
|
|
43
43
|
readonly attemptedAt: string;
|
|
44
44
|
readonly error: string;
|
|
45
45
|
}
|
|
46
|
+
export type ReleaseCleanupWorkerStatus = "scheduled" | "running" | "completed" | "blocked" | "continued" | "failed" | "skipped";
|
|
47
|
+
export interface ReleaseCleanupWorkerRun {
|
|
48
|
+
readonly runId: string;
|
|
49
|
+
readonly status: ReleaseCleanupWorkerStatus;
|
|
50
|
+
readonly scheduledAt: string;
|
|
51
|
+
readonly startedAt: string | null;
|
|
52
|
+
readonly completedAt: string | null;
|
|
53
|
+
readonly pid: number | null;
|
|
54
|
+
readonly batches: number;
|
|
55
|
+
readonly attempted: number;
|
|
56
|
+
readonly completed: number;
|
|
57
|
+
readonly remaining: number;
|
|
58
|
+
readonly error: string | null;
|
|
59
|
+
}
|
|
60
|
+
export interface ReleaseCleanupWorkerSummary {
|
|
61
|
+
readonly batches: number;
|
|
62
|
+
readonly attempted: number;
|
|
63
|
+
readonly completed: number;
|
|
64
|
+
readonly remaining: number;
|
|
65
|
+
}
|
|
46
66
|
export interface ReleaseCleanupState {
|
|
47
67
|
readonly pending: Readonly<Record<string, ReleaseCleanupDisposition>>;
|
|
48
68
|
readonly diagnostics: readonly ReleaseCleanupDiagnostic[];
|
|
69
|
+
readonly workerRuns: readonly ReleaseCleanupWorkerRun[];
|
|
49
70
|
}
|
|
50
71
|
export interface CohortState {
|
|
51
72
|
readonly schema: typeof RELEASE_COHORT_SCHEMA;
|
|
@@ -126,6 +147,10 @@ export declare class CohortStateStore {
|
|
|
126
147
|
markCleanupTrash(releaseId: string, trashPath: string): Promise<CohortState>;
|
|
127
148
|
recordCleanupFailure(releaseId: string, stage: string, error: unknown): Promise<CohortState>;
|
|
128
149
|
completeCleanup(releaseId: string): Promise<CohortState>;
|
|
150
|
+
recordCleanupWorkerScheduled(runId: string): Promise<CohortState>;
|
|
151
|
+
recordCleanupWorkerStarted(runId: string, pid: number): Promise<CohortState>;
|
|
152
|
+
recordCleanupWorkerProgress(runId: string, summary: ReleaseCleanupWorkerSummary): Promise<CohortState>;
|
|
153
|
+
recordCleanupWorkerFinished(runId: string, status: Exclude<ReleaseCleanupWorkerStatus, "scheduled" | "running">, summary: ReleaseCleanupWorkerSummary, error?: unknown): Promise<CohortState>;
|
|
129
154
|
removeUnreferencedRelease(releaseId: string, externalReferences: readonly string[]): Promise<CohortState>;
|
|
130
155
|
}
|
|
131
156
|
export declare function emptyState(): CohortState;
|
|
@@ -219,6 +219,7 @@ export class CohortStateStore {
|
|
|
219
219
|
return {
|
|
220
220
|
...current,
|
|
221
221
|
cleanup: {
|
|
222
|
+
...current.cleanup,
|
|
222
223
|
pending,
|
|
223
224
|
diagnostics: [...current.cleanup.diagnostics, { releaseId, stage, attemptedAt, error: message }].slice(-64),
|
|
224
225
|
},
|
|
@@ -232,6 +233,55 @@ export class CohortStateStore {
|
|
|
232
233
|
return { ...current, cleanup: { ...current.cleanup, pending } };
|
|
233
234
|
});
|
|
234
235
|
}
|
|
236
|
+
async recordCleanupWorkerScheduled(runId) {
|
|
237
|
+
return await this.update(current => {
|
|
238
|
+
const scheduledAt = new Date().toISOString();
|
|
239
|
+
const run = {
|
|
240
|
+
runId,
|
|
241
|
+
status: "scheduled",
|
|
242
|
+
scheduledAt,
|
|
243
|
+
startedAt: null,
|
|
244
|
+
completedAt: null,
|
|
245
|
+
pid: null,
|
|
246
|
+
batches: 0,
|
|
247
|
+
attempted: 0,
|
|
248
|
+
completed: 0,
|
|
249
|
+
remaining: Object.keys(current.cleanup.pending).length,
|
|
250
|
+
error: null,
|
|
251
|
+
};
|
|
252
|
+
return { ...current, cleanup: { ...current.cleanup, workerRuns: [...current.cleanup.workerRuns, run].slice(-16) } };
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
async recordCleanupWorkerStarted(runId, pid) {
|
|
256
|
+
return await this.update(current => replaceCleanupWorkerRun(current, runId, existing => ({
|
|
257
|
+
...(existing ?? cleanupWorkerRun(runId, Object.keys(current.cleanup.pending).length)),
|
|
258
|
+
status: "running",
|
|
259
|
+
startedAt: new Date().toISOString(),
|
|
260
|
+
completedAt: null,
|
|
261
|
+
pid,
|
|
262
|
+
error: null,
|
|
263
|
+
})));
|
|
264
|
+
}
|
|
265
|
+
async recordCleanupWorkerProgress(runId, summary) {
|
|
266
|
+
return await this.update(current => replaceCleanupWorkerRun(current, runId, existing => ({
|
|
267
|
+
...(existing ?? cleanupWorkerRun(runId, summary.remaining)),
|
|
268
|
+
status: "running",
|
|
269
|
+
startedAt: existing?.startedAt ?? new Date().toISOString(),
|
|
270
|
+
pid: existing?.pid ?? process.pid,
|
|
271
|
+
...summary,
|
|
272
|
+
})));
|
|
273
|
+
}
|
|
274
|
+
async recordCleanupWorkerFinished(runId, status, summary, error = null) {
|
|
275
|
+
return await this.update(current => replaceCleanupWorkerRun(current, runId, existing => ({
|
|
276
|
+
...(existing ?? cleanupWorkerRun(runId, summary.remaining)),
|
|
277
|
+
status,
|
|
278
|
+
startedAt: existing?.startedAt ?? new Date().toISOString(),
|
|
279
|
+
completedAt: new Date().toISOString(),
|
|
280
|
+
pid: existing?.pid ?? process.pid,
|
|
281
|
+
...summary,
|
|
282
|
+
error: error === null ? null : boundedError(error),
|
|
283
|
+
})));
|
|
284
|
+
}
|
|
235
285
|
async removeUnreferencedRelease(releaseId, externalReferences) {
|
|
236
286
|
return await this.update(current => {
|
|
237
287
|
const release = requiredRelease(current, releaseId);
|
|
@@ -264,7 +314,7 @@ export function emptyState() {
|
|
|
264
314
|
revision: 0,
|
|
265
315
|
releases: {},
|
|
266
316
|
references: { active: null, pending: null, approved: null, rollback: null, retention: [] },
|
|
267
|
-
cleanup: { pending: {}, diagnostics: [] },
|
|
317
|
+
cleanup: { pending: {}, diagnostics: [], workerRuns: [] },
|
|
268
318
|
activation: activation("idle", null),
|
|
269
319
|
};
|
|
270
320
|
}
|
|
@@ -274,6 +324,28 @@ function cleanupDisposition(release, stage, trashPath) {
|
|
|
274
324
|
function replaceCleanup(current, releaseId, disposition) {
|
|
275
325
|
return { ...current, cleanup: { ...current.cleanup, pending: { ...current.cleanup.pending, [releaseId]: disposition } } };
|
|
276
326
|
}
|
|
327
|
+
function cleanupWorkerRun(runId, remaining) {
|
|
328
|
+
return {
|
|
329
|
+
runId,
|
|
330
|
+
status: "scheduled",
|
|
331
|
+
scheduledAt: new Date().toISOString(),
|
|
332
|
+
startedAt: null,
|
|
333
|
+
completedAt: null,
|
|
334
|
+
pid: null,
|
|
335
|
+
batches: 0,
|
|
336
|
+
attempted: 0,
|
|
337
|
+
completed: 0,
|
|
338
|
+
remaining,
|
|
339
|
+
error: null,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
function replaceCleanupWorkerRun(current, runId, operation) {
|
|
343
|
+
const existing = current.cleanup.workerRuns.find(run => run.runId === runId);
|
|
344
|
+
const workerRuns = existing
|
|
345
|
+
? current.cleanup.workerRuns.map(run => run.runId === runId ? operation(run) : run)
|
|
346
|
+
: [...current.cleanup.workerRuns, operation(undefined)];
|
|
347
|
+
return { ...current, cleanup: { ...current.cleanup, workerRuns: workerRuns.slice(-16) } };
|
|
348
|
+
}
|
|
277
349
|
function activation(state, reason) {
|
|
278
350
|
return { state, reason, blockerGenerationIds: [], updatedAt: new Date().toISOString() };
|
|
279
351
|
}
|
|
@@ -342,17 +414,32 @@ function normalizeState(value) {
|
|
|
342
414
|
if (!value || typeof value !== "object")
|
|
343
415
|
return value;
|
|
344
416
|
const state = value;
|
|
417
|
+
const cleanup = state.cleanup;
|
|
345
418
|
return {
|
|
346
419
|
...state,
|
|
347
|
-
cleanup:
|
|
420
|
+
cleanup: cleanup
|
|
421
|
+
? { ...cleanup, workerRuns: cleanup.workerRuns ?? [] }
|
|
422
|
+
: { pending: {}, diagnostics: [], workerRuns: [] },
|
|
348
423
|
};
|
|
349
424
|
}
|
|
350
425
|
function validateState(state) {
|
|
351
426
|
if (!state || state.schema !== RELEASE_COHORT_SCHEMA || !Number.isSafeInteger(state.revision) || state.revision < 0
|
|
352
427
|
|| !state.releases || typeof state.releases !== "object" || !state.references || !Array.isArray(state.references.retention)
|
|
353
|
-
|| !state.cleanup || typeof state.cleanup.pending !== "object" || !Array.isArray(state.cleanup.diagnostics)
|
|
428
|
+
|| !state.cleanup || typeof state.cleanup.pending !== "object" || !Array.isArray(state.cleanup.diagnostics)
|
|
429
|
+
|| !Array.isArray(state.cleanup.workerRuns)) {
|
|
354
430
|
throw new Error("invalid release cohort state");
|
|
355
431
|
}
|
|
432
|
+
for (const run of state.cleanup.workerRuns) {
|
|
433
|
+
if (!run || typeof run.runId !== "string" || run.runId.length === 0
|
|
434
|
+
|| !["scheduled", "running", "completed", "blocked", "continued", "failed", "skipped"].includes(run.status)
|
|
435
|
+
|| typeof run.scheduledAt !== "string" || (run.startedAt !== null && typeof run.startedAt !== "string")
|
|
436
|
+
|| (run.completedAt !== null && typeof run.completedAt !== "string") || (run.pid !== null && (!Number.isSafeInteger(run.pid) || run.pid < 1))
|
|
437
|
+
|| !Number.isSafeInteger(run.batches) || run.batches < 0 || !Number.isSafeInteger(run.attempted) || run.attempted < 0
|
|
438
|
+
|| !Number.isSafeInteger(run.completed) || run.completed < 0 || run.completed > run.attempted
|
|
439
|
+
|| !Number.isSafeInteger(run.remaining) || run.remaining < 0 || (run.error !== null && typeof run.error !== "string")) {
|
|
440
|
+
throw new Error("invalid release cleanup worker run");
|
|
441
|
+
}
|
|
442
|
+
}
|
|
356
443
|
for (const [releaseId, release] of Object.entries(state.releases))
|
|
357
444
|
validateReleaseRecord(releaseId, release);
|
|
358
445
|
for (const reference of [state.references.active, state.references.pending, state.references.approved, state.references.rollback]) {
|
|
@@ -19,6 +19,17 @@ export interface ReleaseCleanupOptions {
|
|
|
19
19
|
readonly now?: () => number;
|
|
20
20
|
/** Fault-injection seam for bounded filesystem recovery tests. */
|
|
21
21
|
readonly operations?: ReleaseCleanupOperations;
|
|
22
|
+
/** Process-start seam for scheduler and continuation tests. */
|
|
23
|
+
readonly workerSpawner?: (entry: string, environment: NodeJS.ProcessEnv) => Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
export interface ReleaseCleanupWorkerOptions {
|
|
26
|
+
readonly cleanup?: ReleaseCleanupOptions;
|
|
27
|
+
readonly maxDurationMs?: number;
|
|
28
|
+
readonly maxNoProgressBatches?: number;
|
|
29
|
+
readonly retryDelayMs?: number;
|
|
30
|
+
readonly now?: () => number;
|
|
31
|
+
readonly sleep?: (durationMs: number) => Promise<void>;
|
|
32
|
+
readonly continueWorker?: (dataDir: string, paths: ProductPaths, options: ReleaseCleanupOptions) => Promise<void>;
|
|
22
33
|
}
|
|
23
34
|
export interface ReleaseCleanupResult {
|
|
24
35
|
readonly planned: number;
|
|
@@ -54,7 +65,7 @@ export declare function runBoundedReleaseCleanup(dataDir: string, paths?: Produc
|
|
|
54
65
|
*/
|
|
55
66
|
export declare function scheduleReleaseCleanup(dataDir: string, paths: ProductPaths, options?: ReleaseCleanupOptions): Promise<void>;
|
|
56
67
|
/** Entry used by the private cleanup executable shipped in every immutable release. */
|
|
57
|
-
export declare function runReleaseCleanupWorker(environment?: NodeJS.ProcessEnv): Promise<ReleaseCleanupResult>;
|
|
68
|
+
export declare function runReleaseCleanupWorker(environment?: NodeJS.ProcessEnv, workerOptions?: ReleaseCleanupWorkerOptions): Promise<ReleaseCleanupResult>;
|
|
58
69
|
/** Compatibility helper: detach one requested release and execute a one-item pass. */
|
|
59
70
|
export declare function collectRelease(store: CohortStateStore, dataDir: string, releaseId: string, externalReferences: readonly string[], paths?: ProductPaths): Promise<void>;
|
|
60
71
|
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { spawn } from "node:child_process";
|
|
3
|
-
import { lstat, mkdir, readFile, readdir, realpath, rename, rm } from "node:fs/promises";
|
|
3
|
+
import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, stat } from "node:fs/promises";
|
|
4
4
|
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { CohortStateStore, planProtectedReleases, } from "./cohort-state.js";
|
|
@@ -13,7 +13,12 @@ import { PRODUCT_IDENTITY } from "../../product-identity.js";
|
|
|
13
13
|
import { collectCompileCaches, startupCompileCachePath } from "../startup/index.js";
|
|
14
14
|
const DEFAULT_CANDIDATE_AGE_MS = 60 * 60 * 1_000;
|
|
15
15
|
const DEFAULT_LIMITS = { maxItems: 8, maxDurationMs: 2_000, concurrency: 1 };
|
|
16
|
+
const DEFAULT_WORKER_DURATION_MS = 10 * 60 * 1_000;
|
|
17
|
+
const DEFAULT_NO_PROGRESS_BATCHES = 5;
|
|
18
|
+
const DEFAULT_RETRY_DELAY_MS = 500;
|
|
16
19
|
const WORKER_HOLDS_ENV = "A1_RELEASE_CLEANUP_HOLDS";
|
|
20
|
+
const WORKER_RUN_ID_ENV = "A1_RELEASE_CLEANUP_RUN_ID";
|
|
21
|
+
const WORKER_LEASE_FILENAME = "release-cleanup-worker.lock";
|
|
17
22
|
/**
|
|
18
23
|
* Reconcile current ownership under the cohort-state lock and durably detach obsolete releases.
|
|
19
24
|
* This operation intentionally reads only manifests and directory metadata, never payload bytes.
|
|
@@ -30,52 +35,85 @@ export async function prepareReleaseCleanup(dataDir, paths, options = {}) {
|
|
|
30
35
|
}
|
|
31
36
|
/** Run a bounded, restart-safe physical cleanup pass over detached releases and abandoned artifacts. */
|
|
32
37
|
export async function runBoundedReleaseCleanup(dataDir, paths, options = {}) {
|
|
33
|
-
const startedAt = (options.now ?? Date.now)();
|
|
34
38
|
const now = options.now ?? Date.now;
|
|
35
39
|
const limits = normalizeLimits(options.limits);
|
|
36
40
|
const { store, discovered } = await prepareReleaseCleanup(dataDir, paths, options);
|
|
41
|
+
// Performance: The physical-work allowance begins after discovery, ownership probes, and the
|
|
42
|
+
// durable reconciliation write. A slow preparation phase must not starve every worker invocation.
|
|
43
|
+
const startedAt = now();
|
|
37
44
|
const initial = await store.read();
|
|
38
|
-
const pending = Object.values(initial.cleanup.pending)
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
for (let offset = 0; offset < pending.length && attempted < limits.maxItems; offset += limits.concurrency) {
|
|
42
|
-
if (now() - startedAt >= limits.maxDurationMs)
|
|
43
|
-
break;
|
|
44
|
-
const batch = pending.slice(offset, Math.min(offset + limits.concurrency, offset + (limits.maxItems - attempted)));
|
|
45
|
-
attempted += batch.length;
|
|
46
|
-
const results = await Promise.all(batch.map(async (disposition) => await collectDisposition(store, dataDir, paths, disposition, options)));
|
|
47
|
-
completed += results.filter(Boolean).length;
|
|
48
|
-
}
|
|
49
|
-
const transaction = await (options.transactionStore ?? new UpdateTransactionStore(dataDir)).read();
|
|
45
|
+
const pending = sortCleanupDispositions(Object.values(initial.cleanup.pending));
|
|
46
|
+
const transactionStore = options.transactionStore ?? new UpdateTransactionStore(dataDir);
|
|
47
|
+
const transaction = await transactionStore.read();
|
|
50
48
|
const activeTransaction = transaction?.status === "active";
|
|
51
|
-
const
|
|
52
|
-
const artifacts = [
|
|
49
|
+
const artifacts = sortArtifacts([
|
|
53
50
|
...(!activeTransaction ? discovered.candidates : []),
|
|
54
51
|
...discovered.unmanagedTrash,
|
|
55
52
|
...discovered.certifications,
|
|
56
|
-
]
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
53
|
+
], initial);
|
|
54
|
+
let attempted = 0;
|
|
55
|
+
let completed = 0;
|
|
56
|
+
let pendingOffset = 0;
|
|
57
|
+
let artifactOffset = 0;
|
|
58
|
+
const hasTime = (guaranteed = false) => guaranteed || attempted === 0 || now() - startedAt < limits.maxDurationMs;
|
|
59
|
+
const attemptPending = async (maximum, guaranteed = false) => {
|
|
60
|
+
while (pendingOffset < pending.length && attempted < limits.maxItems && maximum > 0 && hasTime(guaranteed)) {
|
|
61
|
+
const size = Math.min(limits.concurrency, maximum, limits.maxItems - attempted, pending.length - pendingOffset);
|
|
62
|
+
const batch = pending.slice(pendingOffset, pendingOffset + size);
|
|
63
|
+
pendingOffset += size;
|
|
64
|
+
maximum -= size;
|
|
65
|
+
attempted += batch.length;
|
|
66
|
+
const results = await Promise.all(batch.map(async (disposition) => await collectDisposition(store, dataDir, paths, disposition, options)));
|
|
67
|
+
completed += results.filter(Boolean).length;
|
|
68
|
+
guaranteed = false;
|
|
70
69
|
}
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
};
|
|
71
|
+
const attemptArtifacts = async (maximum, guaranteed = false) => {
|
|
72
|
+
while (artifactOffset < artifacts.length && attempted < limits.maxItems && maximum > 0 && hasTime(guaranteed)) {
|
|
73
|
+
const artifact = artifacts[artifactOffset++];
|
|
74
|
+
maximum -= 1;
|
|
75
|
+
const certificationId = certificationReleaseId(artifact);
|
|
76
|
+
if (certificationId !== null) {
|
|
77
|
+
const inputs = await protectionInputs(paths, transactionStore, options.externalHolds ?? []);
|
|
78
|
+
if (isProtectedDetachedRelease(await store.read(), inputs, certificationId))
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
attempted += 1;
|
|
82
|
+
try {
|
|
83
|
+
await removeAbandonedArtifact(dataDir, artifact, options.operations);
|
|
84
|
+
completed += 1;
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
await store.recordCleanupFailure(artifactName(artifact), "artifact-delete", error);
|
|
88
|
+
}
|
|
89
|
+
guaranteed = false;
|
|
73
90
|
}
|
|
91
|
+
};
|
|
92
|
+
// Concurrency: Give each available class one opportunity before filling the batch. This keeps
|
|
93
|
+
// a locked release or a large release backlog from indefinitely starving candidates and evidence.
|
|
94
|
+
if (pending.length > 0)
|
|
95
|
+
await attemptPending(1, true);
|
|
96
|
+
if (artifacts.length > 0 && attempted < limits.maxItems)
|
|
97
|
+
await attemptArtifacts(1, true);
|
|
98
|
+
if (!activeTransaction && attempted < limits.maxItems) {
|
|
99
|
+
const layerResult = await collectUnreferencedDependencyLayers(store, dataDir, 1, options);
|
|
100
|
+
attempted += layerResult.attempted;
|
|
101
|
+
completed += layerResult.completed;
|
|
102
|
+
}
|
|
103
|
+
while (attempted < limits.maxItems && hasTime()) {
|
|
104
|
+
const before = attempted;
|
|
105
|
+
await attemptPending(1);
|
|
106
|
+
if (attempted < limits.maxItems)
|
|
107
|
+
await attemptArtifacts(1);
|
|
108
|
+
if (attempted === before)
|
|
109
|
+
break;
|
|
74
110
|
}
|
|
75
|
-
if (!activeTransaction && attempted < limits.maxItems &&
|
|
111
|
+
if (!activeTransaction && attempted < limits.maxItems && hasTime()) {
|
|
76
112
|
const layerResult = await collectUnreferencedDependencyLayers(store, dataDir, limits.maxItems - attempted, options);
|
|
77
113
|
attempted += layerResult.attempted;
|
|
78
114
|
completed += layerResult.completed;
|
|
115
|
+
}
|
|
116
|
+
if (!activeTransaction && hasTime()) {
|
|
79
117
|
await collectCompileCaches(dataDir, await protectedCompileCachePaths(dataDir, await store.read())).catch(() => { });
|
|
80
118
|
}
|
|
81
119
|
const remaining = Object.keys((await store.read()).cleanup.pending).length;
|
|
@@ -87,33 +125,102 @@ export async function runBoundedReleaseCleanup(dataDir, paths, options = {}) {
|
|
|
87
125
|
*/
|
|
88
126
|
export async function scheduleReleaseCleanup(dataDir, paths, options = {}) {
|
|
89
127
|
const { store } = await prepareReleaseCleanup(dataDir, paths, options);
|
|
128
|
+
const runId = randomUUID();
|
|
129
|
+
await store.recordCleanupWorkerScheduled(runId);
|
|
90
130
|
const entry = fileURLToPath(new URL("../../../bin/release-cleanup.js", import.meta.url));
|
|
131
|
+
const environment = {
|
|
132
|
+
...process.env,
|
|
133
|
+
[PRODUCT_IDENTITY.environment.dataDir]: dataDir,
|
|
134
|
+
[PRODUCT_IDENTITY.environment.runtimeDir]: paths.runtimeDir,
|
|
135
|
+
[WORKER_HOLDS_ENV]: JSON.stringify(options.externalHolds ?? []),
|
|
136
|
+
[WORKER_RUN_ID_ENV]: runId,
|
|
137
|
+
};
|
|
91
138
|
try {
|
|
92
|
-
|
|
93
|
-
detached: true,
|
|
94
|
-
windowsHide: true,
|
|
95
|
-
stdio: "ignore",
|
|
96
|
-
env: {
|
|
97
|
-
...process.env,
|
|
98
|
-
[PRODUCT_IDENTITY.environment.dataDir]: dataDir,
|
|
99
|
-
[PRODUCT_IDENTITY.environment.runtimeDir]: paths.runtimeDir,
|
|
100
|
-
[WORKER_HOLDS_ENV]: JSON.stringify(options.externalHolds ?? []),
|
|
101
|
-
},
|
|
102
|
-
});
|
|
103
|
-
await new Promise((resolvePromise, rejectPromise) => {
|
|
104
|
-
child.once("spawn", resolvePromise);
|
|
105
|
-
child.once("error", rejectPromise);
|
|
106
|
-
});
|
|
107
|
-
child.unref();
|
|
139
|
+
await (options.workerSpawner ?? spawnCleanupWorker)(entry, environment);
|
|
108
140
|
}
|
|
109
141
|
catch (error) {
|
|
142
|
+
const summary = { batches: 0, attempted: 0, completed: 0, remaining: Object.keys((await store.read()).cleanup.pending).length };
|
|
143
|
+
await store.recordCleanupWorkerFinished(runId, "failed", summary, error);
|
|
110
144
|
await store.recordCleanupFailure("worker", "spawn", error);
|
|
111
145
|
}
|
|
112
146
|
}
|
|
113
147
|
/** Entry used by the private cleanup executable shipped in every immutable release. */
|
|
114
|
-
export async function runReleaseCleanupWorker(environment = process.env) {
|
|
148
|
+
export async function runReleaseCleanupWorker(environment = process.env, workerOptions = {}) {
|
|
115
149
|
const paths = resolveProductPaths(environment);
|
|
116
|
-
|
|
150
|
+
const store = new CohortStateStore(paths.dataDir);
|
|
151
|
+
const runId = environment[WORKER_RUN_ID_ENV] ?? randomUUID();
|
|
152
|
+
const now = workerOptions.now ?? Date.now;
|
|
153
|
+
const sleep = workerOptions.sleep ?? (async (durationMs) => await new Promise(resolvePromise => setTimeout(resolvePromise, durationMs)));
|
|
154
|
+
const cleanupOptions = {
|
|
155
|
+
...workerOptions.cleanup,
|
|
156
|
+
externalHolds: workerOptions.cleanup?.externalHolds ?? parseWorkerHolds(environment[WORKER_HOLDS_ENV]),
|
|
157
|
+
};
|
|
158
|
+
const summary = { batches: 0, attempted: 0, completed: 0, remaining: Object.keys((await store.read()).cleanup.pending).length };
|
|
159
|
+
let lease = null;
|
|
160
|
+
let continuation = false;
|
|
161
|
+
let status = "completed";
|
|
162
|
+
let failure = null;
|
|
163
|
+
const startedAt = now();
|
|
164
|
+
try {
|
|
165
|
+
lease = await acquireWorkerLease(paths.dataDir, runId);
|
|
166
|
+
if (lease === null) {
|
|
167
|
+
status = "skipped";
|
|
168
|
+
failure = "another release cleanup worker owns the data root";
|
|
169
|
+
return { planned: 0, attempted: 0, completed: 0, remaining: summary.remaining, durationMs: 0 };
|
|
170
|
+
}
|
|
171
|
+
await store.recordCleanupWorkerStarted(runId, process.pid);
|
|
172
|
+
let noProgressBatches = 0;
|
|
173
|
+
while (true) {
|
|
174
|
+
const result = await runBoundedReleaseCleanup(paths.dataDir, paths, cleanupOptions);
|
|
175
|
+
summary.batches += 1;
|
|
176
|
+
summary.attempted += result.attempted;
|
|
177
|
+
summary.completed += result.completed;
|
|
178
|
+
summary.remaining = result.remaining;
|
|
179
|
+
await store.recordCleanupWorkerProgress(runId, summary);
|
|
180
|
+
if (result.attempted === 0)
|
|
181
|
+
break;
|
|
182
|
+
if (result.completed === 0) {
|
|
183
|
+
noProgressBatches += 1;
|
|
184
|
+
if (noProgressBatches >= (workerOptions.maxNoProgressBatches ?? DEFAULT_NO_PROGRESS_BATCHES)) {
|
|
185
|
+
status = "blocked";
|
|
186
|
+
failure = "cleanup made no progress within its bounded retry allowance";
|
|
187
|
+
break;
|
|
188
|
+
}
|
|
189
|
+
await sleep((workerOptions.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS) * (2 ** (noProgressBatches - 1)));
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
noProgressBatches = 0;
|
|
193
|
+
}
|
|
194
|
+
if (now() - startedAt >= (workerOptions.maxDurationMs ?? DEFAULT_WORKER_DURATION_MS)) {
|
|
195
|
+
status = "continued";
|
|
196
|
+
continuation = true;
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
status = "failed";
|
|
203
|
+
failure = error;
|
|
204
|
+
}
|
|
205
|
+
finally {
|
|
206
|
+
if (lease !== null)
|
|
207
|
+
await releaseWorkerLease(paths.dataDir, runId, lease);
|
|
208
|
+
await store.recordCleanupWorkerFinished(runId, status, summary, failure);
|
|
209
|
+
}
|
|
210
|
+
if (continuation) {
|
|
211
|
+
await (workerOptions.continueWorker ?? scheduleReleaseCleanup)(paths.dataDir, paths, cleanupOptions).catch(async (error) => {
|
|
212
|
+
await store.recordCleanupFailure("worker", "continuation", error);
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
if (status === "failed")
|
|
216
|
+
throw failure instanceof Error ? failure : new Error(errorMessage(failure));
|
|
217
|
+
return {
|
|
218
|
+
planned: summary.remaining + summary.completed,
|
|
219
|
+
attempted: summary.attempted,
|
|
220
|
+
completed: summary.completed,
|
|
221
|
+
remaining: summary.remaining,
|
|
222
|
+
durationMs: Math.max(0, now() - startedAt),
|
|
223
|
+
};
|
|
117
224
|
}
|
|
118
225
|
/** Compatibility helper: detach one requested release and execute a one-item pass. */
|
|
119
226
|
export async function collectRelease(store, dataDir, releaseId, externalReferences, paths) {
|
|
@@ -305,11 +412,9 @@ async function assertReleaseDirectory(path, parent, release, expectedName) {
|
|
|
305
412
|
async function removeCertificationEvidence(dataDir, release, store) {
|
|
306
413
|
if (release.diagnosticsPath === null)
|
|
307
414
|
return;
|
|
415
|
+
// Security: the persisted path is historical metadata, not deletion authority. Derive the
|
|
416
|
+
// only permitted target from validated identity so Windows casing changes cannot strand it.
|
|
308
417
|
const expected = resolve(dataDir, `certification-${release.releaseId}.json`);
|
|
309
|
-
if (resolve(release.diagnosticsPath) !== expected) {
|
|
310
|
-
await store.recordCleanupFailure(release.releaseId, "certification", "refused to delete certification evidence outside its managed identity path");
|
|
311
|
-
return;
|
|
312
|
-
}
|
|
313
418
|
const metadata = await lstat(expected).catch(error => missingOrThrow(error));
|
|
314
419
|
if (metadata === null)
|
|
315
420
|
return;
|
|
@@ -317,7 +422,13 @@ async function removeCertificationEvidence(dataDir, release, store) {
|
|
|
317
422
|
await store.recordCleanupFailure(release.releaseId, "certification", "refused to delete linked or non-file certification evidence");
|
|
318
423
|
return;
|
|
319
424
|
}
|
|
320
|
-
|
|
425
|
+
const [canonicalDataDir, canonicalEvidence] = await Promise.all([realpath(dataDir), realpath(expected)]);
|
|
426
|
+
assertDirectChild(canonicalDataDir, canonicalEvidence);
|
|
427
|
+
if (artifactName(canonicalEvidence) !== `certification-${release.releaseId}.json`) {
|
|
428
|
+
await store.recordCleanupFailure(release.releaseId, "certification", "refused to delete certification evidence with a different managed identity");
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
await rm(canonicalEvidence, { force: true });
|
|
321
432
|
}
|
|
322
433
|
async function canonicalReleaseStore(dataDir) {
|
|
323
434
|
const root = resolve(dataDir, "releases");
|
|
@@ -511,6 +622,91 @@ function normalizeLimits(input) {
|
|
|
511
622
|
}
|
|
512
623
|
return limits;
|
|
513
624
|
}
|
|
625
|
+
function sortCleanupDispositions(dispositions) {
|
|
626
|
+
return [...dispositions].sort((left, right) => left.attempts - right.attempts
|
|
627
|
+
|| (left.lastAttemptAt ?? "").localeCompare(right.lastAttemptAt ?? "")
|
|
628
|
+
|| left.release.releaseId.localeCompare(right.release.releaseId));
|
|
629
|
+
}
|
|
630
|
+
function sortArtifacts(artifacts, state) {
|
|
631
|
+
const attempts = new Map();
|
|
632
|
+
for (const diagnostic of state.cleanup.diagnostics) {
|
|
633
|
+
attempts.set(diagnostic.releaseId, (attempts.get(diagnostic.releaseId) ?? 0) + 1);
|
|
634
|
+
}
|
|
635
|
+
return [...artifacts].sort((left, right) => (attempts.get(artifactName(left)) ?? 0) - (attempts.get(artifactName(right)) ?? 0)
|
|
636
|
+
|| artifactName(left).localeCompare(artifactName(right)));
|
|
637
|
+
}
|
|
638
|
+
async function spawnCleanupWorker(entry, environment) {
|
|
639
|
+
const child = spawn(process.execPath, [entry], {
|
|
640
|
+
detached: true,
|
|
641
|
+
windowsHide: true,
|
|
642
|
+
stdio: "ignore",
|
|
643
|
+
env: environment,
|
|
644
|
+
});
|
|
645
|
+
await new Promise((resolvePromise, rejectPromise) => {
|
|
646
|
+
child.once("spawn", resolvePromise);
|
|
647
|
+
child.once("error", rejectPromise);
|
|
648
|
+
});
|
|
649
|
+
child.unref();
|
|
650
|
+
}
|
|
651
|
+
async function acquireWorkerLease(dataDir, runId) {
|
|
652
|
+
const path = resolve(dataDir, WORKER_LEASE_FILENAME);
|
|
653
|
+
await mkdir(dataDir, { recursive: true, mode: 0o700 });
|
|
654
|
+
try {
|
|
655
|
+
const lease = await open(path, "wx", 0o600);
|
|
656
|
+
await lease.writeFile(JSON.stringify({ runId, pid: process.pid, createdAt: new Date().toISOString() }));
|
|
657
|
+
await lease.sync();
|
|
658
|
+
return lease;
|
|
659
|
+
}
|
|
660
|
+
catch (error) {
|
|
661
|
+
if (!(error instanceof Error && "code" in error && error.code === "EEXIST"))
|
|
662
|
+
throw error;
|
|
663
|
+
}
|
|
664
|
+
const owner = await readWorkerLease(path);
|
|
665
|
+
if (owner?.pid && processIsAlive(owner.pid))
|
|
666
|
+
return null;
|
|
667
|
+
if (owner === null) {
|
|
668
|
+
const metadata = await stat(path).catch(() => null);
|
|
669
|
+
if (metadata !== null && Date.now() - metadata.mtimeMs < 10_000)
|
|
670
|
+
return null;
|
|
671
|
+
}
|
|
672
|
+
const quarantine = `${path}.${randomUUID()}.abandoned`;
|
|
673
|
+
try {
|
|
674
|
+
await rename(path, quarantine);
|
|
675
|
+
await rm(quarantine, { force: true });
|
|
676
|
+
}
|
|
677
|
+
catch {
|
|
678
|
+
return null;
|
|
679
|
+
}
|
|
680
|
+
return await acquireWorkerLease(dataDir, runId);
|
|
681
|
+
}
|
|
682
|
+
async function releaseWorkerLease(dataDir, runId, lease) {
|
|
683
|
+
const path = resolve(dataDir, WORKER_LEASE_FILENAME);
|
|
684
|
+
await lease.close().catch(() => { });
|
|
685
|
+
const owner = await readWorkerLease(path);
|
|
686
|
+
if (owner?.runId === runId)
|
|
687
|
+
await rm(path, { force: true });
|
|
688
|
+
}
|
|
689
|
+
async function readWorkerLease(path) {
|
|
690
|
+
try {
|
|
691
|
+
const value = JSON.parse(await readFile(path, "utf8"));
|
|
692
|
+
return {
|
|
693
|
+
...(typeof value.runId === "string" ? { runId: value.runId } : {}),
|
|
694
|
+
...(typeof value.pid === "number" ? { pid: value.pid } : {}),
|
|
695
|
+
};
|
|
696
|
+
}
|
|
697
|
+
catch {
|
|
698
|
+
return null;
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
function processIsAlive(pid) {
|
|
702
|
+
try {
|
|
703
|
+
process.kill(pid, 0);
|
|
704
|
+
return true;
|
|
705
|
+
}
|
|
706
|
+
catch (error) {
|
|
707
|
+
return error instanceof Error && "code" in error && error.code === "EPERM";
|
|
708
|
+
}
|
|
709
|
+
}
|
|
514
710
|
function parseWorkerHolds(value) {
|
|
515
711
|
if (!value)
|
|
516
712
|
return [];
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"platform": "darwin",
|
|
6
6
|
"architecture": "arm64",
|
|
7
7
|
"capability": "unsupported",
|
|
8
|
-
"builtAt": "2026-09-
|
|
8
|
+
"builtAt": "2026-09-03T18:06:08.893Z",
|
|
9
9
|
"artifact": {
|
|
10
10
|
"filename": "process-guardian",
|
|
11
11
|
"sha256": "7524bf568992517f50ed53196c861c5edeba0d7275633bb4735ab45bd5963a28",
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"platform": "linux",
|
|
6
6
|
"architecture": "x64",
|
|
7
7
|
"capability": "supported",
|
|
8
|
-
"builtAt": "2026-09-
|
|
8
|
+
"builtAt": "2026-09-03T18:05:58.567Z",
|
|
9
9
|
"artifact": {
|
|
10
10
|
"filename": "process-guardian",
|
|
11
11
|
"sha256": "ee8a00eaaf79c707459bbbfb52518e9739314967049fbe5fa625f36ce33db9ee",
|
|
@@ -5,10 +5,10 @@
|
|
|
5
5
|
"platform": "win32",
|
|
6
6
|
"architecture": "x64",
|
|
7
7
|
"capability": "supported",
|
|
8
|
-
"builtAt": "2026-09-
|
|
8
|
+
"builtAt": "2026-09-03T18:06:42.554Z",
|
|
9
9
|
"artifact": {
|
|
10
10
|
"filename": "process-guardian.exe",
|
|
11
|
-
"sha256": "
|
|
11
|
+
"sha256": "5e8247ce99c85457ecef2f8f0fd56bbd42796613037563a570fa8ad2b2fb2567",
|
|
12
12
|
"size": 177664
|
|
13
13
|
},
|
|
14
14
|
"provenance": {
|
|
Binary file
|