@nowcrew/daemon 0.6.68 → 0.6.69

package/dist/storage-status-replay.js

@@ -0,0 +1,125 @@
+import { readdir } from "node:fs/promises";
+import { readStorageMigrationReceipt, storageMigrationReplayStatuses, } from "./storage-migration-controller.js";
+import { readStorageWorkerFailureStatus, readStorageWorkerHeartbeat, } from "./storage-worker-status.js";
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
+export async function latestSourceStorageMigrationReceipt(backupsRoot, expected) {
+    let entries;
+    try {
+        entries = await readdir(backupsRoot, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return null;
+        throw error;
+    }
+    if (entries.length > 10_000)
+        throw new Error("storage migration receipt directory limit exceeded");
+    let latest = null;
+    for (const entry of entries) {
+        if (!entry.isDirectory() || !UUID.test(entry.name))
+            continue;
+        const receipt = await readStorageMigrationReceipt(backupsRoot, entry.name);
+        if (receipt === null || receipt.environment !== expected.environment || receipt.profile !== expected.profile)
+            continue;
+        const safeTerminal = receipt.status === "rolled_back"
+            || (receipt.status === "failed" && receipt.sourceStopAttempted === false);
+        if (!safeTerminal)
+            continue;
+        const at = new Date(receipt.history.at(-1)?.at ?? 0).getTime();
+        if (latest === null || at > latest.at)
+            latest = { receipt, at };
+    }
+    return latest?.receipt ?? null;
+}
+export function storageMigrationStatusReplay(receipt, expected) {
+    if (expected !== undefined && (receipt.migrationId !== expected.migrationId
+        || receipt.environment !== expected.environment
+        || receipt.workspaceId !== expected.workspaceId
+        || receipt.profile !== expected.profile)) {
+        throw new Error("storage migration receipt identity mismatch");
+    }
+    return Object.freeze(storageMigrationReplayStatuses(receipt).map((status) => Object.freeze({
+        type: "daemon:storage-migration-status",
+        migrationId: receipt.migrationId,
+        status,
+        environment: receipt.environment,
+        workspaceId: receipt.workspaceId,
+        profile: receipt.profile,
+        ...(status === "verified" ? { finalManifestDigest: receipt.manifestDigest } : {}),
+        ...(status === "failed"
+            ? {
+                errorCode: receipt.errorCode ?? "storage_migration_failed",
+                ...(receipt.errorMessage === undefined ? {} : { errorMessage: receipt.errorMessage }),
+                ...(receipt.sourceStopAttempted === false ? { reopenAdmission: true } : {}),
+            }
+            : {}),
+    })));
+}
+const wait = (milliseconds) => new Promise((resolvePromise) => {
+    setTimeout(resolvePromise, milliseconds);
+});
+export async function monitorStorageMigrationStatus(input, send, injected = {}) {
+    const readReceipt = injected.readReceipt ?? readStorageMigrationReceipt;
+    const readWorkerFailure = injected.readWorkerFailure ?? readStorageWorkerFailureStatus;
+    const readWorkerHeartbeatStatus = injected.readWorkerHeartbeat ?? readStorageWorkerHeartbeat;
+    const now = injected.now ?? (() => new Date());
+    const delay = injected.delay ?? wait;
+    const pollIntervalMs = input.pollIntervalMs ?? 250;
+    const heartbeatTimeoutMs = input.workerHeartbeatTimeoutMs ?? 30_000;
+    const monitorStartedAt = now().getTime();
+    let sentCount = 0;
+    for (;;) {
+        const [receipt, failure] = await Promise.all([
+            readReceipt(input.backupsRoot, input.migrationId),
+            readWorkerFailure(input.stateRoot, input.migrationId),
+        ]);
+        if (receipt !== null) {
+            const frames = storageMigrationStatusReplay(receipt, input);
+            for (const frame of frames.slice(sentCount))
+                send(frame);
+            sentCount = frames.length;
+            if (receipt.status === "service_switched" || receipt.status === "completed") {
+                return { status: receipt.status, reopenAdmission: false };
+            }
+            if (receipt.status === "rolled_back")
+                return { status: "rolled_back", reopenAdmission: true };
+            if (receipt.status === "rollback_failed")
+                return { status: "rollback_failed", reopenAdmission: false };
+            if (receipt.status === "failed" && receipt.sourceStopAttempted === false) {
+                return { status: "failed", reopenAdmission: true };
+            }
+        }
+        if (failure !== null) {
+            const workerFailure = failure;
+            send({
+                type: "daemon:storage-migration-status",
+                migrationId: input.migrationId,
+                status: "failed",
+                environment: input.environment,
+                workspaceId: input.workspaceId,
+                profile: input.profile,
+                errorCode: workerFailure.errorCode,
+                errorMessage: workerFailure.errorMessage,
+                ...(receipt?.sourceStopAttempted === false || receipt === null ? { reopenAdmission: true } : {}),
+            });
+            return { status: "failed", reopenAdmission: receipt?.sourceStopAttempted === false || receipt === null };
+        }
+        const heartbeat = await readWorkerHeartbeatStatus(input.stateRoot, input.migrationId);
+        const heartbeatAt = heartbeat === null ? monitorStartedAt : new Date(heartbeat.at).getTime();
+        if (now().getTime() - heartbeatAt > heartbeatTimeoutMs) {
+            send({
+                type: "daemon:storage-migration-status",
+                migrationId: input.migrationId,
+                status: "failed",
+                environment: input.environment,
+                workspaceId: input.workspaceId,
+                profile: input.profile,
+                errorCode: "worker_heartbeat_stale_recovery_required",
+                errorMessage: "storage migration worker heartbeat expired; explicit recovery is required",
+                ...(receipt?.sourceStopAttempted === false || receipt === null ? { reopenAdmission: true } : {}),
+            });
+            return { status: "failed", reopenAdmission: receipt?.sourceStopAttempted === false || receipt === null };
+        }
+        await delay(pollIntervalMs);
+    }
+}

package/dist/storage-target-confirmation.js

@@ -0,0 +1,129 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, join } from "node:path";
+import { z } from "zod";
+import { atomicPrivateWrite } from "./atomic-private-write.js";
+import { readStorageMigrationReceipt } from "./storage-migration-controller.js";
+const FingerprintSchema = z.string().regex(/^sha256:[0-9a-f]{64}$/u);
+const IdentitySchema = z.object({
+    type: z.literal("daemon:storage-migration-confirmed"),
+    migrationId: z.string().uuid(),
+    environment: z.string().regex(/^[a-z0-9][a-z0-9_-]{0,31}$/u),
+    workspaceId: z.string().uuid(),
+    profile: z.string().regex(/^[a-z0-9][a-z0-9_-]{0,47}$/u),
+    manifestDigest: z.string().regex(/^[0-9a-f]{64}$/u),
+    targetStorageFingerprint: FingerprintSchema,
+}).strict();
+const AcknowledgementSchema = IdentitySchema.omit({ type: true }).extend({
+    version: z.literal(1),
+    confirmedAt: z.string().datetime({ offset: true }),
+}).strict();
+const delay = (milliseconds) => new Promise((resolvePromise) => {
+    setTimeout(resolvePromise, milliseconds);
+});
+export function storageMigrationAcknowledgementPath(backupsRoot, migrationId) {
+    if (!isAbsolute(backupsRoot))
+        throw new Error("storage migration backups root must be absolute");
+    return join(backupsRoot, z.string().uuid().parse(migrationId), "hello-confirmed.json");
+}
+export async function waitForStorageMigrationAcknowledgement(input) {
+    const expected = IdentitySchema.omit({ type: true }).parse({
+        migrationId: input.migrationId,
+        environment: input.environment,
+        workspaceId: input.workspaceId,
+        profile: input.profile,
+        manifestDigest: input.manifestDigest,
+        targetStorageFingerprint: input.targetStorageFingerprint,
+    });
+    const timeoutMs = z.number().int().min(10).max(600_000).parse(input.timeoutMs ?? 120_000);
+    const pollIntervalMs = z.number().int().min(1).max(5_000).parse(input.pollIntervalMs ?? 250);
+    const path = storageMigrationAcknowledgementPath(input.backupsRoot, expected.migrationId);
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() <= deadline) {
+        try {
+            const acknowledgement = AcknowledgementSchema.parse(JSON.parse(await readFile(path, "utf8")));
+            const { version: _version, confirmedAt: _confirmedAt, ...identity } = acknowledgement;
+            if (JSON.stringify(identity) !== JSON.stringify(expected)) {
+                throw new Error("storage migration acknowledgement identity mismatch");
+            }
+            return;
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+        await delay(pollIntervalMs);
+    }
+    throw new Error("storage migration hello confirmation timeout");
+}
+export function createStorageTargetConfirmationHandler(input) {
+    const profile = z.string().regex(/^[a-z0-9][a-z0-9_-]{0,47}$/u).parse(input.profile);
+    const completionTimeoutMs = z.number().int().min(10).max(600_000).parse(input.completionTimeoutMs ?? 120_000);
+    const pollIntervalMs = z.number().int().min(1).max(5_000).parse(input.pollIntervalMs ?? 250);
+    if (input.identity.storageMigrationId === undefined) {
+        throw new Error("storage target confirmation requires a migration identity");
+    }
+    let active = null;
+    const handle = async (raw, send) => {
+        const rawType = typeof raw === "object" && raw !== null && "type" in raw ? raw.type : undefined;
+        if (rawType !== "daemon:storage-migration-confirmed")
+            return false;
+        const confirmation = IdentitySchema.parse(raw);
+        if (confirmation.migrationId !== input.identity.storageMigrationId
+            || confirmation.environment !== input.identity.storageEnvironment
+            || confirmation.workspaceId !== input.identity.storageWorkspaceId
+            || confirmation.profile !== profile
+            || confirmation.targetStorageFingerprint !== input.identity.storageIdFingerprint) {
+            throw new Error("storage migration confirmation identity mismatch");
+        }
+        const key = JSON.stringify(confirmation);
+        if (active !== null) {
+            if (active.key !== key)
+                throw new Error("storage migration confirmation changed while processing");
+            return active.promise;
+        }
+        const processConfirmation = async () => {
+            const receipt = await readStorageMigrationReceipt(input.backupsRoot, confirmation.migrationId);
+            if (receipt === null || receipt.status !== "service_switched"
+                || receipt.environment !== confirmation.environment
+                || receipt.workspaceId !== confirmation.workspaceId
+                || receipt.profile !== confirmation.profile
+                || receipt.manifestDigest !== confirmation.manifestDigest) {
+                throw new Error("storage migration confirmation receipt identity mismatch");
+            }
+            const { type: _type, ...confirmationIdentity } = confirmation;
+            await atomicPrivateWrite(storageMigrationAcknowledgementPath(input.backupsRoot, confirmation.migrationId), `${JSON.stringify(AcknowledgementSchema.parse({
+                version: 1,
+                ...confirmationIdentity,
+                confirmedAt: new Date().toISOString(),
+            }), null, 2)}\n`);
+            const deadline = Date.now() + completionTimeoutMs;
+            while (Date.now() <= deadline) {
+                const current = await readStorageMigrationReceipt(input.backupsRoot, confirmation.migrationId);
+                if (current?.status === "completed") {
+                    send({
+                        type: "daemon:storage-migration-status",
+                        migrationId: confirmation.migrationId,
+                        status: "completed",
+                        environment: confirmation.environment,
+                        workspaceId: confirmation.workspaceId,
+                        profile: confirmation.profile,
+                    });
+                    return true;
+                }
+                if (current !== null && ["failed", "rolled_back", "rollback_failed"].includes(current.status)) {
+                    throw new Error(`storage migration ended before completion: ${current.status}`);
+                }
+                await delay(pollIntervalMs);
+            }
+            throw new Error("storage migration worker completion timeout");
+        };
+        const promise = processConfirmation();
+        active = { key, promise };
+        void promise.catch(() => {
+            if (active?.key === key)
+                active = null;
+        });
+        return promise;
+    };
+    return Object.freeze({ handle });
+}

package/dist/storage-worker-command.js

@@ -0,0 +1,37 @@
+import { posix, win32 } from "node:path";
+import { executeStorageMigrationWorker, recoverStorageMigrationWorker, } from "./storage-worker-executor.js";
+import { completeStorageMigrationWorkerServiceForHandoff } from "./storage-worker-handoff.js";
+import { startStorageWorkerHeartbeat, writeStorageWorkerFailureStatus, } from "./storage-worker-status.js";
+export async function runStorageWorkerCommand(args, execute = executeStorageMigrationWorker, writeFailure = writeStorageWorkerFailureStatus, startHeartbeat = startStorageWorkerHeartbeat, completeService = completeStorageMigrationWorkerServiceForHandoff, recover = recoverStorageMigrationWorker) {
+    const command = args[0];
+    if (command !== "storage-migration-worker" && command !== "storage-migration-recover")
+        return null;
+    const path = args[1] === "--job" ? args[2] : undefined;
+    if (args.length !== 3 || path === undefined || path.length === 0
+        || /[\0\r\n]/u.test(path) || (!posix.isAbsolute(path) && !win32.isAbsolute(path))) {
+        throw new Error("Invalid storage migration worker command");
+    }
+    const stopHeartbeat = await startHeartbeat(path);
+    const assertHeartbeatHealthy = stopHeartbeat.assertHealthy;
+    try {
+        if (command === "storage-migration-recover") {
+            await (assertHeartbeatHealthy === undefined
+                ? recover(path)
+                : recover(path, { assertHeartbeatHealthy }));
+        }
+        else {
+            await (assertHeartbeatHealthy === undefined
+                ? execute(path)
+                : execute(path, { assertHeartbeatHealthy }));
+        }
+    }
+    catch (error) {
+        await writeFailure(path, error).catch(() => undefined);
+        throw error;
+    }
+    finally {
+        await stopHeartbeat().catch(() => undefined);
+        await completeService(path);
+    }
+    return 0;
+}