@smithers-orchestrator/cli 0.16.0

package/src/supervisor.js

@@ -0,0 +1,418 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
+import { Effect, Schedule } from "effect";
+import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
+import { trackEvent } from "@smithers-orchestrator/observability/metrics";
+import { isPidAlive, parseRuntimeOwnerPid } from "@smithers-orchestrator/engine/runtime-owner";
+import { SmithersError } from "@smithers-orchestrator/errors";
+import { resumeRunDetached } from "./resume-detached.js";
+/** @typedef {import("./RunAutoResumeSkipReason.ts").RunAutoResumeSkipReason} RunAutoResumeSkipReason */
+/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
+/** @typedef {import("./SupervisorOptions.ts").SupervisorOptions} SupervisorOptions */
+/** @typedef {import("./SupervisorPollSummary.ts").SupervisorPollSummary} SupervisorPollSummary */
+
+export const DEFAULT_SUPERVISOR_INTERVAL_MS = 10_000;
+export const DEFAULT_SUPERVISOR_STALE_THRESHOLD_MS = 30_000;
+export const DEFAULT_SUPERVISOR_MAX_CONCURRENT = 3;
+export const SUPERVISOR_EVENT_RUN_ID = "__supervisor__";
+const durationMultipliers = {
+    ms: 1,
+    s: 1_000,
+    m: 60_000,
+    h: 3_600_000,
+    d: 86_400_000,
+};
+/**
+ * @param {string} raw
+ * @param {string} fieldName
+ * @returns {number}
+ */
+export function parseDurationMs(raw, fieldName) {
+    const input = raw.trim().toLowerCase();
+    const match = input.match(/^(\d+(?:\.\d+)?)(ms|s|m|h|d)?$/);
+    if (!match) {
+        throw new SmithersError("INVALID_DURATION", `Invalid ${fieldName}: "${raw}". Use formats like 500ms, 10s, 2m.`, { fieldName, raw });
+    }
+    const value = Number(match[1]);
+    const unit = match[2] ?? "ms";
+    const multiplier = durationMultipliers[unit];
+    const ms = Math.floor(value * multiplier);
+    if (!Number.isFinite(ms) || ms <= 0) {
+        throw new SmithersError("INVALID_DURATION", `Invalid ${fieldName}: "${raw}" must be > 0.`, { fieldName, raw });
+    }
+    return ms;
+}
+export { isPidAlive, parseRuntimeOwnerPid } from "@smithers-orchestrator/engine/runtime-owner";
+/**
+ * @param {SupervisorOptions} options
+ * @returns {NormalizedSupervisorOptions}
+ */
+function normalizeSupervisorOptions(options) {
+    const deps = {
+        now: () => Date.now(),
+        workflowExists: (workflowPath) => existsSync(workflowPath),
+        parseRuntimeOwnerPid,
+        isPidAlive,
+        spawnResumeDetached: resumeRunDetached,
+        ...options.deps,
+    };
+    return {
+        adapter: options.adapter,
+        pollIntervalMs: options.pollIntervalMs ?? DEFAULT_SUPERVISOR_INTERVAL_MS,
+        staleThresholdMs: options.staleThresholdMs ?? DEFAULT_SUPERVISOR_STALE_THRESHOLD_MS,
+        maxConcurrent: options.maxConcurrent ?? DEFAULT_SUPERVISOR_MAX_CONCURRENT,
+        dryRun: Boolean(options.dryRun),
+        supervisorId: options.supervisorId ?? randomUUID(),
+        supervisorRunId: options.supervisorRunId ?? SUPERVISOR_EVENT_RUN_ID,
+        deps,
+    };
+}
+/**
+ * @param {string | null} workflowPath
+ * @returns {string | null}
+ */
+function resolveWorkflowPath(workflowPath) {
+    if (!workflowPath)
+        return null;
+    return isAbsolute(workflowPath)
+        ? workflowPath
+        : resolve(process.cwd(), workflowPath);
+}
+/**
+ * @param {string | null} [metaJson]
+ * @returns {number | null}
+ */
+function parseTimerFiresAtMs(metaJson) {
+    if (!metaJson)
+        return null;
+    try {
+        const parsed = JSON.parse(metaJson);
+        const firesAt = Number(parsed?.timer?.firesAtMs);
+        return Number.isFinite(firesAt) ? Math.floor(firesAt) : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * @param {NormalizedSupervisorOptions} options
+ * @param {string} runId
+ * @param {number} now
+ * @returns {Effect.Effect<boolean, never>}
+ */
+function runHasDueTimerEffect(options, runId, now) {
+    return Effect.gen(function* () {
+        const nodes = yield* options.adapter.listNodesEffect(runId).pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed to list nodes for timer run ${runId}: ${error instanceof Error ? error.message : String(error)}`).pipe(Effect.as([]))));
+        const waitingTimerNodes = nodes.filter((node) => node.state === "waiting-timer");
+        if (waitingTimerNodes.length === 0) {
+            return false;
+        }
+        for (const node of waitingTimerNodes) {
+            const attempts = yield* options.adapter
+                .listAttemptsEffect(runId, node.nodeId, node.iteration ?? 0)
+                .pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed to list attempts for timer ${runId}/${node.nodeId}: ${error instanceof Error ? error.message : String(error)}`).pipe(Effect.as([]))));
+            const waitingAttempt = attempts.find((attempt) => attempt.state === "waiting-timer") ??
+                attempts[0];
+            const firesAtMs = parseTimerFiresAtMs(waitingAttempt?.metaJson);
+            if (typeof firesAtMs === "number" && firesAtMs <= now) {
+                return true;
+            }
+        }
+        return false;
+    });
+}
+/**
+ * @param {SmithersDb} adapter
+ * @param {SmithersEvent} event
+ * @returns {Effect.Effect<void, never>}
+ */
+function emitEventEffect(adapter, event) {
+    return Effect.all([
+        trackEvent(event),
+        adapter.insertEventWithNextSeqEffect({
+            runId: event.runId,
+            timestampMs: event.timestampMs,
+            type: event.type,
+            payloadJson: JSON.stringify(event),
+        }).pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed to persist event ${event.type}: ${error instanceof Error ? error.message : String(error)}`))),
+    ], { discard: true });
+}
+/**
+ * @param {NormalizedSupervisorOptions} options
+ * @param {string} runId
+ * @param {RunAutoResumeSkipReason} reason
+ * @returns {Effect.Effect<void, never>}
+ */
+function emitSkipEventEffect(options, runId, reason) {
+    return emitEventEffect(options.adapter, {
+        type: "RunAutoResumeSkipped",
+        runId,
+        reason,
+        timestampMs: options.deps.now(),
+    });
+}
+/**
+ * @param {NormalizedSupervisorOptions} options
+ * @param {StaleRunRecord} staleRun
+ * @param {number} staleBeforeMs
+ * @returns {Effect.Effect<"resumed" | "skipped", never>}
+ */
+function processCandidateEffect(options, staleRun, staleBeforeMs) {
+    const workflowPath = resolveWorkflowPath(staleRun.workflowPath);
+    const now = options.deps.now();
+    const staleDurationMs = typeof staleRun.heartbeatAtMs === "number"
+        ? Math.max(0, now - staleRun.heartbeatAtMs)
+        : options.staleThresholdMs;
+    const runAnnotations = {
+        runId: staleRun.runId,
+        staleDurationMs,
+        runtimeOwnerId: staleRun.runtimeOwnerId ?? null,
+    };
+    const claimOwnerId = `supervisor:${options.supervisorId}`;
+    return Effect.withLogSpan("supervisor:resume")(Effect.gen(function* () {
+        if (!workflowPath || !options.deps.workflowExists(workflowPath)) {
+            yield* Effect.logWarning(`Skipping run ${staleRun.runId}: workflow file not found at ${workflowPath ?? "(missing path)"}`);
+            yield* emitSkipEventEffect(options, staleRun.runId, "missing-workflow");
+            return "skipped";
+        }
+        const ownerPid = options.deps.parseRuntimeOwnerPid(staleRun.runtimeOwnerId);
+        if (ownerPid !== null && options.deps.isPidAlive(ownerPid)) {
+            yield* Effect.logDebug(`Skipping run ${staleRun.runId}: runtime owner pid ${ownerPid} is still alive`);
+            yield* emitSkipEventEffect(options, staleRun.runId, "pid-alive");
+            return "skipped";
+        }
+        if (options.dryRun) {
+            yield* Effect.logInfo(`Dry-run: would resume stale run ${staleRun.runId} (last heartbeat ${staleDurationMs}ms ago)`);
+            return "skipped";
+        }
+        const claimHeartbeatAtMs = options.deps.now();
+        const claimed = yield* options.adapter
+            .claimRunForResumeEffect({
+            runId: staleRun.runId,
+            expectedRuntimeOwnerId: staleRun.runtimeOwnerId ?? null,
+            expectedHeartbeatAtMs: staleRun.heartbeatAtMs ?? null,
+            staleBeforeMs,
+            claimOwnerId,
+            claimHeartbeatAtMs,
+        })
+            .pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed to claim run ${staleRun.runId}: ${error instanceof Error ? error.message : String(error)}`).pipe(Effect.as(false))));
+        if (!claimed) {
+            yield* Effect.logDebug(`Skipping run ${staleRun.runId}: claim not acquired`);
+            return "skipped";
+        }
+        const spawnResult = yield* Effect.try({
+            try: () => options.deps.spawnResumeDetached(workflowPath, staleRun.runId, {
+                claimOwnerId,
+                claimHeartbeatAtMs,
+                restoreRuntimeOwnerId: staleRun.runtimeOwnerId ?? null,
+                restoreHeartbeatAtMs: staleRun.heartbeatAtMs ?? null,
+            }),
+            catch: (cause) => toSmithersError(cause, `resume stale run ${staleRun.runId}`, {
+                code: "PROCESS_SPAWN_FAILED",
+                details: { runId: staleRun.runId, workflowPath },
+            }),
+        }).pipe(Effect.either);
+        if (spawnResult._tag === "Left") {
+            yield* Effect.logWarning(`[supervisor] failed to resume run ${staleRun.runId}: ${spawnResult.left.message}`);
+            yield* options.adapter
+                .releaseRunResumeClaimEffect({
+                runId: staleRun.runId,
+                claimOwnerId,
+                restoreRuntimeOwnerId: staleRun.runtimeOwnerId ?? null,
+                restoreHeartbeatAtMs: staleRun.heartbeatAtMs ?? null,
+            })
+                .pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed to release claim for run ${staleRun.runId}: ${error instanceof Error ? error.message : String(error)}`)));
+            return "skipped";
+        }
+        const resumePid = spawnResult.right;
+        yield* Effect.logInfo(`Resuming stale run ${staleRun.runId} (last heartbeat ${staleDurationMs}ms ago)${resumePid ? ` with pid ${resumePid}` : ""}`);
+        yield* emitEventEffect(options.adapter, {
+            type: "RunAutoResumed",
+            runId: staleRun.runId,
+            lastHeartbeatAtMs: staleRun.heartbeatAtMs ?? null,
+            staleDurationMs,
+            timestampMs: options.deps.now(),
+        });
+        return "resumed";
+    }).pipe(Effect.annotateLogs(runAnnotations))).pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed while processing stale run ${staleRun.runId}: ${String(error)}`).pipe(Effect.as("skipped"))));
+}
+/**
+ * @param {NormalizedSupervisorOptions} options
+ * @param {any} run
+ * @param {number} staleBeforeMs
+ * @returns {Effect.Effect<"resumed" | "skipped", never>}
+ */
+function processTimerCandidateEffect(options, run, staleBeforeMs) {
+    const workflowPath = resolveWorkflowPath(run.workflowPath ?? null);
+    const runAnnotations = {
+        runId: run.runId,
+        status: run.status ?? null,
+        runtimeOwnerId: run.runtimeOwnerId ?? null,
+    };
+    return Effect.withLogSpan("supervisor:timer-resume")(Effect.gen(function* () {
+        if (!workflowPath || !options.deps.workflowExists(workflowPath)) {
+            yield* Effect.logWarning(`Skipping timer run ${run.runId}: workflow file not found at ${workflowPath ?? "(missing path)"}`);
+            yield* emitSkipEventEffect(options, run.runId, "missing-workflow");
+            return "skipped";
+        }
+        const ownerPid = options.deps.parseRuntimeOwnerPid(run.runtimeOwnerId);
+        if (ownerPid !== null && options.deps.isPidAlive(ownerPid)) {
+            yield* Effect.logDebug(`Skipping timer run ${run.runId}: runtime owner pid ${ownerPid} is still alive`);
+            yield* emitSkipEventEffect(options, run.runId, "pid-alive");
+            return "skipped";
+        }
+        if (options.dryRun) {
+            yield* Effect.logInfo(`Dry-run: would resume due timer run ${run.runId}`);
+            return "skipped";
+        }
+        const claimOwnerId = `supervisor:${options.supervisorId}`;
+        const claimHeartbeatAtMs = options.deps.now();
+        const claimed = yield* options.adapter
+            .claimRunForResumeEffect({
+            runId: run.runId,
+            expectedStatus: "waiting-timer",
+            expectedRuntimeOwnerId: run.runtimeOwnerId ?? null,
+            expectedHeartbeatAtMs: run.heartbeatAtMs ?? null,
+            staleBeforeMs,
+            claimOwnerId,
+            claimHeartbeatAtMs,
+            requireStale: true,
+        })
+            .pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed to claim timer run ${run.runId}: ${error instanceof Error ? error.message : String(error)}`).pipe(Effect.as(false))));
+        if (!claimed) {
+            yield* Effect.logDebug(`Skipping timer run ${run.runId}: claim not acquired`);
+            return "skipped";
+        }
+        const spawnResult = yield* Effect.try({
+            try: () => options.deps.spawnResumeDetached(workflowPath, run.runId, {
+                claimOwnerId,
+                claimHeartbeatAtMs,
+                restoreRuntimeOwnerId: run.runtimeOwnerId ?? null,
+                restoreHeartbeatAtMs: run.heartbeatAtMs ?? null,
+            }),
+            catch: (cause) => toSmithersError(cause, `resume timer run ${run.runId}`, {
+                code: "PROCESS_SPAWN_FAILED",
+                details: { runId: run.runId, workflowPath },
+            }),
+        }).pipe(Effect.either);
+        if (spawnResult._tag === "Left") {
+            yield* Effect.logWarning(`[supervisor] failed to resume timer run ${run.runId}: ${spawnResult.left.message}`);
+            yield* options.adapter
+                .releaseRunResumeClaimEffect({
+                runId: run.runId,
+                claimOwnerId,
+                restoreRuntimeOwnerId: run.runtimeOwnerId ?? null,
+                restoreHeartbeatAtMs: run.heartbeatAtMs ?? null,
+            })
+                .pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed to release timer claim for run ${run.runId}: ${error instanceof Error ? error.message : String(error)}`)));
+            return "skipped";
+        }
+        const resumePid = spawnResult.right;
+        yield* Effect.logInfo(`Resuming timer-blocked run ${run.runId}${resumePid ? ` with pid ${resumePid}` : ""}`);
+        yield* emitEventEffect(options.adapter, {
+            type: "RunAutoResumed",
+            runId: run.runId,
+            lastHeartbeatAtMs: run.heartbeatAtMs ?? null,
+            staleDurationMs: typeof run.heartbeatAtMs === "number"
+                ? Math.max(0, options.deps.now() - run.heartbeatAtMs)
+                : 0,
+            timestampMs: options.deps.now(),
+        });
+        return "resumed";
+    }).pipe(Effect.annotateLogs(runAnnotations))).pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] failed while processing timer run ${run.runId}: ${String(error)}`).pipe(Effect.as("skipped"))));
+}
+/**
+ * @param {NormalizedSupervisorOptions} options
+ * @returns {Effect.Effect<SupervisorPollSummary, never>}
+ */
+function pollEffect(options) {
+    return Effect.withLogSpan("supervisor:poll")(Effect.gen(function* () {
+        const pollStartedAtMs = options.deps.now();
+        const staleBeforeMs = pollStartedAtMs - options.staleThresholdMs;
+        const staleRuns = yield* options.adapter
+            .listStaleRunningRunsEffect(staleBeforeMs)
+            .pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] stale-run query failed: ${error instanceof Error ? error.message : String(error)}`).pipe(Effect.as([]))));
+        if (staleRuns.length === 0) {
+            yield* Effect.logDebug("Supervisor poll found no stale runs");
+        }
+        const resumable = staleRuns.slice(0, options.maxConcurrent);
+        const rateLimited = staleRuns.slice(options.maxConcurrent);
+        if (rateLimited.length > 0) {
+            for (const run of rateLimited) {
+                yield* Effect.logDebug(`Skipping run ${run.runId}: rate limited (max-concurrent=${options.maxConcurrent})`);
+                yield* emitSkipEventEffect(options, run.runId, "rate-limited");
+            }
+        }
+        const results = yield* Effect.all(resumable.map((run) => processCandidateEffect(options, run, staleBeforeMs)), { concurrency: options.maxConcurrent });
+        const staleResumedCount = results.filter((result) => result === "resumed").length;
+        const staleSkippedCount = rateLimited.length +
+            results.filter((result) => result === "skipped").length;
+        const waitingTimerRuns = yield* options.adapter
+            .listRunsEffect(500, "waiting-timer")
+            .pipe(Effect.catchAll((error) => Effect.logWarning(`[supervisor] waiting-timer query failed: ${error instanceof Error ? error.message : String(error)}`).pipe(Effect.as([]))));
+        const claimableTimerRuns = waitingTimerRuns.filter((run) => run.heartbeatAtMs == null || run.heartbeatAtMs < staleBeforeMs);
+        const timerDueChecks = yield* Effect.all(claimableTimerRuns.map((run) => runHasDueTimerEffect(options, run.runId, pollStartedAtMs)), { concurrency: options.maxConcurrent });
+        const dueTimerRuns = claimableTimerRuns.filter((_run, index) => timerDueChecks[index]);
+        const timerSlots = Math.max(0, options.maxConcurrent - staleResumedCount);
+        const timerResumable = dueTimerRuns.slice(0, timerSlots);
+        const timerRateLimited = dueTimerRuns.slice(timerSlots);
+        for (const run of timerRateLimited) {
+            yield* emitSkipEventEffect(options, run.runId, "rate-limited");
+        }
+        const timerResults = yield* Effect.all(timerResumable.map((run) => processTimerCandidateEffect(options, run, staleBeforeMs)), { concurrency: options.maxConcurrent });
+        const resumedCount = staleResumedCount +
+            timerResults.filter((result) => result === "resumed").length;
+        const skippedCount = staleSkippedCount +
+            timerRateLimited.length +
+            timerResults.filter((result) => result === "skipped").length;
+        const durationMs = Math.max(0, options.deps.now() - pollStartedAtMs);
+        yield* emitEventEffect(options.adapter, {
+            type: "SupervisorPollCompleted",
+            runId: options.supervisorRunId,
+            staleCount: staleRuns.length,
+            resumedCount,
+            skippedCount,
+            durationMs,
+            timestampMs: options.deps.now(),
+        });
+        return {
+            staleCount: staleRuns.length,
+            resumedCount,
+            skippedCount,
+            durationMs,
+        };
+    }));
+}
+/**
+ * @param {SupervisorOptions} options
+ * @returns {Effect.Effect<SupervisorPollSummary, never>}
+ */
+export function supervisorPollEffect(options) {
+    return pollEffect(normalizeSupervisorOptions(options));
+}
+/**
+ * @param {SupervisorOptions} options
+ * @returns {Effect.Effect<void, never>}
+ */
+export function supervisorLoopEffect(options) {
+    const normalized = normalizeSupervisorOptions(options);
+    return Effect.gen(function* () {
+        yield* Effect.logInfo(`[supervisor] started (interval=${normalized.pollIntervalMs}ms, staleThreshold=${normalized.staleThresholdMs}ms, maxConcurrent=${normalized.maxConcurrent}, dryRun=${normalized.dryRun})`);
+        yield* emitEventEffect(normalized.adapter, {
+            type: "SupervisorStarted",
+            runId: normalized.supervisorRunId,
+            pollIntervalMs: normalized.pollIntervalMs,
+            staleThresholdMs: normalized.staleThresholdMs,
+            timestampMs: normalized.deps.now(),
+        });
+        yield* pollEffect(normalized).pipe(Effect.repeat(Schedule.spaced(`${normalized.pollIntervalMs} millis`)));
+    }).pipe(Effect.annotateLogs({
+        component: "supervisor",
+        supervisorId: normalized.supervisorId,
+        pollIntervalMs: normalized.pollIntervalMs,
+        staleThresholdMs: normalized.staleThresholdMs,
+        maxConcurrent: normalized.maxConcurrent,
+        dryRun: normalized.dryRun,
+    }), Effect.asVoid);
+}