@tea-agent/loop-agent 0.28.2-beta.1 → 0.28.3

package/dist/worker/scheduler/dispatcher.js

@@ -0,0 +1,503 @@
+import path from "node:path";
+import { readTaskPoolState } from "../pool/run-store.js";
+import { executePreparedTaskSpec } from "../run-task/execute-prepared-task.js";
+import { runSingleTaskAttempt, } from "../runner/single-task-attempt.js";
+import { revalidateNightWorkspace } from "./workspace-adapter.js";
+import { archiveExecutionEvidence } from "./evidence.js";
+import { createNightClosingCommit } from "./night-git-finalizer.js";
+import { heartbeatExecutionLease, releaseExecutionLease, } from "./lease.js";
+import { transitionSchedule } from "./lifecycle.js";
+import { maybeEnqueueNightFollowUp } from "./auto-followup.js";
+import { runVerifyOnlyRerun } from "./prepared-attempt-recovery.js";
+import { maybeQueueAutoRetry } from "./retry.js";
+import { listExecutionsForSchedule, readAdmission, readExecution, readSchedule, writeExecution, } from "./store.js";
+import { SCHEDULER_ERROR_CODES, SchedulerError } from "./types.js";
+/**
+ * Run one claimed night execution outside the short mutation lock.
+ * Revalidates admission/worktree, executes prepared task in workspace, archives evidence.
+ */
+export async function dispatchClaimedExecution(input) {
+    const now = input.now ?? new Date();
+    const schedule = await readSchedule(input.controlRepoRoot, input.scheduleId);
+    if (!schedule) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.NOT_FOUND, `schedule not found: ${input.scheduleId}`);
+    }
+    const admission = await readAdmission(input.controlRepoRoot, input.scheduleId);
+    if (!admission?.workspace ||
+        !admission.gateApproval?.token ||
+        !admission.gateApproval.approvedAt) {
+        await failHumanRequired(input, schedule, "admission-incomplete", "admission/workspace/gate missing at dispatch", now);
+        const execution = await finalizeExecution(input, "human_required", now, {
+            code: "admission-incomplete",
+            message: "missing approved admission fields",
+        });
+        await releaseExecutionLease({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            token: input.lease.token,
+        }).catch(() => { });
+        return {
+            schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+            execution,
+        };
+    }
+    const poolState = await readTaskPoolState(input.controlRepoRoot, {
+        featureId: schedule.featureId,
+        taskId: schedule.taskId,
+    });
+    if (poolState?.status !== "Queued" ||
+        poolState.nightScheduleId !== schedule.id) {
+        await failHumanRequired(input, schedule, "reservation-mismatch", "Task Pool reservation is missing or no longer owned by this schedule", now);
+        const execution = await finalizeExecution(input, "human_required", now, {
+            code: "reservation-mismatch",
+            message: "Task Pool reservation mismatch",
+        });
+        await releaseExecutionLease({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            token: input.lease.token,
+        }).catch(() => { });
+        return {
+            schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+            execution,
+        };
+    }
+    const workspaceRoot = path.resolve(input.controlRepoRoot, admission.workspace.path);
+    try {
+        const revalidate = input.revalidateWorkspace ?? revalidateNightWorkspace;
+        await revalidate({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            worktreePath: admission.workspace.path,
+            branch: admission.workspace.branch,
+            baseCommit: admission.workspace.baseCommit,
+            ownershipToken: admission.workspace.ownershipToken,
+        });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        await failHumanRequired(input, schedule, "worktree-revalidate-failed", message, now);
+        await releaseExecutionLease({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            token: input.lease.token,
+        }).catch(() => { });
+        return {
+            schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+            execution: await finalizeExecution(input, "human_required", now, {
+                code: "worktree-revalidate-failed",
+                message,
+            }),
+        };
+    }
+    // Associate running
+    await transitionSchedule({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: input.scheduleId,
+        toStatus: "running",
+        event: "worker_run_associated",
+        patch: {
+            currentExecutionId: input.executionId,
+            isolation: {
+                mode: "git-worktree",
+                required: true,
+                baseBranch: admission.workspace.baseBranch,
+                baseCommit: admission.workspace.baseCommit,
+                worktreePath: admission.workspace.path,
+                branch: admission.workspace.branch,
+                mergePolicy: schedule.isolation?.mergePolicy ??
+                    schedule.policySnapshot?.mergePolicy ??
+                    "manual-on-success",
+                mergeState: "pending-execution",
+            },
+        },
+        now,
+    });
+    const batchRunId = `night-${input.scheduleId}-${input.executionId}`;
+    const workerRunId = `wr-night-${input.executionId}`;
+    const existingExecution = await readExecution(input.controlRepoRoot, input.executionId);
+    const priorCount = (await listExecutionsForSchedule(input.controlRepoRoot, input.scheduleId)).length;
+    const attemptNumber = existingExecution?.attempt ?? (priorCount > 0 ? priorCount : 1);
+    await writeExecution(input.controlRepoRoot, {
+        schemaVersion: 1,
+        executionId: input.executionId,
+        scheduleId: input.scheduleId,
+        attempt: attemptNumber,
+        status: "running",
+        claimedAt: input.lease.acquiredAt,
+        startedAt: now.toISOString(),
+        finishedAt: null,
+        lease: {
+            ownerPid: input.lease.ownerPid,
+            hostname: input.lease.hostname,
+            token: input.lease.token,
+            heartbeatAt: now.toISOString(),
+        },
+        workerRunId,
+        workspacePath: admission.workspace.path,
+        failure: null,
+    });
+    await heartbeatExecutionLease({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: input.scheduleId,
+        token: input.lease.token,
+        workerRunId,
+        now,
+    }).catch(() => { });
+    const runAttempt = input.runAttempt ?? runSingleTaskAttempt;
+    const executePrepared = input.executePrepared ?? executePreparedTaskSpec;
+    const archiveEvidence = input.archiveEvidence ?? archiveExecutionEvidence;
+    const createClosingCommit = input.createClosingCommit ?? createNightClosingCommit;
+    const stopLeaseHeartbeat = startExecutionLeaseHeartbeat(input, workerRunId);
+    let attempt;
+    try {
+        attempt = await runAttempt({
+            ctx: {
+                controlRepoRoot: input.controlRepoRoot,
+                workspaceRoot,
+                featureId: schedule.featureId,
+                taskId: schedule.taskId,
+                batchRunId,
+                workerRunId,
+                night: {
+                    nightScheduleId: schedule.id,
+                    admissionPath: schedule.admissionRef ??
+                        `.harness/task-pool/scheduler/admissions/${schedule.id}.json`,
+                    nightWorktreePath: admission.workspace.path,
+                    nightBranch: admission.workspace.branch,
+                },
+                controllerIdentity: admission.controllerIdentity,
+                client: input.client,
+                now,
+            },
+            execute: async () => executePrepared({
+                controlRepoRoot: input.controlRepoRoot,
+                workspaceRoot,
+                admission,
+                client: input.client,
+                workerRunId,
+                now,
+                controllerIdentity: admission.controllerIdentity,
+            }),
+        });
+    }
+    catch (error) {
+        stopLeaseHeartbeat();
+        const message = error instanceof Error ? error.message : String(error);
+        const failedAt = new Date();
+        const execution = await finalizeExecution(input, "human_required", failedAt, { code: "dispatch-infrastructure-failed", message }, workerRunId);
+        let evidenceArchivePath;
+        try {
+            const evidence = await archiveEvidence({
+                controlRepoRoot: input.controlRepoRoot,
+                executionId: input.executionId,
+                schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+                admission,
+                execution,
+                workspaceRoot,
+                now: failedAt,
+            });
+            evidenceArchivePath = evidence.archiveRoot;
+        }
+        catch {
+            // Keep the primary dispatch failure; missing archive is reflected by doctor/harvest.
+        }
+        await transitionSchedule({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            toStatus: "human_required",
+            event: "human_required",
+            reason: message,
+            reasonCode: "dispatch-infrastructure-failed",
+            ...(evidenceArchivePath
+                ? { evidencePaths: [evidenceArchivePath] }
+                : {}),
+            now: failedAt,
+        });
+        await releaseExecutionLease({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            token: input.lease.token,
+        }).catch(() => { });
+        return {
+            schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+            execution,
+            ...(evidenceArchivePath ? { evidenceArchivePath } : {}),
+        };
+    }
+    stopLeaseHeartbeat();
+    await heartbeatExecutionLease({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: input.scheduleId,
+        token: input.lease.token,
+        workerRunId: attempt.workerRunId,
+        now: new Date(),
+    }).catch(() => { });
+    if (attempt.status === "succeeded") {
+        let closing;
+        try {
+            closing = await createClosingCommit({
+                workspaceRoot,
+                schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+                admission,
+                workerRunId: attempt.workerRunId,
+                executionId: input.executionId,
+                writeSet: admission.dag?.writeSet,
+            });
+        }
+        catch (error) {
+            closing = {
+                ok: false,
+                reason: error instanceof Error ? error.message : String(error),
+            };
+        }
+        if (!closing.ok) {
+            const execution = await finalizeExecution(input, "human_required", new Date(), { code: "closing-commit-failed", message: closing.reason }, attempt.workerRunId);
+            const evidence = await archiveEvidence({
+                controlRepoRoot: input.controlRepoRoot,
+                executionId: input.executionId,
+                schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+                admission,
+                execution,
+                workspaceRoot,
+                now: new Date(),
+            });
+            await transitionSchedule({
+                controlRepoRoot: input.controlRepoRoot,
+                scheduleId: input.scheduleId,
+                toStatus: "human_required",
+                event: "human_required",
+                reason: closing.reason,
+                reasonCode: "closing-commit-failed",
+                patch: {
+                    currentExecutionId: input.executionId,
+                    isolation: {
+                        mode: "git-worktree",
+                        required: true,
+                        baseBranch: admission.workspace.baseBranch,
+                        baseCommit: admission.workspace.baseCommit,
+                        worktreePath: admission.workspace.path,
+                        branch: admission.workspace.branch,
+                        mergePolicy: schedule.isolation?.mergePolicy ?? "manual-on-success",
+                        mergeState: "not-merged",
+                    },
+                },
+                evidencePaths: [evidence.archiveRoot],
+                now: new Date(),
+            });
+            await releaseExecutionLease({
+                controlRepoRoot: input.controlRepoRoot,
+                scheduleId: input.scheduleId,
+                token: input.lease.token,
+            }).catch(() => { });
+            return {
+                schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+                execution,
+                attempt,
+                evidenceArchivePath: evidence.archiveRoot,
+            };
+        }
+        const execution = await finalizeExecution(input, "succeeded", new Date(), null, attempt.workerRunId);
+        const evidence = await archiveEvidence({
+            controlRepoRoot: input.controlRepoRoot,
+            executionId: input.executionId,
+            schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+            admission,
+            execution: {
+                ...execution,
+                evidenceArchivePath: `.harness/task-pool/scheduler/evidence/${input.executionId}`,
+            },
+            workspaceRoot,
+            now: new Date(),
+        });
+        await writeExecution(input.controlRepoRoot, {
+            ...execution,
+            evidenceArchivePath: evidence.archiveRoot,
+        });
+        await transitionSchedule({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            toStatus: "succeeded",
+            event: "execution_finished",
+            patch: {
+                currentExecutionId: input.executionId,
+                isolation: {
+                    mode: "git-worktree",
+                    required: true,
+                    baseBranch: admission.workspace.baseBranch,
+                    baseCommit: admission.workspace.baseCommit,
+                    worktreePath: admission.workspace.path,
+                    branch: admission.workspace.branch,
+                    mergePolicy: schedule.isolation?.mergePolicy ?? "manual-on-success",
+                    mergeState: "pending-harvest",
+                },
+            },
+            evidencePaths: [evidence.archiveRoot],
+            now: new Date(),
+        });
+        await releaseExecutionLease({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            token: input.lease.token,
+        }).catch(() => { });
+        return {
+            schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+            execution: {
+                ...execution,
+                evidenceArchivePath: evidence.archiveRoot,
+            },
+            attempt,
+            closingCommitSha: closing.commitSha,
+            evidenceArchivePath: evidence.archiveRoot,
+        };
+    }
+    // failed / run-error
+    const execution = await finalizeExecution(input, attempt.status === "run-error" ? "failed" : "failed", new Date(), {
+        code: attempt.status,
+        message: attempt.error ?? "prepared execution failed",
+    }, attempt.workerRunId);
+    const evidence = await archiveEvidence({
+        controlRepoRoot: input.controlRepoRoot,
+        executionId: input.executionId,
+        schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+        admission,
+        execution,
+        workspaceRoot,
+        now: new Date(),
+    });
+    await writeExecution(input.controlRepoRoot, {
+        ...execution,
+        evidenceArchivePath: evidence.archiveRoot,
+    });
+    await transitionSchedule({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: input.scheduleId,
+        toStatus: "failed",
+        event: "execution_finished",
+        reason: attempt.error ?? "prepared execution failed",
+        reasonCode: attempt.status,
+        patch: {
+            currentExecutionId: input.executionId,
+            isolation: {
+                mode: "git-worktree",
+                required: true,
+                baseBranch: admission.workspace.baseBranch,
+                baseCommit: admission.workspace.baseCommit,
+                worktreePath: admission.workspace.path,
+                branch: admission.workspace.branch,
+                mergePolicy: schedule.isolation?.mergePolicy ?? "manual-on-success",
+                mergeState: "not-merged",
+            },
+        },
+        evidencePaths: [evidence.archiveRoot],
+        now: new Date(),
+    });
+    await releaseExecutionLease({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: input.scheduleId,
+        token: input.lease.token,
+    }).catch(() => { });
+    // Phase 4 complete: recovery adapter (EnvFailure full re-attempt / FlakyTest verify-only).
+    const failureCategory = attempt.run?.failure?.category ??
+        attempt.poolState?.failure?.category ??
+        (attempt.status === "run-error" ? "EnvFailure" : undefined);
+    if (failureCategory) {
+        const retryDecision = await maybeQueueAutoRetry({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            failureCategory,
+            attempt,
+            workspaceRoot,
+            now: new Date(),
+        }).catch(() => undefined);
+        // FlakyTest verify-only: run bounded dag rerun in workspace without re-queueing full writer.
+        if (retryDecision?.ok &&
+            retryDecision.action === "verify-only" &&
+            retryDecision.fromNodeId &&
+            retryDecision.parentDagRunId) {
+            await runVerifyOnlyRerun({
+                workspaceRoot,
+                client: input.client,
+                parentDagRunId: retryDecision.parentDagRunId,
+                fromNodeId: retryDecision.fromNodeId,
+                requestId: `night-verify-${input.executionId}`,
+                reason: `night-scheduler-flakytest-verify-only:${retryDecision.fromNodeId}`,
+            }).catch(() => undefined);
+            // Verify-only does not flip schedule to succeeded; remain failed for morning.
+        }
+    }
+    // ADR 0010: opt-in Follow-up draft enqueue (never approve). Best-effort after failed terminal.
+    const failedSchedule = await readSchedule(input.controlRepoRoot, input.scheduleId);
+    if (failedSchedule?.status === "failed") {
+        await maybeEnqueueNightFollowUp({
+            controlRepoRoot: input.controlRepoRoot,
+            schedule: failedSchedule,
+            workerRunId: attempt.workerRunId,
+        }).catch(() => undefined);
+    }
+    return {
+        schedule: (await readSchedule(input.controlRepoRoot, input.scheduleId)),
+        execution: {
+            ...execution,
+            evidenceArchivePath: evidence.archiveRoot,
+        },
+        attempt,
+        evidenceArchivePath: evidence.archiveRoot,
+    };
+}
+function startExecutionLeaseHeartbeat(input, workerRunId) {
+    let heartbeatInFlight = false;
+    const timer = setInterval(() => {
+        if (heartbeatInFlight)
+            return;
+        heartbeatInFlight = true;
+        void heartbeatExecutionLease({
+            controlRepoRoot: input.controlRepoRoot,
+            scheduleId: input.scheduleId,
+            token: input.lease.token,
+            workerRunId,
+            now: new Date(),
+        })
+            .catch(() => { })
+            .finally(() => {
+            heartbeatInFlight = false;
+        });
+    }, 15_000);
+    timer.unref();
+    return () => clearInterval(timer);
+}
+async function failHumanRequired(input, schedule, code, message, now) {
+    await transitionSchedule({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: schedule.id,
+        toStatus: "human_required",
+        event: "human_required",
+        reason: message,
+        reasonCode: code,
+        now,
+    });
+}
+async function finalizeExecution(input, status, now, failure, workerRunId) {
+    const existing = await readExecution(input.controlRepoRoot, input.executionId);
+    const execution = {
+        schemaVersion: 1,
+        executionId: input.executionId,
+        scheduleId: input.scheduleId,
+        attempt: existing?.attempt ?? 1,
+        status,
+        claimedAt: input.lease.acquiredAt,
+        startedAt: input.lease.acquiredAt,
+        finishedAt: now.toISOString(),
+        lease: {
+            ownerPid: input.lease.ownerPid,
+            hostname: input.lease.hostname,
+            token: input.lease.token,
+            heartbeatAt: now.toISOString(),
+        },
+        ...(workerRunId ? { workerRunId } : {}),
+        workspacePath: input.lease.workspacePath,
+        failure: failure ?? null,
+    };
+    await writeExecution(input.controlRepoRoot, execution);
+    return execution;
+}