@tea-agent/loop-agent 0.28.2-beta.1 → 0.28.2

package/dist/worker/scheduler/recovery.js

@@ -0,0 +1,277 @@
+import { readdir } from "node:fs/promises";
+import { readExecutionLease, releaseExecutionLease, } from "./lease.js";
+import { transitionSchedule } from "./lifecycle.js";
+import { getLeasesDir } from "./paths.js";
+import { listExecutionsForSchedule, listSchedules, readExecution, writeExecution, } from "./store.js";
+function isPidAlive(pid) {
+    if (!Number.isFinite(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function classifyLeaseLiveness(lease, now = new Date(), staleMs = 5 * 60_000) {
+    if (!lease)
+        return "missing";
+    const heartbeatMs = Date.parse(lease.heartbeatAt);
+    const heartbeatStale = !Number.isFinite(heartbeatMs) || now.getTime() - heartbeatMs > staleMs;
+    const pidAlive = isPidAlive(lease.ownerPid);
+    if (!heartbeatStale && pidAlive)
+        return "active";
+    if (!pidAlive)
+        return "stale-dead-pid";
+    return "stale-heartbeat";
+}
+/**
+ * Classify whether a non-terminal schedule with a lease/execution can be
+ * safely reclaimed or must go human_required (termination-unconfirmed).
+ *
+ * Rules (ADR 0009 / Phase 4):
+ * - No workerRunId and not running → safe reclaim (crash after claim, before child)
+ * - workerRunId or running/dispatching with dead owner → termination-unconfirmed
+ *   (DAG may still be alive; never start a second writer automatically)
+ */
+export function classifyLeaseRecovery(input) {
+    const now = input.now ?? new Date();
+    const liveness = classifyLeaseLiveness(input.lease, now, input.staleMs);
+    const schedule = input.schedule;
+    const execution = input.execution;
+    if (schedule.status === "succeeded" ||
+        schedule.status === "failed" ||
+        schedule.status === "human_required" ||
+        schedule.status === "cancelled" ||
+        schedule.status === "rejected") {
+        if (input.lease) {
+            return {
+                kind: "already-terminal",
+                reason: `schedule terminal (${schedule.status}) but lease file remains`,
+                ...(execution ? { execution } : {}),
+            };
+        }
+        return { kind: "none", reason: "schedule-terminal" };
+    }
+    if (!input.lease && !execution) {
+        return { kind: "none", reason: "no-lease-or-execution" };
+    }
+    if (execution && isExecutionTerminal(execution.status)) {
+        return {
+            kind: "already-terminal",
+            reason: `execution already terminal (${execution.status})`,
+            execution,
+            ...(input.lease ? {} : {}),
+        };
+    }
+    const workerAssociated = Boolean(execution?.workerRunId || input.lease?.workerRunId);
+    const inFlight = schedule.status === "running" ||
+        schedule.status === "dispatching" ||
+        schedule.status === "cancel_requested" ||
+        execution?.status === "running" ||
+        execution?.status === "claimed";
+    if (liveness === "active") {
+        return {
+            kind: "none",
+            reason: "lease-active",
+            ...(input.lease ? { lease: input.lease } : {}),
+            ...(execution ? { execution } : {}),
+        };
+    }
+    // Stale or missing owner.
+    if (!workerAssociated && execution?.status === "claimed") {
+        return {
+            kind: "safe-reclaim",
+            reason: "orphan-claim-before-child",
+            ...(input.lease ? { lease: input.lease } : {}),
+            execution,
+        };
+    }
+    if (!workerAssociated && schedule.status === "dispatching" && !execution) {
+        return {
+            kind: "safe-reclaim",
+            reason: "orphan-claim-before-child",
+            ...(input.lease ? { lease: input.lease } : {}),
+        };
+    }
+    if (inFlight || workerAssociated) {
+        return {
+            kind: "termination-unconfirmed",
+            reason: workerAssociated
+                ? "worker-or-dag-associated-but-owner-stale"
+                : "in-flight-without-confirmed-terminal",
+            ...(input.lease ? { lease: input.lease } : {}),
+            ...(execution ? { execution } : {}),
+        };
+    }
+    return {
+        kind: "none",
+        reason: `no-recovery-action (liveness=${liveness}, status=${schedule.status})`,
+    };
+}
+function isExecutionTerminal(status) {
+    return (status === "succeeded" ||
+        status === "failed" ||
+        status === "human_required" ||
+        status === "cancelled");
+}
+/**
+ * Scan schedules/leases and apply safe recovery:
+ * - release orphan claims (re-queue to waiting)
+ * - mark termination-unconfirmed as human_required
+ * - drop leftover leases on terminal schedules
+ */
+export async function recoverStaleSchedulerState(input) {
+    const now = input.now ?? new Date();
+    const results = [];
+    const schedules = await listSchedules(input.controlRepoRoot);
+    // Also scan lease files that may not match loaded schedules.
+    const leaseScheduleIds = new Set();
+    try {
+        const names = await readdir(getLeasesDir(input.controlRepoRoot));
+        for (const name of names) {
+            if (name.endsWith(".json"))
+                leaseScheduleIds.add(name.slice(0, -5));
+        }
+    }
+    catch {
+        // no leases dir
+    }
+    for (const schedule of schedules)
+        leaseScheduleIds.add(schedule.id);
+    for (const scheduleId of leaseScheduleIds) {
+        const schedule = schedules.find((item) => item.id === scheduleId);
+        const lease = await readExecutionLease(input.controlRepoRoot, scheduleId);
+        let execution;
+        if (schedule?.currentExecutionId) {
+            execution = await readExecution(input.controlRepoRoot, schedule.currentExecutionId);
+        }
+        if (!execution && lease?.executionId) {
+            execution = await readExecution(input.controlRepoRoot, lease.executionId);
+        }
+        if (!schedule) {
+            if (lease) {
+                const liveness = classifyLeaseLiveness(lease, now, input.staleMs);
+                if (liveness !== "active") {
+                    await releaseExecutionLease({
+                        controlRepoRoot: input.controlRepoRoot,
+                        scheduleId,
+                    }).catch(() => { });
+                    results.push({
+                        scheduleId,
+                        action: "released-stale-lease-on-terminal",
+                        detail: "lease without schedule snapshot released",
+                    });
+                }
+            }
+            continue;
+        }
+        const classification = classifyLeaseRecovery({
+            schedule,
+            ...(lease ? { lease } : {}),
+            ...(execution ? { execution } : {}),
+            now,
+            ...(input.staleMs !== undefined ? { staleMs: input.staleMs } : {}),
+        });
+        if (classification.kind === "already-terminal") {
+            if (lease) {
+                await releaseExecutionLease({
+                    controlRepoRoot: input.controlRepoRoot,
+                    scheduleId,
+                }).catch(() => { });
+                results.push({
+                    scheduleId,
+                    action: "released-stale-lease-on-terminal",
+                    detail: classification.reason,
+                });
+            }
+            else {
+                results.push({
+                    scheduleId,
+                    action: "none",
+                    detail: classification.reason,
+                });
+            }
+            continue;
+        }
+        if (classification.kind === "safe-reclaim") {
+            if (execution && execution.status === "claimed") {
+                await writeExecution(input.controlRepoRoot, {
+                    ...execution,
+                    status: "cancelled",
+                    finishedAt: now.toISOString(),
+                    failure: {
+                        code: "orphan-claim-before-child",
+                        message: "reclaimed after crash before worker association",
+                    },
+                });
+            }
+            await releaseExecutionLease({
+                controlRepoRoot: input.controlRepoRoot,
+                scheduleId,
+            }).catch(() => { });
+            if (schedule.status === "dispatching" ||
+                schedule.status === "running" ||
+                schedule.status === "cancel_requested") {
+                await transitionSchedule({
+                    controlRepoRoot: input.controlRepoRoot,
+                    scheduleId,
+                    toStatus: "waiting",
+                    event: "lease_recovered",
+                    reason: "orphan claim reclaimed; re-queue for next tick",
+                    reasonCode: "orphan-claim-before-child",
+                    patch: {
+                        currentExecutionId: null,
+                        waitingReason: "lease-recovered-orphan-claim",
+                        nextAttemptAt: now.toISOString(),
+                    },
+                    now,
+                });
+            }
+            results.push({
+                scheduleId,
+                action: "released-orphan-claim",
+                detail: classification.reason,
+            });
+            continue;
+        }
+        if (classification.kind === "termination-unconfirmed") {
+            if (schedule.status !== "human_required" &&
+                schedule.status !== "failed" &&
+                schedule.status !== "cancelled" &&
+                schedule.status !== "succeeded") {
+                await transitionSchedule({
+                    controlRepoRoot: input.controlRepoRoot,
+                    scheduleId,
+                    toStatus: "human_required",
+                    event: "human_required",
+                    reason: classification.reason,
+                    reasonCode: "termination-unconfirmed",
+                    now,
+                });
+            }
+            // Do NOT steal lease / do NOT start second writer.
+            results.push({
+                scheduleId,
+                action: "human-required-unconfirmed",
+                detail: classification.reason,
+            });
+            continue;
+        }
+        results.push({
+            scheduleId,
+            action: "none",
+            detail: classification.reason,
+        });
+    }
+    return results;
+}
+export async function countAttemptsForSchedule(controlRepoRoot, scheduleId) {
+    const executions = await listExecutionsForSchedule(controlRepoRoot, scheduleId);
+    return executions.length;
+}
+/** Test helper re-export surface for PID checks without exporting isPidAlive. */
+export function leaseOwnerLooksAlive(lease) {
+    return isPidAlive(lease.ownerPid);
+}

package/dist/worker/scheduler/reservation.js

@@ -0,0 +1,146 @@
+import { readTaskPoolState, writeTaskPoolState } from "../pool/run-store.js";
+import { SCHEDULER_ERROR_CODES, SchedulerError } from "./types.js";
+/**
+ * Project schedule reservation onto Task Pool: Ready → Queued with night fields.
+ * Fail-closed if another schedule already owns the reservation or status is not Ready/Queued(same id).
+ */
+export async function reserveTaskPoolForSchedule(input) {
+    const now = (input.now ?? new Date()).toISOString();
+    const ref = { featureId: input.featureId, taskId: input.taskId };
+    const current = await readTaskPoolState(input.controlRepoRoot, ref);
+    if (!current) {
+        // Create Queued projection when no pool state exists yet (admission-created schedule).
+        const created = {
+            schemaVersion: 2,
+            featureId: input.featureId,
+            taskId: input.taskId,
+            status: "Queued",
+            updatedAt: now,
+            nightScheduleId: input.nightScheduleId,
+            admissionPath: input.admissionPath,
+            nightWorktreePath: input.nightWorktreePath,
+            nightBranch: input.nightBranch,
+        };
+        await writeTaskPoolState(input.controlRepoRoot, created);
+        return created;
+    }
+    if (current.nightScheduleId &&
+        current.nightScheduleId !== input.nightScheduleId) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.DUPLICATE_ACTIVE, `Task Pool ${input.featureId}/${input.taskId} already reserved by ${current.nightScheduleId}`);
+    }
+    if (current.status === "Queued" &&
+        current.nightScheduleId === input.nightScheduleId) {
+        const refreshed = {
+            ...current,
+            schemaVersion: 2,
+            featureId: input.featureId,
+            taskId: input.taskId,
+            status: "Queued",
+            updatedAt: now,
+            nightScheduleId: input.nightScheduleId,
+            admissionPath: input.admissionPath,
+            nightWorktreePath: input.nightWorktreePath,
+            nightBranch: input.nightBranch,
+        };
+        await writeTaskPoolState(input.controlRepoRoot, refreshed);
+        return refreshed;
+    }
+    if (current.status !== "Ready") {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.INVALID_TRANSITION, `cannot reserve Task Pool ${input.featureId}/${input.taskId} from status ${current.status}`);
+    }
+    const next = {
+        ...current,
+        schemaVersion: 2,
+        featureId: input.featureId,
+        taskId: input.taskId,
+        status: "Queued",
+        updatedAt: now,
+        nightScheduleId: input.nightScheduleId,
+        admissionPath: input.admissionPath,
+        nightWorktreePath: input.nightWorktreePath,
+        nightBranch: input.nightBranch,
+    };
+    await writeTaskPoolState(input.controlRepoRoot, next);
+    return next;
+}
+/**
+ * Restore a failed night-owned Task Pool projection to Queued for a bounded
+ * prepared re-attempt. Refuses to steal another schedule or non-failed state.
+ */
+export async function requeueTaskPoolReservation(input) {
+    const now = (input.now ?? new Date()).toISOString();
+    const current = await readTaskPoolState(input.controlRepoRoot, {
+        featureId: input.featureId,
+        taskId: input.taskId,
+    });
+    if (!current) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.INVALID_RECORD, `cannot requeue missing Task Pool ${input.featureId}/${input.taskId}`);
+    }
+    if (current.nightScheduleId !== input.nightScheduleId) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.DUPLICATE_ACTIVE, `cannot requeue Task Pool ${input.featureId}/${input.taskId}: reservation is not owned by ${input.nightScheduleId}`);
+    }
+    if (current.status !== "Failed") {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.INVALID_TRANSITION, `cannot requeue Task Pool ${input.featureId}/${input.taskId} from status ${current.status}`);
+    }
+    const next = {
+        ...current,
+        status: "Queued",
+        updatedAt: now,
+    };
+    await writeTaskPoolState(input.controlRepoRoot, next);
+    return next;
+}
+/**
+ * Release Queued reservation when cancelling before dispatch.
+ * Only restores Ready when nightScheduleId still matches.
+ */
+export async function releaseTaskPoolReservation(input) {
+    const now = (input.now ?? new Date()).toISOString();
+    const ref = { featureId: input.featureId, taskId: input.taskId };
+    const current = await readTaskPoolState(input.controlRepoRoot, ref);
+    if (!current)
+        return undefined;
+    if (current.nightScheduleId !== input.nightScheduleId) {
+        return current;
+    }
+    if (current.status !== "Queued") {
+        // Clear night fields but do not force Ready if already Running/Done/Failed.
+        const cleared = {
+            ...current,
+            schemaVersion: 2,
+            updatedAt: now,
+            nightScheduleId: undefined,
+            admissionPath: undefined,
+            nightWorktreePath: undefined,
+            nightBranch: undefined,
+        };
+        // Explicitly omit night fields by rewriting without them.
+        const { nightScheduleId: _a, admissionPath: _b, nightWorktreePath: _c, nightBranch: _d, ...rest } = cleared;
+        void _a;
+        void _b;
+        void _c;
+        void _d;
+        await writeTaskPoolState(input.controlRepoRoot, {
+            ...rest,
+            schemaVersion: 2,
+            featureId: input.featureId,
+            taskId: input.taskId,
+            status: current.status,
+            updatedAt: now,
+        });
+        return rest;
+    }
+    const next = {
+        schemaVersion: 2,
+        featureId: input.featureId,
+        taskId: input.taskId,
+        status: "Ready",
+        updatedAt: now,
+        ...(current.workerRunId ? { workerRunId: current.workerRunId } : {}),
+        ...(current.retryOfWorkerRunId
+            ? { retryOfWorkerRunId: current.retryOfWorkerRunId }
+            : {}),
+    };
+    await writeTaskPoolState(input.controlRepoRoot, next);
+    return next;
+}

package/dist/worker/scheduler/retry.js

@@ -0,0 +1,199 @@
+import { listExecutionsForSchedule, readAdmission, readSchedule, } from "./store.js";
+import { transitionSchedule } from "./lifecycle.js";
+import { requeueTaskPoolReservation } from "./reservation.js";
+import { assessPreparedAttemptRecovery, } from "./prepared-attempt-recovery.js";
+import { SCHEDULER_ERROR_CODES, SchedulerError } from "./types.js";
+/** Categories eligible for any Scheduler-level auto recovery (Phase 4 complete). */
+export const AUTO_RETRY_FAILURE_CATEGORIES = new Set([
+    "EnvFailure",
+    "FlakyTest",
+]);
+/**
+ * Pure decision (legacy/simple path): whether a failed schedule may be
+ * re-queued without adapter I/O. Prefer assessAndDecideAutoRetry for
+ * full recovery-adapter semantics.
+ */
+export function decideAutoRetry(input) {
+    if (input.schedule.status !== "failed") {
+        return {
+            ok: false,
+            action: "stop",
+            reason: "not-failed",
+            detail: `status is ${input.schedule.status}`,
+        };
+    }
+    if (input.terminationUnconfirmed) {
+        return {
+            ok: false,
+            action: "stop",
+            reason: "termination-unconfirmed",
+            detail: "refusing auto retry while prior attempt termination is unconfirmed",
+        };
+    }
+    const maxAutoAttempts = input.schedule.policySnapshot?.maxAutoAttempts ?? 1;
+    if (input.attemptCount >= maxAutoAttempts) {
+        return {
+            ok: false,
+            action: "stop",
+            reason: "budget-exhausted",
+            detail: `attempts=${input.attemptCount} maxAutoAttempts=${maxAutoAttempts}`,
+        };
+    }
+    const category = input.failureCategory ?? "Unknown";
+    if (input.recoveryMode === "stop") {
+        return {
+            ok: false,
+            action: "stop",
+            reason: "recovery-adapter-stop",
+            detail: `recovery adapter refused category ${category}`,
+        };
+    }
+    if (input.recoveryMode === "verify-only-rerun") {
+        return {
+            ok: true,
+            action: "verify-only",
+            mode: "verify-only-rerun",
+            attempt: input.attemptCount + 1,
+            maxAutoAttempts,
+            category,
+        };
+    }
+    if (input.recoveryMode === "full-prepared-reattempt") {
+        return {
+            ok: true,
+            action: "queue-waiting",
+            mode: "full-prepared-reattempt",
+            attempt: input.attemptCount + 1,
+            maxAutoAttempts,
+            category,
+        };
+    }
+    // Fallback without adapter: EnvFailure only (legacy Phase 4.0 behaviour).
+    if (!AUTO_RETRY_FAILURE_CATEGORIES.has(category)) {
+        return {
+            ok: false,
+            action: "stop",
+            reason: "category-not-retryable",
+            detail: `category ${category} is not in EnvFailure/FlakyTest allowlist`,
+        };
+    }
+    if (category === "FlakyTest") {
+        // Without adapter assessment we cannot prove verify-safe node → stop.
+        return {
+            ok: false,
+            action: "stop",
+            reason: "flaky-verify-unsafe",
+            detail: "FlakyTest requires recovery adapter verify-only assessment (missing)",
+        };
+    }
+    return {
+        ok: true,
+        action: "queue-waiting",
+        mode: "full-prepared-reattempt",
+        attempt: input.attemptCount + 1,
+        maxAutoAttempts,
+        category,
+    };
+}
+/**
+ * Full recovery-adapter path: assess admission/worktree + category policy,
+ * then optionally re-queue waiting for full prepared re-attempt.
+ * Verify-only decisions are returned without transitioning (dispatcher runs them).
+ */
+export async function assessAndDecideAutoRetry(input) {
+    const schedule = await readSchedule(input.controlRepoRoot, input.scheduleId);
+    if (!schedule) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.NOT_FOUND, `schedule not found: ${input.scheduleId}`);
+    }
+    const executions = await listExecutionsForSchedule(input.controlRepoRoot, input.scheduleId);
+    const lastExecution = [...executions]
+        .reverse()
+        .find((item) => item.status === "failed" || item.status === "succeeded");
+    const admission = await readAdmission(input.controlRepoRoot, input.scheduleId);
+    const assessment = await assessPreparedAttemptRecovery({
+        controlRepoRoot: input.controlRepoRoot,
+        schedule,
+        admission,
+        lastExecution,
+        attemptCount: executions.length,
+        ...(input.failureCategory
+            ? { failureCategory: input.failureCategory }
+            : {}),
+        ...(input.terminationUnconfirmed !== undefined
+            ? { terminationUnconfirmed: input.terminationUnconfirmed }
+            : {}),
+        ...(input.attempt ? { attempt: input.attempt } : {}),
+        ...(input.workspaceRoot ? { workspaceRoot: input.workspaceRoot } : {}),
+        ...(input.revalidateWorkspace
+            ? { revalidateWorkspace: input.revalidateWorkspace }
+            : {}),
+        ...(input.inspectWorkspaceChanges
+            ? { inspectWorkspaceChanges: input.inspectWorkspaceChanges }
+            : {}),
+    });
+    const decision = decideAutoRetry({
+        schedule,
+        attemptCount: executions.length,
+        ...(input.failureCategory
+            ? { failureCategory: input.failureCategory }
+            : { failureCategory: assessment.category }),
+        ...(input.terminationUnconfirmed !== undefined
+            ? { terminationUnconfirmed: input.terminationUnconfirmed }
+            : { terminationUnconfirmed: assessment.terminationUnconfirmed }),
+        recoveryMode: assessment.mode,
+    });
+    if (!decision.ok) {
+        return { ...decision, assessment, schedule };
+    }
+    if (decision.action === "verify-only") {
+        return {
+            ...decision,
+            fromNodeId: assessment.fromNodeId,
+            parentDagRunId: assessment.parentDagRunId,
+            assessment,
+            schedule,
+        };
+    }
+    // full-prepared-reattempt → re-queue waiting
+    const now = input.now ?? new Date();
+    await requeueTaskPoolReservation({
+        controlRepoRoot: input.controlRepoRoot,
+        featureId: schedule.featureId,
+        taskId: schedule.taskId,
+        nightScheduleId: schedule.id,
+        now,
+    });
+    const transitioned = await transitionSchedule({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: input.scheduleId,
+        toStatus: "waiting",
+        event: "auto_retry_queued",
+        reason: `auto retry ${decision.attempt}/${decision.maxAutoAttempts} for ${decision.category} (${decision.mode})`,
+        reasonCode: decision.category === "EnvFailure"
+            ? "env-failure-auto-retry"
+            : "prepared-attempt-auto-retry",
+        patch: {
+            waitingReason: `auto-retry:${decision.category}:${decision.mode}`,
+            nextAttemptAt: now.toISOString(),
+            currentExecutionId: null,
+            reservation: schedule.reservation ?? {
+                taskPoolStatus: "Queued",
+                reservedAt: now.toISOString(),
+            },
+        },
+        now,
+    });
+    return {
+        ...decision,
+        assessment,
+        schedule: transitioned.schedule,
+    };
+}
+/**
+ * Re-queue a failed schedule for another tick attempt when policy allows.
+ * Keeps Queued reservation / admission; never re-generates DAG.
+ * Uses full recovery adapter when controlRepoRoot is available.
+ */
+export async function maybeQueueAutoRetry(input) {
+    return assessAndDecideAutoRetry(input);
+}