@tea-agent/loop-agent 0.28.2-beta.1 → 0.28.2

package/dist/worker/scheduler/lease.js

@@ -0,0 +1,114 @@
+import { randomBytes } from "node:crypto";
+import { readFile, rm, unlink } from "node:fs/promises";
+import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
+import { getLeasePath } from "./paths.js";
+import { ensureSchedulerDirs } from "./store.js";
+import { SCHEDULER_ERROR_CODES, SchedulerError } from "./types.js";
+export async function readExecutionLease(controlRepoRoot, scheduleId) {
+    try {
+        const raw = await readFile(getLeasePath(controlRepoRoot, scheduleId), "utf-8");
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        if (error &&
+            typeof error === "object" &&
+            "code" in error &&
+            error.code === "ENOENT") {
+            return undefined;
+        }
+        throw error;
+    }
+}
+export async function acquireExecutionLease(input) {
+    const now = input.now ?? new Date();
+    const existing = await readExecutionLease(input.controlRepoRoot, input.scheduleId);
+    if (existing) {
+        const staleMs = input.staleMs ?? 5 * 60_000;
+        const heartbeatMs = Date.parse(existing.heartbeatAt);
+        const isStale = !Number.isFinite(heartbeatMs) ||
+            now.getTime() - heartbeatMs > staleMs ||
+            !isPidAlive(existing.ownerPid);
+        if (!isStale) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.LOCK_HELD, `execution lease held for ${input.scheduleId} by pid ${existing.ownerPid}`);
+        }
+        if (!input.allowStealStale) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.LOCK_HELD, `stale execution lease present for ${input.scheduleId}; refusing automatic steal without allowStealStale`);
+        }
+    }
+    const lease = {
+        schemaVersion: 1,
+        scheduleId: input.scheduleId,
+        executionId: input.executionId,
+        ownerPid: process.pid,
+        hostname: input.hostname ?? "local",
+        token: `lease-${randomBytes(8).toString("hex")}`,
+        acquiredAt: now.toISOString(),
+        heartbeatAt: now.toISOString(),
+        ...(input.workspacePath ? { workspacePath: input.workspacePath } : {}),
+    };
+    await ensureSchedulerDirs(input.controlRepoRoot);
+    await writeJsonAtomic(getLeasePath(input.controlRepoRoot, input.scheduleId), lease, {
+        repoRoot: input.controlRepoRoot,
+    });
+    return lease;
+}
+export async function heartbeatExecutionLease(input) {
+    const current = await readExecutionLease(input.controlRepoRoot, input.scheduleId);
+    if (!current || current.token !== input.token) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.LOCK_HELD, `cannot heartbeat lease for ${input.scheduleId}: token mismatch or missing`);
+    }
+    const now = input.now ?? new Date();
+    const next = {
+        ...current,
+        heartbeatAt: now.toISOString(),
+        ...(input.workerRunId ? { workerRunId: input.workerRunId } : {}),
+        ...(input.dagRunId ? { dagRunId: input.dagRunId } : {}),
+    };
+    await writeJsonAtomic(getLeasePath(input.controlRepoRoot, input.scheduleId), next, {
+        repoRoot: input.controlRepoRoot,
+    });
+    return next;
+}
+export async function releaseExecutionLease(input) {
+    const current = await readExecutionLease(input.controlRepoRoot, input.scheduleId);
+    if (!current)
+        return;
+    if (input.token && current.token !== input.token) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.LOCK_HELD, `cannot release lease for ${input.scheduleId}: token mismatch`);
+    }
+    try {
+        await unlink(getLeasePath(input.controlRepoRoot, input.scheduleId));
+    }
+    catch (error) {
+        if (!(error &&
+            typeof error === "object" &&
+            "code" in error &&
+            error.code === "ENOENT")) {
+            // directory-style cleanup fallback
+            await rm(getLeasePath(input.controlRepoRoot, input.scheduleId), {
+                force: true,
+            });
+        }
+    }
+}
+export function countActiveLeases(leases, now = new Date(), staleMs = 5 * 60_000) {
+    return leases.filter((lease) => {
+        const heartbeatMs = Date.parse(lease.heartbeatAt);
+        if (!Number.isFinite(heartbeatMs))
+            return false;
+        if (now.getTime() - heartbeatMs > staleMs)
+            return false;
+        return isPidAlive(lease.ownerPid);
+    }).length;
+}
+function isPidAlive(pid) {
+    if (!Number.isFinite(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/worker/scheduler/lifecycle.js

@@ -0,0 +1,348 @@
+import { withSchedulerMutationLock } from "./lock.js";
+import { appendLedgerEvent, createEventId, createScheduleId, createTransitionId, deleteTransitionJournal, listActiveSchedulesForTask, listSchedules, readSchedule, recoverAllPendingJournals, writeScheduleSnapshot, writeTransitionJournal, } from "./store.js";
+import { canTransitionSchedule, defaultEventForTransition, isTerminalScheduleStatus, SCHEDULER_ERROR_CODES, SchedulerError, } from "./types.js";
+/**
+ * Create a submitted schedule fact. Enforces single active schedule per task
+ * under the mutation lock. Does not run admission or dispatch.
+ */
+export async function submitSchedule(input) {
+    const now = input.now ?? new Date();
+    return withSchedulerMutationLock({
+        controlRepoRoot: input.controlRepoRoot,
+        hostname: input.hostname,
+        now: () => now,
+    }, async () => {
+        await recoverAllPendingJournals(input.controlRepoRoot);
+        const active = await listActiveSchedulesForTask(input.controlRepoRoot, input.featureId, input.taskId);
+        if (active.length > 0) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.DUPLICATE_ACTIVE, `active schedule already exists for ${input.featureId}/${input.taskId}: ${active.map((item) => item.id).join(", ")}`);
+        }
+        const scheduleId = createScheduleId(now);
+        const transitionId = createTransitionId(now);
+        const eventId = createEventId(now);
+        const submittedAt = now.toISOString();
+        const schedule = {
+            schemaVersion: 1,
+            id: scheduleId,
+            revision: 0,
+            featureId: input.featureId,
+            taskId: input.taskId,
+            ...(input.featureDir ? { featureDir: input.featureDir } : {}),
+            status: "submitted",
+            dispatch: { mode: "single-task" },
+            ...(input.trigger ? { trigger: input.trigger } : {}),
+            ...(input.policySnapshot
+                ? { policySnapshot: input.policySnapshot }
+                : {}),
+            ...(input.isolation ? { isolation: input.isolation } : {}),
+            ...(input.traceability || input.workItemRef
+                ? {
+                    traceability: {
+                        kind: "task-card",
+                        label: input.traceability?.label ?? "Task Card",
+                        requiredOnCreate: input.traceability?.requiredOnCreate ?? true,
+                        requiredOnEveryPushableCommit: input.traceability?.requiredOnEveryPushableCommit ?? true,
+                        ...(input.workItemRef
+                            ? { workItemRef: input.workItemRef }
+                            : {}),
+                        ...(input.traceability?.pattern
+                            ? { pattern: input.traceability.pattern }
+                            : {}),
+                        ...(input.traceability?.commitMessageTemplate
+                            ? {
+                                commitMessageTemplate: input.traceability.commitMessageTemplate,
+                            }
+                            : {}),
+                        ...(input.traceability?.workItemRef
+                            ? { workItemRef: input.traceability.workItemRef }
+                            : {}),
+                    },
+                }
+                : {}),
+            timestamps: {
+                submittedAt,
+                validatedAt: null,
+                scheduledAt: null,
+                startedAt: null,
+                finishedAt: null,
+            },
+            currentExecutionId: null,
+            lastTransitionId: transitionId,
+            lastLedgerEventId: eventId,
+            waitingReason: null,
+            nextAttemptAt: null,
+            cancelReason: null,
+            rejectReason: null,
+        };
+        const ledgerEvent = {
+            schemaVersion: 1,
+            eventId,
+            transitionId,
+            revision: 0,
+            at: submittedAt,
+            scheduleId,
+            featureId: input.featureId,
+            taskId: input.taskId,
+            fromStatus: null,
+            toStatus: "submitted",
+            event: "plan_submitted",
+            ...(input.trigger?.executeAtUtc
+                ? { plannedExecuteAtUtc: input.trigger.executeAtUtc }
+                : {}),
+            reasonCode: null,
+            reason: null,
+            executionId: null,
+            worktreePath: input.isolation?.worktreePath ?? null,
+            evidencePaths: [],
+        };
+        return commitTransition({
+            controlRepoRoot: input.controlRepoRoot,
+            fromStatus: null,
+            toStatus: "submitted",
+            expectedRevision: null,
+            schedule,
+            ledgerEvent,
+            transitionId,
+            event: "plan_submitted",
+            now,
+        });
+    });
+}
+/**
+ * Single mutation entry for schedule status changes.
+ * Writes transition journal → snapshot → ledger → delete journal.
+ */
+export async function transitionSchedule(input) {
+    const now = input.now ?? new Date();
+    const run = async () => {
+        await recoverAllPendingJournals(input.controlRepoRoot);
+        const current = await readSchedule(input.controlRepoRoot, input.scheduleId);
+        if (!current) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.NOT_FOUND, `schedule not found: ${input.scheduleId}`);
+        }
+        if (input.expectedRevision !== undefined &&
+            input.expectedRevision !== current.revision) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.REVISION_CONFLICT, `schedule ${input.scheduleId} revision conflict: expected ${input.expectedRevision}, actual ${current.revision}`);
+        }
+        // Phase 4: failed → waiting is an explicit auto-retry edge; all other
+        // transitions out of terminal statuses remain forbidden.
+        const autoRetryEdge = current.status === "failed" && input.toStatus === "waiting";
+        if (isTerminalScheduleStatus(current.status) && !autoRetryEdge) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.TERMINAL, `schedule ${input.scheduleId} is terminal (${current.status})`);
+        }
+        if (!canTransitionSchedule(current.status, input.toStatus)) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.INVALID_TRANSITION, `invalid transition ${current.status} → ${input.toStatus} for ${input.scheduleId}`);
+        }
+        const transitionId = createTransitionId(now);
+        const eventId = createEventId(now);
+        const at = now.toISOString();
+        const eventName = input.event ?? defaultEventForTransition(current.status, input.toStatus);
+        const nextRevision = current.revision + 1;
+        const timestamps = { ...current.timestamps };
+        if (input.toStatus === "validated")
+            timestamps.validatedAt = at;
+        if (input.toStatus === "scheduled")
+            timestamps.scheduledAt = at;
+        if (input.toStatus === "running" || input.toStatus === "dispatching") {
+            timestamps.startedAt = timestamps.startedAt ?? at;
+        }
+        if (input.toStatus === "succeeded" ||
+            input.toStatus === "failed" ||
+            input.toStatus === "human_required" ||
+            input.toStatus === "cancelled" ||
+            input.toStatus === "rejected") {
+            timestamps.finishedAt = at;
+        }
+        const schedule = {
+            ...current,
+            ...input.patch,
+            status: input.toStatus,
+            revision: nextRevision,
+            timestamps,
+            lastTransitionId: transitionId,
+            lastLedgerEventId: eventId,
+            ...(input.toStatus === "cancelled" && input.reason
+                ? { cancelReason: input.reason }
+                : {}),
+            ...(input.toStatus === "rejected" && input.reason
+                ? { rejectReason: input.reason }
+                : {}),
+        };
+        const ledgerEvent = {
+            schemaVersion: 1,
+            eventId,
+            transitionId,
+            revision: nextRevision,
+            at,
+            scheduleId: current.id,
+            featureId: current.featureId,
+            taskId: current.taskId,
+            fromStatus: current.status,
+            toStatus: input.toStatus,
+            event: eventName,
+            ...(schedule.trigger?.executeAtUtc
+                ? { plannedExecuteAtUtc: schedule.trigger.executeAtUtc }
+                : {}),
+            reasonCode: input.reasonCode ?? null,
+            reason: input.reason ?? null,
+            executionId: schedule.currentExecutionId ?? null,
+            worktreePath: schedule.isolation?.worktreePath ?? null,
+            evidencePaths: input.evidencePaths ?? [],
+        };
+        return commitTransition({
+            controlRepoRoot: input.controlRepoRoot,
+            fromStatus: current.status,
+            toStatus: input.toStatus,
+            expectedRevision: current.revision,
+            schedule,
+            ledgerEvent,
+            transitionId,
+            event: eventName,
+            now,
+        });
+    };
+    if (input.alreadyLocked) {
+        return run();
+    }
+    return withSchedulerMutationLock({
+        controlRepoRoot: input.controlRepoRoot,
+        hostname: input.hostname,
+        now: () => now,
+    }, run);
+}
+/**
+ * Cancel a non-running schedule (submitted|validating|validated|scheduled|waiting).
+ * Running/dispatching must use cancel_requested (Phase 3).
+ */
+export async function cancelSchedule(input) {
+    const current = await readSchedule(input.controlRepoRoot, input.scheduleId);
+    if (!current) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.NOT_FOUND, `schedule not found: ${input.scheduleId}`);
+    }
+    const cancellable = new Set([
+        "submitted",
+        "validating",
+        "validated",
+        "scheduled",
+        "waiting",
+    ]);
+    if (!cancellable.has(current.status)) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.INVALID_TRANSITION, `cannot cancel schedule in status ${current.status}; use cancel_requested for running attempts`);
+    }
+    return transitionSchedule({
+        controlRepoRoot: input.controlRepoRoot,
+        scheduleId: input.scheduleId,
+        toStatus: "cancelled",
+        event: "schedule_cancelled",
+        reason: input.reason,
+        reasonCode: "operator-cancel",
+        now: input.now,
+        hostname: input.hostname,
+    });
+}
+export async function patchScheduleIsolation(input) {
+    const now = input.now ?? new Date();
+    return withSchedulerMutationLock({
+        controlRepoRoot: input.controlRepoRoot,
+        hostname: input.hostname,
+        now: () => now,
+    }, async () => {
+        await recoverAllPendingJournals(input.controlRepoRoot);
+        const current = await readSchedule(input.controlRepoRoot, input.scheduleId);
+        if (!current) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.NOT_FOUND, `schedule not found: ${input.scheduleId}`);
+        }
+        if (input.expectedRevision !== undefined &&
+            input.expectedRevision !== current.revision) {
+            throw new SchedulerError(SCHEDULER_ERROR_CODES.REVISION_CONFLICT, `schedule ${input.scheduleId} revision conflict: expected ${input.expectedRevision}, actual ${current.revision}`);
+        }
+        const transitionId = createTransitionId(now);
+        const eventId = createEventId(now);
+        const next = {
+            ...current,
+            revision: current.revision + 1,
+            isolation: input.isolation,
+            lastTransitionId: transitionId,
+            lastLedgerEventId: eventId,
+        };
+        const ledgerEvent = {
+            schemaVersion: 1,
+            eventId,
+            transitionId,
+            revision: next.revision,
+            at: now.toISOString(),
+            scheduleId: current.id,
+            featureId: current.featureId,
+            taskId: current.taskId,
+            fromStatus: current.status,
+            toStatus: current.status,
+            event: input.event,
+            reasonCode: input.reasonCode,
+            reason: input.reason,
+            executionId: current.currentExecutionId ?? null,
+            worktreePath: input.isolation.worktreePath ?? null,
+            evidencePaths: input.evidencePaths ?? [],
+        };
+        return commitTransition({
+            controlRepoRoot: input.controlRepoRoot,
+            fromStatus: current.status,
+            toStatus: current.status,
+            expectedRevision: current.revision,
+            schedule: next,
+            ledgerEvent,
+            transitionId,
+            event: input.event,
+            now,
+        });
+    });
+}
+export async function getScheduleStatus(input) {
+    await recoverAllPendingJournals(input.controlRepoRoot);
+    const schedule = await readSchedule(input.controlRepoRoot, input.scheduleId);
+    if (!schedule) {
+        throw new SchedulerError(SCHEDULER_ERROR_CODES.NOT_FOUND, `schedule not found: ${input.scheduleId}`);
+    }
+    return schedule;
+}
+export async function listScheduleSnapshots(input) {
+    await recoverAllPendingJournals(input.controlRepoRoot);
+    const all = await listSchedules(input.controlRepoRoot);
+    if (!input.status)
+        return all;
+    const allowed = new Set(Array.isArray(input.status) ? input.status : [input.status]);
+    return all.filter((item) => allowed.has(item.status));
+}
+async function commitTransition(input) {
+    const journal = {
+        schemaVersion: 1,
+        transitionId: input.transitionId,
+        scheduleId: input.schedule.id,
+        status: "pending",
+        createdAt: input.now.toISOString(),
+        expectedRevision: input.expectedRevision,
+        fromStatus: input.fromStatus,
+        toStatus: input.toStatus,
+        event: input.event,
+        patch: {},
+        ledgerEvent: input.ledgerEvent,
+        scheduleSnapshot: input.schedule,
+    };
+    // 1. journal pending
+    await writeTransitionJournal(input.controlRepoRoot, journal);
+    // 2. schedule snapshot
+    await writeScheduleSnapshot(input.controlRepoRoot, input.schedule);
+    // 3. ledger append (idempotent by eventId)
+    await appendLedgerEvent(input.controlRepoRoot, input.ledgerEvent);
+    // 4. mark committed then delete
+    await writeTransitionJournal(input.controlRepoRoot, {
+        ...journal,
+        status: "committed",
+        committedAt: new Date().toISOString(),
+    });
+    await deleteTransitionJournal(input.controlRepoRoot, input.transitionId);
+    return {
+        schedule: input.schedule,
+        ledgerEvent: input.ledgerEvent,
+        transitionId: input.transitionId,
+    };
+}

package/dist/worker/scheduler/lock.js

@@ -0,0 +1,80 @@
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { getMutationLockPath, getSchedulerRoot } from "./paths.js";
+import { SCHEDULER_ERROR_CODES, SchedulerError } from "./types.js";
+/**
+ * Short-lived exclusive mutation lock for schedule add/cancel/claim/transition.
+ * Uses mkdir-as-lock (same pattern as update/evaluation locks). Must not be held
+ * across multi-hour DAG execution.
+ */
+export async function acquireSchedulerMutationLock(options) {
+    const now = options.now ?? (() => new Date());
+    const staleMs = options.staleMs ?? 30_000;
+    const lockPath = getMutationLockPath(options.controlRepoRoot);
+    const owner = {
+        pid: process.pid,
+        hostname: options.hostname ?? "local",
+        token: options.token ?? `tok-${process.pid}-${now().getTime()}`,
+        acquiredAt: now().toISOString(),
+    };
+    await mkdir(getSchedulerRoot(options.controlRepoRoot), { recursive: true });
+    const tryAcquire = async () => {
+        try {
+            await mkdir(lockPath);
+            await writeFile(path.join(lockPath, "owner.json"), `${JSON.stringify(owner, null, 2)}\n`, "utf-8");
+            return true;
+        }
+        catch (error) {
+            if (!isExist(error))
+                throw error;
+            return false;
+        }
+    };
+    if (await tryAcquire()) {
+        return {
+            owner,
+            release: async () => {
+                await rm(lockPath, { recursive: true, force: true });
+            },
+        };
+    }
+    // Stale recovery: if owner is old enough, reclaim.
+    try {
+        const raw = await readFile(path.join(lockPath, "owner.json"), "utf-8");
+        const existing = JSON.parse(raw);
+        const acquiredAtMs = existing.acquiredAt
+            ? Date.parse(existing.acquiredAt)
+            : Number.NaN;
+        if (Number.isFinite(acquiredAtMs) &&
+            now().getTime() - acquiredAtMs > staleMs) {
+            await rm(lockPath, { recursive: true, force: true });
+            if (await tryAcquire()) {
+                return {
+                    owner,
+                    release: async () => {
+                        await rm(lockPath, { recursive: true, force: true });
+                    },
+                };
+            }
+        }
+    }
+    catch {
+        // fall through to lock-held
+    }
+    throw new SchedulerError(SCHEDULER_ERROR_CODES.LOCK_HELD, `Scheduler mutation lock is held: ${lockPath}`);
+}
+export async function withSchedulerMutationLock(options, fn) {
+    const handle = await acquireSchedulerMutationLock(options);
+    try {
+        return await fn(handle);
+    }
+    finally {
+        await handle.release();
+    }
+}
+function isExist(error) {
+    return Boolean(error &&
+        typeof error === "object" &&
+        "code" in error &&
+        error.code === "EEXIST");
+}