agent-dealer 0.2.0 → 1.0.0

package/bundle/server/dist/coordinator/abort.integration.test.js

@@ -0,0 +1,74 @@
+// packages/server/src/coordinator/abort.integration.test.ts
+//
+// NOT-83 acceptance scenario that needs the real worker loop wired up: abort while a
+// developer effect handler is actually in flight (leased, session running), then let that
+// zombie attempt's late completion arrive — it must be fenced (a no-op) and must never
+// reopen or mutate the now-closed issue. Mirrors execution-env.integration.test.ts's
+// fake-effect-handler + runCoordinatorTick/drainCoordinator pump pattern.
+import { test, before, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-abort-"));
+const { migrate, getDb } = await import("../db/index.js");
+const { BUILTIN_AGENT_CLAUDE_ID, BUILTIN_AGENT_CURSOR_ID } = await import("@agent-dealer/shared");
+const { createIssue, getIssue } = await import("../repository/issues.js");
+const { getWorkItem, listWorkItemsForIssue } = await import("../repository/work-items.js");
+const { getWorkerSession } = await import("../repository/worker-sessions.js");
+const { startWorkflow, abortIssue } = await import("./commands.js");
+const { registerEffectHandler, resetEffectHandlers } = await import("./effect-registry.js");
+const { runCoordinatorTick, drainCoordinator } = await import("./worker-loop.js");
+before(() => migrate());
+beforeEach(() => getDb().exec("DELETE FROM work_items"));
+afterEach(() => resetEffectHandlers());
+function newIssue() {
+    return createIssue({
+        title: "Abort mid-flight",
+        description: "d",
+        acceptanceCriteria: "It works",
+        repo: "acme/app",
+        developerAgentId: BUILTIN_AGENT_CLAUDE_ID,
+        reviewerAgentId: BUILTIN_AGENT_CURSOR_ID,
+        baseBranch: "main",
+        maxReviewRounds: 3,
+        maxInfraAttempts: 3,
+        source: "manual"
+    }).id;
+}
+test("abortIssue fences a developer effect that completes after the abort already committed", async () => {
+    const issueId = newIssue();
+    startWorkflow(issueId);
+    // A deferred the test controls: the fake developer effect blocks here until the test
+    // lets it resolve, simulating a session still genuinely running when the abort lands.
+    let releaseEffect;
+    const blocked = new Promise((resolve) => {
+        releaseEffect = resolve;
+    });
+    registerEffectHandler("developer", async () => {
+        await blocked;
+        return { kind: "clean_handoff", branch: "issue-x", headSha: "deadbeef", baseSha: "base1", prNumber: 1, prUrl: "https://gh/pr/1" };
+    });
+    // Synchronous up to the handler's own first await (session create/bind/start all run
+    // inside one non-async transaction before the handler is ever invoked) — by the time
+    // this resolves, the work item is already `leased` and its session `running`.
+    const started = await runCoordinatorTick({ leaseOwner: "test" });
+    assert.equal(started, 1);
+    const item = listWorkItemsForIssue(issueId)[0];
+    assert.equal(item.status, "leased");
+    const sessionId = item.workerSessionId;
+    assert.ok(sessionId, "expected the session to already be bound before the handler blocks");
+    assert.equal(getWorkerSession(sessionId).status, "running");
+    const killed = [];
+    const result = abortIssue(issueId, "tester", { killProcess: (id) => (killed.push(id), true) });
+    assert.deepEqual(result, { ok: true, issueStatus: "closed", alreadyClosed: false });
+    assert.deepEqual(killed, [sessionId], "the running session's registered child process must be terminated");
+    assert.equal(getWorkItem(item.id).status, "cancelled");
+    assert.equal(getWorkerSession(sessionId).status, "cancelled");
+    // The zombie attempt now finishes and tries to hand off its (otherwise valid) outcome.
+    releaseEffect();
+    await drainCoordinator();
+    assert.equal(getWorkItem(item.id).status, "cancelled", "a late completion must never re-advance a cancelled work item");
+    assert.equal(getIssue(issueId).status, "closed", "a late completion must never reopen or mutate a closed issue");
+    assert.equal(getIssue(issueId).prNumber, null, "the late clean_handoff must never be projected onto the issue");
+});

package/bundle/server/dist/coordinator/admission.js

@@ -0,0 +1,330 @@
+// packages/server/src/coordinator/admission.ts
+//
+// NOT-103: when to call startWorkflowCore on which queued issue.
+// Order (queue_entries) + eligibility rules + CapacityPolicy — not a second execution engine.
+//
+// NOT-118: this is also the *only* entry point a route may use to start an issue
+// (`startIssueViaQueue`). Manual Start moves the entry to the front and admits it when a
+// slot is free — it never bypasses the queue. The single ungated exception is a human
+// action resolving into startWorkflowCore (commands.ts).
+import { checkAgentDeckHealth } from "../adapters/agent-deck.js";
+import { healthForAgent } from "../adapters/agent-health.js";
+import { getDb } from "../db/index.js";
+import { getAgent } from "../repository/agents.js";
+import { getIssue, listIssues } from "../repository/issues.js";
+import { enqueueIssue, getQueuedEntryForIssue, listQueuedEntries, markQueueEntryAdmitted, markQueueEntryRemoved, moveQueueEntryToTop, setQueueWaitReason, } from "../repository/queue-entries.js";
+import { runtimeAvailability } from "../repository/runtime-availability.js";
+import { getActiveWorkflowInstance } from "../repository/workflow-events.js";
+import { listWorkItemsForIssue } from "../repository/work-items.js";
+import { activeWorkflowConflictMessage, checkIssueReadiness, startWorkflowCore, StartPreconditionError, } from "./commands.js";
+import { blockerSnapshotFor, DEPENDENCY_STATE_UNAVAILABLE, isDependencyTracked, unsatisfiedBlockerReason, } from "./dependencies.js";
+/** Statuses that occupy an admission slot (Decision 2). */
+export const occupyingStatuses = new Set(["developing", "reviewing", "repairing"]);
+/**
+ * Statuses admission can actually start: `ready`, or `needs_human` parked back for a human
+ * to unblock. Everything else — mid-flight (`developing`/`reviewing`/`repairing`), awaiting
+ * a human's merge call (`final_review`), or terminal — is not startable.
+ */
+export const startableStatuses = new Set(["ready", "needs_human"]);
+/**
+ * The single "may this issue enter the queue" predicate, shared by the eligibility rule,
+ * Manual Start and the create route's idempotent re-enqueue. Enqueueing on a looser
+ * predicate creates a row admission can never accept: a migrated `final_review` issue has
+ * no *active* instance but a completed one, so "nonterminal + no active workflow" would
+ * queue it forever with `status final_review — not startable`.
+ */
+export function isStartable(issue) {
+    return startableStatuses.has(issue.status) && !getActiveWorkflowInstance(issue.id);
+}
+export const sequentialCapacityPolicy = (activeIssues) => {
+    const occupying = activeIssues.filter((i) => occupyingStatuses.has(i.status)).length;
+    return Math.max(0, 1 - occupying);
+};
+let capacityPolicy = sequentialCapacityPolicy;
+/** Test / future parallel wiring — swap the single CapacityPolicy function. */
+export function setCapacityPolicyForTests(policy) {
+    capacityPolicy = policy;
+}
+export function resetCapacityPolicyForTests() {
+    capacityPolicy = sequentialCapacityPolicy;
+}
+export function countOccupyingIssues() {
+    return listOccupyingIssues().length;
+}
+function listOccupyingIssues() {
+    return listIssues([...occupyingStatuses]).map((i) => ({ id: i.id, status: i.status }));
+}
+let healthChecker = null;
+/** Tests inject a pure checker so admission does not hit real CLIs. */
+export function setAdmissionHealthCheckerForTests(checker) {
+    healthChecker = checker;
+}
+async function defaultAgentHealth(agent, deckOnline) {
+    const health = await healthForAgent(agent, deckOnline);
+    // usage_capped is owned by runtimeAvailable — keep agentsHealthy for CLI/workspace/deck.
+    const nonCap = health.issues.filter((i) => i.code !== "usage_capped");
+    if (nonCap.length > 0) {
+        return {
+            ok: false,
+            reason: `agent unhealthy: ${agent.name} — ${nonCap.map((i) => i.message).join("; ")}`,
+        };
+    }
+    return { ok: true };
+}
+async function checkAgentsHealthy(issue, ctx) {
+    const check = healthChecker ?? ((agent) => defaultAgentHealth(agent, ctx.deckOnline));
+    for (const role of ["developer", "reviewer"]) {
+        const agentId = role === "developer" ? issue.developerAgentId : issue.reviewerAgentId;
+        if (!agentId)
+            return { ok: false, reason: `missing ${role} agent` };
+        const agent = getAgent(agentId);
+        if (!agent)
+            return { ok: false, reason: `${role} agent not found` };
+        const result = await check(agent);
+        if (!result.ok)
+            return result;
+    }
+    return { ok: true };
+}
+/**
+ * Status must be startable (`ready`, or `needs_human` with no active workflow) and
+ * checkIssueReadiness must pass — never open product_scope_decision.
+ */
+function issueReadinessRule(issue) {
+    if (!startableStatuses.has(issue.status)) {
+        return { ok: false, reason: `issue status is ${issue.status} — not startable` };
+    }
+    if (getActiveWorkflowInstance(issue.id)) {
+        return { ok: false, reason: "issue already has an active workflow" };
+    }
+    const readiness = checkIssueReadiness(issue);
+    if (!readiness.ok) {
+        if (readiness.missing.includes("acceptance criteria") && readiness.missing.length === 1) {
+            return { ok: false, reason: "missing acceptance criteria" };
+        }
+        return { ok: false, reason: `missing ${readiness.missing.join(", ")}` };
+    }
+    return { ok: true };
+}
+function runtimeAvailableRule(issue) {
+    for (const role of ["developer", "reviewer"]) {
+        const agentId = role === "developer" ? issue.developerAgentId : issue.reviewerAgentId;
+        if (!agentId)
+            continue;
+        const agent = getAgent(agentId);
+        if (!agent)
+            continue;
+        const avail = runtimeAvailability(agent.runtime);
+        if (!avail.available) {
+            return {
+                ok: false,
+                reason: `runtime capped: ${agent.runtime} until ${avail.until} (${avail.reason})`,
+            };
+        }
+    }
+    return { ok: true };
+}
+/**
+ * NOT-104: park a Linear-sourced issue while its *declared* blockers are unsatisfied, so a
+ * dependent never branches from a base that is missing its upstream work.
+ *
+ * Manual (`source != 'linear'`) issues have no declared dependency data, so this is a
+ * permanent no-op for them — including while Linear is unreachable, which is what keeps a
+ * Linear outage from pausing the whole queue.
+ *
+ * There is no bypass flag: the escape hatch is editing Linear (drop the relation, or cancel
+ * an abandoned blocker), which fixes the cause for every future dependent too. Cycles are
+ * not detected — both entries simply park pointing at each other, which is a stalled pair
+ * the operator can see and fix, never a wrong run.
+ */
+function blockedByDependency(issue, ctx) {
+    if (!isDependencyTracked(issue))
+        return { ok: true };
+    const blockers = ctx.blockers.get(issue.externalId);
+    // Absent means unknown, not unblocked (see BlockerSnapshot) — park until the next fetch.
+    if (!blockers)
+        return { ok: false, reason: DEPENDENCY_STATE_UNAVAILABLE };
+    const reason = unsatisfiedBlockerReason(blockers);
+    return reason ? { ok: false, reason } : { ok: true };
+}
+/** Initial eligibility rules — NOT-104 adds blockedByDependency here, not new machinery. */
+export const defaultEligibilityRules = [
+    issueReadinessRule,
+    checkAgentsHealthy,
+    runtimeAvailableRule,
+    blockedByDependency,
+];
+let eligibilityRules = defaultEligibilityRules;
+export function setEligibilityRulesForTests(rules) {
+    eligibilityRules = rules;
+}
+export function resetEligibilityRulesForTests() {
+    eligibilityRules = defaultEligibilityRules;
+}
+async function evaluateEligibility(issue, ctx) {
+    for (const rule of eligibilityRules) {
+        const result = await rule(issue, ctx);
+        if (!result.ok)
+            return result;
+    }
+    return { ok: true };
+}
+export async function admitNext() {
+    const freeSlots = capacityPolicy(listOccupyingIssues());
+    const entries = listQueuedEntries();
+    // Housekeeping pass — independent of free slots.
+    for (const entry of entries) {
+        const issue = getIssue(entry.issueId);
+        if (!issue || issue.status === "done" || issue.status === "closed") {
+            markQueueEntryRemoved(entry.issueId);
+            continue;
+        }
+        if (getActiveWorkflowInstance(entry.issueId)) {
+            markQueueEntryAdmitted(entry.issueId);
+        }
+    }
+    if (freeSlots <= 0)
+        return null;
+    // Re-list after housekeeping so admitted/removed rows are gone.
+    const remaining = listQueuedEntries();
+    if (remaining.length === 0)
+        return null;
+    // One deck-health check per tick, shared across every queued entry and role. Each
+    // agent's health check otherwise re-hits agent-deck's uncached /health endpoint per
+    // entry per role; with several issues queued and agent-deck slow/unreachable that
+    // serially stalls this loop, which worker-loop runs before any real work dispatch.
+    //
+    // Same shape for NOT-104's blocker snapshot: one batched, timeout-bounded Linear query
+    // for the whole remaining queue, and only on a tick that actually has a slot to fill —
+    // the early return above means a busy system makes zero Linear calls.
+    const queuedIssues = remaining
+        .map((entry) => getIssue(entry.issueId))
+        .filter((issue) => issue !== null);
+    const [deckOnline, blockers] = await Promise.all([
+        checkAgentDeckHealth(),
+        blockerSnapshotFor(queuedIssues),
+    ]);
+    const ctx = { deckOnline, blockers };
+    for (const entry of remaining) {
+        const issue = getIssue(entry.issueId);
+        if (!issue) {
+            markQueueEntryRemoved(entry.issueId);
+            continue;
+        }
+        const eligibility = await evaluateEligibility(issue, ctx);
+        if (!eligibility.ok) {
+            setQueueWaitReason(entry.id, eligibility.reason);
+            continue;
+        }
+        try {
+            const started = getDb().transaction(() => {
+                // Dequeue race: operator may have removed the entry during async eligibility checks.
+                if (!getQueuedEntryForIssue(entry.issueId)) {
+                    throw new StartPreconditionError(409, "queue entry no longer queued");
+                }
+                // Re-check capacity inside the txn so a concurrent Manual Start cannot double-admit.
+                if (capacityPolicy(listOccupyingIssues()) <= 0) {
+                    throw new StartPreconditionError(409, "no free admission slots");
+                }
+                // Force-admit lives inside startWorkflowCore — single owner for start-path-queue-sync.
+                return startWorkflowCore(entry.issueId);
+            })();
+            return { issueId: entry.issueId, ...started };
+        }
+        catch (err) {
+            const message = err instanceof StartPreconditionError
+                ? err.message
+                : err instanceof Error
+                    ? err.message
+                    : String(err);
+            // Entry was dequeued — nothing to record; try the next entry.
+            if (message.includes("no longer queued"))
+                continue;
+            // Capacity exhausted mid-walk (concurrent Manual Start) — stop; do not write wait reasons.
+            if (message.includes("no free admission slots"))
+                return null;
+            setQueueWaitReason(entry.id, message);
+        }
+    }
+    return null;
+}
+/**
+ * NOT-118 read-time wait reason. `admitNext` returns before evaluating anything when no
+ * slot is free, so a full system would otherwise leave every queued entry with no reason
+ * (or a stale one from an earlier tick). Derived on read instead of written every tick.
+ */
+export function slotWaitReason() {
+    const running = listIssues([...occupyingStatuses]);
+    if (capacityPolicy(running.map((i) => ({ id: i.id, status: i.status }))) > 0)
+        return null;
+    const titles = running.map((i) => i.title).join(", ");
+    return titles ? `waiting for slot — running: ${titles}` : "waiting for slot";
+}
+/**
+ * The queue as operators read it: 1-based positions (rank, not the raw stored column, which
+ * legacy rows may have left sparse) and the current blocker per entry. While capacity is
+ * full that blocker *is* the missing slot, so the derived reason wins over whatever an
+ * earlier tick persisted; once a slot frees, the entry's own reason (missing acceptance
+ * criteria, unhealthy agent, capped runtime) surfaces again on the next tick.
+ */
+export function listQueuedEntriesForRead() {
+    const slotReason = slotWaitReason();
+    return listQueuedEntries().map((entry, index) => ({
+        ...entry,
+        position: index + 1,
+        waitReason: slotReason ?? entry.waitReason,
+    }));
+}
+export function queueStatusForIssue(issueId) {
+    const entry = listQueuedEntriesForRead().find((e) => e.issueId === issueId);
+    return entry ? { position: entry.position, waitReason: entry.waitReason } : null;
+}
+/** The round-1 developer item an admitted start enqueued, for a start that raced a tick. */
+function developerWorkItemFor(issueId, instanceId) {
+    return (listWorkItemsForIssue(issueId).find((w) => w.workflowInstanceId === instanceId && w.kind === "developer") ?? null);
+}
+/**
+ * NOT-118 Start: make sure the issue is queued, move its entry to position 1, then admit
+ * synchronously so an idle system still starts immediately. There is no "start now, skip
+ * the queue" escape hatch — when no slot is free (or the issue isn't eligible yet) it waits
+ * at the top of the queue with a reason, and is the next one admitted.
+ *
+ * Every route/CLI/agent start path goes through here. The one ungated exception is a human
+ * action resuming a workflow (resolveHumanActionAndAdvance → startWorkflowCore), which may
+ * briefly exceed capacity by design (NOT-103 decision 5).
+ */
+export async function startIssueViaQueue(issueId) {
+    const issue = getIssue(issueId);
+    if (!issue)
+        return { state: "error", code: 404, error: "Issue not found" };
+    if (getActiveWorkflowInstance(issueId)) {
+        return { state: "error", code: 409, error: activeWorkflowConflictMessage(issueId) };
+    }
+    if (!startableStatuses.has(issue.status)) {
+        return { state: "error", code: 409, error: `Issue is ${issue.status} — not startable` };
+    }
+    try {
+        enqueueIssue(issueId); // idempotent when it is already queued
+    }
+    catch (err) {
+        // Only reachable if the issue turned terminal / started between the checks above and
+        // here; enqueueIssue tags those with an HTTP code.
+        const code = err.code ?? 409;
+        return { state: "error", code, error: err instanceof Error ? err.message : String(err) };
+    }
+    moveQueueEntryToTop(issueId);
+    const admitted = await admitNext();
+    if (admitted?.issueId === issueId) {
+        return { state: "admitted", instance: admitted.instance, workItem: admitted.workItem };
+    }
+    const queued = queueStatusForIssue(issueId);
+    if (queued)
+        return { state: "queued", ...queued };
+    // No longer queued and not admitted by this call — a concurrent coordinator tick admitted
+    // it between the enqueue and the walk, or an operator dequeued it mid-flight.
+    const instance = getActiveWorkflowInstance(issueId);
+    const workItem = instance ? developerWorkItemFor(issueId, instance.id) : null;
+    if (instance && workItem)
+        return { state: "admitted", instance, workItem };
+    return { state: "error", code: 409, error: "Issue left the queue before it could start" };
+}