agent-dealer 0.2.0 → 1.0.0

package/bundle/server/dist/coordinator/deck-outage.integration.test.js

@@ -0,0 +1,200 @@
+// packages/server/src/coordinator/deck-outage.integration.test.ts
+//
+// NOT-136 acceptance: an Agent Deck outage observed at preflight is a wait, not a failed
+// attempt. Four preflight failures in nine seconds used to exhaust the whole infra budget
+// and park a healthy round on a human; here the same outage must cost nothing and resolve
+// itself the moment the deck answers again.
+import { test, before, beforeEach, after } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-deckout-int-"));
+process.env.MAX_COORDINATOR_CONCURRENCY = "1";
+process.env.COORDINATOR_HEARTBEAT_MS = "20";
+process.env.COORDINATOR_FAIL_BACKOFF_MS = "0";
+process.env.COORDINATOR_POLL_INTERVAL_MS = "50";
+// Keep the backoff curve observable inside a test: 20ms, 40ms, 80ms, ...
+process.env.DECK_OUTAGE_BACKOFF_BASE_MS = "20";
+process.env.DECK_OUTAGE_BACKOFF_MAX_MS = "5000";
+const { migrate, getDb } = await import("../db/index.js");
+const { createAgent } = await import("../repository/agents.js");
+const { createIssue, getIssue } = await import("../repository/issues.js");
+const { listWorkItemsForIssue } = await import("../repository/work-items.js");
+const { listWorkflowEventsForIssue } = await import("../repository/workflow-events.js");
+const { listHumanActionsForIssue } = await import("../repository/human-actions.js");
+const { listWorkerSessionsForIssue } = await import("../repository/worker-sessions.js");
+const { startWorkflow } = await import("./commands.js");
+const { registerEffectHandler, resetEffectHandlers } = await import("./effect-registry.js");
+const { runCoordinatorTick, drainCoordinator } = await import("./worker-loop.js");
+const { deckOutageBackoffMs } = await import("./deck-outage-config.js");
+const UNREACHABLE = "Agent Deck is unreachable — fetch failed";
+before(() => migrate());
+beforeEach(() => {
+    getDb().exec(`
+    DELETE FROM review_publications;
+    DELETE FROM work_items;
+    DELETE FROM human_actions;
+    DELETE FROM workflow_events;
+    DELETE FROM findings;
+    DELETE FROM worker_sessions;
+    DELETE FROM artifacts;
+    DELETE FROM usage_events;
+    DELETE FROM workflow_instances;
+    DELETE FROM issues;
+    DELETE FROM runtime_availability;
+  `);
+});
+after(() => resetEffectHandlers());
+async function pump(max = 10) {
+    for (let i = 0; i < max; i++) {
+        await runCoordinatorTick();
+        await drainCoordinator();
+        await new Promise((r) => setTimeout(r, 30));
+    }
+}
+function makeIssue(name, maxInfraAttempts = 3) {
+    const repo = fs.mkdtempSync(path.join(os.tmpdir(), `dealer-deckout-${name}-`));
+    const dev = createAgent({ name: `dev-${name}`, runtime: "codex_local", deckId: "00000000-0000-4000-a000-000000000099" });
+    const rev = createAgent({ name: `rev-${name}`, runtime: "codex_local", deckId: "00000000-0000-4000-a000-000000000099" });
+    return createIssue({
+        title: `Deck outage ${name}`,
+        description: "d",
+        acceptanceCriteria: "ac",
+        repo,
+        baseBranch: "main",
+        developerAgentId: dev.id,
+        reviewerAgentId: rev.id,
+        maxReviewRounds: 2,
+        maxInfraAttempts,
+        source: "manual"
+    }).id;
+}
+test("a deck outage at preflight spends no infra attempts and never reaches needs_human", async () => {
+    const issueId = makeIssue("outage", 1);
+    let preflights = 0;
+    registerEffectHandler("developer", async () => {
+        preflights++;
+        return { kind: "deck_unavailable", reason: UNREACHABLE };
+    });
+    assert.equal(startWorkflow(issueId).ok, true);
+    await pump();
+    // The original incident: four attempts in nine seconds against a max of three.
+    assert.ok(preflights > 1, "the item must be retried, not dropped");
+    const issue = getIssue(issueId);
+    assert.equal(issue.infraAttempts, 0, "a deck outage must not spend the infra budget");
+    assert.notEqual(issue.status, "needs_human");
+    assert.match(issue.currentIntent ?? "", /Waiting for Agent Deck/);
+    const items = listWorkItemsForIssue(issueId);
+    assert.equal(items.length, 1);
+    assert.equal(items[0].status, "pending", "the item waits, it is not dead-lettered");
+    assert.equal(items[0].attemptCount, 0, "the claim-time attempt bump is reverted");
+    // Measured against when the deferral was written, not against "now": the backoff here is
+    // 20ms and the pump iterations that follow the last deferral can outlast it on a loaded
+    // runner, so a wall-clock comparison flakes without asserting anything extra.
+    assert.ok(Date.parse(items[0].availableAt) > Date.parse(items[0].updatedAt), "the next preflight is gated behind a wait");
+    assert.equal(listHumanActionsForIssue(issueId).filter((a) => a.status === "open").length, 0);
+    const events = listWorkflowEventsForIssue(issueId);
+    const deferrals = events.filter((e) => e.type === "worker.deferred");
+    assert.ok(deferrals.length > 0);
+    const payload = JSON.parse(deferrals[0].payloadJson ?? "{}");
+    assert.equal(payload.outcome, "deck_unavailable", "the timeline says the deck is unavailable");
+    assert.match(payload.reason, /unreachable/);
+    assert.equal(events.some((e) => e.type === "worker.failed"), false, "no worker failed here");
+    // Nothing spawned, so the session is cancelled rather than recorded as a crash.
+    assert.equal(listWorkerSessionsForIssue(issueId).every((s) => s.status === "cancelled"), true);
+});
+test("the issue proceeds on its own once the deck is reachable again", async () => {
+    const issueId = makeIssue("recover", 1);
+    let attempts = 0;
+    registerEffectHandler("developer", async () => {
+        attempts++;
+        // The deck comes back between the first and second preflight — as it would after a
+        // restart. No human action, no resume command.
+        if (attempts === 1) {
+            return { kind: "deck_unavailable", reason: UNREACHABLE };
+        }
+        return { kind: "no_pr" };
+    });
+    assert.equal(startWorkflow(issueId).ok, true);
+    await pump();
+    assert.ok(attempts >= 2, "the deferred item must be re-attempted without human input");
+    const issue = getIssue(issueId);
+    // The recovered attempt consumed exactly one infra attempt — the no_pr one, not the outage.
+    assert.equal(issue.infraAttempts, 1);
+});
+test("consecutive outages back off exponentially instead of retrying every ~3s", async () => {
+    const issueId = makeIssue("backoff");
+    registerEffectHandler("developer", async () => {
+        return { kind: "deck_unavailable", reason: UNREACHABLE };
+    });
+    assert.equal(startWorkflow(issueId).ok, true);
+    const waits = [];
+    for (let i = 0; i < 4; i++) {
+        await runCoordinatorTick();
+        await drainCoordinator();
+        const item = listWorkItemsForIssue(issueId)[0];
+        const deferrals = JSON.parse(item.payloadJson ?? "{}").deckUnavailableDeferrals;
+        waits.push(deferrals);
+        // Wait out this deferral so the next tick can claim the item again.
+        await new Promise((r) => setTimeout(r, Math.max(0, Date.parse(item.availableAt) - Date.now()) + 5));
+    }
+    assert.deepEqual(waits, [1, 2, 3, 4], "each outage increments this item's deferral count");
+    assert.ok(deckOutageBackoffMs(3) > deckOutageBackoffMs(0), "later deferrals wait strictly longer than the first");
+});
+test("a prolonged outage keeps waiting — it says how long, and still recovers on its own", async () => {
+    const prev = process.env.DECK_OUTAGE_PROLONGED_AFTER_MS;
+    // Long enough that the wait stops reading as "the deck is restarting".
+    process.env.DECK_OUTAGE_PROLONGED_AFTER_MS = "40";
+    try {
+        const issueId = makeIssue("prolonged");
+        let down = true;
+        let ranAfterRecovery = 0;
+        registerEffectHandler("developer", async () => {
+            if (down)
+                return { kind: "deck_unavailable", reason: UNREACHABLE };
+            ranAfterRecovery++;
+            return { kind: "no_pr" };
+        });
+        assert.equal(startWorkflow(issueId).ok, true);
+        await pump(1); // first observation — starts the outage clock
+        await new Promise((r) => setTimeout(r, 60)); // outlive the "prolonged" threshold
+        await pump();
+        const waiting = getIssue(issueId);
+        assert.notEqual(waiting.status, "needs_human", "a long outage is still a wait, not a handoff");
+        assert.equal(waiting.infraAttempts, 0);
+        assert.match(waiting.currentIntent ?? "", /Waiting for Agent Deck/);
+        assert.match(waiting.currentIntent ?? "", /unreachable for \d+s/, "the timeline names the outage length");
+        assert.equal(listHumanActionsForIssue(issueId).filter((a) => a.status === "open").length, 0);
+        assert.equal(listWorkItemsForIssue(issueId)[0].status, "pending", "the item is still retryable");
+        // The whole point of not escalating: whenever the deck returns, the issue moves again.
+        down = false;
+        const pending = listWorkItemsForIssue(issueId)[0];
+        await new Promise((r) => setTimeout(r, Math.max(0, Date.parse(pending.availableAt) - Date.now()) + 5));
+        await pump();
+        assert.ok(ranAfterRecovery > 0, "the deferred item resumed on its own once the deck answered");
+    }
+    finally {
+        if (prev === undefined)
+            delete process.env.DECK_OUTAGE_PROLONGED_AFTER_MS;
+        else
+            process.env.DECK_OUTAGE_PROLONGED_AFTER_MS = prev;
+    }
+});
+test("a deck that is reachable but answers with an error still fails the attempt", async () => {
+    const issueId = makeIssue("realerror", 1);
+    registerEffectHandler("developer", async () => {
+        return {
+            kind: "deck_failure",
+            reason: "preflight failed: get_playbook(pb-x) returned an error: missing"
+        };
+    });
+    assert.equal(startWorkflow(issueId).ok, true);
+    await pump();
+    const issue = getIssue(issueId);
+    assert.equal(issue.infraAttempts, 1, "a real deck error still spends the infra budget");
+    assert.equal(issue.status, "needs_human");
+    const events = listWorkflowEventsForIssue(issueId);
+    assert.ok(events.some((e) => e.type === "worker.failed"));
+    assert.equal(events.some((e) => e.type === "worker.deferred"), false);
+});

package/bundle/server/dist/coordinator/dependencies.js

@@ -0,0 +1,242 @@
+// packages/server/src/coordinator/dependencies.ts
+//
+// NOT-104: dependency readiness from *declared* Linear blockers.
+//
+// Dealer never infers a dependency graph — no file overlap, no same-repo ordering. The only
+// edge that gates admission is an explicit `blocks` relation a human wrote in Linear. This
+// module owns the three pieces the `blockedByDependency` rule needs:
+//
+//   1. a batched, TTL-cached, injectable provider that turns queued issues into a snapshot
+//      of their declared blockers (`blockerSnapshotFor`),
+//   2. the per-blocker satisfaction rule (dealer `done` beats Linear's "Done"),
+//   3. the operator-facing wait reason.
+//
+// There is no `issue_dependencies` table: a dealer-linked blocker is a pure local join and is
+// always live, so only the edge itself and the Linear state of *unlinked* blockers are remote.
+// A persisted table can replace the provider later without touching the rule.
+import { isTerminalIssueStatus } from "@agent-dealer/shared";
+import { fetchLinearBlockers } from "../adapters/linear-inbox.js";
+import { listIssuesByExternalId } from "../repository/issues.js";
+const EMPTY_SNAPSHOT = new Map();
+const DEFAULT_CACHE_TTL_MS = 60_000;
+/**
+ * Admission runs inside the ~3s coordinator tick and this is the first remote call in that
+ * path, so it is hard-bounded: a hanging Linear API degrades to "unavailable" (everything
+ * parks, nothing is dropped) instead of stalling every worker dispatch behind it. The budget
+ * plus both graces below stays under that tick — a healthy Linear answers in well under a
+ * second, and one slow read must not cost the loop a whole cycle.
+ */
+const DEFAULT_FETCH_TIMEOUT_MS = 1_500;
+/** Grace on top of the per-request budget, so the request's own error wins over the wrapper. */
+const TIMEOUT_GRACE_MS = 250;
+/**
+ * After a failed fetch, wait this long before asking Linear again. Only *successes* are
+ * cached, so without this the 3s coordinator tick retries an outage ~1200 times an hour —
+ * against a 1500/hr personal-key limit that the inbox poller and status write-back share.
+ * Entries stay parked either way; recovery is just delayed by at most this window.
+ */
+const FAILURE_BACKOFF_MS = 10_000;
+let cacheTtlMs = DEFAULT_CACHE_TTL_MS;
+let fetchTimeoutMs = DEFAULT_FETCH_TIMEOUT_MS;
+let failureBackoffMs = FAILURE_BACKOFF_MS;
+let blockersProvider = null;
+let linearFetcher = null;
+const cache = new Map();
+let backoffUntil = 0;
+let inFlight = null;
+let lastLoggedError = null;
+/** Tests inject a pure provider so admission never reaches the network. */
+export function setBlockersProviderForTests(provider) {
+    blockersProvider = provider;
+}
+/** Narrower seam: keeps the real caching/TTL policy, replaces only the Linear call. */
+export function setLinearBlockerFetcherForTests(fetcher) {
+    linearFetcher = fetcher;
+}
+export function setBlockerCacheTtlForTests(ms) {
+    cacheTtlMs = ms ?? DEFAULT_CACHE_TTL_MS;
+}
+export function setBlockerFetchTimeoutForTests(ms) {
+    fetchTimeoutMs = ms ?? DEFAULT_FETCH_TIMEOUT_MS;
+}
+/** Also re-scales a window already running, so a test can lapse one without sleeping it out. */
+export function setBlockerFailureBackoffForTests(ms) {
+    failureBackoffMs = ms ?? FAILURE_BACKOFF_MS;
+    backoffUntil = Math.min(backoffUntil, Date.now() + failureBackoffMs);
+}
+export function clearBlockerCacheForTests() {
+    cache.clear();
+    backoffUntil = 0;
+    inFlight = null;
+    lastLoggedError = null;
+}
+export function resetDependenciesForTests() {
+    blockersProvider = null;
+    linearFetcher = null;
+    cacheTtlMs = DEFAULT_CACHE_TTL_MS;
+    fetchTimeoutMs = DEFAULT_FETCH_TIMEOUT_MS;
+    failureBackoffMs = FAILURE_BACKOFF_MS;
+    clearBlockerCacheForTests();
+}
+/** Only a Linear-sourced issue carries declared dependencies; manual issues are a no-op. */
+export function isDependencyTracked(issue) {
+    return issue.source === "linear" && Boolean(issue.externalId);
+}
+/**
+ * The default provider: one batched Linear query per `admitNext()` that finds a free slot,
+ * memoized per issue id for `cacheTtlMs`.
+ *
+ * **No stale-while-error.** An entry past its TTL is dropped rather than reused when the
+ * refresh fails — a dependency declared *during* a Linear outage would otherwise be
+ * invisible for as long as the outage lasts, which is exactly the doomed run this prevents.
+ */
+export async function blockersFor(issues) {
+    const ids = [...new Set(issues.filter(isDependencyTracked).map((i) => i.externalId))];
+    const snapshot = new Map();
+    const now = Date.now();
+    const stale = [];
+    // Expired rows are never served, so drop them outright — otherwise every Linear issue ever
+    // queued stays in the map for the life of the process.
+    for (const [id, hit] of cache)
+        if (now - hit.at >= cacheTtlMs)
+            cache.delete(id);
+    for (const id of ids) {
+        const hit = cache.get(id);
+        if (hit)
+            snapshot.set(id, hit.blockers);
+        else
+            stale.push(id);
+    }
+    if (stale.length === 0)
+        return snapshot;
+    if (now < backoffUntil)
+        throw new Error("blocker fetch backing off after a recent failure");
+    // Ticks overlap (the 3s loop plus a Manual Start): the second one parks rather than opening
+    // a second connection to an API that is, by definition, slower than a tick right now.
+    if (inFlight)
+        throw new Error("blocker fetch already in flight");
+    // Throwing (rather than falling back to the expired rows) is what makes the TTL a hard
+    // edge: the caller turns a throw into "unavailable", never into "no blockers".
+    const pending = withTimeout((linearFetcher ?? defaultLinearFetcher)(stale), fetchTimeoutMs + TIMEOUT_GRACE_MS);
+    inFlight = pending;
+    let fetched;
+    try {
+        fetched = await pending;
+    }
+    catch (err) {
+        backoffUntil = Date.now() + failureBackoffMs;
+        throw err;
+    }
+    finally {
+        if (inFlight === pending)
+            inFlight = null;
+    }
+    backoffUntil = 0;
+    lastLoggedError = null; // a later recurrence is news again
+    const fetchedAt = Date.now();
+    for (const id of stale) {
+        const blockers = fetched.get(id);
+        // An id Linear did not return is unknowable, not unblocked: leave it out of the
+        // snapshot (→ unavailable) and out of the cache so the next tick retries it.
+        if (!blockers) {
+            cache.delete(id);
+            continue;
+        }
+        cache.set(id, { at: fetchedAt, blockers });
+        snapshot.set(id, blockers);
+    }
+    return snapshot;
+}
+function defaultLinearFetcher(issueIds) {
+    return fetchLinearBlockers(issueIds, { timeoutMs: fetchTimeoutMs });
+}
+function withTimeout(promise, ms) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`blocker fetch timed out after ${ms}ms`)), ms);
+        promise.then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, (err) => {
+            clearTimeout(timer);
+            reject(err instanceof Error ? err : new Error(String(err)));
+        });
+    });
+}
+/**
+ * What `admitNext()` calls: the active provider, hard-bounded in time and guaranteed not to
+ * throw into the coordinator tick. Zero Linear traffic when nothing Linear-sourced is queued
+ * (and none at all on a tick with no free slot — `admitNext` returns before reaching here).
+ */
+export async function blockerSnapshotFor(issues) {
+    const tracked = issues.filter(isDependencyTracked);
+    if (tracked.length === 0)
+        return EMPTY_SNAPSHOT;
+    try {
+        // Backstop over the batch's own bound, so an injected provider cannot stall the tick either.
+        return await withTimeout((blockersProvider ?? blockersFor)(tracked), fetchTimeoutMs + 2 * TIMEOUT_GRACE_MS);
+    }
+    catch (err) {
+        // The operator only ever sees the one constant reason, so the *cause* — missing API key,
+        // an id Linear does not know, HTTP 500, timeout — has to reach the log or a permanent
+        // misconfiguration is indistinguishable from a passing blip. Logged on change only, the
+        // same discipline wait_reason uses: an outage is one line, not one per 3s tick.
+        const message = err instanceof Error ? err.message : String(err);
+        if (message !== lastLoggedError) {
+            lastLoggedError = message;
+            console.error("[coordinator] blocker fetch", err);
+        }
+        // Fail closed *and* park: an empty snapshot marks every Linear-sourced entry unavailable,
+        // which writes a wait reason and leaves the entry `queued`. Nothing is dequeued, failed
+        // or canceled, and the next successful fetch admits it with no operator action.
+        return EMPTY_SNAPSHOT;
+    }
+}
+/** Linear state types that mean "this work will never land in a form we must wait for". */
+const SATISFIED_STATE_TYPES = new Set(["completed", "canceled"]);
+/**
+ * Decision 2. Resolve the blocker against `issues.external_id`:
+ *
+ * - **dealer-linked** → satisfied only once the dealer issue is `done`, i.e. the PR actually
+ *   merged. Linear status alone is not enough; issues get marked Done at PR-approval time,
+ *   before the code lands, which is precisely the doomed run this rule prevents. A ticket may
+ *   hold several dealer passes (NOT-141); the live one decides, terminal ones only when none
+ *   is live.
+ * - **unlinked** → satisfied when the Linear state type is `completed` or `canceled`.
+ *
+ * Once a blocker resolves to a dealer issue, dealer status is the *only* answer — a Linear
+ * `canceled` on a ticket dealer is still developing would admit the dependent onto a base
+ * without that work. Canceling remains decision 6's escape hatch for a blocker dealer never
+ * picked up; for one it did, the fix is to drop the `blocks` relation.
+ */
+export function blockerVerdict(blocker) {
+    // NOT-141: a ticket may have several dealer passes (newest first) once a terminal one no
+    // longer blocks a re-import. The *live* pass is authoritative whenever there is one: a
+    // regression or follow-up pass reopened on a merged ticket is work the dependent must
+    // still wait for, exactly as for a first pass. Only when every pass has ended do the
+    // historical ones answer — and a merged one stays merged, so an abandoned follow-up
+    // (`closed`, never landed) cannot un-satisfy a blocker whose code is already in.
+    const dealerPasses = blocker.id ? listIssuesByExternalId("linear", blocker.id) : [];
+    if (dealerPasses.length > 0) {
+        const active = dealerPasses.find((issue) => !isTerminalIssueStatus(issue.status));
+        // The dealer status is the honest answer to "why am I still waiting" — a blocker
+        // sitting at `waiting on NOT-123 (Done)` reads like a bug. An active pass is never
+        // `done`, so naming it is always naming the pass that still has to land.
+        if (active)
+            return { satisfied: false, state: active.status };
+        const merged = dealerPasses.find((issue) => issue.status === "done");
+        return { satisfied: merged != null, state: (merged ?? dealerPasses[0]).status };
+    }
+    return { satisfied: SATISFIED_STATE_TYPES.has(blocker.stateType), state: blocker.stateName };
+}
+/** `waiting on NOT-123 (In Progress)` — names every unsatisfied blocker and its state. */
+export function unsatisfiedBlockerReason(blockers) {
+    const unsatisfied = blockers
+        .map((b) => ({ blocker: b, verdict: blockerVerdict(b) }))
+        .filter((x) => !x.verdict.satisfied);
+    if (unsatisfied.length === 0)
+        return null;
+    return `waiting on ${unsatisfied
+        .map(({ blocker, verdict }) => `${blocker.identifier} (${verdict.state})`)
+        .join(", ")}`;
+}
+export const DEPENDENCY_STATE_UNAVAILABLE = "dependency state unavailable";