@agent-relay/factory 0.1.16 → 0.1.18

package/dist/orchestrator/factory.js

@@ -15,7 +15,7 @@ import { GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlack
 import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared.js';
 import { issueKey } from './batch-tracker.js';
 import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
-import { terminatePids } from './reaper.js';
+import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
 const ISSUE_ROOT = '/linear/issues';
 const GITHUB_ISSUE_ROOT = '/github/repos';
 const READY_EVENTS_LIMIT = 100;
@@ -315,6 +315,7 @@ export class FactoryLoop {
             return;
         }
         this.#wireFleetEvents();
+        await this.#adoptInFlightAgents();
         if ((opts.mode ?? 'live') === 'live') {
             this.#started = true;
             try {
@@ -1384,6 +1385,28 @@ export class FactoryLoop {
             });
         }
     }
+    // Remote backends survive orchestrator restarts: re-adopt the agents recorded
+    // in the in-flight registry, then reconcile once so exits that happened while
+    // this process was down are handled before any new dispatch.
+    async #adoptInFlightAgents() {
+        if (!this.#fleet.hydrateTracked)
+            return;
+        try {
+            const registry = await readFactoryInFlightRegistry(this.#config.loop.registryPath);
+            const agents = (registry?.agents ?? []).filter((agent) => agent.invocationId || agent.node);
+            if (agents.length > 0) {
+                this.#fleet.hydrateTracked(agents.map((agent) => ({
+                    name: agent.name,
+                    invocationId: agent.invocationId,
+                    node: agent.node,
+                })));
+            }
+            await this.#fleet.reconcileTrackedAgents?.();
+        }
+        catch (error) {
+            this.#logger.warn?.('[factory] failed to re-adopt in-flight agents from the registry', { error });
+        }
+    }
     async #backfillReadyIssues() {
         const page = await this.#mount.getEvents({ limit: READY_EVENTS_LIMIT });
         const allPaths = page.events.map((event) => changeEventPath(event)).filter((p) => Boolean(p));
@@ -2106,6 +2129,7 @@ export class FactoryLoop {
                     processes.push({ ...identity, agentName });
                 }
             }
+            const fleetTracked = this.#fleet.trackedAgents?.().get(agentName);
             agents.push({
                 name: agentName,
                 role: tracked.spec.role,
@@ -2113,6 +2137,8 @@ export class FactoryLoop {
                 sessionRef: tracked.sessionRef,
                 pids,
                 processes,
+                ...(fleetTracked?.invocationId ? { invocationId: fleetTracked.invocationId } : {}),
+                ...(fleetTracked?.node ? { node: fleetTracked.node } : {}),
             });
         };
         if (!empty) {
@@ -2245,14 +2271,40 @@ export class FactoryLoop {
                 await this.#completeIssue(record);
                 return;
             }
+            // The implementer's turn ended without a PR of record. Agents reliably
+            // COMMIT their work to a feature branch but often exit (turn-end / idle)
+            // before running `gh pr create` — the root reason a dispatch never reached
+            // human-review even after the #67 fixes. Rather than respawn a done agent
+            // and hope, factory finalizes the branch into a PR itself (the same
+            // publish path the completion flow uses), then advances to the babysitter
+            // / human-review path. Best-effort: with no publishable branch (no commits
+            // ahead of base, clone gone) it returns undefined and we fall through.
+            if (tracked.spec.role === 'implementer') {
+                const publishedPr = await this.#tryPublishImplementerPr(record, tracked);
+                if (publishedPr) {
+                    if (this.#config.babysitter.enabled) {
+                        await this.#ensureBabysitter(record, {
+                            repo: publishedPr.repo,
+                            prNumber: publishedPr.number,
+                            url: publishedPr.url,
+                        });
+                    }
+                    else {
+                        await this.#completeIssue(record);
+                    }
+                    return;
+                }
+            }
             if (tracked.sessionRef) {
                 const resumeKey = `${issueKey(record.issue)}:${name}:${tracked.sessionRef}`;
                 if (await this.#state.isResumed(this.#workspaceId, resumeKey)) {
                     // Already resumed once and STILL exiting with no completion PR — the
-                    // agent isn't making progress. Escalate so a human notices, instead of
-                    // leaving the issue silently in-flight forever.
+                    // agent isn't making progress. Conclude the dispatch so a human
+                    // notices AND the reviewer waiting on the implementer's DM is torn
+                    // down, instead of leaving the issue silently in-flight forever with
+                    // a live reviewer stalling the owned-broker dispose-wait (#67).
                     if (tracked.spec.role === 'implementer') {
-                        await this.#escalateStalledIssue(record, name);
+                        await this.#concludeTerminalImplementer(record, name, 'stalled-no-pr');
                     }
                     return;
                 }
@@ -2271,17 +2323,24 @@ export class FactoryLoop {
                     if (isAgentAlreadyExistsError(error)) {
                         // The broker never released this agent's name on exit
                         // (relay#1116-family), so re-registering collides with the stuck
-                        // name. The error is marked retryable but isn't — retrying just
-                        // re-collides forever. Treat it as terminal for this name: record
-                        // the resume key so subsequent exit events short-circuit, count it,
-                        // and warn once instead of spamming a 500 stack trace. The external
-                        // reaper / a broker restart reclaims the leaked name.
+                        // name. Retrying just re-collides forever. Treat it as terminal for
+                        // this name: record the resume key so subsequent exit events
+                        // short-circuit, count it, and warn once. The external reaper / a
+                        // broker restart reclaims the leaked name.
+                        this.#fleet.markAgentTerminal?.(name, 'resume-already-exists');
                         await this.#state.markResumed(this.#workspaceId, resumeKey);
                         this.#increment('resumeNameCollisions');
                         this.#logger.warn?.('[factory] resume skipped: broker still holds agent name (relay#1116); not retrying', {
                             issue: record.issue.key,
                             name,
                         });
+                        // The implementer is now terminal but never sent the completion DM
+                        // the reviewer is blocked on ("Wait for a DM from the
+                        // implementer(s)"). Resolve the dispatch so the reviewer does not
+                        // stay live forever and stall the owned-broker dispose-wait (#67).
+                        if (tracked.spec.role === 'implementer') {
+                            await this.#concludeTerminalImplementer(record, name, 'resume-already-exists');
+                        }
                     }
                     else {
                         throw error;
@@ -2293,25 +2352,78 @@ export class FactoryLoop {
             }
             else {
                 const invocationId = `${batch.invocationIdFor(record.issue, tracked.spec)}:restart:${this.#clock.now()}`;
-                const result = await this.#fleet.spawn({
-                    name: tracked.spec.name,
-                    capability: tracked.spec.capability,
-                    node: tracked.spec.node ?? 'self',
-                    task: tracked.spec.task,
-                    model: tracked.spec.model,
-                    cwd: tracked.spec.clonePath,
-                    sessionRef: tracked.spec.sessionRef,
-                    invocationId,
-                    restartPolicy: defaultRestartPolicy(tracked.spec),
-                    channel: tracked.spec.channel,
-                });
-                batch.recordSpawn(record, tracked.spec, invocationId, result);
+                try {
+                    const result = await this.#fleet.spawn({
+                        name: tracked.spec.name,
+                        capability: tracked.spec.capability,
+                        node: tracked.spec.node ?? 'self',
+                        task: tracked.spec.task,
+                        model: tracked.spec.model,
+                        cwd: tracked.spec.clonePath,
+                        sessionRef: tracked.spec.sessionRef,
+                        invocationId,
+                        restartPolicy: defaultRestartPolicy(tracked.spec),
+                        channel: tracked.spec.channel,
+                    });
+                    batch.recordSpawn(record, tracked.spec, invocationId, result);
+                }
+                catch (error) {
+                    if (!isAgentAlreadyExistsError(error)) {
+                        throw error;
+                    }
+                    // Same leaked-broker-name collision as the resume path, on a
+                    // no-sessionRef respawn: the broker's own restartPolicy already
+                    // re-registered the name (relay#1116-family), so our respawn collides.
+                    // Retrying just re-collides forever. Treat it as terminal and resolve
+                    // the dispatch so a reviewer blocked on the implementer's DM does not
+                    // hang the owned-broker dispose-wait (#67).
+                    this.#fleet.markAgentTerminal?.(name, 'respawn-already-exists');
+                    this.#increment('resumeNameCollisions');
+                    this.#logger.warn?.('[factory] respawn skipped: broker still holds agent name (relay#1116); not retrying', {
+                        issue: record.issue.key,
+                        name,
+                    });
+                    if (tracked.spec.role === 'implementer') {
+                        await this.#concludeTerminalImplementer(record, name, 'respawn-already-exists');
+                    }
+                }
             }
         }
         catch (error) {
             this.#error(error, record.issue);
         }
     }
+    // Publish a PR from the implementer's committed branch when it exited without
+    // opening one. Best-effort and idempotent: returns undefined when there is no
+    // GitHub write path, no clone, or nothing publishable (no branch / no commits
+    // ahead of base — `#publishImplementerPullRequest` refuses head==base), so the
+    // caller falls back to its normal restart/conclude handling.
+    async #tryPublishImplementerPr(record, implementer) {
+        if (record.dryRun || !implementer.spec.clonePath || !this.#mount.githubWrite) {
+            return undefined;
+        }
+        try {
+            const published = await this.#publishImplementerPullRequest(record, implementer);
+            if (published) {
+                this.#increment('implementerPrsPublishedOnExit');
+                this.#logger.info?.('[factory] published PR from implementer clone after it exited without opening one', {
+                    issue: record.issue.key,
+                    repo: published.repo,
+                    prNumber: published.number,
+                });
+            }
+            return published;
+        }
+        catch (error) {
+            this.#increment('exitPrPublishSkipped');
+            this.#logger.warn?.('[factory] could not publish implementer PR on exit; falling back', {
+                issue: record.issue.key,
+                name: implementer.result?.name ?? implementer.spec.name,
+                error: describeError(error).errorMessage,
+            });
+            return undefined;
+        }
+    }
     async #publishImplementerPullRequest(record, implementer) {
         const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
         if (this.#publishedPullRequests.has(key))
@@ -2396,6 +2508,75 @@ export class FactoryLoop {
             this.#logger.warn?.('[factory] failed to post stalled-issue escalation to Slack', error);
         }
     }
+    // An implementer that collided on its leaked broker name, or that exited a
+    // second time with no PR, is terminal: it will never send the completion DM
+    // the reviewer is blocked on (its prompt is literally "Wait for a DM from the
+    // implementer(s)"). Left alone, the reviewer stays live forever and the
+    // owned-broker dispose-wait stalls until FACTORY_AGENT_EXIT_TIMEOUT_MS
+    // (default 30 min), so the issue never reaches human-review (#67). Signal any
+    // waiting reviewer(s) so they can proceed, then resolve the dispatch
+    // deterministically: complete to the terminal state when a PR is now visible
+    // (mount lag at exit time is why we fell through to resume in the first
+    // place), otherwise escalate and tear down the stuck agents so dispose drains.
+    async #concludeTerminalImplementer(record, implementerName, reason) {
+        await this.#signalReviewersImplementerDone(record, implementerName, reason);
+        if (await this.#issueHasCompletionPr(record)) {
+            await this.#completeIssue(record);
+            return;
+        }
+        await this.#escalateStalledIssue(record, implementerName);
+        await this.#abandonStuckDispatch(record, reason);
+    }
+    // Deliver a synthetic "implementer done" DM so a reviewer blocked on the
+    // implementer's message unblocks and proceeds (review the PR if one is open,
+    // otherwise conclude). Best-effort: the deterministic teardown below is the
+    // backstop when the reviewer cannot act on it.
+    async #signalReviewersImplementerDone(record, implementerName, reason) {
+        const reviewers = new Set([...record.agents.values()]
+            .filter((agent) => agent.spec.role === 'reviewer')
+            .map((agent) => agent.result?.name ?? agent.spec.name)
+            .filter((name) => Boolean(name)));
+        for (const reviewer of reviewers) {
+            try {
+                await this.#fleet.sendMessage({
+                    to: reviewer,
+                    from: implementerName,
+                    text: `[factory] implementer ${implementerName} has terminated (${reason}) and will send no further messages. If a pull request is open for this issue, review it now; otherwise conclude your review and DM \`broker\` that you are done.`,
+                });
+                this.#increment('reviewerImplementerTerminatedSignals');
+            }
+            catch (error) {
+                this.#logger.warn?.('[factory] failed to signal reviewer of implementer termination', {
+                    issue: record.issue.key,
+                    reviewer,
+                    error,
+                });
+            }
+        }
+    }
+    // Tear down agents still live after a terminal implementer produced no PR —
+    // chiefly the reviewer, blocked on a DM that will never arrive — and free the
+    // batch slot, so the owned-broker dispose-wait drains instead of stalling for
+    // the full agent-exit timeout. Marking each agent terminal first suppresses
+    // the release-driven exit event so it cannot re-trigger a resume before the
+    // record leaves the batch.
+    async #abandonStuckDispatch(record, reason) {
+        const remaining = [...record.agents].filter(([, tracked]) => tracked.spec.role !== 'implementer');
+        for (const [agentName] of remaining) {
+            this.#fleet.markAgentTerminal?.(agentName, `implementer-terminal:${reason}`);
+        }
+        if (remaining.length > 0) {
+            await this.#releaseAndTerminateAgents(remaining, 'issue-abandoned', 'completion');
+        }
+        await this.#recordDispatchTerminal(record.issue);
+        const next = (await this.#batch()).complete(record.issue);
+        await this.#stopSlackWatcher(record.issue);
+        await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
+        await this.#writeInFlightRegistry();
+        if (next) {
+            await this.dispatch(next.decision, { dryRun: next.dryRun });
+        }
+    }
     async #issueHasCompletionPr(record) {
         try {
             const issue = await this.#readIssue(record.issue.path);