@agent-relay/factory 0.1.57 → 0.1.58

package/dist/orchestrator/factory.js

@@ -132,6 +132,44 @@ const STALE_LOCAL_AGENT_RECLAIM_MAX_ATTEMPTS = 3;
 const STALE_LOCAL_AGENT_RECLAIM_BACKOFF_MS = 500;
 export const DEFAULT_FACTORY_LOOP_HEARTBEAT_PATH = '/tmp/factory-run/factory-loop-heartbeat.json';
 export const DEFAULT_FACTORY_LOOP_REGISTRY_PATH = '/tmp/factory-run/factory-loop-registry.json';
+// #githubApiFallbackIssues is append-only in memory for the life of the
+// process; bound it so a long-running `factory start --mode live` daemon
+// doesn't accumulate one entry per fallback dispatch forever.
+const GITHUB_API_FALLBACK_ISSUES_MAX = 2_000;
+// A small, separately-bounded record of recently evicted identities, so a
+// later miss can be reported as "eligibility was evicted" rather than
+// silently folded into the same signal as "never was eligible" — an
+// operator investigating a false phantom-skip needs to tell those apart.
+const GITHUB_API_FALLBACK_ISSUES_EVICTED_MAX = 200;
+/**
+ * Adds `identity` to `eligible`, evicting the least-recently-registered
+ * entry once at capacity (Sets preserve insertion order; delete+add
+ * refreshes an existing identity's recency instead of leaving it at its
+ * original position). An evicted identity moves into `evicted` — itself
+ * bounded — so a later caller can tell "eligibility was evicted" apart from
+ * "never was eligible" instead of collapsing both into the same signal.
+ * Exported standalone (pure, no class state) so the bounding and eviction
+ * behavior is directly testable without driving thousands of dispatches
+ * through a full FactoryLoop to fill the production-sized set.
+ */
+export function rememberBoundedFallbackEligibility(eligible, evicted, identity, maxEligible, maxEvicted) {
+    evicted.delete(identity);
+    eligible.delete(identity);
+    eligible.add(identity);
+    if (eligible.size <= maxEligible)
+        return;
+    const oldest = eligible.values().next().value;
+    if (oldest === undefined)
+        return;
+    eligible.delete(oldest);
+    evicted.delete(oldest);
+    evicted.add(oldest);
+    if (evicted.size <= maxEvicted)
+        return;
+    const oldestEvicted = evicted.values().next().value;
+    if (oldestEvicted !== undefined)
+        evicted.delete(oldestEvicted);
+}
 class DispatchLifecycleCapacityError extends Error {
 }
 class DispatchLifecycleOwnedElsewhereError extends Error {
@@ -199,6 +237,8 @@ export class FactoryLoop {
     #githubIssueAuthors = new Map();
     #githubIssueAuthorLookups = new Map();
     #githubIssuePreferredPaths = new Map();
+    #githubApiFallbackIssues = new Set();
+    #githubApiFallbackIssuesEvicted = new Set();
     #githubIssuePathIndexReady = false;
     #slackReporterUserIds = new Map();
     #slackReporterUserIdLookups = new Map();
@@ -251,6 +291,10 @@ export class FactoryLoop {
     #subscription;
     #livePollTimer;
     #livePollInFlight = false;
+    // The effective transport includes per-start overrides, which can differ
+    // from config.liveSubscription. Persist it so status/heartbeat describes
+    // the listener that was actually registered.
+    #liveTransport;
     #liveEventCursor;
     #liveEventHighWatermark;
     #liveConnectStartedAtMs = 0;
@@ -622,6 +666,11 @@ export class FactoryLoop {
             this.#schedulePreviewSweep();
             try {
                 await this.#startLiveSubscription(issueSource, opts.liveSubscription);
+                // The initial live heartbeat is intentionally written before startup
+                // so crash reapers can see a daemon while it bootstraps. Write again
+                // once the listener is actually registered so external `factory
+                // status` can distinguish a quiet feed from no listener.
+                await this.#writeLiveHeartbeat('running');
                 await this.#rearmSlackReplyWatchers();
                 await this.#drainReadyClarificationWake();
                 await this.#rearmGithubIssueCommentWatchers();
@@ -825,6 +874,7 @@ export class FactoryLoop {
     }
     async #startLiveSubscription(issueSource, overrides = {}) {
         const options = this.#liveOptions(overrides);
+        this.#liveTransport = options.transport;
         this.#liveConnectStartedAtMs = this.#clock.now();
         this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
         const highWatermark = await this.#currentEventHighWatermark();
@@ -2067,7 +2117,10 @@ export class FactoryLoop {
         const dispatched = this.#dispatchUnlocked(decision, opts);
         this.#dispatchInFlight.set(key, dispatched);
         try {
-            return await dispatched;
+            const result = await dispatched;
+            if (decision.issueResolution)
+                result.issueResolution = structuredClone(decision.issueResolution);
+            return result;
         }
         finally {
             if (this.#dispatchInFlight.get(key) === dispatched) {
@@ -2101,7 +2154,11 @@ export class FactoryLoop {
             this.#error(error, decision.issue);
             throw error;
         }
-        const liveIssue = await this.#readIssue(decision.issue.path);
+        // This read runs before `decision` has any tracked record for
+        // #deriveGithubApiFallbackEligibility to find (batch insertion happens
+        // later, once scope/readiness are validated), so pass it directly
+        // rather than registering it into the cache first.
+        const liveIssue = await this.#readIssue(decision.issue.path, decision);
         if (!liveIssue || !isInFactoryScope(liveIssue, this.#config.safety)) {
             const error = new Error(`Refusing to dispatch ${decision.issue.key}: not factory-e2e scope`);
             this.#error(error, decision.issue);
@@ -2456,8 +2513,27 @@ export class FactoryLoop {
             counters: { ...this.#counters },
             slackDegraded: this.#slackDegraded,
             slackDegradedReason: this.#slackDegradedReason,
+            eventListener: this.#eventListenerStatus(),
         };
     }
+    #eventListenerStatus() {
+        if (this.#startMode !== 'live') {
+            return {
+                state: 'not-listening',
+                reason: this.#startMode ? `factory mode is ${this.#startMode}` : 'factory has not started',
+            };
+        }
+        if (!this.#liveHeartbeatActive) {
+            return { state: 'not-listening', reason: 'live daemon heartbeat is inactive' };
+        }
+        if (this.#subscription) {
+            return { state: 'subscribed' };
+        }
+        if (this.#liveTransport === 'poll') {
+            return { state: 'polling' };
+        }
+        return { state: 'starting' };
+    }
     on(event, listener) {
         let listeners = this.#listeners.get(event);
         if (!listeners) {
@@ -3524,7 +3600,11 @@ export class FactoryLoop {
     async #resumeDurableDispatch(record) {
         let liveIssue;
         if (!record.dryRun) {
-            liveIssue = await this.#readIssue(record.issue.path);
+            // record is already inserted into the batch by the time any phase
+            // handler runs (BatchTracker.restore, called before this), so
+            // #deriveGithubApiFallbackEligibility would find it there too — the
+            // explicit hint is defense in depth, not a requirement.
+            liveIssue = await this.#readIssue(record.issue.path, record.decision);
             if (!liveIssue) {
                 throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is not currently readable`);
             }
@@ -4126,7 +4206,62 @@ export class FactoryLoop {
             this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
         }
     }
-    async #readGithubIssue(path) {
+    #rememberGithubApiFallbackEligible(identity) {
+        rememberBoundedFallbackEligibility(this.#githubApiFallbackIssues, this.#githubApiFallbackIssuesEvicted, identity, GITHUB_API_FALLBACK_ISSUES_MAX, GITHUB_API_FALLBACK_ISSUES_EVICTED_MAX);
+    }
+    /**
+     * `#githubApiFallbackIssues` is a cache, not a source of truth: eligibility
+     * is derived here, not remembered by every caller. A restart (or any read
+     * path added later that never registered anything) still resolves
+     * correctly because this checks the actual current state instead of
+     * requiring every entry point to have called a registration method first.
+     *
+     * A decision can durably outlive the original dispatch call in three
+     * shapes, and this checks all three — a fourth found later means this
+     * enumeration is incomplete, not that the approach is wrong:
+     *   1. `batch.inFlight` — an active live-dispatched or durably-restored
+     *      record (`BatchTracker.restore`/`start` insert it before any phase
+     *      handler runs).
+     *   2. `#state.listWaitingClarifications` — an issue parked awaiting a
+     *      human reply. Restoring it (`#restoreClarifications` et al.)
+     *      rebuilds an `InFlightIssue`-shaped record locally to re-arm the
+     *      watcher and does not insert it into the batch, so (1) alone misses
+     *      it.
+     *   3. `#state.listDispatchLifecycles`, non-terminal — a durably-tracked
+     *      dispatch that is not currently in-memory at all yet (e.g. a
+     *      non-durable-fleet dispatch never reaches the batch either).
+     *
+     * `decisionHint` stays a pure optimization ahead of all three: the one
+     * read that happens before its own record exists anywhere (the initial
+     * live dispatch, validating scope before it is tracked) can skip the
+     * scan entirely, but no read site is required to pass it — omitting it
+     * only costs the scan, never correctness.
+     *
+     * Deliberately scans by GitHub identity (owner/repo/number) against each
+     * candidate's own issue path rather than looking up the durable record by
+     * its composite key (uuid/key/path): the real uuid is built from GitHub's
+     * node_id/id when content is available, which a bare path cannot
+     * reconstruct, so a keyed lookup would not reliably match.
+     */
+    async #deriveGithubApiFallbackEligibility(identity, decisionHint) {
+        if (decisionHint?.issueResolution?.source === 'github-api-fallback' &&
+            githubIssueRefIdentity(decisionHint.issue) === identity) {
+            return true;
+        }
+        const inFlight = (await this.#batch()).inFlight;
+        if (isGithubApiFallbackEligible(inFlight, identity))
+            return true;
+        const [waitingClarifications, dispatchLifecycles] = await Promise.all([
+            this.#state.listWaitingClarifications(this.#workspaceId),
+            this.#state.listDispatchLifecycles(this.#workspaceId),
+        ]);
+        const durable = [
+            ...githubApiFallbackCandidatesFromWaitingClarifications(waitingClarifications),
+            ...githubApiFallbackCandidatesFromDispatchLifecycles(dispatchLifecycles),
+        ];
+        return isGithubApiFallbackEligible(durable, identity);
+    }
+    async #readGithubIssue(path, decisionHint) {
         const preferredPath = await this.#preferredGithubIssuePath(path);
         const candidatePaths = [...new Set([
                 ...githubIssueReadCandidatePaths(preferredPath),
@@ -4150,8 +4285,55 @@ export class FactoryLoop {
         }
         catch (error) {
             if (isMissingIssueFileError(error)) {
+                const parts = githubIssuePathParts(path) ?? githubIssueDirectoryPathParts(path);
+                const identity = parts ? githubIssueIdentity(parts.owner, parts.repo, parts.number) : undefined;
+                const eligible = identity
+                    ? this.#githubApiFallbackIssues.has(identity) ||
+                        await this.#deriveGithubApiFallbackEligibility(identity, decisionHint)
+                    : false;
+                if (eligible && identity)
+                    this.#rememberGithubApiFallbackEligible(identity);
+                if (parts && identity && eligible && this.#mount.githubRead) {
+                    const lookup = await this.#mount.githubRead.getIssue(`${parts.owner}/${parts.repo}`, parts.number);
+                    if (lookup.outcome === 'found') {
+                        const githubIssue = parseGithubIssue(lookup.issue.path, lookup.issue.content);
+                        this.#indexDependencyIssue(githubIssueAsFactoryIssue(githubIssue));
+                        this.#logger.warn?.('[factory] Relayfile projection missed GitHub issue; using GitHub API fallback', {
+                            repo: lookup.issue.repo,
+                            number: lookup.issue.number,
+                            source: 'github-api-fallback',
+                        });
+                        return githubIssue;
+                    }
+                    if (lookup.outcome === 'indeterminate') {
+                        // The provider lookup could not confirm absence (e.g. an
+                        // unauthenticated 404 against a repo it cannot prove is public).
+                        // That is not the same claim as "confirmed gone" — count and log
+                        // it separately so it stays visible instead of being folded into
+                        // the phantom-skip metric.
+                        this.#increment('githubIssueUnverifiable');
+                        this.#logger.warn?.('[factory] GitHub API fallback could not determine issue existence', {
+                            path,
+                            repo: `${parts.owner}/${parts.repo}`,
+                            number: parts.number,
+                            reason: lookup.reason,
+                        });
+                        return undefined;
+                    }
+                }
+                if (identity && this.#githubApiFallbackIssuesEvicted.has(identity)) {
+                    // identity was eligible for the API fallback but aged out of the
+                    // bounded set — a different claim from "never was eligible", and
+                    // one this signal must not silently collapse into a confirmed miss.
+                    this.#increment('githubIssueApiFallbackEligibilityEvicted');
+                    this.#logger.warn?.('[factory] GitHub API fallback eligibility was evicted from the bounded cache before this read', {
+                        path,
+                        identity,
+                    });
+                    return undefined;
+                }
                 this.#increment('githubIssuePhantomSkipped');
-                this.#logger.debug?.('[factory] skipped missing GitHub issue file discovered from issue tree', { path });
+                this.#logger.debug?.('[factory] skipped missing GitHub issue after projection and provider lookup', { path });
                 return undefined;
             }
             throw error;
@@ -4569,6 +4751,7 @@ export class FactoryLoop {
             updatedAt: new Date(updatedAtMs).toISOString(),
             updatedAtMs,
             registryPath,
+            eventListener: this.#eventListenerStatus(),
         };
         await mkdir(dirname(path), { recursive: true });
         await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
@@ -4736,10 +4919,10 @@ export class FactoryLoop {
         }
         return [...pathsByKey.values()].sort();
     }
-    async #readIssue(path) {
+    async #readIssue(path, decisionHint) {
         try {
             if (isGithubIssueFilePath(path)) {
-                const githubIssue = await this.#readGithubIssue(path);
+                const githubIssue = await this.#readGithubIssue(path, decisionHint);
                 return githubIssue ? githubIssueAsFactoryIssue(githubIssue) : undefined;
             }
             // Newly-synced issues land as a change-event STUB at the primary
@@ -5604,6 +5787,7 @@ export class FactoryLoop {
             baseRef,
             title: `${issue.key}: ${issue.title}`,
             body: githubPullRequestBody(issue, implementer.spec.preview),
+            ...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
         });
         const published = result.author
             ? result
@@ -6258,6 +6442,20 @@ export class FactoryLoop {
         if (!tracked.sessionRef) {
             return;
         }
+        // Never resume an agent against a read-denied mirror: the mount lacks the
+        // filesystem scope it needs, so the spawn would fail opaquely (roster PID
+        // never resolves) and risk operating on stale integration state. Skip with
+        // one clear message; a later attempt succeeds once the session is re-authed
+        // and Factory restarted.
+        if (this.#mount.isLocalMountAuthDegraded?.()) {
+            this.#increment('resumeSkippedMountAuthDegraded');
+            this.#logger.warn?.('[factory] tracked agent resume skipped: local mount is auth-degraded (cloud session missing relayfile fs scope); re-authenticate and restart Factory', {
+                issue: record.issue.key,
+                name,
+                role: tracked.spec.role,
+            });
+            return;
+        }
         this.#logger.debug?.('[factory] tracked agent resume preparation started', {
             issue: record.issue.key,
             name,
@@ -11403,7 +11601,7 @@ export class FactoryLoop {
         }, Math.max(1_000, Math.floor(CLARIFICATION_WAKE_LEASE_MS / 3)));
         heartbeat.unref?.();
         try {
-            if (!await this.#clarificationIssueStillActive(waiting.issue)) {
+            if (!await this.#clarificationIssueStillActive(waiting.issue, waiting.decision)) {
                 this.#assertClarificationWakeRunning();
                 await renewLease();
                 const completed = await this.#state.completeClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner);
@@ -11579,8 +11777,8 @@ export class FactoryLoop {
         if (this.#stopping)
             throw new ClarificationWakeStoppedError('factory is stopping');
     }
-    async #clarificationIssueStillActive(issueRef) {
-        const issue = await this.#readIssue(issueRef.path);
+    async #clarificationIssueStillActive(issueRef, decisionHint) {
+        const issue = await this.#readIssue(issueRef.path, decisionHint);
         if (!issue || !isInFactoryScope(issue, this.#config.safety) || !isDispatchableIssue(issue)) {
             this.#logger.info?.('[factory] clarification wake cancelled because issue left factory scope', {
                 issue: issueRef.key,
@@ -11746,12 +11944,11 @@ export class FactoryLoop {
             this.#emit('issue-queued', { issue: decision.issue });
         }
     }
-    // Absolute path to the local .integrations mount the daemon manages. The mount
-    // is created at the daemon's cwd (see ensureLocalMount), and spawned agents run
-    // in their repo clonePath, so writeback paths handed to agents must be absolute
-    // against this root rather than a bare relative `.integrations/...`.
+    // Absolute path to the one registered workspace mirror. Spawned agents run in
+    // repo clone paths, so writeback instructions must name the shared mirror,
+    // not a relative `.integrations` path or a per-repository re-home attempt.
     #integrationsMountRoot() {
-        return resolve(process.cwd(), '.integrations');
+        return this.#mount.getLocalMountRoot?.() ?? resolve(process.cwd(), '.integrations');
     }
     async #slackChannelDir() {
         if (!this.#config.slack) {
@@ -12269,8 +12466,17 @@ const pidsFromSpawnResult = (result) => {
     }
     return [...pids].sort((a, b) => a - b);
 };
-const dispatchComment = (decision, agents) => [
+// `issueResolution.detail` and `projection.localMountDegradedReason` are
+// free text that can carry an absolute local filesystem path (e.g. "mount
+// state is missing at /Users/<name>/...") — safe in the JSON result and in
+// logs, but this comment is posted to a public GitHub issue. Emit only
+// `source`, a closed enum, rather than trying to scrub paths out of free
+// text; free text is not a safe thing to scrub, only a safe thing to omit.
+export const dispatchComment = (decision, agents) => [
     `Factory dispatch for ${decision.issue.key}`,
+    decision.issueResolution
+        ? `Issue resolution: ${decision.issueResolution.source}`
+        : undefined,
     `Implementers: ${agents.filter((agent) => agent.role === 'implementer').map((agent) => agent.name).join(', ') || 'none'}`,
     decision.scope === 'workflow' ? `Workflow: ${agents.find((agent) => agent.role === 'workflow')?.name ?? 'none'}` : undefined,
     `Reviewer: ${agents.find((agent) => agent.role === 'reviewer')?.name ?? 'none'}`,
@@ -12788,11 +12994,51 @@ export const githubIssuePathParts = (path) => {
         slug: match[7],
     };
 };
-const githubIssueIdentity = (owner, repo, number) => `${owner.toLowerCase()}/${repo.toLowerCase()}#${number}`;
+export const githubIssueIdentity = (owner, repo, number) => `${owner.toLowerCase()}/${repo.toLowerCase()}#${number}`;
 const githubIssueRefIdentity = (issue) => {
     const parts = githubIssuePathParts(issue.path);
     return parts ? githubIssueIdentity(parts.owner, parts.repo, parts.number) : undefined;
 };
+/**
+ * Whether a GitHub API fallback read is eligible for `identity`, derived
+ * from currently-tracked state rather than a separately-remembered flag:
+ * `decisionHint` (the one read that happens before its own record is
+ * tracked anywhere) or any `{issue, decision}` candidate — drawn from
+ * whichever durable or in-memory store the caller has gathered — whose
+ * decision was resolved through the fallback. Exported standalone so this
+ * is directly testable — no caller has to have registered anything for
+ * this to return true for a candidate that is genuinely present, which is
+ * the property that keeps a future read call site from silently
+ * reintroducing the restart-eligibility bug by omission.
+ *
+ * A decision durably outlives the in-memory batch in more than one shape
+ * (in-flight dispatch, a parked clarification, a durable dispatch
+ * lifecycle — see #deriveGithubApiFallbackEligibility for the full list
+ * this class gathers); this function does not care which shape a
+ * candidate came from, only whether one matches.
+ */
+export function isGithubApiFallbackEligible(candidates, identity, decisionHint) {
+    if (decisionHint?.issueResolution?.source === 'github-api-fallback' &&
+        githubIssueRefIdentity(decisionHint.issue) === identity) {
+        return true;
+    }
+    return candidates.some((candidate) => candidate.decision.issueResolution?.source === 'github-api-fallback' &&
+        githubIssueRefIdentity(candidate.issue) === identity);
+}
+/** Maps durable waiting-clarification records to isGithubApiFallbackEligible candidates. */
+export function githubApiFallbackCandidatesFromWaitingClarifications(waitingClarifications) {
+    return waitingClarifications.map(([, waiting]) => ({ issue: waiting.issue, decision: waiting.decision }));
+}
+/**
+ * Maps durable dispatch-lifecycle records to isGithubApiFallbackEligible
+ * candidates, excluding terminal ones — a completed or abandoned dispatch's
+ * decision is no longer a reason to trust a live read for that issue.
+ */
+export function githubApiFallbackCandidatesFromDispatchLifecycles(dispatchLifecycles) {
+    return dispatchLifecycles
+        .filter(([, lifecycle]) => !isTerminalDispatchLifecycle(lifecycle))
+        .map(([, lifecycle]) => ({ issue: lifecycle.issue, decision: lifecycle.decision }));
+}
 const githubIssuePathPreference = (path) => {
     if (path.endsWith('/meta.json'))
         return 0;