@agent-relay/factory 0.1.72 → 0.1.73

package/dist/orchestrator/factory.js

@@ -36,6 +36,7 @@ import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../obse
 import { telemetryErrorClass } from '../observability/error-class.js';
 import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
 import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
+import { factoryDispatchFailureReasonCodeForErrorClass, factoryDispatchFailureReasonCounts, } from './dispatch-failure-reason.js';
 import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
 import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
 import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
@@ -553,18 +554,30 @@ export class FactoryLoop {
     #readinessReconcileLastError;
     #readinessReconcileLastErrorClass;
     /**
-     * The last *completed* sweep's arithmetic (#355).
+     * The last *enumerating* sweep's arithmetic (#355).
      *
-     * Held as one record rather than four fields so it can only ever be replaced
+     * Held as one record rather than three fields so it can only ever be replaced
      * whole: publishing a `dispatched` from one pass beside a `candidates` from
      * another would be worse than publishing neither, since the whole use of
      * these numbers is comparing them to each other.
      *
-     * `undefined` until a sweep completes, and never initialised to zeroes —
+     * `undefined` until a sweep enumerates, and never initialised to zeroes —
      * "this daemon has not finished a sweep" and "a sweep finished and found
      * nothing" are the two readings #355 has to tell apart.
      */
     #readinessReconcileLastSweep;
+    /**
+     * Whether the MOST RECENT pass deferred, tracked apart from the counts above
+     * (#358 review, CodeRabbit — Major, and right).
+     *
+     * A deferred pass enumerates nothing and settles in milliseconds, so folding
+     * it into the snapshot overwrote the last real sweep's numbers with zeroes.
+     * On a container where another process holds the lease for any length of time
+     * — the #347/#349 condition — every pass would publish `candidates: 0` and the
+     * last actual enumeration would be unrecoverable, destroying the measurement
+     * this whole change exists to provide.
+     */
+    #readinessReconcileLastSweepDeferred;
     #liveEventQueue = [];
     #liveEventDrainScheduled = false;
     #liveEventDrainActive = false;
@@ -1331,9 +1344,11 @@ export class FactoryLoop {
                 // cold container it is the first — and for the next interval, only —
                 // sweep whose counts exist. Leaving it unrecorded would make a daemon
                 // that has completed a full pass still read as "never ran" (#355).
-                this.#recordReadinessSweepOutcome(await this.runOnce());
+                const report = await this.runOnce();
                 this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
-                this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
+                const completedAtMs = this.#clock.now();
+                this.#readinessReconcileLastCompletedAtMs = completedAtMs;
+                this.#recordReadinessSweepOutcome(report, completedAtMs);
             }
             catch (error) {
                 this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
@@ -1559,20 +1574,28 @@ export class FactoryLoop {
             const report = await this.#runOnceWithReadinessDeadline();
             this.#readinessReconcileConsecutiveFailures = 0;
             this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
-            this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
+            const completedAtMs = this.#clock.now();
+            this.#readinessReconcileLastCompletedAtMs = completedAtMs;
             this.#readinessReconcileLastError = undefined;
             this.#readinessReconcileLastErrorClass = undefined;
             // The three integers below have gone to stdout since this loop existed,
             // and stdout does not reach the deployed container's operator (#355).
             // Publishing them is what lets a reader tell a sweep that saw eligible
             // work and rejected it from one that never pulled it at all.
-            this.#recordReadinessSweepOutcome(report);
+            this.#recordReadinessSweepOutcome(report, completedAtMs);
             this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
                 durationMs: this.#readinessReconcileLastDurationMs,
                 candidates: report.pulled.length,
                 dispatched: report.dispatched.length,
                 skipped: report.skipped.length,
-                skipReasons: this.#readinessReconcileLastSweep?.skipReasons,
+                // THIS pass's breakdown, never the retained snapshot (#359 review,
+                // codex P2). Logging the retained one beside a deferred pass's zeroes
+                // produced a line that contradicted its own arithmetic —
+                // `skipped: 0` next to a non-empty breakdown — and this log is what a
+                // local operator reads.
+                skipReasons: factorySweepSkipReasonCounts(report.skipped),
+                dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
+                dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
                 discoveryDeferred: report.discoveryDeferred,
             });
         }
@@ -1596,6 +1619,10 @@ export class FactoryLoop {
             this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
             this.#readinessReconcileLastFailureAtMs = this.#clock.now();
             this.#readinessReconcileLastError = errorMessage;
+            // This failure is now the latest settled pass. A deferral marker left by
+            // an older pass would falsely describe this one as lease contention when
+            // the timestamps/error below prove that it acquired the lease and failed.
+            this.#readinessReconcileLastSweepDeferred = undefined;
             // The class, unlike the message, is publishable: #295 puts it on the
             // unauthenticated health surface through the same allowlist.
             this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
@@ -2703,6 +2730,10 @@ export class FactoryLoop {
                     continue;
                 }
                 const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
+                // Reset per work unit, and advanced by assignment immediately before
+                // each stage rather than inferred in the catch: the whole value of the
+                // `unclassified-*` codes is that they name the stage honestly (#355).
+                let attemptPhase = 'gate';
                 try {
                     if (recoveredOrphan) {
                         const dispatchBlock = await this.#dispatchBlockReason(issue);
@@ -2723,8 +2754,10 @@ export class FactoryLoop {
                         });
                         continue;
                     }
+                    attemptPhase = 'triage';
                     const decision = await this.triageIssue(issue);
                     triaged.push(decision);
+                    attemptPhase = 'dispatch';
                     const result = await this.dispatch(decision, { dryRun });
                     // A completed dispatch — even one that parks or escalates the issue —
                     // proves the pipeline still works, so the fuse below starts over.
@@ -2825,6 +2858,11 @@ export class FactoryLoop {
                         issue: issueRef(issue),
                         reason: perItemDispatchSkipReason(error),
                         code: 'dispatch-failed',
+                        // The publishable half of the same classification. `reason` is the
+                        // operator's sentence and stays off the health surface; this token
+                        // is what tells a reader watching `dispatch-failed: 5` which of
+                        // five very different bugs they are looking at (#355).
+                        failureCode: perItemDispatchFailureCode(error, attemptPhase),
                     });
                     continue;
                 }
@@ -4488,6 +4526,9 @@ export class FactoryLoop {
             })) ?? [],
             counters: { ...this.#counters },
             fleetControlPlane: this.#fleetControlPlane.status(),
+            // Optional on the port: a backend with no socket omits it, and an absent
+            // value stays absent rather than being invented as healthy.
+            ...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
             slackDegraded: this.#slackDegraded,
             slackDegradedReason: this.#slackDegradedReason,
             eventListener: this.#eventListenerStatus(),
@@ -4519,16 +4560,37 @@ export class FactoryLoop {
      *
      * Only successful passes reach here: a pass that threw has no report, and
      * inventing zeroes for it would publish "found nothing" for a sweep that
-     * never got to look. The previous pass's numbers stay put instead, dated by
-     * `lastCompletedAtMs`, which is the honest reading.
+     * never got to look. The previous enumerating pass's numbers stay put
+     * instead, dated by `lastEnumeratedAtMs`, which is the honest reading.
+     *
+     * A deferred pass gets the same treatment for the same reason. It settles
+     * successfully, and `lastCompletedAtMs` moves — deliberately, because the
+     * #295/#296 stall derivation reads that timestamp against `lastStartedAtMs`,
+     * and freezing it would report a functioning daemon as a hung one after ten
+     * intervals of deferring correctly to another owner. But it enumerated
+     * nothing, so its zeroes are not a measurement of anything and must not
+     * replace one. Only the marker is recorded.
      */
-    #recordReadinessSweepOutcome(report) {
+    #recordReadinessSweepOutcome(report, completedAtMs) {
+        if (report.discoveryDeferred) {
+            this.#readinessReconcileLastSweepDeferred = report.discoveryDeferred;
+            return;
+        }
+        this.#readinessReconcileLastSweepDeferred = undefined;
         this.#readinessReconcileLastSweep = {
             candidates: report.pulled.length,
             dispatched: report.dispatched.length,
             skipped: report.skipped.length,
             skipReasons: factorySweepSkipReasonCounts(report.skipped),
-            ...(report.discoveryDeferred ? { discoveryDeferred: report.discoveryDeferred } : {}),
+            // Counted from the same entries `skipReasons` counts, so the parts sum to
+            // `skipReasons['dispatch-failed']` by construction rather than by a
+            // second traversal agreeing with the first.
+            dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
+            dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
+            // The caller's completion stamp, not a fresh clock read: on a pass that
+            // enumerated, `lastEnumeratedAtMs` and `lastCompletedAtMs` describe the
+            // same instant and must not drift apart by a tick.
+            enumeratedAtMs: completedAtMs,
         };
     }
     #readinessReconcileStatus() {
@@ -4616,11 +4678,21 @@ export class FactoryLoop {
                     ...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
                         ? { skipReasons: { ...this.#readinessReconcileLastSweep.skipReasons } }
                         : {}),
-                    ...(this.#readinessReconcileLastSweep.discoveryDeferred
-                        ? { discoveryDeferred: this.#readinessReconcileLastSweep.discoveryDeferred }
+                    // Unconditional, unlike the breakdown below it: a zero here is the
+                    // fact "this sweep attempted dispatches and none of them failed",
+                    // which no other field on this surface can express.
+                    dispatchFailures: this.#readinessReconcileLastSweep.dispatchFailures,
+                    ...(Object.keys(this.#readinessReconcileLastSweep.dispatchFailureReasons).length > 0
+                        ? { dispatchFailureReasons: { ...this.#readinessReconcileLastSweep.dispatchFailureReasons } }
                         : {}),
+                    lastEnumeratedAtMs: this.#readinessReconcileLastSweep.enumeratedAtMs,
                 }
                 : {}),
+            // Independent of the trio: a daemon whose FIRST pass deferred has no
+            // counts to publish and still needs to say why.
+            ...(this.#readinessReconcileLastSweepDeferred
+                ? { discoveryDeferred: this.#readinessReconcileLastSweepDeferred }
+                : {}),
             ...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
             ...(this.#readinessReconcileLastErrorClass
                 ? { lastErrorClass: this.#readinessReconcileLastErrorClass }
@@ -7453,6 +7525,9 @@ export class FactoryLoop {
             readinessReconcile: this.#readinessReconcileStatus(),
             dispatchCapacity: this.#dispatchCapacityStatus(),
             fleetControlPlane: this.#fleetControlPlane.status(),
+            // Optional on the port: a backend with no socket omits it, and an absent
+            // value stays absent rather than being invented as healthy.
+            ...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
         };
         // The deployed container serves `/healthz` straight out of this file and
         // has no redaction logic of its own, so publish the already-safe view here
@@ -18271,21 +18346,28 @@ const githubIssueIndexRepoRoots = (path) => {
         `/github/repos/${owner}__${repo}/issues`,
     ];
 };
-const relayfileOverload = (error) => {
-    const flat = asRecord(error) ?? {};
+/** How far to follow wrapped provider failures without trusting an unbounded chain. */
+const RELAYFILE_OVERLOAD_CAUSE_DEPTH = 4;
+const relayfileOverload = (error, depth = 0) => {
+    if (depth > RELAYFILE_OVERLOAD_CAUSE_DEPTH)
+        return undefined;
+    const flat = asRecord(error);
+    if (!flat)
+        return undefined;
     const response = asRecord(flat.response) ?? {};
     const data = asRecord(flat.data) ?? asRecord(response.data) ?? {};
     const details = asRecord(flat.details) ?? asRecord(data.details) ?? {};
     const statusValue = flat.status ?? flat.statusCode ?? response.status ?? response.statusCode;
     const status = typeof statusValue === 'number' ? statusValue : Number(statusValue);
-    if (status !== 429)
-        return undefined;
-    const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
-    const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
-    const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
-    const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
-        stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
-    return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
+    if (status === 429) {
+        const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
+        const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
+        const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
+        const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
+            stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
+        return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
+    }
+    return relayfileOverload(flat.cause, depth + 1);
 };
 /**
  * relayfile's overload reason codes, allowlisted.
@@ -18804,13 +18886,14 @@ const PASS_FATAL_CAUSE_DEPTH = 4;
  * classification has to follow the cause chain rather than trust the outermost
  * type.
  */
-const wrapsErrorOfType = (error, type, depth = 0) => {
+const findWrappedErrorOfType = (error, type, depth = 0) => {
     if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
-        return false;
+        return undefined;
     if (error instanceof type)
-        return true;
-    return wrapsErrorOfType(error.cause, type, depth + 1);
+        return error;
+    return findWrappedErrorOfType(error.cause, type, depth + 1);
 };
+const wrapsErrorOfType = (error, type) => findWrappedErrorOfType(error, type) !== undefined;
 /**
  * How many *unclassified* per-item failures without an intervening successful
  * dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
@@ -18875,6 +18958,54 @@ const perItemDispatchSkipReason = (error) => {
     }
     return `dispatch failed (${telemetryErrorClass(error)})`;
 };
+const UNCLASSIFIED_PHASE_CODES = {
+    gate: 'unclassified-gate',
+    triage: 'unclassified-triage',
+    dispatch: 'unclassified-dispatch',
+};
+/**
+ * Why a dispatch attempt failed, as a code from the published vocabulary (#355).
+ *
+ * The sibling of `perItemDispatchSkipReason`: that returns the operator's
+ * sentence, this returns the one token that may cross onto the unauthenticated
+ * health surface, and both are recorded at the skip site from the same thrown
+ * value. The token is never parsed back out of the sentence — a reworded
+ * message would silently empty a bucket, and this vocabulary is what an
+ * operator reads when the daemon's stdout does not reach them.
+ *
+ * Ordered most specific first. Every branch follows the bounded cause chain,
+ * because `contextualError` and the control-plane guard both rethrow wrapped.
+ * `relayfileOverload` is also the loop's shedding predicate, so widening it at
+ * the source keeps the health code, skip counter, fuse, and durable overload
+ * ratchet on one verdict instead of merely relabelling the published bucket.
+ */
+const perItemDispatchFailureCode = (error, phase) => {
+    if (relayfileOverload(error) !== undefined)
+        return 'relayfile-overloaded';
+    if (wrapsErrorOfType(error, LiveDispatchStateChangedError))
+        return 'live-state-changed';
+    if (wrapsErrorOfType(error, LatePlacementReleasedError))
+        return 'late-placement-released';
+    const refused = findWrappedErrorOfType(error, DispatchLifecycleClaimRefusedError);
+    if (refused)
+        return refused.refusal === 'terminal' ? 'lifecycle-terminal' : 'lifecycle-owned-elsewhere';
+    if (wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError))
+        return 'control-plane-open';
+    // The class-name tail of the vocabulary; see its own doc comment for why
+    // these five are not `instanceof`. Walked down the cause chain like the
+    // branches above, because `contextualError` wraps in a plain `Error` and
+    // reading only the outermost name would miss every wrapped spawn failure.
+    // `telemetryErrorClass` is the same allowlist that guards every other
+    // identifier leaving this process, so a hostile `name` cannot invent a key
+    // here either — it collapses to `Error`, which the map does not hold.
+    for (let cursor = error, depth = 0; cursor instanceof Error && depth <= PASS_FATAL_CAUSE_DEPTH; depth += 1) {
+        const named = factoryDispatchFailureReasonCodeForErrorClass(telemetryErrorClass(cursor));
+        if (named)
+            return named;
+        cursor = cursor.cause;
+    }
+    return UNCLASSIFIED_PHASE_CODES[phase];
+};
 const triageEscalationQuestion = (decision, issue) => {
     const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
     const subject = issue?.title?.trim() || decision.issue.key;