@agent-relay/factory 0.1.71 → 0.1.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/diagnose.d.ts +13 -1
- package/dist/cli/diagnose.d.ts.map +1 -1
- package/dist/cli/diagnose.js +92 -1
- package/dist/cli/diagnose.js.map +1 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +5 -0
- package/dist/cli/fleet.js.map +1 -1
- package/dist/fleet/control-plane-circuit.d.ts +10 -0
- package/dist/fleet/control-plane-circuit.d.ts.map +1 -1
- package/dist/fleet/control-plane-circuit.js +9 -1
- package/dist/fleet/control-plane-circuit.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +21 -2
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +97 -17
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/observability/events.d.ts +14 -14
- package/dist/orchestrator/dispatch-failure-reason.d.ts +60 -0
- package/dist/orchestrator/dispatch-failure-reason.d.ts.map +1 -0
- package/dist/orchestrator/dispatch-failure-reason.js +127 -0
- package/dist/orchestrator/dispatch-failure-reason.js.map +1 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +253 -31
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts.map +1 -1
- package/dist/orchestrator/public-health.js +185 -0
- package/dist/orchestrator/public-health.js.map +1 -1
- package/dist/orchestrator/sweep-skip-reason.d.ts +45 -0
- package/dist/orchestrator/sweep-skip-reason.d.ts.map +1 -0
- package/dist/orchestrator/sweep-skip-reason.js +82 -0
- package/dist/orchestrator/sweep-skip-reason.js.map +1 -0
- package/dist/ports/fleet.d.ts +38 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -1
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/types.d.ts +156 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -35,6 +35,8 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
|
35
35
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
36
36
|
import { telemetryErrorClass } from '../observability/error-class.js';
|
|
37
37
|
import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
|
|
38
|
+
import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
|
|
39
|
+
import { factoryDispatchFailureReasonCodeForErrorClass, factoryDispatchFailureReasonCounts, } from './dispatch-failure-reason.js';
|
|
38
40
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
39
41
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
40
42
|
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
@@ -551,6 +553,31 @@ export class FactoryLoop {
|
|
|
551
553
|
#readinessReconcileLastFailureAtMs;
|
|
552
554
|
#readinessReconcileLastError;
|
|
553
555
|
#readinessReconcileLastErrorClass;
|
|
556
|
+
/**
|
|
557
|
+
* The last *enumerating* sweep's arithmetic (#355).
|
|
558
|
+
*
|
|
559
|
+
* Held as one record rather than three fields so it can only ever be replaced
|
|
560
|
+
* whole: publishing a `dispatched` from one pass beside a `candidates` from
|
|
561
|
+
* another would be worse than publishing neither, since the whole use of
|
|
562
|
+
* these numbers is comparing them to each other.
|
|
563
|
+
*
|
|
564
|
+
* `undefined` until a sweep enumerates, and never initialised to zeroes —
|
|
565
|
+
* "this daemon has not finished a sweep" and "a sweep finished and found
|
|
566
|
+
* nothing" are the two readings #355 has to tell apart.
|
|
567
|
+
*/
|
|
568
|
+
#readinessReconcileLastSweep;
|
|
569
|
+
/**
|
|
570
|
+
* Whether the MOST RECENT pass deferred, tracked apart from the counts above
|
|
571
|
+
* (#358 review, CodeRabbit — Major, and right).
|
|
572
|
+
*
|
|
573
|
+
* A deferred pass enumerates nothing and settles in milliseconds, so folding
|
|
574
|
+
* it into the snapshot overwrote the last real sweep's numbers with zeroes.
|
|
575
|
+
* On a container where another process holds the lease for any length of time
|
|
576
|
+
* — the #347/#349 condition — every pass would publish `candidates: 0` and the
|
|
577
|
+
* last actual enumeration would be unrecoverable, destroying the measurement
|
|
578
|
+
* this whole change exists to provide.
|
|
579
|
+
*/
|
|
580
|
+
#readinessReconcileLastSweepDeferred;
|
|
554
581
|
#liveEventQueue = [];
|
|
555
582
|
#liveEventDrainScheduled = false;
|
|
556
583
|
#liveEventDrainActive = false;
|
|
@@ -1313,9 +1340,15 @@ export class FactoryLoop {
|
|
|
1313
1340
|
const backfillStartedAtMs = this.#clock.now();
|
|
1314
1341
|
this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
|
|
1315
1342
|
try {
|
|
1316
|
-
|
|
1343
|
+
// The startup backfill is a discovery pass like any other, and on a
|
|
1344
|
+
// cold container it is the first — and for the next interval, only —
|
|
1345
|
+
// sweep whose counts exist. Leaving it unrecorded would make a daemon
|
|
1346
|
+
// that has completed a full pass still read as "never ran" (#355).
|
|
1347
|
+
const report = await this.runOnce();
|
|
1317
1348
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1318
|
-
|
|
1349
|
+
const completedAtMs = this.#clock.now();
|
|
1350
|
+
this.#readinessReconcileLastCompletedAtMs = completedAtMs;
|
|
1351
|
+
this.#recordReadinessSweepOutcome(report, completedAtMs);
|
|
1319
1352
|
}
|
|
1320
1353
|
catch (error) {
|
|
1321
1354
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
@@ -1541,14 +1574,29 @@ export class FactoryLoop {
|
|
|
1541
1574
|
const report = await this.#runOnceWithReadinessDeadline();
|
|
1542
1575
|
this.#readinessReconcileConsecutiveFailures = 0;
|
|
1543
1576
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1544
|
-
|
|
1577
|
+
const completedAtMs = this.#clock.now();
|
|
1578
|
+
this.#readinessReconcileLastCompletedAtMs = completedAtMs;
|
|
1545
1579
|
this.#readinessReconcileLastError = undefined;
|
|
1546
1580
|
this.#readinessReconcileLastErrorClass = undefined;
|
|
1581
|
+
// The three integers below have gone to stdout since this loop existed,
|
|
1582
|
+
// and stdout does not reach the deployed container's operator (#355).
|
|
1583
|
+
// Publishing them is what lets a reader tell a sweep that saw eligible
|
|
1584
|
+
// work and rejected it from one that never pulled it at all.
|
|
1585
|
+
this.#recordReadinessSweepOutcome(report, completedAtMs);
|
|
1547
1586
|
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1548
1587
|
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1549
1588
|
candidates: report.pulled.length,
|
|
1550
1589
|
dispatched: report.dispatched.length,
|
|
1551
1590
|
skipped: report.skipped.length,
|
|
1591
|
+
// THIS pass's breakdown, never the retained snapshot (#359 review,
|
|
1592
|
+
// codex P2). Logging the retained one beside a deferred pass's zeroes
|
|
1593
|
+
// produced a line that contradicted its own arithmetic —
|
|
1594
|
+
// `skipped: 0` next to a non-empty breakdown — and this log is what a
|
|
1595
|
+
// local operator reads.
|
|
1596
|
+
skipReasons: factorySweepSkipReasonCounts(report.skipped),
|
|
1597
|
+
dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
|
|
1598
|
+
dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
|
|
1599
|
+
discoveryDeferred: report.discoveryDeferred,
|
|
1552
1600
|
});
|
|
1553
1601
|
}
|
|
1554
1602
|
catch (error) {
|
|
@@ -1571,6 +1619,10 @@ export class FactoryLoop {
|
|
|
1571
1619
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1572
1620
|
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1573
1621
|
this.#readinessReconcileLastError = errorMessage;
|
|
1622
|
+
// This failure is now the latest settled pass. A deferral marker left by
|
|
1623
|
+
// an older pass would falsely describe this one as lease contention when
|
|
1624
|
+
// the timestamps/error below prove that it acquired the lease and failed.
|
|
1625
|
+
this.#readinessReconcileLastSweepDeferred = undefined;
|
|
1574
1626
|
// The class, unlike the message, is publishable: #295 puts it on the
|
|
1575
1627
|
// unauthenticated health surface through the same allowlist.
|
|
1576
1628
|
this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
|
|
@@ -2540,6 +2592,7 @@ export class FactoryLoop {
|
|
|
2540
2592
|
issue: entry.issue.key,
|
|
2541
2593
|
path: entry.issue.path,
|
|
2542
2594
|
reason: entry.reason,
|
|
2595
|
+
code: entry.code,
|
|
2543
2596
|
});
|
|
2544
2597
|
};
|
|
2545
2598
|
// Backstop for the skip-by-default catch below: see #292. Reset only by
|
|
@@ -2579,7 +2632,11 @@ export class FactoryLoop {
|
|
|
2579
2632
|
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2580
2633
|
sweepOverloads: this.#discoverySweepOverloads,
|
|
2581
2634
|
});
|
|
2582
|
-
recordSkip({
|
|
2635
|
+
recordSkip({
|
|
2636
|
+
issue: issueRefFromPath(path),
|
|
2637
|
+
reason: perItemDispatchSkipReason(error),
|
|
2638
|
+
code: 'read-failed',
|
|
2639
|
+
});
|
|
2583
2640
|
}
|
|
2584
2641
|
readyIssueReads += 1;
|
|
2585
2642
|
// Relayfile served this work unit's read: the dependency is shedding
|
|
@@ -2640,7 +2697,7 @@ export class FactoryLoop {
|
|
|
2640
2697
|
if (!mayRecoverGithubOrphan) {
|
|
2641
2698
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
2642
2699
|
if (dispatchBlock) {
|
|
2643
|
-
recordSkip({ issue: issueRef(issue),
|
|
2700
|
+
recordSkip({ issue: issueRef(issue), ...dispatchBlock });
|
|
2644
2701
|
continue;
|
|
2645
2702
|
}
|
|
2646
2703
|
}
|
|
@@ -2650,42 +2707,57 @@ export class FactoryLoop {
|
|
|
2650
2707
|
const recoveredOrphan = orphanResult.recovered;
|
|
2651
2708
|
const batch = await this.#batch();
|
|
2652
2709
|
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
2653
|
-
recordSkip({
|
|
2710
|
+
recordSkip({
|
|
2711
|
+
issue: issueRef(issue),
|
|
2712
|
+
reason: orphanResult.reason ?? 'already tracked',
|
|
2713
|
+
code: 'already-tracked',
|
|
2714
|
+
});
|
|
2654
2715
|
continue;
|
|
2655
2716
|
}
|
|
2656
2717
|
if (!wasReady && !recoveredOrphan) {
|
|
2657
2718
|
if (mayRecoverGithubOrphan) {
|
|
2658
2719
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
2659
2720
|
if (dispatchBlock) {
|
|
2660
|
-
recordSkip({ issue: issueRef(issue),
|
|
2721
|
+
recordSkip({ issue: issueRef(issue), ...dispatchBlock });
|
|
2661
2722
|
continue;
|
|
2662
2723
|
}
|
|
2663
2724
|
}
|
|
2664
2725
|
recordSkip({
|
|
2665
2726
|
issue: issueRef(issue),
|
|
2666
2727
|
reason: orphanResult.reason ?? 'live state is not ready-for-agent',
|
|
2728
|
+
code: 'not-ready',
|
|
2667
2729
|
});
|
|
2668
2730
|
continue;
|
|
2669
2731
|
}
|
|
2670
2732
|
const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
|
|
2733
|
+
// Reset per work unit, and advanced by assignment immediately before
|
|
2734
|
+
// each stage rather than inferred in the catch: the whole value of the
|
|
2735
|
+
// `unclassified-*` codes is that they name the stage honestly (#355).
|
|
2736
|
+
let attemptPhase = 'gate';
|
|
2671
2737
|
try {
|
|
2672
2738
|
if (recoveredOrphan) {
|
|
2673
2739
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
2674
2740
|
if (dispatchBlock) {
|
|
2675
|
-
recordSkip({ issue: issueRef(issue),
|
|
2741
|
+
recordSkip({ issue: issueRef(issue), ...dispatchBlock });
|
|
2676
2742
|
continue;
|
|
2677
2743
|
}
|
|
2678
2744
|
}
|
|
2679
2745
|
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
2680
|
-
recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
|
|
2746
|
+
recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope', code: 'out-of-scope' });
|
|
2681
2747
|
continue;
|
|
2682
2748
|
}
|
|
2683
2749
|
if (!isDispatchableIssue(issue)) {
|
|
2684
|
-
recordSkip({
|
|
2750
|
+
recordSkip({
|
|
2751
|
+
issue: issueRef(issue),
|
|
2752
|
+
reason: 'not reconciled real Linear issue',
|
|
2753
|
+
code: 'not-dispatchable',
|
|
2754
|
+
});
|
|
2685
2755
|
continue;
|
|
2686
2756
|
}
|
|
2757
|
+
attemptPhase = 'triage';
|
|
2687
2758
|
const decision = await this.triageIssue(issue);
|
|
2688
2759
|
triaged.push(decision);
|
|
2760
|
+
attemptPhase = 'dispatch';
|
|
2689
2761
|
const result = await this.dispatch(decision, { dryRun });
|
|
2690
2762
|
// A completed dispatch — even one that parks or escalates the issue —
|
|
2691
2763
|
// proves the pipeline still works, so the fuse below starts over.
|
|
@@ -2694,12 +2766,17 @@ export class FactoryLoop {
|
|
|
2694
2766
|
// decays the durable overload ratchet (#297).
|
|
2695
2767
|
this.#discoverySweepProgress = true;
|
|
2696
2768
|
if (result.agents.length === 0 && !dryRun) {
|
|
2769
|
+
const code = result.hold?.kind === 'dependency-cycle'
|
|
2770
|
+
? 'dependency-cycle'
|
|
2771
|
+
: result.hold?.kind === 'dependency'
|
|
2772
|
+
? 'parked-dependency'
|
|
2773
|
+
: 'queued-or-escalated';
|
|
2697
2774
|
const reason = result.hold?.kind === 'dependency-cycle'
|
|
2698
2775
|
? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
|
|
2699
2776
|
: result.hold?.kind === 'dependency'
|
|
2700
2777
|
? `parked on dependencies: ${result.hold.blockers?.join(', ') ?? 'unresolved dependency'}`
|
|
2701
2778
|
: 'queued or escalated';
|
|
2702
|
-
recordSkip({ issue: decision.issue, reason });
|
|
2779
|
+
recordSkip({ issue: decision.issue, reason, code });
|
|
2703
2780
|
}
|
|
2704
2781
|
else {
|
|
2705
2782
|
dispatched.push(result);
|
|
@@ -2777,7 +2854,16 @@ export class FactoryLoop {
|
|
|
2777
2854
|
error: describeError(error).errorMessage,
|
|
2778
2855
|
});
|
|
2779
2856
|
}
|
|
2780
|
-
recordSkip({
|
|
2857
|
+
recordSkip({
|
|
2858
|
+
issue: issueRef(issue),
|
|
2859
|
+
reason: perItemDispatchSkipReason(error),
|
|
2860
|
+
code: 'dispatch-failed',
|
|
2861
|
+
// The publishable half of the same classification. `reason` is the
|
|
2862
|
+
// operator's sentence and stays off the health surface; this token
|
|
2863
|
+
// is what tells a reader watching `dispatch-failed: 5` which of
|
|
2864
|
+
// five very different bugs they are looking at (#355).
|
|
2865
|
+
failureCode: perItemDispatchFailureCode(error, attemptPhase),
|
|
2866
|
+
});
|
|
2781
2867
|
continue;
|
|
2782
2868
|
}
|
|
2783
2869
|
finally {
|
|
@@ -4014,7 +4100,7 @@ export class FactoryLoop {
|
|
|
4014
4100
|
}
|
|
4015
4101
|
const blockReason = await this.#dispatchBlockReason(decision.issue);
|
|
4016
4102
|
if (blockReason) {
|
|
4017
|
-
const error = new Error(`Refusing to dispatch ${decision.issue.key}: ${blockReason}`);
|
|
4103
|
+
const error = new Error(`Refusing to dispatch ${decision.issue.key}: ${blockReason.reason}`);
|
|
4018
4104
|
this.#error(error, decision.issue);
|
|
4019
4105
|
throw error;
|
|
4020
4106
|
}
|
|
@@ -4440,6 +4526,9 @@ export class FactoryLoop {
|
|
|
4440
4526
|
})) ?? [],
|
|
4441
4527
|
counters: { ...this.#counters },
|
|
4442
4528
|
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
4529
|
+
// Optional on the port: a backend with no socket omits it, and an absent
|
|
4530
|
+
// value stays absent rather than being invented as healthy.
|
|
4531
|
+
...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
|
|
4443
4532
|
slackDegraded: this.#slackDegraded,
|
|
4444
4533
|
slackDegradedReason: this.#slackDegradedReason,
|
|
4445
4534
|
eventListener: this.#eventListenerStatus(),
|
|
@@ -4466,6 +4555,44 @@ export class FactoryLoop {
|
|
|
4466
4555
|
}
|
|
4467
4556
|
return { state: 'starting' };
|
|
4468
4557
|
}
|
|
4558
|
+
/**
|
|
4559
|
+
* Snapshot a settled sweep's counts onto the readiness surface (#355).
|
|
4560
|
+
*
|
|
4561
|
+
* Only successful passes reach here: a pass that threw has no report, and
|
|
4562
|
+
* inventing zeroes for it would publish "found nothing" for a sweep that
|
|
4563
|
+
* never got to look. The previous enumerating pass's numbers stay put
|
|
4564
|
+
* instead, dated by `lastEnumeratedAtMs`, which is the honest reading.
|
|
4565
|
+
*
|
|
4566
|
+
* A deferred pass gets the same treatment for the same reason. It settles
|
|
4567
|
+
* successfully, and `lastCompletedAtMs` moves — deliberately, because the
|
|
4568
|
+
* #295/#296 stall derivation reads that timestamp against `lastStartedAtMs`,
|
|
4569
|
+
* and freezing it would report a functioning daemon as a hung one after ten
|
|
4570
|
+
* intervals of deferring correctly to another owner. But it enumerated
|
|
4571
|
+
* nothing, so its zeroes are not a measurement of anything and must not
|
|
4572
|
+
* replace one. Only the marker is recorded.
|
|
4573
|
+
*/
|
|
4574
|
+
#recordReadinessSweepOutcome(report, completedAtMs) {
|
|
4575
|
+
if (report.discoveryDeferred) {
|
|
4576
|
+
this.#readinessReconcileLastSweepDeferred = report.discoveryDeferred;
|
|
4577
|
+
return;
|
|
4578
|
+
}
|
|
4579
|
+
this.#readinessReconcileLastSweepDeferred = undefined;
|
|
4580
|
+
this.#readinessReconcileLastSweep = {
|
|
4581
|
+
candidates: report.pulled.length,
|
|
4582
|
+
dispatched: report.dispatched.length,
|
|
4583
|
+
skipped: report.skipped.length,
|
|
4584
|
+
skipReasons: factorySweepSkipReasonCounts(report.skipped),
|
|
4585
|
+
// Counted from the same entries `skipReasons` counts, so the parts sum to
|
|
4586
|
+
// `skipReasons['dispatch-failed']` by construction rather than by a
|
|
4587
|
+
// second traversal agreeing with the first.
|
|
4588
|
+
dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
|
|
4589
|
+
dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
|
|
4590
|
+
// The caller's completion stamp, not a fresh clock read: on a pass that
|
|
4591
|
+
// enumerated, `lastEnumeratedAtMs` and `lastCompletedAtMs` describe the
|
|
4592
|
+
// same instant and must not drift apart by a tick.
|
|
4593
|
+
enumeratedAtMs: completedAtMs,
|
|
4594
|
+
};
|
|
4595
|
+
}
|
|
4469
4596
|
#readinessReconcileStatus() {
|
|
4470
4597
|
const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
|
|
4471
4598
|
// #296 owns the numerator here, #295/#300 own the derivation. The earliest
|
|
@@ -4540,6 +4667,32 @@ export class FactoryLoop {
|
|
|
4540
4667
|
...(this.#readinessReconcileLastFailureAtMs !== undefined
|
|
4541
4668
|
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
4542
4669
|
: {}),
|
|
4670
|
+
// Spread whole or not at all: see `#readinessReconcileLastSweep`. Zeroes
|
|
4671
|
+
// are published, which is the entire point — an absent `candidates` and
|
|
4672
|
+
// a zero `candidates` are different diagnoses (#355).
|
|
4673
|
+
...(this.#readinessReconcileLastSweep
|
|
4674
|
+
? {
|
|
4675
|
+
candidates: this.#readinessReconcileLastSweep.candidates,
|
|
4676
|
+
dispatched: this.#readinessReconcileLastSweep.dispatched,
|
|
4677
|
+
skipped: this.#readinessReconcileLastSweep.skipped,
|
|
4678
|
+
...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
|
|
4679
|
+
? { skipReasons: { ...this.#readinessReconcileLastSweep.skipReasons } }
|
|
4680
|
+
: {}),
|
|
4681
|
+
// Unconditional, unlike the breakdown below it: a zero here is the
|
|
4682
|
+
// fact "this sweep attempted dispatches and none of them failed",
|
|
4683
|
+
// which no other field on this surface can express.
|
|
4684
|
+
dispatchFailures: this.#readinessReconcileLastSweep.dispatchFailures,
|
|
4685
|
+
...(Object.keys(this.#readinessReconcileLastSweep.dispatchFailureReasons).length > 0
|
|
4686
|
+
? { dispatchFailureReasons: { ...this.#readinessReconcileLastSweep.dispatchFailureReasons } }
|
|
4687
|
+
: {}),
|
|
4688
|
+
lastEnumeratedAtMs: this.#readinessReconcileLastSweep.enumeratedAtMs,
|
|
4689
|
+
}
|
|
4690
|
+
: {}),
|
|
4691
|
+
// Independent of the trio: a daemon whose FIRST pass deferred has no
|
|
4692
|
+
// counts to publish and still needs to say why.
|
|
4693
|
+
...(this.#readinessReconcileLastSweepDeferred
|
|
4694
|
+
? { discoveryDeferred: this.#readinessReconcileLastSweepDeferred }
|
|
4695
|
+
: {}),
|
|
4543
4696
|
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
4544
4697
|
...(this.#readinessReconcileLastErrorClass
|
|
4545
4698
|
? { lastErrorClass: this.#readinessReconcileLastErrorClass }
|
|
@@ -7234,23 +7387,33 @@ export class FactoryLoop {
|
|
|
7234
7387
|
}
|
|
7235
7388
|
}
|
|
7236
7389
|
}
|
|
7390
|
+
/**
|
|
7391
|
+
* Why durable dispatch state refuses this issue, or `undefined` to proceed.
|
|
7392
|
+
*
|
|
7393
|
+
* Returns the operator text *and* its #355 code together rather than letting
|
|
7394
|
+
* the sweep re-derive one from the other: these four conditions are the ones
|
|
7395
|
+
* most likely to explain a sweep that saw eligible issues and dispatched
|
|
7396
|
+
* nothing, and two of them (`terminal`, `retry-limit`) never clear on their
|
|
7397
|
+
* own. Deriving the code by matching the message would put that distinction
|
|
7398
|
+
* one rename away from collapsing into `other`.
|
|
7399
|
+
*/
|
|
7237
7400
|
async #dispatchBlockReason(issue) {
|
|
7238
7401
|
const key = issueStateKey(issue);
|
|
7239
7402
|
const state = await this.#state.getDispatchAttempts(this.#workspaceId, key);
|
|
7240
7403
|
if (!state)
|
|
7241
7404
|
return undefined;
|
|
7242
7405
|
if (state.terminal)
|
|
7243
|
-
return 'dispatch already terminal';
|
|
7406
|
+
return { reason: 'dispatch already terminal', code: 'dispatch-terminal' };
|
|
7244
7407
|
if (state.inFlight)
|
|
7245
|
-
return 'dispatch already in-flight';
|
|
7408
|
+
return { reason: 'dispatch already in-flight', code: 'dispatch-in-flight' };
|
|
7246
7409
|
const now = this.#clock.now();
|
|
7247
7410
|
if (state.backoffUntilMs > now) {
|
|
7248
|
-
return 'dispatch backoff active';
|
|
7411
|
+
return { reason: 'dispatch backoff active', code: 'dispatch-backoff' };
|
|
7249
7412
|
}
|
|
7250
7413
|
if (state.attempts >= this.#config.dispatch.maxAttempts) {
|
|
7251
7414
|
state.terminal = true;
|
|
7252
7415
|
await this.#state.recordDispatchAttempt(this.#workspaceId, key, state);
|
|
7253
|
-
return 'dispatch retry limit reached';
|
|
7416
|
+
return { reason: 'dispatch retry limit reached', code: 'dispatch-retry-limit' };
|
|
7254
7417
|
}
|
|
7255
7418
|
return undefined;
|
|
7256
7419
|
}
|
|
@@ -7362,6 +7525,9 @@ export class FactoryLoop {
|
|
|
7362
7525
|
readinessReconcile: this.#readinessReconcileStatus(),
|
|
7363
7526
|
dispatchCapacity: this.#dispatchCapacityStatus(),
|
|
7364
7527
|
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
7528
|
+
// Optional on the port: a backend with no socket omits it, and an absent
|
|
7529
|
+
// value stays absent rather than being invented as healthy.
|
|
7530
|
+
...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
|
|
7365
7531
|
};
|
|
7366
7532
|
// The deployed container serves `/healthz` straight out of this file and
|
|
7367
7533
|
// has no redaction logic of its own, so publish the already-safe view here
|
|
@@ -18180,21 +18346,28 @@ const githubIssueIndexRepoRoots = (path) => {
|
|
|
18180
18346
|
`/github/repos/${owner}__${repo}/issues`,
|
|
18181
18347
|
];
|
|
18182
18348
|
};
|
|
18183
|
-
|
|
18184
|
-
|
|
18349
|
+
/** How far to follow wrapped provider failures without trusting an unbounded chain. */
|
|
18350
|
+
const RELAYFILE_OVERLOAD_CAUSE_DEPTH = 4;
|
|
18351
|
+
const relayfileOverload = (error, depth = 0) => {
|
|
18352
|
+
if (depth > RELAYFILE_OVERLOAD_CAUSE_DEPTH)
|
|
18353
|
+
return undefined;
|
|
18354
|
+
const flat = asRecord(error);
|
|
18355
|
+
if (!flat)
|
|
18356
|
+
return undefined;
|
|
18185
18357
|
const response = asRecord(flat.response) ?? {};
|
|
18186
18358
|
const data = asRecord(flat.data) ?? asRecord(response.data) ?? {};
|
|
18187
18359
|
const details = asRecord(flat.details) ?? asRecord(data.details) ?? {};
|
|
18188
18360
|
const statusValue = flat.status ?? flat.statusCode ?? response.status ?? response.statusCode;
|
|
18189
18361
|
const status = typeof statusValue === 'number' ? statusValue : Number(statusValue);
|
|
18190
|
-
if (status
|
|
18191
|
-
|
|
18192
|
-
|
|
18193
|
-
|
|
18194
|
-
|
|
18195
|
-
|
|
18196
|
-
|
|
18197
|
-
|
|
18362
|
+
if (status === 429) {
|
|
18363
|
+
const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
|
|
18364
|
+
const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
|
|
18365
|
+
const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
|
|
18366
|
+
const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
|
|
18367
|
+
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
18368
|
+
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
18369
|
+
}
|
|
18370
|
+
return relayfileOverload(flat.cause, depth + 1);
|
|
18198
18371
|
};
|
|
18199
18372
|
/**
|
|
18200
18373
|
* relayfile's overload reason codes, allowlisted.
|
|
@@ -18713,13 +18886,14 @@ const PASS_FATAL_CAUSE_DEPTH = 4;
|
|
|
18713
18886
|
* classification has to follow the cause chain rather than trust the outermost
|
|
18714
18887
|
* type.
|
|
18715
18888
|
*/
|
|
18716
|
-
const
|
|
18889
|
+
const findWrappedErrorOfType = (error, type, depth = 0) => {
|
|
18717
18890
|
if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
|
|
18718
|
-
return
|
|
18891
|
+
return undefined;
|
|
18719
18892
|
if (error instanceof type)
|
|
18720
|
-
return
|
|
18721
|
-
return
|
|
18893
|
+
return error;
|
|
18894
|
+
return findWrappedErrorOfType(error.cause, type, depth + 1);
|
|
18722
18895
|
};
|
|
18896
|
+
const wrapsErrorOfType = (error, type) => findWrappedErrorOfType(error, type) !== undefined;
|
|
18723
18897
|
/**
|
|
18724
18898
|
* How many *unclassified* per-item failures without an intervening successful
|
|
18725
18899
|
* dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
|
|
@@ -18784,6 +18958,54 @@ const perItemDispatchSkipReason = (error) => {
|
|
|
18784
18958
|
}
|
|
18785
18959
|
return `dispatch failed (${telemetryErrorClass(error)})`;
|
|
18786
18960
|
};
|
|
18961
|
+
const UNCLASSIFIED_PHASE_CODES = {
|
|
18962
|
+
gate: 'unclassified-gate',
|
|
18963
|
+
triage: 'unclassified-triage',
|
|
18964
|
+
dispatch: 'unclassified-dispatch',
|
|
18965
|
+
};
|
|
18966
|
+
/**
|
|
18967
|
+
* Why a dispatch attempt failed, as a code from the published vocabulary (#355).
|
|
18968
|
+
*
|
|
18969
|
+
* The sibling of `perItemDispatchSkipReason`: that returns the operator's
|
|
18970
|
+
* sentence, this returns the one token that may cross onto the unauthenticated
|
|
18971
|
+
* health surface, and both are recorded at the skip site from the same thrown
|
|
18972
|
+
* value. The token is never parsed back out of the sentence — a reworded
|
|
18973
|
+
* message would silently empty a bucket, and this vocabulary is what an
|
|
18974
|
+
* operator reads when the daemon's stdout does not reach them.
|
|
18975
|
+
*
|
|
18976
|
+
* Ordered most specific first. Every branch follows the bounded cause chain,
|
|
18977
|
+
* because `contextualError` and the control-plane guard both rethrow wrapped.
|
|
18978
|
+
* `relayfileOverload` is also the loop's shedding predicate, so widening it at
|
|
18979
|
+
* the source keeps the health code, skip counter, fuse, and durable overload
|
|
18980
|
+
* ratchet on one verdict instead of merely relabelling the published bucket.
|
|
18981
|
+
*/
|
|
18982
|
+
const perItemDispatchFailureCode = (error, phase) => {
|
|
18983
|
+
if (relayfileOverload(error) !== undefined)
|
|
18984
|
+
return 'relayfile-overloaded';
|
|
18985
|
+
if (wrapsErrorOfType(error, LiveDispatchStateChangedError))
|
|
18986
|
+
return 'live-state-changed';
|
|
18987
|
+
if (wrapsErrorOfType(error, LatePlacementReleasedError))
|
|
18988
|
+
return 'late-placement-released';
|
|
18989
|
+
const refused = findWrappedErrorOfType(error, DispatchLifecycleClaimRefusedError);
|
|
18990
|
+
if (refused)
|
|
18991
|
+
return refused.refusal === 'terminal' ? 'lifecycle-terminal' : 'lifecycle-owned-elsewhere';
|
|
18992
|
+
if (wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError))
|
|
18993
|
+
return 'control-plane-open';
|
|
18994
|
+
// The class-name tail of the vocabulary; see its own doc comment for why
|
|
18995
|
+
// these five are not `instanceof`. Walked down the cause chain like the
|
|
18996
|
+
// branches above, because `contextualError` wraps in a plain `Error` and
|
|
18997
|
+
// reading only the outermost name would miss every wrapped spawn failure.
|
|
18998
|
+
// `telemetryErrorClass` is the same allowlist that guards every other
|
|
18999
|
+
// identifier leaving this process, so a hostile `name` cannot invent a key
|
|
19000
|
+
// here either — it collapses to `Error`, which the map does not hold.
|
|
19001
|
+
for (let cursor = error, depth = 0; cursor instanceof Error && depth <= PASS_FATAL_CAUSE_DEPTH; depth += 1) {
|
|
19002
|
+
const named = factoryDispatchFailureReasonCodeForErrorClass(telemetryErrorClass(cursor));
|
|
19003
|
+
if (named)
|
|
19004
|
+
return named;
|
|
19005
|
+
cursor = cursor.cause;
|
|
19006
|
+
}
|
|
19007
|
+
return UNCLASSIFIED_PHASE_CODES[phase];
|
|
19008
|
+
};
|
|
18787
19009
|
const triageEscalationQuestion = (decision, issue) => {
|
|
18788
19010
|
const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
|
|
18789
19011
|
const subject = issue?.title?.trim() || decision.issue.key;
|