@agent-relay/factory 0.1.70 → 0.1.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/diagnose.d.ts +12 -1
- package/dist/cli/diagnose.d.ts.map +1 -1
- package/dist/cli/diagnose.js +37 -1
- package/dist/cli/diagnose.js.map +1 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +3 -0
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +84 -40
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +14 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/environments/connection-registry.d.ts +20 -20
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +57 -19
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +10 -2
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +90 -14
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/mount/relayfile-operation-timeout.d.ts +97 -0
- package/dist/mount/relayfile-operation-timeout.d.ts.map +1 -0
- package/dist/mount/relayfile-operation-timeout.js +149 -0
- package/dist/mount/relayfile-operation-timeout.js.map +1 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +223 -32
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts.map +1 -1
- package/dist/orchestrator/public-health.js +75 -0
- package/dist/orchestrator/public-health.js.map +1 -1
- package/dist/orchestrator/sweep-skip-reason.d.ts +45 -0
- package/dist/orchestrator/sweep-skip-reason.d.ts.map +1 -0
- package/dist/orchestrator/sweep-skip-reason.js +82 -0
- package/dist/orchestrator/sweep-skip-reason.js.map +1 -0
- package/dist/types.d.ts +69 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
3
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
4
4
|
import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
|
|
5
|
+
import { DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS, RelayfileOperationTimeoutError, relayfileTimeoutWithPhase, withRelayfileCallDeadline, } from '../mount/relayfile-operation-timeout.js';
|
|
5
6
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
7
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
8
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
@@ -34,6 +35,7 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
|
34
35
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
35
36
|
import { telemetryErrorClass } from '../observability/error-class.js';
|
|
36
37
|
import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
|
|
38
|
+
import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
|
|
37
39
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
38
40
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
39
41
|
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
@@ -204,9 +206,25 @@ const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
|
|
|
204
206
|
const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
205
207
|
const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
|
|
206
208
|
const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
|
|
209
|
+
/** How far the cycle-level relayfile backstop sits above the transport deadline. */
|
|
210
|
+
const RELAYFILE_OPERATION_BACKSTOP_RATIO = 1.25;
|
|
207
211
|
const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
|
|
208
212
|
const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
209
213
|
const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
|
|
214
|
+
/**
|
|
215
|
+
* A relayfile fault a swallowing catch must not turn into "no result".
|
|
216
|
+
*
|
|
217
|
+
* A 429 already escaped every one of these (#297): it is a fact about the
|
|
218
|
+
* dependency, not about the one item being read, so folding it into an empty
|
|
219
|
+
* result reports a clean pass over work that was never served.
|
|
220
|
+
*
|
|
221
|
+
* A per-call timeout (#351) is the same fact in a worse costume. The read that
|
|
222
|
+
* hung will hang for the next item too, and `#githubIssuePaths` swallowing it
|
|
223
|
+
* turns a wedged dependency into a *successful* sweep that discovered zero
|
|
224
|
+
* issues — `consecutiveFailures: 0`, no `lastError`, nothing dispatched. That
|
|
225
|
+
* is the exact silence this bound exists to remove, so it escapes here too.
|
|
226
|
+
*/
|
|
227
|
+
const isPassWideRelayfileFault = (error) => Boolean(relayfileOverload(error)) || error instanceof RelayfileOperationTimeoutError;
|
|
210
228
|
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
211
229
|
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
212
230
|
/** First rung of the ladder when the 429 advertises no `Retry-After`. */
|
|
@@ -492,6 +510,14 @@ export class FactoryLoop {
|
|
|
492
510
|
#readinessReconcileInFlight;
|
|
493
511
|
#readinessReconcileIntervalMs = 60_000;
|
|
494
512
|
#readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
|
|
513
|
+
/**
|
|
514
|
+
* Deadline for ONE relayfile call (#351).
|
|
515
|
+
*
|
|
516
|
+
* `#readinessReconcileTimeoutMs` bounds the sweep; this bounds the calls
|
|
517
|
+
* inside it. Only the second one can reach a dependency call that never
|
|
518
|
+
* returns: a deadline checked between awaits never regains control to check.
|
|
519
|
+
*/
|
|
520
|
+
#relayfileOperationTimeoutMs = DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS;
|
|
495
521
|
// Set for exactly as long as a sweep is running. `state` is derived from
|
|
496
522
|
// this, so an in-flight pass can no longer masquerade as the last settled one.
|
|
497
523
|
#readinessReconcileInFlightSinceMs;
|
|
@@ -526,6 +552,19 @@ export class FactoryLoop {
|
|
|
526
552
|
#readinessReconcileLastFailureAtMs;
|
|
527
553
|
#readinessReconcileLastError;
|
|
528
554
|
#readinessReconcileLastErrorClass;
|
|
555
|
+
/**
|
|
556
|
+
* The last *completed* sweep's arithmetic (#355).
|
|
557
|
+
*
|
|
558
|
+
* Held as one record rather than four fields so it can only ever be replaced
|
|
559
|
+
* whole: publishing a `dispatched` from one pass beside a `candidates` from
|
|
560
|
+
* another would be worse than publishing neither, since the whole use of
|
|
561
|
+
* these numbers is comparing them to each other.
|
|
562
|
+
*
|
|
563
|
+
* `undefined` until a sweep completes, and never initialised to zeroes —
|
|
564
|
+
* "this daemon has not finished a sweep" and "a sweep finished and found
|
|
565
|
+
* nothing" are the two readings #355 has to tell apart.
|
|
566
|
+
*/
|
|
567
|
+
#readinessReconcileLastSweep;
|
|
529
568
|
#liveEventQueue = [];
|
|
530
569
|
#liveEventDrainScheduled = false;
|
|
531
570
|
#liveEventDrainActive = false;
|
|
@@ -652,6 +691,9 @@ export class FactoryLoop {
|
|
|
652
691
|
constructor(config, ports) {
|
|
653
692
|
this.#readOnly = ports.readOnly ?? false;
|
|
654
693
|
this.#config = config;
|
|
694
|
+
// Also read here, not only in `#startLiveSubscription`: a standalone
|
|
695
|
+
// `runOnce()` never starts the live subscription and must still be bounded.
|
|
696
|
+
this.#relayfileOperationTimeoutMs = config.liveSubscription.relayfileOperationTimeoutMs;
|
|
655
697
|
this.#mount = ports.mount;
|
|
656
698
|
// Resolved role<->state mapping. The CLI injects a name-resolved, per-team
|
|
657
699
|
// resolution via ports; fall back to one built from explicit stateIds plus
|
|
@@ -1234,6 +1276,7 @@ export class FactoryLoop {
|
|
|
1234
1276
|
// `start()` overrides skip the schema's cross-field check, so re-apply its
|
|
1235
1277
|
// floor here: a deadline under one interval would kill every pass.
|
|
1236
1278
|
this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
|
|
1279
|
+
this.#relayfileOperationTimeoutMs = options.relayfileOperationTimeoutMs;
|
|
1237
1280
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
1238
1281
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
1239
1282
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1284,7 +1327,11 @@ export class FactoryLoop {
|
|
|
1284
1327
|
const backfillStartedAtMs = this.#clock.now();
|
|
1285
1328
|
this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
|
|
1286
1329
|
try {
|
|
1287
|
-
|
|
1330
|
+
// The startup backfill is a discovery pass like any other, and on a
|
|
1331
|
+
// cold container it is the first — and for the next interval, only —
|
|
1332
|
+
// sweep whose counts exist. Leaving it unrecorded would make a daemon
|
|
1333
|
+
// that has completed a full pass still read as "never ran" (#355).
|
|
1334
|
+
this.#recordReadinessSweepOutcome(await this.runOnce());
|
|
1288
1335
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1289
1336
|
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1290
1337
|
}
|
|
@@ -1389,6 +1436,8 @@ export class FactoryLoop {
|
|
|
1389
1436
|
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
1390
1437
|
reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
|
|
1391
1438
|
reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
|
|
1439
|
+
relayfileOperationTimeoutMs: overrides.relayfileOperationTimeoutMs
|
|
1440
|
+
?? this.#config.liveSubscription.relayfileOperationTimeoutMs,
|
|
1392
1441
|
};
|
|
1393
1442
|
}
|
|
1394
1443
|
async #currentEventCursor(limit) {
|
|
@@ -1513,11 +1562,18 @@ export class FactoryLoop {
|
|
|
1513
1562
|
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1514
1563
|
this.#readinessReconcileLastError = undefined;
|
|
1515
1564
|
this.#readinessReconcileLastErrorClass = undefined;
|
|
1565
|
+
// The three integers below have gone to stdout since this loop existed,
|
|
1566
|
+
// and stdout does not reach the deployed container's operator (#355).
|
|
1567
|
+
// Publishing them is what lets a reader tell a sweep that saw eligible
|
|
1568
|
+
// work and rejected it from one that never pulled it at all.
|
|
1569
|
+
this.#recordReadinessSweepOutcome(report);
|
|
1516
1570
|
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1517
1571
|
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1518
1572
|
candidates: report.pulled.length,
|
|
1519
1573
|
dispatched: report.dispatched.length,
|
|
1520
1574
|
skipped: report.skipped.length,
|
|
1575
|
+
skipReasons: this.#readinessReconcileLastSweep?.skipReasons,
|
|
1576
|
+
discoveryDeferred: report.discoveryDeferred,
|
|
1521
1577
|
});
|
|
1522
1578
|
}
|
|
1523
1579
|
catch (error) {
|
|
@@ -2491,7 +2547,13 @@ export class FactoryLoop {
|
|
|
2491
2547
|
}
|
|
2492
2548
|
const paths = await this.#readyIssuePaths();
|
|
2493
2549
|
const orphanRecovery = issueSource === 'github'
|
|
2494
|
-
? await this.#githubOrphanRecoveryContext()
|
|
2550
|
+
? await this.#githubOrphanRecoveryContext(dryRun)
|
|
2551
|
+
: undefined;
|
|
2552
|
+
// Preserved-not-reconciled is a fact the caller needs either way, but the
|
|
2553
|
+
// two causes are not the same event: a dry run skips the context by
|
|
2554
|
+
// design, a live sweep that has none failed to build one.
|
|
2555
|
+
const orphanRecoveryDegraded = issueSource === 'github' && orphanRecovery === undefined
|
|
2556
|
+
? (dryRun ? 'dry-run' : 'context-unavailable')
|
|
2495
2557
|
: undefined;
|
|
2496
2558
|
const pulled = [];
|
|
2497
2559
|
const triaged = [];
|
|
@@ -2503,6 +2565,7 @@ export class FactoryLoop {
|
|
|
2503
2565
|
issue: entry.issue.key,
|
|
2504
2566
|
path: entry.issue.path,
|
|
2505
2567
|
reason: entry.reason,
|
|
2568
|
+
code: entry.code,
|
|
2506
2569
|
});
|
|
2507
2570
|
};
|
|
2508
2571
|
// Backstop for the skip-by-default catch below: see #292. Reset only by
|
|
@@ -2542,7 +2605,11 @@ export class FactoryLoop {
|
|
|
2542
2605
|
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2543
2606
|
sweepOverloads: this.#discoverySweepOverloads,
|
|
2544
2607
|
});
|
|
2545
|
-
recordSkip({
|
|
2608
|
+
recordSkip({
|
|
2609
|
+
issue: issueRefFromPath(path),
|
|
2610
|
+
reason: perItemDispatchSkipReason(error),
|
|
2611
|
+
code: 'read-failed',
|
|
2612
|
+
});
|
|
2546
2613
|
}
|
|
2547
2614
|
readyIssueReads += 1;
|
|
2548
2615
|
// Relayfile served this work unit's read: the dependency is shedding
|
|
@@ -2603,7 +2670,7 @@ export class FactoryLoop {
|
|
|
2603
2670
|
if (!mayRecoverGithubOrphan) {
|
|
2604
2671
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
2605
2672
|
if (dispatchBlock) {
|
|
2606
|
-
recordSkip({ issue: issueRef(issue),
|
|
2673
|
+
recordSkip({ issue: issueRef(issue), ...dispatchBlock });
|
|
2607
2674
|
continue;
|
|
2608
2675
|
}
|
|
2609
2676
|
}
|
|
@@ -2613,20 +2680,25 @@ export class FactoryLoop {
|
|
|
2613
2680
|
const recoveredOrphan = orphanResult.recovered;
|
|
2614
2681
|
const batch = await this.#batch();
|
|
2615
2682
|
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
2616
|
-
recordSkip({
|
|
2683
|
+
recordSkip({
|
|
2684
|
+
issue: issueRef(issue),
|
|
2685
|
+
reason: orphanResult.reason ?? 'already tracked',
|
|
2686
|
+
code: 'already-tracked',
|
|
2687
|
+
});
|
|
2617
2688
|
continue;
|
|
2618
2689
|
}
|
|
2619
2690
|
if (!wasReady && !recoveredOrphan) {
|
|
2620
2691
|
if (mayRecoverGithubOrphan) {
|
|
2621
2692
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
2622
2693
|
if (dispatchBlock) {
|
|
2623
|
-
recordSkip({ issue: issueRef(issue),
|
|
2694
|
+
recordSkip({ issue: issueRef(issue), ...dispatchBlock });
|
|
2624
2695
|
continue;
|
|
2625
2696
|
}
|
|
2626
2697
|
}
|
|
2627
2698
|
recordSkip({
|
|
2628
2699
|
issue: issueRef(issue),
|
|
2629
2700
|
reason: orphanResult.reason ?? 'live state is not ready-for-agent',
|
|
2701
|
+
code: 'not-ready',
|
|
2630
2702
|
});
|
|
2631
2703
|
continue;
|
|
2632
2704
|
}
|
|
@@ -2635,16 +2707,20 @@ export class FactoryLoop {
|
|
|
2635
2707
|
if (recoveredOrphan) {
|
|
2636
2708
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
2637
2709
|
if (dispatchBlock) {
|
|
2638
|
-
recordSkip({ issue: issueRef(issue),
|
|
2710
|
+
recordSkip({ issue: issueRef(issue), ...dispatchBlock });
|
|
2639
2711
|
continue;
|
|
2640
2712
|
}
|
|
2641
2713
|
}
|
|
2642
2714
|
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
2643
|
-
recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
|
|
2715
|
+
recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope', code: 'out-of-scope' });
|
|
2644
2716
|
continue;
|
|
2645
2717
|
}
|
|
2646
2718
|
if (!isDispatchableIssue(issue)) {
|
|
2647
|
-
recordSkip({
|
|
2719
|
+
recordSkip({
|
|
2720
|
+
issue: issueRef(issue),
|
|
2721
|
+
reason: 'not reconciled real Linear issue',
|
|
2722
|
+
code: 'not-dispatchable',
|
|
2723
|
+
});
|
|
2648
2724
|
continue;
|
|
2649
2725
|
}
|
|
2650
2726
|
const decision = await this.triageIssue(issue);
|
|
@@ -2657,12 +2733,17 @@ export class FactoryLoop {
|
|
|
2657
2733
|
// decays the durable overload ratchet (#297).
|
|
2658
2734
|
this.#discoverySweepProgress = true;
|
|
2659
2735
|
if (result.agents.length === 0 && !dryRun) {
|
|
2736
|
+
const code = result.hold?.kind === 'dependency-cycle'
|
|
2737
|
+
? 'dependency-cycle'
|
|
2738
|
+
: result.hold?.kind === 'dependency'
|
|
2739
|
+
? 'parked-dependency'
|
|
2740
|
+
: 'queued-or-escalated';
|
|
2660
2741
|
const reason = result.hold?.kind === 'dependency-cycle'
|
|
2661
2742
|
? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
|
|
2662
2743
|
: result.hold?.kind === 'dependency'
|
|
2663
2744
|
? `parked on dependencies: ${result.hold.blockers?.join(', ') ?? 'unresolved dependency'}`
|
|
2664
2745
|
: 'queued or escalated';
|
|
2665
|
-
recordSkip({ issue: decision.issue, reason });
|
|
2746
|
+
recordSkip({ issue: decision.issue, reason, code });
|
|
2666
2747
|
}
|
|
2667
2748
|
else {
|
|
2668
2749
|
dispatched.push(result);
|
|
@@ -2740,7 +2821,11 @@ export class FactoryLoop {
|
|
|
2740
2821
|
error: describeError(error).errorMessage,
|
|
2741
2822
|
});
|
|
2742
2823
|
}
|
|
2743
|
-
recordSkip({
|
|
2824
|
+
recordSkip({
|
|
2825
|
+
issue: issueRef(issue),
|
|
2826
|
+
reason: perItemDispatchSkipReason(error),
|
|
2827
|
+
code: 'dispatch-failed',
|
|
2828
|
+
});
|
|
2744
2829
|
continue;
|
|
2745
2830
|
}
|
|
2746
2831
|
finally {
|
|
@@ -2748,7 +2833,15 @@ export class FactoryLoop {
|
|
|
2748
2833
|
this.#reconciledGithubInProgress.delete(recoveredIdentity);
|
|
2749
2834
|
}
|
|
2750
2835
|
}
|
|
2751
|
-
report = {
|
|
2836
|
+
report = {
|
|
2837
|
+
pulled,
|
|
2838
|
+
triaged,
|
|
2839
|
+
dispatched,
|
|
2840
|
+
skipped,
|
|
2841
|
+
dryRun,
|
|
2842
|
+
slackDegraded: this.#slackDegraded,
|
|
2843
|
+
...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
|
|
2844
|
+
};
|
|
2752
2845
|
return report;
|
|
2753
2846
|
}
|
|
2754
2847
|
catch (error) {
|
|
@@ -2769,6 +2862,7 @@ export class FactoryLoop {
|
|
|
2769
2862
|
dispatched: report.dispatched.length,
|
|
2770
2863
|
skipped: report.skipped.length,
|
|
2771
2864
|
slackDegraded: report.slackDegraded ?? false,
|
|
2865
|
+
orphanRecoveryDegraded: report.orphanRecoveryDegraded,
|
|
2772
2866
|
relayfileWaitWarnings: (this.#counters.relayfileOperationWaitWarnings ?? 0) - relayfileWaitWarningsAtStart,
|
|
2773
2867
|
relayfileSlowOperations: (this.#counters.relayfileSlowOperations ?? 0) - relayfileSlowOperationsAtStart,
|
|
2774
2868
|
relayfileOperationFailures: (this.#counters.relayfileOperationFailures ?? 0) - relayfileOperationFailuresAtStart,
|
|
@@ -3039,7 +3133,7 @@ export class FactoryLoop {
|
|
|
3039
3133
|
: { available: true, highWatermark };
|
|
3040
3134
|
}
|
|
3041
3135
|
catch (error) {
|
|
3042
|
-
if (
|
|
3136
|
+
if (isPassWideRelayfileFault(error))
|
|
3043
3137
|
throw error;
|
|
3044
3138
|
this.#increment('discoveryHighWatermarkFailures');
|
|
3045
3139
|
this.#logger.warn?.('[factory] discovery high-watermark unavailable; using a full sweep without caching', {
|
|
@@ -3062,7 +3156,7 @@ export class FactoryLoop {
|
|
|
3062
3156
|
events = [...page.events].sort(compareDiscoveryEvents);
|
|
3063
3157
|
}
|
|
3064
3158
|
catch (error) {
|
|
3065
|
-
if (
|
|
3159
|
+
if (isPassWideRelayfileFault(error))
|
|
3066
3160
|
throw error;
|
|
3067
3161
|
this.#logger.warn?.('[factory] discovery change feed unavailable; refreshing tree prefixes once', {
|
|
3068
3162
|
error: describeError(error).errorMessage,
|
|
@@ -3124,7 +3218,34 @@ export class FactoryLoop {
|
|
|
3124
3218
|
}
|
|
3125
3219
|
}
|
|
3126
3220
|
}
|
|
3127
|
-
|
|
3221
|
+
/**
|
|
3222
|
+
* The safety context orphan recovery needs before it may release a leaked
|
|
3223
|
+
* `factory:in-progress` claim: who is online, which issues are actively
|
|
3224
|
+
* owned, and which durable lifecycle rows look abandoned.
|
|
3225
|
+
*
|
|
3226
|
+
* A dry run never builds it. `#reconcileOrphanedGithubInProgress` refuses to
|
|
3227
|
+
* release a claim under `dryRun` before it so much as looks at the context,
|
|
3228
|
+
* and `mayRecoverGithubOrphan` in `#performRunOnce` is itself `!dryRun` —
|
|
3229
|
+
* so the context was already provably unused on that path. Gathering it
|
|
3230
|
+
* anyway was not free: `fleet.roster()` mints this process's workspace
|
|
3231
|
+
* identity on demand, which is exactly what a read-only client refuses
|
|
3232
|
+
* (#343) and exactly what a dry run must not need. Deciding what a sweep
|
|
3233
|
+
* WOULD do must not require an identity to do it with.
|
|
3234
|
+
*
|
|
3235
|
+
* Returning `undefined` is therefore the honest answer for a dry run, not a
|
|
3236
|
+
* failure: the caller reports the sweep as `orphanRecoveryDegraded:
|
|
3237
|
+
* 'dry-run'` and preserves in-progress issues, which is what a dry run does
|
|
3238
|
+
* regardless. The failure counter and the warn below stay for the live path,
|
|
3239
|
+
* where an absent context IS a degradation.
|
|
3240
|
+
*/
|
|
3241
|
+
async #githubOrphanRecoveryContext(dryRun) {
|
|
3242
|
+
if (dryRun) {
|
|
3243
|
+
this.#increment('githubOrphanRecoveryContextSkippedDryRun');
|
|
3244
|
+
this.#logger.info?.('[factory] dry run skipped the orphan-recovery safety context; in-progress issues are preserved', {
|
|
3245
|
+
reason: 'a dry run never releases an in-progress claim, so it needs no workspace identity to build one',
|
|
3246
|
+
});
|
|
3247
|
+
return undefined;
|
|
3248
|
+
}
|
|
3128
3249
|
try {
|
|
3129
3250
|
const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
|
|
3130
3251
|
readFactoryInFlightRegistry(this.#config.loop.registryPath),
|
|
@@ -3701,7 +3822,17 @@ export class FactoryLoop {
|
|
|
3701
3822
|
progressTimer.unref?.();
|
|
3702
3823
|
}
|
|
3703
3824
|
try {
|
|
3704
|
-
|
|
3825
|
+
// The bound the 2026-08-23 wedge needed (#351). `#readinessReconcileTimeoutMs`
|
|
3826
|
+
// is 90 minutes and rejects only the WAIT — the sweep keeps its discovery
|
|
3827
|
+
// lease and the next cycle coalesces onto it — so a call that never
|
|
3828
|
+
// returns stopped dispatch for as long as the process lived. Bounding the
|
|
3829
|
+
// call instead makes the pass unwind, which releases the lease and lets
|
|
3830
|
+
// the next cycle actually start.
|
|
3831
|
+
//
|
|
3832
|
+
// Slightly above the transport's own deadline so the mount's cancellation
|
|
3833
|
+
// wins the race and reports the more precise error; this is the backstop
|
|
3834
|
+
// for mounts that cannot honour a signal.
|
|
3835
|
+
const result = await withRelayfileCallDeadline(operation, details.phase, this.#relayfileOperationBackstopMs(), fn);
|
|
3705
3836
|
const elapsedMs = this.#elapsedSince(startedAtMs);
|
|
3706
3837
|
const count = opts.count?.(result);
|
|
3707
3838
|
if (opts.logComplete) {
|
|
@@ -3758,7 +3889,11 @@ export class FactoryLoop {
|
|
|
3758
3889
|
error: describeError(error).errorMessage,
|
|
3759
3890
|
});
|
|
3760
3891
|
}
|
|
3761
|
-
|
|
3892
|
+
// The transport deadline is meant to win the race above, and the mount
|
|
3893
|
+
// does not know which phase it was serving. Without this, `lastError`
|
|
3894
|
+
// would name the call but not the context — one of many list/read sites
|
|
3895
|
+
// (codex on #354).
|
|
3896
|
+
throw relayfileTimeoutWithPhase(error, details.phase);
|
|
3762
3897
|
}
|
|
3763
3898
|
finally {
|
|
3764
3899
|
if (progressTimer) {
|
|
@@ -3766,6 +3901,19 @@ export class FactoryLoop {
|
|
|
3766
3901
|
}
|
|
3767
3902
|
}
|
|
3768
3903
|
}
|
|
3904
|
+
/**
|
|
3905
|
+
* The cycle-level backstop budget: the transport deadline plus a margin.
|
|
3906
|
+
*
|
|
3907
|
+
* Proportional rather than a flat grace so it holds at both ends of the
|
|
3908
|
+
* range — a five-minute production budget and a millisecond test budget both
|
|
3909
|
+
* leave the transport room to fire first.
|
|
3910
|
+
*/
|
|
3911
|
+
#relayfileOperationBackstopMs() {
|
|
3912
|
+
const timeoutMs = this.#relayfileOperationTimeoutMs;
|
|
3913
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
|
|
3914
|
+
return undefined;
|
|
3915
|
+
return Math.ceil(timeoutMs * RELAYFILE_OPERATION_BACKSTOP_RATIO);
|
|
3916
|
+
}
|
|
3769
3917
|
#elapsedSince(startedAtMs) {
|
|
3770
3918
|
return Math.max(0, this.#clock.now() - startedAtMs);
|
|
3771
3919
|
}
|
|
@@ -3914,7 +4062,7 @@ export class FactoryLoop {
|
|
|
3914
4062
|
}
|
|
3915
4063
|
const blockReason = await this.#dispatchBlockReason(decision.issue);
|
|
3916
4064
|
if (blockReason) {
|
|
3917
|
-
const error = new Error(`Refusing to dispatch ${decision.issue.key}: ${blockReason}`);
|
|
4065
|
+
const error = new Error(`Refusing to dispatch ${decision.issue.key}: ${blockReason.reason}`);
|
|
3918
4066
|
this.#error(error, decision.issue);
|
|
3919
4067
|
throw error;
|
|
3920
4068
|
}
|
|
@@ -4366,6 +4514,23 @@ export class FactoryLoop {
|
|
|
4366
4514
|
}
|
|
4367
4515
|
return { state: 'starting' };
|
|
4368
4516
|
}
|
|
4517
|
+
/**
|
|
4518
|
+
* Snapshot a settled sweep's counts onto the readiness surface (#355).
|
|
4519
|
+
*
|
|
4520
|
+
* Only successful passes reach here: a pass that threw has no report, and
|
|
4521
|
+
* inventing zeroes for it would publish "found nothing" for a sweep that
|
|
4522
|
+
* never got to look. The previous pass's numbers stay put instead, dated by
|
|
4523
|
+
* `lastCompletedAtMs`, which is the honest reading.
|
|
4524
|
+
*/
|
|
4525
|
+
#recordReadinessSweepOutcome(report) {
|
|
4526
|
+
this.#readinessReconcileLastSweep = {
|
|
4527
|
+
candidates: report.pulled.length,
|
|
4528
|
+
dispatched: report.dispatched.length,
|
|
4529
|
+
skipped: report.skipped.length,
|
|
4530
|
+
skipReasons: factorySweepSkipReasonCounts(report.skipped),
|
|
4531
|
+
...(report.discoveryDeferred ? { discoveryDeferred: report.discoveryDeferred } : {}),
|
|
4532
|
+
};
|
|
4533
|
+
}
|
|
4369
4534
|
#readinessReconcileStatus() {
|
|
4370
4535
|
const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
|
|
4371
4536
|
// #296 owns the numerator here, #295/#300 own the derivation. The earliest
|
|
@@ -4440,6 +4605,22 @@ export class FactoryLoop {
|
|
|
4440
4605
|
...(this.#readinessReconcileLastFailureAtMs !== undefined
|
|
4441
4606
|
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
4442
4607
|
: {}),
|
|
4608
|
+
// Spread whole or not at all: see `#readinessReconcileLastSweep`. Zeroes
|
|
4609
|
+
// are published, which is the entire point — an absent `candidates` and
|
|
4610
|
+
// a zero `candidates` are different diagnoses (#355).
|
|
4611
|
+
...(this.#readinessReconcileLastSweep
|
|
4612
|
+
? {
|
|
4613
|
+
candidates: this.#readinessReconcileLastSweep.candidates,
|
|
4614
|
+
dispatched: this.#readinessReconcileLastSweep.dispatched,
|
|
4615
|
+
skipped: this.#readinessReconcileLastSweep.skipped,
|
|
4616
|
+
...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
|
|
4617
|
+
? { skipReasons: { ...this.#readinessReconcileLastSweep.skipReasons } }
|
|
4618
|
+
: {}),
|
|
4619
|
+
...(this.#readinessReconcileLastSweep.discoveryDeferred
|
|
4620
|
+
? { discoveryDeferred: this.#readinessReconcileLastSweep.discoveryDeferred }
|
|
4621
|
+
: {}),
|
|
4622
|
+
}
|
|
4623
|
+
: {}),
|
|
4443
4624
|
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
4444
4625
|
...(this.#readinessReconcileLastErrorClass
|
|
4445
4626
|
? { lastErrorClass: this.#readinessReconcileLastErrorClass }
|
|
@@ -6571,7 +6752,7 @@ export class FactoryLoop {
|
|
|
6571
6752
|
return [...issuePaths.values()].sort();
|
|
6572
6753
|
}
|
|
6573
6754
|
catch (error) {
|
|
6574
|
-
if (
|
|
6755
|
+
if (isPassWideRelayfileFault(error))
|
|
6575
6756
|
throw error;
|
|
6576
6757
|
this.#githubIssuePathIndexReady = false;
|
|
6577
6758
|
this.#increment('githubIssueListFailures');
|
|
@@ -6587,7 +6768,7 @@ export class FactoryLoop {
|
|
|
6587
6768
|
parsed = parseJsonContent(content);
|
|
6588
6769
|
}
|
|
6589
6770
|
catch (error) {
|
|
6590
|
-
if (
|
|
6771
|
+
if (isPassWideRelayfileFault(error))
|
|
6591
6772
|
throw error;
|
|
6592
6773
|
return undefined;
|
|
6593
6774
|
}
|
|
@@ -6684,7 +6865,7 @@ export class FactoryLoop {
|
|
|
6684
6865
|
this.#increment('githubIssueMirrorsCreated');
|
|
6685
6866
|
}
|
|
6686
6867
|
catch (error) {
|
|
6687
|
-
if (
|
|
6868
|
+
if (isPassWideRelayfileFault(error))
|
|
6688
6869
|
throw error;
|
|
6689
6870
|
this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
|
|
6690
6871
|
}
|
|
@@ -7134,23 +7315,33 @@ export class FactoryLoop {
|
|
|
7134
7315
|
}
|
|
7135
7316
|
}
|
|
7136
7317
|
}
|
|
7318
|
+
/**
|
|
7319
|
+
* Why durable dispatch state refuses this issue, or `undefined` to proceed.
|
|
7320
|
+
*
|
|
7321
|
+
* Returns the operator text *and* its #355 code together rather than letting
|
|
7322
|
+
* the sweep re-derive one from the other: these four conditions are the ones
|
|
7323
|
+
* most likely to explain a sweep that saw eligible issues and dispatched
|
|
7324
|
+
* nothing, and two of them (`terminal`, `retry-limit`) never clear on their
|
|
7325
|
+
* own. Deriving the code by matching the message would put that distinction
|
|
7326
|
+
* one rename away from collapsing into `other`.
|
|
7327
|
+
*/
|
|
7137
7328
|
async #dispatchBlockReason(issue) {
|
|
7138
7329
|
const key = issueStateKey(issue);
|
|
7139
7330
|
const state = await this.#state.getDispatchAttempts(this.#workspaceId, key);
|
|
7140
7331
|
if (!state)
|
|
7141
7332
|
return undefined;
|
|
7142
7333
|
if (state.terminal)
|
|
7143
|
-
return 'dispatch already terminal';
|
|
7334
|
+
return { reason: 'dispatch already terminal', code: 'dispatch-terminal' };
|
|
7144
7335
|
if (state.inFlight)
|
|
7145
|
-
return 'dispatch already in-flight';
|
|
7336
|
+
return { reason: 'dispatch already in-flight', code: 'dispatch-in-flight' };
|
|
7146
7337
|
const now = this.#clock.now();
|
|
7147
7338
|
if (state.backoffUntilMs > now) {
|
|
7148
|
-
return 'dispatch backoff active';
|
|
7339
|
+
return { reason: 'dispatch backoff active', code: 'dispatch-backoff' };
|
|
7149
7340
|
}
|
|
7150
7341
|
if (state.attempts >= this.#config.dispatch.maxAttempts) {
|
|
7151
7342
|
state.terminal = true;
|
|
7152
7343
|
await this.#state.recordDispatchAttempt(this.#workspaceId, key, state);
|
|
7153
|
-
return 'dispatch retry limit reached';
|
|
7344
|
+
return { reason: 'dispatch retry limit reached', code: 'dispatch-retry-limit' };
|
|
7154
7345
|
}
|
|
7155
7346
|
return undefined;
|
|
7156
7347
|
}
|
|
@@ -7488,7 +7679,7 @@ export class FactoryLoop {
|
|
|
7488
7679
|
return resolvedIssue;
|
|
7489
7680
|
}
|
|
7490
7681
|
catch (error) {
|
|
7491
|
-
if (
|
|
7682
|
+
if (isPassWideRelayfileFault(error))
|
|
7492
7683
|
throw error;
|
|
7493
7684
|
if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
|
|
7494
7685
|
this.#increment('phantomSkipped');
|
|
@@ -8584,7 +8775,7 @@ export class FactoryLoop {
|
|
|
8584
8775
|
paths = await this.#listRelayfileTree(root, 'exact-head PR confirmation');
|
|
8585
8776
|
}
|
|
8586
8777
|
catch (error) {
|
|
8587
|
-
if (
|
|
8778
|
+
if (isPassWideRelayfileFault(error))
|
|
8588
8779
|
throw error;
|
|
8589
8780
|
continue;
|
|
8590
8781
|
}
|
|
@@ -8854,7 +9045,7 @@ export class FactoryLoop {
|
|
|
8854
9045
|
return await this.#listRelayfileTree(root, 'published PR confirmation');
|
|
8855
9046
|
}
|
|
8856
9047
|
catch (error) {
|
|
8857
|
-
if (
|
|
9048
|
+
if (isPassWideRelayfileFault(error))
|
|
8858
9049
|
throw error;
|
|
8859
9050
|
return [];
|
|
8860
9051
|
}
|
|
@@ -10436,7 +10627,7 @@ export class FactoryLoop {
|
|
|
10436
10627
|
}
|
|
10437
10628
|
}
|
|
10438
10629
|
catch (error) {
|
|
10439
|
-
if (
|
|
10630
|
+
if (isPassWideRelayfileFault(error))
|
|
10440
10631
|
throw error;
|
|
10441
10632
|
this.#logger.warn?.('[factory] unable to list GitHub issue comments for replay', { prefix, error });
|
|
10442
10633
|
}
|
|
@@ -12982,7 +13173,7 @@ export class FactoryLoop {
|
|
|
12982
13173
|
found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
|
|
12983
13174
|
}
|
|
12984
13175
|
catch (error) {
|
|
12985
|
-
if (
|
|
13176
|
+
if (isPassWideRelayfileFault(error))
|
|
12986
13177
|
throw error;
|
|
12987
13178
|
// try the next root
|
|
12988
13179
|
}
|
|
@@ -14161,7 +14352,7 @@ export class FactoryLoop {
|
|
|
14161
14352
|
return await this.#listRelayfileTree(prefix, 'Slack identity lookup');
|
|
14162
14353
|
}
|
|
14163
14354
|
catch (error) {
|
|
14164
|
-
if (
|
|
14355
|
+
if (isPassWideRelayfileFault(error))
|
|
14165
14356
|
throw error;
|
|
14166
14357
|
return [];
|
|
14167
14358
|
}
|
|
@@ -17088,7 +17279,7 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
17088
17279
|
paths.add(path);
|
|
17089
17280
|
}
|
|
17090
17281
|
catch (error) {
|
|
17091
|
-
if (
|
|
17282
|
+
if (isPassWideRelayfileFault(error))
|
|
17092
17283
|
throw error;
|
|
17093
17284
|
listErrors.push(error);
|
|
17094
17285
|
}
|