@agent-relay/factory 0.1.75 → 0.1.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/schema.d.ts +110 -2
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +61 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/orchestrator/batch-tracker.d.ts +9 -0
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +795 -92
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/sweep-budget.d.ts +137 -0
- package/dist/orchestrator/sweep-budget.d.ts.map +1 -0
- package/dist/orchestrator/sweep-budget.js +208 -0
- package/dist/orchestrator/sweep-budget.js.map +1 -0
- package/dist/ports/state.d.ts +9 -1
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +1 -1
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +2 -2
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/types.d.ts +7 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
3
|
import { readFile } from 'node:fs/promises';
|
|
4
4
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
5
|
-
import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
|
|
5
|
+
import { DEFAULT_DISCOVERY_SWEEP_BUDGET_MS, DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema, resolvedSweepBudgetMs, } from '../config/schema.js';
|
|
6
6
|
import { DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS, RelayfileOperationTimeoutError, relayfileTimeoutWithPhase, withRelayfileCallDeadline, } from '../mount/relayfile-operation-timeout.js';
|
|
7
7
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
8
8
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
@@ -11,8 +11,9 @@ import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftNam
|
|
|
11
11
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
12
12
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
13
13
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
14
|
+
import { DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS, DiscoverySweepBudgetExceededError, startDiscoverySweepBudget, withSweepTeardownDeadline, } from './sweep-budget.js';
|
|
14
15
|
import { dispatchHandedOffToBabysitters, dispatchLifecycleOccupiesSlot, dispatchPhaseOccupiesSlot, } from '../state/dispatch-lifecycle-slot.js';
|
|
15
|
-
import { branchImplementsIssue, containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue, prBodyDisclaimsClosing, prClosureAuthority } from '../issue-key-match.js';
|
|
16
|
+
import { ISSUE_KEY_PARTS, branchImplementsIssue, containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue, prBodyDisclaimsClosing, prClosureAuthority } from '../issue-key-match.js';
|
|
16
17
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
17
18
|
import { isInFactoryScope } from '../safety/factory-scope.js';
|
|
18
19
|
import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
|
|
@@ -479,6 +480,10 @@ export class FactoryLoop {
|
|
|
479
480
|
// owner/repo#number identity. GitHub-native records outrank Linear mirrors.
|
|
480
481
|
#dependencyIssues = new Map();
|
|
481
482
|
#terminalDependencyIdentities = new Set();
|
|
483
|
+
/** Sweep-scoped memo of `#dependencyIsTerminalOrMerged`'s mount walk, including
|
|
484
|
+
* the negative answer that `#terminalDependencyIdentities` cannot hold. Cleared
|
|
485
|
+
* with it at the top of every sweep. */
|
|
486
|
+
#dependencyPrProbes = new Map();
|
|
482
487
|
#dependencyParkNotices = new Map();
|
|
483
488
|
#dependencyGithubPathsByIdentity;
|
|
484
489
|
#dependencyLinearTreeLoaded = false;
|
|
@@ -557,6 +562,31 @@ export class FactoryLoop {
|
|
|
557
562
|
* returns: a deadline checked between awaits never regains control to check.
|
|
558
563
|
*/
|
|
559
564
|
#relayfileOperationTimeoutMs = DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS;
|
|
565
|
+
/**
|
|
566
|
+
* Aggregate budget for ONE sweep (#372).
|
|
567
|
+
*
|
|
568
|
+
* The third bound on this path and the only transport-agnostic one.
|
|
569
|
+
* `#relayfileOperationTimeoutMs` bounds one relayfile call and cannot see the
|
|
570
|
+
* retry loop around it; `#readinessReconcileTimeoutMs` bounds the *wait* and
|
|
571
|
+
* leaves `runOnce()` running for the next cycle to coalesce onto. This is
|
|
572
|
+
* charged against one timer for the whole pass, so however many calls,
|
|
573
|
+
* retries and transports the sweep is spread across, it cannot outlive this.
|
|
574
|
+
*
|
|
575
|
+
* Held at `min(configured, #readinessReconcileTimeoutMs)`: a budget looser
|
|
576
|
+
* than the wait would let the wait give up first, which is the behaviour it
|
|
577
|
+
* exists to remove.
|
|
578
|
+
*/
|
|
579
|
+
#discoverySweepBudgetMs = DEFAULT_DISCOVERY_SWEEP_BUDGET_MS;
|
|
580
|
+
/**
|
|
581
|
+
* The budgets of every sweep currently in flight.
|
|
582
|
+
*
|
|
583
|
+
* `stop()` drains the sweep it started (see `#readinessReconcileAbandonedWait`
|
|
584
|
+
* below), so a wedged sweep holds shutdown open for the whole budget — 90
|
|
585
|
+
* minutes at the default, and unbounded before this budget existed. A set
|
|
586
|
+
* rather than one field because a live sweep and a mismatched dry-run one can
|
|
587
|
+
* be in flight at the same time.
|
|
588
|
+
*/
|
|
589
|
+
#discoverySweepBudgets = new Set();
|
|
560
590
|
// Set for exactly as long as a sweep is running. `state` is derived from
|
|
561
591
|
// this, so an in-flight pass can no longer masquerade as the last settled one.
|
|
562
592
|
#readinessReconcileInFlightSinceMs;
|
|
@@ -772,6 +802,7 @@ export class FactoryLoop {
|
|
|
772
802
|
// Also read here, not only in `#startLiveSubscription`: a standalone
|
|
773
803
|
// `runOnce()` never starts the live subscription and must still be bounded.
|
|
774
804
|
this.#relayfileOperationTimeoutMs = config.liveSubscription.relayfileOperationTimeoutMs;
|
|
805
|
+
this.#discoverySweepBudgetMs = resolvedSweepBudgetMs(config.liveSubscription.sweepBudgetMs, config.liveSubscription.reconcileTimeoutMs);
|
|
775
806
|
this.#mount = ports.mount;
|
|
776
807
|
// Resolved role<->state mapping. The CLI injects a name-resolved, per-team
|
|
777
808
|
// resolution via ports; fall back to one built from explicit stateIds plus
|
|
@@ -806,7 +837,8 @@ export class FactoryLoop {
|
|
|
806
837
|
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
807
838
|
this.#hasProbePrGhRunner = Boolean(ports.probePrGhRunner);
|
|
808
839
|
this.#probePrGhRunner = ports.probePrGhRunner ?? failClosedGhRunner;
|
|
809
|
-
this.#probePrResolver = ports.probePrResolver ??
|
|
840
|
+
this.#probePrResolver = ports.probePrResolver ??
|
|
841
|
+
((issue) => this.#resolveIssuePr(issue, { repo: this.#probeRepoForIssue(issue) }));
|
|
810
842
|
this.#logger = normalizeLogger(ports.logger ?? console);
|
|
811
843
|
this.#clock = ports.clock ?? realClock;
|
|
812
844
|
this.#fleetControlPlane = new FleetControlPlaneCircuit({
|
|
@@ -1202,31 +1234,50 @@ export class FactoryLoop {
|
|
|
1202
1234
|
clearTimeout(this.#heldAgentDeadlineTimer);
|
|
1203
1235
|
this.#heldAgentDeadlineTimer = undefined;
|
|
1204
1236
|
this.#heldAgentDeadlineDueAtMs = undefined;
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1237
|
+
// #372: the drain below is unbounded in the one case that matters — a
|
|
1238
|
+
// wedged sweep — so shutdown inherits the sweep budget, 90 minutes at the
|
|
1239
|
+
// default. Spending it after one teardown window bounds shutdown without
|
|
1240
|
+
// discarding a sweep that was about to finish.
|
|
1241
|
+
//
|
|
1242
|
+
// Armed BEFORE the first shutdown await, not just before the sweep drain
|
|
1243
|
+
// (cubic-dev-ai, #374 review). The grace is a timer, so arming it costs
|
|
1244
|
+
// nothing and starts the clock at the moment shutdown starts; arming it
|
|
1245
|
+
// after `#heldAgentDeadlineSweepInFlight` made the lever's start depend on
|
|
1246
|
+
// an UNRELATED in-flight sweep finishing first, so a slow held-agent pass
|
|
1247
|
+
// simply added its own latency to the wedged discovery sweep's reprieve —
|
|
1248
|
+
// in the limit the lever never arms at all, which is the bound this whole
|
|
1249
|
+
// change exists to provide.
|
|
1250
|
+
const releaseSweepBudgetGrace = this.#cutSweepBudgetsShortForStop();
|
|
1251
|
+
try {
|
|
1252
|
+
await this.#heldAgentDeadlineSweepInFlight;
|
|
1253
|
+
for (const timer of this.#dispatchLifecycleRetryTimers.values())
|
|
1254
|
+
clearTimeout(timer);
|
|
1255
|
+
this.#dispatchLifecycleRetryTimers.clear();
|
|
1256
|
+
this.#abandonedDispatchReasons.clear();
|
|
1257
|
+
this.#dispatchLifecycleCapacityWaits.clear();
|
|
1258
|
+
this.#dispatchLifecycleOwnershipWaitLogged.clear();
|
|
1259
|
+
if (this.#completionSweepTimer)
|
|
1260
|
+
clearTimeout(this.#completionSweepTimer);
|
|
1261
|
+
this.#completionSweepTimer = undefined;
|
|
1262
|
+
if (this.#readinessReconcileTimer)
|
|
1263
|
+
clearTimeout(this.#readinessReconcileTimer);
|
|
1264
|
+
this.#readinessReconcileTimer = undefined;
|
|
1265
|
+
if (this.#previewSweepTimer)
|
|
1266
|
+
clearTimeout(this.#previewSweepTimer);
|
|
1267
|
+
this.#previewSweepTimer = undefined;
|
|
1268
|
+
await this.#readinessReconcileInFlight;
|
|
1269
|
+
// #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
|
|
1270
|
+
// can settle with its `runOnce()` still live. Shutdown releases dispatch
|
|
1271
|
+
// lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
|
|
1272
|
+
// fences a stopping sweep once something in it throws — so a sweep whose
|
|
1273
|
+
// dependency recovers cleanly would otherwise dispatch through torn-down
|
|
1274
|
+
// state. Draining here restores exactly the pre-deadline shutdown contract:
|
|
1275
|
+
// stop() outlives the sweep it started.
|
|
1276
|
+
await this.#readinessReconcileAbandonedWait;
|
|
1277
|
+
}
|
|
1278
|
+
finally {
|
|
1279
|
+
releaseSweepBudgetGrace();
|
|
1280
|
+
}
|
|
1230
1281
|
await this.#previewSweepInFlight;
|
|
1231
1282
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
1232
1283
|
try {
|
|
@@ -1613,6 +1664,10 @@ export class FactoryLoop {
|
|
|
1613
1664
|
// floor here: a deadline under one interval would kill every pass.
|
|
1614
1665
|
this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
|
|
1615
1666
|
this.#relayfileOperationTimeoutMs = options.relayfileOperationTimeoutMs;
|
|
1667
|
+
// Same re-application as the deadline above: `start()` overrides bypass the
|
|
1668
|
+
// schema's cross-field check, and a budget looser than the wait is not a
|
|
1669
|
+
// budget.
|
|
1670
|
+
this.#discoverySweepBudgetMs = resolvedSweepBudgetMs(options.sweepBudgetMs, this.#readinessReconcileTimeoutMs);
|
|
1616
1671
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
1617
1672
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
1618
1673
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1776,6 +1831,7 @@ export class FactoryLoop {
|
|
|
1776
1831
|
reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
|
|
1777
1832
|
relayfileOperationTimeoutMs: overrides.relayfileOperationTimeoutMs
|
|
1778
1833
|
?? this.#config.liveSubscription.relayfileOperationTimeoutMs,
|
|
1834
|
+
sweepBudgetMs: overrides.sweepBudgetMs ?? this.#config.liveSubscription.sweepBudgetMs,
|
|
1779
1835
|
};
|
|
1780
1836
|
}
|
|
1781
1837
|
async #currentEventCursor(limit) {
|
|
@@ -2304,7 +2360,11 @@ export class FactoryLoop {
|
|
|
2304
2360
|
}
|
|
2305
2361
|
if (pr.draft) {
|
|
2306
2362
|
this.#increment('completionSweepDraftPr');
|
|
2307
|
-
|
|
2363
|
+
// Must match the key `#completionPrForIssue` resolves under —
|
|
2364
|
+
// same options, same repo scope — or this backoff is written
|
|
2365
|
+
// under a name nothing reads and the draft PR is re-fetched from
|
|
2366
|
+
// gh on every pass.
|
|
2367
|
+
this.#probePrGhBackoffUntilMs.set(this.#probePrCacheKey(issue, { repo: this.#probeRepoForIssue(issue) }), this.#clock.now() + PROBE_PR_GH_BACKOFF_MS);
|
|
2308
2368
|
return undefined;
|
|
2309
2369
|
}
|
|
2310
2370
|
if (record.decision.implementers.length > 1 && !await this.#allImplementersHaveCompletionPr(record)) {
|
|
@@ -2545,6 +2605,7 @@ export class FactoryLoop {
|
|
|
2545
2605
|
}
|
|
2546
2606
|
return this.#resolveIssuePr(issue, {
|
|
2547
2607
|
titleMarker: FACTORY_E2E_MARKER,
|
|
2608
|
+
repo: this.#probeRepoForIssue(issue),
|
|
2548
2609
|
});
|
|
2549
2610
|
}
|
|
2550
2611
|
async #openPrForIssue(issue) {
|
|
@@ -2554,18 +2615,82 @@ export class FactoryLoop {
|
|
|
2554
2615
|
return this.#resolveIssuePr(issue, {
|
|
2555
2616
|
titleMarker: FACTORY_E2E_MARKER,
|
|
2556
2617
|
openOnly: true,
|
|
2618
|
+
repo: this.#probeRepoForIssue(issue),
|
|
2557
2619
|
});
|
|
2558
2620
|
}
|
|
2559
|
-
|
|
2621
|
+
/**
|
|
2622
|
+
* Which repository a probe walk may scope itself to.
|
|
2623
|
+
*
|
|
2624
|
+
* `resolveIssuePrFromMount` has always accepted `opts.repo`, but
|
|
2625
|
+
* `#resolveIssuePr` had no way to express one, so every probe crawled the PR
|
|
2626
|
+
* tree of EVERY configured repository — 21 of them in the live workspace — to
|
|
2627
|
+
* find a PR that can only ever live in one. This is the same routing answer
|
|
2628
|
+
* `#dependencyIsTerminalOrMerged` already uses for its own probe, so the two
|
|
2629
|
+
* probes now scope identically.
|
|
2630
|
+
*
|
|
2631
|
+
* `undefined` means the routing is genuinely ambiguous (a decision spanning
|
|
2632
|
+
* several routes, an issue whose labels match no single repo). The walk then
|
|
2633
|
+
* stays unscoped, exactly as before: narrowing on a guess would silently fail
|
|
2634
|
+
* to find a PR that is really there, which is worse than a slow walk.
|
|
2635
|
+
*
|
|
2636
|
+
* `allowDefault: false` is the whole point of this wrapper. Routing precedence
|
|
2637
|
+
* is byLabel, byProject, keywordRules, default, and `dependencyRepoForIssue`
|
|
2638
|
+
* can see neither the triage decision nor `keywordRules`. Left to its own
|
|
2639
|
+
* fallback it answers `repos.default` for every keyword-routed issue, while
|
|
2640
|
+
* dispatch opened that PR in the keyword-selected repository — so the probe
|
|
2641
|
+
* would walk one repository, confidently, and find nothing. Reporting "no PR"
|
|
2642
|
+
* for an issue that has one is a correctness bug; the unscoped walk it falls
|
|
2643
|
+
* back to instead is merely slow, and the dedupe and cache in this same change
|
|
2644
|
+
* already blunt that cost.
|
|
2645
|
+
*/
|
|
2646
|
+
#probeRepoForIssue(issue) {
|
|
2647
|
+
return dependencyRepoForIssue(issue, undefined, this.#config, { allowDefault: false });
|
|
2648
|
+
}
|
|
2649
|
+
/**
|
|
2650
|
+
* The key for BOTH `#probePrResolvedCache` and `#probePrGhBackoffUntilMs`.
|
|
2651
|
+
*
|
|
2652
|
+
* Shared rather than inlined because those two maps are written from more than
|
|
2653
|
+
* one place: `#resolveIssuePr` writes both, and the completion sweep writes a
|
|
2654
|
+
* draft-PR backoff directly. Those writes must agree on the key or the backoff
|
|
2655
|
+
* is set under a name nothing reads, which silently costs a gh call per pass.
|
|
2656
|
+
*
|
|
2657
|
+
* `repo` is part of the key because it narrows which pull requests the
|
|
2658
|
+
* resolution can even see — an issue whose route changes must not be served
|
|
2659
|
+
* the previous repository's PR, or held off gh by the previous repository's
|
|
2660
|
+
* negative backoff, because the completion path probes and CLOSES what this
|
|
2661
|
+
* returns. `*` marks the unscoped walk, a genuinely different resolution that
|
|
2662
|
+
* must not share an entry with any single-repo one.
|
|
2663
|
+
*
|
|
2664
|
+
* Every dimension is a trailing `:`-prefixed segment so the completion
|
|
2665
|
+
* invalidation — which clears `stateKey` plus everything starting
|
|
2666
|
+
* `${stateKey}:` — keeps clearing the whole family as the key grows.
|
|
2667
|
+
*/
|
|
2668
|
+
#probePrCacheKey(issue, opts = {}) {
|
|
2560
2669
|
const issueKey = issueStateKey(issueRef(issue));
|
|
2561
|
-
const
|
|
2670
|
+
const repoScope = opts.repo ? opts.repo.trim().toLowerCase() : '*';
|
|
2671
|
+
return `${opts.openOnly ? `${issueKey}:open` : issueKey}${opts.allowLegacyGithubBranch ? ':legacy' : ''}:repo=${repoScope}`;
|
|
2672
|
+
}
|
|
2673
|
+
async #resolveIssuePr(issue, opts = {}) {
|
|
2674
|
+
const key = this.#probePrCacheKey(issue, opts);
|
|
2562
2675
|
const now = this.#clock.now();
|
|
2563
2676
|
const cached = this.#probePrResolvedCache.get(key);
|
|
2564
2677
|
if (cached && cached.expiresAtMs > now) {
|
|
2565
2678
|
return cached.pr;
|
|
2566
2679
|
}
|
|
2567
|
-
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts, (prefix) => this.#listRelayfileTree(prefix, 'PR probe resolution'));
|
|
2680
|
+
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts, (prefix) => this.#listRelayfileTree(prefix, 'PR probe resolution'), this.#probeMountWalkProgress('[factory] PR probe mount read progress', issue));
|
|
2568
2681
|
if (mountPr) {
|
|
2682
|
+
// The mount branch runs first and is the common hit, and until now it was
|
|
2683
|
+
// the one branch that never wrote the cache it reads at the top of this
|
|
2684
|
+
// method. The cache had a reader and no writer on the hot path, so the
|
|
2685
|
+
// full tree walk repeated for every caller, on every sweep, forever.
|
|
2686
|
+
//
|
|
2687
|
+
// Cached on the same terms as the gh branch below — same key, same TTL,
|
|
2688
|
+
// same draft exclusion — because the reason to keep a draft uncached is a
|
|
2689
|
+
// property of the PR (its state is about to flip and the caller wants to
|
|
2690
|
+
// see that promptly), not of which resolver observed it.
|
|
2691
|
+
if (!mountPr.draft) {
|
|
2692
|
+
this.#probePrResolvedCache.set(key, { pr: mountPr, expiresAtMs: now + PROBE_PR_GH_BACKOFF_MS });
|
|
2693
|
+
}
|
|
2569
2694
|
return mountPr;
|
|
2570
2695
|
}
|
|
2571
2696
|
const backoffUntil = this.#probePrGhBackoffUntilMs.get(key) ?? 0;
|
|
@@ -2638,12 +2763,131 @@ export class FactoryLoop {
|
|
|
2638
2763
|
throw contextualError('Factory dispatch paused because the fleet control plane is unavailable', error);
|
|
2639
2764
|
}
|
|
2640
2765
|
}
|
|
2766
|
+
/**
|
|
2767
|
+
* One sweep, under one aggregate budget (#372).
|
|
2768
|
+
*
|
|
2769
|
+
* The budget wraps the fence rather than the fence's caller, and that is the
|
|
2770
|
+
* whole point. #296's deadline lives in `#runOnceWithReadinessDeadline`,
|
|
2771
|
+
* outside `runOnce()`: expiry rejects the wait and leaves the sweep running,
|
|
2772
|
+
* so the next cycle coalesces onto the same wedged promise (`runOnce()`, the
|
|
2773
|
+
* `#runOnceInFlight` branch) and the daemon never recovers. Expiring in HERE
|
|
2774
|
+
* unwinds the body below, which releases the discovery lease on its way out
|
|
2775
|
+
* and lets `runOnce()` clear `#runOnceInFlight` — so the next cycle claims a
|
|
2776
|
+
* fresh lease and runs clean.
|
|
2777
|
+
*
|
|
2778
|
+
* See `sweep-budget.ts` for what the mechanism can and cannot interrupt. In
|
|
2779
|
+
* short: it abandons the in-flight await, it does not cancel the call.
|
|
2780
|
+
*/
|
|
2641
2781
|
async #runOnceWithDiscoveryFence(opts) {
|
|
2782
|
+
const budget = startDiscoverySweepBudget(this.#discoverySweepBudgetMs);
|
|
2783
|
+
this.#discoverySweepBudgets.add(budget);
|
|
2784
|
+
const sweepStartedAtMs = this.#clock.now();
|
|
2785
|
+
// A sweep that starts while shutdown is already draining would otherwise
|
|
2786
|
+
// get a full fresh budget to hold `stop()` open with.
|
|
2787
|
+
if (this.#stopping)
|
|
2788
|
+
budget.expire();
|
|
2789
|
+
try {
|
|
2790
|
+
return await this.#runDiscoverySweep(opts, budget);
|
|
2791
|
+
}
|
|
2792
|
+
catch (error) {
|
|
2793
|
+
if (error instanceof DiscoverySweepBudgetExceededError) {
|
|
2794
|
+
this.#increment('discoverySweepBudgetExceeded');
|
|
2795
|
+
this.#logger.error?.('[factory] discovery sweep aborted at its aggregate budget', {
|
|
2796
|
+
budgetMs: error.budgetMs,
|
|
2797
|
+
// The await the sweep was abandoned on. The one diagnostic no
|
|
2798
|
+
// per-call bound can produce once the sweep is already wedged: it
|
|
2799
|
+
// says WHICH transport this wedge is on without anyone having to
|
|
2800
|
+
// guess which layer to bound next.
|
|
2801
|
+
phase: error.phase,
|
|
2802
|
+
elapsedMs: this.#elapsedSince(sweepStartedAtMs),
|
|
2803
|
+
});
|
|
2804
|
+
}
|
|
2805
|
+
throw error;
|
|
2806
|
+
}
|
|
2807
|
+
finally {
|
|
2808
|
+
this.#discoverySweepBudgets.delete(budget);
|
|
2809
|
+
budget.dispose();
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
/**
|
|
2813
|
+
* Cut every in-flight sweep's budget short once shutdown starts draining.
|
|
2814
|
+
*
|
|
2815
|
+
* `stop()` deliberately outlives the sweep it started (#301), so a wedged
|
|
2816
|
+
* sweep makes shutdown as long as the budget. The grace window is what keeps
|
|
2817
|
+
* an ordinary restart from throwing away a sweep that was about to commit;
|
|
2818
|
+
* after it, expiry routes the sweep into exactly the abort path a real expiry
|
|
2819
|
+
* takes — lease released, teardown bounded — instead of holding the process.
|
|
2820
|
+
*
|
|
2821
|
+
* Returns the cancel for the grace timer. Always call it: on a prompt
|
|
2822
|
+
* shutdown there is nothing to cut short.
|
|
2823
|
+
*/
|
|
2824
|
+
#cutSweepBudgetsShortForStop() {
|
|
2825
|
+
if (this.#discoverySweepBudgets.size === 0)
|
|
2826
|
+
return () => undefined;
|
|
2827
|
+
const timer = setTimeout(() => {
|
|
2828
|
+
for (const budget of this.#discoverySweepBudgets) {
|
|
2829
|
+
if (budget.expired())
|
|
2830
|
+
continue;
|
|
2831
|
+
this.#increment('discoverySweepBudgetsCutShortForStop');
|
|
2832
|
+
this.#logger.warn?.('[factory] shutdown is draining a sweep; spending its budget now', {
|
|
2833
|
+
graceMs: STOP_TEARDOWN_TIMEOUT_MS,
|
|
2834
|
+
budgetMs: budget.budgetMs,
|
|
2835
|
+
});
|
|
2836
|
+
budget.expire();
|
|
2837
|
+
}
|
|
2838
|
+
}, STOP_TEARDOWN_TIMEOUT_MS);
|
|
2839
|
+
timer.unref?.();
|
|
2840
|
+
return () => clearTimeout(timer);
|
|
2841
|
+
}
|
|
2842
|
+
/**
|
|
2843
|
+
* Claim the sweep lease under the budget, compensating for a claim that
|
|
2844
|
+
* lands after we stopped waiting for it.
|
|
2845
|
+
*
|
|
2846
|
+
* The budget abandons the wait, not the call, so the store can still persist
|
|
2847
|
+
* a lease for a claim this sweep has already given up on — and that lease
|
|
2848
|
+
* would be held by an owner that will never renew, commit or release it, so
|
|
2849
|
+
* every later sweep defers until it expires. It is self-expiring and a later
|
|
2850
|
+
* sweep reclaims it as an orphan, which makes this a latency fix rather than
|
|
2851
|
+
* a correctness one; the latency is one whole lease window with no discovery,
|
|
2852
|
+
* which is the thing this PR exists to stop paying.
|
|
2853
|
+
*/
|
|
2854
|
+
async #claimDiscoverySweepUnderBudget(budget) {
|
|
2855
|
+
// Issued INSIDE the budget callback, so a budget that is already spent
|
|
2856
|
+
// rejects the phase without opening a lease it could only hand straight
|
|
2857
|
+
// back. The handle is kept out here because the compensation below needs
|
|
2858
|
+
// the promise the wait was abandoned on.
|
|
2859
|
+
let claim;
|
|
2860
|
+
try {
|
|
2861
|
+
return await budget.run('discovery-lease-claim', () => {
|
|
2862
|
+
claim = this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
2863
|
+
return claim;
|
|
2864
|
+
});
|
|
2865
|
+
}
|
|
2866
|
+
catch (error) {
|
|
2867
|
+
if (!(error instanceof DiscoverySweepBudgetExceededError) || claim === undefined)
|
|
2868
|
+
throw error;
|
|
2869
|
+
void claim.then(async (late) => {
|
|
2870
|
+
if (!late.acquired || !late.lease)
|
|
2871
|
+
return;
|
|
2872
|
+
this.#increment('discoverySweepStrandedClaimsReleased');
|
|
2873
|
+
this.#logger.warn?.('[factory] releasing a discovery lease that was claimed after the sweep budget expired', {
|
|
2874
|
+
epoch: late.lease.epoch,
|
|
2875
|
+
budgetMs: error.budgetMs,
|
|
2876
|
+
});
|
|
2877
|
+
await this.#sweepTeardownStep('stranded discovery lease release', () => this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, late.lease.epoch));
|
|
2878
|
+
}, () => undefined).catch(() => undefined);
|
|
2879
|
+
throw error;
|
|
2880
|
+
}
|
|
2881
|
+
}
|
|
2882
|
+
async #runDiscoverySweep(opts, budget) {
|
|
2642
2883
|
const sweepStartedAtMs = this.#clock.now();
|
|
2643
2884
|
if (!(opts.dryRun ?? this.#config.dryRun)) {
|
|
2644
|
-
|
|
2885
|
+
// Under the budget, and first, because this is where the 2026-08-25
|
|
2886
|
+
// 07:52:59Z wedge sat: a pre-claim probe on a transport neither #351 nor
|
|
2887
|
+
// #368 covers. The budget does not care which one it is.
|
|
2888
|
+
await budget.run('fleet-control-plane-probe', () => this.#assertFleetControlPlaneAvailable());
|
|
2645
2889
|
}
|
|
2646
|
-
let claim = await this.#
|
|
2890
|
+
let claim = await this.#claimDiscoverySweepUnderBudget(budget);
|
|
2647
2891
|
if (!claim.acquired && claim.reason === 'backoff') {
|
|
2648
2892
|
const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
|
|
2649
2893
|
this.#increment('discoveryBackoffWaits');
|
|
@@ -2652,8 +2896,8 @@ export class FactoryLoop {
|
|
|
2652
2896
|
backoffUntilMs: claim.state.backoffUntilMs,
|
|
2653
2897
|
consecutiveOverloads: claim.state.consecutiveOverloads,
|
|
2654
2898
|
});
|
|
2655
|
-
await this.#clock.sleep(delayMs);
|
|
2656
|
-
claim = await this.#
|
|
2899
|
+
await budget.run('discovery-backoff-wait', () => this.#clock.sleep(delayMs));
|
|
2900
|
+
claim = await this.#claimDiscoverySweepUnderBudget(budget);
|
|
2657
2901
|
}
|
|
2658
2902
|
if (!claim.acquired || !claim.lease) {
|
|
2659
2903
|
this.#increment('discoverySweepsSkippedInFlight');
|
|
@@ -2698,13 +2942,13 @@ export class FactoryLoop {
|
|
|
2698
2942
|
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
2699
2943
|
let leaseReleased = false;
|
|
2700
2944
|
try {
|
|
2701
|
-
this.#discoverySession = await this.#prepareDiscoverySession(claim);
|
|
2945
|
+
this.#discoverySession = await budget.run('discovery-session', () => this.#prepareDiscoverySession(claim));
|
|
2702
2946
|
// #297: a 429 raised anywhere in the sweep used to latch and be rethrown
|
|
2703
2947
|
// here, discarding a completed pass — every issue read, every dispatch —
|
|
2704
2948
|
// because of one transient shed operation. The work this sweep did is
|
|
2705
2949
|
// now kept instead, and the ratchet below records that the dependency is
|
|
2706
2950
|
// shedding but still serving.
|
|
2707
|
-
const report = await this.#performRunOnce(opts);
|
|
2951
|
+
const report = await budget.run('run-once', () => this.#performRunOnce(opts, budget));
|
|
2708
2952
|
// The exception, and the reason skipping shed units cannot make a sweep
|
|
2709
2953
|
// unconditionally green: a sweep that was shed AND got no work unit
|
|
2710
2954
|
// through accomplished nothing. There is no progress to preserve, and
|
|
@@ -2715,17 +2959,20 @@ export class FactoryLoop {
|
|
|
2715
2959
|
if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
|
|
2716
2960
|
throw this.#discoveryOverloadError;
|
|
2717
2961
|
}
|
|
2718
|
-
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
2962
|
+
const checkpoint = await budget.run('discovery-checkpoint', () => this.#finalizeDiscoveryCheckpoint());
|
|
2719
2963
|
// Do not clear the durable lease while a renewal can still be waiting on
|
|
2720
2964
|
// the same state-file lock. A late renewal that observes the completed
|
|
2721
2965
|
// (lease-less) checkpoint is a false lease-loss signal and can poison an
|
|
2722
2966
|
// otherwise successful reconcile cycle.
|
|
2723
|
-
await this.#stopDiscoverySweepRenewal();
|
|
2967
|
+
await budget.run('discovery-renewal-stop', () => this.#stopDiscoverySweepRenewal());
|
|
2724
2968
|
if (this.#discoverySweepLeaseLost) {
|
|
2725
2969
|
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
2726
2970
|
}
|
|
2727
2971
|
const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
|
|
2728
|
-
|
|
2972
|
+
// Under the budget too. A hung commit is the same class of wedge as a
|
|
2973
|
+
// hung read, and the epoch guard in the store makes a late one a no-op:
|
|
2974
|
+
// the teardown below has already released this epoch's lease.
|
|
2975
|
+
const completed = await budget.run('discovery-commit', () => this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual));
|
|
2729
2976
|
leaseReleased = completed;
|
|
2730
2977
|
if (!completed)
|
|
2731
2978
|
throw new Error('discovery sweep lease was lost before completion');
|
|
@@ -2742,11 +2989,14 @@ export class FactoryLoop {
|
|
|
2742
2989
|
return report;
|
|
2743
2990
|
}
|
|
2744
2991
|
catch (error) {
|
|
2745
|
-
await this.#stopDiscoverySweepRenewal();
|
|
2992
|
+
await this.#sweepTeardownStep('discovery sweep renewal stop', () => this.#stopDiscoverySweepRenewal());
|
|
2746
2993
|
const overload = relayfileOverload(error);
|
|
2747
2994
|
if (overload) {
|
|
2748
2995
|
const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
|
|
2749
|
-
|
|
2996
|
+
// Bounded for the same reason the release below is: this is the other
|
|
2997
|
+
// path that hands the lease back, and an unbounded one would hold
|
|
2998
|
+
// `#runOnceInFlight` open past the budget that just expired.
|
|
2999
|
+
leaseReleased = await withSweepTeardownDeadline(DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS, () => this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads)) ?? false;
|
|
2750
3000
|
this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
|
|
2751
3001
|
status: overload.status,
|
|
2752
3002
|
reason: overload.reason,
|
|
@@ -2767,7 +3017,7 @@ export class FactoryLoop {
|
|
|
2767
3017
|
throw error;
|
|
2768
3018
|
}
|
|
2769
3019
|
finally {
|
|
2770
|
-
await this.#stopDiscoverySweepRenewal();
|
|
3020
|
+
await this.#sweepTeardownStep('discovery sweep renewal stop', () => this.#stopDiscoverySweepRenewal());
|
|
2771
3021
|
this.#discoverySession = undefined;
|
|
2772
3022
|
this.#discoverySweepEpoch = undefined;
|
|
2773
3023
|
this.#discoverySweepStartedAtMs = undefined;
|
|
@@ -2784,10 +3034,39 @@ export class FactoryLoop {
|
|
|
2784
3034
|
// happens to run and reset it at the top of this method.
|
|
2785
3035
|
this.#discoverySweepLeaseLost = false;
|
|
2786
3036
|
if (!leaseReleased) {
|
|
2787
|
-
|
|
3037
|
+
// The half of the budget that makes the NEXT cycle clean, so it gets
|
|
3038
|
+
// its own deadline rather than the spent one: an unbounded release
|
|
3039
|
+
// would re-create the wedge one layer down, which is the pattern this
|
|
3040
|
+
// change exists to end. An abandoned release is survivable — the
|
|
3041
|
+
// durable lease carries its own expiry and a later sweep reclaims it
|
|
3042
|
+
// as an orphan (`claim.reclaimedLease` above).
|
|
3043
|
+
await this.#sweepTeardownStep('discovery sweep lease release', () => this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch));
|
|
2788
3044
|
}
|
|
2789
3045
|
}
|
|
2790
3046
|
}
|
|
3047
|
+
/**
|
|
3048
|
+
* One sweep-teardown step, under its own deadline.
|
|
3049
|
+
*
|
|
3050
|
+
* Teardown cannot run under the sweep's aggregate budget: on the path that
|
|
3051
|
+
* matters the budget is already spent, so every step would reject and the
|
|
3052
|
+
* lease would never be released. It gets a short independent deadline
|
|
3053
|
+
* instead. Abandoning it costs an orphaned lease for one expiry window;
|
|
3054
|
+
* NOT bounding it costs the whole invariant, because a hung release holds
|
|
3055
|
+
* `#runOnceInFlight` open and every later cycle coalesces onto it.
|
|
3056
|
+
*/
|
|
3057
|
+
async #sweepTeardownStep(label, step) {
|
|
3058
|
+
const outcome = await withSweepTeardownDeadline(DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS, async () => {
|
|
3059
|
+
await step();
|
|
3060
|
+
return true;
|
|
3061
|
+
});
|
|
3062
|
+
if (outcome === undefined) {
|
|
3063
|
+
this.#increment('discoverySweepTeardownDeadlineExceeded');
|
|
3064
|
+
this.#logger.warn?.('[factory] discovery sweep teardown step abandoned at its deadline', {
|
|
3065
|
+
step: label,
|
|
3066
|
+
timeoutMs: DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS,
|
|
3067
|
+
});
|
|
3068
|
+
}
|
|
3069
|
+
}
|
|
2791
3070
|
/**
|
|
2792
3071
|
* Commit the sweep, carrying any residual overload backoff into the store.
|
|
2793
3072
|
*
|
|
@@ -2876,7 +3155,7 @@ export class FactoryLoop {
|
|
|
2876
3155
|
...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
|
|
2877
3156
|
};
|
|
2878
3157
|
}
|
|
2879
|
-
async #performRunOnce(opts = {}) {
|
|
3158
|
+
async #performRunOnce(opts = {}, budget) {
|
|
2880
3159
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
2881
3160
|
const startedAtMs = this.#clock.now();
|
|
2882
3161
|
const relayfileWaitWarningsAtStart = this.#counters.relayfileOperationWaitWarnings ?? 0;
|
|
@@ -2890,6 +3169,7 @@ export class FactoryLoop {
|
|
|
2890
3169
|
// current provider snapshots (or merged PR metadata) so a reopened issue
|
|
2891
3170
|
// cannot remain permanently resolved after an earlier close event.
|
|
2892
3171
|
this.#terminalDependencyIdentities.clear();
|
|
3172
|
+
this.#dependencyPrProbes.clear();
|
|
2893
3173
|
this.#dependencyGithubPathsByIdentity = undefined;
|
|
2894
3174
|
this.#dependencyLinearTreeLoaded = false;
|
|
2895
3175
|
const issueSource = await this.#issueSource();
|
|
@@ -2936,6 +3216,12 @@ export class FactoryLoop {
|
|
|
2936
3216
|
let readyIssueReads = 0;
|
|
2937
3217
|
const issueEntries = [];
|
|
2938
3218
|
for (const path of paths) {
|
|
3219
|
+
// A between-await check, worth exactly what #368 said such a check is
|
|
3220
|
+
// worth against a call that never returns: nothing. What it buys is
|
|
3221
|
+
// the other half — a pass already abandoned at the budget unwinds at
|
|
3222
|
+
// its next iteration if it ever regains control, instead of running to
|
|
3223
|
+
// completion beside the sweep that replaced it.
|
|
3224
|
+
budget?.assertNotExpired('run-once');
|
|
2939
3225
|
let issue;
|
|
2940
3226
|
let shed = false;
|
|
2941
3227
|
try {
|
|
@@ -2986,8 +3272,11 @@ export class FactoryLoop {
|
|
|
2986
3272
|
? '[factory] GitHub ready issue read progress'
|
|
2987
3273
|
: '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
|
|
2988
3274
|
if (!shed) {
|
|
2989
|
-
|
|
2990
|
-
|
|
3275
|
+
// Both surfaces. Gating this on Linear is what left GitHub ingestion
|
|
3276
|
+
// with no canonical state at all, so nothing could ever clear the
|
|
3277
|
+
// terminal row a completed run left behind (#334).
|
|
3278
|
+
if (issue) {
|
|
3279
|
+
await this.#recordCanonicalIssueState(issue, this.#issueLifecycleRole(issue));
|
|
2991
3280
|
}
|
|
2992
3281
|
issueEntries.push({ path, issue });
|
|
2993
3282
|
}
|
|
@@ -3009,6 +3298,10 @@ export class FactoryLoop {
|
|
|
3009
3298
|
});
|
|
3010
3299
|
}
|
|
3011
3300
|
for (const { issue } of issueEntries) {
|
|
3301
|
+
// The dispatch half of the same fence as the read loop above. Without
|
|
3302
|
+
// it a pass abandoned during enumeration would go on to dispatch after
|
|
3303
|
+
// its lease had been handed back, racing the sweep that replaced it.
|
|
3304
|
+
budget?.assertNotExpired('run-once');
|
|
3012
3305
|
await this.#refreshLiveHeartbeatIfDue();
|
|
3013
3306
|
if (!issue) {
|
|
3014
3307
|
continue;
|
|
@@ -3650,7 +3943,7 @@ export class FactoryLoop {
|
|
|
3650
3943
|
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
3651
3944
|
const hasLiveAgent = activeAgents.some((agent) => onlineAgents.has(agent.name));
|
|
3652
3945
|
const exitRecoveryActive = activeAgents.some((agent) => this.#agentExitsInFlight.has(agent.name));
|
|
3653
|
-
const dispatchCallActive = this.#
|
|
3946
|
+
const dispatchCallActive = this.#hasDispatchCallInFlight(lifecycle.issue);
|
|
3654
3947
|
// The provider's `factory:in-progress` transition happens immediately
|
|
3655
3948
|
// before the durable lifecycle advances from dispatching to running.
|
|
3656
3949
|
// A crash can therefore leave any nonterminal phase behind while the
|
|
@@ -3965,6 +4258,7 @@ export class FactoryLoop {
|
|
|
3965
4258
|
openOnly: true,
|
|
3966
4259
|
failOnLookupError: true,
|
|
3967
4260
|
allowLegacyGithubBranch: true,
|
|
4261
|
+
repo: this.#probeRepoForIssue(issue),
|
|
3968
4262
|
});
|
|
3969
4263
|
}
|
|
3970
4264
|
async #adoptOrphanedGithubPullRequest(issue, pr, legacyUnownedAgents = []) {
|
|
@@ -4194,10 +4488,35 @@ export class FactoryLoop {
|
|
|
4194
4488
|
const paths = session.checkpoint.trees[prefix];
|
|
4195
4489
|
return paths ? [...paths] : undefined;
|
|
4196
4490
|
}
|
|
4491
|
+
/**
|
|
4492
|
+
* True when this call is a continuation of a sweep that has already ended.
|
|
4493
|
+
*
|
|
4494
|
+
* #372: the aggregate budget abandons the WAIT, not the call, so a read
|
|
4495
|
+
* issued by an aborted sweep can still resolve — by which time the shared
|
|
4496
|
+
* `#discoverySession` and `#discoverySweepEpoch` may belong to the sweep
|
|
4497
|
+
* that replaced it. `discoveryEnumerationPass` is an `AsyncLocalStorage`, so
|
|
4498
|
+
* its store follows the async continuation and still carries the epoch that
|
|
4499
|
+
* ISSUED the read; comparing the two is what tells a live write from a
|
|
4500
|
+
* late one.
|
|
4501
|
+
*
|
|
4502
|
+
* Only when a store exists: a caller outside a discovery pass legitimately
|
|
4503
|
+
* has none, and treating that as stale would silence the live event drain.
|
|
4504
|
+
*/
|
|
4505
|
+
#isStaleDiscoveryContinuation() {
|
|
4506
|
+
const issuingPass = discoveryEnumerationPass.getStore();
|
|
4507
|
+
return issuingPass !== undefined && issuingPass.epoch !== this.#discoverySweepEpoch;
|
|
4508
|
+
}
|
|
4197
4509
|
async #rememberDiscoveryTree(prefix, paths) {
|
|
4198
4510
|
const session = this.#discoverySession;
|
|
4199
4511
|
if (!session)
|
|
4200
4512
|
return;
|
|
4513
|
+
// Committing this listing would put a tree from an abandoned pass into the
|
|
4514
|
+
// replacement sweep's checkpoint, under a watermark that claims to describe
|
|
4515
|
+
// the replacement. That is checkpoint corruption, and it outlives the sweep.
|
|
4516
|
+
if (this.#isStaleDiscoveryContinuation()) {
|
|
4517
|
+
this.#increment('discoveryStaleTreeWritesDropped');
|
|
4518
|
+
return;
|
|
4519
|
+
}
|
|
4201
4520
|
const uniquePaths = new Set();
|
|
4202
4521
|
for (let index = 0; index < paths.length; index += 1) {
|
|
4203
4522
|
uniquePaths.add(paths[index]);
|
|
@@ -4272,7 +4591,10 @@ export class FactoryLoop {
|
|
|
4272
4591
|
}
|
|
4273
4592
|
catch (error) {
|
|
4274
4593
|
const overload = relayfileOverload(error);
|
|
4275
|
-
|
|
4594
|
+
// The stale check for the same reason as the tree write above: a 429 that
|
|
4595
|
+
// arrives after its sweep was abandoned is not this sweep's evidence, and
|
|
4596
|
+
// attributing it here would drive the replacement's overload ratchet.
|
|
4597
|
+
if (overload && this.#discoverySweepEpoch !== undefined && !this.#isStaleDiscoveryContinuation()) {
|
|
4276
4598
|
this.#discoveryOverloadError ??= error;
|
|
4277
4599
|
this.#discoverySweepOverloads += 1;
|
|
4278
4600
|
if (overload.retryAfterSeconds !== undefined) {
|
|
@@ -4336,6 +4658,51 @@ export class FactoryLoop {
|
|
|
4336
4658
|
#elapsedSince(startedAtMs) {
|
|
4337
4659
|
return Math.max(0, this.#clock.now() - startedAtMs);
|
|
4338
4660
|
}
|
|
4661
|
+
/**
|
|
4662
|
+
* Progress reporting for the mount PR walk.
|
|
4663
|
+
*
|
|
4664
|
+
* `listTree` inside that walk is wrapped by `#listRelayfileTree` — named,
|
|
4665
|
+
* timed and logged. The `readFile` per candidate path was not: it ran inside a
|
|
4666
|
+
* bare try/catch that swallows failures into `undefined`, with no logger, no
|
|
4667
|
+
* counter and no progress line. A walk of several thousand paths at ~175ms
|
|
4668
|
+
* each therefore emitted its last log line at the final `listTree` and then
|
|
4669
|
+
* went silent for twelve minutes, which is indistinguishable from a hung
|
|
4670
|
+
* process. Three investigation layers could not tell those apart from the logs
|
|
4671
|
+
* alone, so the missing instrumentation is a defect in its own right and not a
|
|
4672
|
+
* nice-to-have.
|
|
4673
|
+
*
|
|
4674
|
+
* Same cadence helper the ready-issue read loop uses, so a long PR probe reads
|
|
4675
|
+
* like a long issue read; the counter carries the same signal into /evidence.
|
|
4676
|
+
*/
|
|
4677
|
+
#probeMountWalkProgress(message, issue) {
|
|
4678
|
+
const startedAtMs = this.#clock.now();
|
|
4679
|
+
let lastLoggedAtMs = startedAtMs;
|
|
4680
|
+
return {
|
|
4681
|
+
onRead: (progress) => {
|
|
4682
|
+
this.#increment('probePrMountReads');
|
|
4683
|
+
lastLoggedAtMs = this.#logTimedProgress(message, startedAtMs, lastLoggedAtMs, {
|
|
4684
|
+
issue: issue.key,
|
|
4685
|
+
read: progress.read,
|
|
4686
|
+
total: progress.total,
|
|
4687
|
+
path: progress.path,
|
|
4688
|
+
});
|
|
4689
|
+
},
|
|
4690
|
+
// One line per repository saying why the tree walk was necessary. The walk
|
|
4691
|
+
// is a permanent silent fallback today, and a silent permanent fallback is
|
|
4692
|
+
// indistinguishable from a fast path that is working. This makes the day
|
|
4693
|
+
// the index becomes usable (relayfile-adapters#271) show up in the logs
|
|
4694
|
+
// instead of passing unnoticed.
|
|
4695
|
+
onIndexFallback: (repo, reason) => {
|
|
4696
|
+
this.#increment('probePrIndexFallbacks');
|
|
4697
|
+
this.#increment(PROBE_PR_INDEX_FALLBACK_COUNTERS[reason]);
|
|
4698
|
+
this.#logger.debug?.('[factory] PR probe fell back to a full mount walk', {
|
|
4699
|
+
issue: issue.key,
|
|
4700
|
+
repo,
|
|
4701
|
+
reason,
|
|
4702
|
+
});
|
|
4703
|
+
},
|
|
4704
|
+
};
|
|
4705
|
+
}
|
|
4339
4706
|
#logTimedProgress(message, startedAtMs, lastLoggedAtMs, metadata) {
|
|
4340
4707
|
const now = this.#clock.now();
|
|
4341
4708
|
if (now - lastLoggedAtMs < REMOTE_OPERATION_PROGRESS_INTERVAL_MS) {
|
|
@@ -4459,6 +4826,44 @@ export class FactoryLoop {
|
|
|
4459
4826
|
}
|
|
4460
4827
|
}
|
|
4461
4828
|
}
|
|
4829
|
+
/**
|
|
4830
|
+
* Is a `dispatch()` call that could own this work unit's durable lifecycle
|
|
4831
|
+
* running in this process right now?
|
|
4832
|
+
*
|
|
4833
|
+
* `#dispatchInFlight` is keyed by work unit *plus* the dry-run flag and phase
|
|
4834
|
+
* the call was made under, so this must be built from the same three parts
|
|
4835
|
+
* `dispatch()` writes rather than from the bare identity.
|
|
4836
|
+
*
|
|
4837
|
+
* Live only, but BOTH phases — the same `` `${key}:live:` `` prefix `stop()`
|
|
4838
|
+
* uses at `:1531`, and for the same reason.
|
|
4839
|
+
*
|
|
4840
|
+
* The dry-run half of the key is decided once and is stable across both
|
|
4841
|
+
* functions, and `durableDispatch` is `!dryRun && …`, so a `:dry-run:*` call
|
|
4842
|
+
* provably never claims a lifecycle and matching it would preserve a
|
|
4843
|
+
* genuinely orphaned claim (#369 review, codex).
|
|
4844
|
+
*
|
|
4845
|
+
* The phase half is NOT stable. `dispatch()` derives it from the incoming
|
|
4846
|
+
* decision, while `#dispatchUnlocked` re-derives the escalation reason from
|
|
4847
|
+
* the post-routing decision — and `authoritativeRoutedDecision` upgrades a
|
|
4848
|
+
* routeless `confidence: 'low'` triage to `'high'` when the live labels
|
|
4849
|
+
* resolve a repository. A call keyed `:live:escalation` therefore does reach
|
|
4850
|
+
* lifecycle creation, so excluding that phase would reopen exactly the defect
|
|
4851
|
+
* below for it (#369 review, cubic).
|
|
4852
|
+
*
|
|
4853
|
+
* #367: this was `#dispatchInFlight.has(issueKey(issue))`, which no key in
|
|
4854
|
+
* the map can ever equal. `issueKey` is `<key>:<uuid>:<path>`, while every
|
|
4855
|
+
* entry is `<dispatchLifecycleKey>:<dry-run|live>:<phase>` — so the guard was
|
|
4856
|
+
* unsatisfiable by construction and orphan recovery could class a lifecycle
|
|
4857
|
+
* as abandoned while its own dispatch was still mid-flight.
|
|
4858
|
+
*/
|
|
4859
|
+
#hasDispatchCallInFlight(issue) {
|
|
4860
|
+
const livePrefix = `${dispatchLifecycleKey(issue)}:live:`;
|
|
4861
|
+
for (const key of this.#dispatchInFlight.keys()) {
|
|
4862
|
+
if (key.startsWith(livePrefix))
|
|
4863
|
+
return true;
|
|
4864
|
+
}
|
|
4865
|
+
return false;
|
|
4866
|
+
}
|
|
4462
4867
|
async #dispatchUnlocked(decision, opts = {}) {
|
|
4463
4868
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
4464
4869
|
const batch = await this.#batch();
|
|
@@ -7041,6 +7446,10 @@ export class FactoryLoop {
|
|
|
7041
7446
|
}
|
|
7042
7447
|
if (!await this.#saveDispatchLifecycle(record, 'complete'))
|
|
7043
7448
|
return false;
|
|
7449
|
+
// The terminal rows now exist. If a reopen was observed while they were
|
|
7450
|
+
// still being written, it found nothing to clear and consumed the edge; do
|
|
7451
|
+
// the clear it could not do (#334, #375 review).
|
|
7452
|
+
await this.#reconcileReopenObservedDuringCompletion(record);
|
|
7044
7453
|
this.#increment(releaseReason === 'issue-human-review' ? 'humanReview' : 'done');
|
|
7045
7454
|
this.#emit('issue-done', { issue: record.issue });
|
|
7046
7455
|
await this.#writeInFlightRegistry();
|
|
@@ -7084,8 +7493,8 @@ export class FactoryLoop {
|
|
|
7084
7493
|
}
|
|
7085
7494
|
try {
|
|
7086
7495
|
const issue = await this.#readIssue(path);
|
|
7087
|
-
if (issue
|
|
7088
|
-
await this.#recordCanonicalIssueState(issue);
|
|
7496
|
+
if (issue) {
|
|
7497
|
+
await this.#recordCanonicalIssueState(issue, this.#issueLifecycleRole(issue));
|
|
7089
7498
|
}
|
|
7090
7499
|
if (issue && this.#dependencyIssueIsTerminal(issue)) {
|
|
7091
7500
|
await this.#markDependencyTerminalAndReconcile(issue);
|
|
@@ -8040,12 +8449,25 @@ export class FactoryLoop {
|
|
|
8040
8449
|
const repo = dependencyRepoForIssue(issue, undefined, this.#config);
|
|
8041
8450
|
if (!repo)
|
|
8042
8451
|
return false;
|
|
8452
|
+
// This probe does NOT go through `#resolveIssuePr` — it must not fall back to
|
|
8453
|
+
// gh — so it never saw that method's cache, and `#terminalDependencyIdentities`
|
|
8454
|
+
// only ever memoises the TRUE answer. A dependency that is not merged was
|
|
8455
|
+
// therefore re-walked in full for every issue declaring it, on every sweep;
|
|
8456
|
+
// several issues blocked on one dependency multiplied a single tree walk by
|
|
8457
|
+
// the number of blocked issues. Memoise the negative answer too, on exactly
|
|
8458
|
+
// the lifetime of the terminal set beside it: cleared at the top of each
|
|
8459
|
+
// sweep, so a PR that merges between sweeps is still observed.
|
|
8460
|
+
const memoized = this.#dependencyPrProbes.get(identity);
|
|
8461
|
+
if (memoized !== undefined)
|
|
8462
|
+
return memoized;
|
|
8043
8463
|
const pullRequest = await resolveIssuePrFromMount(this.#mount, this.#config, issue, {
|
|
8044
8464
|
allowLegacyGithubBranch: true,
|
|
8045
8465
|
repo,
|
|
8046
|
-
}, (prefix) => this.#listRelayfileTree(prefix, 'dependency PR probe resolution'));
|
|
8047
|
-
if (normalizePrState(pullRequest?.state) !== 'MERGED')
|
|
8466
|
+
}, (prefix) => this.#listRelayfileTree(prefix, 'dependency PR probe resolution'), this.#probeMountWalkProgress('[factory] dependency PR probe mount read progress', issue));
|
|
8467
|
+
if (normalizePrState(pullRequest?.state) !== 'MERGED') {
|
|
8468
|
+
this.#dependencyPrProbes.set(identity, false);
|
|
8048
8469
|
return false;
|
|
8470
|
+
}
|
|
8049
8471
|
this.#terminalDependencyIdentities.add(identity);
|
|
8050
8472
|
return true;
|
|
8051
8473
|
}
|
|
@@ -8195,38 +8617,152 @@ export class FactoryLoop {
|
|
|
8195
8617
|
state.backoffUntilMs = 0;
|
|
8196
8618
|
await this.#state.recordDispatchAttempt(this.#workspaceId, key, state);
|
|
8197
8619
|
}
|
|
8198
|
-
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8620
|
+
/**
|
|
8621
|
+
* The lifecycle role a work unit is in, from whichever surface described it.
|
|
8622
|
+
*
|
|
8623
|
+
* A GitHub issue carries no Linear workflow state — `githubIssueAsFactoryIssue`
|
|
8624
|
+
* sets `stateId: ''` — so its role lives in the `factory:*` labels and the
|
|
8625
|
+
* open/closed flag instead, exactly as `#isIssueReady` and
|
|
8626
|
+
* `#isIssueExternallyTerminal` already read it. Canonical state is recorded as
|
|
8627
|
+
* this role rather than as a raw state id so one provider-neutral value means
|
|
8628
|
+
* the same thing on both surfaces (#334).
|
|
8629
|
+
*/
|
|
8630
|
+
#issueLifecycleRole(issue) {
|
|
8631
|
+
if (!isGithubIssue(issue))
|
|
8632
|
+
return this.#states.roleOf(issue.stateId);
|
|
8633
|
+
if (githubFactoryIssueIsClosed(issue))
|
|
8634
|
+
return 'done';
|
|
8635
|
+
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
8636
|
+
if (labels.has('factory:human-review'))
|
|
8637
|
+
return 'humanReview';
|
|
8638
|
+
if (this.#isIssueReady(issue))
|
|
8639
|
+
return 'readyForAgent';
|
|
8640
|
+
if (labels.has('factory:in-progress'))
|
|
8641
|
+
return 'agentImplementing';
|
|
8642
|
+
return undefined;
|
|
8643
|
+
}
|
|
8644
|
+
/**
|
|
8645
|
+
* Remember the role this work unit was last seen in, and clear the durable
|
|
8646
|
+
* refusals a completed run left behind when it comes back ready.
|
|
8647
|
+
*
|
|
8648
|
+
* Called for BOTH surfaces. It used to be gated to Linear, so on a
|
|
8649
|
+
* GitHub-sourced Factory the reopen cleanup below was never reached at all
|
|
8650
|
+
* and a terminal row refused the reopened work forever — seven work units
|
|
8651
|
+
* across five repositories on 2026-08-25 (#334).
|
|
8652
|
+
*/
|
|
8653
|
+
async #recordCanonicalIssueState(issue, role) {
|
|
8654
|
+
const ref = issueRefForState(issue);
|
|
8655
|
+
const canonicalKey = canonicalStateKey(ref);
|
|
8656
|
+
const previousRole = await this.#state.getCanonicalState(this.#workspaceId, canonicalKey);
|
|
8202
8657
|
const reopenedFromTerminal = previousRole === 'done' || previousRole === 'humanReview';
|
|
8203
|
-
if (reopenedFromTerminal &&
|
|
8204
|
-
|
|
8205
|
-
|
|
8206
|
-
|
|
8207
|
-
|
|
8208
|
-
|
|
8209
|
-
|
|
8210
|
-
|
|
8211
|
-
|
|
8212
|
-
|
|
8213
|
-
|
|
8214
|
-
|
|
8215
|
-
|
|
8216
|
-
|
|
8217
|
-
|
|
8218
|
-
|
|
8219
|
-
|
|
8220
|
-
|
|
8221
|
-
|
|
8222
|
-
|
|
8223
|
-
|
|
8224
|
-
|
|
8225
|
-
|
|
8226
|
-
|
|
8227
|
-
|
|
8658
|
+
if (reopenedFromTerminal && role === 'readyForAgent') {
|
|
8659
|
+
await this.#clearTerminalRefusals(ref);
|
|
8660
|
+
}
|
|
8661
|
+
// Never erase a remembered terminal role on an intermediate observation.
|
|
8662
|
+
// A reopened issue is routinely seen in a non-ready shape first — reopened
|
|
8663
|
+
// while `factory:in-progress` is still hanging on it, or reopened one event
|
|
8664
|
+
// before the readiness label is reapplied — and writing `role ?? ''` there
|
|
8665
|
+
// overwrote the remembered `done`, disarming the reopen edge so the later
|
|
8666
|
+
// ready observation refused the work exactly as #334 did. Only the three
|
|
8667
|
+
// roles the comparison above acts on are recorded, and that comparison is
|
|
8668
|
+
// this row's only consumer: `getCanonicalState` has no other caller
|
|
8669
|
+
// (cubic-dev-ai, #375 review).
|
|
8670
|
+
if (role === 'done' || role === 'humanReview' || role === 'readyForAgent') {
|
|
8671
|
+
await this.#state.recordCanonicalState(this.#workspaceId, canonicalKey, role);
|
|
8672
|
+
}
|
|
8673
|
+
}
|
|
8674
|
+
/**
|
|
8675
|
+
* Clear the durable refusals a completed run left behind for this work unit,
|
|
8676
|
+
* and report whether anything was actually cleared.
|
|
8677
|
+
*
|
|
8678
|
+
* One increment per reopened work unit, not per cleared row: the attempt
|
|
8679
|
+
* counters and the durable lifecycle are two records of the same refusal and
|
|
8680
|
+
* a unit that clears both reopened once. Counted here rather than at the
|
|
8681
|
+
* attempt clear alone so a reopen that only had a durable row — the shape a
|
|
8682
|
+
* restart leaves, and the one #334 saw in production — is still visible to an
|
|
8683
|
+
* operator.
|
|
8684
|
+
*/
|
|
8685
|
+
async #clearTerminalRefusals(ref) {
|
|
8686
|
+
// Dispatch attempts stay on the surface key: that is the key every other
|
|
8687
|
+
// reader and writer of the attempt counters uses, and rekeying only this
|
|
8688
|
+
// one site is the writer/reader mismatch #367 was filed for.
|
|
8689
|
+
const attemptKey = issueStateKey(ref);
|
|
8690
|
+
let reopened = false;
|
|
8691
|
+
const dispatchState = await this.#state.getDispatchAttempts(this.#workspaceId, attemptKey);
|
|
8692
|
+
if (dispatchState?.terminal) {
|
|
8693
|
+
dispatchState.attempts = 0;
|
|
8694
|
+
dispatchState.inFlight = false;
|
|
8695
|
+
dispatchState.terminal = false;
|
|
8696
|
+
dispatchState.backoffUntilMs = 0;
|
|
8697
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, attemptKey, dispatchState);
|
|
8698
|
+
reopened = true;
|
|
8699
|
+
}
|
|
8700
|
+
// Match on the work unit, not the surface key: a completed Linear mirror
|
|
8701
|
+
// persists `AR-448` while the GitHub-native arrival of the same issue is
|
|
8702
|
+
// `448`, and comparing surface keys would leave that terminal row in
|
|
8703
|
+
// place to refuse the reopened work forever.
|
|
8704
|
+
const reopenedIdentity = safeDispatchLifecycleKey(ref);
|
|
8705
|
+
for (const [lifecycleKey, lifecycle] of await this.#state.listDispatchLifecycles(this.#workspaceId)) {
|
|
8706
|
+
if (!isTerminalDispatchLifecycle(lifecycle))
|
|
8707
|
+
continue;
|
|
8708
|
+
const lifecycleIdentity = safeDispatchLifecycleKey(lifecycle.issue);
|
|
8709
|
+
const matches = reopenedIdentity !== undefined && lifecycleIdentity === reopenedIdentity;
|
|
8710
|
+
// The surface-key fallback covers only a row so old it cannot produce a
|
|
8711
|
+
// work-unit identity at all, and only for a Linear-shaped key. A GitHub
|
|
8712
|
+
// issue key is a bare number that repeats in every repository, so
|
|
8713
|
+
// `key === key` there would let a reopened `factory#364` clear
|
|
8714
|
+
// `cloud#364`'s completed row — the #334 evidence spans five
|
|
8715
|
+
// repositories with overlapping numbering.
|
|
8716
|
+
const legacySurfaceMatch = lifecycleIdentity === undefined &&
|
|
8717
|
+
ISSUE_KEY_PARTS.test(ref.key) &&
|
|
8718
|
+
lifecycle.issue.key === ref.key;
|
|
8719
|
+
if (!matches && !legacySurfaceMatch)
|
|
8720
|
+
continue;
|
|
8721
|
+
await this.#state.clearDispatchLifecycle(this.#workspaceId, lifecycleKey);
|
|
8722
|
+
this.#dispatchLifecycleEpochs.delete(lifecycleKey);
|
|
8723
|
+
reopened = true;
|
|
8724
|
+
}
|
|
8725
|
+
if (reopened)
|
|
8726
|
+
this.#increment('dispatchTerminalReopened');
|
|
8727
|
+
return reopened;
|
|
8728
|
+
}
|
|
8729
|
+
/**
|
|
8730
|
+
* Consume a reopen that was observed *while* completion was still writing its
|
|
8731
|
+
* terminal rows.
|
|
8732
|
+
*
|
|
8733
|
+
* Completion records the terminal role at the provider write, but the durable
|
|
8734
|
+
* rows that refuse a redispatch land much later — `#recordDispatchTerminal`
|
|
8735
|
+
* and the `complete` lifecycle save happen after the completion comment, the
|
|
8736
|
+
* Slack thread and every agent release. A reopen observed inside that window
|
|
8737
|
+
* reads terminal → ready, finds nothing terminal to clear, and *consumes the
|
|
8738
|
+
* edge anyway* by recording `readyForAgent`. The terminal rows then land on a
|
|
8739
|
+
* work unit whose canonical role will never again read terminal → ready, so
|
|
8740
|
+
* the refusal is permanent: #334's exact failure reached through the
|
|
8741
|
+
* completion window instead of through a missing canonical row
|
|
8742
|
+
* (chatgpt-codex-connector P1, #375 review).
|
|
8743
|
+
*
|
|
8744
|
+
* So recheck at the terminal save, which is the first instant the rows the
|
|
8745
|
+
* reopen would have cleared actually exist. This clears nothing the reopen
|
|
8746
|
+
* path would not have cleared itself had it run a moment later — it only
|
|
8747
|
+
* repairs the ordering — and it is armed solely for a completion that
|
|
8748
|
+
* recorded a terminal role of its own.
|
|
8749
|
+
*/
|
|
8750
|
+
async #reconcileReopenObservedDuringCompletion(record) {
|
|
8751
|
+
if (!record.canonicalTerminalRoleRecorded)
|
|
8752
|
+
return;
|
|
8753
|
+
const ref = issueRefForState(record.issue);
|
|
8754
|
+
const role = await this.#state.getCanonicalState(this.#workspaceId, canonicalStateKey(ref));
|
|
8755
|
+
if (role !== 'readyForAgent')
|
|
8756
|
+
return;
|
|
8757
|
+
// Counted from the repair, not from the observation. A reopen that landed
|
|
8758
|
+
// after the terminal rows existed was already cleared by the observation
|
|
8759
|
+
// path itself and leaves the same `readyForAgent` row behind, so counting
|
|
8760
|
+
// the read would put a reopen completion never raced into the
|
|
8761
|
+
// completion-window bucket (cubic-dev-ai P3, #375 review). Only a clear
|
|
8762
|
+
// that actually found a refusal to remove is one this recheck repaired.
|
|
8763
|
+
if (await this.#clearTerminalRefusals(ref)) {
|
|
8764
|
+
this.#increment('dispatchTerminalReopenedDuringCompletion');
|
|
8228
8765
|
}
|
|
8229
|
-
await this.#state.recordCanonicalState(this.#workspaceId, key, issue.stateId);
|
|
8230
8766
|
}
|
|
8231
8767
|
async #writeLoopHeartbeat(path, registryPath, status, iteration, maxIterations) {
|
|
8232
8768
|
const updatedAtMs = this.#clock.now();
|
|
@@ -13455,14 +13991,34 @@ export class FactoryLoop {
|
|
|
13455
13991
|
this.#postMergeDoneAdvances.add(advanceKey);
|
|
13456
13992
|
try {
|
|
13457
13993
|
const githubIssue = isGithubIssue(issue);
|
|
13994
|
+
let terminalStateObserved = true;
|
|
13458
13995
|
if (githubIssue) {
|
|
13459
|
-
await this.#githubWriteback.closeIssue(issue, `Factory observed pull request #${snapshot.number} merge and completed this issue.\n\nClosing authority: ${authority.evidence}.`);
|
|
13996
|
+
const closeWrite = await this.#githubWriteback.closeIssue(issue, `Factory observed pull request #${snapshot.number} merge and completed this issue.\n\nClosing authority: ${authority.evidence}.`);
|
|
13997
|
+
if (closeWrite === undefined)
|
|
13998
|
+
this.#recordMissingGithubWritebackReceipt('closeIssue');
|
|
13999
|
+
terminalStateObserved = closeWrite !== undefined;
|
|
13460
14000
|
}
|
|
13461
14001
|
else {
|
|
13462
14002
|
const doneStateId = this.#states.idFor(issue.team, 'done');
|
|
13463
14003
|
await this.#linear.setState(issue, doneStateId);
|
|
13464
|
-
await this.#recordCanonicalIssueState({ ...issueRef(issue), stateId: doneStateId });
|
|
13465
14004
|
}
|
|
14005
|
+
// Recorded from the write, not left to the next sweep's read: a reopen
|
|
14006
|
+
// landing in that gap would otherwise be read as the FIRST terminal
|
|
14007
|
+
// observation of this unit and the reopen would never be seen (#334).
|
|
14008
|
+
//
|
|
14009
|
+
// Any defined close receipt records it, because canonical state answers
|
|
14010
|
+
// "what state is this unit in", which is a different question from the
|
|
14011
|
+
// "who created this transition" that `applied` alone answers and that
|
|
14012
|
+
// ownership gates elsewhere require. Every receipt in
|
|
14013
|
+
// `GithubIssueCloseWriteResult` is a state observation: `already-matched`
|
|
14014
|
+
// is a provider read of the closed state, `acknowledged` is by contract
|
|
14015
|
+
// an unattributed *visible* transition, and the shipped `gh` adapter
|
|
14016
|
+
// throws unless it reads the issue back as closed. Only a legacy void
|
|
14017
|
+
// adapter (`undefined`) carries no state evidence at all; there the next
|
|
14018
|
+
// sweep records whatever the provider does say (cubic-dev-ai, #375
|
|
14019
|
+
// review).
|
|
14020
|
+
if (terminalStateObserved)
|
|
14021
|
+
await this.#recordCanonicalIssueState(issueRef(issue), 'done');
|
|
13466
14022
|
this.#emit('writeback-verified', { issue: issueRef(issue), path: issue.path });
|
|
13467
14023
|
await this.#markDependencyTerminalAndReconcile(issue);
|
|
13468
14024
|
this.#increment('mergedPrAdvancedDone');
|
|
@@ -14200,6 +14756,16 @@ export class FactoryLoop {
|
|
|
14200
14756
|
// park during it must still abort immediately rather than waiting on a
|
|
14201
14757
|
// possibly long completion path.
|
|
14202
14758
|
this.#issueWritebackInFlight.set(completionKey, issueWritebackSettled);
|
|
14759
|
+
// Whether the PROVIDER's visible state is now terminal. Deliberately a
|
|
14760
|
+
// different question from `issueWritebackConfirmedAtMs`, which asks
|
|
14761
|
+
// whether THIS dispatch owns that transition and stays `applied`-only:
|
|
14762
|
+
// canonical state exists to answer "what state is this unit in", and
|
|
14763
|
+
// every defined receipt answers that (`already-matched` is a provider
|
|
14764
|
+
// read of the target state, `acknowledged` is by contract an
|
|
14765
|
+
// unattributed *visible* transition). Only a legacy void adapter
|
|
14766
|
+
// carries no state evidence, and there the next sweep still records
|
|
14767
|
+
// whatever the provider says (cubic-dev-ai, #375 review).
|
|
14768
|
+
let terminalStateObserved = false;
|
|
14203
14769
|
if (githubIssue) {
|
|
14204
14770
|
if (humanReview) {
|
|
14205
14771
|
const statusWrite = await this.#githubWriteback.setStatus(issue, 'human-review');
|
|
@@ -14211,6 +14777,7 @@ export class FactoryLoop {
|
|
|
14211
14777
|
if (statusWrite === 'applied') {
|
|
14212
14778
|
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
14213
14779
|
}
|
|
14780
|
+
terminalStateObserved = statusWrite !== undefined;
|
|
14214
14781
|
// The lifecycle-state outcome is now known. Unblock the concurrent
|
|
14215
14782
|
// post-spawn read before the separate completion comment write.
|
|
14216
14783
|
settleIssueWritebackOnce();
|
|
@@ -14227,6 +14794,7 @@ export class FactoryLoop {
|
|
|
14227
14794
|
if (closeWrite === 'applied') {
|
|
14228
14795
|
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
14229
14796
|
}
|
|
14797
|
+
terminalStateObserved = closeWrite !== undefined;
|
|
14230
14798
|
}
|
|
14231
14799
|
}
|
|
14232
14800
|
else {
|
|
@@ -14235,7 +14803,17 @@ export class FactoryLoop {
|
|
|
14235
14803
|
: this.#states.idFor(issueTeam, 'done');
|
|
14236
14804
|
await this.#linear.setState(issue, targetState);
|
|
14237
14805
|
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
14238
|
-
|
|
14806
|
+
terminalStateObserved = true;
|
|
14807
|
+
}
|
|
14808
|
+
if (terminalStateObserved) {
|
|
14809
|
+
// Both surfaces now, and recorded from the write rather than left to
|
|
14810
|
+
// the next sweep so a reopen cannot slip into the gap (#334). The
|
|
14811
|
+
// marker is what lets the terminal save — which happens after the
|
|
14812
|
+
// completion comment, the Slack thread and every agent release —
|
|
14813
|
+
// detect a reopen that landed inside that window and consumed the
|
|
14814
|
+
// reopen edge before there was anything to clear.
|
|
14815
|
+
await this.#recordCanonicalIssueState(record.issue, humanReview ? 'humanReview' : 'done');
|
|
14816
|
+
record.canonicalTerminalRoleRecorded = true;
|
|
14239
14817
|
}
|
|
14240
14818
|
if (record.issueWritebackConfirmedAtMs !== undefined) {
|
|
14241
14819
|
this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
|
|
@@ -14326,8 +14904,25 @@ export class FactoryLoop {
|
|
|
14326
14904
|
settleIssueWritebackOnce();
|
|
14327
14905
|
this.#completionInFlight.delete(completionKey);
|
|
14328
14906
|
const stateKey = issueStateKey(record.issue);
|
|
14329
|
-
|
|
14330
|
-
|
|
14907
|
+
// Both maps are keyed by issue state key PLUS the option suffixes
|
|
14908
|
+
// `#resolveIssuePr` appends (`:open`, `:legacy`, `:open:legacy`), but this
|
|
14909
|
+
// invalidation only ever deleted the bare key. Every `openOnly` probe —
|
|
14910
|
+
// `#openPrForIssue` and `#openCompletionPr`, i.e. the completion path — was
|
|
14911
|
+
// therefore never invalidated at all. That was survivable only because the
|
|
14912
|
+
// mount branch never wrote the cache; now that it does, a stale OPEN entry
|
|
14913
|
+
// outliving completion would be a live correctness bug, so clear the whole
|
|
14914
|
+
// family. Suffixes always start with ':', and no other issue's state key can
|
|
14915
|
+
// be this one followed by ':', so the prefix test cannot over-delete.
|
|
14916
|
+
for (const cacheKey of [...this.#probePrResolvedCache.keys()]) {
|
|
14917
|
+
if (cacheKey === stateKey || cacheKey.startsWith(`${stateKey}:`)) {
|
|
14918
|
+
this.#probePrResolvedCache.delete(cacheKey);
|
|
14919
|
+
}
|
|
14920
|
+
}
|
|
14921
|
+
for (const backoffKey of [...this.#probePrGhBackoffUntilMs.keys()]) {
|
|
14922
|
+
if (backoffKey === stateKey || backoffKey.startsWith(`${stateKey}:`)) {
|
|
14923
|
+
this.#probePrGhBackoffUntilMs.delete(backoffKey);
|
|
14924
|
+
}
|
|
14925
|
+
}
|
|
14331
14926
|
// Cancellation must see the subscription identity so it can issue the
|
|
14332
14927
|
// idempotent Relayfile DELETE before clearing the local owner maps.
|
|
14333
14928
|
await this.#cancelBabysittersForIssue(record.issue);
|
|
@@ -16995,6 +17590,7 @@ export class FactoryLoop {
|
|
|
16995
17590
|
? await this.#probePrResolver(issue)
|
|
16996
17591
|
: await this.#resolveIssuePr(issue, {
|
|
16997
17592
|
titleMarker: FACTORY_E2E_MARKER,
|
|
17593
|
+
repo: this.#probeRepoForIssue(issue),
|
|
16998
17594
|
});
|
|
16999
17595
|
if (!probe) {
|
|
17000
17596
|
return;
|
|
@@ -17267,7 +17863,7 @@ const githubIssueSourceRef = (issue) => {
|
|
|
17267
17863
|
}
|
|
17268
17864
|
return { owner, repo, number: number, url };
|
|
17269
17865
|
};
|
|
17270
|
-
function dependencyRepoForIssue(issue, decision, config) {
|
|
17866
|
+
function dependencyRepoForIssue(issue, decision, config, opts = {}) {
|
|
17271
17867
|
const source = githubIssueSourceRef(issue);
|
|
17272
17868
|
if (source)
|
|
17273
17869
|
return `${source.owner}/${source.repo}`;
|
|
@@ -17304,7 +17900,19 @@ function dependencyRepoForIssue(issue, decision, config) {
|
|
|
17304
17900
|
? Object.entries(config.repos.byProject)
|
|
17305
17901
|
.find(([project]) => project.trim().toLowerCase() === issue.project.trim().toLowerCase())?.[1]
|
|
17306
17902
|
: undefined;
|
|
17307
|
-
|
|
17903
|
+
const projectResolved = normalize(projectRepo);
|
|
17904
|
+
if (projectResolved)
|
|
17905
|
+
return projectResolved;
|
|
17906
|
+
// `repos.default` is a DISPATCH fallback, not evidence of where a PR already
|
|
17907
|
+
// lives. Routing precedence is byLabel, byProject, keywordRules, default —
|
|
17908
|
+
// and this helper cannot see `keywordRules` at all, because those match on
|
|
17909
|
+
// title/description through triage rather than on the issue's own fields. So
|
|
17910
|
+
// for a keyword-routed issue the default answer here is confidently wrong:
|
|
17911
|
+
// dispatch opened the PR in the keyword-selected repository, not in
|
|
17912
|
+
// `repos.default`. Callers that only need a dispatch target still want that
|
|
17913
|
+
// fallback; a PROBE must not have it, because a mis-scoped probe reports "no
|
|
17914
|
+
// PR" for an issue that has one, which is worse than a slow unscoped walk.
|
|
17915
|
+
return opts.allowDefault === false ? undefined : normalize(config.repos.default);
|
|
17308
17916
|
}
|
|
17309
17917
|
function dependencyIdentityForIssue(issue, repo) {
|
|
17310
17918
|
const source = githubIssueSourceRef(issue);
|
|
@@ -17386,6 +17994,21 @@ const dispatchLifecycleKey = (issue) => dispatchIssueIdentity(issue);
|
|
|
17386
17994
|
// Preserve the historical Linear state namespace while keeping GitHub-native
|
|
17387
17995
|
// issue numbers independent across repositories in the same workspace.
|
|
17388
17996
|
const issueStateKey = (issue) => githubIssuePathParts(issue.path) ? issueKey(issue) : issue.key;
|
|
17997
|
+
/**
|
|
17998
|
+
* The canonical-state key: the work unit, the same identity the dispatch
|
|
17999
|
+
* lifecycle it gates is keyed under (#211, #329, #367).
|
|
18000
|
+
*
|
|
18001
|
+
* Canonical state exists for exactly one decision — did this work unit reach a
|
|
18002
|
+
* terminal role and then come back ready — and that decision clears a durable
|
|
18003
|
+
* row keyed on the work unit. Keying the question per surface and the answer
|
|
18004
|
+
* per work unit is what made #334 reachable: a GitHub-native ref could not see
|
|
18005
|
+
* the terminal role its own Linear mirror had recorded.
|
|
18006
|
+
*
|
|
18007
|
+
* Falls back to the surface key for a ref with no derivable provider identity,
|
|
18008
|
+
* so a malformed row degrades to today's behaviour instead of throwing out of
|
|
18009
|
+
* ingestion.
|
|
18010
|
+
*/
|
|
18011
|
+
const canonicalStateKey = (issue) => safeDispatchLifecycleKey(issue) ?? issueStateKey(issue);
|
|
17389
18012
|
const pidsFromSpawnResult = (result) => {
|
|
17390
18013
|
const pids = new Set();
|
|
17391
18014
|
for (const pid of result?.pids ?? []) {
|
|
@@ -18277,10 +18900,59 @@ const linearIssueMirrorsGithubIssue = (issue, ghIssue) => {
|
|
|
18277
18900
|
.split(/\r?\n/u)
|
|
18278
18901
|
.some((line) => line.trim() === `${GITHUB_MIRROR_SOURCE_PREFIX}${ghIssue.url}`);
|
|
18279
18902
|
};
|
|
18280
|
-
const
|
|
18903
|
+
const PROBE_PR_INDEX_FALLBACK_COUNTERS = {
|
|
18904
|
+
'index-absent': 'probePrIndexAbsent',
|
|
18905
|
+
'index-shape-unrecognised': 'probePrIndexShapeUnrecognised',
|
|
18906
|
+
'index-without-head-ref': 'probePrIndexWithoutHeadRef',
|
|
18907
|
+
// Not a fallback condition so much as a standing invitation: rows carry
|
|
18908
|
+
// `headRef`, so the walk is now avoidable and this counter going non-zero is
|
|
18909
|
+
// the cue to build the fast path.
|
|
18910
|
+
'index-usable': 'probePrIndexUsableButUnused',
|
|
18911
|
+
};
|
|
18912
|
+
/**
|
|
18913
|
+
* Classify the pull index without acting on it.
|
|
18914
|
+
*
|
|
18915
|
+
* Deliberately diagnostic only. Factory's issue-side reader
|
|
18916
|
+
* (`#githubIssuePathsFromIndex`) uses its index as a NARROWING FILTER over
|
|
18917
|
+
* paths it still reads individually, which is safe because it can only
|
|
18918
|
+
* over-select. A pull index cannot be used that way today: it carries no
|
|
18919
|
+
* `headRef`, so it cannot exclude any pull request from a branch match, and a
|
|
18920
|
+
* title hit alone (score 20) must never be returned as the answer while an
|
|
18921
|
+
* unread branch match (score 30) could outrank it. Until the row contract
|
|
18922
|
+
* carries `headRef` (relayfile-adapters#271), the only honest thing to do with
|
|
18923
|
+
* the index is say out loud why it did not help.
|
|
18924
|
+
*/
|
|
18925
|
+
const classifyPullIndexFallback = async (mount, repo) => {
|
|
18926
|
+
const [owner, name] = repo.split('/');
|
|
18927
|
+
if (!owner || !name)
|
|
18928
|
+
return 'index-absent';
|
|
18929
|
+
let parsed;
|
|
18930
|
+
try {
|
|
18931
|
+
const { content } = await mount.readFile(`${GITHUB_ISSUE_ROOT}/${owner}/${name}/pulls/_index.json`);
|
|
18932
|
+
parsed = parseJsonContent(content);
|
|
18933
|
+
}
|
|
18934
|
+
catch (error) {
|
|
18935
|
+
// A mount that is failing pass-wide must not be reported as "no index" —
|
|
18936
|
+
// that is the same convention the tree reads below already follow.
|
|
18937
|
+
if (isPassWideRelayfileFault(error))
|
|
18938
|
+
throw error;
|
|
18939
|
+
return 'index-absent';
|
|
18940
|
+
}
|
|
18941
|
+
if (!Array.isArray(parsed))
|
|
18942
|
+
return 'index-shape-unrecognised';
|
|
18943
|
+
// An empty index proves nothing about the row contract, so it must not read
|
|
18944
|
+
// as usable — `[].every(...)` is vacuously true.
|
|
18945
|
+
return parsed.length > 0 && parsed.every((entry) => typeof asRecord(entry)?.headRef === 'string')
|
|
18946
|
+
? 'index-usable'
|
|
18947
|
+
: 'index-without-head-ref';
|
|
18948
|
+
};
|
|
18949
|
+
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix), observer = {}) => {
|
|
18281
18950
|
const candidates = [];
|
|
18282
18951
|
const listErrors = [];
|
|
18283
18952
|
for (const repo of opts.repo ? [opts.repo] : reposFromConfig(config)) {
|
|
18953
|
+
if (observer.onIndexFallback) {
|
|
18954
|
+
observer.onIndexFallback(repo, await classifyPullIndexFallback(mount, repo));
|
|
18955
|
+
}
|
|
18284
18956
|
const paths = new Set();
|
|
18285
18957
|
for (const root of githubPullRoots(repo)) {
|
|
18286
18958
|
try {
|
|
@@ -18293,10 +18965,41 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
18293
18965
|
listErrors.push(error);
|
|
18294
18966
|
}
|
|
18295
18967
|
}
|
|
18968
|
+
// `githubPullRoots` is plural, and both of its roots describe the SAME
|
|
18969
|
+
// repository: the nested `<owner>/<repo>/pulls/` layout and the flat
|
|
18970
|
+
// `<owner>__<repo>/pulls/by-id/` one. A pull request is generally present
|
|
18971
|
+
// under both, spelled differently, so the union walked most PRs twice — in
|
|
18972
|
+
// the live workspace 2877 nested paths plus 1156 by-id paths for roughly
|
|
18973
|
+
// 1156 actual pull requests.
|
|
18974
|
+
//
|
|
18975
|
+
// Collapse them on the identity the PATH already carries. `githubPullPathParts`
|
|
18976
|
+
// reads owner/repo/number out of either spelling and costs no mount read, so
|
|
18977
|
+
// the dedupe is free. Insertion order is preserved and the first spelling
|
|
18978
|
+
// encountered wins, which is the same candidate the existing stable sort kept
|
|
18979
|
+
// when two spellings of one PR tied — so the winner does not move.
|
|
18980
|
+
//
|
|
18981
|
+
// Paths the matcher does not recognise (`pulls/_index.json`, per-PR
|
|
18982
|
+
// `comments/*.json`) carry no PR identity, so they are left in the walk and
|
|
18983
|
+
// still read exactly as before rather than being filtered on a guess.
|
|
18984
|
+
const walk = [];
|
|
18985
|
+
const seenPulls = new Set();
|
|
18296
18986
|
for (const path of paths) {
|
|
18297
18987
|
if (!path.endsWith('.json'))
|
|
18298
18988
|
continue;
|
|
18989
|
+
const parts = githubPullPathParts(path);
|
|
18990
|
+
if (parts) {
|
|
18991
|
+
const identity = `${parts.owner}/${parts.repo}#${parts.number}`.toLowerCase();
|
|
18992
|
+
if (seenPulls.has(identity))
|
|
18993
|
+
continue;
|
|
18994
|
+
seenPulls.add(identity);
|
|
18995
|
+
}
|
|
18996
|
+
walk.push(path);
|
|
18997
|
+
}
|
|
18998
|
+
let read = 0;
|
|
18999
|
+
for (const path of walk) {
|
|
18299
19000
|
const pr = await readProbePrCandidate(mount, path);
|
|
19001
|
+
read += 1;
|
|
19002
|
+
observer.onRead?.({ read, total: walk.length, path });
|
|
18300
19003
|
if (opts.openOnly && normalizePrState(pr?.state) !== 'OPEN')
|
|
18301
19004
|
continue;
|
|
18302
19005
|
const score = pr
|