@agent-relay/factory 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/cli/diagnose.d.ts +18 -2
- package/dist/cli/diagnose.d.ts.map +1 -1
- package/dist/cli/diagnose.js +94 -5
- package/dist/cli/diagnose.js.map +1 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +10 -0
- package/dist/cli/fleet.js.map +1 -1
- package/dist/cli/teammate-mcp.d.ts +19 -0
- package/dist/cli/teammate-mcp.d.ts.map +1 -0
- package/dist/cli/teammate-mcp.js +102 -0
- package/dist/cli/teammate-mcp.js.map +1 -0
- package/dist/environments/load-profile.d.ts +4 -4
- package/dist/environments/stack-descriptor.d.ts +17 -17
- package/dist/environments/verification-stack-descriptor.d.ts +32 -32
- package/dist/fleet/control-plane-circuit.d.ts +10 -0
- package/dist/fleet/control-plane-circuit.d.ts.map +1 -1
- package/dist/fleet/control-plane-circuit.js +9 -1
- package/dist/fleet/control-plane-circuit.js.map +1 -1
- package/dist/fleet/create-fleet.d.ts +2 -0
- package/dist/fleet/create-fleet.d.ts.map +1 -1
- package/dist/fleet/create-fleet.js +1 -0
- package/dist/fleet/create-fleet.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +29 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +161 -11
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +43 -2
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +264 -19
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/fleet/teammates.d.ts +46 -0
- package/dist/fleet/teammates.d.ts.map +1 -0
- package/dist/fleet/teammates.js +424 -0
- package/dist/fleet/teammates.js.map +1 -0
- package/dist/index.d.ts +11 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +4 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +18 -6
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/mount/relayfile-github-connection-write.d.ts +3 -2
- package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
- package/dist/mount/relayfile-github-connection-write.js +33 -0
- package/dist/mount/relayfile-github-connection-write.js.map +1 -1
- package/dist/node/factory-node-runtime.d.ts +17 -0
- package/dist/node/factory-node-runtime.d.ts.map +1 -0
- package/dist/node/factory-node-runtime.js +100 -0
- package/dist/node/factory-node-runtime.js.map +1 -0
- package/dist/node/factory-node.d.ts +10 -1
- package/dist/node/factory-node.d.ts.map +1 -1
- package/dist/node/factory-node.js +13 -2
- package/dist/node/factory-node.js.map +1 -1
- package/dist/node/factory-persona-card.d.ts +38 -0
- package/dist/node/factory-persona-card.d.ts.map +1 -0
- package/dist/node/factory-persona-card.js +163 -0
- package/dist/node/factory-persona-card.js.map +1 -0
- package/dist/node/factory.node.d.ts +1 -1
- package/dist/observability/events.d.ts +38 -38
- package/dist/orchestrator/dispatch-failure-reason.d.ts +60 -0
- package/dist/orchestrator/dispatch-failure-reason.d.ts.map +1 -0
- package/dist/orchestrator/dispatch-failure-reason.js +127 -0
- package/dist/orchestrator/dispatch-failure-reason.js.map +1 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1175 -103
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts.map +1 -1
- package/dist/orchestrator/public-health.js +152 -6
- package/dist/orchestrator/public-health.js.map +1 -1
- package/dist/orchestrator/release-error.d.ts +26 -0
- package/dist/orchestrator/release-error.d.ts.map +1 -0
- package/dist/orchestrator/release-error.js +33 -0
- package/dist/orchestrator/release-error.js.map +1 -0
- package/dist/ports/fleet.d.ts +104 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/fleet.js +10 -1
- package/dist/ports/fleet.js.map +1 -1
- package/dist/ports/index.d.ts +4 -3
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/index.js +1 -1
- package/dist/ports/index.js.map +1 -1
- package/dist/ports/mount.d.ts +10 -2
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/writeback.d.ts +45 -4
- package/dist/ports/writeback.d.ts.map +1 -1
- package/dist/state/watch-state-document.js +4 -1
- package/dist/state/watch-state-document.js.map +1 -1
- package/dist/testing/fakes.d.ts +8 -3
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js +25 -1
- package/dist/testing/fakes.js.map +1 -1
- package/dist/triage/schema.d.ts +14 -14
- package/dist/types.d.ts +170 -14
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/github.d.ts +17 -9
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +331 -9
- package/dist/writeback/github.js.map +1 -1
- package/dist/writeback/linear.d.ts +5 -1
- package/dist/writeback/linear.d.ts.map +1 -1
- package/dist/writeback/linear.js +89 -1
- package/dist/writeback/linear.js.map +1 -1
- package/package.json +4 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
1
2
|
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import { readFile } from 'node:fs/promises';
|
|
3
4
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
@@ -36,6 +37,8 @@ import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../obse
|
|
|
36
37
|
import { telemetryErrorClass } from '../observability/error-class.js';
|
|
37
38
|
import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
|
|
38
39
|
import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
|
|
40
|
+
import { factoryDispatchFailureReasonCodeForErrorClass, factoryDispatchFailureReasonCounts, } from './dispatch-failure-reason.js';
|
|
41
|
+
import { isAgentAlreadyGoneOnRelease } from './release-error.js';
|
|
39
42
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
40
43
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
41
44
|
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
@@ -143,6 +146,12 @@ const CLARIFICATION_ESCALATION_LEASE_MS = 2 * 60_000;
|
|
|
143
146
|
const CLARIFICATION_ESCALATION_RETRY_MS = 5_000;
|
|
144
147
|
const CLARIFICATION_STALE_WARN_MS = 7 * 24 * 60 * 60_000;
|
|
145
148
|
const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
149
|
+
// A rejected post-spawn dispatch normally gets a final opportunity to undo an
|
|
150
|
+
// external claim before shutdown relinquishes its lifecycle lease. Provider
|
|
151
|
+
// writes are not guaranteed to settle, so that opportunity must stay inside a
|
|
152
|
+
// bounded shutdown budget; a successor can recover from the retained durable
|
|
153
|
+
// lifecycle after this process releases its local agents.
|
|
154
|
+
const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
|
|
146
155
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
147
156
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
148
157
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
@@ -347,6 +356,16 @@ class ReadinessReconcileTimeoutError extends Error {
|
|
|
347
356
|
this.name = 'ReadinessReconcileTimeoutError';
|
|
348
357
|
}
|
|
349
358
|
}
|
|
359
|
+
class PostSpawnDispatchWaitRejectedError extends Error {
|
|
360
|
+
issueKey;
|
|
361
|
+
compensationError;
|
|
362
|
+
constructor(issueKey, compensationError) {
|
|
363
|
+
super(`Post-spawn dispatch waits were rejected for ${issueKey}`);
|
|
364
|
+
this.issueKey = issueKey;
|
|
365
|
+
this.compensationError = compensationError;
|
|
366
|
+
this.name = 'PostSpawnDispatchWaitRejectedError';
|
|
367
|
+
}
|
|
368
|
+
}
|
|
350
369
|
const realClock = {
|
|
351
370
|
now: () => Date.now(),
|
|
352
371
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -354,6 +373,26 @@ const realClock = {
|
|
|
354
373
|
export function createFactory(config, ports) {
|
|
355
374
|
return new FactoryLoop(FactoryConfigSchema.parse(config), ports);
|
|
356
375
|
}
|
|
376
|
+
/**
|
|
377
|
+
* The discovery pass a tree read was issued by, carried per async call (#363
|
|
378
|
+
* review, CodeRabbit).
|
|
379
|
+
*
|
|
380
|
+
* `#discoverySweepEpoch` says a sweep is in flight; it cannot say the read in
|
|
381
|
+
* hand belongs to it. In live mode an event drain reaches the very same
|
|
382
|
+
* enumeration helpers — `#handlePrChange` -> `#advanceMergedPrToDone` ->
|
|
383
|
+
* `#findMergeAdvanceIssueForPr` -> `#githubIssuePaths()` is the concrete path —
|
|
384
|
+
* so a drain's full-root walk would land in a concurrent sweep's ratio while
|
|
385
|
+
* measuring a different instant. That is enough to hide a mount that went
|
|
386
|
+
* silent mid-sweep.
|
|
387
|
+
*
|
|
388
|
+
* A drain does not inherit this store: its continuation begins at the
|
|
389
|
+
* subscription callback, outside the `run()` below. Threading an epoch argument
|
|
390
|
+
* through `#ingestGithubIssues` / `#handleGithubIssueChange` /
|
|
391
|
+
* `#findGithubIssueMirror` / `#loadLinearMirrorCandidates` would express the
|
|
392
|
+
* same fact and put a parameter on every hot path that must never be passed
|
|
393
|
+
* wrong.
|
|
394
|
+
*/
|
|
395
|
+
const discoveryEnumerationPass = new AsyncLocalStorage();
|
|
357
396
|
export class FactoryLoop {
|
|
358
397
|
#config;
|
|
359
398
|
#mount;
|
|
@@ -553,18 +592,30 @@ export class FactoryLoop {
|
|
|
553
592
|
#readinessReconcileLastError;
|
|
554
593
|
#readinessReconcileLastErrorClass;
|
|
555
594
|
/**
|
|
556
|
-
* The last *
|
|
595
|
+
* The last *enumerating* sweep's arithmetic (#355).
|
|
557
596
|
*
|
|
558
|
-
* Held as one record rather than
|
|
597
|
+
* Held as one record rather than three fields so it can only ever be replaced
|
|
559
598
|
* whole: publishing a `dispatched` from one pass beside a `candidates` from
|
|
560
599
|
* another would be worse than publishing neither, since the whole use of
|
|
561
600
|
* these numbers is comparing them to each other.
|
|
562
601
|
*
|
|
563
|
-
* `undefined` until a sweep
|
|
602
|
+
* `undefined` until a sweep enumerates, and never initialised to zeroes —
|
|
564
603
|
* "this daemon has not finished a sweep" and "a sweep finished and found
|
|
565
604
|
* nothing" are the two readings #355 has to tell apart.
|
|
566
605
|
*/
|
|
567
606
|
#readinessReconcileLastSweep;
|
|
607
|
+
/**
|
|
608
|
+
* Whether the MOST RECENT pass deferred, tracked apart from the counts above
|
|
609
|
+
* (#358 review, CodeRabbit — Major, and right).
|
|
610
|
+
*
|
|
611
|
+
* A deferred pass enumerates nothing and settles in milliseconds, so folding
|
|
612
|
+
* it into the snapshot overwrote the last real sweep's numbers with zeroes.
|
|
613
|
+
* On a container where another process holds the lease for any length of time
|
|
614
|
+
* — the #347/#349 condition — every pass would publish `candidates: 0` and the
|
|
615
|
+
* last actual enumeration would be unrecoverable, destroying the measurement
|
|
616
|
+
* this whole change exists to provide.
|
|
617
|
+
*/
|
|
618
|
+
#readinessReconcileLastSweepDeferred;
|
|
568
619
|
#liveEventQueue = [];
|
|
569
620
|
#liveEventDrainScheduled = false;
|
|
570
621
|
#liveEventDrainActive = false;
|
|
@@ -577,6 +628,29 @@ export class FactoryLoop {
|
|
|
577
628
|
#previewSweepTimer;
|
|
578
629
|
#previewSweepInFlight;
|
|
579
630
|
#completionInFlight = new Set();
|
|
631
|
+
#issueWritebackInFlight = new Map();
|
|
632
|
+
// A fast completion can make terminal issue state visible while dispatch is
|
|
633
|
+
// still performing its post-spawn readiness read. Completion publishes its
|
|
634
|
+
// provider receipt, then waits for that read to classify the state as owned
|
|
635
|
+
// or foreign before it releases agents under a terminal success reason.
|
|
636
|
+
#postSpawnIssueObservations = new Map();
|
|
637
|
+
// The post-spawn ready read and following dispatch claim form one local
|
|
638
|
+
// write boundary. A completion arriving first makes dispatch wait and
|
|
639
|
+
// re-read; one arriving after claim entry waits for the claim to finish.
|
|
640
|
+
#postSpawnDispatchClaimFences = new Map();
|
|
641
|
+
// Shutdown must not lose a local placement merely because a rejected
|
|
642
|
+
// provider claim unwinds before #releaseInFlightAgents snapshots the batch.
|
|
643
|
+
// The capture set makes both interleavings explicit: the dispatch catch may
|
|
644
|
+
// remove a record only after shutdown has copied its agents, otherwise the
|
|
645
|
+
// release sweep removes it after the copy is safe.
|
|
646
|
+
#shutdownReleaseCapturedDispatches = new Set();
|
|
647
|
+
#postSpawnDispatchesRejectedDuringStop = new Set();
|
|
648
|
+
// A rejected provider claim is unsafe to release until its conditional
|
|
649
|
+
// rollback succeeds or a newer provider status supersedes it. This runtime
|
|
650
|
+
// set closes the race before the durable dispatchClaim flag is persisted.
|
|
651
|
+
#uncompensatedDispatchClaims = new Set();
|
|
652
|
+
#pendingDispatchClaims = new Set();
|
|
653
|
+
#dispatchClaimSettlementsInFlight = new Set();
|
|
580
654
|
#agentExitsInFlight = new Map();
|
|
581
655
|
#reconciledAgentExitsActive = 0;
|
|
582
656
|
#reconciledAgentExitWaiters = [];
|
|
@@ -665,6 +739,10 @@ export class FactoryLoop {
|
|
|
665
739
|
#discoveryOverloadError;
|
|
666
740
|
/** Relayfile operations this sweep has been shed on. */
|
|
667
741
|
#discoverySweepOverloads = 0;
|
|
742
|
+
/** Tree reads this sweep issued that the backend served (#351 follow-up). */
|
|
743
|
+
#discoverySweepTreeReads = 0;
|
|
744
|
+
/** How many of those were served with zero entries. */
|
|
745
|
+
#discoverySweepEmptyTreeReads = 0;
|
|
668
746
|
/**
|
|
669
747
|
* The longest `Retry-After` any operation in this sweep advertised.
|
|
670
748
|
*
|
|
@@ -971,6 +1049,15 @@ export class FactoryLoop {
|
|
|
971
1049
|
}
|
|
972
1050
|
}
|
|
973
1051
|
async #start(opts) {
|
|
1052
|
+
// These sets coordinate one stop snapshot with dispatches unwinding during
|
|
1053
|
+
// that same stop. They are not durable lifecycle state: retaining their
|
|
1054
|
+
// keys across a later start would make the next stop mistake a fresh
|
|
1055
|
+
// dispatch for one already captured by the previous shutdown.
|
|
1056
|
+
this.#shutdownReleaseCapturedDispatches.clear();
|
|
1057
|
+
this.#postSpawnDispatchesRejectedDuringStop.clear();
|
|
1058
|
+
this.#uncompensatedDispatchClaims.clear();
|
|
1059
|
+
this.#pendingDispatchClaims.clear();
|
|
1060
|
+
this.#dispatchClaimSettlementsInFlight.clear();
|
|
974
1061
|
this.#stopping = false;
|
|
975
1062
|
this.#startMode = opts.mode ?? 'live';
|
|
976
1063
|
const issueSource = await this.#issueSource();
|
|
@@ -1085,15 +1172,32 @@ export class FactoryLoop {
|
|
|
1085
1172
|
async stop() {
|
|
1086
1173
|
this.#started = false;
|
|
1087
1174
|
this.#stopping = true;
|
|
1175
|
+
// These waits are armed before every planned spawn returns. Resolve them
|
|
1176
|
+
// before the first shutdown await: a fast first agent may be completing
|
|
1177
|
+
// while a later spawn is hung, and #drainAgentExitsInFlight would otherwise
|
|
1178
|
+
// wait forever on a resolver stranded inside that dispatch stack.
|
|
1179
|
+
const postSpawnWaitKeys = new Set([
|
|
1180
|
+
...this.#postSpawnIssueObservations.keys(),
|
|
1181
|
+
...this.#postSpawnDispatchClaimFences.keys(),
|
|
1182
|
+
]);
|
|
1183
|
+
const rejectedClaimDispatches = new Set();
|
|
1184
|
+
for (const [key, fence] of this.#postSpawnDispatchClaimFences) {
|
|
1185
|
+
if (!fence.claimStarted)
|
|
1186
|
+
continue;
|
|
1187
|
+
const dispatchKeyPrefix = `${key}:live:`;
|
|
1188
|
+
for (const [dispatchKey, dispatch] of this.#dispatchInFlight) {
|
|
1189
|
+
if (dispatchKey.startsWith(dispatchKeyPrefix))
|
|
1190
|
+
rejectedClaimDispatches.add(dispatch);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
for (const key of postSpawnWaitKeys)
|
|
1194
|
+
this.#settlePostSpawnDispatchWaits(key, false, 'stop');
|
|
1088
1195
|
if (this.#babysitterResourceDeliveryRetryTimer)
|
|
1089
1196
|
clearTimeout(this.#babysitterResourceDeliveryRetryTimer);
|
|
1090
1197
|
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
1091
1198
|
if (this.#babysitterResourceSubscriptionRenewTimer)
|
|
1092
1199
|
clearTimeout(this.#babysitterResourceSubscriptionRenewTimer);
|
|
1093
1200
|
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
1094
|
-
if (this.#dispatchLifecycleRenewTimer)
|
|
1095
|
-
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
1096
|
-
this.#dispatchLifecycleRenewTimer = undefined;
|
|
1097
1201
|
if (this.#heldAgentDeadlineTimer)
|
|
1098
1202
|
clearTimeout(this.#heldAgentDeadlineTimer);
|
|
1099
1203
|
this.#heldAgentDeadlineTimer = undefined;
|
|
@@ -1126,12 +1230,25 @@ export class FactoryLoop {
|
|
|
1126
1230
|
await this.#previewSweepInFlight;
|
|
1127
1231
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
1128
1232
|
try {
|
|
1233
|
+
// Only dispatches that had entered the provider-claim boundary can leave
|
|
1234
|
+
// an external claim behind. Drain those rejected promises before
|
|
1235
|
+
// relinquishing their lifecycle leases or tearing down the fleet/mount.
|
|
1236
|
+
// A pre-claim spawn hang is deliberately absent from this set, so it
|
|
1237
|
+
// cannot hold shutdown open.
|
|
1238
|
+
await this.#drainRejectedClaimDispatchesForStop(rejectedClaimDispatches);
|
|
1239
|
+
await this.#persistOutstandingClaimCancellationBlocks();
|
|
1240
|
+
// Keep renewing ownership through the potentially slow provider drain
|
|
1241
|
+
// and cancellation-fence handoff. Stopping renewal earlier can let the
|
|
1242
|
+
// lease expire before the fence is saved, allowing a successor to reap
|
|
1243
|
+
// agents while the old provider write can still land.
|
|
1129
1244
|
// Relinquish durable ownership before waiting on mount-backed lifecycle
|
|
1130
1245
|
// drives. A slow Relayfile scan must not consume the shutdown deadline
|
|
1131
1246
|
// while every issue remains fenced to a publisher that is already
|
|
1132
|
-
// stopping.
|
|
1133
|
-
//
|
|
1134
|
-
|
|
1247
|
+
// stopping. Preserve leases for provider claims whose bounded drain
|
|
1248
|
+
// expired: the original caller still needs its epoch to persist a late
|
|
1249
|
+
// conclusive settlement. A second sweep below catches claims racing this
|
|
1250
|
+
// one.
|
|
1251
|
+
await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
|
|
1135
1252
|
await Promise.allSettled([...this.#dispatchLifecycleDrives]);
|
|
1136
1253
|
// Fence every source of new clarification work before touching the fleet.
|
|
1137
1254
|
// A wake already past the fence is allowed to unwind, and is awaited
|
|
@@ -1154,13 +1271,18 @@ export class FactoryLoop {
|
|
|
1154
1271
|
// non-durable (local/internal) records; terminal completion performs the
|
|
1155
1272
|
// normal remote release before clearing the lifecycle.
|
|
1156
1273
|
await this.#releaseInFlightAgents('factory-stopped', { preserveDurable: true });
|
|
1157
|
-
|
|
1274
|
+
if (this.#dispatchLifecycleRenewTimer)
|
|
1275
|
+
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
1276
|
+
this.#dispatchLifecycleRenewTimer = undefined;
|
|
1277
|
+
await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
|
|
1158
1278
|
if (this.#livePollTimer)
|
|
1159
1279
|
clearTimeout(this.#livePollTimer);
|
|
1160
1280
|
this.#livePollTimer = undefined;
|
|
1161
1281
|
this.#livePollInFlight = false;
|
|
1162
1282
|
this.#liveEventQueue.length = 0;
|
|
1163
1283
|
this.#completionInFlight.clear();
|
|
1284
|
+
this.#postSpawnIssueObservations.clear();
|
|
1285
|
+
this.#postSpawnDispatchClaimFences.clear();
|
|
1164
1286
|
this.#babysitterSpawned.clear();
|
|
1165
1287
|
this.#babysitterPr.clear();
|
|
1166
1288
|
this.#babysitterIssueRefs.clear();
|
|
@@ -1207,10 +1329,13 @@ export class FactoryLoop {
|
|
|
1207
1329
|
await this.#fleet.dispose();
|
|
1208
1330
|
}
|
|
1209
1331
|
finally {
|
|
1332
|
+
if (this.#dispatchLifecycleRenewTimer)
|
|
1333
|
+
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
1334
|
+
this.#dispatchLifecycleRenewTimer = undefined;
|
|
1210
1335
|
this.#stoppingHeartbeatRefreshActive = false;
|
|
1211
1336
|
}
|
|
1212
1337
|
}
|
|
1213
|
-
async #releaseOwnedDispatchLifecycleLeases() {
|
|
1338
|
+
async #releaseOwnedDispatchLifecycleLeases(preserve = () => false) {
|
|
1214
1339
|
// The epoch cache is an execution optimization, not the durable ownership
|
|
1215
1340
|
// authority. Error/fence paths may evict a cached epoch while its persisted
|
|
1216
1341
|
// lease is still ours, so enumerate state before shutdown relinquishment.
|
|
@@ -1221,6 +1346,8 @@ export class FactoryLoop {
|
|
|
1221
1346
|
}
|
|
1222
1347
|
}
|
|
1223
1348
|
for (const [key, epoch] of owned) {
|
|
1349
|
+
if (preserve(key))
|
|
1350
|
+
continue;
|
|
1224
1351
|
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
1225
1352
|
if (this.#dispatchLifecycleEpochs.get(key) === epoch) {
|
|
1226
1353
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
@@ -1244,6 +1371,215 @@ export class FactoryLoop {
|
|
|
1244
1371
|
await Promise.allSettled([...this.#clarificationQuestionDeliveryInFlight.values()]);
|
|
1245
1372
|
}
|
|
1246
1373
|
}
|
|
1374
|
+
async #drainRejectedClaimDispatchesForStop(dispatches) {
|
|
1375
|
+
if (dispatches.size === 0)
|
|
1376
|
+
return;
|
|
1377
|
+
let timer;
|
|
1378
|
+
const drained = Promise.allSettled([...dispatches]).then(() => true);
|
|
1379
|
+
// A durable fleet can recover ownership after this process exits. Local
|
|
1380
|
+
// and other non-durable fleets cannot: releasing their placements before
|
|
1381
|
+
// the provider claim has compensated would leave no successor able to
|
|
1382
|
+
// finish the lifecycle, and a late write could strand the issue in its
|
|
1383
|
+
// claimed state. Keep draining those fleets without the shutdown escape
|
|
1384
|
+
// hatch.
|
|
1385
|
+
if (this.#fleet.durableOwnership !== true) {
|
|
1386
|
+
await drained;
|
|
1387
|
+
return;
|
|
1388
|
+
}
|
|
1389
|
+
const timedOut = new Promise((resolve) => {
|
|
1390
|
+
timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
|
|
1391
|
+
timer.unref?.();
|
|
1392
|
+
});
|
|
1393
|
+
const completed = await Promise.race([drained, timedOut]);
|
|
1394
|
+
if (timer)
|
|
1395
|
+
clearTimeout(timer);
|
|
1396
|
+
if (completed)
|
|
1397
|
+
return;
|
|
1398
|
+
this.#increment('postSpawnDispatchClaimDrainTimeouts');
|
|
1399
|
+
this.#logger.warn?.('[factory] rejected post-spawn dispatch compensation timed out; continuing shutdown', {
|
|
1400
|
+
dispatches: dispatches.size,
|
|
1401
|
+
timeoutMs: STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS,
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1404
|
+
async #persistOutstandingClaimCancellationBlocks() {
|
|
1405
|
+
const batch = await this.#batch();
|
|
1406
|
+
for (const record of batch.inFlight) {
|
|
1407
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1408
|
+
if (!this.#pendingDispatchClaims.has(key))
|
|
1409
|
+
continue;
|
|
1410
|
+
// The bounded durable drain expired before the provider operation could
|
|
1411
|
+
// report whether rollback was safe. Persist the provisional release
|
|
1412
|
+
// fence while this publisher still owns the lifecycle lease, so a
|
|
1413
|
+
// successor cannot mistake the unresolved claim for abandonable work.
|
|
1414
|
+
const persisted = await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not settle before shutdown'), { preferExistingError: true, pending: true });
|
|
1415
|
+
if (!persisted) {
|
|
1416
|
+
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
1417
|
+
if (durable?.dispatchClaim?.cancellationPending !== true) {
|
|
1418
|
+
throw new Error(`Unable to persist unresolved dispatch-claim fence for ${record.issue.key}; refusing shutdown handoff`);
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
async #awaitRejectedClaimCompensation(settled) {
|
|
1424
|
+
let timer;
|
|
1425
|
+
const timedOut = new Promise((resolve) => {
|
|
1426
|
+
timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
|
|
1427
|
+
timer.unref?.();
|
|
1428
|
+
});
|
|
1429
|
+
const compensated = await Promise.race([settled, timedOut]);
|
|
1430
|
+
if (timer)
|
|
1431
|
+
clearTimeout(timer);
|
|
1432
|
+
return compensated;
|
|
1433
|
+
}
|
|
1434
|
+
/**
|
|
1435
|
+
* Re-adopt this publisher's own lease epoch after a refused lifecycle write.
|
|
1436
|
+
*
|
|
1437
|
+
* `#saveDispatchLifecycle` cannot tell "the store rejected our fence" from
|
|
1438
|
+
* "the store declined this row's contents", so it evicts the cached epoch for
|
|
1439
|
+
* both. Only the first is a loss of ownership. Re-read the durable row and
|
|
1440
|
+
* take the epoch back when the lease is still ours and still live; a row that
|
|
1441
|
+
* is gone, terminal, expired, or owned by someone else keeps the eviction,
|
|
1442
|
+
* because for those the eviction was correct.
|
|
1443
|
+
*/
|
|
1444
|
+
async #recoverDispatchLifecycleLeaseEpoch(key) {
|
|
1445
|
+
if (this.#dispatchLifecycleEpochs.has(key))
|
|
1446
|
+
return true;
|
|
1447
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
1448
|
+
const lease = lifecycle?.lease;
|
|
1449
|
+
if (!lifecycle || !lease || isTerminalDispatchLifecycle(lifecycle))
|
|
1450
|
+
return false;
|
|
1451
|
+
if (lease.owner !== this.#dispatchLifecycleOwner)
|
|
1452
|
+
return false;
|
|
1453
|
+
if (lease.leaseUntilMs <= this.#clock.now())
|
|
1454
|
+
return false;
|
|
1455
|
+
this.#dispatchLifecycleEpochs.set(key, lease.epoch);
|
|
1456
|
+
this.#increment('dispatchLifecycleLeaseEpochsRecovered');
|
|
1457
|
+
return true;
|
|
1458
|
+
}
|
|
1459
|
+
/**
|
|
1460
|
+
* Persist a dispatch-claim fence without letting a refused write disarm the
|
|
1461
|
+
* lease the next one needs.
|
|
1462
|
+
*
|
|
1463
|
+
* The shutdown handoff deliberately probes with a provisional
|
|
1464
|
+
* `cancellationPending` fence, and a store may refuse that row on its
|
|
1465
|
+
* contents while this process still holds the lease. Losing the epoch there
|
|
1466
|
+
* would make every later write for this lifecycle short-circuit on
|
|
1467
|
+
* `epoch === undefined` — including the conclusive compensated settlement
|
|
1468
|
+
* that clears the block and frees the agents and the batch slot. That turns
|
|
1469
|
+
* one refused probe into a lifecycle retained until restart, so bracket the
|
|
1470
|
+
* write with epoch recovery instead.
|
|
1471
|
+
*/
|
|
1472
|
+
async #persistDispatchClaimFence(record) {
|
|
1473
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1474
|
+
await this.#recoverDispatchLifecycleLeaseEpoch(key);
|
|
1475
|
+
const saved = await this.#saveDispatchLifecycle(record, 'running');
|
|
1476
|
+
if (!saved)
|
|
1477
|
+
await this.#recoverDispatchLifecycleLeaseEpoch(key);
|
|
1478
|
+
return saved;
|
|
1479
|
+
}
|
|
1480
|
+
async #retainUncompensatedDispatchClaim(record, error, opts = {}) {
|
|
1481
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1482
|
+
const alreadyBlocked = record.dispatchClaim?.cancellationBlocked === true;
|
|
1483
|
+
const errorMessage = describeError(error).errorMessage;
|
|
1484
|
+
this.#uncompensatedDispatchClaims.add(key);
|
|
1485
|
+
const { cancellationPending: _pending, ...existingClaim } = record.dispatchClaim ?? { state: 'degraded' };
|
|
1486
|
+
record.dispatchClaim = {
|
|
1487
|
+
...existingClaim,
|
|
1488
|
+
state: 'degraded',
|
|
1489
|
+
cancellationBlocked: true,
|
|
1490
|
+
...(opts.pending ? { cancellationPending: true } : {}),
|
|
1491
|
+
write: 'rejected dispatch claim rollback',
|
|
1492
|
+
error: opts.preferExistingError ? record.dispatchClaim?.error ?? errorMessage : errorMessage,
|
|
1493
|
+
deadLettered: true,
|
|
1494
|
+
updatedAtMs: this.#clock.now(),
|
|
1495
|
+
};
|
|
1496
|
+
this.#dispatchClaimStatuses.set(key, record.dispatchClaim);
|
|
1497
|
+
if (!alreadyBlocked) {
|
|
1498
|
+
this.#increment('postSpawnDispatchClaimRecoveryRetentions');
|
|
1499
|
+
this.#logger.warn?.('[factory] retaining dispatch lifecycle because rejected claim compensation is unproven', {
|
|
1500
|
+
issue: record.issue.key,
|
|
1501
|
+
});
|
|
1502
|
+
}
|
|
1503
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
1504
|
+
return await this.#persistDispatchClaimFence(record);
|
|
1505
|
+
}
|
|
1506
|
+
else {
|
|
1507
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
1508
|
+
return true;
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
async #clearDispatchCancellationBlock(record) {
|
|
1512
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1513
|
+
if (record.dispatchClaim?.cancellationBlocked !== true) {
|
|
1514
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
1515
|
+
return true;
|
|
1516
|
+
}
|
|
1517
|
+
const previousClaim = record.dispatchClaim;
|
|
1518
|
+
const { cancellationBlocked: _blocked, cancellationPending: _pending, ...claim } = previousClaim;
|
|
1519
|
+
record.dispatchClaim = claim;
|
|
1520
|
+
this.#dispatchClaimStatuses.set(key, claim);
|
|
1521
|
+
try {
|
|
1522
|
+
const persisted = this.#usesDurableDispatchLifecycle()
|
|
1523
|
+
? await this.#persistDispatchClaimFence(record)
|
|
1524
|
+
: await this.#writeDispatchClaimRegistry(record.issue).then(() => true);
|
|
1525
|
+
if (!persisted) {
|
|
1526
|
+
record.dispatchClaim = previousClaim;
|
|
1527
|
+
this.#dispatchClaimStatuses.set(key, previousClaim);
|
|
1528
|
+
return false;
|
|
1529
|
+
}
|
|
1530
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
1531
|
+
return true;
|
|
1532
|
+
}
|
|
1533
|
+
catch (error) {
|
|
1534
|
+
record.dispatchClaim = previousClaim;
|
|
1535
|
+
this.#dispatchClaimStatuses.set(key, previousClaim);
|
|
1536
|
+
throw error;
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
async #dispatchClaimBlocksAbandonment(record) {
|
|
1540
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1541
|
+
if (!this.#uncompensatedDispatchClaims.has(key) && record.dispatchClaim?.cancellationBlocked !== true) {
|
|
1542
|
+
return false;
|
|
1543
|
+
}
|
|
1544
|
+
// A provider request that was still unresolved at handoff can land after
|
|
1545
|
+
// any current-state read. No observation is proof of supersession until
|
|
1546
|
+
// the original caller records that the operation conclusively settled.
|
|
1547
|
+
if (record.dispatchClaim?.cancellationPending === true)
|
|
1548
|
+
return true;
|
|
1549
|
+
try {
|
|
1550
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
1551
|
+
if (!issue)
|
|
1552
|
+
return true;
|
|
1553
|
+
if (isGithubIssue(issue)) {
|
|
1554
|
+
if (!this.#githubWriteback.getIssueStatus)
|
|
1555
|
+
return true;
|
|
1556
|
+
const status = await this.#githubWriteback.getIssueStatus(issue, {
|
|
1557
|
+
requireFresh: true,
|
|
1558
|
+
freshAfterMs: record.dispatchClaim?.claimStartedAtMs,
|
|
1559
|
+
});
|
|
1560
|
+
if (!status || status === 'in-progress')
|
|
1561
|
+
return true;
|
|
1562
|
+
}
|
|
1563
|
+
else {
|
|
1564
|
+
if (!this.#linear.getIssueStateId)
|
|
1565
|
+
return true;
|
|
1566
|
+
const implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
1567
|
+
const currentStateId = await this.#linear.getIssueStateId(issue);
|
|
1568
|
+
if (!implementingStateId || !currentStateId || currentStateId === implementingStateId)
|
|
1569
|
+
return true;
|
|
1570
|
+
}
|
|
1571
|
+
if (!await this.#clearDispatchCancellationBlock(record))
|
|
1572
|
+
return true;
|
|
1573
|
+
return false;
|
|
1574
|
+
}
|
|
1575
|
+
catch (error) {
|
|
1576
|
+
this.#logger.warn?.('[factory] unable to verify whether a blocked dispatch claim was superseded', {
|
|
1577
|
+
issue: record.issue.key,
|
|
1578
|
+
error: describeError(error).errorMessage,
|
|
1579
|
+
});
|
|
1580
|
+
return true;
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1247
1583
|
async #boundedStopTeardown(label, teardown) {
|
|
1248
1584
|
let timer;
|
|
1249
1585
|
const action = Promise.resolve()
|
|
@@ -1331,9 +1667,11 @@ export class FactoryLoop {
|
|
|
1331
1667
|
// cold container it is the first — and for the next interval, only —
|
|
1332
1668
|
// sweep whose counts exist. Leaving it unrecorded would make a daemon
|
|
1333
1669
|
// that has completed a full pass still read as "never ran" (#355).
|
|
1334
|
-
|
|
1670
|
+
const report = await this.runOnce();
|
|
1335
1671
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1336
|
-
|
|
1672
|
+
const completedAtMs = this.#clock.now();
|
|
1673
|
+
this.#readinessReconcileLastCompletedAtMs = completedAtMs;
|
|
1674
|
+
this.#recordReadinessSweepOutcome(report, completedAtMs);
|
|
1337
1675
|
}
|
|
1338
1676
|
catch (error) {
|
|
1339
1677
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
@@ -1559,20 +1897,28 @@ export class FactoryLoop {
|
|
|
1559
1897
|
const report = await this.#runOnceWithReadinessDeadline();
|
|
1560
1898
|
this.#readinessReconcileConsecutiveFailures = 0;
|
|
1561
1899
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1562
|
-
|
|
1900
|
+
const completedAtMs = this.#clock.now();
|
|
1901
|
+
this.#readinessReconcileLastCompletedAtMs = completedAtMs;
|
|
1563
1902
|
this.#readinessReconcileLastError = undefined;
|
|
1564
1903
|
this.#readinessReconcileLastErrorClass = undefined;
|
|
1565
1904
|
// The three integers below have gone to stdout since this loop existed,
|
|
1566
1905
|
// and stdout does not reach the deployed container's operator (#355).
|
|
1567
1906
|
// Publishing them is what lets a reader tell a sweep that saw eligible
|
|
1568
1907
|
// work and rejected it from one that never pulled it at all.
|
|
1569
|
-
this.#recordReadinessSweepOutcome(report);
|
|
1908
|
+
this.#recordReadinessSweepOutcome(report, completedAtMs);
|
|
1570
1909
|
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1571
1910
|
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1572
1911
|
candidates: report.pulled.length,
|
|
1573
1912
|
dispatched: report.dispatched.length,
|
|
1574
1913
|
skipped: report.skipped.length,
|
|
1575
|
-
|
|
1914
|
+
// THIS pass's breakdown, never the retained snapshot (#359 review,
|
|
1915
|
+
// codex P2). Logging the retained one beside a deferred pass's zeroes
|
|
1916
|
+
// produced a line that contradicted its own arithmetic —
|
|
1917
|
+
// `skipped: 0` next to a non-empty breakdown — and this log is what a
|
|
1918
|
+
// local operator reads.
|
|
1919
|
+
skipReasons: factorySweepSkipReasonCounts(report.skipped),
|
|
1920
|
+
dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
|
|
1921
|
+
dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
|
|
1576
1922
|
discoveryDeferred: report.discoveryDeferred,
|
|
1577
1923
|
});
|
|
1578
1924
|
}
|
|
@@ -1596,6 +1942,10 @@ export class FactoryLoop {
|
|
|
1596
1942
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1597
1943
|
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1598
1944
|
this.#readinessReconcileLastError = errorMessage;
|
|
1945
|
+
// This failure is now the latest settled pass. A deferral marker left by
|
|
1946
|
+
// an older pass would falsely describe this one as lease contention when
|
|
1947
|
+
// the timestamps/error below prove that it acquired the lease and failed.
|
|
1948
|
+
this.#readinessReconcileLastSweepDeferred = undefined;
|
|
1599
1949
|
// The class, unlike the message, is publishable: #295 puts it on the
|
|
1600
1950
|
// unauthenticated health surface through the same allowlist.
|
|
1601
1951
|
this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
|
|
@@ -2341,6 +2691,8 @@ export class FactoryLoop {
|
|
|
2341
2691
|
this.#discoverySweepLeaseLost = false;
|
|
2342
2692
|
this.#discoveryOverloadError = undefined;
|
|
2343
2693
|
this.#discoverySweepOverloads = 0;
|
|
2694
|
+
this.#discoverySweepTreeReads = 0;
|
|
2695
|
+
this.#discoverySweepEmptyTreeReads = 0;
|
|
2344
2696
|
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2345
2697
|
this.#discoverySweepProgress = false;
|
|
2346
2698
|
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
@@ -2421,6 +2773,8 @@ export class FactoryLoop {
|
|
|
2421
2773
|
this.#discoverySweepStartedAtMs = undefined;
|
|
2422
2774
|
this.#discoveryOverloadError = undefined;
|
|
2423
2775
|
this.#discoverySweepOverloads = 0;
|
|
2776
|
+
this.#discoverySweepTreeReads = 0;
|
|
2777
|
+
this.#discoverySweepEmptyTreeReads = 0;
|
|
2424
2778
|
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2425
2779
|
this.#discoverySweepProgress = false;
|
|
2426
2780
|
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
@@ -2539,13 +2893,19 @@ export class FactoryLoop {
|
|
|
2539
2893
|
this.#dependencyGithubPathsByIdentity = undefined;
|
|
2540
2894
|
this.#dependencyLinearTreeLoaded = false;
|
|
2541
2895
|
const issueSource = await this.#issueSource();
|
|
2896
|
+
// The sweep's own discovery pass, and the only region whose full-root
|
|
2897
|
+
// walks may enter the tree-read ratio (#363 review). Everything reached
|
|
2898
|
+
// from here inherits the marker; a concurrently running drain does not.
|
|
2899
|
+
const enumerate = (fn) => this.#discoverySweepEpoch === undefined
|
|
2900
|
+
? fn()
|
|
2901
|
+
: discoveryEnumerationPass.run({ epoch: this.#discoverySweepEpoch }, fn);
|
|
2542
2902
|
if (issueSource === 'linear') {
|
|
2543
|
-
await this.#ingestGithubIssues({ dryRun });
|
|
2903
|
+
await enumerate(() => this.#ingestGithubIssues({ dryRun }));
|
|
2544
2904
|
}
|
|
2545
2905
|
else {
|
|
2546
2906
|
await this.#ensureGithubIngestionReady();
|
|
2547
2907
|
}
|
|
2548
|
-
const paths = await this.#readyIssuePaths();
|
|
2908
|
+
const paths = await enumerate(() => this.#readyIssuePaths());
|
|
2549
2909
|
const orphanRecovery = issueSource === 'github'
|
|
2550
2910
|
? await this.#githubOrphanRecoveryContext(dryRun)
|
|
2551
2911
|
: undefined;
|
|
@@ -2703,6 +3063,10 @@ export class FactoryLoop {
|
|
|
2703
3063
|
continue;
|
|
2704
3064
|
}
|
|
2705
3065
|
const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
|
|
3066
|
+
// Reset per work unit, and advanced by assignment immediately before
|
|
3067
|
+
// each stage rather than inferred in the catch: the whole value of the
|
|
3068
|
+
// `unclassified-*` codes is that they name the stage honestly (#355).
|
|
3069
|
+
let attemptPhase = 'gate';
|
|
2706
3070
|
try {
|
|
2707
3071
|
if (recoveredOrphan) {
|
|
2708
3072
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
@@ -2723,8 +3087,10 @@ export class FactoryLoop {
|
|
|
2723
3087
|
});
|
|
2724
3088
|
continue;
|
|
2725
3089
|
}
|
|
3090
|
+
attemptPhase = 'triage';
|
|
2726
3091
|
const decision = await this.triageIssue(issue);
|
|
2727
3092
|
triaged.push(decision);
|
|
3093
|
+
attemptPhase = 'dispatch';
|
|
2728
3094
|
const result = await this.dispatch(decision, { dryRun });
|
|
2729
3095
|
// A completed dispatch — even one that parks or escalates the issue —
|
|
2730
3096
|
// proves the pipeline still works, so the fuse below starts over.
|
|
@@ -2825,6 +3191,11 @@ export class FactoryLoop {
|
|
|
2825
3191
|
issue: issueRef(issue),
|
|
2826
3192
|
reason: perItemDispatchSkipReason(error),
|
|
2827
3193
|
code: 'dispatch-failed',
|
|
3194
|
+
// The publishable half of the same classification. `reason` is the
|
|
3195
|
+
// operator's sentence and stays off the health surface; this token
|
|
3196
|
+
// is what tells a reader watching `dispatch-failed: 5` which of
|
|
3197
|
+
// five very different bugs they are looking at (#355).
|
|
3198
|
+
failureCode: perItemDispatchFailureCode(error, attemptPhase),
|
|
2828
3199
|
});
|
|
2829
3200
|
continue;
|
|
2830
3201
|
}
|
|
@@ -2839,6 +3210,10 @@ export class FactoryLoop {
|
|
|
2839
3210
|
dispatched,
|
|
2840
3211
|
skipped,
|
|
2841
3212
|
dryRun,
|
|
3213
|
+
// Read before `#runOnceWithDiscoveryFence`'s finally resets them: this
|
|
3214
|
+
// is still inside that try, so the counts are this sweep's own.
|
|
3215
|
+
treeReads: this.#discoverySweepTreeReads,
|
|
3216
|
+
emptyTreeReads: this.#discoverySweepEmptyTreeReads,
|
|
2842
3217
|
slackDegraded: this.#slackDegraded,
|
|
2843
3218
|
...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
|
|
2844
3219
|
};
|
|
@@ -3744,6 +4119,10 @@ export class FactoryLoop {
|
|
|
3744
4119
|
// that expects a specific PR to appear, or an escalation/comment-replay
|
|
3745
4120
|
// scan that must not miss a marker or reply that landed after the cache
|
|
3746
4121
|
// was populated. Those callers must omit `cache` and pay for a fresh list.
|
|
4122
|
+
//
|
|
4123
|
+
// `enumeration` marks the call sites that ARE the readiness sweep's
|
|
4124
|
+
// discovery pass — a full-root walk looking for candidate work. Only those
|
|
4125
|
+
// feed the tree-read pair below, and only while a sweep holds the lease.
|
|
3747
4126
|
async #listRelayfileTree(prefix, phase, opts = {}) {
|
|
3748
4127
|
if (this.#discoverySweepLeaseLost) {
|
|
3749
4128
|
throw new Error('discovery sweep lease was lost; refusing another tree request');
|
|
@@ -3762,6 +4141,46 @@ export class FactoryLoop {
|
|
|
3762
4141
|
logStart: true,
|
|
3763
4142
|
logComplete: true,
|
|
3764
4143
|
});
|
|
4144
|
+
// #351 follow-up: tree reads the backend SERVED, and how many of them it
|
|
4145
|
+
// answered with nothing.
|
|
4146
|
+
//
|
|
4147
|
+
// The per-call deadline made a hung dependency loud. This is its
|
|
4148
|
+
// companion: a mount that starts returning empty trees instead of hanging
|
|
4149
|
+
// raises no timeout, no failure and no `lastError`, and produces a sweep
|
|
4150
|
+
// that completes `healthy` having dispatched nothing.
|
|
4151
|
+
//
|
|
4152
|
+
// BOTH numbers, because one is not a signal. A healthy sweep lists two
|
|
4153
|
+
// path forms per repo and only one of them exists, so an empty read is
|
|
4154
|
+
// ordinary and a bare count of them fires constantly. What separates the
|
|
4155
|
+
// fault is the RATIO: `emptyTreeReads === treeReads` means the mount
|
|
4156
|
+
// served nothing at all, which `candidates: 0` cannot distinguish from a
|
|
4157
|
+
// workspace that simply has no ready work.
|
|
4158
|
+
//
|
|
4159
|
+
// SCOPED TWICE, and both guards are load-bearing (#363 review, codex P2).
|
|
4160
|
+
// The ratio is only readable if every read in it came from the same
|
|
4161
|
+
// question. In live mode a Slack identity lookup, a PR-confirmation poll
|
|
4162
|
+
// or a comment-replay scan can run concurrently with `runOnce()` — each is
|
|
4163
|
+
// a `listTree`, none is discovery, and one populated result among them
|
|
4164
|
+
// makes `emptyTreeReads < treeReads` on an all-empty sweep, silently
|
|
4165
|
+
// erasing exactly the signal this pair exists to raise. `enumeration`
|
|
4166
|
+
// keeps the numerator and denominator to the discovery pass; the epoch
|
|
4167
|
+
// check keeps a discovery walk issued outside any sweep — startup
|
|
4168
|
+
// backfill, most obviously — out of a sweep's totals.
|
|
4169
|
+
// `enumeration` says this is a full-root discovery walk rather than a point
|
|
4170
|
+
// lookup; the context says THIS sweep's discovery pass is what issued it.
|
|
4171
|
+
// Both, because either alone admits a read the ratio cannot use. Compared
|
|
4172
|
+
// by value rather than presence: an absent store and an absent epoch are
|
|
4173
|
+
// both `undefined` and must not read as a match.
|
|
4174
|
+
const issuingPass = discoveryEnumerationPass.getStore();
|
|
4175
|
+
if (opts.enumeration &&
|
|
4176
|
+
issuingPass !== undefined &&
|
|
4177
|
+
issuingPass.epoch === this.#discoverySweepEpoch) {
|
|
4178
|
+
this.#discoverySweepTreeReads += 1;
|
|
4179
|
+
if (paths.length === 0) {
|
|
4180
|
+
this.#increment('relayfileEmptyTreeReads');
|
|
4181
|
+
this.#discoverySweepEmptyTreeReads += 1;
|
|
4182
|
+
}
|
|
4183
|
+
}
|
|
3765
4184
|
if (opts.cache)
|
|
3766
4185
|
await this.#rememberDiscoveryTree(prefix, paths);
|
|
3767
4186
|
return paths;
|
|
@@ -4277,6 +4696,66 @@ export class FactoryLoop {
|
|
|
4277
4696
|
if (!dryRun)
|
|
4278
4697
|
await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
|
|
4279
4698
|
const spawnedForReaperHandoff = [];
|
|
4699
|
+
// These waits belong to the durable work unit, not the ingestion surface.
|
|
4700
|
+
// Completion and lifecycle ownership use the same identity (#211); using
|
|
4701
|
+
// issueKey here would make every GitHub-native dispatch miss its own fence.
|
|
4702
|
+
const postSpawnKey = dispatchLifecycleKey(record.issue);
|
|
4703
|
+
let resolvePostSpawnIssueObservation;
|
|
4704
|
+
let postSpawnIssueObservationDidSettle = false;
|
|
4705
|
+
const postSpawnIssueObservationSettled = new Promise((resolve) => {
|
|
4706
|
+
resolvePostSpawnIssueObservation = resolve;
|
|
4707
|
+
});
|
|
4708
|
+
let postSpawnIssueObservation;
|
|
4709
|
+
postSpawnIssueObservation = {
|
|
4710
|
+
settled: postSpawnIssueObservationSettled,
|
|
4711
|
+
settle: (accepted) => {
|
|
4712
|
+
if (postSpawnIssueObservationDidSettle)
|
|
4713
|
+
return;
|
|
4714
|
+
postSpawnIssueObservationDidSettle = true;
|
|
4715
|
+
resolvePostSpawnIssueObservation(accepted);
|
|
4716
|
+
if (this.#postSpawnIssueObservations.get(postSpawnKey) === postSpawnIssueObservation) {
|
|
4717
|
+
this.#postSpawnIssueObservations.delete(postSpawnKey);
|
|
4718
|
+
}
|
|
4719
|
+
},
|
|
4720
|
+
};
|
|
4721
|
+
this.#postSpawnIssueObservations.set(postSpawnKey, postSpawnIssueObservation);
|
|
4722
|
+
let resolvePostSpawnDispatchClaim;
|
|
4723
|
+
let postSpawnDispatchClaimSettled = false;
|
|
4724
|
+
const postSpawnDispatchClaim = new Promise((resolve) => {
|
|
4725
|
+
resolvePostSpawnDispatchClaim = resolve;
|
|
4726
|
+
});
|
|
4727
|
+
let resolveRejectedClaimCompensation;
|
|
4728
|
+
let rejectedClaimCompensationSettled = false;
|
|
4729
|
+
const rejectedClaimCompensation = new Promise((resolve) => {
|
|
4730
|
+
resolveRejectedClaimCompensation = resolve;
|
|
4731
|
+
});
|
|
4732
|
+
let postSpawnDispatchClaimFence;
|
|
4733
|
+
postSpawnDispatchClaimFence = {
|
|
4734
|
+
completionAtWriteBoundary: false,
|
|
4735
|
+
claimStarted: false,
|
|
4736
|
+
settled: postSpawnDispatchClaim,
|
|
4737
|
+
rejectionSettled: rejectedClaimCompensation,
|
|
4738
|
+
settle: (accepted) => {
|
|
4739
|
+
if (postSpawnDispatchClaimSettled)
|
|
4740
|
+
return;
|
|
4741
|
+
postSpawnDispatchClaimSettled = true;
|
|
4742
|
+
postSpawnDispatchClaimFence.accepted = accepted;
|
|
4743
|
+
resolvePostSpawnDispatchClaim(accepted);
|
|
4744
|
+
if (this.#postSpawnDispatchClaimFences.get(postSpawnKey) === postSpawnDispatchClaimFence) {
|
|
4745
|
+
this.#postSpawnDispatchClaimFences.delete(postSpawnKey);
|
|
4746
|
+
}
|
|
4747
|
+
},
|
|
4748
|
+
settleRejection: (compensated) => {
|
|
4749
|
+
if (rejectedClaimCompensationSettled)
|
|
4750
|
+
return;
|
|
4751
|
+
rejectedClaimCompensationSettled = true;
|
|
4752
|
+
resolveRejectedClaimCompensation(compensated);
|
|
4753
|
+
},
|
|
4754
|
+
};
|
|
4755
|
+
this.#postSpawnDispatchClaimFences.set(postSpawnKey, postSpawnDispatchClaimFence);
|
|
4756
|
+
const settlePostSpawnIssueObservation = postSpawnIssueObservation.settle;
|
|
4757
|
+
const settlePostSpawnDispatchClaim = postSpawnDispatchClaimFence.settle;
|
|
4758
|
+
let rejectDispatchClaim;
|
|
4280
4759
|
try {
|
|
4281
4760
|
if (!dryRun) {
|
|
4282
4761
|
const issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
@@ -4301,47 +4780,92 @@ export class FactoryLoop {
|
|
|
4301
4780
|
agents.push({ name: spawned.name, role: spec.role });
|
|
4302
4781
|
}
|
|
4303
4782
|
if (!dryRun) {
|
|
4783
|
+
const claimStartedAtMs = this.#clock.now();
|
|
4304
4784
|
record.dispatchClaim = {
|
|
4305
4785
|
state: 'pending',
|
|
4306
|
-
|
|
4786
|
+
claimStartedAtMs,
|
|
4787
|
+
updatedAtMs: claimStartedAtMs,
|
|
4307
4788
|
};
|
|
4308
4789
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
4309
4790
|
}
|
|
4310
4791
|
await this.#writeInFlightRegistry();
|
|
4311
4792
|
const comment = dispatchComment(dispatchDecision, agents);
|
|
4312
4793
|
let implementingStateId;
|
|
4794
|
+
const rejectDispatchClaimIfCancelled = async () => {
|
|
4795
|
+
if (postSpawnDispatchClaimFence.accepted !== false)
|
|
4796
|
+
return;
|
|
4797
|
+
if (rejectDispatchClaim)
|
|
4798
|
+
await rejectDispatchClaim();
|
|
4799
|
+
throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
|
|
4800
|
+
};
|
|
4313
4801
|
if (!dryRun) {
|
|
4314
|
-
|
|
4315
|
-
if (
|
|
4316
|
-
//
|
|
4317
|
-
//
|
|
4318
|
-
//
|
|
4319
|
-
//
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
//
|
|
4802
|
+
let issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
4803
|
+
if (postSpawnDispatchClaimFence.completionAtWriteBoundary) {
|
|
4804
|
+
// Completion reached its terminal provider-write boundary before
|
|
4805
|
+
// dispatch entered the claim boundary. Always re-read after its
|
|
4806
|
+
// receipt, even when the first snapshot was ready: the terminal
|
|
4807
|
+
// write may have started immediately after that snapshot.
|
|
4808
|
+
const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
|
|
4809
|
+
if (issueWriteback)
|
|
4810
|
+
await issueWriteback;
|
|
4811
|
+
issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
4812
|
+
}
|
|
4813
|
+
else if ((!issue || !this.#isIssueReady(issue)) && record.issueWritebackConfirmedAtMs === undefined) {
|
|
4814
|
+
// A very fast agent can finish while its own dispatch is still
|
|
4815
|
+
// performing this post-spawn read. Do not guess authorship from the
|
|
4816
|
+
// first visible state: an idempotent provider mutation cannot tell us
|
|
4817
|
+
// whether Factory or another actor won the race. Instead, wait for
|
|
4818
|
+
// this record's concurrent completion writeback to settle, then use
|
|
4819
|
+
// only its provider-confirmed outcome (factory#319).
|
|
4327
4820
|
//
|
|
4328
|
-
//
|
|
4329
|
-
//
|
|
4330
|
-
//
|
|
4331
|
-
// the
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4821
|
+
// The completion promise settles as soon as terminal issue writeback
|
|
4822
|
+
// succeeds or fails, before Slack/release cleanup. If there is no
|
|
4823
|
+
// completion in flight, this is an ordinary foreign state change and
|
|
4824
|
+
// the existing abort remains immediate.
|
|
4825
|
+
const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
|
|
4826
|
+
if (issueWriteback) {
|
|
4827
|
+
await issueWriteback;
|
|
4828
|
+
issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
4829
|
+
}
|
|
4830
|
+
}
|
|
4831
|
+
if (!issue || !this.#isIssueReady(issue)) {
|
|
4832
|
+
// Only a terminal writeback applied and confirmed by the completion
|
|
4833
|
+
// already in flight for this exact record may excuse the change. The
|
|
4834
|
+
// marker is stamped after provider acknowledgement/readback, never at
|
|
4835
|
+
// the earlier locally-visible write boundary. A confirmed no-op does
|
|
4836
|
+
// not establish authorship: another actor may have parked the issue
|
|
4837
|
+
// before our adapter's first provider read.
|
|
4336
4838
|
if (record.issueWritebackConfirmedAtMs === undefined) {
|
|
4337
4839
|
throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
|
|
4338
4840
|
}
|
|
4339
4841
|
// The claim is moot and would be wrong to write: it would drag an
|
|
4340
4842
|
// issue our own lifecycle has already parked back to `implementing`.
|
|
4341
4843
|
}
|
|
4844
|
+
else if (this.#stopping) {
|
|
4845
|
+
// Shutdown snapshots the post-spawn fence maps once and settles only
|
|
4846
|
+
// what it finds there. A dispatch that armed its fence after that
|
|
4847
|
+
// snapshot — it can still be inside the preview render, the
|
|
4848
|
+
// `dispatching` save or the question-watch setup when `stop()`
|
|
4849
|
+
// begins — would otherwise enter the claim boundary below against a
|
|
4850
|
+
// fleet already being disposed and leases already relinquished,
|
|
4851
|
+
// leaving a claimed issue with no lifecycle to recover it. Reject
|
|
4852
|
+
// here instead. No claim has started, so there is nothing to
|
|
4853
|
+
// compensate, and this stays synchronous so the completion-race
|
|
4854
|
+
// invariant documented below is untouched (#346 review, cubic).
|
|
4855
|
+
this.#increment('postSpawnDispatchClaimsRefusedDuringStop');
|
|
4856
|
+
postSpawnDispatchClaimFence.settle(false);
|
|
4857
|
+
throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
|
|
4858
|
+
}
|
|
4342
4859
|
else {
|
|
4343
|
-
|
|
4344
|
-
|
|
4860
|
+
// This assignment is synchronous with the preceding completion flag
|
|
4861
|
+
// check. Completion either arrived first (the branch above) or will
|
|
4862
|
+
// now observe claimStarted and wait; no await-sized gap remains.
|
|
4863
|
+
postSpawnDispatchClaimFence.claimStarted = true;
|
|
4864
|
+
const appliedClaim = await this.#applyDispatchClaim(record, issue, comment, postSpawnDispatchClaimFence);
|
|
4865
|
+
implementingStateId = appliedClaim.implementingStateId;
|
|
4866
|
+
rejectDispatchClaim = appliedClaim.reject;
|
|
4867
|
+
await rejectDispatchClaimIfCancelled();
|
|
4868
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
4345
4869
|
this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
|
|
4346
4870
|
}
|
|
4347
4871
|
}
|
|
@@ -4356,19 +4880,110 @@ export class FactoryLoop {
|
|
|
4356
4880
|
dryRun,
|
|
4357
4881
|
};
|
|
4358
4882
|
record.result = result;
|
|
4359
|
-
|
|
4883
|
+
const runningSaved = await this.#saveDispatchLifecycle(record, 'running');
|
|
4884
|
+
await rejectDispatchClaimIfCancelled();
|
|
4885
|
+
if (!runningSaved) {
|
|
4886
|
+
settlePostSpawnDispatchClaim(true);
|
|
4887
|
+
settlePostSpawnIssueObservation(true);
|
|
4360
4888
|
return result;
|
|
4361
|
-
|
|
4362
|
-
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
4889
|
+
}
|
|
4363
4890
|
if (!dryRun && this.#config.hooks?.onTicketDispatch) {
|
|
4364
4891
|
await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
|
|
4892
|
+
await rejectDispatchClaimIfCancelled();
|
|
4365
4893
|
}
|
|
4366
4894
|
if (!dryRun) {
|
|
4367
4895
|
await this.#ensureSlackDispatchThread(record, result, liveIssue);
|
|
4896
|
+
await rejectDispatchClaimIfCancelled();
|
|
4368
4897
|
}
|
|
4898
|
+
// Accept without an intervening await only after every durable
|
|
4899
|
+
// publication and post-dispatch side effect has completed. Stop or held
|
|
4900
|
+
// abandonment can therefore reject any still-publishing dispatch.
|
|
4901
|
+
settlePostSpawnDispatchClaim(true);
|
|
4902
|
+
settlePostSpawnIssueObservation(true);
|
|
4903
|
+
this.#increment('dispatched');
|
|
4904
|
+
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
4369
4905
|
return result;
|
|
4370
4906
|
}
|
|
4371
|
-
catch (
|
|
4907
|
+
catch (caughtError) {
|
|
4908
|
+
// Stop/deadline may reject the fence while any awaited provider or
|
|
4909
|
+
// lifecycle operation is still in flight. If that operation then fails,
|
|
4910
|
+
// its raw error must not fall into ordinary dispatch cleanup: shutdown
|
|
4911
|
+
// still owns the already-spawned placements and needs them retained
|
|
4912
|
+
// until its release snapshot. If the claim receipt is already available,
|
|
4913
|
+
// conditionally reject it before releasing those placements. Without a
|
|
4914
|
+
// receipt the provider outcome is ambiguous, so preserve the original
|
|
4915
|
+
// failure as the compensation detail after the claim boundary began.
|
|
4916
|
+
let error = caughtError;
|
|
4917
|
+
if (postSpawnDispatchClaimFence.accepted === false
|
|
4918
|
+
&& !(caughtError instanceof PostSpawnDispatchWaitRejectedError)) {
|
|
4919
|
+
if (rejectDispatchClaim) {
|
|
4920
|
+
try {
|
|
4921
|
+
await this.#rejectDispatchClaimAfterOperationError(dispatchDecision.issue.key, caughtError, rejectDispatchClaim);
|
|
4922
|
+
}
|
|
4923
|
+
catch (rejectionError) {
|
|
4924
|
+
error = rejectionError;
|
|
4925
|
+
}
|
|
4926
|
+
}
|
|
4927
|
+
else {
|
|
4928
|
+
error = new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key, postSpawnDispatchClaimFence.claimStarted ? caughtError : undefined);
|
|
4929
|
+
}
|
|
4930
|
+
}
|
|
4931
|
+
settlePostSpawnDispatchClaim(false);
|
|
4932
|
+
if (error instanceof PostSpawnDispatchWaitRejectedError) {
|
|
4933
|
+
settlePostSpawnIssueObservation(false);
|
|
4934
|
+
const compensated = error.compensationError === undefined;
|
|
4935
|
+
// The original provider call has conclusively returned. Stop the
|
|
4936
|
+
// timeout handoff from writing a provisional pending fence, but keep a
|
|
4937
|
+
// separate lease fence until the definitive result is durable. Without
|
|
4938
|
+
// that second set, shutdown can relinquish this epoch in the await
|
|
4939
|
+
// below and make the late save fail after the provider compensated.
|
|
4940
|
+
this.#dispatchClaimSettlementsInFlight.add(postSpawnKey);
|
|
4941
|
+
this.#pendingDispatchClaims.delete(postSpawnKey);
|
|
4942
|
+
let settlementPersisted = false;
|
|
4943
|
+
try {
|
|
4944
|
+
settlementPersisted = compensated
|
|
4945
|
+
? await this.#clearDispatchCancellationBlock(record)
|
|
4946
|
+
: await this.#retainUncompensatedDispatchClaim(record, error.compensationError);
|
|
4947
|
+
if (!settlementPersisted) {
|
|
4948
|
+
throw new Error(`Unable to persist settled dispatch-claim compensation for ${decision.issue.key}`);
|
|
4949
|
+
}
|
|
4950
|
+
postSpawnDispatchClaimFence.settleRejection(compensated);
|
|
4951
|
+
if (error.compensationError !== undefined) {
|
|
4952
|
+
this.#increment('postSpawnDispatchClaimCompensationFailures');
|
|
4953
|
+
this.#logger.error?.('[factory] rejected dispatch claim could not be safely compensated', {
|
|
4954
|
+
issue: decision.issue.key,
|
|
4955
|
+
error: describeError(error.compensationError).errorMessage,
|
|
4956
|
+
});
|
|
4957
|
+
}
|
|
4958
|
+
// External stop/abandonment owns agent and durable-lifecycle cleanup.
|
|
4959
|
+
// Remove only the process-local slot here; running the ordinary
|
|
4960
|
+
// dispatch-failure path would race or duplicate that cleanup. During
|
|
4961
|
+
// stop, however, retain local placements until the shutdown release
|
|
4962
|
+
// sweep has captured them. If it already has, removal is safe now.
|
|
4963
|
+
if (!compensated) {
|
|
4964
|
+
// The provider still exposes a claim this lifecycle authored. Keep
|
|
4965
|
+
// the process-local slot and durable row recoverable; releasing its
|
|
4966
|
+
// agents would leave the claimed issue with no successor lifecycle.
|
|
4967
|
+
}
|
|
4968
|
+
else if (this.#stopping) {
|
|
4969
|
+
this.#postSpawnDispatchesRejectedDuringStop.add(postSpawnKey);
|
|
4970
|
+
if (this.#shutdownReleaseCapturedDispatches.has(postSpawnKey)) {
|
|
4971
|
+
batch.abandon(decision.issue);
|
|
4972
|
+
this.#postSpawnDispatchesRejectedDuringStop.delete(postSpawnKey);
|
|
4973
|
+
}
|
|
4974
|
+
}
|
|
4975
|
+
else {
|
|
4976
|
+
batch.abandon(decision.issue);
|
|
4977
|
+
}
|
|
4978
|
+
}
|
|
4979
|
+
finally {
|
|
4980
|
+
if (!settlementPersisted)
|
|
4981
|
+
this.#pendingDispatchClaims.add(postSpawnKey);
|
|
4982
|
+
this.#dispatchClaimSettlementsInFlight.delete(postSpawnKey);
|
|
4983
|
+
}
|
|
4984
|
+
throw error;
|
|
4985
|
+
}
|
|
4986
|
+
settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
|
|
4372
4987
|
// A spawn can fail after the broker accepted it but before its ack
|
|
4373
4988
|
// reached Factory. Include every planned worktree agent, not only the
|
|
4374
4989
|
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
@@ -4488,6 +5103,9 @@ export class FactoryLoop {
|
|
|
4488
5103
|
})) ?? [],
|
|
4489
5104
|
counters: { ...this.#counters },
|
|
4490
5105
|
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
5106
|
+
// Optional on the port: a backend with no socket omits it, and an absent
|
|
5107
|
+
// value stays absent rather than being invented as healthy.
|
|
5108
|
+
...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
|
|
4491
5109
|
slackDegraded: this.#slackDegraded,
|
|
4492
5110
|
slackDegradedReason: this.#slackDegradedReason,
|
|
4493
5111
|
eventListener: this.#eventListenerStatus(),
|
|
@@ -4519,16 +5137,39 @@ export class FactoryLoop {
|
|
|
4519
5137
|
*
|
|
4520
5138
|
* Only successful passes reach here: a pass that threw has no report, and
|
|
4521
5139
|
* inventing zeroes for it would publish "found nothing" for a sweep that
|
|
4522
|
-
* never got to look. The previous pass's numbers stay put
|
|
4523
|
-
* `
|
|
5140
|
+
* never got to look. The previous enumerating pass's numbers stay put
|
|
5141
|
+
* instead, dated by `lastEnumeratedAtMs`, which is the honest reading.
|
|
5142
|
+
*
|
|
5143
|
+
* A deferred pass gets the same treatment for the same reason. It settles
|
|
5144
|
+
* successfully, and `lastCompletedAtMs` moves — deliberately, because the
|
|
5145
|
+
* #295/#296 stall derivation reads that timestamp against `lastStartedAtMs`,
|
|
5146
|
+
* and freezing it would report a functioning daemon as a hung one after ten
|
|
5147
|
+
* intervals of deferring correctly to another owner. But it enumerated
|
|
5148
|
+
* nothing, so its zeroes are not a measurement of anything and must not
|
|
5149
|
+
* replace one. Only the marker is recorded.
|
|
4524
5150
|
*/
|
|
4525
|
-
#recordReadinessSweepOutcome(report) {
|
|
5151
|
+
#recordReadinessSweepOutcome(report, completedAtMs) {
|
|
5152
|
+
if (report.discoveryDeferred) {
|
|
5153
|
+
this.#readinessReconcileLastSweepDeferred = report.discoveryDeferred;
|
|
5154
|
+
return;
|
|
5155
|
+
}
|
|
5156
|
+
this.#readinessReconcileLastSweepDeferred = undefined;
|
|
4526
5157
|
this.#readinessReconcileLastSweep = {
|
|
4527
5158
|
candidates: report.pulled.length,
|
|
5159
|
+
treeReads: report.treeReads ?? 0,
|
|
5160
|
+
emptyTreeReads: report.emptyTreeReads ?? 0,
|
|
4528
5161
|
dispatched: report.dispatched.length,
|
|
4529
5162
|
skipped: report.skipped.length,
|
|
4530
5163
|
skipReasons: factorySweepSkipReasonCounts(report.skipped),
|
|
4531
|
-
|
|
5164
|
+
// Counted from the same entries `skipReasons` counts, so the parts sum to
|
|
5165
|
+
// `skipReasons['dispatch-failed']` by construction rather than by a
|
|
5166
|
+
// second traversal agreeing with the first.
|
|
5167
|
+
dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
|
|
5168
|
+
dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
|
|
5169
|
+
// The caller's completion stamp, not a fresh clock read: on a pass that
|
|
5170
|
+
// enumerated, `lastEnumeratedAtMs` and `lastCompletedAtMs` describe the
|
|
5171
|
+
// same instant and must not drift apart by a tick.
|
|
5172
|
+
enumeratedAtMs: completedAtMs,
|
|
4532
5173
|
};
|
|
4533
5174
|
}
|
|
4534
5175
|
#readinessReconcileStatus() {
|
|
@@ -4611,16 +5252,33 @@ export class FactoryLoop {
|
|
|
4611
5252
|
...(this.#readinessReconcileLastSweep
|
|
4612
5253
|
? {
|
|
4613
5254
|
candidates: this.#readinessReconcileLastSweep.candidates,
|
|
5255
|
+
// Unconditional for the same reason `dispatchFailures` is: these
|
|
5256
|
+
// are only meaningful as a pair, and only a published zero lets a
|
|
5257
|
+
// reader see that `emptyTreeReads < treeReads` — i.e. that the
|
|
5258
|
+
// mount served real content and a zero `candidates` beside it means
|
|
5259
|
+
// an empty workspace, not a silent mount (#351 follow-up).
|
|
5260
|
+
treeReads: this.#readinessReconcileLastSweep.treeReads,
|
|
5261
|
+
emptyTreeReads: this.#readinessReconcileLastSweep.emptyTreeReads,
|
|
4614
5262
|
dispatched: this.#readinessReconcileLastSweep.dispatched,
|
|
4615
5263
|
skipped: this.#readinessReconcileLastSweep.skipped,
|
|
4616
5264
|
...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
|
|
4617
5265
|
? { skipReasons: { ...this.#readinessReconcileLastSweep.skipReasons } }
|
|
4618
5266
|
: {}),
|
|
4619
|
-
|
|
4620
|
-
|
|
5267
|
+
// Unconditional, unlike the breakdown below it: a zero here is the
|
|
5268
|
+
// fact "this sweep attempted dispatches and none of them failed",
|
|
5269
|
+
// which no other field on this surface can express.
|
|
5270
|
+
dispatchFailures: this.#readinessReconcileLastSweep.dispatchFailures,
|
|
5271
|
+
...(Object.keys(this.#readinessReconcileLastSweep.dispatchFailureReasons).length > 0
|
|
5272
|
+
? { dispatchFailureReasons: { ...this.#readinessReconcileLastSweep.dispatchFailureReasons } }
|
|
4621
5273
|
: {}),
|
|
5274
|
+
lastEnumeratedAtMs: this.#readinessReconcileLastSweep.enumeratedAtMs,
|
|
4622
5275
|
}
|
|
4623
5276
|
: {}),
|
|
5277
|
+
// Independent of the trio: a daemon whose FIRST pass deferred has no
|
|
5278
|
+
// counts to publish and still needs to say why.
|
|
5279
|
+
...(this.#readinessReconcileLastSweepDeferred
|
|
5280
|
+
? { discoveryDeferred: this.#readinessReconcileLastSweepDeferred }
|
|
5281
|
+
: {}),
|
|
4624
5282
|
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
4625
5283
|
...(this.#readinessReconcileLastErrorClass
|
|
4626
5284
|
? { lastErrorClass: this.#readinessReconcileLastErrorClass }
|
|
@@ -6187,9 +6845,11 @@ export class FactoryLoop {
|
|
|
6187
6845
|
const comment = dispatchComment(record.decision, agents);
|
|
6188
6846
|
let implementingStateId;
|
|
6189
6847
|
if (!record.dryRun) {
|
|
6848
|
+
const claimStartedAtMs = this.#clock.now();
|
|
6190
6849
|
record.dispatchClaim = {
|
|
6191
6850
|
state: 'pending',
|
|
6192
|
-
|
|
6851
|
+
claimStartedAtMs,
|
|
6852
|
+
updatedAtMs: claimStartedAtMs,
|
|
6193
6853
|
};
|
|
6194
6854
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
6195
6855
|
}
|
|
@@ -6199,7 +6859,7 @@ export class FactoryLoop {
|
|
|
6199
6859
|
if (!issue)
|
|
6200
6860
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
6201
6861
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
6202
|
-
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
6862
|
+
implementingStateId = (await this.#applyDispatchClaim(record, issue, comment)).implementingStateId;
|
|
6203
6863
|
}
|
|
6204
6864
|
const recoveredPreviews = uniquePreviewReferences([
|
|
6205
6865
|
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
@@ -6508,33 +7168,161 @@ export class FactoryLoop {
|
|
|
6508
7168
|
}
|
|
6509
7169
|
await this.#linear.postComment(issue, body);
|
|
6510
7170
|
}
|
|
6511
|
-
async #applyDispatchClaim(record, issue, comment) {
|
|
7171
|
+
async #applyDispatchClaim(record, issue, comment, claimFence) {
|
|
7172
|
+
const claimWasRejected = () => claimFence?.accepted === false;
|
|
7173
|
+
// stop() can reject this fence after the post-spawn readiness read but
|
|
7174
|
+
// before this method starts. Never begin a provider claim in that state;
|
|
7175
|
+
// shutdown intentionally does not drain dispatches that never crossed the
|
|
7176
|
+
// claim boundary.
|
|
7177
|
+
if (claimWasRejected()) {
|
|
7178
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key);
|
|
7179
|
+
}
|
|
6512
7180
|
let implementingStateId;
|
|
7181
|
+
let rejectClaim;
|
|
6513
7182
|
if (isGithubIssue(issue)) {
|
|
7183
|
+
const statusClaim = {
|
|
7184
|
+
value: { result: undefined },
|
|
7185
|
+
};
|
|
6514
7186
|
await this.#retryDispatchWriteback(record, issue, 'GitHub label factory:in-progress', async () => {
|
|
6515
|
-
|
|
7187
|
+
statusClaim.value = this.#githubWriteback.claimStatus
|
|
7188
|
+
? await this.#githubWriteback.claimStatus(issue, 'in-progress')
|
|
7189
|
+
: { result: await this.#githubWriteback.setStatus(issue, 'in-progress') };
|
|
6516
7190
|
});
|
|
7191
|
+
rejectClaim = async () => await this.#rejectGithubDispatchClaim(record, issue, statusClaim.value);
|
|
7192
|
+
if (claimWasRejected()) {
|
|
7193
|
+
await rejectClaim();
|
|
7194
|
+
}
|
|
6517
7195
|
const commentApplied = this.#githubWriteback.hasCommentMarker
|
|
6518
7196
|
? async () => this.#githubWriteback.hasCommentMarker(issue, comment)
|
|
6519
7197
|
: undefined;
|
|
6520
|
-
|
|
7198
|
+
try {
|
|
7199
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => {
|
|
7200
|
+
if (claimWasRejected())
|
|
7201
|
+
await rejectClaim();
|
|
7202
|
+
await this.#githubWriteback.postComment(issue, comment);
|
|
7203
|
+
}, commentApplied);
|
|
7204
|
+
}
|
|
7205
|
+
catch (error) {
|
|
7206
|
+
if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
|
|
7207
|
+
await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
|
|
7208
|
+
}
|
|
7209
|
+
throw error;
|
|
7210
|
+
}
|
|
7211
|
+
if (claimWasRejected())
|
|
7212
|
+
await rejectClaim();
|
|
6521
7213
|
}
|
|
6522
7214
|
else {
|
|
6523
7215
|
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
7216
|
+
const readyStateId = issue.stateId;
|
|
7217
|
+
let claimToken;
|
|
6524
7218
|
await this.#retryDispatchWriteback(record, issue, `Linear state ${implementingStateId}`, async () => {
|
|
6525
|
-
await this.#linear.setState(issue, implementingStateId);
|
|
6526
|
-
});
|
|
6527
|
-
await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
|
|
6528
|
-
await this.#linear.postComment(issue, comment);
|
|
7219
|
+
claimToken = (await this.#linear.setState(issue, implementingStateId))?.claimToken;
|
|
6529
7220
|
});
|
|
7221
|
+
rejectClaim = async () => await this.#rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken);
|
|
7222
|
+
if (claimWasRejected()) {
|
|
7223
|
+
await rejectClaim();
|
|
7224
|
+
}
|
|
7225
|
+
try {
|
|
7226
|
+
await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
|
|
7227
|
+
if (claimWasRejected())
|
|
7228
|
+
await rejectClaim();
|
|
7229
|
+
await this.#linear.postComment(issue, comment);
|
|
7230
|
+
});
|
|
7231
|
+
}
|
|
7232
|
+
catch (error) {
|
|
7233
|
+
if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
|
|
7234
|
+
await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
|
|
7235
|
+
}
|
|
7236
|
+
throw error;
|
|
7237
|
+
}
|
|
7238
|
+
if (claimWasRejected())
|
|
7239
|
+
await rejectClaim();
|
|
6530
7240
|
}
|
|
6531
7241
|
record.dispatchClaim = {
|
|
6532
7242
|
state: 'verified',
|
|
7243
|
+
// Cancellation can still arrive after this returns — while the running
|
|
7244
|
+
// lifecycle save, the dispatch hook or the Slack publication is pending.
|
|
7245
|
+
// The rejection fence rebuilds the claim from this record, so a verified
|
|
7246
|
+
// state that drops the claim-start stamp leaves `cancellationBlocked`
|
|
7247
|
+
// with no watermark for `getIssueStatus(..., { requireFresh: true })`.
|
|
7248
|
+
// A private repository has no unauthenticated fallback to fall back on,
|
|
7249
|
+
// so that lifecycle, its agents and its batch slot would then be retained
|
|
7250
|
+
// for good, even once a human superseded the claim (#346 review, codex).
|
|
7251
|
+
...(record.dispatchClaim?.claimStartedAtMs === undefined
|
|
7252
|
+
? {}
|
|
7253
|
+
: { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
|
|
6533
7254
|
updatedAtMs: this.#clock.now(),
|
|
6534
7255
|
};
|
|
6535
7256
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
6536
7257
|
await this.#writeDispatchClaimRegistry(record.issue);
|
|
6537
|
-
|
|
7258
|
+
if (claimWasRejected())
|
|
7259
|
+
await rejectClaim?.();
|
|
7260
|
+
return {
|
|
7261
|
+
...(implementingStateId ? { implementingStateId } : {}),
|
|
7262
|
+
reject: rejectClaim,
|
|
7263
|
+
};
|
|
7264
|
+
}
|
|
7265
|
+
async #rejectGithubDispatchClaim(record, issue, claim) {
|
|
7266
|
+
let compensationError;
|
|
7267
|
+
if (claim.result !== 'already-matched') {
|
|
7268
|
+
try {
|
|
7269
|
+
const claimToken = claim.claimToken;
|
|
7270
|
+
if (!claimToken) {
|
|
7271
|
+
throw new Error('GitHub rejected dispatch claim has no immutable ownership token');
|
|
7272
|
+
}
|
|
7273
|
+
const rollbackStatusClaim = this.#githubWriteback.rollbackStatusClaim?.bind(this.#githubWriteback);
|
|
7274
|
+
if (!rollbackStatusClaim) {
|
|
7275
|
+
throw new Error('GitHub writeback cannot conditionally roll back a rejected dispatch claim');
|
|
7276
|
+
}
|
|
7277
|
+
let outcome = 'unproven';
|
|
7278
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub rejected dispatch claim rollback', async () => {
|
|
7279
|
+
outcome = await rollbackStatusClaim(issue, 'in-progress', claimToken);
|
|
7280
|
+
if (outcome === 'unproven') {
|
|
7281
|
+
throw new Error('GitHub rejected dispatch claim rollback could not prove ownership');
|
|
7282
|
+
}
|
|
7283
|
+
});
|
|
7284
|
+
}
|
|
7285
|
+
catch (error) {
|
|
7286
|
+
compensationError = error;
|
|
7287
|
+
}
|
|
7288
|
+
}
|
|
7289
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
|
|
7290
|
+
}
|
|
7291
|
+
async #rejectDispatchClaimAfterOperationError(issueKey, operationError, rejectClaim) {
|
|
7292
|
+
try {
|
|
7293
|
+
await rejectClaim();
|
|
7294
|
+
}
|
|
7295
|
+
catch (rejectionError) {
|
|
7296
|
+
if (rejectionError instanceof PostSpawnDispatchWaitRejectedError) {
|
|
7297
|
+
if (rejectionError.compensationError === undefined)
|
|
7298
|
+
throw rejectionError;
|
|
7299
|
+
throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError.compensationError], 'Cancelled dispatch operation failed and its claim could not be safely compensated'));
|
|
7300
|
+
}
|
|
7301
|
+
throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError], 'Cancelled dispatch operation failed and its claim rejection threw unexpectedly'));
|
|
7302
|
+
}
|
|
7303
|
+
throw new PostSpawnDispatchWaitRejectedError(issueKey);
|
|
7304
|
+
}
|
|
7305
|
+
async #rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken) {
|
|
7306
|
+
let compensationError;
|
|
7307
|
+
try {
|
|
7308
|
+
const compareAndSetState = this.#linear.compareAndSetState?.bind(this.#linear);
|
|
7309
|
+
if (!compareAndSetState) {
|
|
7310
|
+
throw new Error('Linear writeback cannot conditionally roll back a rejected dispatch claim');
|
|
7311
|
+
}
|
|
7312
|
+
if (!claimToken) {
|
|
7313
|
+
throw new Error('Linear rejected dispatch claim has no immutable ownership token');
|
|
7314
|
+
}
|
|
7315
|
+
await this.#retryDispatchWriteback(record, issue, `Linear rejected dispatch claim rollback to ${readyStateId}`, async () => {
|
|
7316
|
+
const outcome = await compareAndSetState(issue, implementingStateId, claimToken, readyStateId);
|
|
7317
|
+
if (outcome === 'unproven') {
|
|
7318
|
+
throw new Error('Linear rejected dispatch claim rollback could not prove an atomic provider transition');
|
|
7319
|
+
}
|
|
7320
|
+
});
|
|
7321
|
+
}
|
|
7322
|
+
catch (error) {
|
|
7323
|
+
compensationError = error;
|
|
7324
|
+
}
|
|
7325
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
|
|
6538
7326
|
}
|
|
6539
7327
|
async #retryDispatchWriteback(record, issue, write, apply, isApplied) {
|
|
6540
7328
|
let lastError;
|
|
@@ -6549,11 +7337,23 @@ export class FactoryLoop {
|
|
|
6549
7337
|
return;
|
|
6550
7338
|
}
|
|
6551
7339
|
catch (error) {
|
|
7340
|
+
// Claim rejection is terminal for this dispatch. In particular, do not
|
|
7341
|
+
// turn a cancellation discovered between comment attempts into another
|
|
7342
|
+
// provider retry that can publish after shutdown began.
|
|
7343
|
+
if (error instanceof PostSpawnDispatchWaitRejectedError) {
|
|
7344
|
+
if (lastError !== undefined && error.compensationError !== undefined) {
|
|
7345
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key, new AggregateError([lastError, error.compensationError], 'Dispatch writeback failed before cancellation and its claim could not be safely compensated'));
|
|
7346
|
+
}
|
|
7347
|
+
throw error;
|
|
7348
|
+
}
|
|
6552
7349
|
lastError = error;
|
|
6553
7350
|
const deadLettered = attempt === DISPATCH_WRITEBACK_MAX_ATTEMPTS;
|
|
6554
7351
|
this.#increment('dispatchWritebackFailures');
|
|
6555
7352
|
record.dispatchClaim = {
|
|
6556
7353
|
state: 'degraded',
|
|
7354
|
+
...(record.dispatchClaim?.claimStartedAtMs === undefined
|
|
7355
|
+
? {}
|
|
7356
|
+
: { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
|
|
6557
7357
|
write,
|
|
6558
7358
|
attempts: attempt,
|
|
6559
7359
|
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
@@ -6656,7 +7456,7 @@ export class FactoryLoop {
|
|
|
6656
7456
|
const candidates = [];
|
|
6657
7457
|
let scanned = 0;
|
|
6658
7458
|
let lastProgressAtMs = startedAtMs;
|
|
6659
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true })) {
|
|
7459
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true, enumeration: true })) {
|
|
6660
7460
|
await this.#refreshLiveHeartbeatIfDue();
|
|
6661
7461
|
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
6662
7462
|
continue;
|
|
@@ -6706,7 +7506,7 @@ export class FactoryLoop {
|
|
|
6706
7506
|
else {
|
|
6707
7507
|
pathBatches = [];
|
|
6708
7508
|
for (const root of roots) {
|
|
6709
|
-
pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true }));
|
|
7509
|
+
pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true, enumeration: true }));
|
|
6710
7510
|
}
|
|
6711
7511
|
this.#increment('githubIssueIndexFallbacks');
|
|
6712
7512
|
}
|
|
@@ -7184,6 +7984,10 @@ export class FactoryLoop {
|
|
|
7184
7984
|
// The pass-scoped flag also lets every later blocked issue reuse the
|
|
7185
7985
|
// resulting dependency index instead of rescanning the full tree.
|
|
7186
7986
|
this.#dependencyLinearTreeLoaded = true;
|
|
7987
|
+
// Cached like the enumeration walks, but deliberately NOT `enumeration`:
|
|
7988
|
+
// this is a lookup for named blocker identities, not the sweep asking
|
|
7989
|
+
// what work exists, and mixing a lookup's result into the ratio is the
|
|
7990
|
+
// masking the pair's scoping exists to prevent (#363 review).
|
|
7187
7991
|
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery', { cache: true })) {
|
|
7188
7992
|
if (isIssueFilePath(path))
|
|
7189
7993
|
await this.#readIssue(path);
|
|
@@ -7453,6 +8257,9 @@ export class FactoryLoop {
|
|
|
7453
8257
|
readinessReconcile: this.#readinessReconcileStatus(),
|
|
7454
8258
|
dispatchCapacity: this.#dispatchCapacityStatus(),
|
|
7455
8259
|
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
8260
|
+
// Optional on the port: a backend with no socket omits it, and an absent
|
|
8261
|
+
// value stays absent rather than being invented as healthy.
|
|
8262
|
+
...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
|
|
7456
8263
|
};
|
|
7457
8264
|
// The deployed container serves `/healthz` straight out of this file and
|
|
7458
8265
|
// has no redaction logic of its own, so publish the already-safe view here
|
|
@@ -7631,14 +8438,14 @@ export class FactoryLoop {
|
|
|
7631
8438
|
}
|
|
7632
8439
|
const pathsByKey = new Map();
|
|
7633
8440
|
const canonicalPathsByKey = new Map();
|
|
7634
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true })) {
|
|
8441
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true, enumeration: true })) {
|
|
7635
8442
|
if (isIssueFilePath(path)) {
|
|
7636
8443
|
const key = keyFromPath(path);
|
|
7637
8444
|
canonicalPathsByKey.set(key, path);
|
|
7638
8445
|
pathsByKey.set(key, path);
|
|
7639
8446
|
}
|
|
7640
8447
|
}
|
|
7641
|
-
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true })) {
|
|
8448
|
+
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true, enumeration: true })) {
|
|
7642
8449
|
if (isIssueAliasFilePath(path)) {
|
|
7643
8450
|
const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
|
|
7644
8451
|
if (canonicalPath) {
|
|
@@ -7692,10 +8499,20 @@ export class FactoryLoop {
|
|
|
7692
8499
|
}
|
|
7693
8500
|
async #releaseInFlightAgents(reason, opts = {}) {
|
|
7694
8501
|
const agents = new Map();
|
|
7695
|
-
|
|
8502
|
+
const blockedDispatches = new Set();
|
|
8503
|
+
const batch = await this.#batch();
|
|
8504
|
+
const records = [...batch.inFlight];
|
|
8505
|
+
for (const record of records) {
|
|
8506
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
7696
8507
|
if (record.dryRun) {
|
|
8508
|
+
this.#shutdownReleaseCapturedDispatches.add(key);
|
|
8509
|
+
continue;
|
|
8510
|
+
}
|
|
8511
|
+
if (this.#uncompensatedDispatchClaims.has(key) || record.dispatchClaim?.cancellationBlocked === true) {
|
|
8512
|
+
blockedDispatches.add(key);
|
|
7697
8513
|
continue;
|
|
7698
8514
|
}
|
|
8515
|
+
this.#shutdownReleaseCapturedDispatches.add(key);
|
|
7699
8516
|
if (opts.preserveDurable && [...record.agents.values()].some((tracked) => tracked.result?.locality === 'remote')) {
|
|
7700
8517
|
continue;
|
|
7701
8518
|
}
|
|
@@ -7704,7 +8521,15 @@ export class FactoryLoop {
|
|
|
7704
8521
|
}
|
|
7705
8522
|
}
|
|
7706
8523
|
await this.#releaseAndTerminateAgents([...agents], reason, 'stop');
|
|
7707
|
-
|
|
8524
|
+
for (const record of records) {
|
|
8525
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
8526
|
+
if (blockedDispatches.has(key))
|
|
8527
|
+
continue;
|
|
8528
|
+
if (!this.#postSpawnDispatchesRejectedDuringStop.delete(key))
|
|
8529
|
+
continue;
|
|
8530
|
+
batch.abandon(record.issue);
|
|
8531
|
+
}
|
|
8532
|
+
await this.#writeInFlightRegistry(undefined, undefined, true, blockedDispatches);
|
|
7708
8533
|
}
|
|
7709
8534
|
async #releaseAndTerminateAgents(agents, reason, context) {
|
|
7710
8535
|
const failed = [];
|
|
@@ -7763,17 +8588,40 @@ export class FactoryLoop {
|
|
|
7763
8588
|
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
7764
8589
|
}
|
|
7765
8590
|
catch (error) {
|
|
7766
|
-
|
|
7767
|
-
|
|
7768
|
-
|
|
7769
|
-
|
|
7770
|
-
|
|
7771
|
-
|
|
7772
|
-
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
7776
|
-
|
|
8591
|
+
if (isAgentAlreadyGoneOnRelease(error)) {
|
|
8592
|
+
// Do the same bookkeeping the success path does — the agent IS
|
|
8593
|
+
// gone. Skipping this would leave `batch.recordRelease` unset and
|
|
8594
|
+
// the next retry attempt would think the invocation is still
|
|
8595
|
+
// dispatchable.
|
|
8596
|
+
this.#increment('releaseAgentAlreadyGone');
|
|
8597
|
+
this.#logger.info?.(`[factory] release skipped: agent already gone during ${context}`, { agentName, reason });
|
|
8598
|
+
if (record && batch && context !== 'stop') {
|
|
8599
|
+
const releasedInvocationId = batch.recordRelease(record, agentName, this.#clock.now());
|
|
8600
|
+
if (releasedInvocationId) {
|
|
8601
|
+
this.#logger.debug?.('[factory] released agent invocation is no longer dispatchable', {
|
|
8602
|
+
issue: record.issue.key,
|
|
8603
|
+
agentName,
|
|
8604
|
+
reason,
|
|
8605
|
+
invocationId: releasedInvocationId,
|
|
8606
|
+
});
|
|
8607
|
+
}
|
|
8608
|
+
}
|
|
8609
|
+
if (record)
|
|
8610
|
+
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
8611
|
+
}
|
|
8612
|
+
else {
|
|
8613
|
+
failed.push(agentName);
|
|
8614
|
+
this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
|
|
8615
|
+
if (record) {
|
|
8616
|
+
const lifecycle = await this.#state
|
|
8617
|
+
.getDispatchLifecycle(this.#workspaceId, dispatchLifecycleKey(record.issue))
|
|
8618
|
+
.catch(() => undefined);
|
|
8619
|
+
if (lifecycle) {
|
|
8620
|
+
await this.#reportLifecycle(lifecycle, 'factory.failure', {
|
|
8621
|
+
level: 'error',
|
|
8622
|
+
errorCode: 'release_failed',
|
|
8623
|
+
});
|
|
8624
|
+
}
|
|
7777
8625
|
}
|
|
7778
8626
|
}
|
|
7779
8627
|
}
|
|
@@ -7923,7 +8771,7 @@ export class FactoryLoop {
|
|
|
7923
8771
|
this.#increment('agentWorktreesCleaned');
|
|
7924
8772
|
}
|
|
7925
8773
|
}
|
|
7926
|
-
async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false) {
|
|
8774
|
+
async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false, retainIssueKeys) {
|
|
7927
8775
|
const updatedAtMs = this.#clock.now();
|
|
7928
8776
|
const agents = [];
|
|
7929
8777
|
const seenAgents = new Set();
|
|
@@ -7961,10 +8809,12 @@ export class FactoryLoop {
|
|
|
7961
8809
|
} : {}),
|
|
7962
8810
|
});
|
|
7963
8811
|
};
|
|
7964
|
-
if (!empty) {
|
|
8812
|
+
if (!empty || retainIssueKeys?.size) {
|
|
7965
8813
|
for (const record of (await this.#batch()).inFlight) {
|
|
7966
8814
|
if (record.dryRun)
|
|
7967
8815
|
continue;
|
|
8816
|
+
if (empty && !retainIssueKeys?.has(dispatchLifecycleKey(record.issue)))
|
|
8817
|
+
continue;
|
|
7968
8818
|
if (record.dispatchClaim) {
|
|
7969
8819
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
7970
8820
|
}
|
|
@@ -9179,10 +10029,59 @@ export class FactoryLoop {
|
|
|
9179
10029
|
// the release-driven exit event so it cannot re-trigger a resume before the
|
|
9180
10030
|
// record leaves the batch.
|
|
9181
10031
|
async #abandonStuckDispatch(record, reason) {
|
|
9182
|
-
const key =
|
|
10032
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
10033
|
+
// Fence the periodic held sweep before any await below. A blocked claim is
|
|
10034
|
+
// retried by the single keyed abandonment timer; letting the held sweep
|
|
10035
|
+
// enter concurrently can release the same placements twice when provider
|
|
10036
|
+
// supersession becomes visible.
|
|
10037
|
+
this.#abandonedDispatchReasons.set(key, reason);
|
|
10038
|
+
try {
|
|
10039
|
+
await this.#abandonStuckDispatchFenced(record, reason, key);
|
|
10040
|
+
}
|
|
10041
|
+
catch (error) {
|
|
10042
|
+
// The early fence above suppresses the periodic held sweep. Any
|
|
10043
|
+
// exceptional exit must therefore install the keyed retry before
|
|
10044
|
+
// propagating, or one transient persistence/read failure becomes a
|
|
10045
|
+
// permanent absorbing state.
|
|
10046
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
10047
|
+
this.#logger.warn?.('[factory] abandoned dispatch cleanup failed before retry was armed', {
|
|
10048
|
+
issue: record.issue.key,
|
|
10049
|
+
error: describeError(error).errorMessage,
|
|
10050
|
+
});
|
|
10051
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10052
|
+
throw error;
|
|
10053
|
+
}
|
|
10054
|
+
}
|
|
10055
|
+
async #abandonStuckDispatchFenced(record, reason, key) {
|
|
10056
|
+
const claimFence = this.#postSpawnDispatchClaimFences.get(key);
|
|
10057
|
+
const rejectedClaimCompensation = claimFence?.claimStarted
|
|
10058
|
+
? claimFence.rejectionSettled
|
|
10059
|
+
: undefined;
|
|
10060
|
+
// A dispatch can be stuck inside a later spawn after an earlier agent has
|
|
10061
|
+
// already started completion. Release both post-spawn waits before the
|
|
10062
|
+
// first abandonment await so terminal processing cannot remain an
|
|
10063
|
+
// absorbing promise after this lifecycle is reaped.
|
|
10064
|
+
this.#settlePostSpawnDispatchWaits(key, false, 'abandonment');
|
|
10065
|
+
if (rejectedClaimCompensation) {
|
|
10066
|
+
const compensated = await this.#awaitRejectedClaimCompensation(rejectedClaimCompensation);
|
|
10067
|
+
if (!compensated) {
|
|
10068
|
+
if (!this.#pendingDispatchClaims.has(key)) {
|
|
10069
|
+
// The provider call settled while this bounded wait expired; its
|
|
10070
|
+
// caller is recording the definitive compensated/unproven result.
|
|
10071
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10072
|
+
return;
|
|
10073
|
+
}
|
|
10074
|
+
await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not complete before abandonment'), { preferExistingError: true, pending: true });
|
|
10075
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10076
|
+
return;
|
|
10077
|
+
}
|
|
10078
|
+
}
|
|
10079
|
+
if (await this.#dispatchClaimBlocksAbandonment(record)) {
|
|
10080
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10081
|
+
return;
|
|
10082
|
+
}
|
|
9183
10083
|
const heldPastDeadline = reason === HELD_PAST_DEADLINE_RELEASE_REASON;
|
|
9184
10084
|
const agentReleaseReason = heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : 'issue-abandoned';
|
|
9185
|
-
this.#abandonedDispatchReasons.set(key, reason);
|
|
9186
10085
|
if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
9187
10086
|
this.#increment('abandonedDispatchReleaseRetries');
|
|
9188
10087
|
// The generic durable retry can recover the in-memory reason in this
|
|
@@ -13210,6 +14109,22 @@ export class FactoryLoop {
|
|
|
13210
14109
|
return;
|
|
13211
14110
|
}
|
|
13212
14111
|
this.#completionInFlight.add(completionKey);
|
|
14112
|
+
const postSpawnIssueObservation = this.#postSpawnIssueObservations.get(completionKey);
|
|
14113
|
+
const postSpawnDispatchClaimFence = this.#postSpawnDispatchClaimFences.get(completionKey);
|
|
14114
|
+
let settleIssueWriteback;
|
|
14115
|
+
const issueWritebackSettled = new Promise((resolve) => {
|
|
14116
|
+
settleIssueWriteback = resolve;
|
|
14117
|
+
});
|
|
14118
|
+
let issueWritebackDidSettle = false;
|
|
14119
|
+
const settleIssueWritebackOnce = () => {
|
|
14120
|
+
if (issueWritebackDidSettle)
|
|
14121
|
+
return;
|
|
14122
|
+
issueWritebackDidSettle = true;
|
|
14123
|
+
settleIssueWriteback();
|
|
14124
|
+
if (this.#issueWritebackInFlight.get(completionKey) === issueWritebackSettled) {
|
|
14125
|
+
this.#issueWritebackInFlight.delete(completionKey);
|
|
14126
|
+
}
|
|
14127
|
+
};
|
|
13213
14128
|
let releaseReasonForRetry;
|
|
13214
14129
|
try {
|
|
13215
14130
|
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
@@ -13259,13 +14174,59 @@ export class FactoryLoop {
|
|
|
13259
14174
|
const humanReview = configuredHumanReview || (githubIssue && !githubMerged);
|
|
13260
14175
|
const statusLabel = humanReview ? 'In Human Review' : 'Done';
|
|
13261
14176
|
if (issue) {
|
|
14177
|
+
if (postSpawnDispatchClaimFence) {
|
|
14178
|
+
// Publish boundary arrival before any await. Dispatch performs the
|
|
14179
|
+
// paired synchronous claimStarted assignment after its ready read,
|
|
14180
|
+
// so exactly one side wins: an earlier completion is awaited and
|
|
14181
|
+
// re-read; a claim already entering its provider write finishes
|
|
14182
|
+
// before this terminal write begins.
|
|
14183
|
+
postSpawnDispatchClaimFence.completionAtWriteBoundary = true;
|
|
14184
|
+
if (postSpawnDispatchClaimFence.claimStarted || postSpawnDispatchClaimFence.accepted !== undefined) {
|
|
14185
|
+
let claimAccepted = postSpawnDispatchClaimFence.accepted;
|
|
14186
|
+
if (claimAccepted === undefined) {
|
|
14187
|
+
// This counter makes the otherwise internal promise boundary
|
|
14188
|
+
// observable in deterministic race regressions and in incident
|
|
14189
|
+
// telemetry. External abandonment/stop settlement must release
|
|
14190
|
+
// every completion counted here.
|
|
14191
|
+
this.#increment('postSpawnDispatchClaimFenceWaits');
|
|
14192
|
+
claimAccepted = await postSpawnDispatchClaimFence.settled;
|
|
14193
|
+
}
|
|
14194
|
+
if (!claimAccepted)
|
|
14195
|
+
return;
|
|
14196
|
+
}
|
|
14197
|
+
}
|
|
14198
|
+
// Register only at the provider-write boundary. Work before this point
|
|
14199
|
+
// (PR discovery/merge gating) has not changed the issue, so a foreign
|
|
14200
|
+
// park during it must still abort immediately rather than waiting on a
|
|
14201
|
+
// possibly long completion path.
|
|
14202
|
+
this.#issueWritebackInFlight.set(completionKey, issueWritebackSettled);
|
|
13262
14203
|
if (githubIssue) {
|
|
13263
14204
|
if (humanReview) {
|
|
13264
|
-
await this.#githubWriteback.setStatus(issue, 'human-review');
|
|
14205
|
+
const statusWrite = await this.#githubWriteback.setStatus(issue, 'human-review');
|
|
14206
|
+
if (statusWrite === undefined)
|
|
14207
|
+
this.#recordMissingGithubWritebackReceipt('setStatus');
|
|
14208
|
+
// Only an explicit provider-proven transition establishes that
|
|
14209
|
+
// this dispatch owns the visible park. Legacy void adapters and
|
|
14210
|
+
// App acknowledgements remain deliberately untrusted.
|
|
14211
|
+
if (statusWrite === 'applied') {
|
|
14212
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
14213
|
+
}
|
|
14214
|
+
// The lifecycle-state outcome is now known. Unblock the concurrent
|
|
14215
|
+
// post-spawn read before the separate completion comment write.
|
|
14216
|
+
settleIssueWritebackOnce();
|
|
13265
14217
|
await this.#githubWriteback.postComment(issue, `Factory agents completed; this issue is awaiting human review. The pull request remains open.\n\nMerge policy: ${this.#config.mergePolicy}`);
|
|
13266
14218
|
}
|
|
13267
14219
|
else {
|
|
13268
|
-
await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
|
|
14220
|
+
const closeWrite = await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
|
|
14221
|
+
if (closeWrite === undefined)
|
|
14222
|
+
this.#recordMissingGithubWritebackReceipt('closeIssue');
|
|
14223
|
+
// A provider-confirmed, actor-attributed close is the only safe
|
|
14224
|
+
// proof that this dispatch owns the visible terminal state. An
|
|
14225
|
+
// idempotent no-op, legacy void adapter, or App acknowledgement
|
|
14226
|
+
// may instead mean a third party won the close race.
|
|
14227
|
+
if (closeWrite === 'applied') {
|
|
14228
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
14229
|
+
}
|
|
13269
14230
|
}
|
|
13270
14231
|
}
|
|
13271
14232
|
else {
|
|
@@ -13273,13 +14234,28 @@ export class FactoryLoop {
|
|
|
13273
14234
|
? this.#states.idFor(issueTeam, 'humanReview')
|
|
13274
14235
|
: this.#states.idFor(issueTeam, 'done');
|
|
13275
14236
|
await this.#linear.setState(issue, targetState);
|
|
14237
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
13276
14238
|
await this.#recordCanonicalIssueState({ ...record.issue, stateId: targetState });
|
|
13277
14239
|
}
|
|
13278
|
-
record.issueWritebackConfirmedAtMs
|
|
13279
|
-
|
|
14240
|
+
if (record.issueWritebackConfirmedAtMs !== undefined) {
|
|
14241
|
+
this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
|
|
14242
|
+
}
|
|
14243
|
+
// Unblock a concurrent post-spawn read as soon as the issue writeback
|
|
14244
|
+
// outcome is known. Completion still has dependency, Slack and release
|
|
14245
|
+
// work to do; none determines whether the observed issue state is safe.
|
|
14246
|
+
settleIssueWritebackOnce();
|
|
14247
|
+
// If completion raced dispatch's post-spawn read, let that read consume
|
|
14248
|
+
// the receipt before claiming terminal success. An unproven provider
|
|
14249
|
+
// transition is a foreign live-state change: dispatch owns the
|
|
14250
|
+
// abandonment and must release agents with that reason, not issue-done.
|
|
14251
|
+
if (postSpawnIssueObservation && !await postSpawnIssueObservation.settled)
|
|
14252
|
+
return;
|
|
13280
14253
|
if (!humanReview)
|
|
13281
14254
|
await this.#markDependencyTerminalAndReconcile(issue);
|
|
13282
14255
|
}
|
|
14256
|
+
else {
|
|
14257
|
+
settleIssueWritebackOnce();
|
|
14258
|
+
}
|
|
13283
14259
|
if (!await this.#saveDispatchLifecycle(record, 'writeback-applied'))
|
|
13284
14260
|
return;
|
|
13285
14261
|
if (issue && this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
|
|
@@ -13344,6 +14320,10 @@ export class FactoryLoop {
|
|
|
13344
14320
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
13345
14321
|
}
|
|
13346
14322
|
finally {
|
|
14323
|
+
// Errors before or during provider confirmation leave the marker unset;
|
|
14324
|
+
// the waiting dispatch will re-read and preserve the foreign-change
|
|
14325
|
+
// abort. Always settle so a failed write cannot strand that dispatch.
|
|
14326
|
+
settleIssueWritebackOnce();
|
|
13347
14327
|
this.#completionInFlight.delete(completionKey);
|
|
13348
14328
|
const stateKey = issueStateKey(record.issue);
|
|
13349
14329
|
this.#probePrGhBackoffUntilMs.delete(stateKey);
|
|
@@ -13360,6 +14340,35 @@ export class FactoryLoop {
|
|
|
13360
14340
|
}
|
|
13361
14341
|
}
|
|
13362
14342
|
}
|
|
14343
|
+
#recordMissingGithubWritebackReceipt(method) {
|
|
14344
|
+
const implementation = this.#githubWriteback.constructor.name || 'anonymous GithubWriteback';
|
|
14345
|
+
this.#increment('githubWritebackReceiptMissing');
|
|
14346
|
+
this.#logger.warn?.('[factory] GitHub writeback returned no ownership receipt', {
|
|
14347
|
+
implementation,
|
|
14348
|
+
method,
|
|
14349
|
+
});
|
|
14350
|
+
}
|
|
14351
|
+
#settlePostSpawnDispatchWaits(key, accepted, source) {
|
|
14352
|
+
const observation = this.#postSpawnIssueObservations.get(key);
|
|
14353
|
+
const claimFence = this.#postSpawnDispatchClaimFences.get(key);
|
|
14354
|
+
if (!observation && !claimFence)
|
|
14355
|
+
return false;
|
|
14356
|
+
// Capture both before either settlement deletes its own map entry.
|
|
14357
|
+
if (!accepted && claimFence?.claimStarted) {
|
|
14358
|
+
// The rejection handler clears this provisional block only after a
|
|
14359
|
+
// conditional provider rollback succeeds. Until then, shutdown and held
|
|
14360
|
+
// abandonment must not release the lifecycle's agents.
|
|
14361
|
+
this.#uncompensatedDispatchClaims.add(key);
|
|
14362
|
+
this.#pendingDispatchClaims.add(key);
|
|
14363
|
+
}
|
|
14364
|
+
observation?.settle(accepted);
|
|
14365
|
+
claimFence?.settle(accepted);
|
|
14366
|
+
if (source === 'abandonment')
|
|
14367
|
+
this.#increment('postSpawnWaitsSettledByAbandonment');
|
|
14368
|
+
if (source === 'stop')
|
|
14369
|
+
this.#increment('postSpawnWaitsSettledByStop');
|
|
14370
|
+
return true;
|
|
14371
|
+
}
|
|
13363
14372
|
#emit(event, payload) {
|
|
13364
14373
|
for (const listener of this.#listeners.get(event) ?? []) {
|
|
13365
14374
|
try {
|
|
@@ -16030,7 +17039,7 @@ const defaultGithubWriteback = (config, mount) => {
|
|
|
16030
17039
|
if (!mount.githubWrite) {
|
|
16031
17040
|
throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
|
|
16032
17041
|
}
|
|
16033
|
-
return new AppGithubWriteback(mount.githubWrite);
|
|
17042
|
+
return new AppGithubWriteback(mount.githubWrite, mount.githubRead);
|
|
16034
17043
|
};
|
|
16035
17044
|
export function parseLinearIssue(path, content) {
|
|
16036
17045
|
const parsed = parseJsonContent(content);
|
|
@@ -18271,21 +19280,28 @@ const githubIssueIndexRepoRoots = (path) => {
|
|
|
18271
19280
|
`/github/repos/${owner}__${repo}/issues`,
|
|
18272
19281
|
];
|
|
18273
19282
|
};
|
|
18274
|
-
|
|
18275
|
-
|
|
19283
|
+
/** How far to follow wrapped provider failures without trusting an unbounded chain. */
|
|
19284
|
+
const RELAYFILE_OVERLOAD_CAUSE_DEPTH = 4;
|
|
19285
|
+
const relayfileOverload = (error, depth = 0) => {
|
|
19286
|
+
if (depth > RELAYFILE_OVERLOAD_CAUSE_DEPTH)
|
|
19287
|
+
return undefined;
|
|
19288
|
+
const flat = asRecord(error);
|
|
19289
|
+
if (!flat)
|
|
19290
|
+
return undefined;
|
|
18276
19291
|
const response = asRecord(flat.response) ?? {};
|
|
18277
19292
|
const data = asRecord(flat.data) ?? asRecord(response.data) ?? {};
|
|
18278
19293
|
const details = asRecord(flat.details) ?? asRecord(data.details) ?? {};
|
|
18279
19294
|
const statusValue = flat.status ?? flat.statusCode ?? response.status ?? response.statusCode;
|
|
18280
19295
|
const status = typeof statusValue === 'number' ? statusValue : Number(statusValue);
|
|
18281
|
-
if (status
|
|
18282
|
-
|
|
18283
|
-
|
|
18284
|
-
|
|
18285
|
-
|
|
18286
|
-
|
|
18287
|
-
|
|
18288
|
-
|
|
19296
|
+
if (status === 429) {
|
|
19297
|
+
const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
|
|
19298
|
+
const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
|
|
19299
|
+
const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
|
|
19300
|
+
const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
|
|
19301
|
+
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
19302
|
+
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
19303
|
+
}
|
|
19304
|
+
return relayfileOverload(flat.cause, depth + 1);
|
|
18289
19305
|
};
|
|
18290
19306
|
/**
|
|
18291
19307
|
* relayfile's overload reason codes, allowlisted.
|
|
@@ -18804,13 +19820,14 @@ const PASS_FATAL_CAUSE_DEPTH = 4;
|
|
|
18804
19820
|
* classification has to follow the cause chain rather than trust the outermost
|
|
18805
19821
|
* type.
|
|
18806
19822
|
*/
|
|
18807
|
-
const
|
|
19823
|
+
const findWrappedErrorOfType = (error, type, depth = 0) => {
|
|
18808
19824
|
if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
|
|
18809
|
-
return
|
|
19825
|
+
return undefined;
|
|
18810
19826
|
if (error instanceof type)
|
|
18811
|
-
return
|
|
18812
|
-
return
|
|
19827
|
+
return error;
|
|
19828
|
+
return findWrappedErrorOfType(error.cause, type, depth + 1);
|
|
18813
19829
|
};
|
|
19830
|
+
const wrapsErrorOfType = (error, type) => findWrappedErrorOfType(error, type) !== undefined;
|
|
18814
19831
|
/**
|
|
18815
19832
|
* How many *unclassified* per-item failures without an intervening successful
|
|
18816
19833
|
* dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
|
|
@@ -18826,6 +19843,10 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
|
|
|
18826
19843
|
*/
|
|
18827
19844
|
const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
|
|
18828
19845
|
error instanceof DispatchLifecycleClaimRefusedError ||
|
|
19846
|
+
// An external stop/held-agent deadline rejected the post-spawn boundary
|
|
19847
|
+
// while a provider claim was already in flight. That lifecycle's cleanup is
|
|
19848
|
+
// owned by the rejecting path, so the late claim unwind is an expected skip.
|
|
19849
|
+
error instanceof PostSpawnDispatchWaitRejectedError ||
|
|
18829
19850
|
// #303: the never-placed deadline released this dispatch while its spawn was
|
|
18830
19851
|
// still in flight. Named, expected and self-healing — the issue goes back to
|
|
18831
19852
|
// the queue — and it recurs under exactly the slow-spawn conditions the
|
|
@@ -18850,7 +19871,8 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
|
|
|
18850
19871
|
* of a missed one is leaked agents and duplicate workers on the next retry.
|
|
18851
19872
|
*/
|
|
18852
19873
|
const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
|
|
18853
|
-
!(error instanceof DispatchLifecycleClaimRefusedError)
|
|
19874
|
+
!(error instanceof DispatchLifecycleClaimRefusedError) &&
|
|
19875
|
+
!(error instanceof PostSpawnDispatchWaitRejectedError);
|
|
18854
19876
|
/**
|
|
18855
19877
|
* The run-report reason recorded for a work unit the pass could not dispatch.
|
|
18856
19878
|
*
|
|
@@ -18868,6 +19890,8 @@ const perItemDispatchSkipReason = (error) => {
|
|
|
18868
19890
|
return 'live state changed during dispatch';
|
|
18869
19891
|
if (error instanceof LatePlacementReleasedError)
|
|
18870
19892
|
return 'dispatch released while its agent was still spawning';
|
|
19893
|
+
if (error instanceof PostSpawnDispatchWaitRejectedError)
|
|
19894
|
+
return 'dispatch terminated during post-spawn claim';
|
|
18871
19895
|
if (error instanceof DispatchLifecycleClaimRefusedError) {
|
|
18872
19896
|
return error.refusal === 'terminal'
|
|
18873
19897
|
? 'dispatch lifecycle already terminal'
|
|
@@ -18875,6 +19899,54 @@ const perItemDispatchSkipReason = (error) => {
|
|
|
18875
19899
|
}
|
|
18876
19900
|
return `dispatch failed (${telemetryErrorClass(error)})`;
|
|
18877
19901
|
};
|
|
19902
|
+
const UNCLASSIFIED_PHASE_CODES = {
|
|
19903
|
+
gate: 'unclassified-gate',
|
|
19904
|
+
triage: 'unclassified-triage',
|
|
19905
|
+
dispatch: 'unclassified-dispatch',
|
|
19906
|
+
};
|
|
19907
|
+
/**
|
|
19908
|
+
* Why a dispatch attempt failed, as a code from the published vocabulary (#355).
|
|
19909
|
+
*
|
|
19910
|
+
* The sibling of `perItemDispatchSkipReason`: that returns the operator's
|
|
19911
|
+
* sentence, this returns the one token that may cross onto the unauthenticated
|
|
19912
|
+
* health surface, and both are recorded at the skip site from the same thrown
|
|
19913
|
+
* value. The token is never parsed back out of the sentence — a reworded
|
|
19914
|
+
* message would silently empty a bucket, and this vocabulary is what an
|
|
19915
|
+
* operator reads when the daemon's stdout does not reach them.
|
|
19916
|
+
*
|
|
19917
|
+
* Ordered most specific first. Every branch follows the bounded cause chain,
|
|
19918
|
+
* because `contextualError` and the control-plane guard both rethrow wrapped.
|
|
19919
|
+
* `relayfileOverload` is also the loop's shedding predicate, so widening it at
|
|
19920
|
+
* the source keeps the health code, skip counter, fuse, and durable overload
|
|
19921
|
+
* ratchet on one verdict instead of merely relabelling the published bucket.
|
|
19922
|
+
*/
|
|
19923
|
+
const perItemDispatchFailureCode = (error, phase) => {
|
|
19924
|
+
if (relayfileOverload(error) !== undefined)
|
|
19925
|
+
return 'relayfile-overloaded';
|
|
19926
|
+
if (wrapsErrorOfType(error, LiveDispatchStateChangedError))
|
|
19927
|
+
return 'live-state-changed';
|
|
19928
|
+
if (wrapsErrorOfType(error, LatePlacementReleasedError))
|
|
19929
|
+
return 'late-placement-released';
|
|
19930
|
+
const refused = findWrappedErrorOfType(error, DispatchLifecycleClaimRefusedError);
|
|
19931
|
+
if (refused)
|
|
19932
|
+
return refused.refusal === 'terminal' ? 'lifecycle-terminal' : 'lifecycle-owned-elsewhere';
|
|
19933
|
+
if (wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError))
|
|
19934
|
+
return 'control-plane-open';
|
|
19935
|
+
// The class-name tail of the vocabulary; see its own doc comment for why
|
|
19936
|
+
// these five are not `instanceof`. Walked down the cause chain like the
|
|
19937
|
+
// branches above, because `contextualError` wraps in a plain `Error` and
|
|
19938
|
+
// reading only the outermost name would miss every wrapped spawn failure.
|
|
19939
|
+
// `telemetryErrorClass` is the same allowlist that guards every other
|
|
19940
|
+
// identifier leaving this process, so a hostile `name` cannot invent a key
|
|
19941
|
+
// here either — it collapses to `Error`, which the map does not hold.
|
|
19942
|
+
for (let cursor = error, depth = 0; cursor instanceof Error && depth <= PASS_FATAL_CAUSE_DEPTH; depth += 1) {
|
|
19943
|
+
const named = factoryDispatchFailureReasonCodeForErrorClass(telemetryErrorClass(cursor));
|
|
19944
|
+
if (named)
|
|
19945
|
+
return named;
|
|
19946
|
+
cursor = cursor.cause;
|
|
19947
|
+
}
|
|
19948
|
+
return UNCLASSIFIED_PHASE_CODES[phase];
|
|
19949
|
+
};
|
|
18878
19950
|
const triageEscalationQuestion = (decision, issue) => {
|
|
18879
19951
|
const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
|
|
18880
19952
|
const subject = issue?.title?.trim() || decision.issue.key;
|