@agent-relay/factory 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/cli/diagnose.d.ts +15 -0
- package/dist/cli/diagnose.d.ts.map +1 -1
- package/dist/cli/diagnose.js +34 -0
- package/dist/cli/diagnose.js.map +1 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +5 -0
- package/dist/cli/fleet.js.map +1 -1
- package/dist/cli/teammate-mcp.d.ts +19 -0
- package/dist/cli/teammate-mcp.d.ts.map +1 -0
- package/dist/cli/teammate-mcp.js +102 -0
- package/dist/cli/teammate-mcp.js.map +1 -0
- package/dist/environments/load-profile.d.ts +4 -4
- package/dist/environments/stack-descriptor.d.ts +17 -17
- package/dist/environments/verification-stack-descriptor.d.ts +32 -32
- package/dist/fleet/create-fleet.d.ts +2 -0
- package/dist/fleet/create-fleet.d.ts.map +1 -1
- package/dist/fleet/create-fleet.js +1 -0
- package/dist/fleet/create-fleet.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +29 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +161 -11
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +23 -1
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +167 -2
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/fleet/teammates.d.ts +46 -0
- package/dist/fleet/teammates.d.ts.map +1 -0
- package/dist/fleet/teammates.js +424 -0
- package/dist/fleet/teammates.js.map +1 -0
- package/dist/index.d.ts +9 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +4 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +40 -9
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/mount/relayfile-github-connection-write.d.ts +3 -2
- package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
- package/dist/mount/relayfile-github-connection-write.js +33 -0
- package/dist/mount/relayfile-github-connection-write.js.map +1 -1
- package/dist/node/factory-node-runtime.d.ts +17 -0
- package/dist/node/factory-node-runtime.d.ts.map +1 -0
- package/dist/node/factory-node-runtime.js +100 -0
- package/dist/node/factory-node-runtime.js.map +1 -0
- package/dist/node/factory-node.d.ts +10 -1
- package/dist/node/factory-node.d.ts.map +1 -1
- package/dist/node/factory-node.js +13 -2
- package/dist/node/factory-node.js.map +1 -1
- package/dist/node/factory-persona-card.d.ts +38 -0
- package/dist/node/factory-persona-card.d.ts.map +1 -0
- package/dist/node/factory-persona-card.js +163 -0
- package/dist/node/factory-persona-card.js.map +1 -0
- package/dist/node/factory.node.d.ts +1 -1
- package/dist/observability/events.d.ts +24 -24
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1016 -75
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts.map +1 -1
- package/dist/orchestrator/public-health.js +36 -0
- package/dist/orchestrator/public-health.js.map +1 -1
- package/dist/orchestrator/release-error.d.ts +26 -0
- package/dist/orchestrator/release-error.d.ts.map +1 -0
- package/dist/orchestrator/release-error.js +33 -0
- package/dist/orchestrator/release-error.js.map +1 -0
- package/dist/ports/fleet.d.ts +66 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/fleet.js +10 -1
- package/dist/ports/fleet.js.map +1 -1
- package/dist/ports/index.d.ts +4 -3
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/index.js +1 -1
- package/dist/ports/index.js.map +1 -1
- package/dist/ports/mount.d.ts +10 -2
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/writeback.d.ts +45 -4
- package/dist/ports/writeback.d.ts.map +1 -1
- package/dist/state/watch-state-document.js +4 -1
- package/dist/state/watch-state-document.js.map +1 -1
- package/dist/testing/fakes.d.ts +8 -3
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js +25 -1
- package/dist/testing/fakes.js.map +1 -1
- package/dist/triage/schema.d.ts +14 -14
- package/dist/types.d.ts +52 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/github.d.ts +17 -9
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +331 -9
- package/dist/writeback/github.js.map +1 -1
- package/dist/writeback/linear.d.ts +5 -1
- package/dist/writeback/linear.d.ts.map +1 -1
- package/dist/writeback/linear.js +89 -1
- package/dist/writeback/linear.js.map +1 -1
- package/package.json +4 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
1
2
|
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import { readFile } from 'node:fs/promises';
|
|
3
4
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
@@ -37,6 +38,7 @@ import { telemetryErrorClass } from '../observability/error-class.js';
|
|
|
37
38
|
import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
|
|
38
39
|
import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
|
|
39
40
|
import { factoryDispatchFailureReasonCodeForErrorClass, factoryDispatchFailureReasonCounts, } from './dispatch-failure-reason.js';
|
|
41
|
+
import { isAgentAlreadyGoneOnRelease } from './release-error.js';
|
|
40
42
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
41
43
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
42
44
|
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
@@ -144,6 +146,12 @@ const CLARIFICATION_ESCALATION_LEASE_MS = 2 * 60_000;
|
|
|
144
146
|
const CLARIFICATION_ESCALATION_RETRY_MS = 5_000;
|
|
145
147
|
const CLARIFICATION_STALE_WARN_MS = 7 * 24 * 60 * 60_000;
|
|
146
148
|
const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
149
|
+
// A rejected post-spawn dispatch normally gets a final opportunity to undo an
|
|
150
|
+
// external claim before shutdown relinquishes its lifecycle lease. Provider
|
|
151
|
+
// writes are not guaranteed to settle, so that opportunity must stay inside a
|
|
152
|
+
// bounded shutdown budget; a successor can recover from the retained durable
|
|
153
|
+
// lifecycle after this process releases its local agents.
|
|
154
|
+
const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
|
|
147
155
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
148
156
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
149
157
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
@@ -348,6 +356,16 @@ class ReadinessReconcileTimeoutError extends Error {
|
|
|
348
356
|
this.name = 'ReadinessReconcileTimeoutError';
|
|
349
357
|
}
|
|
350
358
|
}
|
|
359
|
+
class PostSpawnDispatchWaitRejectedError extends Error {
|
|
360
|
+
issueKey;
|
|
361
|
+
compensationError;
|
|
362
|
+
constructor(issueKey, compensationError) {
|
|
363
|
+
super(`Post-spawn dispatch waits were rejected for ${issueKey}`);
|
|
364
|
+
this.issueKey = issueKey;
|
|
365
|
+
this.compensationError = compensationError;
|
|
366
|
+
this.name = 'PostSpawnDispatchWaitRejectedError';
|
|
367
|
+
}
|
|
368
|
+
}
|
|
351
369
|
const realClock = {
|
|
352
370
|
now: () => Date.now(),
|
|
353
371
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -355,6 +373,26 @@ const realClock = {
|
|
|
355
373
|
export function createFactory(config, ports) {
|
|
356
374
|
return new FactoryLoop(FactoryConfigSchema.parse(config), ports);
|
|
357
375
|
}
|
|
376
|
+
/**
|
|
377
|
+
* The discovery pass a tree read was issued by, carried per async call (#363
|
|
378
|
+
* review, CodeRabbit).
|
|
379
|
+
*
|
|
380
|
+
* `#discoverySweepEpoch` says a sweep is in flight; it cannot say the read in
|
|
381
|
+
* hand belongs to it. In live mode an event drain reaches the very same
|
|
382
|
+
* enumeration helpers — `#handlePrChange` -> `#advanceMergedPrToDone` ->
|
|
383
|
+
* `#findMergeAdvanceIssueForPr` -> `#githubIssuePaths()` is the concrete path —
|
|
384
|
+
* so a drain's full-root walk would land in a concurrent sweep's ratio while
|
|
385
|
+
* measuring a different instant. That is enough to hide a mount that went
|
|
386
|
+
* silent mid-sweep.
|
|
387
|
+
*
|
|
388
|
+
* A drain does not inherit this store: its continuation begins at the
|
|
389
|
+
* subscription callback, outside the `run()` below. Threading an epoch argument
|
|
390
|
+
* through `#ingestGithubIssues` / `#handleGithubIssueChange` /
|
|
391
|
+
* `#findGithubIssueMirror` / `#loadLinearMirrorCandidates` would express the
|
|
392
|
+
* same fact and put a parameter on every hot path that must never be passed
|
|
393
|
+
* wrong.
|
|
394
|
+
*/
|
|
395
|
+
const discoveryEnumerationPass = new AsyncLocalStorage();
|
|
358
396
|
export class FactoryLoop {
|
|
359
397
|
#config;
|
|
360
398
|
#mount;
|
|
@@ -590,6 +628,29 @@ export class FactoryLoop {
|
|
|
590
628
|
#previewSweepTimer;
|
|
591
629
|
#previewSweepInFlight;
|
|
592
630
|
#completionInFlight = new Set();
|
|
631
|
+
#issueWritebackInFlight = new Map();
|
|
632
|
+
// A fast completion can make terminal issue state visible while dispatch is
|
|
633
|
+
// still performing its post-spawn readiness read. Completion publishes its
|
|
634
|
+
// provider receipt, then waits for that read to classify the state as owned
|
|
635
|
+
// or foreign before it releases agents under a terminal success reason.
|
|
636
|
+
#postSpawnIssueObservations = new Map();
|
|
637
|
+
// The post-spawn ready read and following dispatch claim form one local
|
|
638
|
+
// write boundary. A completion arriving first makes dispatch wait and
|
|
639
|
+
// re-read; one arriving after claim entry waits for the claim to finish.
|
|
640
|
+
#postSpawnDispatchClaimFences = new Map();
|
|
641
|
+
// Shutdown must not lose a local placement merely because a rejected
|
|
642
|
+
// provider claim unwinds before #releaseInFlightAgents snapshots the batch.
|
|
643
|
+
// The capture set makes both interleavings explicit: the dispatch catch may
|
|
644
|
+
// remove a record only after shutdown has copied its agents, otherwise the
|
|
645
|
+
// release sweep removes it after the copy is safe.
|
|
646
|
+
#shutdownReleaseCapturedDispatches = new Set();
|
|
647
|
+
#postSpawnDispatchesRejectedDuringStop = new Set();
|
|
648
|
+
// A rejected provider claim is unsafe to release until its conditional
|
|
649
|
+
// rollback succeeds or a newer provider status supersedes it. This runtime
|
|
650
|
+
// set closes the race before the durable dispatchClaim flag is persisted.
|
|
651
|
+
#uncompensatedDispatchClaims = new Set();
|
|
652
|
+
#pendingDispatchClaims = new Set();
|
|
653
|
+
#dispatchClaimSettlementsInFlight = new Set();
|
|
593
654
|
#agentExitsInFlight = new Map();
|
|
594
655
|
#reconciledAgentExitsActive = 0;
|
|
595
656
|
#reconciledAgentExitWaiters = [];
|
|
@@ -678,6 +739,10 @@ export class FactoryLoop {
|
|
|
678
739
|
#discoveryOverloadError;
|
|
679
740
|
/** Relayfile operations this sweep has been shed on. */
|
|
680
741
|
#discoverySweepOverloads = 0;
|
|
742
|
+
/** Tree reads this sweep issued that the backend served (#351 follow-up). */
|
|
743
|
+
#discoverySweepTreeReads = 0;
|
|
744
|
+
/** How many of those were served with zero entries. */
|
|
745
|
+
#discoverySweepEmptyTreeReads = 0;
|
|
681
746
|
/**
|
|
682
747
|
* The longest `Retry-After` any operation in this sweep advertised.
|
|
683
748
|
*
|
|
@@ -984,6 +1049,15 @@ export class FactoryLoop {
|
|
|
984
1049
|
}
|
|
985
1050
|
}
|
|
986
1051
|
async #start(opts) {
|
|
1052
|
+
// These sets coordinate one stop snapshot with dispatches unwinding during
|
|
1053
|
+
// that same stop. They are not durable lifecycle state: retaining their
|
|
1054
|
+
// keys across a later start would make the next stop mistake a fresh
|
|
1055
|
+
// dispatch for one already captured by the previous shutdown.
|
|
1056
|
+
this.#shutdownReleaseCapturedDispatches.clear();
|
|
1057
|
+
this.#postSpawnDispatchesRejectedDuringStop.clear();
|
|
1058
|
+
this.#uncompensatedDispatchClaims.clear();
|
|
1059
|
+
this.#pendingDispatchClaims.clear();
|
|
1060
|
+
this.#dispatchClaimSettlementsInFlight.clear();
|
|
987
1061
|
this.#stopping = false;
|
|
988
1062
|
this.#startMode = opts.mode ?? 'live';
|
|
989
1063
|
const issueSource = await this.#issueSource();
|
|
@@ -1098,15 +1172,32 @@ export class FactoryLoop {
|
|
|
1098
1172
|
async stop() {
|
|
1099
1173
|
this.#started = false;
|
|
1100
1174
|
this.#stopping = true;
|
|
1175
|
+
// These waits are armed before every planned spawn returns. Resolve them
|
|
1176
|
+
// before the first shutdown await: a fast first agent may be completing
|
|
1177
|
+
// while a later spawn is hung, and #drainAgentExitsInFlight would otherwise
|
|
1178
|
+
// wait forever on a resolver stranded inside that dispatch stack.
|
|
1179
|
+
const postSpawnWaitKeys = new Set([
|
|
1180
|
+
...this.#postSpawnIssueObservations.keys(),
|
|
1181
|
+
...this.#postSpawnDispatchClaimFences.keys(),
|
|
1182
|
+
]);
|
|
1183
|
+
const rejectedClaimDispatches = new Set();
|
|
1184
|
+
for (const [key, fence] of this.#postSpawnDispatchClaimFences) {
|
|
1185
|
+
if (!fence.claimStarted)
|
|
1186
|
+
continue;
|
|
1187
|
+
const dispatchKeyPrefix = `${key}:live:`;
|
|
1188
|
+
for (const [dispatchKey, dispatch] of this.#dispatchInFlight) {
|
|
1189
|
+
if (dispatchKey.startsWith(dispatchKeyPrefix))
|
|
1190
|
+
rejectedClaimDispatches.add(dispatch);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
for (const key of postSpawnWaitKeys)
|
|
1194
|
+
this.#settlePostSpawnDispatchWaits(key, false, 'stop');
|
|
1101
1195
|
if (this.#babysitterResourceDeliveryRetryTimer)
|
|
1102
1196
|
clearTimeout(this.#babysitterResourceDeliveryRetryTimer);
|
|
1103
1197
|
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
1104
1198
|
if (this.#babysitterResourceSubscriptionRenewTimer)
|
|
1105
1199
|
clearTimeout(this.#babysitterResourceSubscriptionRenewTimer);
|
|
1106
1200
|
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
1107
|
-
if (this.#dispatchLifecycleRenewTimer)
|
|
1108
|
-
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
1109
|
-
this.#dispatchLifecycleRenewTimer = undefined;
|
|
1110
1201
|
if (this.#heldAgentDeadlineTimer)
|
|
1111
1202
|
clearTimeout(this.#heldAgentDeadlineTimer);
|
|
1112
1203
|
this.#heldAgentDeadlineTimer = undefined;
|
|
@@ -1139,12 +1230,25 @@ export class FactoryLoop {
|
|
|
1139
1230
|
await this.#previewSweepInFlight;
|
|
1140
1231
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
1141
1232
|
try {
|
|
1233
|
+
// Only dispatches that had entered the provider-claim boundary can leave
|
|
1234
|
+
// an external claim behind. Drain those rejected promises before
|
|
1235
|
+
// relinquishing their lifecycle leases or tearing down the fleet/mount.
|
|
1236
|
+
// A pre-claim spawn hang is deliberately absent from this set, so it
|
|
1237
|
+
// cannot hold shutdown open.
|
|
1238
|
+
await this.#drainRejectedClaimDispatchesForStop(rejectedClaimDispatches);
|
|
1239
|
+
await this.#persistOutstandingClaimCancellationBlocks();
|
|
1240
|
+
// Keep renewing ownership through the potentially slow provider drain
|
|
1241
|
+
// and cancellation-fence handoff. Stopping renewal earlier can let the
|
|
1242
|
+
// lease expire before the fence is saved, allowing a successor to reap
|
|
1243
|
+
// agents while the old provider write can still land.
|
|
1142
1244
|
// Relinquish durable ownership before waiting on mount-backed lifecycle
|
|
1143
1245
|
// drives. A slow Relayfile scan must not consume the shutdown deadline
|
|
1144
1246
|
// while every issue remains fenced to a publisher that is already
|
|
1145
|
-
// stopping.
|
|
1146
|
-
//
|
|
1147
|
-
|
|
1247
|
+
// stopping. Preserve leases for provider claims whose bounded drain
|
|
1248
|
+
// expired: the original caller still needs its epoch to persist a late
|
|
1249
|
+
// conclusive settlement. A second sweep below catches claims racing this
|
|
1250
|
+
// one.
|
|
1251
|
+
await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
|
|
1148
1252
|
await Promise.allSettled([...this.#dispatchLifecycleDrives]);
|
|
1149
1253
|
// Fence every source of new clarification work before touching the fleet.
|
|
1150
1254
|
// A wake already past the fence is allowed to unwind, and is awaited
|
|
@@ -1167,13 +1271,18 @@ export class FactoryLoop {
|
|
|
1167
1271
|
// non-durable (local/internal) records; terminal completion performs the
|
|
1168
1272
|
// normal remote release before clearing the lifecycle.
|
|
1169
1273
|
await this.#releaseInFlightAgents('factory-stopped', { preserveDurable: true });
|
|
1170
|
-
|
|
1274
|
+
if (this.#dispatchLifecycleRenewTimer)
|
|
1275
|
+
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
1276
|
+
this.#dispatchLifecycleRenewTimer = undefined;
|
|
1277
|
+
await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
|
|
1171
1278
|
if (this.#livePollTimer)
|
|
1172
1279
|
clearTimeout(this.#livePollTimer);
|
|
1173
1280
|
this.#livePollTimer = undefined;
|
|
1174
1281
|
this.#livePollInFlight = false;
|
|
1175
1282
|
this.#liveEventQueue.length = 0;
|
|
1176
1283
|
this.#completionInFlight.clear();
|
|
1284
|
+
this.#postSpawnIssueObservations.clear();
|
|
1285
|
+
this.#postSpawnDispatchClaimFences.clear();
|
|
1177
1286
|
this.#babysitterSpawned.clear();
|
|
1178
1287
|
this.#babysitterPr.clear();
|
|
1179
1288
|
this.#babysitterIssueRefs.clear();
|
|
@@ -1220,10 +1329,13 @@ export class FactoryLoop {
|
|
|
1220
1329
|
await this.#fleet.dispose();
|
|
1221
1330
|
}
|
|
1222
1331
|
finally {
|
|
1332
|
+
if (this.#dispatchLifecycleRenewTimer)
|
|
1333
|
+
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
1334
|
+
this.#dispatchLifecycleRenewTimer = undefined;
|
|
1223
1335
|
this.#stoppingHeartbeatRefreshActive = false;
|
|
1224
1336
|
}
|
|
1225
1337
|
}
|
|
1226
|
-
async #releaseOwnedDispatchLifecycleLeases() {
|
|
1338
|
+
async #releaseOwnedDispatchLifecycleLeases(preserve = () => false) {
|
|
1227
1339
|
// The epoch cache is an execution optimization, not the durable ownership
|
|
1228
1340
|
// authority. Error/fence paths may evict a cached epoch while its persisted
|
|
1229
1341
|
// lease is still ours, so enumerate state before shutdown relinquishment.
|
|
@@ -1234,6 +1346,8 @@ export class FactoryLoop {
|
|
|
1234
1346
|
}
|
|
1235
1347
|
}
|
|
1236
1348
|
for (const [key, epoch] of owned) {
|
|
1349
|
+
if (preserve(key))
|
|
1350
|
+
continue;
|
|
1237
1351
|
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
1238
1352
|
if (this.#dispatchLifecycleEpochs.get(key) === epoch) {
|
|
1239
1353
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
@@ -1257,6 +1371,215 @@ export class FactoryLoop {
|
|
|
1257
1371
|
await Promise.allSettled([...this.#clarificationQuestionDeliveryInFlight.values()]);
|
|
1258
1372
|
}
|
|
1259
1373
|
}
|
|
1374
|
+
async #drainRejectedClaimDispatchesForStop(dispatches) {
|
|
1375
|
+
if (dispatches.size === 0)
|
|
1376
|
+
return;
|
|
1377
|
+
let timer;
|
|
1378
|
+
const drained = Promise.allSettled([...dispatches]).then(() => true);
|
|
1379
|
+
// A durable fleet can recover ownership after this process exits. Local
|
|
1380
|
+
// and other non-durable fleets cannot: releasing their placements before
|
|
1381
|
+
// the provider claim has compensated would leave no successor able to
|
|
1382
|
+
// finish the lifecycle, and a late write could strand the issue in its
|
|
1383
|
+
// claimed state. Keep draining those fleets without the shutdown escape
|
|
1384
|
+
// hatch.
|
|
1385
|
+
if (this.#fleet.durableOwnership !== true) {
|
|
1386
|
+
await drained;
|
|
1387
|
+
return;
|
|
1388
|
+
}
|
|
1389
|
+
const timedOut = new Promise((resolve) => {
|
|
1390
|
+
timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
|
|
1391
|
+
timer.unref?.();
|
|
1392
|
+
});
|
|
1393
|
+
const completed = await Promise.race([drained, timedOut]);
|
|
1394
|
+
if (timer)
|
|
1395
|
+
clearTimeout(timer);
|
|
1396
|
+
if (completed)
|
|
1397
|
+
return;
|
|
1398
|
+
this.#increment('postSpawnDispatchClaimDrainTimeouts');
|
|
1399
|
+
this.#logger.warn?.('[factory] rejected post-spawn dispatch compensation timed out; continuing shutdown', {
|
|
1400
|
+
dispatches: dispatches.size,
|
|
1401
|
+
timeoutMs: STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS,
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1404
|
+
async #persistOutstandingClaimCancellationBlocks() {
|
|
1405
|
+
const batch = await this.#batch();
|
|
1406
|
+
for (const record of batch.inFlight) {
|
|
1407
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1408
|
+
if (!this.#pendingDispatchClaims.has(key))
|
|
1409
|
+
continue;
|
|
1410
|
+
// The bounded durable drain expired before the provider operation could
|
|
1411
|
+
// report whether rollback was safe. Persist the provisional release
|
|
1412
|
+
// fence while this publisher still owns the lifecycle lease, so a
|
|
1413
|
+
// successor cannot mistake the unresolved claim for abandonable work.
|
|
1414
|
+
const persisted = await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not settle before shutdown'), { preferExistingError: true, pending: true });
|
|
1415
|
+
if (!persisted) {
|
|
1416
|
+
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
1417
|
+
if (durable?.dispatchClaim?.cancellationPending !== true) {
|
|
1418
|
+
throw new Error(`Unable to persist unresolved dispatch-claim fence for ${record.issue.key}; refusing shutdown handoff`);
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
async #awaitRejectedClaimCompensation(settled) {
|
|
1424
|
+
let timer;
|
|
1425
|
+
const timedOut = new Promise((resolve) => {
|
|
1426
|
+
timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
|
|
1427
|
+
timer.unref?.();
|
|
1428
|
+
});
|
|
1429
|
+
const compensated = await Promise.race([settled, timedOut]);
|
|
1430
|
+
if (timer)
|
|
1431
|
+
clearTimeout(timer);
|
|
1432
|
+
return compensated;
|
|
1433
|
+
}
|
|
1434
|
+
/**
|
|
1435
|
+
* Re-adopt this publisher's own lease epoch after a refused lifecycle write.
|
|
1436
|
+
*
|
|
1437
|
+
* `#saveDispatchLifecycle` cannot tell "the store rejected our fence" from
|
|
1438
|
+
* "the store declined this row's contents", so it evicts the cached epoch for
|
|
1439
|
+
* both. Only the first is a loss of ownership. Re-read the durable row and
|
|
1440
|
+
* take the epoch back when the lease is still ours and still live; a row that
|
|
1441
|
+
* is gone, terminal, expired, or owned by someone else keeps the eviction,
|
|
1442
|
+
* because for those the eviction was correct.
|
|
1443
|
+
*/
|
|
1444
|
+
async #recoverDispatchLifecycleLeaseEpoch(key) {
|
|
1445
|
+
if (this.#dispatchLifecycleEpochs.has(key))
|
|
1446
|
+
return true;
|
|
1447
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
1448
|
+
const lease = lifecycle?.lease;
|
|
1449
|
+
if (!lifecycle || !lease || isTerminalDispatchLifecycle(lifecycle))
|
|
1450
|
+
return false;
|
|
1451
|
+
if (lease.owner !== this.#dispatchLifecycleOwner)
|
|
1452
|
+
return false;
|
|
1453
|
+
if (lease.leaseUntilMs <= this.#clock.now())
|
|
1454
|
+
return false;
|
|
1455
|
+
this.#dispatchLifecycleEpochs.set(key, lease.epoch);
|
|
1456
|
+
this.#increment('dispatchLifecycleLeaseEpochsRecovered');
|
|
1457
|
+
return true;
|
|
1458
|
+
}
|
|
1459
|
+
/**
|
|
1460
|
+
* Persist a dispatch-claim fence without letting a refused write disarm the
|
|
1461
|
+
* lease the next one needs.
|
|
1462
|
+
*
|
|
1463
|
+
* The shutdown handoff deliberately probes with a provisional
|
|
1464
|
+
* `cancellationPending` fence, and a store may refuse that row on its
|
|
1465
|
+
* contents while this process still holds the lease. Losing the epoch there
|
|
1466
|
+
* would make every later write for this lifecycle short-circuit on
|
|
1467
|
+
* `epoch === undefined` — including the conclusive compensated settlement
|
|
1468
|
+
* that clears the block and frees the agents and the batch slot. That turns
|
|
1469
|
+
* one refused probe into a lifecycle retained until restart, so bracket the
|
|
1470
|
+
* write with epoch recovery instead.
|
|
1471
|
+
*/
|
|
1472
|
+
async #persistDispatchClaimFence(record) {
|
|
1473
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1474
|
+
await this.#recoverDispatchLifecycleLeaseEpoch(key);
|
|
1475
|
+
const saved = await this.#saveDispatchLifecycle(record, 'running');
|
|
1476
|
+
if (!saved)
|
|
1477
|
+
await this.#recoverDispatchLifecycleLeaseEpoch(key);
|
|
1478
|
+
return saved;
|
|
1479
|
+
}
|
|
1480
|
+
async #retainUncompensatedDispatchClaim(record, error, opts = {}) {
|
|
1481
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1482
|
+
const alreadyBlocked = record.dispatchClaim?.cancellationBlocked === true;
|
|
1483
|
+
const errorMessage = describeError(error).errorMessage;
|
|
1484
|
+
this.#uncompensatedDispatchClaims.add(key);
|
|
1485
|
+
const { cancellationPending: _pending, ...existingClaim } = record.dispatchClaim ?? { state: 'degraded' };
|
|
1486
|
+
record.dispatchClaim = {
|
|
1487
|
+
...existingClaim,
|
|
1488
|
+
state: 'degraded',
|
|
1489
|
+
cancellationBlocked: true,
|
|
1490
|
+
...(opts.pending ? { cancellationPending: true } : {}),
|
|
1491
|
+
write: 'rejected dispatch claim rollback',
|
|
1492
|
+
error: opts.preferExistingError ? record.dispatchClaim?.error ?? errorMessage : errorMessage,
|
|
1493
|
+
deadLettered: true,
|
|
1494
|
+
updatedAtMs: this.#clock.now(),
|
|
1495
|
+
};
|
|
1496
|
+
this.#dispatchClaimStatuses.set(key, record.dispatchClaim);
|
|
1497
|
+
if (!alreadyBlocked) {
|
|
1498
|
+
this.#increment('postSpawnDispatchClaimRecoveryRetentions');
|
|
1499
|
+
this.#logger.warn?.('[factory] retaining dispatch lifecycle because rejected claim compensation is unproven', {
|
|
1500
|
+
issue: record.issue.key,
|
|
1501
|
+
});
|
|
1502
|
+
}
|
|
1503
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
1504
|
+
return await this.#persistDispatchClaimFence(record);
|
|
1505
|
+
}
|
|
1506
|
+
else {
|
|
1507
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
1508
|
+
return true;
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
async #clearDispatchCancellationBlock(record) {
|
|
1512
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1513
|
+
if (record.dispatchClaim?.cancellationBlocked !== true) {
|
|
1514
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
1515
|
+
return true;
|
|
1516
|
+
}
|
|
1517
|
+
const previousClaim = record.dispatchClaim;
|
|
1518
|
+
const { cancellationBlocked: _blocked, cancellationPending: _pending, ...claim } = previousClaim;
|
|
1519
|
+
record.dispatchClaim = claim;
|
|
1520
|
+
this.#dispatchClaimStatuses.set(key, claim);
|
|
1521
|
+
try {
|
|
1522
|
+
const persisted = this.#usesDurableDispatchLifecycle()
|
|
1523
|
+
? await this.#persistDispatchClaimFence(record)
|
|
1524
|
+
: await this.#writeDispatchClaimRegistry(record.issue).then(() => true);
|
|
1525
|
+
if (!persisted) {
|
|
1526
|
+
record.dispatchClaim = previousClaim;
|
|
1527
|
+
this.#dispatchClaimStatuses.set(key, previousClaim);
|
|
1528
|
+
return false;
|
|
1529
|
+
}
|
|
1530
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
1531
|
+
return true;
|
|
1532
|
+
}
|
|
1533
|
+
catch (error) {
|
|
1534
|
+
record.dispatchClaim = previousClaim;
|
|
1535
|
+
this.#dispatchClaimStatuses.set(key, previousClaim);
|
|
1536
|
+
throw error;
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
async #dispatchClaimBlocksAbandonment(record) {
|
|
1540
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
1541
|
+
if (!this.#uncompensatedDispatchClaims.has(key) && record.dispatchClaim?.cancellationBlocked !== true) {
|
|
1542
|
+
return false;
|
|
1543
|
+
}
|
|
1544
|
+
// A provider request that was still unresolved at handoff can land after
|
|
1545
|
+
// any current-state read. No observation is proof of supersession until
|
|
1546
|
+
// the original caller records that the operation conclusively settled.
|
|
1547
|
+
if (record.dispatchClaim?.cancellationPending === true)
|
|
1548
|
+
return true;
|
|
1549
|
+
try {
|
|
1550
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
1551
|
+
if (!issue)
|
|
1552
|
+
return true;
|
|
1553
|
+
if (isGithubIssue(issue)) {
|
|
1554
|
+
if (!this.#githubWriteback.getIssueStatus)
|
|
1555
|
+
return true;
|
|
1556
|
+
const status = await this.#githubWriteback.getIssueStatus(issue, {
|
|
1557
|
+
requireFresh: true,
|
|
1558
|
+
freshAfterMs: record.dispatchClaim?.claimStartedAtMs,
|
|
1559
|
+
});
|
|
1560
|
+
if (!status || status === 'in-progress')
|
|
1561
|
+
return true;
|
|
1562
|
+
}
|
|
1563
|
+
else {
|
|
1564
|
+
if (!this.#linear.getIssueStateId)
|
|
1565
|
+
return true;
|
|
1566
|
+
const implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
1567
|
+
const currentStateId = await this.#linear.getIssueStateId(issue);
|
|
1568
|
+
if (!implementingStateId || !currentStateId || currentStateId === implementingStateId)
|
|
1569
|
+
return true;
|
|
1570
|
+
}
|
|
1571
|
+
if (!await this.#clearDispatchCancellationBlock(record))
|
|
1572
|
+
return true;
|
|
1573
|
+
return false;
|
|
1574
|
+
}
|
|
1575
|
+
catch (error) {
|
|
1576
|
+
this.#logger.warn?.('[factory] unable to verify whether a blocked dispatch claim was superseded', {
|
|
1577
|
+
issue: record.issue.key,
|
|
1578
|
+
error: describeError(error).errorMessage,
|
|
1579
|
+
});
|
|
1580
|
+
return true;
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1260
1583
|
async #boundedStopTeardown(label, teardown) {
|
|
1261
1584
|
let timer;
|
|
1262
1585
|
const action = Promise.resolve()
|
|
@@ -2368,6 +2691,8 @@ export class FactoryLoop {
|
|
|
2368
2691
|
this.#discoverySweepLeaseLost = false;
|
|
2369
2692
|
this.#discoveryOverloadError = undefined;
|
|
2370
2693
|
this.#discoverySweepOverloads = 0;
|
|
2694
|
+
this.#discoverySweepTreeReads = 0;
|
|
2695
|
+
this.#discoverySweepEmptyTreeReads = 0;
|
|
2371
2696
|
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2372
2697
|
this.#discoverySweepProgress = false;
|
|
2373
2698
|
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
@@ -2448,6 +2773,8 @@ export class FactoryLoop {
|
|
|
2448
2773
|
this.#discoverySweepStartedAtMs = undefined;
|
|
2449
2774
|
this.#discoveryOverloadError = undefined;
|
|
2450
2775
|
this.#discoverySweepOverloads = 0;
|
|
2776
|
+
this.#discoverySweepTreeReads = 0;
|
|
2777
|
+
this.#discoverySweepEmptyTreeReads = 0;
|
|
2451
2778
|
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2452
2779
|
this.#discoverySweepProgress = false;
|
|
2453
2780
|
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
@@ -2566,13 +2893,19 @@ export class FactoryLoop {
|
|
|
2566
2893
|
this.#dependencyGithubPathsByIdentity = undefined;
|
|
2567
2894
|
this.#dependencyLinearTreeLoaded = false;
|
|
2568
2895
|
const issueSource = await this.#issueSource();
|
|
2896
|
+
// The sweep's own discovery pass, and the only region whose full-root
|
|
2897
|
+
// walks may enter the tree-read ratio (#363 review). Everything reached
|
|
2898
|
+
// from here inherits the marker; a concurrently running drain does not.
|
|
2899
|
+
const enumerate = (fn) => this.#discoverySweepEpoch === undefined
|
|
2900
|
+
? fn()
|
|
2901
|
+
: discoveryEnumerationPass.run({ epoch: this.#discoverySweepEpoch }, fn);
|
|
2569
2902
|
if (issueSource === 'linear') {
|
|
2570
|
-
await this.#ingestGithubIssues({ dryRun });
|
|
2903
|
+
await enumerate(() => this.#ingestGithubIssues({ dryRun }));
|
|
2571
2904
|
}
|
|
2572
2905
|
else {
|
|
2573
2906
|
await this.#ensureGithubIngestionReady();
|
|
2574
2907
|
}
|
|
2575
|
-
const paths = await this.#readyIssuePaths();
|
|
2908
|
+
const paths = await enumerate(() => this.#readyIssuePaths());
|
|
2576
2909
|
const orphanRecovery = issueSource === 'github'
|
|
2577
2910
|
? await this.#githubOrphanRecoveryContext(dryRun)
|
|
2578
2911
|
: undefined;
|
|
@@ -2877,6 +3210,10 @@ export class FactoryLoop {
|
|
|
2877
3210
|
dispatched,
|
|
2878
3211
|
skipped,
|
|
2879
3212
|
dryRun,
|
|
3213
|
+
// Read before `#runOnceWithDiscoveryFence`'s finally resets them: this
|
|
3214
|
+
// is still inside that try, so the counts are this sweep's own.
|
|
3215
|
+
treeReads: this.#discoverySweepTreeReads,
|
|
3216
|
+
emptyTreeReads: this.#discoverySweepEmptyTreeReads,
|
|
2880
3217
|
slackDegraded: this.#slackDegraded,
|
|
2881
3218
|
...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
|
|
2882
3219
|
};
|
|
@@ -3782,6 +4119,10 @@ export class FactoryLoop {
|
|
|
3782
4119
|
// that expects a specific PR to appear, or an escalation/comment-replay
|
|
3783
4120
|
// scan that must not miss a marker or reply that landed after the cache
|
|
3784
4121
|
// was populated. Those callers must omit `cache` and pay for a fresh list.
|
|
4122
|
+
//
|
|
4123
|
+
// `enumeration` marks the call sites that ARE the readiness sweep's
|
|
4124
|
+
// discovery pass — a full-root walk looking for candidate work. Only those
|
|
4125
|
+
// feed the tree-read pair below, and only while a sweep holds the lease.
|
|
3785
4126
|
async #listRelayfileTree(prefix, phase, opts = {}) {
|
|
3786
4127
|
if (this.#discoverySweepLeaseLost) {
|
|
3787
4128
|
throw new Error('discovery sweep lease was lost; refusing another tree request');
|
|
@@ -3800,6 +4141,46 @@ export class FactoryLoop {
|
|
|
3800
4141
|
logStart: true,
|
|
3801
4142
|
logComplete: true,
|
|
3802
4143
|
});
|
|
4144
|
+
// #351 follow-up: tree reads the backend SERVED, and how many of them it
|
|
4145
|
+
// answered with nothing.
|
|
4146
|
+
//
|
|
4147
|
+
// The per-call deadline made a hung dependency loud. This is its
|
|
4148
|
+
// companion: a mount that starts returning empty trees instead of hanging
|
|
4149
|
+
// raises no timeout, no failure and no `lastError`, and produces a sweep
|
|
4150
|
+
// that completes `healthy` having dispatched nothing.
|
|
4151
|
+
//
|
|
4152
|
+
// BOTH numbers, because one is not a signal. A healthy sweep lists two
|
|
4153
|
+
// path forms per repo and only one of them exists, so an empty read is
|
|
4154
|
+
// ordinary and a bare count of them fires constantly. What separates the
|
|
4155
|
+
// fault is the RATIO: `emptyTreeReads === treeReads` means the mount
|
|
4156
|
+
// served nothing at all, which `candidates: 0` cannot distinguish from a
|
|
4157
|
+
// workspace that simply has no ready work.
|
|
4158
|
+
//
|
|
4159
|
+
// SCOPED TWICE, and both guards are load-bearing (#363 review, codex P2).
|
|
4160
|
+
// The ratio is only readable if every read in it came from the same
|
|
4161
|
+
// question. In live mode a Slack identity lookup, a PR-confirmation poll
|
|
4162
|
+
// or a comment-replay scan can run concurrently with `runOnce()` — each is
|
|
4163
|
+
// a `listTree`, none is discovery, and one populated result among them
|
|
4164
|
+
// makes `emptyTreeReads < treeReads` on an all-empty sweep, silently
|
|
4165
|
+
// erasing exactly the signal this pair exists to raise. `enumeration`
|
|
4166
|
+
// keeps the numerator and denominator to the discovery pass; the epoch
|
|
4167
|
+
// check keeps a discovery walk issued outside any sweep — startup
|
|
4168
|
+
// backfill, most obviously — out of a sweep's totals.
|
|
4169
|
+
// `enumeration` says this is a full-root discovery walk rather than a point
|
|
4170
|
+
// lookup; the context says THIS sweep's discovery pass is what issued it.
|
|
4171
|
+
// Both, because either alone admits a read the ratio cannot use. Compared
|
|
4172
|
+
// by value rather than presence: an absent store and an absent epoch are
|
|
4173
|
+
// both `undefined` and must not read as a match.
|
|
4174
|
+
const issuingPass = discoveryEnumerationPass.getStore();
|
|
4175
|
+
if (opts.enumeration &&
|
|
4176
|
+
issuingPass !== undefined &&
|
|
4177
|
+
issuingPass.epoch === this.#discoverySweepEpoch) {
|
|
4178
|
+
this.#discoverySweepTreeReads += 1;
|
|
4179
|
+
if (paths.length === 0) {
|
|
4180
|
+
this.#increment('relayfileEmptyTreeReads');
|
|
4181
|
+
this.#discoverySweepEmptyTreeReads += 1;
|
|
4182
|
+
}
|
|
4183
|
+
}
|
|
3803
4184
|
if (opts.cache)
|
|
3804
4185
|
await this.#rememberDiscoveryTree(prefix, paths);
|
|
3805
4186
|
return paths;
|
|
@@ -4315,6 +4696,66 @@ export class FactoryLoop {
|
|
|
4315
4696
|
if (!dryRun)
|
|
4316
4697
|
await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
|
|
4317
4698
|
const spawnedForReaperHandoff = [];
|
|
4699
|
+
// These waits belong to the durable work unit, not the ingestion surface.
|
|
4700
|
+
// Completion and lifecycle ownership use the same identity (#211); using
|
|
4701
|
+
// issueKey here would make every GitHub-native dispatch miss its own fence.
|
|
4702
|
+
const postSpawnKey = dispatchLifecycleKey(record.issue);
|
|
4703
|
+
let resolvePostSpawnIssueObservation;
|
|
4704
|
+
let postSpawnIssueObservationDidSettle = false;
|
|
4705
|
+
const postSpawnIssueObservationSettled = new Promise((resolve) => {
|
|
4706
|
+
resolvePostSpawnIssueObservation = resolve;
|
|
4707
|
+
});
|
|
4708
|
+
let postSpawnIssueObservation;
|
|
4709
|
+
postSpawnIssueObservation = {
|
|
4710
|
+
settled: postSpawnIssueObservationSettled,
|
|
4711
|
+
settle: (accepted) => {
|
|
4712
|
+
if (postSpawnIssueObservationDidSettle)
|
|
4713
|
+
return;
|
|
4714
|
+
postSpawnIssueObservationDidSettle = true;
|
|
4715
|
+
resolvePostSpawnIssueObservation(accepted);
|
|
4716
|
+
if (this.#postSpawnIssueObservations.get(postSpawnKey) === postSpawnIssueObservation) {
|
|
4717
|
+
this.#postSpawnIssueObservations.delete(postSpawnKey);
|
|
4718
|
+
}
|
|
4719
|
+
},
|
|
4720
|
+
};
|
|
4721
|
+
this.#postSpawnIssueObservations.set(postSpawnKey, postSpawnIssueObservation);
|
|
4722
|
+
let resolvePostSpawnDispatchClaim;
|
|
4723
|
+
let postSpawnDispatchClaimSettled = false;
|
|
4724
|
+
const postSpawnDispatchClaim = new Promise((resolve) => {
|
|
4725
|
+
resolvePostSpawnDispatchClaim = resolve;
|
|
4726
|
+
});
|
|
4727
|
+
let resolveRejectedClaimCompensation;
|
|
4728
|
+
let rejectedClaimCompensationSettled = false;
|
|
4729
|
+
const rejectedClaimCompensation = new Promise((resolve) => {
|
|
4730
|
+
resolveRejectedClaimCompensation = resolve;
|
|
4731
|
+
});
|
|
4732
|
+
let postSpawnDispatchClaimFence;
|
|
4733
|
+
postSpawnDispatchClaimFence = {
|
|
4734
|
+
completionAtWriteBoundary: false,
|
|
4735
|
+
claimStarted: false,
|
|
4736
|
+
settled: postSpawnDispatchClaim,
|
|
4737
|
+
rejectionSettled: rejectedClaimCompensation,
|
|
4738
|
+
settle: (accepted) => {
|
|
4739
|
+
if (postSpawnDispatchClaimSettled)
|
|
4740
|
+
return;
|
|
4741
|
+
postSpawnDispatchClaimSettled = true;
|
|
4742
|
+
postSpawnDispatchClaimFence.accepted = accepted;
|
|
4743
|
+
resolvePostSpawnDispatchClaim(accepted);
|
|
4744
|
+
if (this.#postSpawnDispatchClaimFences.get(postSpawnKey) === postSpawnDispatchClaimFence) {
|
|
4745
|
+
this.#postSpawnDispatchClaimFences.delete(postSpawnKey);
|
|
4746
|
+
}
|
|
4747
|
+
},
|
|
4748
|
+
settleRejection: (compensated) => {
|
|
4749
|
+
if (rejectedClaimCompensationSettled)
|
|
4750
|
+
return;
|
|
4751
|
+
rejectedClaimCompensationSettled = true;
|
|
4752
|
+
resolveRejectedClaimCompensation(compensated);
|
|
4753
|
+
},
|
|
4754
|
+
};
|
|
4755
|
+
this.#postSpawnDispatchClaimFences.set(postSpawnKey, postSpawnDispatchClaimFence);
|
|
4756
|
+
const settlePostSpawnIssueObservation = postSpawnIssueObservation.settle;
|
|
4757
|
+
const settlePostSpawnDispatchClaim = postSpawnDispatchClaimFence.settle;
|
|
4758
|
+
let rejectDispatchClaim;
|
|
4318
4759
|
try {
|
|
4319
4760
|
if (!dryRun) {
|
|
4320
4761
|
const issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
@@ -4339,47 +4780,92 @@ export class FactoryLoop {
|
|
|
4339
4780
|
agents.push({ name: spawned.name, role: spec.role });
|
|
4340
4781
|
}
|
|
4341
4782
|
if (!dryRun) {
|
|
4783
|
+
const claimStartedAtMs = this.#clock.now();
|
|
4342
4784
|
record.dispatchClaim = {
|
|
4343
4785
|
state: 'pending',
|
|
4344
|
-
|
|
4786
|
+
claimStartedAtMs,
|
|
4787
|
+
updatedAtMs: claimStartedAtMs,
|
|
4345
4788
|
};
|
|
4346
4789
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
4347
4790
|
}
|
|
4348
4791
|
await this.#writeInFlightRegistry();
|
|
4349
4792
|
const comment = dispatchComment(dispatchDecision, agents);
|
|
4350
4793
|
let implementingStateId;
|
|
4794
|
+
const rejectDispatchClaimIfCancelled = async () => {
|
|
4795
|
+
if (postSpawnDispatchClaimFence.accepted !== false)
|
|
4796
|
+
return;
|
|
4797
|
+
if (rejectDispatchClaim)
|
|
4798
|
+
await rejectDispatchClaim();
|
|
4799
|
+
throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
|
|
4800
|
+
};
|
|
4351
4801
|
if (!dryRun) {
|
|
4352
|
-
|
|
4353
|
-
if (
|
|
4354
|
-
//
|
|
4355
|
-
//
|
|
4356
|
-
//
|
|
4357
|
-
//
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
//
|
|
4802
|
+
let issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
4803
|
+
if (postSpawnDispatchClaimFence.completionAtWriteBoundary) {
|
|
4804
|
+
// Completion reached its terminal provider-write boundary before
|
|
4805
|
+
// dispatch entered the claim boundary. Always re-read after its
|
|
4806
|
+
// receipt, even when the first snapshot was ready: the terminal
|
|
4807
|
+
// write may have started immediately after that snapshot.
|
|
4808
|
+
const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
|
|
4809
|
+
if (issueWriteback)
|
|
4810
|
+
await issueWriteback;
|
|
4811
|
+
issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
4812
|
+
}
|
|
4813
|
+
else if ((!issue || !this.#isIssueReady(issue)) && record.issueWritebackConfirmedAtMs === undefined) {
|
|
4814
|
+
// A very fast agent can finish while its own dispatch is still
|
|
4815
|
+
// performing this post-spawn read. Do not guess authorship from the
|
|
4816
|
+
// first visible state: an idempotent provider mutation cannot tell us
|
|
4817
|
+
// whether Factory or another actor won the race. Instead, wait for
|
|
4818
|
+
// this record's concurrent completion writeback to settle, then use
|
|
4819
|
+
// only its provider-confirmed outcome (factory#319).
|
|
4365
4820
|
//
|
|
4366
|
-
//
|
|
4367
|
-
//
|
|
4368
|
-
//
|
|
4369
|
-
// the
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4821
|
+
// The completion promise settles as soon as terminal issue writeback
|
|
4822
|
+
// succeeds or fails, before Slack/release cleanup. If there is no
|
|
4823
|
+
// completion in flight, this is an ordinary foreign state change and
|
|
4824
|
+
// the existing abort remains immediate.
|
|
4825
|
+
const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
|
|
4826
|
+
if (issueWriteback) {
|
|
4827
|
+
await issueWriteback;
|
|
4828
|
+
issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
4829
|
+
}
|
|
4830
|
+
}
|
|
4831
|
+
if (!issue || !this.#isIssueReady(issue)) {
|
|
4832
|
+
// Only a terminal writeback applied and confirmed by the completion
|
|
4833
|
+
// already in flight for this exact record may excuse the change. The
|
|
4834
|
+
// marker is stamped after provider acknowledgement/readback, never at
|
|
4835
|
+
// the earlier locally-visible write boundary. A confirmed no-op does
|
|
4836
|
+
// not establish authorship: another actor may have parked the issue
|
|
4837
|
+
// before our adapter's first provider read.
|
|
4374
4838
|
if (record.issueWritebackConfirmedAtMs === undefined) {
|
|
4375
4839
|
throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
|
|
4376
4840
|
}
|
|
4377
4841
|
// The claim is moot and would be wrong to write: it would drag an
|
|
4378
4842
|
// issue our own lifecycle has already parked back to `implementing`.
|
|
4379
4843
|
}
|
|
4844
|
+
else if (this.#stopping) {
|
|
4845
|
+
// Shutdown snapshots the post-spawn fence maps once and settles only
|
|
4846
|
+
// what it finds there. A dispatch that armed its fence after that
|
|
4847
|
+
// snapshot — it can still be inside the preview render, the
|
|
4848
|
+
// `dispatching` save or the question-watch setup when `stop()`
|
|
4849
|
+
// begins — would otherwise enter the claim boundary below against a
|
|
4850
|
+
// fleet already being disposed and leases already relinquished,
|
|
4851
|
+
// leaving a claimed issue with no lifecycle to recover it. Reject
|
|
4852
|
+
// here instead. No claim has started, so there is nothing to
|
|
4853
|
+
// compensate, and this stays synchronous so the completion-race
|
|
4854
|
+
// invariant documented below is untouched (#346 review, cubic).
|
|
4855
|
+
this.#increment('postSpawnDispatchClaimsRefusedDuringStop');
|
|
4856
|
+
postSpawnDispatchClaimFence.settle(false);
|
|
4857
|
+
throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
|
|
4858
|
+
}
|
|
4380
4859
|
else {
|
|
4381
|
-
|
|
4382
|
-
|
|
4860
|
+
// This assignment is synchronous with the preceding completion flag
|
|
4861
|
+
// check. Completion either arrived first (the branch above) or will
|
|
4862
|
+
// now observe claimStarted and wait; no await-sized gap remains.
|
|
4863
|
+
postSpawnDispatchClaimFence.claimStarted = true;
|
|
4864
|
+
const appliedClaim = await this.#applyDispatchClaim(record, issue, comment, postSpawnDispatchClaimFence);
|
|
4865
|
+
implementingStateId = appliedClaim.implementingStateId;
|
|
4866
|
+
rejectDispatchClaim = appliedClaim.reject;
|
|
4867
|
+
await rejectDispatchClaimIfCancelled();
|
|
4868
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
4383
4869
|
this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
|
|
4384
4870
|
}
|
|
4385
4871
|
}
|
|
@@ -4394,19 +4880,110 @@ export class FactoryLoop {
|
|
|
4394
4880
|
dryRun,
|
|
4395
4881
|
};
|
|
4396
4882
|
record.result = result;
|
|
4397
|
-
|
|
4883
|
+
const runningSaved = await this.#saveDispatchLifecycle(record, 'running');
|
|
4884
|
+
await rejectDispatchClaimIfCancelled();
|
|
4885
|
+
if (!runningSaved) {
|
|
4886
|
+
settlePostSpawnDispatchClaim(true);
|
|
4887
|
+
settlePostSpawnIssueObservation(true);
|
|
4398
4888
|
return result;
|
|
4399
|
-
|
|
4400
|
-
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
4889
|
+
}
|
|
4401
4890
|
if (!dryRun && this.#config.hooks?.onTicketDispatch) {
|
|
4402
4891
|
await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
|
|
4892
|
+
await rejectDispatchClaimIfCancelled();
|
|
4403
4893
|
}
|
|
4404
4894
|
if (!dryRun) {
|
|
4405
4895
|
await this.#ensureSlackDispatchThread(record, result, liveIssue);
|
|
4896
|
+
await rejectDispatchClaimIfCancelled();
|
|
4406
4897
|
}
|
|
4898
|
+
// Accept without an intervening await only after every durable
|
|
4899
|
+
// publication and post-dispatch side effect has completed. Stop or held
|
|
4900
|
+
// abandonment can therefore reject any still-publishing dispatch.
|
|
4901
|
+
settlePostSpawnDispatchClaim(true);
|
|
4902
|
+
settlePostSpawnIssueObservation(true);
|
|
4903
|
+
this.#increment('dispatched');
|
|
4904
|
+
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
4407
4905
|
return result;
|
|
4408
4906
|
}
|
|
4409
|
-
catch (
|
|
4907
|
+
catch (caughtError) {
|
|
4908
|
+
// Stop/deadline may reject the fence while any awaited provider or
|
|
4909
|
+
// lifecycle operation is still in flight. If that operation then fails,
|
|
4910
|
+
// its raw error must not fall into ordinary dispatch cleanup: shutdown
|
|
4911
|
+
// still owns the already-spawned placements and needs them retained
|
|
4912
|
+
// until its release snapshot. If the claim receipt is already available,
|
|
4913
|
+
// conditionally reject it before releasing those placements. Without a
|
|
4914
|
+
// receipt the provider outcome is ambiguous, so preserve the original
|
|
4915
|
+
// failure as the compensation detail after the claim boundary began.
|
|
4916
|
+
let error = caughtError;
|
|
4917
|
+
if (postSpawnDispatchClaimFence.accepted === false
|
|
4918
|
+
&& !(caughtError instanceof PostSpawnDispatchWaitRejectedError)) {
|
|
4919
|
+
if (rejectDispatchClaim) {
|
|
4920
|
+
try {
|
|
4921
|
+
await this.#rejectDispatchClaimAfterOperationError(dispatchDecision.issue.key, caughtError, rejectDispatchClaim);
|
|
4922
|
+
}
|
|
4923
|
+
catch (rejectionError) {
|
|
4924
|
+
error = rejectionError;
|
|
4925
|
+
}
|
|
4926
|
+
}
|
|
4927
|
+
else {
|
|
4928
|
+
error = new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key, postSpawnDispatchClaimFence.claimStarted ? caughtError : undefined);
|
|
4929
|
+
}
|
|
4930
|
+
}
|
|
4931
|
+
settlePostSpawnDispatchClaim(false);
|
|
4932
|
+
if (error instanceof PostSpawnDispatchWaitRejectedError) {
|
|
4933
|
+
settlePostSpawnIssueObservation(false);
|
|
4934
|
+
const compensated = error.compensationError === undefined;
|
|
4935
|
+
// The original provider call has conclusively returned. Stop the
|
|
4936
|
+
// timeout handoff from writing a provisional pending fence, but keep a
|
|
4937
|
+
// separate lease fence until the definitive result is durable. Without
|
|
4938
|
+
// that second set, shutdown can relinquish this epoch in the await
|
|
4939
|
+
// below and make the late save fail after the provider compensated.
|
|
4940
|
+
this.#dispatchClaimSettlementsInFlight.add(postSpawnKey);
|
|
4941
|
+
this.#pendingDispatchClaims.delete(postSpawnKey);
|
|
4942
|
+
let settlementPersisted = false;
|
|
4943
|
+
try {
|
|
4944
|
+
settlementPersisted = compensated
|
|
4945
|
+
? await this.#clearDispatchCancellationBlock(record)
|
|
4946
|
+
: await this.#retainUncompensatedDispatchClaim(record, error.compensationError);
|
|
4947
|
+
if (!settlementPersisted) {
|
|
4948
|
+
throw new Error(`Unable to persist settled dispatch-claim compensation for ${decision.issue.key}`);
|
|
4949
|
+
}
|
|
4950
|
+
postSpawnDispatchClaimFence.settleRejection(compensated);
|
|
4951
|
+
if (error.compensationError !== undefined) {
|
|
4952
|
+
this.#increment('postSpawnDispatchClaimCompensationFailures');
|
|
4953
|
+
this.#logger.error?.('[factory] rejected dispatch claim could not be safely compensated', {
|
|
4954
|
+
issue: decision.issue.key,
|
|
4955
|
+
error: describeError(error.compensationError).errorMessage,
|
|
4956
|
+
});
|
|
4957
|
+
}
|
|
4958
|
+
// External stop/abandonment owns agent and durable-lifecycle cleanup.
|
|
4959
|
+
// Remove only the process-local slot here; running the ordinary
|
|
4960
|
+
// dispatch-failure path would race or duplicate that cleanup. During
|
|
4961
|
+
// stop, however, retain local placements until the shutdown release
|
|
4962
|
+
// sweep has captured them. If it already has, removal is safe now.
|
|
4963
|
+
if (!compensated) {
|
|
4964
|
+
// The provider still exposes a claim this lifecycle authored. Keep
|
|
4965
|
+
// the process-local slot and durable row recoverable; releasing its
|
|
4966
|
+
// agents would leave the claimed issue with no successor lifecycle.
|
|
4967
|
+
}
|
|
4968
|
+
else if (this.#stopping) {
|
|
4969
|
+
this.#postSpawnDispatchesRejectedDuringStop.add(postSpawnKey);
|
|
4970
|
+
if (this.#shutdownReleaseCapturedDispatches.has(postSpawnKey)) {
|
|
4971
|
+
batch.abandon(decision.issue);
|
|
4972
|
+
this.#postSpawnDispatchesRejectedDuringStop.delete(postSpawnKey);
|
|
4973
|
+
}
|
|
4974
|
+
}
|
|
4975
|
+
else {
|
|
4976
|
+
batch.abandon(decision.issue);
|
|
4977
|
+
}
|
|
4978
|
+
}
|
|
4979
|
+
finally {
|
|
4980
|
+
if (!settlementPersisted)
|
|
4981
|
+
this.#pendingDispatchClaims.add(postSpawnKey);
|
|
4982
|
+
this.#dispatchClaimSettlementsInFlight.delete(postSpawnKey);
|
|
4983
|
+
}
|
|
4984
|
+
throw error;
|
|
4985
|
+
}
|
|
4986
|
+
settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
|
|
4410
4987
|
// A spawn can fail after the broker accepted it but before its ack
|
|
4411
4988
|
// reached Factory. Include every planned worktree agent, not only the
|
|
4412
4989
|
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
@@ -4579,6 +5156,8 @@ export class FactoryLoop {
|
|
|
4579
5156
|
this.#readinessReconcileLastSweepDeferred = undefined;
|
|
4580
5157
|
this.#readinessReconcileLastSweep = {
|
|
4581
5158
|
candidates: report.pulled.length,
|
|
5159
|
+
treeReads: report.treeReads ?? 0,
|
|
5160
|
+
emptyTreeReads: report.emptyTreeReads ?? 0,
|
|
4582
5161
|
dispatched: report.dispatched.length,
|
|
4583
5162
|
skipped: report.skipped.length,
|
|
4584
5163
|
skipReasons: factorySweepSkipReasonCounts(report.skipped),
|
|
@@ -4673,6 +5252,13 @@ export class FactoryLoop {
|
|
|
4673
5252
|
...(this.#readinessReconcileLastSweep
|
|
4674
5253
|
? {
|
|
4675
5254
|
candidates: this.#readinessReconcileLastSweep.candidates,
|
|
5255
|
+
// Unconditional for the same reason `dispatchFailures` is: these
|
|
5256
|
+
// are only meaningful as a pair, and only a published zero lets a
|
|
5257
|
+
// reader see that `emptyTreeReads < treeReads` — i.e. that the
|
|
5258
|
+
// mount served real content and a zero `candidates` beside it means
|
|
5259
|
+
// an empty workspace, not a silent mount (#351 follow-up).
|
|
5260
|
+
treeReads: this.#readinessReconcileLastSweep.treeReads,
|
|
5261
|
+
emptyTreeReads: this.#readinessReconcileLastSweep.emptyTreeReads,
|
|
4676
5262
|
dispatched: this.#readinessReconcileLastSweep.dispatched,
|
|
4677
5263
|
skipped: this.#readinessReconcileLastSweep.skipped,
|
|
4678
5264
|
...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
|
|
@@ -6259,9 +6845,11 @@ export class FactoryLoop {
|
|
|
6259
6845
|
const comment = dispatchComment(record.decision, agents);
|
|
6260
6846
|
let implementingStateId;
|
|
6261
6847
|
if (!record.dryRun) {
|
|
6848
|
+
const claimStartedAtMs = this.#clock.now();
|
|
6262
6849
|
record.dispatchClaim = {
|
|
6263
6850
|
state: 'pending',
|
|
6264
|
-
|
|
6851
|
+
claimStartedAtMs,
|
|
6852
|
+
updatedAtMs: claimStartedAtMs,
|
|
6265
6853
|
};
|
|
6266
6854
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
6267
6855
|
}
|
|
@@ -6271,7 +6859,7 @@ export class FactoryLoop {
|
|
|
6271
6859
|
if (!issue)
|
|
6272
6860
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
6273
6861
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
6274
|
-
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
6862
|
+
implementingStateId = (await this.#applyDispatchClaim(record, issue, comment)).implementingStateId;
|
|
6275
6863
|
}
|
|
6276
6864
|
const recoveredPreviews = uniquePreviewReferences([
|
|
6277
6865
|
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
@@ -6580,33 +7168,161 @@ export class FactoryLoop {
|
|
|
6580
7168
|
}
|
|
6581
7169
|
await this.#linear.postComment(issue, body);
|
|
6582
7170
|
}
|
|
6583
|
-
async #applyDispatchClaim(record, issue, comment) {
|
|
7171
|
+
async #applyDispatchClaim(record, issue, comment, claimFence) {
|
|
7172
|
+
const claimWasRejected = () => claimFence?.accepted === false;
|
|
7173
|
+
// stop() can reject this fence after the post-spawn readiness read but
|
|
7174
|
+
// before this method starts. Never begin a provider claim in that state;
|
|
7175
|
+
// shutdown intentionally does not drain dispatches that never crossed the
|
|
7176
|
+
// claim boundary.
|
|
7177
|
+
if (claimWasRejected()) {
|
|
7178
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key);
|
|
7179
|
+
}
|
|
6584
7180
|
let implementingStateId;
|
|
7181
|
+
let rejectClaim;
|
|
6585
7182
|
if (isGithubIssue(issue)) {
|
|
7183
|
+
const statusClaim = {
|
|
7184
|
+
value: { result: undefined },
|
|
7185
|
+
};
|
|
6586
7186
|
await this.#retryDispatchWriteback(record, issue, 'GitHub label factory:in-progress', async () => {
|
|
6587
|
-
|
|
7187
|
+
statusClaim.value = this.#githubWriteback.claimStatus
|
|
7188
|
+
? await this.#githubWriteback.claimStatus(issue, 'in-progress')
|
|
7189
|
+
: { result: await this.#githubWriteback.setStatus(issue, 'in-progress') };
|
|
6588
7190
|
});
|
|
7191
|
+
rejectClaim = async () => await this.#rejectGithubDispatchClaim(record, issue, statusClaim.value);
|
|
7192
|
+
if (claimWasRejected()) {
|
|
7193
|
+
await rejectClaim();
|
|
7194
|
+
}
|
|
6589
7195
|
const commentApplied = this.#githubWriteback.hasCommentMarker
|
|
6590
7196
|
? async () => this.#githubWriteback.hasCommentMarker(issue, comment)
|
|
6591
7197
|
: undefined;
|
|
6592
|
-
|
|
7198
|
+
try {
|
|
7199
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => {
|
|
7200
|
+
if (claimWasRejected())
|
|
7201
|
+
await rejectClaim();
|
|
7202
|
+
await this.#githubWriteback.postComment(issue, comment);
|
|
7203
|
+
}, commentApplied);
|
|
7204
|
+
}
|
|
7205
|
+
catch (error) {
|
|
7206
|
+
if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
|
|
7207
|
+
await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
|
|
7208
|
+
}
|
|
7209
|
+
throw error;
|
|
7210
|
+
}
|
|
7211
|
+
if (claimWasRejected())
|
|
7212
|
+
await rejectClaim();
|
|
6593
7213
|
}
|
|
6594
7214
|
else {
|
|
6595
7215
|
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
7216
|
+
const readyStateId = issue.stateId;
|
|
7217
|
+
let claimToken;
|
|
6596
7218
|
await this.#retryDispatchWriteback(record, issue, `Linear state ${implementingStateId}`, async () => {
|
|
6597
|
-
await this.#linear.setState(issue, implementingStateId);
|
|
6598
|
-
});
|
|
6599
|
-
await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
|
|
6600
|
-
await this.#linear.postComment(issue, comment);
|
|
7219
|
+
claimToken = (await this.#linear.setState(issue, implementingStateId))?.claimToken;
|
|
6601
7220
|
});
|
|
7221
|
+
rejectClaim = async () => await this.#rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken);
|
|
7222
|
+
if (claimWasRejected()) {
|
|
7223
|
+
await rejectClaim();
|
|
7224
|
+
}
|
|
7225
|
+
try {
|
|
7226
|
+
await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
|
|
7227
|
+
if (claimWasRejected())
|
|
7228
|
+
await rejectClaim();
|
|
7229
|
+
await this.#linear.postComment(issue, comment);
|
|
7230
|
+
});
|
|
7231
|
+
}
|
|
7232
|
+
catch (error) {
|
|
7233
|
+
if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
|
|
7234
|
+
await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
|
|
7235
|
+
}
|
|
7236
|
+
throw error;
|
|
7237
|
+
}
|
|
7238
|
+
if (claimWasRejected())
|
|
7239
|
+
await rejectClaim();
|
|
6602
7240
|
}
|
|
6603
7241
|
record.dispatchClaim = {
|
|
6604
7242
|
state: 'verified',
|
|
7243
|
+
// Cancellation can still arrive after this returns — while the running
|
|
7244
|
+
// lifecycle save, the dispatch hook or the Slack publication is pending.
|
|
7245
|
+
// The rejection fence rebuilds the claim from this record, so a verified
|
|
7246
|
+
// state that drops the claim-start stamp leaves `cancellationBlocked`
|
|
7247
|
+
// with no watermark for `getIssueStatus(..., { requireFresh: true })`.
|
|
7248
|
+
// A private repository has no unauthenticated fallback to fall back on,
|
|
7249
|
+
// so that lifecycle, its agents and its batch slot would then be retained
|
|
7250
|
+
// for good, even once a human superseded the claim (#346 review, codex).
|
|
7251
|
+
...(record.dispatchClaim?.claimStartedAtMs === undefined
|
|
7252
|
+
? {}
|
|
7253
|
+
: { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
|
|
6605
7254
|
updatedAtMs: this.#clock.now(),
|
|
6606
7255
|
};
|
|
6607
7256
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
6608
7257
|
await this.#writeDispatchClaimRegistry(record.issue);
|
|
6609
|
-
|
|
7258
|
+
if (claimWasRejected())
|
|
7259
|
+
await rejectClaim?.();
|
|
7260
|
+
return {
|
|
7261
|
+
...(implementingStateId ? { implementingStateId } : {}),
|
|
7262
|
+
reject: rejectClaim,
|
|
7263
|
+
};
|
|
7264
|
+
}
|
|
7265
|
+
async #rejectGithubDispatchClaim(record, issue, claim) {
|
|
7266
|
+
let compensationError;
|
|
7267
|
+
if (claim.result !== 'already-matched') {
|
|
7268
|
+
try {
|
|
7269
|
+
const claimToken = claim.claimToken;
|
|
7270
|
+
if (!claimToken) {
|
|
7271
|
+
throw new Error('GitHub rejected dispatch claim has no immutable ownership token');
|
|
7272
|
+
}
|
|
7273
|
+
const rollbackStatusClaim = this.#githubWriteback.rollbackStatusClaim?.bind(this.#githubWriteback);
|
|
7274
|
+
if (!rollbackStatusClaim) {
|
|
7275
|
+
throw new Error('GitHub writeback cannot conditionally roll back a rejected dispatch claim');
|
|
7276
|
+
}
|
|
7277
|
+
let outcome = 'unproven';
|
|
7278
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub rejected dispatch claim rollback', async () => {
|
|
7279
|
+
outcome = await rollbackStatusClaim(issue, 'in-progress', claimToken);
|
|
7280
|
+
if (outcome === 'unproven') {
|
|
7281
|
+
throw new Error('GitHub rejected dispatch claim rollback could not prove ownership');
|
|
7282
|
+
}
|
|
7283
|
+
});
|
|
7284
|
+
}
|
|
7285
|
+
catch (error) {
|
|
7286
|
+
compensationError = error;
|
|
7287
|
+
}
|
|
7288
|
+
}
|
|
7289
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
|
|
7290
|
+
}
|
|
7291
|
+
async #rejectDispatchClaimAfterOperationError(issueKey, operationError, rejectClaim) {
|
|
7292
|
+
try {
|
|
7293
|
+
await rejectClaim();
|
|
7294
|
+
}
|
|
7295
|
+
catch (rejectionError) {
|
|
7296
|
+
if (rejectionError instanceof PostSpawnDispatchWaitRejectedError) {
|
|
7297
|
+
if (rejectionError.compensationError === undefined)
|
|
7298
|
+
throw rejectionError;
|
|
7299
|
+
throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError.compensationError], 'Cancelled dispatch operation failed and its claim could not be safely compensated'));
|
|
7300
|
+
}
|
|
7301
|
+
throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError], 'Cancelled dispatch operation failed and its claim rejection threw unexpectedly'));
|
|
7302
|
+
}
|
|
7303
|
+
throw new PostSpawnDispatchWaitRejectedError(issueKey);
|
|
7304
|
+
}
|
|
7305
|
+
async #rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken) {
|
|
7306
|
+
let compensationError;
|
|
7307
|
+
try {
|
|
7308
|
+
const compareAndSetState = this.#linear.compareAndSetState?.bind(this.#linear);
|
|
7309
|
+
if (!compareAndSetState) {
|
|
7310
|
+
throw new Error('Linear writeback cannot conditionally roll back a rejected dispatch claim');
|
|
7311
|
+
}
|
|
7312
|
+
if (!claimToken) {
|
|
7313
|
+
throw new Error('Linear rejected dispatch claim has no immutable ownership token');
|
|
7314
|
+
}
|
|
7315
|
+
await this.#retryDispatchWriteback(record, issue, `Linear rejected dispatch claim rollback to ${readyStateId}`, async () => {
|
|
7316
|
+
const outcome = await compareAndSetState(issue, implementingStateId, claimToken, readyStateId);
|
|
7317
|
+
if (outcome === 'unproven') {
|
|
7318
|
+
throw new Error('Linear rejected dispatch claim rollback could not prove an atomic provider transition');
|
|
7319
|
+
}
|
|
7320
|
+
});
|
|
7321
|
+
}
|
|
7322
|
+
catch (error) {
|
|
7323
|
+
compensationError = error;
|
|
7324
|
+
}
|
|
7325
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
|
|
6610
7326
|
}
|
|
6611
7327
|
async #retryDispatchWriteback(record, issue, write, apply, isApplied) {
|
|
6612
7328
|
let lastError;
|
|
@@ -6621,11 +7337,23 @@ export class FactoryLoop {
|
|
|
6621
7337
|
return;
|
|
6622
7338
|
}
|
|
6623
7339
|
catch (error) {
|
|
7340
|
+
// Claim rejection is terminal for this dispatch. In particular, do not
|
|
7341
|
+
// turn a cancellation discovered between comment attempts into another
|
|
7342
|
+
// provider retry that can publish after shutdown began.
|
|
7343
|
+
if (error instanceof PostSpawnDispatchWaitRejectedError) {
|
|
7344
|
+
if (lastError !== undefined && error.compensationError !== undefined) {
|
|
7345
|
+
throw new PostSpawnDispatchWaitRejectedError(record.issue.key, new AggregateError([lastError, error.compensationError], 'Dispatch writeback failed before cancellation and its claim could not be safely compensated'));
|
|
7346
|
+
}
|
|
7347
|
+
throw error;
|
|
7348
|
+
}
|
|
6624
7349
|
lastError = error;
|
|
6625
7350
|
const deadLettered = attempt === DISPATCH_WRITEBACK_MAX_ATTEMPTS;
|
|
6626
7351
|
this.#increment('dispatchWritebackFailures');
|
|
6627
7352
|
record.dispatchClaim = {
|
|
6628
7353
|
state: 'degraded',
|
|
7354
|
+
...(record.dispatchClaim?.claimStartedAtMs === undefined
|
|
7355
|
+
? {}
|
|
7356
|
+
: { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
|
|
6629
7357
|
write,
|
|
6630
7358
|
attempts: attempt,
|
|
6631
7359
|
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
@@ -6728,7 +7456,7 @@ export class FactoryLoop {
|
|
|
6728
7456
|
const candidates = [];
|
|
6729
7457
|
let scanned = 0;
|
|
6730
7458
|
let lastProgressAtMs = startedAtMs;
|
|
6731
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true })) {
|
|
7459
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true, enumeration: true })) {
|
|
6732
7460
|
await this.#refreshLiveHeartbeatIfDue();
|
|
6733
7461
|
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
6734
7462
|
continue;
|
|
@@ -6778,7 +7506,7 @@ export class FactoryLoop {
|
|
|
6778
7506
|
else {
|
|
6779
7507
|
pathBatches = [];
|
|
6780
7508
|
for (const root of roots) {
|
|
6781
|
-
pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true }));
|
|
7509
|
+
pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true, enumeration: true }));
|
|
6782
7510
|
}
|
|
6783
7511
|
this.#increment('githubIssueIndexFallbacks');
|
|
6784
7512
|
}
|
|
@@ -7256,6 +7984,10 @@ export class FactoryLoop {
|
|
|
7256
7984
|
// The pass-scoped flag also lets every later blocked issue reuse the
|
|
7257
7985
|
// resulting dependency index instead of rescanning the full tree.
|
|
7258
7986
|
this.#dependencyLinearTreeLoaded = true;
|
|
7987
|
+
// Cached like the enumeration walks, but deliberately NOT `enumeration`:
|
|
7988
|
+
// this is a lookup for named blocker identities, not the sweep asking
|
|
7989
|
+
// what work exists, and mixing a lookup's result into the ratio is the
|
|
7990
|
+
// masking the pair's scoping exists to prevent (#363 review).
|
|
7259
7991
|
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery', { cache: true })) {
|
|
7260
7992
|
if (isIssueFilePath(path))
|
|
7261
7993
|
await this.#readIssue(path);
|
|
@@ -7706,14 +8438,14 @@ export class FactoryLoop {
|
|
|
7706
8438
|
}
|
|
7707
8439
|
const pathsByKey = new Map();
|
|
7708
8440
|
const canonicalPathsByKey = new Map();
|
|
7709
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true })) {
|
|
8441
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true, enumeration: true })) {
|
|
7710
8442
|
if (isIssueFilePath(path)) {
|
|
7711
8443
|
const key = keyFromPath(path);
|
|
7712
8444
|
canonicalPathsByKey.set(key, path);
|
|
7713
8445
|
pathsByKey.set(key, path);
|
|
7714
8446
|
}
|
|
7715
8447
|
}
|
|
7716
|
-
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true })) {
|
|
8448
|
+
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true, enumeration: true })) {
|
|
7717
8449
|
if (isIssueAliasFilePath(path)) {
|
|
7718
8450
|
const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
|
|
7719
8451
|
if (canonicalPath) {
|
|
@@ -7767,10 +8499,20 @@ export class FactoryLoop {
|
|
|
7767
8499
|
}
|
|
7768
8500
|
async #releaseInFlightAgents(reason, opts = {}) {
|
|
7769
8501
|
const agents = new Map();
|
|
7770
|
-
|
|
8502
|
+
const blockedDispatches = new Set();
|
|
8503
|
+
const batch = await this.#batch();
|
|
8504
|
+
const records = [...batch.inFlight];
|
|
8505
|
+
for (const record of records) {
|
|
8506
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
7771
8507
|
if (record.dryRun) {
|
|
8508
|
+
this.#shutdownReleaseCapturedDispatches.add(key);
|
|
7772
8509
|
continue;
|
|
7773
8510
|
}
|
|
8511
|
+
if (this.#uncompensatedDispatchClaims.has(key) || record.dispatchClaim?.cancellationBlocked === true) {
|
|
8512
|
+
blockedDispatches.add(key);
|
|
8513
|
+
continue;
|
|
8514
|
+
}
|
|
8515
|
+
this.#shutdownReleaseCapturedDispatches.add(key);
|
|
7774
8516
|
if (opts.preserveDurable && [...record.agents.values()].some((tracked) => tracked.result?.locality === 'remote')) {
|
|
7775
8517
|
continue;
|
|
7776
8518
|
}
|
|
@@ -7779,7 +8521,15 @@ export class FactoryLoop {
|
|
|
7779
8521
|
}
|
|
7780
8522
|
}
|
|
7781
8523
|
await this.#releaseAndTerminateAgents([...agents], reason, 'stop');
|
|
7782
|
-
|
|
8524
|
+
for (const record of records) {
|
|
8525
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
8526
|
+
if (blockedDispatches.has(key))
|
|
8527
|
+
continue;
|
|
8528
|
+
if (!this.#postSpawnDispatchesRejectedDuringStop.delete(key))
|
|
8529
|
+
continue;
|
|
8530
|
+
batch.abandon(record.issue);
|
|
8531
|
+
}
|
|
8532
|
+
await this.#writeInFlightRegistry(undefined, undefined, true, blockedDispatches);
|
|
7783
8533
|
}
|
|
7784
8534
|
async #releaseAndTerminateAgents(agents, reason, context) {
|
|
7785
8535
|
const failed = [];
|
|
@@ -7838,17 +8588,40 @@ export class FactoryLoop {
|
|
|
7838
8588
|
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
7839
8589
|
}
|
|
7840
8590
|
catch (error) {
|
|
7841
|
-
|
|
7842
|
-
|
|
7843
|
-
|
|
7844
|
-
|
|
7845
|
-
|
|
7846
|
-
|
|
7847
|
-
|
|
7848
|
-
|
|
7849
|
-
|
|
7850
|
-
|
|
7851
|
-
|
|
8591
|
+
if (isAgentAlreadyGoneOnRelease(error)) {
|
|
8592
|
+
// Do the same bookkeeping the success path does — the agent IS
|
|
8593
|
+
// gone. Skipping this would leave `batch.recordRelease` unset and
|
|
8594
|
+
// the next retry attempt would think the invocation is still
|
|
8595
|
+
// dispatchable.
|
|
8596
|
+
this.#increment('releaseAgentAlreadyGone');
|
|
8597
|
+
this.#logger.info?.(`[factory] release skipped: agent already gone during ${context}`, { agentName, reason });
|
|
8598
|
+
if (record && batch && context !== 'stop') {
|
|
8599
|
+
const releasedInvocationId = batch.recordRelease(record, agentName, this.#clock.now());
|
|
8600
|
+
if (releasedInvocationId) {
|
|
8601
|
+
this.#logger.debug?.('[factory] released agent invocation is no longer dispatchable', {
|
|
8602
|
+
issue: record.issue.key,
|
|
8603
|
+
agentName,
|
|
8604
|
+
reason,
|
|
8605
|
+
invocationId: releasedInvocationId,
|
|
8606
|
+
});
|
|
8607
|
+
}
|
|
8608
|
+
}
|
|
8609
|
+
if (record)
|
|
8610
|
+
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
8611
|
+
}
|
|
8612
|
+
else {
|
|
8613
|
+
failed.push(agentName);
|
|
8614
|
+
this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
|
|
8615
|
+
if (record) {
|
|
8616
|
+
const lifecycle = await this.#state
|
|
8617
|
+
.getDispatchLifecycle(this.#workspaceId, dispatchLifecycleKey(record.issue))
|
|
8618
|
+
.catch(() => undefined);
|
|
8619
|
+
if (lifecycle) {
|
|
8620
|
+
await this.#reportLifecycle(lifecycle, 'factory.failure', {
|
|
8621
|
+
level: 'error',
|
|
8622
|
+
errorCode: 'release_failed',
|
|
8623
|
+
});
|
|
8624
|
+
}
|
|
7852
8625
|
}
|
|
7853
8626
|
}
|
|
7854
8627
|
}
|
|
@@ -7998,7 +8771,7 @@ export class FactoryLoop {
|
|
|
7998
8771
|
this.#increment('agentWorktreesCleaned');
|
|
7999
8772
|
}
|
|
8000
8773
|
}
|
|
8001
|
-
async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false) {
|
|
8774
|
+
async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false, retainIssueKeys) {
|
|
8002
8775
|
const updatedAtMs = this.#clock.now();
|
|
8003
8776
|
const agents = [];
|
|
8004
8777
|
const seenAgents = new Set();
|
|
@@ -8036,10 +8809,12 @@ export class FactoryLoop {
|
|
|
8036
8809
|
} : {}),
|
|
8037
8810
|
});
|
|
8038
8811
|
};
|
|
8039
|
-
if (!empty) {
|
|
8812
|
+
if (!empty || retainIssueKeys?.size) {
|
|
8040
8813
|
for (const record of (await this.#batch()).inFlight) {
|
|
8041
8814
|
if (record.dryRun)
|
|
8042
8815
|
continue;
|
|
8816
|
+
if (empty && !retainIssueKeys?.has(dispatchLifecycleKey(record.issue)))
|
|
8817
|
+
continue;
|
|
8043
8818
|
if (record.dispatchClaim) {
|
|
8044
8819
|
this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
|
|
8045
8820
|
}
|
|
@@ -9254,10 +10029,59 @@ export class FactoryLoop {
|
|
|
9254
10029
|
// the release-driven exit event so it cannot re-trigger a resume before the
|
|
9255
10030
|
// record leaves the batch.
|
|
9256
10031
|
async #abandonStuckDispatch(record, reason) {
|
|
9257
|
-
const key =
|
|
10032
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
10033
|
+
// Fence the periodic held sweep before any await below. A blocked claim is
|
|
10034
|
+
// retried by the single keyed abandonment timer; letting the held sweep
|
|
10035
|
+
// enter concurrently can release the same placements twice when provider
|
|
10036
|
+
// supersession becomes visible.
|
|
10037
|
+
this.#abandonedDispatchReasons.set(key, reason);
|
|
10038
|
+
try {
|
|
10039
|
+
await this.#abandonStuckDispatchFenced(record, reason, key);
|
|
10040
|
+
}
|
|
10041
|
+
catch (error) {
|
|
10042
|
+
// The early fence above suppresses the periodic held sweep. Any
|
|
10043
|
+
// exceptional exit must therefore install the keyed retry before
|
|
10044
|
+
// propagating, or one transient persistence/read failure becomes a
|
|
10045
|
+
// permanent absorbing state.
|
|
10046
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
10047
|
+
this.#logger.warn?.('[factory] abandoned dispatch cleanup failed before retry was armed', {
|
|
10048
|
+
issue: record.issue.key,
|
|
10049
|
+
error: describeError(error).errorMessage,
|
|
10050
|
+
});
|
|
10051
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10052
|
+
throw error;
|
|
10053
|
+
}
|
|
10054
|
+
}
|
|
10055
|
+
async #abandonStuckDispatchFenced(record, reason, key) {
|
|
10056
|
+
const claimFence = this.#postSpawnDispatchClaimFences.get(key);
|
|
10057
|
+
const rejectedClaimCompensation = claimFence?.claimStarted
|
|
10058
|
+
? claimFence.rejectionSettled
|
|
10059
|
+
: undefined;
|
|
10060
|
+
// A dispatch can be stuck inside a later spawn after an earlier agent has
|
|
10061
|
+
// already started completion. Release both post-spawn waits before the
|
|
10062
|
+
// first abandonment await so terminal processing cannot remain an
|
|
10063
|
+
// absorbing promise after this lifecycle is reaped.
|
|
10064
|
+
this.#settlePostSpawnDispatchWaits(key, false, 'abandonment');
|
|
10065
|
+
if (rejectedClaimCompensation) {
|
|
10066
|
+
const compensated = await this.#awaitRejectedClaimCompensation(rejectedClaimCompensation);
|
|
10067
|
+
if (!compensated) {
|
|
10068
|
+
if (!this.#pendingDispatchClaims.has(key)) {
|
|
10069
|
+
// The provider call settled while this bounded wait expired; its
|
|
10070
|
+
// caller is recording the definitive compensated/unproven result.
|
|
10071
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10072
|
+
return;
|
|
10073
|
+
}
|
|
10074
|
+
await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not complete before abandonment'), { preferExistingError: true, pending: true });
|
|
10075
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10076
|
+
return;
|
|
10077
|
+
}
|
|
10078
|
+
}
|
|
10079
|
+
if (await this.#dispatchClaimBlocksAbandonment(record)) {
|
|
10080
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
10081
|
+
return;
|
|
10082
|
+
}
|
|
9258
10083
|
const heldPastDeadline = reason === HELD_PAST_DEADLINE_RELEASE_REASON;
|
|
9259
10084
|
const agentReleaseReason = heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : 'issue-abandoned';
|
|
9260
|
-
this.#abandonedDispatchReasons.set(key, reason);
|
|
9261
10085
|
if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
9262
10086
|
this.#increment('abandonedDispatchReleaseRetries');
|
|
9263
10087
|
// The generic durable retry can recover the in-memory reason in this
|
|
@@ -13285,6 +14109,22 @@ export class FactoryLoop {
|
|
|
13285
14109
|
return;
|
|
13286
14110
|
}
|
|
13287
14111
|
this.#completionInFlight.add(completionKey);
|
|
14112
|
+
const postSpawnIssueObservation = this.#postSpawnIssueObservations.get(completionKey);
|
|
14113
|
+
const postSpawnDispatchClaimFence = this.#postSpawnDispatchClaimFences.get(completionKey);
|
|
14114
|
+
let settleIssueWriteback;
|
|
14115
|
+
const issueWritebackSettled = new Promise((resolve) => {
|
|
14116
|
+
settleIssueWriteback = resolve;
|
|
14117
|
+
});
|
|
14118
|
+
let issueWritebackDidSettle = false;
|
|
14119
|
+
const settleIssueWritebackOnce = () => {
|
|
14120
|
+
if (issueWritebackDidSettle)
|
|
14121
|
+
return;
|
|
14122
|
+
issueWritebackDidSettle = true;
|
|
14123
|
+
settleIssueWriteback();
|
|
14124
|
+
if (this.#issueWritebackInFlight.get(completionKey) === issueWritebackSettled) {
|
|
14125
|
+
this.#issueWritebackInFlight.delete(completionKey);
|
|
14126
|
+
}
|
|
14127
|
+
};
|
|
13288
14128
|
let releaseReasonForRetry;
|
|
13289
14129
|
try {
|
|
13290
14130
|
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
@@ -13334,13 +14174,59 @@ export class FactoryLoop {
|
|
|
13334
14174
|
const humanReview = configuredHumanReview || (githubIssue && !githubMerged);
|
|
13335
14175
|
const statusLabel = humanReview ? 'In Human Review' : 'Done';
|
|
13336
14176
|
if (issue) {
|
|
14177
|
+
if (postSpawnDispatchClaimFence) {
|
|
14178
|
+
// Publish boundary arrival before any await. Dispatch performs the
|
|
14179
|
+
// paired synchronous claimStarted assignment after its ready read,
|
|
14180
|
+
// so exactly one side wins: an earlier completion is awaited and
|
|
14181
|
+
// re-read; a claim already entering its provider write finishes
|
|
14182
|
+
// before this terminal write begins.
|
|
14183
|
+
postSpawnDispatchClaimFence.completionAtWriteBoundary = true;
|
|
14184
|
+
if (postSpawnDispatchClaimFence.claimStarted || postSpawnDispatchClaimFence.accepted !== undefined) {
|
|
14185
|
+
let claimAccepted = postSpawnDispatchClaimFence.accepted;
|
|
14186
|
+
if (claimAccepted === undefined) {
|
|
14187
|
+
// This counter makes the otherwise internal promise boundary
|
|
14188
|
+
// observable in deterministic race regressions and in incident
|
|
14189
|
+
// telemetry. External abandonment/stop settlement must release
|
|
14190
|
+
// every completion counted here.
|
|
14191
|
+
this.#increment('postSpawnDispatchClaimFenceWaits');
|
|
14192
|
+
claimAccepted = await postSpawnDispatchClaimFence.settled;
|
|
14193
|
+
}
|
|
14194
|
+
if (!claimAccepted)
|
|
14195
|
+
return;
|
|
14196
|
+
}
|
|
14197
|
+
}
|
|
14198
|
+
// Register only at the provider-write boundary. Work before this point
|
|
14199
|
+
// (PR discovery/merge gating) has not changed the issue, so a foreign
|
|
14200
|
+
// park during it must still abort immediately rather than waiting on a
|
|
14201
|
+
// possibly long completion path.
|
|
14202
|
+
this.#issueWritebackInFlight.set(completionKey, issueWritebackSettled);
|
|
13337
14203
|
if (githubIssue) {
|
|
13338
14204
|
if (humanReview) {
|
|
13339
|
-
await this.#githubWriteback.setStatus(issue, 'human-review');
|
|
14205
|
+
const statusWrite = await this.#githubWriteback.setStatus(issue, 'human-review');
|
|
14206
|
+
if (statusWrite === undefined)
|
|
14207
|
+
this.#recordMissingGithubWritebackReceipt('setStatus');
|
|
14208
|
+
// Only an explicit provider-proven transition establishes that
|
|
14209
|
+
// this dispatch owns the visible park. Legacy void adapters and
|
|
14210
|
+
// App acknowledgements remain deliberately untrusted.
|
|
14211
|
+
if (statusWrite === 'applied') {
|
|
14212
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
14213
|
+
}
|
|
14214
|
+
// The lifecycle-state outcome is now known. Unblock the concurrent
|
|
14215
|
+
// post-spawn read before the separate completion comment write.
|
|
14216
|
+
settleIssueWritebackOnce();
|
|
13340
14217
|
await this.#githubWriteback.postComment(issue, `Factory agents completed; this issue is awaiting human review. The pull request remains open.\n\nMerge policy: ${this.#config.mergePolicy}`);
|
|
13341
14218
|
}
|
|
13342
14219
|
else {
|
|
13343
|
-
await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
|
|
14220
|
+
const closeWrite = await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
|
|
14221
|
+
if (closeWrite === undefined)
|
|
14222
|
+
this.#recordMissingGithubWritebackReceipt('closeIssue');
|
|
14223
|
+
// A provider-confirmed, actor-attributed close is the only safe
|
|
14224
|
+
// proof that this dispatch owns the visible terminal state. An
|
|
14225
|
+
// idempotent no-op, legacy void adapter, or App acknowledgement
|
|
14226
|
+
// may instead mean a third party won the close race.
|
|
14227
|
+
if (closeWrite === 'applied') {
|
|
14228
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
14229
|
+
}
|
|
13344
14230
|
}
|
|
13345
14231
|
}
|
|
13346
14232
|
else {
|
|
@@ -13348,13 +14234,28 @@ export class FactoryLoop {
|
|
|
13348
14234
|
? this.#states.idFor(issueTeam, 'humanReview')
|
|
13349
14235
|
: this.#states.idFor(issueTeam, 'done');
|
|
13350
14236
|
await this.#linear.setState(issue, targetState);
|
|
14237
|
+
record.issueWritebackConfirmedAtMs ??= this.#clock.now();
|
|
13351
14238
|
await this.#recordCanonicalIssueState({ ...record.issue, stateId: targetState });
|
|
13352
14239
|
}
|
|
13353
|
-
record.issueWritebackConfirmedAtMs
|
|
13354
|
-
|
|
14240
|
+
if (record.issueWritebackConfirmedAtMs !== undefined) {
|
|
14241
|
+
this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
|
|
14242
|
+
}
|
|
14243
|
+
// Unblock a concurrent post-spawn read as soon as the issue writeback
|
|
14244
|
+
// outcome is known. Completion still has dependency, Slack and release
|
|
14245
|
+
// work to do; none determines whether the observed issue state is safe.
|
|
14246
|
+
settleIssueWritebackOnce();
|
|
14247
|
+
// If completion raced dispatch's post-spawn read, let that read consume
|
|
14248
|
+
// the receipt before claiming terminal success. An unproven provider
|
|
14249
|
+
// transition is a foreign live-state change: dispatch owns the
|
|
14250
|
+
// abandonment and must release agents with that reason, not issue-done.
|
|
14251
|
+
if (postSpawnIssueObservation && !await postSpawnIssueObservation.settled)
|
|
14252
|
+
return;
|
|
13355
14253
|
if (!humanReview)
|
|
13356
14254
|
await this.#markDependencyTerminalAndReconcile(issue);
|
|
13357
14255
|
}
|
|
14256
|
+
else {
|
|
14257
|
+
settleIssueWritebackOnce();
|
|
14258
|
+
}
|
|
13358
14259
|
if (!await this.#saveDispatchLifecycle(record, 'writeback-applied'))
|
|
13359
14260
|
return;
|
|
13360
14261
|
if (issue && this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
|
|
@@ -13419,6 +14320,10 @@ export class FactoryLoop {
|
|
|
13419
14320
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
13420
14321
|
}
|
|
13421
14322
|
finally {
|
|
14323
|
+
// Errors before or during provider confirmation leave the marker unset;
|
|
14324
|
+
// the waiting dispatch will re-read and preserve the foreign-change
|
|
14325
|
+
// abort. Always settle so a failed write cannot strand that dispatch.
|
|
14326
|
+
settleIssueWritebackOnce();
|
|
13422
14327
|
this.#completionInFlight.delete(completionKey);
|
|
13423
14328
|
const stateKey = issueStateKey(record.issue);
|
|
13424
14329
|
this.#probePrGhBackoffUntilMs.delete(stateKey);
|
|
@@ -13435,6 +14340,35 @@ export class FactoryLoop {
|
|
|
13435
14340
|
}
|
|
13436
14341
|
}
|
|
13437
14342
|
}
|
|
14343
|
+
#recordMissingGithubWritebackReceipt(method) {
|
|
14344
|
+
const implementation = this.#githubWriteback.constructor.name || 'anonymous GithubWriteback';
|
|
14345
|
+
this.#increment('githubWritebackReceiptMissing');
|
|
14346
|
+
this.#logger.warn?.('[factory] GitHub writeback returned no ownership receipt', {
|
|
14347
|
+
implementation,
|
|
14348
|
+
method,
|
|
14349
|
+
});
|
|
14350
|
+
}
|
|
14351
|
+
#settlePostSpawnDispatchWaits(key, accepted, source) {
|
|
14352
|
+
const observation = this.#postSpawnIssueObservations.get(key);
|
|
14353
|
+
const claimFence = this.#postSpawnDispatchClaimFences.get(key);
|
|
14354
|
+
if (!observation && !claimFence)
|
|
14355
|
+
return false;
|
|
14356
|
+
// Capture both before either settlement deletes its own map entry.
|
|
14357
|
+
if (!accepted && claimFence?.claimStarted) {
|
|
14358
|
+
// The rejection handler clears this provisional block only after a
|
|
14359
|
+
// conditional provider rollback succeeds. Until then, shutdown and held
|
|
14360
|
+
// abandonment must not release the lifecycle's agents.
|
|
14361
|
+
this.#uncompensatedDispatchClaims.add(key);
|
|
14362
|
+
this.#pendingDispatchClaims.add(key);
|
|
14363
|
+
}
|
|
14364
|
+
observation?.settle(accepted);
|
|
14365
|
+
claimFence?.settle(accepted);
|
|
14366
|
+
if (source === 'abandonment')
|
|
14367
|
+
this.#increment('postSpawnWaitsSettledByAbandonment');
|
|
14368
|
+
if (source === 'stop')
|
|
14369
|
+
this.#increment('postSpawnWaitsSettledByStop');
|
|
14370
|
+
return true;
|
|
14371
|
+
}
|
|
13438
14372
|
#emit(event, payload) {
|
|
13439
14373
|
for (const listener of this.#listeners.get(event) ?? []) {
|
|
13440
14374
|
try {
|
|
@@ -16105,7 +17039,7 @@ const defaultGithubWriteback = (config, mount) => {
|
|
|
16105
17039
|
if (!mount.githubWrite) {
|
|
16106
17040
|
throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
|
|
16107
17041
|
}
|
|
16108
|
-
return new AppGithubWriteback(mount.githubWrite);
|
|
17042
|
+
return new AppGithubWriteback(mount.githubWrite, mount.githubRead);
|
|
16109
17043
|
};
|
|
16110
17044
|
export function parseLinearIssue(path, content) {
|
|
16111
17045
|
const parsed = parseJsonContent(content);
|
|
@@ -18909,6 +19843,10 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
|
|
|
18909
19843
|
*/
|
|
18910
19844
|
const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
|
|
18911
19845
|
error instanceof DispatchLifecycleClaimRefusedError ||
|
|
19846
|
+
// An external stop/held-agent deadline rejected the post-spawn boundary
|
|
19847
|
+
// while a provider claim was already in flight. That lifecycle's cleanup is
|
|
19848
|
+
// owned by the rejecting path, so the late claim unwind is an expected skip.
|
|
19849
|
+
error instanceof PostSpawnDispatchWaitRejectedError ||
|
|
18912
19850
|
// #303: the never-placed deadline released this dispatch while its spawn was
|
|
18913
19851
|
// still in flight. Named, expected and self-healing — the issue goes back to
|
|
18914
19852
|
// the queue — and it recurs under exactly the slow-spawn conditions the
|
|
@@ -18933,7 +19871,8 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
|
|
|
18933
19871
|
* of a missed one is leaked agents and duplicate workers on the next retry.
|
|
18934
19872
|
*/
|
|
18935
19873
|
const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
|
|
18936
|
-
!(error instanceof DispatchLifecycleClaimRefusedError)
|
|
19874
|
+
!(error instanceof DispatchLifecycleClaimRefusedError) &&
|
|
19875
|
+
!(error instanceof PostSpawnDispatchWaitRejectedError);
|
|
18937
19876
|
/**
|
|
18938
19877
|
* The run-report reason recorded for a work unit the pass could not dispatch.
|
|
18939
19878
|
*
|
|
@@ -18951,6 +19890,8 @@ const perItemDispatchSkipReason = (error) => {
|
|
|
18951
19890
|
return 'live state changed during dispatch';
|
|
18952
19891
|
if (error instanceof LatePlacementReleasedError)
|
|
18953
19892
|
return 'dispatch released while its agent was still spawning';
|
|
19893
|
+
if (error instanceof PostSpawnDispatchWaitRejectedError)
|
|
19894
|
+
return 'dispatch terminated during post-spawn claim';
|
|
18954
19895
|
if (error instanceof DispatchLifecycleClaimRefusedError) {
|
|
18955
19896
|
return error.refusal === 'terminal'
|
|
18956
19897
|
? 'dispatch lifecycle already terminal'
|