@agent-relay/factory 0.1.62 → 0.1.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -15
- package/dist/cli/exit-codes.d.ts +81 -0
- package/dist/cli/exit-codes.d.ts.map +1 -0
- package/dist/cli/exit-codes.js +122 -0
- package/dist/cli/exit-codes.js.map +1 -0
- package/dist/cli/fleet.d.ts +26 -2
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +183 -40
- package/dist/cli/fleet.js.map +1 -1
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/config/schema.d.ts +156 -52
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +17 -5
- package/dist/config/schema.js.map +1 -1
- package/dist/dispatch/templates.d.ts +7 -0
- package/dist/dispatch/templates.d.ts.map +1 -1
- package/dist/dispatch/templates.js +64 -12
- package/dist/dispatch/templates.js.map +1 -1
- package/dist/dispatch/work-unit-identity.d.ts +19 -0
- package/dist/dispatch/work-unit-identity.d.ts.map +1 -0
- package/dist/dispatch/work-unit-identity.js +40 -0
- package/dist/dispatch/work-unit-identity.js.map +1 -0
- package/dist/environments/stack-descriptor.d.ts +3 -3
- package/dist/fleet/control-plane-circuit.d.ts +57 -0
- package/dist/fleet/control-plane-circuit.d.ts.map +1 -0
- package/dist/fleet/control-plane-circuit.js +231 -0
- package/dist/fleet/control-plane-circuit.js.map +1 -0
- package/dist/fleet/internal-fleet-client.d.ts +2 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +11 -7
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +3 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +97 -18
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/github/writeback-paths.d.ts +7 -0
- package/dist/github/writeback-paths.d.ts.map +1 -0
- package/dist/github/writeback-paths.js +28 -0
- package/dist/github/writeback-paths.js.map +1 -0
- package/dist/hosted/orchestrator.d.ts.map +1 -1
- package/dist/hosted/orchestrator.js +2 -0
- package/dist/hosted/orchestrator.js.map +1 -1
- package/dist/index.d.ts +10 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/dist/intake/index.d.ts +2 -1
- package/dist/intake/index.d.ts.map +1 -1
- package/dist/intake/index.js +2 -1
- package/dist/intake/index.js.map +1 -1
- package/dist/intake/notion-manifest.d.ts +48 -0
- package/dist/intake/notion-manifest.d.ts.map +1 -0
- package/dist/intake/notion-manifest.js +258 -0
- package/dist/intake/notion-manifest.js.map +1 -0
- package/dist/intake/notion-relay-claim.d.ts +5 -4
- package/dist/intake/notion-relay-claim.d.ts.map +1 -1
- package/dist/intake/notion-relay-claim.js +48 -7
- package/dist/intake/notion-relay-claim.js.map +1 -1
- package/dist/intake/notion.d.ts +6 -3
- package/dist/intake/notion.d.ts.map +1 -1
- package/dist/intake/notion.js +63 -13
- package/dist/intake/notion.js.map +1 -1
- package/dist/issue-key-match.d.ts +6 -0
- package/dist/issue-key-match.d.ts.map +1 -1
- package/dist/issue-key-match.js +6 -1
- package/dist/issue-key-match.js.map +1 -1
- package/dist/mount/local-mount-preflight.d.ts.map +1 -1
- package/dist/mount/local-mount-preflight.js +6 -2
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/mount-auth-error.d.ts +16 -3
- package/dist/mount/mount-auth-error.d.ts.map +1 -1
- package/dist/mount/mount-auth-error.js +34 -7
- package/dist/mount/mount-auth-error.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +36 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +136 -6
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/mount/relayfile-github-connection-write.d.ts +24 -2
- package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
- package/dist/mount/relayfile-github-connection-write.js +90 -1
- package/dist/mount/relayfile-github-connection-write.js.map +1 -1
- package/dist/node/factory-node.d.ts.map +1 -1
- package/dist/node/factory-node.js +18 -4
- package/dist/node/factory-node.js.map +1 -1
- package/dist/observability/cloud-reporter.d.ts +1 -0
- package/dist/observability/cloud-reporter.d.ts.map +1 -1
- package/dist/observability/cloud-reporter.js +182 -50
- package/dist/observability/cloud-reporter.js.map +1 -1
- package/dist/observability/events.d.ts +52 -52
- package/dist/orchestrator/batch-tracker.d.ts +25 -0
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js +48 -2
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +36 -2
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1208 -82
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +1 -1
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -1
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/ports/fleet.d.ts +6 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -1
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +39 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +75 -4
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/document-store.d.ts +33 -0
- package/dist/state/document-store.d.ts.map +1 -0
- package/dist/state/document-store.js +7 -0
- package/dist/state/document-store.js.map +1 -0
- package/dist/state/file-state-store.d.ts +32 -7
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +169 -250
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +13 -2
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +87 -4
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/state/watch-state-document.d.ts +5 -0
- package/dist/state/watch-state-document.d.ts.map +1 -0
- package/dist/state/watch-state-document.js +486 -0
- package/dist/state/watch-state-document.js.map +1 -0
- package/dist/testing/deadline.d.ts +11 -0
- package/dist/testing/deadline.d.ts.map +1 -0
- package/dist/testing/deadline.js +23 -0
- package/dist/testing/deadline.js.map +1 -0
- package/dist/testing/fakes.d.ts +2 -0
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js.map +1 -1
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +1 -0
- package/dist/testing/index.js.map +1 -1
- package/dist/trajectory.d.ts +25 -0
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/trajectory.js +51 -0
- package/dist/trajectory.js.map +1 -0
- package/dist/triage/heuristic.d.ts +5 -0
- package/dist/triage/heuristic.d.ts.map +1 -1
- package/dist/triage/heuristic.js +42 -3
- package/dist/triage/heuristic.js.map +1 -1
- package/dist/triage/index.d.ts +1 -1
- package/dist/triage/index.d.ts.map +1 -1
- package/dist/triage/index.js +1 -1
- package/dist/triage/index.js.map +1 -1
- package/dist/triage/schema.d.ts +73 -55
- package/dist/triage/schema.d.ts.map +1 -1
- package/dist/triage/schema.js +2 -1
- package/dist/triage/schema.js.map +1 -1
- package/dist/types.d.ts +19 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/github.d.ts +22 -7
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +86 -16
- package/dist/writeback/github.js.map +1 -1
- package/dist/writeback/index.d.ts +1 -1
- package/dist/writeback/index.d.ts.map +1 -1
- package/dist/writeback/index.js +1 -1
- package/dist/writeback/index.js.map +1 -1
- package/package.json +13 -8
|
@@ -5,6 +5,7 @@ import { FactoryConfigSchema } from '../config/schema.js';
|
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
8
|
+
import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftName, isFactoryGithubOperationDraftName, } from '../github/writeback-paths.js';
|
|
8
9
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
10
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
10
11
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
@@ -12,13 +13,14 @@ import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsT
|
|
|
12
13
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
13
14
|
import { isInFactoryScope } from '../safety/factory-scope.js';
|
|
14
15
|
import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
|
|
16
|
+
import { dispatchAgentIdentityKey } from '../dispatch/work-unit-identity.js';
|
|
15
17
|
import { deriveDescriptorsFromMount, prescriptiveInstructions, } from '@agent-relay/integration-prompts';
|
|
16
18
|
import { parseGithubHumanInputRequest, renderAgentTask, } from '../dispatch/templates.js';
|
|
17
19
|
import { resolveTestGuidance } from '../dispatch/test-guidance.js';
|
|
18
|
-
import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels } from '../triage/index.js';
|
|
20
|
+
import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels, swarmChannel, swarmMemberSlugs, swarmTaskFor } from '../triage/index.js';
|
|
19
21
|
import { agentNameForRole, sanitizeAgentSlug } from '../triage/agent-names.js';
|
|
20
22
|
import { isResourceSubscriptionsUnavailable } from '../subscriptions/index.js';
|
|
21
|
-
import { GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
|
|
23
|
+
import { AppGithubWriteback, FACTORY_GITHUB_STATUS_LABELS, GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
|
|
22
24
|
import { parseSlackThreadReply, slackThreadReplyGlob } from '../subscriptions/slack-filter.js';
|
|
23
25
|
import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared.js';
|
|
24
26
|
import { issueKey, } from './batch-tracker.js';
|
|
@@ -29,6 +31,8 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
|
29
31
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
30
32
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
31
33
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
34
|
+
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
35
|
+
import { FleetControlPlaneCircuit, FleetControlPlaneCircuitOpenError, guardFleetControlPlane, } from '../fleet/control-plane-circuit.js';
|
|
32
36
|
class ClarificationWakeLeaseLostError extends Error {
|
|
33
37
|
}
|
|
34
38
|
class ClarificationQuestionDeliveryLeaseLostError extends Error {
|
|
@@ -143,8 +147,32 @@ const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
|
143
147
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
144
148
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
145
149
|
const SLACK_CONVERSATION_TURN_LEASE_MS = 60_000;
|
|
150
|
+
// Both receipt leases guard an in-flight Slack writeback, and no fixed lease can
|
|
151
|
+
// cover one: MountSlackWriteback budgets 90s for the confirm alone, on top of an
|
|
152
|
+
// unbounded writeFile. Sizing them past that worst case would only trade a stolen
|
|
153
|
+
// claim for a stranded one — a lease long enough to survive the slowest write is
|
|
154
|
+
// equally long enough to hold the receipt hostage to a dead holder. So these
|
|
155
|
+
// bound the *idle* claim and #withRenewedProviderLease extends them for exactly
|
|
156
|
+
// as long as the write they cover is still running.
|
|
157
|
+
const SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS = 60_000;
|
|
158
|
+
const SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS = 60_000;
|
|
159
|
+
// Renewal without a ceiling is the same defect from the other side: a heartbeat
|
|
160
|
+
// that extends the claim for as long as the write runs also extends it forever
|
|
161
|
+
// when the write never returns, and nothing else can reclaim the receipt short
|
|
162
|
+
// of a restart. So renewal is bounded past the slowest write this daemon budgets
|
|
163
|
+
// for — MountSlackWriteback's 90s confirm on top of its writeFile — and beyond
|
|
164
|
+
// that the write is not slow, it is wedged: the heartbeat stops, the idle lease
|
|
165
|
+
// runs out, and the retry that owns the queued replies can take them back.
|
|
166
|
+
const SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS = 5 * 60_000;
|
|
146
167
|
const SLACK_CONVERSATION_TURN_RETRY_MS = 1_000;
|
|
147
168
|
const SLACK_REPLY_ROUTE_RETRY_MS = 1_000;
|
|
169
|
+
// One pass drains the whole chain (#slackReplyRoutes holds only the newest
|
|
170
|
+
// route per key and every route awaits its predecessor). The extra passes only
|
|
171
|
+
// exist so the drain can prove quiescence rather than assume it.
|
|
172
|
+
const SLACK_REPLY_ROUTE_DRAIN_PASSES = 8;
|
|
173
|
+
const SLACK_TERMINAL_THREAD_GRACE_MS = 24 * 60 * 60_000;
|
|
174
|
+
const SLACK_TERMINAL_RECEIPT_RETRY_MS = 1_000;
|
|
175
|
+
const SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS = 5 * 60_000;
|
|
148
176
|
const MERGE_GATE_MAX_ATTEMPTS = 12;
|
|
149
177
|
const MERGE_GATE_POLL_DELAY_MS = 10_000;
|
|
150
178
|
const MAX_LABEL_IMPLEMENTERS = 4;
|
|
@@ -212,6 +240,25 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
|
|
|
212
240
|
this.leaseUntilMs = leaseUntilMs;
|
|
213
241
|
}
|
|
214
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* The durable dispatch-lifecycle claim was refused for one work unit: its
|
|
245
|
+
* record is already terminal, or another publisher currently holds the lease.
|
|
246
|
+
* Both are facts about that single unit — the rest of the pass is unaffected —
|
|
247
|
+
* so the readiness loop skips it and keeps going (#292).
|
|
248
|
+
*
|
|
249
|
+
* Typed rather than left as a plain `Error` so the loop can classify it by
|
|
250
|
+
* construction instead of by matching on `Refusing to dispatch ...` text.
|
|
251
|
+
*/
|
|
252
|
+
class DispatchLifecycleClaimRefusedError extends Error {
|
|
253
|
+
issueKey;
|
|
254
|
+
refusal;
|
|
255
|
+
constructor(issueKey, refusal, message) {
|
|
256
|
+
super(message);
|
|
257
|
+
this.issueKey = issueKey;
|
|
258
|
+
this.refusal = refusal;
|
|
259
|
+
this.name = 'DispatchLifecycleClaimRefusedError';
|
|
260
|
+
}
|
|
261
|
+
}
|
|
215
262
|
const realClock = {
|
|
216
263
|
now: () => Date.now(),
|
|
217
264
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -224,6 +271,7 @@ export class FactoryLoop {
|
|
|
224
271
|
#mount;
|
|
225
272
|
#states;
|
|
226
273
|
#fleet;
|
|
274
|
+
#fleetControlPlane;
|
|
227
275
|
#ticketDispatchDelivery;
|
|
228
276
|
#triage;
|
|
229
277
|
#linear;
|
|
@@ -262,6 +310,19 @@ export class FactoryLoop {
|
|
|
262
310
|
#dispatchInFlight = new Map();
|
|
263
311
|
#slackWatchers = new Map();
|
|
264
312
|
#slackWatcherStarts = new Map();
|
|
313
|
+
#slackTerminalWatchExpiryTimers = new Map();
|
|
314
|
+
#slackTerminalReceiptRetryTimers = new Map();
|
|
315
|
+
#terminalSlackWatchIssues = new Set();
|
|
316
|
+
/**
|
|
317
|
+
* The one in-memory record of "this work unit has an in-flight Slack side
|
|
318
|
+
* effect". Both ordinary reply routes and the writebacks the terminal fence
|
|
319
|
+
* issues on their behalf register here, because this map is what the terminal
|
|
320
|
+
* drain waits on: anything that touches Slack for a work unit without
|
|
321
|
+
* registering is invisible to the drain, and the watcher teardown that follows
|
|
322
|
+
* a successful drain then pulls that effect's retry timer out from under it.
|
|
323
|
+
*/
|
|
324
|
+
#slackReplyRoutes = new Map();
|
|
325
|
+
#slackReplyRouteDrains = new Set();
|
|
265
326
|
#slackConversationTurns;
|
|
266
327
|
#slackConversationOwner = `${process.pid}:${randomUUID()}`;
|
|
267
328
|
#githubIssueCommentWatchers = new Map();
|
|
@@ -435,6 +496,11 @@ export class FactoryLoop {
|
|
|
435
496
|
#discoverySweepRenewTimer;
|
|
436
497
|
#discoverySweepRenewalInFlight;
|
|
437
498
|
#discoverySweepLeaseLost = false;
|
|
499
|
+
// Registry/heartbeat paths the in-flight runLoop iteration would use. A
|
|
500
|
+
// per-item dispatch failure now skips instead of aborting the pass (#292),
|
|
501
|
+
// so the loop's catch no longer runs the failure-handoff reaper for it; the
|
|
502
|
+
// pass reaps inline and must write to the same paths runLoop would.
|
|
503
|
+
#loopReapPaths;
|
|
438
504
|
#discoveryOverloadError;
|
|
439
505
|
#resolvedIssueSource;
|
|
440
506
|
#integrationInstructions;
|
|
@@ -452,7 +518,6 @@ export class FactoryLoop {
|
|
|
452
518
|
// synced records (state.name but no state.id) without the states catalog.
|
|
453
519
|
this.#states = ports.stateResolution ?? stateResolutionFromIds(config.stateIds, config.linear.states);
|
|
454
520
|
installFactoryDraftPredicate(this.#mount, config);
|
|
455
|
-
this.#fleet = ports.fleet;
|
|
456
521
|
this.#ticketDispatchDelivery = ports.ticketDispatchDelivery ?? createTicketDispatchDelivery({
|
|
457
522
|
mountRoot: config.localMountRoot,
|
|
458
523
|
});
|
|
@@ -461,7 +526,7 @@ export class FactoryLoop {
|
|
|
461
526
|
safety: config.safety,
|
|
462
527
|
});
|
|
463
528
|
this.#githubWritebackProvided = Boolean(ports.githubWriteback);
|
|
464
|
-
this.#githubWriteback = ports.githubWriteback ??
|
|
529
|
+
this.#githubWriteback = ports.githubWriteback ?? defaultGithubWriteback(config, ports.mount);
|
|
465
530
|
this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
|
|
466
531
|
this.#github = ports.github ?? MountGithubRead(ports.mount);
|
|
467
532
|
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
@@ -483,6 +548,13 @@ export class FactoryLoop {
|
|
|
483
548
|
this.#probePrResolver = ports.probePrResolver ?? ((issue) => this.#resolveIssuePr(issue));
|
|
484
549
|
this.#logger = normalizeLogger(ports.logger ?? console);
|
|
485
550
|
this.#clock = ports.clock ?? realClock;
|
|
551
|
+
this.#fleetControlPlane = new FleetControlPlaneCircuit({
|
|
552
|
+
timeoutMs: config.fleetHealth.rosterTimeoutMs,
|
|
553
|
+
failureThreshold: config.fleetHealth.failureThreshold,
|
|
554
|
+
resetTimeoutMs: config.fleetHealth.resetTimeoutMs,
|
|
555
|
+
now: () => this.#clock.now(),
|
|
556
|
+
});
|
|
557
|
+
this.#fleet = guardFleetControlPlane(ports.fleet, this.#fleetControlPlane);
|
|
486
558
|
this.#processIdentityReader = ports.processIdentityReader ?? readProcessIdentity;
|
|
487
559
|
this.#processFinder = ports.processFinder ?? ((agentName, opts) => findAgentProcessByName(agentName, {
|
|
488
560
|
readProcessIdentity: this.#processIdentityReader,
|
|
@@ -590,11 +662,32 @@ export class FactoryLoop {
|
|
|
590
662
|
this.#batchView = batch;
|
|
591
663
|
return batch;
|
|
592
664
|
}
|
|
665
|
+
/**
|
|
666
|
+
* Resolves once this issue's durable dispatch row reaches a terminal phase,
|
|
667
|
+
* and reports which one. A caller that turns the run into an exit code needs
|
|
668
|
+
* the phase: a dispatch that hit capacity returns an empty hold result and
|
|
669
|
+
* schedules a durable retry, so the pre-wait result says nothing about how
|
|
670
|
+
* the run actually ended.
|
|
671
|
+
*
|
|
672
|
+
* `undefined` means no terminal phase was observed — there was no lifecycle
|
|
673
|
+
* row to wait on, or the wait ended because Factory is stopping.
|
|
674
|
+
*/
|
|
593
675
|
async waitForDispatchTerminal(issue) {
|
|
594
676
|
const key = issueKey(issue);
|
|
595
677
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
596
|
-
|
|
597
|
-
|
|
678
|
+
// No durable row means this dispatch never claimed a lifecycle: a
|
|
679
|
+
// dependency park, a triage escalation, and a label refusal all return
|
|
680
|
+
// before the claim. Nothing can ever become terminal, so polling would
|
|
681
|
+
// never stop and the caller would never produce an exit code at all.
|
|
682
|
+
if (!lifecycle)
|
|
683
|
+
return undefined;
|
|
684
|
+
if (isTerminalDispatchLifecycle(lifecycle))
|
|
685
|
+
return lifecycle.phase;
|
|
686
|
+
// Capture the phase at the moment this waiter observes it. The waiters are
|
|
687
|
+
// shared across callers of one row and carry no payload, so a re-read after
|
|
688
|
+
// the fact can race lifecycle cleanup or a reopened dispatch for the same
|
|
689
|
+
// issue and report a different run — or none.
|
|
690
|
+
let observedPhase;
|
|
598
691
|
await new Promise((resolve) => {
|
|
599
692
|
let settled = false;
|
|
600
693
|
let timer;
|
|
@@ -603,10 +696,11 @@ export class FactoryLoop {
|
|
|
603
696
|
waiters = new Set();
|
|
604
697
|
this.#dispatchTerminalWaiters.set(key, waiters);
|
|
605
698
|
}
|
|
606
|
-
const finish = () => {
|
|
699
|
+
const finish = (phase) => {
|
|
607
700
|
if (settled)
|
|
608
701
|
return;
|
|
609
702
|
settled = true;
|
|
703
|
+
observedPhase = phase;
|
|
610
704
|
if (timer)
|
|
611
705
|
clearTimeout(timer);
|
|
612
706
|
const current = this.#dispatchTerminalWaiters.get(key);
|
|
@@ -629,7 +723,7 @@ export class FactoryLoop {
|
|
|
629
723
|
try {
|
|
630
724
|
const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
631
725
|
if (latest && isTerminalDispatchLifecycle(latest)) {
|
|
632
|
-
this.#resolveDispatchTerminalWaiters(issue);
|
|
726
|
+
this.#resolveDispatchTerminalWaiters(issue, latest.phase);
|
|
633
727
|
return;
|
|
634
728
|
}
|
|
635
729
|
if (latest?.phase === 'waiting-for-human' && this.#startMode === 'dispatch-owner') {
|
|
@@ -659,6 +753,10 @@ export class FactoryLoop {
|
|
|
659
753
|
};
|
|
660
754
|
void poll();
|
|
661
755
|
});
|
|
756
|
+
// `observedPhase` is whatever resolved THIS waiter. It stays undefined only
|
|
757
|
+
// when the wait ended without a terminal resolution at all — Factory is
|
|
758
|
+
// stopping — which is exactly what `undefined` reports.
|
|
759
|
+
return observedPhase;
|
|
662
760
|
}
|
|
663
761
|
async start(opts = {}) {
|
|
664
762
|
if (this.#started) {
|
|
@@ -871,6 +969,14 @@ export class FactoryLoop {
|
|
|
871
969
|
await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
|
|
872
970
|
await Promise.all([...this.#slackWatchers.values()].map((watcher) => watcher.stop()));
|
|
873
971
|
this.#slackWatchers.clear();
|
|
972
|
+
for (const timer of this.#slackTerminalWatchExpiryTimers.values())
|
|
973
|
+
clearTimeout(timer);
|
|
974
|
+
this.#slackTerminalWatchExpiryTimers.clear();
|
|
975
|
+
for (const timer of this.#slackTerminalReceiptRetryTimers.values())
|
|
976
|
+
clearTimeout(timer);
|
|
977
|
+
this.#slackTerminalReceiptRetryTimers.clear();
|
|
978
|
+
this.#terminalSlackWatchIssues.clear();
|
|
979
|
+
this.#slackReplyRouteDrains.clear();
|
|
874
980
|
await Promise.all([...this.#githubIssueCommentWatchers.values()].map((watcher) => watcher.stop()));
|
|
875
981
|
this.#githubIssueCommentWatchers.clear();
|
|
876
982
|
this.#githubIssueCommentWatchStates.clear();
|
|
@@ -1831,8 +1937,30 @@ export class FactoryLoop {
|
|
|
1831
1937
|
}
|
|
1832
1938
|
}
|
|
1833
1939
|
}
|
|
1940
|
+
async #assertFleetControlPlaneAvailable() {
|
|
1941
|
+
try {
|
|
1942
|
+
await this.#fleet.roster();
|
|
1943
|
+
this.#increment('fleetControlPlaneProbeSuccesses');
|
|
1944
|
+
}
|
|
1945
|
+
catch (error) {
|
|
1946
|
+
const health = this.#fleetControlPlane.status();
|
|
1947
|
+
this.#increment('fleetControlPlaneProbeFailures');
|
|
1948
|
+
if (health.state === 'open')
|
|
1949
|
+
this.#increment('fleetControlPlaneCircuitOpen');
|
|
1950
|
+
this.#logger.error?.('[factory] fleet control plane unavailable; dispatch paused', {
|
|
1951
|
+
state: health.state,
|
|
1952
|
+
consecutiveFailures: health.consecutiveFailures,
|
|
1953
|
+
retryAtMs: health.retryAtMs,
|
|
1954
|
+
error: health.lastError ?? 'unknown control-plane failure',
|
|
1955
|
+
});
|
|
1956
|
+
throw contextualError('Factory dispatch paused because the fleet control plane is unavailable', error);
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1834
1959
|
async #runOnceWithDiscoveryFence(opts) {
|
|
1835
1960
|
const sweepStartedAtMs = this.#clock.now();
|
|
1961
|
+
if (!(opts.dryRun ?? this.#config.dryRun)) {
|
|
1962
|
+
await this.#assertFleetControlPlaneAvailable();
|
|
1963
|
+
}
|
|
1836
1964
|
let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1837
1965
|
if (!claim.acquired && claim.reason === 'backoff') {
|
|
1838
1966
|
const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
|
|
@@ -1989,6 +2117,10 @@ export class FactoryLoop {
|
|
|
1989
2117
|
reason: entry.reason,
|
|
1990
2118
|
});
|
|
1991
2119
|
};
|
|
2120
|
+
// Backstop for the skip-by-default catch below: see #292. Reset only by
|
|
2121
|
+
// a completed dispatch, so the name says "since a dispatch" rather than
|
|
2122
|
+
// "consecutive" — a benign classified skip in between does not clear it.
|
|
2123
|
+
let unclassifiedFailuresSinceDispatch = 0;
|
|
1992
2124
|
let lastReadyReadProgressAtMs = this.#clock.now();
|
|
1993
2125
|
let readyIssueReads = 0;
|
|
1994
2126
|
const issueEntries = [];
|
|
@@ -2086,19 +2218,10 @@ export class FactoryLoop {
|
|
|
2086
2218
|
}
|
|
2087
2219
|
const decision = await this.triageIssue(issue);
|
|
2088
2220
|
triaged.push(decision);
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
catch (error) {
|
|
2094
|
-
if (!(error instanceof LiveDispatchStateChangedError))
|
|
2095
|
-
throw error;
|
|
2096
|
-
recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
|
|
2097
|
-
this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
|
|
2098
|
-
issue: decision.issue.key,
|
|
2099
|
-
});
|
|
2100
|
-
continue;
|
|
2101
|
-
}
|
|
2221
|
+
const result = await this.dispatch(decision, { dryRun });
|
|
2222
|
+
// A completed dispatch — even one that parks or escalates the issue —
|
|
2223
|
+
// proves the pipeline still works, so the fuse below starts over.
|
|
2224
|
+
unclassifiedFailuresSinceDispatch = 0;
|
|
2102
2225
|
if (result.agents.length === 0 && !dryRun) {
|
|
2103
2226
|
const reason = result.hold?.kind === 'dependency-cycle'
|
|
2104
2227
|
? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
|
|
@@ -2111,6 +2234,52 @@ export class FactoryLoop {
|
|
|
2111
2234
|
dispatched.push(result);
|
|
2112
2235
|
}
|
|
2113
2236
|
}
|
|
2237
|
+
catch (error) {
|
|
2238
|
+
// #292: issues in a pass are independent work units, so a failure
|
|
2239
|
+
// that is about ONE unit costs that unit and nothing else. Only the
|
|
2240
|
+
// conditions named in `#isPassFatalFailure` — the ones where
|
|
2241
|
+
// continuing the pass is meaningless — abort the whole sweep.
|
|
2242
|
+
if (this.#isPassFatalFailure(error, dryRun))
|
|
2243
|
+
throw error;
|
|
2244
|
+
if (!isClassifiedPerItemDispatchFailure(error)) {
|
|
2245
|
+
unclassifiedFailuresSinceDispatch += 1;
|
|
2246
|
+
// A pass-wide fault can arrive disguised as a run of per-item
|
|
2247
|
+
// faults. Skipping every unit would then hand back a green report
|
|
2248
|
+
// that dispatched nothing, which is the same silent wedge #292 is
|
|
2249
|
+
// about, wearing the opposite costume. Fail the pass loudly so
|
|
2250
|
+
// `readinessReconcile.lastError` carries the cause.
|
|
2251
|
+
if (unclassifiedFailuresSinceDispatch >= UNCLASSIFIED_DISPATCH_FAILURE_LIMIT) {
|
|
2252
|
+
throw contextualError(`Aborting readiness pass after ${unclassifiedFailuresSinceDispatch} unclassified dispatch failures without a successful dispatch`, error);
|
|
2253
|
+
}
|
|
2254
|
+
this.#increment('dispatchItemFailuresSkipped');
|
|
2255
|
+
// The raw message is operator-facing only; the run report carries
|
|
2256
|
+
// the sanitized classification from `perItemDispatchSkipReason`.
|
|
2257
|
+
this.#logger.warn?.('[factory] skipped a work unit whose dispatch failed; continuing the pass', {
|
|
2258
|
+
issue: issueRef(issue).key,
|
|
2259
|
+
unclassifiedFailuresSinceDispatch,
|
|
2260
|
+
error: describeError(error).errorMessage,
|
|
2261
|
+
});
|
|
2262
|
+
this.#error(error, issueRef(issue));
|
|
2263
|
+
// The failure may have left half-spawned agents behind. runLoop's
|
|
2264
|
+
// catch used to reap them because this error aborted the pass;
|
|
2265
|
+
// now that the pass survives, the reap has to happen here or the
|
|
2266
|
+
// agents leak until the next failed iteration.
|
|
2267
|
+
await this.#reapDispatchFailureHandoffsNow();
|
|
2268
|
+
}
|
|
2269
|
+
else {
|
|
2270
|
+
// Not an error — the unit simply cannot be dispatched right now —
|
|
2271
|
+
// so this stays out of `counters.errors` and gets its own counter
|
|
2272
|
+
// instead, or a terminal-lifecycle backlog would be invisible to
|
|
2273
|
+
// anyone watching only `dispatchItemFailuresSkipped`.
|
|
2274
|
+
this.#increment('dispatchItemsSkippedUndispatchable');
|
|
2275
|
+
this.#logger.info?.('[factory] skipped a work unit that cannot be dispatched right now', {
|
|
2276
|
+
issue: issueRef(issue).key,
|
|
2277
|
+
error: describeError(error).errorMessage,
|
|
2278
|
+
});
|
|
2279
|
+
}
|
|
2280
|
+
recordSkip({ issue: issueRef(issue), reason: perItemDispatchSkipReason(error) });
|
|
2281
|
+
continue;
|
|
2282
|
+
}
|
|
2114
2283
|
finally {
|
|
2115
2284
|
if (recoveredIdentity)
|
|
2116
2285
|
this.#reconciledGithubInProgress.delete(recoveredIdentity);
|
|
@@ -2144,6 +2313,79 @@ export class FactoryLoop {
|
|
|
2144
2313
|
}
|
|
2145
2314
|
}
|
|
2146
2315
|
}
|
|
2316
|
+
/**
|
|
2317
|
+
* Whether a failure raised while processing ONE work unit must abort the
|
|
2318
|
+
* whole readiness pass instead of skipping that unit.
|
|
2319
|
+
*
|
|
2320
|
+
* The default is the opposite, and that inversion is the fix for #292.
|
|
2321
|
+
* Issues in a pass are independent work units: a failure that is *about one
|
|
2322
|
+
* unit* — its dispatch-lifecycle record, its live state, a provider fault on
|
|
2323
|
+
* its own writeback — costs that unit and nothing else. Before this, every
|
|
2324
|
+
* error except `LiveDispatchStateChangedError` escaped the `for` loop, so a
|
|
2325
|
+
* single issue whose lifecycle record had gone terminal stopped all dispatch
|
|
2326
|
+
* indefinitely, every pass.
|
|
2327
|
+
*
|
|
2328
|
+
* A condition belongs here only when continuing the pass is meaningless or
|
|
2329
|
+
* actively harmful — when the failure is about the *pass*, not the item:
|
|
2330
|
+
*
|
|
2331
|
+
* - The discovery sweep lease is gone. Another process now owns this
|
|
2332
|
+
* workspace's sweep, so every remaining read throws the same way and each
|
|
2333
|
+
* one would be recorded as an ordinary per-issue skip. The run report
|
|
2334
|
+
* would then claim a clean pass over work this process no longer has the
|
|
2335
|
+
* right to touch.
|
|
2336
|
+
* - Relayfile signalled overload for this sweep. The backend is shedding
|
|
2337
|
+
* load; grinding through the remaining units makes it worse, and
|
|
2338
|
+
* `#runOnceWithDiscoveryFence` is going to rethrow this at the fence
|
|
2339
|
+
* anyway.
|
|
2340
|
+
* - The factory is stopping. Teardown is in progress and dispatching more
|
|
2341
|
+
* agents now leaks them past the shutdown deadline.
|
|
2342
|
+
* - The fleet control-plane circuit is no longer closed, **on a live pass**.
|
|
2343
|
+
* Dispatch is globally paused — the same condition
|
|
2344
|
+
* `#assertFleetControlPlaneAvailable` refuses to *start* a live pass on,
|
|
2345
|
+
* so it must also stop one already in flight. A dry run is exempt: it
|
|
2346
|
+
* never calls that admission gate and never spawns, so a paused control
|
|
2347
|
+
* plane is irrelevant to it rather than fatal to it. Without the
|
|
2348
|
+
* exemption, one live pass that trips the circuit would poison every
|
|
2349
|
+
* later dry run — including the boot gate's own `run-once --dry-run`
|
|
2350
|
+
* probe, turning a recoverable circuit-open condition into a failed boot.
|
|
2351
|
+
* That is a nastier version of the wedge this whole change removes.
|
|
2352
|
+
*
|
|
2353
|
+
* Deliberately NOT here: JavaScript builtin error types. Classifying
|
|
2354
|
+
* "programmer faults" such as `TypeError` as fatal is the obvious next rule
|
|
2355
|
+
* and it is a trap — Node reports a failed `fetch` as `TypeError: fetch
|
|
2356
|
+
* failed`, which is precisely the transient per-item roster lookup that
|
|
2357
|
+
* wedged the second instance (#291). A rule keyed on builtin types would
|
|
2358
|
+
* have preserved that outage verbatim.
|
|
2359
|
+
*
|
|
2360
|
+
* Everything else — a refused lifecycle claim, a terminal lifecycle record,
|
|
2361
|
+
* a transient network fault on one issue — is per-item: record a skip and
|
|
2362
|
+
* keep going. The unclassified-failure fuse in `#performRunOnce` is the
|
|
2363
|
+
* backstop for a pass-wide fault that does not announce itself as one.
|
|
2364
|
+
*/
|
|
2365
|
+
#isPassFatalFailure(error, dryRun) {
|
|
2366
|
+
// Sweep-scoped: these are about this process's right or ability to run the
|
|
2367
|
+
// pass at all, so they hold for a dry run exactly as for a live one.
|
|
2368
|
+
if (this.#discoverySweepLeaseLost || this.#discoveryOverloadError !== undefined || this.#stopping) {
|
|
2369
|
+
return true;
|
|
2370
|
+
}
|
|
2371
|
+
// Fleet-scoped, and therefore live-only. See the doc comment above.
|
|
2372
|
+
return !dryRun && this.#isFleetControlPlaneHalted(error);
|
|
2373
|
+
}
|
|
2374
|
+
/**
|
|
2375
|
+
* Whether dispatch is globally paused by the fleet control-plane circuit.
|
|
2376
|
+
*
|
|
2377
|
+
* Two reads, because the circuit announces itself two different ways. The
|
|
2378
|
+
* state read covers `guardedMutation`, which records a mutation's own
|
|
2379
|
+
* transport failure and rethrows the *original* error rather than the
|
|
2380
|
+
* circuit-open type — converting it there would be wrong, since the mutation
|
|
2381
|
+
* may already have reached the broker and callers key spawn-failure handling
|
|
2382
|
+
* off that original error. The type check covers a rejection raised without
|
|
2383
|
+
* any state transition, such as an already-open circuit refusing admission.
|
|
2384
|
+
*/
|
|
2385
|
+
#isFleetControlPlaneHalted(error) {
|
|
2386
|
+
return this.#fleetControlPlane.status().state !== 'closed' ||
|
|
2387
|
+
wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError);
|
|
2388
|
+
}
|
|
2147
2389
|
#startDiscoverySweepRenewal(epoch) {
|
|
2148
2390
|
this.#discoverySweepRenewTimer = setInterval(() => {
|
|
2149
2391
|
if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
|
|
@@ -2699,7 +2941,7 @@ export class FactoryLoop {
|
|
|
2699
2941
|
await this.#state.clearBabysitterSession(this.#workspaceId, issueKey(lifecycle.issue));
|
|
2700
2942
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2701
2943
|
this.#abandonedDispatchReasons.delete(key);
|
|
2702
|
-
this.#resolveDispatchTerminalWaiters(lifecycle.issue);
|
|
2944
|
+
this.#resolveDispatchTerminalWaiters(lifecycle.issue, 'abandoned');
|
|
2703
2945
|
await this.#writeInFlightRegistry().catch((error) => {
|
|
2704
2946
|
this.#logger.warn?.('[factory] failed to rewrite registry after orphaned claim release', {
|
|
2705
2947
|
issue: issue.key,
|
|
@@ -3027,6 +3269,7 @@ export class FactoryLoop {
|
|
|
3027
3269
|
const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
|
|
3028
3270
|
const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
|
|
3029
3271
|
const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
|
|
3272
|
+
this.#loopReapPaths = { heartbeatPath, registryPath };
|
|
3030
3273
|
const reports = [];
|
|
3031
3274
|
let consecutiveFailures = 0;
|
|
3032
3275
|
let completed = false;
|
|
@@ -3054,6 +3297,16 @@ export class FactoryLoop {
|
|
|
3054
3297
|
reports.push(failedIterationReport(error, opts.dryRun ?? this.#config.dryRun));
|
|
3055
3298
|
await this.#reapDispatchFailureHandoffsNow(heartbeatPath, registryPath);
|
|
3056
3299
|
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'running', iteration + 1, maxIterations);
|
|
3300
|
+
const fleetControlPlane = this.#fleetControlPlane.status();
|
|
3301
|
+
if (fleetControlPlane.state !== 'closed') {
|
|
3302
|
+
this.#increment('loopCircuitBreaks');
|
|
3303
|
+
this.#logger.error?.('[factory] stopping loop because dispatch is paused by the fleet control-plane circuit', {
|
|
3304
|
+
state: fleetControlPlane.state,
|
|
3305
|
+
consecutiveFailures: fleetControlPlane.consecutiveFailures,
|
|
3306
|
+
retryAtMs: fleetControlPlane.retryAtMs,
|
|
3307
|
+
});
|
|
3308
|
+
throw new FleetControlPlaneCircuitOpenError(fleetControlPlane.retryAtMs ?? this.#clock.now(), fleetControlPlane.state);
|
|
3309
|
+
}
|
|
3057
3310
|
if (consecutiveFailures >= maxConsecutiveFailures) {
|
|
3058
3311
|
this.#increment('loopCircuitBreaks');
|
|
3059
3312
|
this.#logger.error?.('[factory] stopping loop after consecutive iteration failures', {
|
|
@@ -3072,6 +3325,7 @@ export class FactoryLoop {
|
|
|
3072
3325
|
return reports;
|
|
3073
3326
|
}
|
|
3074
3327
|
finally {
|
|
3328
|
+
this.#loopReapPaths = undefined;
|
|
3075
3329
|
if (!completed) {
|
|
3076
3330
|
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'stopping', reports.length, maxIterations);
|
|
3077
3331
|
}
|
|
@@ -3209,6 +3463,13 @@ export class FactoryLoop {
|
|
|
3209
3463
|
};
|
|
3210
3464
|
}
|
|
3211
3465
|
this.#clearDependencyPark(batch, dispatchDecision.issue);
|
|
3466
|
+
// Event-driven and direct dispatches do not necessarily pass through issue
|
|
3467
|
+
// discovery. Admit them before creating previews, claiming a lifecycle, or
|
|
3468
|
+
// consuming a dispatch attempt. The mutation proxy probes again at the
|
|
3469
|
+
// actual spawn/resume boundary so a later control-plane fault still fails
|
|
3470
|
+
// closed.
|
|
3471
|
+
if (!dryRun)
|
|
3472
|
+
await this.#assertFleetControlPlaneAvailable();
|
|
3212
3473
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
3213
3474
|
// Local dispatches need the same deterministic branch identity as remote
|
|
3214
3475
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -3512,6 +3773,7 @@ export class FactoryLoop {
|
|
|
3512
3773
|
capacityBlocked: parked.capacityBlocked,
|
|
3513
3774
|
})) ?? [],
|
|
3514
3775
|
counters: { ...this.#counters },
|
|
3776
|
+
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
3515
3777
|
slackDegraded: this.#slackDegraded,
|
|
3516
3778
|
slackDegradedReason: this.#slackDegradedReason,
|
|
3517
3779
|
eventListener: this.#eventListenerStatus(),
|
|
@@ -4066,10 +4328,11 @@ export class FactoryLoop {
|
|
|
4066
4328
|
seed.decision = decisionWithLifecycleBranches(seed.decision, seed.runId);
|
|
4067
4329
|
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, seed, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
4068
4330
|
if (!claim.acquired || !claim.lease) {
|
|
4069
|
-
const
|
|
4331
|
+
const terminal = isTerminalDispatchLifecycle(claim.lifecycle);
|
|
4332
|
+
const reason = terminal
|
|
4070
4333
|
? 'dispatch lifecycle is already terminal'
|
|
4071
4334
|
: `dispatch lifecycle is owned by ${claim.lifecycle.lease?.owner ?? 'another publisher'}`;
|
|
4072
|
-
throw new
|
|
4335
|
+
throw new DispatchLifecycleClaimRefusedError(decision.issue.key, terminal ? 'terminal' : 'owned-elsewhere', `Refusing to dispatch ${decision.issue.key}: ${reason}`);
|
|
4073
4336
|
}
|
|
4074
4337
|
this.#dispatchLifecycleEpochs.set(claim.key ?? key, claim.lease.epoch);
|
|
4075
4338
|
this.#hydrateCostLedger(claim.lifecycle);
|
|
@@ -4415,12 +4678,25 @@ export class FactoryLoop {
|
|
|
4415
4678
|
}
|
|
4416
4679
|
for (const agent of lifecycle.agents) {
|
|
4417
4680
|
const previousAgent = previous?.agents.find((candidate) => candidate.name === agent.name);
|
|
4418
|
-
|
|
4419
|
-
|
|
4681
|
+
// The in-flight record is authoritative for release state: the stamp
|
|
4682
|
+
// now lives on the tracked agent and round-trips through the lifecycle,
|
|
4683
|
+
// so a still-released agent brings its own stamp here. Inheriting the
|
|
4684
|
+
// previous row's stamp instead would refile a respawned worker as
|
|
4685
|
+
// released — a respawn reuses both the agent name and the deterministic
|
|
4686
|
+
// invocation id, so neither can tell the generations apart — and every
|
|
4687
|
+
// consumer that filters on `releasedAtMs` would then treat a live
|
|
4688
|
+
// worker as gone.
|
|
4420
4689
|
if (previousAgent?.costUsage)
|
|
4421
4690
|
agent.costUsage = structuredClone(previousAgent.costUsage);
|
|
4422
|
-
if (releasedAgentNames.has(agent.name))
|
|
4423
|
-
agent.releasedAtMs
|
|
4691
|
+
if (releasedAgentNames.has(agent.name)) {
|
|
4692
|
+
const releasedAtMs = agent.releasedAtMs ?? this.#clock.now();
|
|
4693
|
+
agent.releasedAtMs = releasedAtMs;
|
|
4694
|
+
// Keep the record the single source of the stamp, so the next save
|
|
4695
|
+
// carries it without having to read it back off the durable row.
|
|
4696
|
+
const tracked = record.agents.get(agent.name);
|
|
4697
|
+
if (tracked)
|
|
4698
|
+
tracked.releasedAtMs ??= releasedAtMs;
|
|
4699
|
+
}
|
|
4424
4700
|
}
|
|
4425
4701
|
const saved = await this.#state.saveDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), lifecycle);
|
|
4426
4702
|
if (!saved) {
|
|
@@ -4448,10 +4724,14 @@ export class FactoryLoop {
|
|
|
4448
4724
|
return true;
|
|
4449
4725
|
});
|
|
4450
4726
|
}
|
|
4451
|
-
|
|
4727
|
+
// Every caller of this knows the phase the row settled in, and each waiter
|
|
4728
|
+
// must be handed it directly. A waiter that instead re-read the shared row
|
|
4729
|
+
// after release could see it cleared, or see the next dispatch for the same
|
|
4730
|
+
// issue, and classify the wrong run.
|
|
4731
|
+
#resolveDispatchTerminalWaiters(issue, phase) {
|
|
4452
4732
|
const key = issueKey(issue);
|
|
4453
4733
|
for (const resolve of this.#dispatchTerminalWaiters.get(key) ?? [])
|
|
4454
|
-
resolve();
|
|
4734
|
+
resolve(phase);
|
|
4455
4735
|
this.#dispatchTerminalWaiters.delete(key);
|
|
4456
4736
|
}
|
|
4457
4737
|
#scheduleDispatchLifecycleRetry(record) {
|
|
@@ -4538,7 +4818,7 @@ export class FactoryLoop {
|
|
|
4538
4818
|
if (!lifecycle)
|
|
4539
4819
|
return;
|
|
4540
4820
|
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
4541
|
-
this.#resolveDispatchTerminalWaiters(lifecycle.issue);
|
|
4821
|
+
this.#resolveDispatchTerminalWaiters(lifecycle.issue, lifecycle.phase);
|
|
4542
4822
|
return;
|
|
4543
4823
|
}
|
|
4544
4824
|
if (lifecycle.phase === 'waiting-for-human')
|
|
@@ -4915,11 +5195,11 @@ export class FactoryLoop {
|
|
|
4915
5195
|
for (const [name] of record.agents) {
|
|
4916
5196
|
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
4917
5197
|
}
|
|
4918
|
-
await this.#
|
|
5198
|
+
await this.#retireSlackWatcher(record);
|
|
4919
5199
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
4920
5200
|
await this.#writeInFlightRegistry();
|
|
4921
5201
|
this.#increment('dispatchLifecycleStaleIssuesAbandoned');
|
|
4922
|
-
this.#resolveDispatchTerminalWaiters(record.issue);
|
|
5202
|
+
this.#resolveDispatchTerminalWaiters(record.issue, 'abandoned');
|
|
4923
5203
|
this.#logger.info?.('[factory] abandoned durable dispatch whose live issue is no longer ready', {
|
|
4924
5204
|
issue: record.issue.key,
|
|
4925
5205
|
reason,
|
|
@@ -4995,7 +5275,7 @@ export class FactoryLoop {
|
|
|
4995
5275
|
this.#increment(releaseReason === 'issue-human-review' ? 'humanReview' : 'done');
|
|
4996
5276
|
this.#emit('issue-done', { issue: record.issue });
|
|
4997
5277
|
await this.#writeInFlightRegistry();
|
|
4998
|
-
this.#resolveDispatchTerminalWaiters(record.issue);
|
|
5278
|
+
this.#resolveDispatchTerminalWaiters(record.issue, 'complete');
|
|
4999
5279
|
return true;
|
|
5000
5280
|
}
|
|
5001
5281
|
async #assertDispatchLifecycleOwner(record) {
|
|
@@ -6019,6 +6299,7 @@ export class FactoryLoop {
|
|
|
6019
6299
|
registryPath,
|
|
6020
6300
|
eventListener: this.#eventListenerStatus(),
|
|
6021
6301
|
readinessReconcile: this.#readinessReconcileStatus(),
|
|
6302
|
+
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
6022
6303
|
};
|
|
6023
6304
|
await mkdir(dirname(path), { recursive: true });
|
|
6024
6305
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
@@ -6037,7 +6318,7 @@ export class FactoryLoop {
|
|
|
6037
6318
|
},
|
|
6038
6319
|
});
|
|
6039
6320
|
}
|
|
6040
|
-
async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
|
|
6321
|
+
async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
|
|
6041
6322
|
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
6042
6323
|
if (handoffs.length === 0) {
|
|
6043
6324
|
return;
|
|
@@ -6277,6 +6558,22 @@ export class FactoryLoop {
|
|
|
6277
6558
|
}
|
|
6278
6559
|
try {
|
|
6279
6560
|
await this.#fleet.release(agentName, reason);
|
|
6561
|
+
// Invalidate the spawn memory the moment the release is confirmed, so a
|
|
6562
|
+
// retry for the same deterministic invocation spawns a real worker
|
|
6563
|
+
// instead of inheriting this one's claim. Shutdown is excluded on
|
|
6564
|
+
// purpose: nothing is being retried there, and a takeover must still be
|
|
6565
|
+
// free to adopt agents this process merely stopped supervising.
|
|
6566
|
+
if (record && batch && context !== 'stop') {
|
|
6567
|
+
const releasedInvocationId = batch.recordRelease(record, agentName, this.#clock.now());
|
|
6568
|
+
if (releasedInvocationId) {
|
|
6569
|
+
this.#logger.debug?.('[factory] released agent invocation is no longer dispatchable', {
|
|
6570
|
+
issue: record.issue.key,
|
|
6571
|
+
agentName,
|
|
6572
|
+
reason,
|
|
6573
|
+
invocationId: releasedInvocationId,
|
|
6574
|
+
});
|
|
6575
|
+
}
|
|
6576
|
+
}
|
|
6280
6577
|
if (record)
|
|
6281
6578
|
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
6282
6579
|
}
|
|
@@ -6506,7 +6803,10 @@ export class FactoryLoop {
|
|
|
6506
6803
|
const batch = await this.#batch();
|
|
6507
6804
|
const invocationId = batch.invocationIdFor(record.issue, spec);
|
|
6508
6805
|
const existing = record.agents.get(spec.name);
|
|
6509
|
-
|
|
6806
|
+
// A released placement keeps its bookkeeping entry, but it is not a worker.
|
|
6807
|
+
// Answering with its old spawn result here would report a synthetic success
|
|
6808
|
+
// for a process that no longer exists.
|
|
6809
|
+
if (existing?.result && existing.releasedAtMs === undefined) {
|
|
6510
6810
|
this.#scheduleHeldAgentDeadline(record);
|
|
6511
6811
|
return { name: existing.result?.name ?? spec.name };
|
|
6512
6812
|
}
|
|
@@ -6556,6 +6856,7 @@ export class FactoryLoop {
|
|
|
6556
6856
|
result = await this.#fleet.spawn({
|
|
6557
6857
|
name: spec.name,
|
|
6558
6858
|
capability: spec.capability,
|
|
6859
|
+
identityKey: dispatchAgentIdentityKey(record.issue, spec.role),
|
|
6559
6860
|
node: spec.node ?? 'self',
|
|
6560
6861
|
repo: spec.repo,
|
|
6561
6862
|
task: spec.task,
|
|
@@ -6657,6 +6958,16 @@ export class FactoryLoop {
|
|
|
6657
6958
|
return;
|
|
6658
6959
|
}
|
|
6659
6960
|
}
|
|
6961
|
+
// Swarm workers share the lead's checkout and lifecycle branch. If a worker
|
|
6962
|
+
// exit reached the publication/completion paths below, whichever worker
|
|
6963
|
+
// finished first would publish whatever partial state was on the shared
|
|
6964
|
+
// branch and mark every swarm member "done" via the shared-branch PR probe,
|
|
6965
|
+
// releasing the still-working lead. The lead alone is authoritative for
|
|
6966
|
+
// publication and completion in a swarm.
|
|
6967
|
+
if (exiting?.spec.swarmRole === 'worker') {
|
|
6968
|
+
this.#increment('swarmWorkerExitsSuppressed');
|
|
6969
|
+
return;
|
|
6970
|
+
}
|
|
6660
6971
|
if (isCompletionReason(reason)) {
|
|
6661
6972
|
if (exiting?.spec.role === 'implementer' && await this.#issueHasCompletionPr(record, {
|
|
6662
6973
|
openOnly: this.#config.babysitter.enabled,
|
|
@@ -6901,6 +7212,7 @@ export class FactoryLoop {
|
|
|
6901
7212
|
const result = await this.#fleet.spawn({
|
|
6902
7213
|
name: tracked.spec.name,
|
|
6903
7214
|
capability: tracked.spec.capability,
|
|
7215
|
+
identityKey: dispatchAgentIdentityKey(record.issue, tracked.spec.role),
|
|
6904
7216
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
6905
7217
|
repo: tracked.spec.repo,
|
|
6906
7218
|
task: tracked.spec.task,
|
|
@@ -7024,6 +7336,7 @@ export class FactoryLoop {
|
|
|
7024
7336
|
}
|
|
7025
7337
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
7026
7338
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
7339
|
+
const trajectorySessionRef = canonicalTrajectorySessionRef(implementer.sessionRef);
|
|
7027
7340
|
const expectedHeadRef = implementer.spec.branch;
|
|
7028
7341
|
if (!expectedHeadRef) {
|
|
7029
7342
|
throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
|
|
@@ -7089,7 +7402,7 @@ export class FactoryLoop {
|
|
|
7089
7402
|
expectedHeadRef,
|
|
7090
7403
|
baseRef,
|
|
7091
7404
|
title: `${issue.key}: ${issue.title}`,
|
|
7092
|
-
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
7405
|
+
body: githubPullRequestBody(issue, implementer.spec.preview, trajectorySessionRef),
|
|
7093
7406
|
...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
|
|
7094
7407
|
});
|
|
7095
7408
|
const published = result.author
|
|
@@ -7678,7 +7991,7 @@ export class FactoryLoop {
|
|
|
7678
7991
|
await this.#recordDispatchTerminal(record.issue);
|
|
7679
7992
|
const next = (await this.#batch()).complete(record.issue);
|
|
7680
7993
|
await this.#drainReadyClarificationWake();
|
|
7681
|
-
await this.#
|
|
7994
|
+
await this.#retireSlackWatcher(record);
|
|
7682
7995
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
7683
7996
|
await this.#writeInFlightRegistry();
|
|
7684
7997
|
if (next) {
|
|
@@ -7782,6 +8095,7 @@ export class FactoryLoop {
|
|
|
7782
8095
|
const result = await this.#fleet.resume({
|
|
7783
8096
|
name,
|
|
7784
8097
|
sessionRef: tracked.sessionRef,
|
|
8098
|
+
identityKey: dispatchAgentIdentityKey(record.issue, tracked.spec.role),
|
|
7785
8099
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
7786
8100
|
capability: tracked.spec.capability,
|
|
7787
8101
|
repo: tracked.spec.repo,
|
|
@@ -7910,6 +8224,7 @@ export class FactoryLoop {
|
|
|
7910
8224
|
const result = await this.#fleet.spawn({
|
|
7911
8225
|
name: replacementSpec.name,
|
|
7912
8226
|
capability: replacementSpec.capability,
|
|
8227
|
+
identityKey: dispatchAgentIdentityKey(record.issue, replacementSpec.role),
|
|
7913
8228
|
node: tracked.result?.node ?? replacementSpec.node ?? 'self',
|
|
7914
8229
|
repo: replacementSpec.repo,
|
|
7915
8230
|
task: replacementSpec.task,
|
|
@@ -9377,6 +9692,15 @@ export class FactoryLoop {
|
|
|
9377
9692
|
previewStartCommand: spec.preview?.startCommand,
|
|
9378
9693
|
} : {}),
|
|
9379
9694
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
9695
|
+
...(spec.swarmRole && spec.channel ? {
|
|
9696
|
+
swarm: {
|
|
9697
|
+
role: spec.swarmRole,
|
|
9698
|
+
channel: spec.channel,
|
|
9699
|
+
otherMemberNames: decision.implementers
|
|
9700
|
+
.filter((implementer) => implementer.channel === spec.channel && implementer.name !== spec.name)
|
|
9701
|
+
.map((implementer) => implementer.name),
|
|
9702
|
+
},
|
|
9703
|
+
} : {}),
|
|
9380
9704
|
}),
|
|
9381
9705
|
};
|
|
9382
9706
|
};
|
|
@@ -10897,7 +11221,7 @@ export class FactoryLoop {
|
|
|
10897
11221
|
});
|
|
10898
11222
|
return;
|
|
10899
11223
|
}
|
|
10900
|
-
const issue = await this.#findMergeAdvanceIssueForPr(snapshot);
|
|
11224
|
+
const issue = await this.#findMergeAdvanceIssueForPr(snapshot, repo);
|
|
10901
11225
|
if (!issue) {
|
|
10902
11226
|
this.#increment('mergedPrAdvanceNoIssue');
|
|
10903
11227
|
return;
|
|
@@ -10952,7 +11276,7 @@ export class FactoryLoop {
|
|
|
10952
11276
|
this.#error(error, issueRef(issue));
|
|
10953
11277
|
}
|
|
10954
11278
|
}
|
|
10955
|
-
async #findMergeAdvanceIssueForPr(snapshot) {
|
|
11279
|
+
async #findMergeAdvanceIssueForPr(snapshot, eventRepo) {
|
|
10956
11280
|
// An issue is "upstream" of a merge if it sits in the agent-implementing or
|
|
10957
11281
|
// human-review role for its team. UUIDs are globally unique, so the reverse
|
|
10958
11282
|
// role lookup covers every team without per-team scoping here.
|
|
@@ -10961,16 +11285,34 @@ export class FactoryLoop {
|
|
|
10961
11285
|
return role === 'agentImplementing' || role === 'humanReview';
|
|
10962
11286
|
};
|
|
10963
11287
|
let best;
|
|
11288
|
+
let ambiguous = false;
|
|
10964
11289
|
const scanStartedAtMs = this.#clock.now();
|
|
10965
11290
|
// This no-record path runs after agents are released, so there is no
|
|
10966
11291
|
// tracked PR identity left. Keep the scan simple and prefer branch identity
|
|
10967
11292
|
// over title/body references to avoid "related to AR-N" body false positives.
|
|
10968
11293
|
const githubSource = await this.#issueSource() === 'github';
|
|
11294
|
+
// A GitHub-native issue key is a bare number; the same number exists
|
|
11295
|
+
// independently in every configured repository. Only a same-repo candidate
|
|
11296
|
+
// may complete the merge, so a merged PR in one repo can never close an
|
|
11297
|
+
// unrelated issue that happens to share its number in another. #276.
|
|
11298
|
+
const eventRepoKey = validGithubRepo(eventRepo) ? eventRepo.toLowerCase() : undefined;
|
|
11299
|
+
if (githubSource && !eventRepoKey) {
|
|
11300
|
+
this.#logger.warn?.('[factory] merge advance skipped: event repo is not identifiable', {
|
|
11301
|
+
prNumber: snapshot.number,
|
|
11302
|
+
});
|
|
11303
|
+
return undefined;
|
|
11304
|
+
}
|
|
10969
11305
|
const paths = githubSource ? await this.#githubIssuePaths() : await this.#listRelayfileTree(ISSUE_ROOT, 'merge advance issue scan');
|
|
10970
11306
|
for (const path of paths) {
|
|
10971
11307
|
if (githubSource ? !isGithubIssueFilePath(path) : !isIssueFilePath(path)) {
|
|
10972
11308
|
continue;
|
|
10973
11309
|
}
|
|
11310
|
+
if (githubSource) {
|
|
11311
|
+
const issueParts = githubIssuePathParts(path);
|
|
11312
|
+
if (!issueParts || `${issueParts.owner}/${issueParts.repo}`.toLowerCase() !== eventRepoKey) {
|
|
11313
|
+
continue;
|
|
11314
|
+
}
|
|
11315
|
+
}
|
|
10974
11316
|
const issue = await this.#readIssue(path);
|
|
10975
11317
|
if (!issue || (githubSource
|
|
10976
11318
|
? issue.state?.name?.trim().toLowerCase() === 'closed'
|
|
@@ -10983,10 +11325,27 @@ export class FactoryLoop {
|
|
|
10983
11325
|
const score = prSnapshotIssueMatchScore(snapshot, issue.key);
|
|
10984
11326
|
if (score > 0 && (!best || score > best.score)) {
|
|
10985
11327
|
best = { issue, score };
|
|
11328
|
+
ambiguous = false;
|
|
11329
|
+
}
|
|
11330
|
+
else if (score > 0 && best && score === best.score) {
|
|
11331
|
+
ambiguous = true;
|
|
10986
11332
|
}
|
|
10987
11333
|
}
|
|
11334
|
+
// Fail-closed on a tie. A silent first-wins on unrelated same-score
|
|
11335
|
+
// candidates is the exact class of quiet corruption we are removing.
|
|
11336
|
+
if (ambiguous) {
|
|
11337
|
+
this.#increment('mergedPrAdvanceAmbiguous');
|
|
11338
|
+
this.#logger.warn?.('[factory] merge advance skipped: multiple candidates tied on match score', {
|
|
11339
|
+
prNumber: snapshot.number,
|
|
11340
|
+
eventRepo,
|
|
11341
|
+
matchedIssue: best?.issue.key,
|
|
11342
|
+
matchScore: best?.score,
|
|
11343
|
+
});
|
|
11344
|
+
return undefined;
|
|
11345
|
+
}
|
|
10988
11346
|
this.#logger.debug?.(`[factory] scanned ${githubSource ? 'GitHub' : 'Linear'} issues for merged PR advance`, {
|
|
10989
11347
|
prNumber: snapshot.number,
|
|
11348
|
+
eventRepo,
|
|
10990
11349
|
durationMs: this.#clock.now() - scanStartedAtMs,
|
|
10991
11350
|
matchedIssue: best?.issue.key,
|
|
10992
11351
|
matchScore: best?.score,
|
|
@@ -11592,7 +11951,7 @@ export class FactoryLoop {
|
|
|
11592
11951
|
}
|
|
11593
11952
|
if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, releaseReason))
|
|
11594
11953
|
return;
|
|
11595
|
-
await this.#
|
|
11954
|
+
await this.#retireSlackWatcher(record);
|
|
11596
11955
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
11597
11956
|
await this.#recordDispatchTerminal(record.issue);
|
|
11598
11957
|
await this.#finishDurableRelease(record, releaseReason);
|
|
@@ -12070,6 +12429,21 @@ export class FactoryLoop {
|
|
|
12070
12429
|
return;
|
|
12071
12430
|
}
|
|
12072
12431
|
const key = issueKey(record.issue);
|
|
12432
|
+
const previousWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
12433
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
12434
|
+
if (previousWatch?.kind === 'terminal-grace') {
|
|
12435
|
+
// A reopened work unit needs a fresh dispatch notification and a fresh
|
|
12436
|
+
// conversation. Do not let the old grace-period watcher (or its expiry
|
|
12437
|
+
// timer) capture and later tear down the new dispatch.
|
|
12438
|
+
if (!await this.#stopSlackWatcher(record.issue)) {
|
|
12439
|
+
// Fail closed. An undrained reply route still holds the retired thread
|
|
12440
|
+
// and would bind it to this dispatch, delivering a stale human reply to
|
|
12441
|
+
// fresh work. Leave the fence up; the next reconcile retries the drain.
|
|
12442
|
+
this.#logger.warn?.('[factory] deferring Slack dispatch thread for reopened work unit; in-flight reply route not drained', { issue: record.issue.key });
|
|
12443
|
+
this.#increment('slackDispatchThreadsDeferredUndrainedReply');
|
|
12444
|
+
return;
|
|
12445
|
+
}
|
|
12446
|
+
}
|
|
12073
12447
|
const existingThread = await this.#persistedSlackThread(key);
|
|
12074
12448
|
const watcherStart = this.#slackWatcherStarts.get(key);
|
|
12075
12449
|
if (existingThread || watcherStart) {
|
|
@@ -12152,11 +12526,13 @@ export class FactoryLoop {
|
|
|
12152
12526
|
if (existing) {
|
|
12153
12527
|
const sessionRef = owned?.tracked.sessionRef;
|
|
12154
12528
|
const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
|
|
12529
|
+
let rebound = false;
|
|
12155
12530
|
if (owned && sessionRef &&
|
|
12156
|
-
(agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
|
|
12157
|
-
|
|
12531
|
+
(!existing.agent || agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
|
|
12532
|
+
rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
|
|
12158
12533
|
name: agentName,
|
|
12159
12534
|
sessionRef,
|
|
12535
|
+
role: owned.tracked.spec.role,
|
|
12160
12536
|
node: owned.tracked.result?.node ?? owned.tracked.spec.node,
|
|
12161
12537
|
capability: owned.tracked.spec.capability,
|
|
12162
12538
|
repo: owned.tracked.spec.repo,
|
|
@@ -12167,39 +12543,39 @@ export class FactoryLoop {
|
|
|
12167
12543
|
}
|
|
12168
12544
|
if (existing.pending.length > 0 || existing.delivery) {
|
|
12169
12545
|
const waiting = await this.#state.getWaitingClarification(this.#workspaceId, issueKey(existing.issue));
|
|
12170
|
-
if (!waiting)
|
|
12546
|
+
if (!waiting && (existing.agent || rebound))
|
|
12171
12547
|
this.#slackConversationTurns.schedule(conversationId);
|
|
12172
12548
|
}
|
|
12173
12549
|
return;
|
|
12174
12550
|
}
|
|
12175
12551
|
const sessionRef = owned?.tracked.sessionRef;
|
|
12176
|
-
if (!owned || !sessionRef) {
|
|
12177
|
-
this.#increment('slackConversationSessionsSkippedMissingSession');
|
|
12178
|
-
return;
|
|
12179
|
-
}
|
|
12180
12552
|
const channelDir = await this.#slackChannelDir() ?? this.#config.slack?.channel;
|
|
12181
12553
|
if (!channelDir)
|
|
12182
12554
|
return;
|
|
12183
|
-
const agentName = owned.tracked.result?.name ?? owned.name;
|
|
12555
|
+
const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
|
|
12184
12556
|
const reserved = await this.#state.reserveConversationSession(this.#workspaceId, conversationId, {
|
|
12185
12557
|
provider: 'slack',
|
|
12186
12558
|
issue: { ...record.issue },
|
|
12187
12559
|
externalId: threadId,
|
|
12188
12560
|
context: { channelDir },
|
|
12189
|
-
agent: {
|
|
12190
|
-
|
|
12191
|
-
|
|
12192
|
-
|
|
12193
|
-
|
|
12194
|
-
|
|
12195
|
-
|
|
12196
|
-
|
|
12561
|
+
...(owned && sessionRef && agentName ? { agent: {
|
|
12562
|
+
name: agentName,
|
|
12563
|
+
sessionRef,
|
|
12564
|
+
role: owned.tracked.spec.role,
|
|
12565
|
+
node: owned.tracked.result?.node ?? owned.tracked.spec.node,
|
|
12566
|
+
capability: owned.tracked.spec.capability,
|
|
12567
|
+
repo: owned.tracked.spec.repo,
|
|
12568
|
+
clonePath: owned.tracked.spec.clonePath,
|
|
12569
|
+
} } : {}),
|
|
12197
12570
|
history: [],
|
|
12198
12571
|
processedMessageIds: [],
|
|
12572
|
+
acknowledgedMessageIds: [],
|
|
12573
|
+
acknowledgementClaims: {},
|
|
12199
12574
|
pending: [],
|
|
12200
12575
|
});
|
|
12201
|
-
if (reserved)
|
|
12202
|
-
this.#increment('slackConversationSessionsOwned');
|
|
12576
|
+
if (reserved) {
|
|
12577
|
+
this.#increment(owned && sessionRef ? 'slackConversationSessionsOwned' : 'slackConversationSessionsReservedUnowned');
|
|
12578
|
+
}
|
|
12203
12579
|
}
|
|
12204
12580
|
// Called right after a babysitter is spawned/reattached for an issue's PR so
|
|
12205
12581
|
// an already-owned Slack conversation session (reserved earlier by the
|
|
@@ -12227,11 +12603,15 @@ export class FactoryLoop {
|
|
|
12227
12603
|
const claimed = await this.#state.claimConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId, this.#clock.now(), SLACK_CONVERSATION_TURN_LEASE_MS);
|
|
12228
12604
|
if (!claimed?.delivery) {
|
|
12229
12605
|
const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
12230
|
-
if (current && (current.pending.length > 0 || current.delivery)) {
|
|
12606
|
+
if (current?.agent && (current.pending.length > 0 || current.delivery)) {
|
|
12231
12607
|
this.#slackConversationTurns.schedule(conversationId, SLACK_CONVERSATION_TURN_RETRY_MS);
|
|
12232
12608
|
}
|
|
12233
12609
|
return;
|
|
12234
12610
|
}
|
|
12611
|
+
if (!claimed.agent) {
|
|
12612
|
+
await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
|
|
12613
|
+
return;
|
|
12614
|
+
}
|
|
12235
12615
|
if (!await this.#ownsActiveSlackConversationIssue(claimed.issue)) {
|
|
12236
12616
|
await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
|
|
12237
12617
|
this.#increment('slackConversationTurnsSuppressedStaleOwner');
|
|
@@ -12252,9 +12632,20 @@ export class FactoryLoop {
|
|
|
12252
12632
|
}, Math.max(1_000, Math.floor(SLACK_CONVERSATION_TURN_LEASE_MS / 3)));
|
|
12253
12633
|
heartbeat.unref?.();
|
|
12254
12634
|
try {
|
|
12635
|
+
// Sessions bound after this field was introduced carry their role
|
|
12636
|
+
// durably, so resume never depends on a live lookup by the agent's
|
|
12637
|
+
// current name — which a since-completed rename (e.g. babysitter
|
|
12638
|
+
// retarget) can otherwise leave unresolvable forever. Older
|
|
12639
|
+
// already-persisted sessions fall back to the live lookup.
|
|
12640
|
+
const conversationRole = claimed.agent.role
|
|
12641
|
+
?? (await this.#batch()).getIssueByAgent(claimed.agent.name)?.agents.get(claimed.agent.name)?.spec.role;
|
|
12642
|
+
if (!conversationRole) {
|
|
12643
|
+
throw new Error(`Cannot resume ${claimed.agent.name}: its dispatch role is unavailable for identity proof`);
|
|
12644
|
+
}
|
|
12255
12645
|
const result = await this.#fleet.resume({
|
|
12256
12646
|
name: claimed.agent.name,
|
|
12257
12647
|
sessionRef: claimed.agent.sessionRef,
|
|
12648
|
+
identityKey: dispatchAgentIdentityKey(claimed.issue, conversationRole),
|
|
12258
12649
|
node: claimed.agent.node ?? 'self',
|
|
12259
12650
|
capability: claimed.agent.capability,
|
|
12260
12651
|
repo: claimed.agent.repo,
|
|
@@ -12304,10 +12695,13 @@ export class FactoryLoop {
|
|
|
12304
12695
|
}
|
|
12305
12696
|
}
|
|
12306
12697
|
async #recordSlackConversationResume(session, result) {
|
|
12698
|
+
const sessionAgent = session.agent;
|
|
12699
|
+
if (!sessionAgent)
|
|
12700
|
+
return;
|
|
12307
12701
|
const record = (await this.#batch()).getIssue(session.issue);
|
|
12308
12702
|
if (!record)
|
|
12309
12703
|
return;
|
|
12310
|
-
const entry = [...record.agents.entries()].find(([name, tracked]) => name ===
|
|
12704
|
+
const entry = [...record.agents.entries()].find(([name, tracked]) => name === sessionAgent.name || tracked.result?.name === sessionAgent.name);
|
|
12311
12705
|
if (!entry)
|
|
12312
12706
|
return;
|
|
12313
12707
|
const [previousName, tracked] = entry;
|
|
@@ -12633,7 +13027,14 @@ export class FactoryLoop {
|
|
|
12633
13027
|
`Question: ${triageEscalationQuestion(decision, issue)}`,
|
|
12634
13028
|
].join('\n'),
|
|
12635
13029
|
});
|
|
12636
|
-
|
|
13030
|
+
const key = issueKey(decision.issue);
|
|
13031
|
+
await this.#state.setSlackThread(this.#workspaceId, key, root.threadId);
|
|
13032
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
|
|
13033
|
+
kind: 'triage',
|
|
13034
|
+
issue: { ...decision.issue },
|
|
13035
|
+
decision: structuredClone(decision),
|
|
13036
|
+
threadId: root.threadId,
|
|
13037
|
+
});
|
|
12637
13038
|
const replayedResult = await this.#watchSlackThread(escalationWatchRecord(decision), root.threadId);
|
|
12638
13039
|
this.#recordSlackWritebackSuccess('triage-escalation');
|
|
12639
13040
|
return replayedResult;
|
|
@@ -12699,6 +13100,11 @@ export class FactoryLoop {
|
|
|
12699
13100
|
if (!reply || !reply.isThreadReply || reply.threadTs !== threadId || reply.channelDir !== channelDir) {
|
|
12700
13101
|
return;
|
|
12701
13102
|
}
|
|
13103
|
+
if (allowPreExisting &&
|
|
13104
|
+
options.replayAfterMs !== undefined &&
|
|
13105
|
+
slackMessageReceivedAtMs(reply.messageTs, Number.MAX_SAFE_INTEGER) < options.replayAfterMs) {
|
|
13106
|
+
return;
|
|
13107
|
+
}
|
|
12702
13108
|
const replyMessageKey = `${reply.threadTs}:${reply.messageTs}`;
|
|
12703
13109
|
if (seenReplyMessages.has(replyMessageKey)) {
|
|
12704
13110
|
this.#logger.debug?.('[factory] suppressed duplicate Slack reply message', { issue: record.issue.key, path });
|
|
@@ -12877,6 +13283,47 @@ export class FactoryLoop {
|
|
|
12877
13283
|
this.#slackConversationTurns.schedule(conversationId);
|
|
12878
13284
|
}
|
|
12879
13285
|
}
|
|
13286
|
+
for (const [key, watch] of await this.#state.listSlackThreadWatches(this.#workspaceId)) {
|
|
13287
|
+
if (this.#slackWatchers.has(key) || this.#slackWatcherStarts.has(key))
|
|
13288
|
+
continue;
|
|
13289
|
+
if (watch.kind === 'terminal-grace' && watch.expiresAtMs <= this.#clock.now()) {
|
|
13290
|
+
await this.#stopSlackWatcher(watch.issue);
|
|
13291
|
+
continue;
|
|
13292
|
+
}
|
|
13293
|
+
await this.#state.setSlackThread(this.#workspaceId, key, watch.threadId);
|
|
13294
|
+
const watchRecord = escalationWatchRecord(watch.decision);
|
|
13295
|
+
if (watch.kind === 'terminal-grace') {
|
|
13296
|
+
const retiredAtMs = terminalSlackWatchRetiredAtMs(watch);
|
|
13297
|
+
if (watch.retiredAtMs !== retiredAtMs) {
|
|
13298
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, { ...watch, retiredAtMs });
|
|
13299
|
+
}
|
|
13300
|
+
this.#terminalSlackWatchIssues.add(key);
|
|
13301
|
+
const conversationId = slackConversationId(watch.threadId);
|
|
13302
|
+
await this.#slackConversationTurns.cancel(conversationId);
|
|
13303
|
+
try {
|
|
13304
|
+
await this.#surfaceUndeliveredSlackConversation(watch.threadId);
|
|
13305
|
+
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13306
|
+
}
|
|
13307
|
+
catch (error) {
|
|
13308
|
+
// The undelivered-reply receipt needs Slack writeback, which may be
|
|
13309
|
+
// unavailable at startup. That is retryable state maintenance for this
|
|
13310
|
+
// one thread, not a reason to abandon rehydration: aborting here would
|
|
13311
|
+
// leave every remaining thread watched by nobody. Keep the queued
|
|
13312
|
+
// replies (clearing them now would drop replies nobody was told about)
|
|
13313
|
+
// and carry on re-arming.
|
|
13314
|
+
this.#logger.warn?.('[factory] failed to settle undelivered Slack replies for terminal watch; will retry', { issue: watch.issue.key, error });
|
|
13315
|
+
this.#increment('slackTerminalWatchReceiptsDeferred');
|
|
13316
|
+
this.#scheduleSlackTerminalReceiptRetry(watch.issue, watch.threadId, watch.expiresAtMs);
|
|
13317
|
+
}
|
|
13318
|
+
await this.#rearmSlackWatcher(watchRecord, watch.threadId, {
|
|
13319
|
+
replayConversationReplies: true,
|
|
13320
|
+
replayAfterMs: retiredAtMs,
|
|
13321
|
+
});
|
|
13322
|
+
this.#scheduleSlackTerminalWatchExpiry(watch.issue, watch.expiresAtMs);
|
|
13323
|
+
continue;
|
|
13324
|
+
}
|
|
13325
|
+
await this.#rearmSlackWatcher(watchRecord, watch.threadId, { replayConversationReplies: true });
|
|
13326
|
+
}
|
|
12880
13327
|
await this.#sweepWaitingClarifications();
|
|
12881
13328
|
for (const [, waiting] of await this.#state.listWaitingClarifications(this.#workspaceId)) {
|
|
12882
13329
|
if (!waiting.threadId)
|
|
@@ -13022,8 +13469,73 @@ export class FactoryLoop {
|
|
|
13022
13469
|
this.#clarificationSweepTimer = timer;
|
|
13023
13470
|
this.#clarificationSweepDueAtMs = dueAtMs;
|
|
13024
13471
|
}
|
|
13472
|
+
// The terminal fence is the only thing that makes an in-flight reply route
|
|
13473
|
+
// answer "no active agent" instead of binding the retired thread to whatever
|
|
13474
|
+
// dispatch owns this key. Routes are chained per work unit, so awaiting the
|
|
13475
|
+
// newest one drains every reply queued behind it.
|
|
13476
|
+
async #drainSlackReplyRoutes(key) {
|
|
13477
|
+
// Snapshotting #slackReplyRoutes is not enough on its own. A reply handler
|
|
13478
|
+
// that is still inside its mount read when the drain starts registers its
|
|
13479
|
+
// route *after* the snapshot, so it would run once the fence is gone and
|
|
13480
|
+
// bind the retired thread to the next dispatch — the same escape one level
|
|
13481
|
+
// in. Bar *routing* for this key first (the bar and the registration are
|
|
13482
|
+
// both synchronous, so nothing can slip between them), then drain whatever
|
|
13483
|
+
// is already chained, then prove the set is empty before reporting success.
|
|
13484
|
+
// A barred reply still answers the human, and that writeback registers here
|
|
13485
|
+
// like any other effect, so the extra passes are what pick it up: quiescence
|
|
13486
|
+
// means every effect this work unit started has settled, not merely the ones
|
|
13487
|
+
// that existed when the drain began.
|
|
13488
|
+
const nested = this.#slackReplyRouteDrains.has(key);
|
|
13489
|
+
this.#slackReplyRouteDrains.add(key);
|
|
13490
|
+
try {
|
|
13491
|
+
for (let pass = 0; pass < SLACK_REPLY_ROUTE_DRAIN_PASSES; pass += 1) {
|
|
13492
|
+
const route = this.#slackReplyRoutes.get(key);
|
|
13493
|
+
if (!route)
|
|
13494
|
+
return true;
|
|
13495
|
+
try {
|
|
13496
|
+
await route;
|
|
13497
|
+
}
|
|
13498
|
+
catch (error) {
|
|
13499
|
+
// A route that *rejects* is not drained: the watcher replays it after
|
|
13500
|
+
// SLACK_REPLY_ROUTE_RETRY_MS, and that replay would land on the next
|
|
13501
|
+
// dispatch. Fail closed and let the caller keep the fence up rather
|
|
13502
|
+
// than leak a stale human reply onto fresh work.
|
|
13503
|
+
this.#logger.warn?.('[factory] in-flight Slack reply route did not drain; keeping terminal Slack fence', { issue: key, error });
|
|
13504
|
+
this.#increment('slackReplyRouteDrainsFailed');
|
|
13505
|
+
return false;
|
|
13506
|
+
}
|
|
13507
|
+
// The owner clears its own entry when it settles; retiring it here too
|
|
13508
|
+
// keeps the loop monotonic if that finally has not run yet.
|
|
13509
|
+
if (this.#slackReplyRoutes.get(key) === route)
|
|
13510
|
+
this.#slackReplyRoutes.delete(key);
|
|
13511
|
+
}
|
|
13512
|
+
// Not provably quiescent. Fail closed for the same reason as a rejection.
|
|
13513
|
+
this.#logger.warn?.('[factory] Slack reply routes did not quiesce; keeping terminal Slack fence', { issue: key });
|
|
13514
|
+
this.#increment('slackReplyRouteDrainsFailed');
|
|
13515
|
+
return false;
|
|
13516
|
+
}
|
|
13517
|
+
finally {
|
|
13518
|
+
if (!nested)
|
|
13519
|
+
this.#slackReplyRouteDrains.delete(key);
|
|
13520
|
+
}
|
|
13521
|
+
}
|
|
13025
13522
|
async #stopSlackWatcher(issue) {
|
|
13026
13523
|
const key = issueKey(issue);
|
|
13524
|
+
// Drain before clearing the fence. Clearing it first lets a reply that is
|
|
13525
|
+
// already mid-route — or one queued behind it — fall through the fence check
|
|
13526
|
+
// in #routeSlackConversationAnswerUnlocked and rebind the retired thread to
|
|
13527
|
+
// the next dispatch of this work unit.
|
|
13528
|
+
if (!await this.#drainSlackReplyRoutes(key))
|
|
13529
|
+
return false;
|
|
13530
|
+
this.#terminalSlackWatchIssues.delete(key);
|
|
13531
|
+
const expiryTimer = this.#slackTerminalWatchExpiryTimers.get(key);
|
|
13532
|
+
if (expiryTimer)
|
|
13533
|
+
clearTimeout(expiryTimer);
|
|
13534
|
+
this.#slackTerminalWatchExpiryTimers.delete(key);
|
|
13535
|
+
const receiptRetryTimer = this.#slackTerminalReceiptRetryTimers.get(key);
|
|
13536
|
+
if (receiptRetryTimer)
|
|
13537
|
+
clearTimeout(receiptRetryTimer);
|
|
13538
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
13027
13539
|
const watcher = this.#slackWatchers.get(key);
|
|
13028
13540
|
this.#slackWatchers.delete(key);
|
|
13029
13541
|
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
@@ -13034,6 +13546,259 @@ export class FactoryLoop {
|
|
|
13034
13546
|
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13035
13547
|
}
|
|
13036
13548
|
await this.#state.clearSlackThread(this.#workspaceId, key);
|
|
13549
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, key);
|
|
13550
|
+
return true;
|
|
13551
|
+
}
|
|
13552
|
+
async #retireSlackWatcher(record) {
|
|
13553
|
+
const key = issueKey(record.issue);
|
|
13554
|
+
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
13555
|
+
if (!threadId) {
|
|
13556
|
+
await this.#stopSlackWatcher(record.issue);
|
|
13557
|
+
return;
|
|
13558
|
+
}
|
|
13559
|
+
const existingWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
13560
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
13561
|
+
const retiredAtMs = existingWatch?.kind === 'terminal-grace'
|
|
13562
|
+
? terminalSlackWatchRetiredAtMs(existingWatch)
|
|
13563
|
+
: this.#clock.now();
|
|
13564
|
+
const expiresAtMs = existingWatch?.kind === 'terminal-grace'
|
|
13565
|
+
? existingWatch.expiresAtMs
|
|
13566
|
+
: retiredAtMs + SLACK_TERMINAL_THREAD_GRACE_MS;
|
|
13567
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
|
|
13568
|
+
kind: 'terminal-grace',
|
|
13569
|
+
issue: { ...record.issue },
|
|
13570
|
+
decision: structuredClone(record.decision),
|
|
13571
|
+
threadId,
|
|
13572
|
+
retiredAtMs,
|
|
13573
|
+
expiresAtMs,
|
|
13574
|
+
});
|
|
13575
|
+
// A terminal thread must never retain a resumable session for an agent that
|
|
13576
|
+
// has already exited. Keep only the exact-thread listener so a late human
|
|
13577
|
+
// reply receives the explicit no-active-agent writeback below.
|
|
13578
|
+
this.#terminalSlackWatchIssues.add(key);
|
|
13579
|
+
await this.#slackReplyRoutes.get(key)?.catch(() => undefined);
|
|
13580
|
+
const conversationId = slackConversationId(threadId);
|
|
13581
|
+
await this.#slackConversationTurns.cancel(conversationId);
|
|
13582
|
+
try {
|
|
13583
|
+
await this.#surfaceUndeliveredSlackConversation(threadId);
|
|
13584
|
+
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13585
|
+
}
|
|
13586
|
+
catch (error) {
|
|
13587
|
+
// The receipt fails whenever another handler holds the claim or Slack
|
|
13588
|
+
// writeback is down — neither is a reason to abort retirement. Callers
|
|
13589
|
+
// reach here having already committed the terminal phase and dropped the
|
|
13590
|
+
// pending abandon reason, so a rejection escaping would strand the
|
|
13591
|
+
// registry rewrite, the GitHub watcher stop, and the queued next dispatch
|
|
13592
|
+
// with nothing left to re-run them. Keep the queued replies and let the
|
|
13593
|
+
// retry that owns this receipt settle it inside the grace window.
|
|
13594
|
+
this.#logger.warn?.('[factory] failed to settle undelivered Slack replies while retiring the watcher; will retry', { issue: record.issue.key, error });
|
|
13595
|
+
this.#increment('slackTerminalWatchReceiptsDeferred');
|
|
13596
|
+
this.#scheduleSlackTerminalReceiptRetry(record.issue, threadId, expiresAtMs);
|
|
13597
|
+
}
|
|
13598
|
+
if (!this.#slackWatchers.has(key) && !this.#stopping) {
|
|
13599
|
+
await this.#rearmSlackWatcher(record, threadId);
|
|
13600
|
+
}
|
|
13601
|
+
this.#scheduleSlackTerminalWatchExpiry(record.issue, expiresAtMs);
|
|
13602
|
+
this.#increment('slackTerminalWatchersRetained');
|
|
13603
|
+
}
|
|
13604
|
+
// The durable half of the same record. Every caller here follows the receipt
|
|
13605
|
+
// with a state write (clearing the session), and those two cannot be one
|
|
13606
|
+
// durable step: when the state write fails, the retry that owns it must not
|
|
13607
|
+
// read "replies still queued" as "the human has not been told" and post the
|
|
13608
|
+
// notice again. So the receipt is claimed before the provider write and marked
|
|
13609
|
+
// posted after it, and a retry finds it already settled.
|
|
13610
|
+
async #surfaceUndeliveredSlackConversation(threadId) {
|
|
13611
|
+
const conversationId = slackConversationId(threadId);
|
|
13612
|
+
const session = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13613
|
+
const pendingCount = session
|
|
13614
|
+
? session.pending.length + (session.delivery?.messages.length ?? 0)
|
|
13615
|
+
: 0;
|
|
13616
|
+
if (pendingCount === 0)
|
|
13617
|
+
return;
|
|
13618
|
+
if (session?.terminalReceipt?.posted) {
|
|
13619
|
+
this.#increment('slackTerminalReceiptsAlreadySettled');
|
|
13620
|
+
return;
|
|
13621
|
+
}
|
|
13622
|
+
if (!this.#slack)
|
|
13623
|
+
throw new Error(`Slack thread ${threadId} cannot surface undelivered replies without writeback`);
|
|
13624
|
+
const claimId = randomUUID();
|
|
13625
|
+
if (!await this.#state.claimConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now(), SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS)) {
|
|
13626
|
+
const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13627
|
+
if (current?.terminalReceipt?.posted) {
|
|
13628
|
+
this.#increment('slackTerminalReceiptsAlreadySettled');
|
|
13629
|
+
return;
|
|
13630
|
+
}
|
|
13631
|
+
// Another handler is mid-write. Fail closed so the queued replies survive
|
|
13632
|
+
// for whoever settles them rather than racing a second notice onto the
|
|
13633
|
+
// same thread.
|
|
13634
|
+
throw new Error(`Slack thread ${threadId} terminal receipt is claimed by another handler; retrying`);
|
|
13635
|
+
}
|
|
13636
|
+
const noun = pendingCount === 1 ? 'reply' : 'replies';
|
|
13637
|
+
const slack = this.#slack;
|
|
13638
|
+
try {
|
|
13639
|
+
await this.#withRenewedProviderLease('terminal Slack receipt', SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS, () => this.#state.renewConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now()), () => slack.reply(threadId, `Factory could not deliver ${pendingCount} queued ${noun} because this work unit no longer has an active agent. Please continue on the linked issue or pull request.`));
|
|
13640
|
+
}
|
|
13641
|
+
catch (error) {
|
|
13642
|
+
await this.#state.releaseConversationTerminalReceipt(this.#workspaceId, conversationId, claimId);
|
|
13643
|
+
throw error;
|
|
13644
|
+
}
|
|
13645
|
+
if (!await this.#state.completeConversationTerminalReceipt(this.#workspaceId, conversationId, claimId)) {
|
|
13646
|
+
throw new Error(`Slack thread ${threadId} terminal receipt could not be recorded`);
|
|
13647
|
+
}
|
|
13648
|
+
this.#increment('slackConversationRepliesSurfacedTerminal');
|
|
13649
|
+
}
|
|
13650
|
+
// A claim only means something for as long as it outlives the work it covers.
|
|
13651
|
+
// A provider write can legitimately run past a fixed lease, at which point the
|
|
13652
|
+
// claim stops protecting the write it was taken for and a second handler can
|
|
13653
|
+
// post the same thing to the same human. Renewing on a heartbeat scopes the
|
|
13654
|
+
// lease to the work instead of to a guessed duration, and leaves the idle
|
|
13655
|
+
// timeout short enough that a holder that dies mid-write still frees it.
|
|
13656
|
+
//
|
|
13657
|
+
// The heartbeat is bounded on both ends, because a renewal loop that never
|
|
13658
|
+
// stops is a lock with no owner check: a provider write that hangs would hold
|
|
13659
|
+
// the receipt past every retry and past shutdown, and the human whose reply is
|
|
13660
|
+
// queued behind it would be told nothing until the process restarts. So it
|
|
13661
|
+
// stops at the ceiling and it stops when this daemon is stopping, and either
|
|
13662
|
+
// way it says so — from there the claim ages out on its own idle lease and
|
|
13663
|
+
// becomes reclaimable. The write may still land afterwards and duplicate the
|
|
13664
|
+
// notice; a reply nobody can ever reclaim is the worse of the two.
|
|
13665
|
+
async #withRenewedProviderLease(label, leaseMs, renew, run) {
|
|
13666
|
+
const renewUntilMs = this.#clock.now() + SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS;
|
|
13667
|
+
let renewalStopped = false;
|
|
13668
|
+
let renewalInFlight = false;
|
|
13669
|
+
const stopRenewing = (counter, reason) => {
|
|
13670
|
+
renewalStopped = true;
|
|
13671
|
+
this.#increment(counter);
|
|
13672
|
+
this.#logger.warn?.(`[factory] ${label} lease will not be renewed further (${reason}); ` +
|
|
13673
|
+
'its claim expires and the queued replies return to whoever retries them');
|
|
13674
|
+
};
|
|
13675
|
+
const heartbeat = setInterval(() => {
|
|
13676
|
+
if (renewalInFlight || renewalStopped)
|
|
13677
|
+
return;
|
|
13678
|
+
if (this.#stopping) {
|
|
13679
|
+
stopRenewing('slackProviderReceiptLeaseRenewalsStoppedForShutdown', 'shutting down');
|
|
13680
|
+
return;
|
|
13681
|
+
}
|
|
13682
|
+
if (this.#clock.now() >= renewUntilMs) {
|
|
13683
|
+
stopRenewing('slackProviderReceiptLeaseRenewalsExpired', `provider write exceeded ${SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS}ms`);
|
|
13684
|
+
return;
|
|
13685
|
+
}
|
|
13686
|
+
renewalInFlight = true;
|
|
13687
|
+
void renew()
|
|
13688
|
+
.then((renewed) => {
|
|
13689
|
+
if (renewed)
|
|
13690
|
+
return;
|
|
13691
|
+
// Losing the lease mid-write is not recoverable from in here: the
|
|
13692
|
+
// write may already have landed. Stop renewing and let the caller's
|
|
13693
|
+
// completion check fail closed, which keeps the queued replies for
|
|
13694
|
+
// whoever holds the claim now.
|
|
13695
|
+
renewalStopped = true;
|
|
13696
|
+
this.#increment('slackProviderReceiptLeasesLost');
|
|
13697
|
+
this.#logger.warn?.(`[factory] ${label} lease was lost while its provider write was in flight`);
|
|
13698
|
+
})
|
|
13699
|
+
.catch((error) => this.#logger.warn?.(`[factory] ${label} lease renewal failed`, {
|
|
13700
|
+
error: describeError(error).errorMessage,
|
|
13701
|
+
}))
|
|
13702
|
+
.finally(() => { renewalInFlight = false; });
|
|
13703
|
+
}, Math.max(1_000, Math.floor(leaseMs / 3)));
|
|
13704
|
+
heartbeat.unref?.();
|
|
13705
|
+
try {
|
|
13706
|
+
return await run();
|
|
13707
|
+
}
|
|
13708
|
+
finally {
|
|
13709
|
+
clearInterval(heartbeat);
|
|
13710
|
+
}
|
|
13711
|
+
}
|
|
13712
|
+
// A terminal receipt that could not be written leaves the queued replies
|
|
13713
|
+
// pending with the human who wrote them told nothing. That is retryable
|
|
13714
|
+
// maintenance this daemon owns, not work to leave for the next restart: the
|
|
13715
|
+
// grace watch is the only window in which the receipt can still land on the
|
|
13716
|
+
// retired thread, so keep reattempting inside it and give up when it closes.
|
|
13717
|
+
#scheduleSlackTerminalReceiptRetry(issue, threadId, expiresAtMs, attempt = 0) {
|
|
13718
|
+
if (this.#stopping)
|
|
13719
|
+
return;
|
|
13720
|
+
const key = issueKey(issue);
|
|
13721
|
+
const existing = this.#slackTerminalReceiptRetryTimers.get(key);
|
|
13722
|
+
if (existing)
|
|
13723
|
+
clearTimeout(existing);
|
|
13724
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
13725
|
+
const remainingMs = expiresAtMs - this.#clock.now();
|
|
13726
|
+
if (remainingMs <= 0) {
|
|
13727
|
+
this.#increment('slackTerminalWatchReceiptsAbandoned');
|
|
13728
|
+
return;
|
|
13729
|
+
}
|
|
13730
|
+
const backoffMs = Math.min(SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS, SLACK_TERMINAL_RECEIPT_RETRY_MS * 2 ** Math.min(attempt, 16));
|
|
13731
|
+
const timer = setTimeout(() => {
|
|
13732
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
13733
|
+
void this.#retrySlackTerminalReceipt(issue, threadId, attempt);
|
|
13734
|
+
}, Math.max(0, Math.min(backoffMs, remainingMs)));
|
|
13735
|
+
timer.unref?.();
|
|
13736
|
+
this.#slackTerminalReceiptRetryTimers.set(key, timer);
|
|
13737
|
+
}
|
|
13738
|
+
async #retrySlackTerminalReceipt(issue, threadId, attempt) {
|
|
13739
|
+
if (this.#stopping)
|
|
13740
|
+
return;
|
|
13741
|
+
const key = issueKey(issue);
|
|
13742
|
+
const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
13743
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
13744
|
+
// The grace watch is gone (expired, or the work unit reopened): the thread
|
|
13745
|
+
// this receipt would settle no longer exists, so there is nothing to say.
|
|
13746
|
+
if (watch?.kind !== 'terminal-grace' || watch.threadId !== threadId)
|
|
13747
|
+
return;
|
|
13748
|
+
try {
|
|
13749
|
+
await this.#surfaceUndeliveredSlackConversation(threadId);
|
|
13750
|
+
await this.#state.clearConversationSession(this.#workspaceId, slackConversationId(threadId));
|
|
13751
|
+
this.#increment('slackTerminalWatchReceiptsRecovered');
|
|
13752
|
+
}
|
|
13753
|
+
catch (error) {
|
|
13754
|
+
this.#logger.warn?.('[factory] terminal Slack receipt retry failed; rescheduling', {
|
|
13755
|
+
issue: issue.key,
|
|
13756
|
+
error,
|
|
13757
|
+
});
|
|
13758
|
+
this.#increment('slackTerminalWatchReceiptRetryFailures');
|
|
13759
|
+
this.#scheduleSlackTerminalReceiptRetry(issue, threadId, watch.expiresAtMs, attempt + 1);
|
|
13760
|
+
}
|
|
13761
|
+
}
|
|
13762
|
+
#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, retryDelayMs) {
|
|
13763
|
+
if (this.#stopping)
|
|
13764
|
+
return;
|
|
13765
|
+
const key = issueKey(issue);
|
|
13766
|
+
const existing = this.#slackTerminalWatchExpiryTimers.get(key);
|
|
13767
|
+
if (existing)
|
|
13768
|
+
clearTimeout(existing);
|
|
13769
|
+
const timer = setTimeout(() => {
|
|
13770
|
+
this.#slackTerminalWatchExpiryTimers.delete(key);
|
|
13771
|
+
void this.#expireSlackTerminalWatcher(issue, expiresAtMs).catch((error) => {
|
|
13772
|
+
this.#logger.warn?.('[factory] failed to expire terminal Slack reply watcher; retrying', {
|
|
13773
|
+
issue: issue.key,
|
|
13774
|
+
error,
|
|
13775
|
+
});
|
|
13776
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
|
|
13777
|
+
});
|
|
13778
|
+
}, retryDelayMs ?? Math.max(0, expiresAtMs - this.#clock.now()));
|
|
13779
|
+
timer.unref?.();
|
|
13780
|
+
this.#slackTerminalWatchExpiryTimers.set(key, timer);
|
|
13781
|
+
}
|
|
13782
|
+
async #expireSlackTerminalWatcher(issue, expiresAtMs) {
|
|
13783
|
+
const key = issueKey(issue);
|
|
13784
|
+
const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
13785
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
13786
|
+
if (watch?.kind !== 'terminal-grace' || watch.expiresAtMs !== expiresAtMs)
|
|
13787
|
+
return;
|
|
13788
|
+
if (watch.expiresAtMs > this.#clock.now()) {
|
|
13789
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, watch.expiresAtMs);
|
|
13790
|
+
return;
|
|
13791
|
+
}
|
|
13792
|
+
// #stopSlackWatcher fails closed when an in-flight reply route will not
|
|
13793
|
+
// drain, leaving the watch and its terminal fence in place. Counting that as
|
|
13794
|
+
// an expiration retires the watch in the metrics while the real one lives
|
|
13795
|
+
// on unwatched by any expiry timer, so reschedule and count only on success.
|
|
13796
|
+
if (!await this.#stopSlackWatcher(issue)) {
|
|
13797
|
+
this.#increment('slackTerminalWatchExpiriesDeferred');
|
|
13798
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
|
|
13799
|
+
return;
|
|
13800
|
+
}
|
|
13801
|
+
this.#increment('slackTerminalWatchersExpired');
|
|
13037
13802
|
}
|
|
13038
13803
|
async #readSlackReply(path) {
|
|
13039
13804
|
try {
|
|
@@ -13098,33 +13863,147 @@ export class FactoryLoop {
|
|
|
13098
13863
|
await this.#wakeWaitingClarification(clarificationKey, claimed);
|
|
13099
13864
|
return;
|
|
13100
13865
|
}
|
|
13866
|
+
return await this.#routeSlackConversationAnswer(record, reply, text, clarificationKey);
|
|
13867
|
+
}
|
|
13868
|
+
async #routeSlackConversationAnswer(record, reply, text, clarificationKey) {
|
|
13869
|
+
if (this.#slackReplyRouteDrains.has(clarificationKey)) {
|
|
13870
|
+
// The terminal fence for this work unit is being drained right now.
|
|
13871
|
+
// Registering an ordinary route here would put it past the drain's
|
|
13872
|
+
// snapshot and run it once the fence is gone. Answer it the way the fence
|
|
13873
|
+
// would have — but as a tracked effect, because this writeback is still a
|
|
13874
|
+
// side effect of this work unit. Left untracked it is the same escape one
|
|
13875
|
+
// level further in: the drain reports quiescence without it, the watcher
|
|
13876
|
+
// stop clears the retry timer that owns this reply, and a slow or failed
|
|
13877
|
+
// receipt leaves the human told nothing at all.
|
|
13878
|
+
this.#increment('slackReplyRoutesFencedDuringDrain');
|
|
13879
|
+
return await this.#trackSlackWorkUnitEffect(clarificationKey, async () => {
|
|
13880
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
13881
|
+
return undefined;
|
|
13882
|
+
});
|
|
13883
|
+
}
|
|
13884
|
+
return await this.#trackSlackWorkUnitEffect(clarificationKey, () => this.#routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey));
|
|
13885
|
+
}
|
|
13886
|
+
// Every Slack side effect a work unit makes on its own behalf runs through
|
|
13887
|
+
// here, so #slackReplyRoutes stays the single record the terminal drain
|
|
13888
|
+
// consults. Effects are chained per work unit: awaiting the newest one drains
|
|
13889
|
+
// everything queued behind it, and a rejection propagates to the drain, which
|
|
13890
|
+
// fails closed rather than tearing the effect's retry path down.
|
|
13891
|
+
async #trackSlackWorkUnitEffect(key, run) {
|
|
13892
|
+
const preceding = this.#slackReplyRoutes.get(key);
|
|
13893
|
+
const effect = (async () => {
|
|
13894
|
+
await preceding?.catch(() => undefined);
|
|
13895
|
+
return await run();
|
|
13896
|
+
})();
|
|
13897
|
+
this.#slackReplyRoutes.set(key, effect);
|
|
13898
|
+
try {
|
|
13899
|
+
return await effect;
|
|
13900
|
+
}
|
|
13901
|
+
finally {
|
|
13902
|
+
if (this.#slackReplyRoutes.get(key) === effect)
|
|
13903
|
+
this.#slackReplyRoutes.delete(key);
|
|
13904
|
+
}
|
|
13905
|
+
}
|
|
13906
|
+
async #routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey) {
|
|
13907
|
+
if (this.#terminalSlackWatchIssues.has(clarificationKey)) {
|
|
13908
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
13909
|
+
return;
|
|
13910
|
+
}
|
|
13101
13911
|
const conversationId = slackConversationId(reply.threadTs);
|
|
13102
|
-
|
|
13912
|
+
let conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13913
|
+
let liveRecord;
|
|
13914
|
+
if (!conversation) {
|
|
13915
|
+
liveRecord = (await this.#batch()).getIssue(record.issue);
|
|
13916
|
+
if (liveRecord && !liveRecord.dryRun) {
|
|
13917
|
+
await this.#ensureSlackConversationSession(liveRecord, reply.threadTs);
|
|
13918
|
+
conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13919
|
+
}
|
|
13920
|
+
}
|
|
13103
13921
|
if (conversation && issueKey(conversation.issue) === clarificationKey) {
|
|
13922
|
+
const replyId = `${reply.threadTs}:${reply.messageTs}`;
|
|
13104
13923
|
const queued = await this.#state.appendConversationMessage(this.#workspaceId, conversationId, {
|
|
13105
|
-
id:
|
|
13924
|
+
id: replyId,
|
|
13106
13925
|
text,
|
|
13107
13926
|
receivedAtMs: slackMessageReceivedAtMs(reply.messageTs, this.#clock.now()),
|
|
13108
13927
|
providerSequence: reply.messageTs,
|
|
13109
13928
|
author: reply.author,
|
|
13110
13929
|
});
|
|
13111
|
-
|
|
13930
|
+
const durable = queued ?? await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13931
|
+
if (!durable || !durable.processedMessageIds.includes(replyId)) {
|
|
13932
|
+
throw new Error(`Slack reply ${replyId} was not durably queued`);
|
|
13933
|
+
}
|
|
13934
|
+
if (!(durable.acknowledgedMessageIds ?? []).includes(replyId)) {
|
|
13935
|
+
const acknowledgementClaimId = randomUUID();
|
|
13936
|
+
const acknowledgementClaimed = await this.#state.claimConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now(), SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS);
|
|
13937
|
+
if (acknowledgementClaimed) {
|
|
13938
|
+
try {
|
|
13939
|
+
if (!this.#slack)
|
|
13940
|
+
throw new Error(`Slack reply ${replyId} cannot be acknowledged without writeback`);
|
|
13941
|
+
const owner = durable.agent?.role === 'babysitter'
|
|
13942
|
+
? 'the PR babysitter'
|
|
13943
|
+
: durable.agent
|
|
13944
|
+
? 'the issue implementer'
|
|
13945
|
+
: 'an issue agent';
|
|
13946
|
+
const receipt = durable.agent
|
|
13947
|
+
? `Factory received this reply and durably queued it for ${owner}.`
|
|
13948
|
+
: 'Factory received and durably stored this reply; it will route when an issue agent is resumable.';
|
|
13949
|
+
const slack = this.#slack;
|
|
13950
|
+
// Same lease scope as the terminal receipt: this claim covers a
|
|
13951
|
+
// provider write that can outrun any fixed duration, so it is
|
|
13952
|
+
// renewed for as long as that write is actually running.
|
|
13953
|
+
await this.#withRenewedProviderLease('Slack reply acknowledgement', SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS, () => this.#state.renewConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now()), () => slack.reply(reply.threadTs, receipt));
|
|
13954
|
+
if (!await this.#state.completeConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId)) {
|
|
13955
|
+
throw new Error(`Slack reply ${replyId} receipt could not be recorded`);
|
|
13956
|
+
}
|
|
13957
|
+
this.#increment('slackConversationRepliesAcknowledged');
|
|
13958
|
+
}
|
|
13959
|
+
catch (error) {
|
|
13960
|
+
await this.#state.releaseConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId);
|
|
13961
|
+
throw error;
|
|
13962
|
+
}
|
|
13963
|
+
}
|
|
13964
|
+
else {
|
|
13965
|
+
const acknowledgementState = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13966
|
+
if (!(acknowledgementState?.acknowledgedMessageIds ?? []).includes(replyId)) {
|
|
13967
|
+
throw new Error(`Slack reply ${replyId} receipt is claimed by another handler; retrying`);
|
|
13968
|
+
}
|
|
13969
|
+
}
|
|
13970
|
+
}
|
|
13971
|
+
if (queued) {
|
|
13972
|
+
this.#increment('slackConversationRepliesQueued');
|
|
13973
|
+
}
|
|
13974
|
+
else {
|
|
13112
13975
|
this.#increment('slackConversationDuplicateRepliesSuppressed');
|
|
13113
|
-
return;
|
|
13114
13976
|
}
|
|
13115
|
-
|
|
13116
|
-
|
|
13977
|
+
const pending = durable.pending.some((message) => message.id === replyId) ||
|
|
13978
|
+
Boolean(durable.delivery?.messages.some((message) => message.id === replyId));
|
|
13979
|
+
if (pending && durable.agent) {
|
|
13980
|
+
this.#slackConversationTurns.schedule(conversationId);
|
|
13981
|
+
}
|
|
13982
|
+
else if (pending) {
|
|
13983
|
+
this.#increment('slackConversationRepliesWaitingForOwner');
|
|
13984
|
+
}
|
|
13117
13985
|
return;
|
|
13118
13986
|
}
|
|
13119
|
-
|
|
13987
|
+
liveRecord ??= (await this.#batch()).getIssue(record.issue);
|
|
13120
13988
|
if (!liveRecord || liveRecord.dryRun) {
|
|
13121
13989
|
if (isTriageEscalationWatchRecord(record)) {
|
|
13122
13990
|
return await this.#handleTriageEscalationSlackAnswer(record, text);
|
|
13123
13991
|
}
|
|
13124
|
-
this.#
|
|
13992
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
13125
13993
|
return;
|
|
13126
13994
|
}
|
|
13127
13995
|
this.#increment('slackAnswersIgnoredNoConversationSession');
|
|
13996
|
+
if (this.#slack) {
|
|
13997
|
+
await this.#slack.reply(reply.threadTs, 'Factory received this reply but could not create a durable agent route. It will remain replayable; please also continue on the linked issue or pull request.');
|
|
13998
|
+
this.#increment('slackAnswersUnroutableVisible');
|
|
13999
|
+
}
|
|
14000
|
+
}
|
|
14001
|
+
async #writeUnroutableSlackReply(threadId) {
|
|
14002
|
+
this.#increment('slackAnswersIgnoredNoInFlight');
|
|
14003
|
+
if (!this.#slack)
|
|
14004
|
+
return;
|
|
14005
|
+
await this.#slack.reply(threadId, 'Factory received this reply but could not route it because this work unit no longer has an active agent. Please continue on the linked issue or pull request.');
|
|
14006
|
+
this.#increment('slackAnswersUnroutableVisible');
|
|
13128
14007
|
}
|
|
13129
14008
|
async #wakeWaitingClarification(key, waiting) {
|
|
13130
14009
|
const existing = this.#clarificationWakeInFlight.get(key);
|
|
@@ -13414,6 +14293,7 @@ export class FactoryLoop {
|
|
|
13414
14293
|
const resumed = await this.#fleet.resume({
|
|
13415
14294
|
name,
|
|
13416
14295
|
sessionRef: tracked.sessionRef,
|
|
14296
|
+
identityKey: dispatchAgentIdentityKey(waiting.issue, tracked.spec.role),
|
|
13417
14297
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
13418
14298
|
capability: tracked.spec.capability,
|
|
13419
14299
|
repo: tracked.spec.repo,
|
|
@@ -13444,6 +14324,7 @@ export class FactoryLoop {
|
|
|
13444
14324
|
return await this.#fleet.spawn({
|
|
13445
14325
|
name,
|
|
13446
14326
|
capability: tracked.spec.capability,
|
|
14327
|
+
identityKey: dispatchAgentIdentityKey(waiting.issue, tracked.spec.role),
|
|
13447
14328
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
13448
14329
|
task,
|
|
13449
14330
|
workflow: tracked.spec.workflow,
|
|
@@ -13494,6 +14375,7 @@ export class FactoryLoop {
|
|
|
13494
14375
|
const batch = await this.#batch();
|
|
13495
14376
|
if (batch.isInFlight(record.issue) || batch.isQueued(record.issue)) {
|
|
13496
14377
|
this.#increment('slackTriageAnswersIgnoredAlreadyActive');
|
|
14378
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
13497
14379
|
return;
|
|
13498
14380
|
}
|
|
13499
14381
|
if (await this.#dispatchBlockReason(record.issue)) {
|
|
@@ -13510,6 +14392,10 @@ export class FactoryLoop {
|
|
|
13510
14392
|
if (hasDispatchableRoute(decision)) {
|
|
13511
14393
|
this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
|
|
13512
14394
|
const result = await this.#startOrQueueSlackClarifiedDecision(dispatchAfterSlackClarification(decision, escalationReason));
|
|
14395
|
+
const active = await this.#batch();
|
|
14396
|
+
if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
|
|
14397
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
14398
|
+
}
|
|
13513
14399
|
this.#increment('slackTriageAnswersDispatchedWithRemainingEscalation');
|
|
13514
14400
|
return result;
|
|
13515
14401
|
}
|
|
@@ -13522,6 +14408,10 @@ export class FactoryLoop {
|
|
|
13522
14408
|
}
|
|
13523
14409
|
this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
|
|
13524
14410
|
const result = await this.#startOrQueueSlackClarifiedDecision(decision);
|
|
14411
|
+
const active = await this.#batch();
|
|
14412
|
+
if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
|
|
14413
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
14414
|
+
}
|
|
13525
14415
|
this.#increment('slackTriageAnswersDispatched');
|
|
13526
14416
|
return result;
|
|
13527
14417
|
}
|
|
@@ -13757,6 +14647,15 @@ export class FactoryLoop {
|
|
|
13757
14647
|
return hasTitlePrefix(issue.title, FACTORY_E2E_MARKER);
|
|
13758
14648
|
}
|
|
13759
14649
|
}
|
|
14650
|
+
const defaultGithubWriteback = (config, mount) => {
|
|
14651
|
+
if (config.github.identity !== 'app') {
|
|
14652
|
+
return new GhCliGithubWriteback();
|
|
14653
|
+
}
|
|
14654
|
+
if (!mount.githubWrite) {
|
|
14655
|
+
throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
|
|
14656
|
+
}
|
|
14657
|
+
return new AppGithubWriteback(mount.githubWrite);
|
|
14658
|
+
};
|
|
13760
14659
|
export function parseLinearIssue(path, content) {
|
|
13761
14660
|
const parsed = parseJsonContent(content);
|
|
13762
14661
|
const payload = wrappedPayload(parsed);
|
|
@@ -14352,13 +15251,19 @@ function labelDerivedDispatchDecision(liveIssue, decision, config) {
|
|
|
14352
15251
|
maxImplementers,
|
|
14353
15252
|
};
|
|
14354
15253
|
}
|
|
14355
|
-
|
|
14356
|
-
|
|
15254
|
+
// Swarm always shares one checkout (lead + workers collaborate live over a
|
|
15255
|
+
// shared relay channel), so — like single/workflow — only the first matched
|
|
15256
|
+
// label route is used; it is never fanned out across repo labels like team.
|
|
15257
|
+
const selectedRoutes = scope === 'single' || scope === 'swarm'
|
|
15258
|
+
? routesByLabel.routes.slice(0, 1)
|
|
15259
|
+
: routesByLabel.routes;
|
|
14357
15260
|
const selectedImplementers = scope === 'team'
|
|
14358
|
-
?
|
|
15261
|
+
? routesByLabel.routes.map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route))
|
|
14359
15262
|
: scope === 'single'
|
|
14360
|
-
?
|
|
14361
|
-
:
|
|
15263
|
+
? routesByLabel.routes.slice(0, 1).map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route))
|
|
15264
|
+
: scope === 'swarm'
|
|
15265
|
+
? routeSwarmImplementerSpecs(liveIssue, config, selectedRoutes[0]?.route, maxImplementers)
|
|
15266
|
+
: [];
|
|
14362
15267
|
const routes = selectedRoutes.map(({ route }) => route);
|
|
14363
15268
|
const workflow = scope === 'workflow'
|
|
14364
15269
|
? routeWorkflowSpec(liveIssue, config, selectedRoutes, decision.workflow)
|
|
@@ -14487,6 +15392,24 @@ function routeImplementerSpec(issue, config, slug, route) {
|
|
|
14487
15392
|
node: 'self',
|
|
14488
15393
|
};
|
|
14489
15394
|
}
|
|
15395
|
+
function routeSwarmImplementerSpecs(issue, config, route, maxImplementers) {
|
|
15396
|
+
if (!route) {
|
|
15397
|
+
return [];
|
|
15398
|
+
}
|
|
15399
|
+
const channel = swarmChannel(issue);
|
|
15400
|
+
return swarmMemberSlugs(maxImplementers).map((slug) => ({
|
|
15401
|
+
name: agentNameForRole(issue, 'impl', { repo: route.repo, discriminator: slug }),
|
|
15402
|
+
role: 'implementer',
|
|
15403
|
+
capability: config.agentCapabilities.implementer,
|
|
15404
|
+
model: config.models.implementer,
|
|
15405
|
+
task: swarmTaskFor(issue, route, slug, channel),
|
|
15406
|
+
repo: route.repo,
|
|
15407
|
+
clonePath: route.clonePath,
|
|
15408
|
+
channel,
|
|
15409
|
+
swarmRole: slug === 'lead' ? 'lead' : 'worker',
|
|
15410
|
+
node: 'self',
|
|
15411
|
+
}));
|
|
15412
|
+
}
|
|
14490
15413
|
function decisionWithLifecycleBranches(decision, runId, opts = {}) {
|
|
14491
15414
|
const implementerBranch = (spec) => {
|
|
14492
15415
|
const runSuffix = `-${runId.slice(0, 8)}`;
|
|
@@ -15633,12 +16556,123 @@ const isAllowedFactoryDraft = async (path, content, opts, mount, config) => {
|
|
|
15633
16556
|
if (/^\/slack\/channels\/[^/]+\/messages\/.+/u.test(path)) {
|
|
15634
16557
|
return true;
|
|
15635
16558
|
}
|
|
15636
|
-
if (
|
|
16559
|
+
if (await isAllowedFactoryGithubDraft(path, content, opts, mount, config))
|
|
15637
16560
|
return true;
|
|
16561
|
+
return false;
|
|
16562
|
+
};
|
|
16563
|
+
const isFactoryGithubAuthoredArtifactPath = (path) => /^\/github\/repos\/[^/]+\/[^/]+\/(?:pull-requests\/factory-[^/]+\.json|refs\/(?:factory\.json|refs%2Fheads%2Ffactory%2F[^/]+\.json)|pulls\/[1-9]\d*\/close\.json)$/iu.test(path);
|
|
16564
|
+
export const isAllowedFactoryGithubArtifactDraft = (path, opts) => opts?.guarded === true && isFactoryGithubAuthoredArtifactPath(path);
|
|
16565
|
+
const factoryGithubIssueWriteTarget = (path) => {
|
|
16566
|
+
const match = /^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/([1-9]\d*)(?:\.json|\/(comments|labels)\/([^/]+))$/iu.exec(path);
|
|
16567
|
+
if (!match?.[1] || !match[2] || !match[3])
|
|
16568
|
+
return undefined;
|
|
16569
|
+
const child = match[4];
|
|
16570
|
+
const filename = match[5];
|
|
16571
|
+
if (child === 'comments' && (!filename || !isFactoryGithubIssueCommentDraftName(filename)))
|
|
16572
|
+
return undefined;
|
|
16573
|
+
if (child === 'labels' && (!filename || !isFactoryGithubOperationDraftName(filename)))
|
|
16574
|
+
return undefined;
|
|
16575
|
+
try {
|
|
16576
|
+
return {
|
|
16577
|
+
owner: decodeURIComponent(match[1]),
|
|
16578
|
+
repo: decodeURIComponent(match[2]),
|
|
16579
|
+
number: Number(match[3]),
|
|
16580
|
+
kind: child === 'comments' ? 'comment' : child === 'labels' ? 'label-operation' : 'issue-update',
|
|
16581
|
+
};
|
|
16582
|
+
}
|
|
16583
|
+
catch {
|
|
16584
|
+
return undefined;
|
|
16585
|
+
}
|
|
16586
|
+
};
|
|
16587
|
+
const factoryGithubRepositoryLabelWriteTarget = (path) => {
|
|
16588
|
+
const match = /^\/github\/repos\/([^/]+)\/([^/]+)\/labels\/([^/]+)$/iu.exec(path);
|
|
16589
|
+
if (!match?.[1] || !match[2] || !match[3] || !isFactoryGithubOperationDraftName(match[3]))
|
|
16590
|
+
return undefined;
|
|
16591
|
+
try {
|
|
16592
|
+
return { owner: decodeURIComponent(match[1]), repo: decodeURIComponent(match[2]) };
|
|
16593
|
+
}
|
|
16594
|
+
catch {
|
|
16595
|
+
return undefined;
|
|
16596
|
+
}
|
|
16597
|
+
};
|
|
16598
|
+
const githubLifecycleLabel = (name) => Object.values(FACTORY_GITHUB_STATUS_LABELS).find((label) => label.name === name);
|
|
16599
|
+
const hasExactKeys = (value, keys) => {
|
|
16600
|
+
const actual = Object.keys(value).sort();
|
|
16601
|
+
const expected = [...keys].sort();
|
|
16602
|
+
return actual.length === expected.length && actual.every((key, index) => key === expected[index]);
|
|
16603
|
+
};
|
|
16604
|
+
const isAllowedFactoryGithubIssueWriteContent = (kind, content) => {
|
|
16605
|
+
const value = asRecord(content);
|
|
16606
|
+
if (!value)
|
|
16607
|
+
return false;
|
|
16608
|
+
if (kind === 'issue-update') {
|
|
16609
|
+
return hasExactKeys(value, ['state']) && value.state === 'closed';
|
|
16610
|
+
}
|
|
16611
|
+
if (kind === 'comment') {
|
|
16612
|
+
return hasExactKeys(value, ['body']) && typeof value.body === 'string' && value.body.trim().length > 0;
|
|
16613
|
+
}
|
|
16614
|
+
if (value.operation === 'add') {
|
|
16615
|
+
return hasExactKeys(value, ['labels', 'operation']) &&
|
|
16616
|
+
Array.isArray(value.labels) && value.labels.length === 1 && Boolean(githubLifecycleLabel(value.labels[0]));
|
|
16617
|
+
}
|
|
16618
|
+
return value.operation === 'remove' && hasExactKeys(value, ['label', 'operation']) && Boolean(githubLifecycleLabel(value.label));
|
|
16619
|
+
};
|
|
16620
|
+
const isAllowedFactoryGithubRepositoryLabelContent = (content) => {
|
|
16621
|
+
const value = asRecord(content);
|
|
16622
|
+
if (!value || !hasExactKeys(value, ['color', 'description', 'name']))
|
|
16623
|
+
return false;
|
|
16624
|
+
const expected = githubLifecycleLabel(value.name);
|
|
16625
|
+
return Boolean(expected && expected.color === value.color && expected.description === value.description);
|
|
16626
|
+
};
|
|
16627
|
+
/**
|
|
16628
|
+
* Last-resort mount guard for Factory-authored GitHub drafts. PR/ref paths are
|
|
16629
|
+
* intrinsically Factory-owned. Issue mutations additionally require a current
|
|
16630
|
+
* in-scope issue projection in one of Relayfile's supported repository layouts.
|
|
16631
|
+
*/
|
|
16632
|
+
export const isAllowedFactoryGithubDraft = async (path, content, opts, mount, config) => {
|
|
16633
|
+
if (!opts?.guarded)
|
|
16634
|
+
return false;
|
|
16635
|
+
if (isAllowedFactoryGithubArtifactDraft(path, opts))
|
|
16636
|
+
return true;
|
|
16637
|
+
const repositoryLabelTarget = factoryGithubRepositoryLabelWriteTarget(path);
|
|
16638
|
+
if (repositoryLabelTarget) {
|
|
16639
|
+
const repoPath = `/github/repos/${encodeURIComponent(repositoryLabelTarget.owner)}/${encodeURIComponent(repositoryLabelTarget.repo)}/`;
|
|
16640
|
+
return isConfiguredGithubRepoPath(repoPath, config) && isAllowedFactoryGithubRepositoryLabelContent(content);
|
|
16641
|
+
}
|
|
16642
|
+
const target = factoryGithubIssueWriteTarget(path);
|
|
16643
|
+
if (!target)
|
|
16644
|
+
return false;
|
|
16645
|
+
if (!isAllowedFactoryGithubIssueWriteContent(target.kind, content))
|
|
16646
|
+
return false;
|
|
16647
|
+
if (target.kind === 'comment') {
|
|
16648
|
+
const body = asRecord(content)?.body;
|
|
16649
|
+
const draftName = path.slice(path.lastIndexOf('/') + 1);
|
|
16650
|
+
if (typeof body !== 'string' || draftName !== factoryGithubIssueCommentDraftName(body))
|
|
16651
|
+
return false;
|
|
16652
|
+
}
|
|
16653
|
+
const repoPath = `/github/repos/${encodeURIComponent(target.owner)}/${encodeURIComponent(target.repo)}`;
|
|
16654
|
+
if (!isConfiguredGithubRepoPath(`${repoPath}/`, config))
|
|
16655
|
+
return false;
|
|
16656
|
+
const compactRepo = `${encodeURIComponent(target.owner)}__${encodeURIComponent(target.repo)}`;
|
|
16657
|
+
const candidates = [
|
|
16658
|
+
`${repoPath}/issues/by-id/${target.number}.json`,
|
|
16659
|
+
`/github/repos/${compactRepo}/issues/by-id/${target.number}.json`,
|
|
16660
|
+
`${repoPath}/issues/${target.number}/meta.json`,
|
|
16661
|
+
`/github/repos/${compactRepo}/issues/${target.number}/meta.json`,
|
|
16662
|
+
`${repoPath}/issues/${target.number}.json`,
|
|
16663
|
+
`/github/repos/${compactRepo}/issues/${target.number}.json`,
|
|
16664
|
+
];
|
|
16665
|
+
for (const candidate of candidates) {
|
|
16666
|
+
try {
|
|
16667
|
+
const issue = parseGithubFactoryIssue(candidate, (await mount.readFile(candidate)).content);
|
|
16668
|
+
return issue.state?.name === 'open' && isInFactoryScope(issue, config.safety);
|
|
16669
|
+
}
|
|
16670
|
+
catch {
|
|
16671
|
+
// Try the next canonical/alias shape. Any total miss fails closed.
|
|
16672
|
+
}
|
|
15638
16673
|
}
|
|
15639
16674
|
return false;
|
|
15640
16675
|
};
|
|
15641
|
-
const isFactoryGithubWritebackPath = (path) => /^\/github\/repos\/[^/]+\/[^/]+\/(?:pull-requests\/factory-[^/]+\.json|refs\/(?:factory\.json|refs%2Fheads%2Ffactory%2F[^/]+\.json)|pulls\/[1-9]\d*\/close\.json)$/iu.test(path);
|
|
15642
16676
|
const isIssuePathInFactoryScope = async (mount, path, config) => {
|
|
15643
16677
|
try {
|
|
15644
16678
|
return isInFactoryScope(parseLinearIssue(path, (await mount.readFile(path)).content), config.safety);
|
|
@@ -15888,8 +16922,8 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
|
|
|
15888
16922
|
}
|
|
15889
16923
|
return `${owner}/${repo}`;
|
|
15890
16924
|
};
|
|
15891
|
-
const githubPullRequestBody = (issue, preview) => [
|
|
15892
|
-
issue.description,
|
|
16925
|
+
const githubPullRequestBody = (issue, preview, sessionRef) => [
|
|
16926
|
+
stripTrajectoryPointers(issue.description),
|
|
15893
16927
|
'',
|
|
15894
16928
|
isGithubIssue(issue) && /^\d+$/u.test(issue.key)
|
|
15895
16929
|
? `Fixes #${issue.key}`
|
|
@@ -15899,7 +16933,25 @@ const githubPullRequestBody = (issue, preview) => [
|
|
|
15899
16933
|
`Live preview: ${preview.url}`,
|
|
15900
16934
|
'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
|
|
15901
16935
|
] : []),
|
|
16936
|
+
'',
|
|
16937
|
+
renderTrajectoryPointer({
|
|
16938
|
+
...trajectoryWorkUnitForIssue(issue),
|
|
16939
|
+
sessionRef,
|
|
16940
|
+
}),
|
|
15902
16941
|
].join('\n').trim();
|
|
16942
|
+
const trajectoryWorkUnitForIssue = (issue) => {
|
|
16943
|
+
const github = githubIssueSourceRef(issue);
|
|
16944
|
+
if (github) {
|
|
16945
|
+
return {
|
|
16946
|
+
workUnitId: `${github.owner}/${github.repo}#${github.number}`,
|
|
16947
|
+
workUnitSurface: 'github',
|
|
16948
|
+
};
|
|
16949
|
+
}
|
|
16950
|
+
if (isRealLinearIssue(issue)) {
|
|
16951
|
+
return { workUnitId: issue.key, workUnitSurface: 'linear' };
|
|
16952
|
+
}
|
|
16953
|
+
return { workUnitId: `factory:${issue.uuid}`, workUnitSurface: 'factory' };
|
|
16954
|
+
};
|
|
15903
16955
|
// The broker rejects re-registering a name it never released on exit
|
|
15904
16956
|
// (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
|
|
15905
16957
|
// the structured payload or the message so resume can treat it as terminal
|
|
@@ -15928,6 +16980,9 @@ const slackMessageReceivedAtMs = (messageTs, fallback) => {
|
|
|
15928
16980
|
const seconds = Number(messageTs);
|
|
15929
16981
|
return Number.isFinite(seconds) && seconds > 0 ? Math.floor(seconds * 1_000) : fallback;
|
|
15930
16982
|
};
|
|
16983
|
+
const terminalSlackWatchRetiredAtMs = (watch) => typeof watch.retiredAtMs === 'number' && Number.isFinite(watch.retiredAtMs)
|
|
16984
|
+
? watch.retiredAtMs
|
|
16985
|
+
: Math.max(0, watch.expiresAtMs - SLACK_TERMINAL_THREAD_GRACE_MS);
|
|
15931
16986
|
const eventIdentity = (event) => {
|
|
15932
16987
|
const record = event;
|
|
15933
16988
|
const rawId = record.id ?? record.event_id ?? record.seq;
|
|
@@ -16091,6 +17146,7 @@ const cloneTrackedAgent = (tracked) => ({
|
|
|
16091
17146
|
result: tracked.result ? { ...tracked.result } : undefined,
|
|
16092
17147
|
sessionRef: tracked.sessionRef,
|
|
16093
17148
|
unreachableWakeResumedSessionRef: tracked.unreachableWakeResumedSessionRef,
|
|
17149
|
+
releasedAtMs: tracked.releasedAtMs,
|
|
16094
17150
|
});
|
|
16095
17151
|
const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') => ({
|
|
16096
17152
|
spec: {
|
|
@@ -16167,7 +17223,11 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
16167
17223
|
decision: structuredClone(record.decision),
|
|
16168
17224
|
dryRun: record.dryRun,
|
|
16169
17225
|
phase,
|
|
16170
|
-
agents: [...record.agents].map(([name, tracked]) => ({
|
|
17226
|
+
agents: [...record.agents].map(([name, tracked]) => ({
|
|
17227
|
+
name,
|
|
17228
|
+
tracked: cloneTrackedAgent(tracked),
|
|
17229
|
+
...(tracked.releasedAtMs !== undefined ? { releasedAtMs: tracked.releasedAtMs } : {}),
|
|
17230
|
+
})),
|
|
16171
17231
|
invocationIds: [...record.invocationIds],
|
|
16172
17232
|
result: record.result ? structuredClone(record.result) : undefined,
|
|
16173
17233
|
...(record.dispatchClaim ? { dispatchClaim: { ...record.dispatchClaim } } : {}),
|
|
@@ -16182,7 +17242,13 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
|
16182
17242
|
issue: { ...lifecycle.issue },
|
|
16183
17243
|
decision: structuredClone(lifecycle.decision),
|
|
16184
17244
|
dryRun: lifecycle.dryRun,
|
|
16185
|
-
|
|
17245
|
+
// Release state has to survive takeover with the agents themselves. Without
|
|
17246
|
+
// it the rebuilt record reads a released placement as a live worker and the
|
|
17247
|
+
// spawn gate answers for a process that no longer exists.
|
|
17248
|
+
agents: new Map(lifecycle.agents.map((agent) => [agent.name, {
|
|
17249
|
+
...cloneTrackedAgent(agent.tracked),
|
|
17250
|
+
releasedAtMs: agent.releasedAtMs ?? agent.tracked.releasedAtMs,
|
|
17251
|
+
}])),
|
|
16186
17252
|
invocationIds: new Set(lifecycle.invocationIds),
|
|
16187
17253
|
result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
|
|
16188
17254
|
...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
|
|
@@ -16209,7 +17275,14 @@ const triageEscalationReason = (decision) => {
|
|
|
16209
17275
|
}
|
|
16210
17276
|
return `${reasons.join(' and ')}${decision.rationale ? `: ${decision.rationale}` : ''}`;
|
|
16211
17277
|
};
|
|
16212
|
-
|
|
17278
|
+
/**
|
|
17279
|
+
* The dispatch claim race: another writer changed the issue's live state
|
|
17280
|
+
* between this process reading it and writing back, so the dispatch was
|
|
17281
|
+
* abandoned. Exported so callers outside this module — notably the CLI, which
|
|
17282
|
+
* turns it into a distinct exit code — can recognize it by type rather than by
|
|
17283
|
+
* matching on `error.name`, and so tests can construct a genuine instance.
|
|
17284
|
+
*/
|
|
17285
|
+
export class LiveDispatchStateChangedError extends Error {
|
|
16213
17286
|
issueKey;
|
|
16214
17287
|
constructor(issueKey) {
|
|
16215
17288
|
super(`Live state changed before writeback for ${issueKey}`);
|
|
@@ -16217,6 +17290,59 @@ class LiveDispatchStateChangedError extends Error {
|
|
|
16217
17290
|
this.issueKey = issueKey;
|
|
16218
17291
|
}
|
|
16219
17292
|
}
|
|
17293
|
+
/** Whether a thrown value is a {@link LiveDispatchStateChangedError}. */
|
|
17294
|
+
export function isLiveDispatchStateChangedError(error) {
|
|
17295
|
+
return error instanceof LiveDispatchStateChangedError;
|
|
17296
|
+
}
|
|
17297
|
+
/** How deep to follow `cause` when classifying a wrapped failure. */
|
|
17298
|
+
const PASS_FATAL_CAUSE_DEPTH = 4;
|
|
17299
|
+
/**
|
|
17300
|
+
* Whether `error`, or anything it wraps, is an instance of `type`.
|
|
17301
|
+
* `contextualError` and the fleet control-plane guard both rethrow wrapped, so
|
|
17302
|
+
* classification has to follow the cause chain rather than trust the outermost
|
|
17303
|
+
* type.
|
|
17304
|
+
*/
|
|
17305
|
+
const wrapsErrorOfType = (error, type, depth = 0) => {
|
|
17306
|
+
if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
|
|
17307
|
+
return false;
|
|
17308
|
+
if (error instanceof type)
|
|
17309
|
+
return true;
|
|
17310
|
+
return wrapsErrorOfType(error.cause, type, depth + 1);
|
|
17311
|
+
};
|
|
17312
|
+
/**
|
|
17313
|
+
* How many *unclassified* per-item failures without an intervening successful
|
|
17314
|
+
* dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
|
|
17315
|
+
* a live-state race) never count toward it and never reset it: those
|
|
17316
|
+
* legitimately affect many units at once and are exactly the benign case #292
|
|
17317
|
+
* asks the loop to survive, so they are neither evidence of a pass-wide fault
|
|
17318
|
+
* nor evidence against one.
|
|
17319
|
+
*/
|
|
17320
|
+
const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
|
|
17321
|
+
/**
|
|
17322
|
+
* Failures the loop recognizes as belonging to one work unit. They are always
|
|
17323
|
+
* skippable and are exempt from the consecutive-failure fuse.
|
|
17324
|
+
*/
|
|
17325
|
+
const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
|
|
17326
|
+
error instanceof DispatchLifecycleClaimRefusedError;
|
|
17327
|
+
/**
|
|
17328
|
+
* The run-report reason recorded for a work unit the pass could not dispatch.
|
|
17329
|
+
*
|
|
17330
|
+
* `factory run-once` serializes the whole report to stdout, so this string is
|
|
17331
|
+
* a public surface: it stays a fixed classification plus an allowlisted error
|
|
17332
|
+
* class name, never raw provider text or filesystem paths. The full message
|
|
17333
|
+
* goes to the operator log instead, the same split
|
|
17334
|
+
* `describeControlPlaneError` makes for circuit state.
|
|
17335
|
+
*/
|
|
17336
|
+
const perItemDispatchSkipReason = (error) => {
|
|
17337
|
+
if (error instanceof LiveDispatchStateChangedError)
|
|
17338
|
+
return 'live state changed during dispatch';
|
|
17339
|
+
if (error instanceof DispatchLifecycleClaimRefusedError) {
|
|
17340
|
+
return error.refusal === 'terminal'
|
|
17341
|
+
? 'dispatch lifecycle already terminal'
|
|
17342
|
+
: 'dispatch lifecycle owned by another publisher';
|
|
17343
|
+
}
|
|
17344
|
+
return `dispatch failed (${telemetryErrorClass(error)})`;
|
|
17345
|
+
};
|
|
16220
17346
|
const triageEscalationQuestion = (decision, issue) => {
|
|
16221
17347
|
const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
|
|
16222
17348
|
const subject = issue?.title?.trim() || decision.issue.key;
|