@agent-relay/factory 0.1.60 → 0.1.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +140 -16
- package/dist/cli/exit-codes.d.ts +81 -0
- package/dist/cli/exit-codes.d.ts.map +1 -0
- package/dist/cli/exit-codes.js +122 -0
- package/dist/cli/exit-codes.js.map +1 -0
- package/dist/cli/fleet.d.ts +23 -2
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +201 -42
- package/dist/cli/fleet.js.map +1 -1
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/config/schema.d.ts +175 -52
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +24 -5
- package/dist/config/schema.js.map +1 -1
- package/dist/dispatch/templates.d.ts +7 -0
- package/dist/dispatch/templates.d.ts.map +1 -1
- package/dist/dispatch/templates.js +64 -12
- package/dist/dispatch/templates.js.map +1 -1
- package/dist/dispatch/work-unit-identity.d.ts +19 -0
- package/dist/dispatch/work-unit-identity.d.ts.map +1 -0
- package/dist/dispatch/work-unit-identity.js +40 -0
- package/dist/dispatch/work-unit-identity.js.map +1 -0
- package/dist/environments/stack-descriptor.d.ts +3 -3
- package/dist/fleet/control-plane-circuit.d.ts +57 -0
- package/dist/fleet/control-plane-circuit.d.ts.map +1 -0
- package/dist/fleet/control-plane-circuit.js +219 -0
- package/dist/fleet/control-plane-circuit.js.map +1 -0
- package/dist/fleet/internal-fleet-client.d.ts +2 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +11 -7
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +3 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +97 -18
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/github/writeback-paths.d.ts +7 -0
- package/dist/github/writeback-paths.d.ts.map +1 -0
- package/dist/github/writeback-paths.js +28 -0
- package/dist/github/writeback-paths.js.map +1 -0
- package/dist/hosted/orchestrator.d.ts.map +1 -1
- package/dist/hosted/orchestrator.js +2 -0
- package/dist/hosted/orchestrator.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/intake/index.d.ts +2 -1
- package/dist/intake/index.d.ts.map +1 -1
- package/dist/intake/index.js +2 -1
- package/dist/intake/index.js.map +1 -1
- package/dist/intake/notion-manifest.d.ts +48 -0
- package/dist/intake/notion-manifest.d.ts.map +1 -0
- package/dist/intake/notion-manifest.js +258 -0
- package/dist/intake/notion-manifest.js.map +1 -0
- package/dist/intake/notion-relay-claim.d.ts +5 -4
- package/dist/intake/notion-relay-claim.d.ts.map +1 -1
- package/dist/intake/notion-relay-claim.js +48 -7
- package/dist/intake/notion-relay-claim.js.map +1 -1
- package/dist/intake/notion.d.ts +6 -3
- package/dist/intake/notion.d.ts.map +1 -1
- package/dist/intake/notion.js +63 -13
- package/dist/intake/notion.js.map +1 -1
- package/dist/issue-key-match.d.ts +6 -0
- package/dist/issue-key-match.d.ts.map +1 -1
- package/dist/issue-key-match.js +6 -1
- package/dist/issue-key-match.js.map +1 -1
- package/dist/mount/local-mount-preflight.d.ts.map +1 -1
- package/dist/mount/local-mount-preflight.js +6 -2
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/mount-auth-error.d.ts +16 -3
- package/dist/mount/mount-auth-error.d.ts.map +1 -1
- package/dist/mount/mount-auth-error.js +34 -7
- package/dist/mount/mount-auth-error.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +14 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +107 -6
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/mount/relayfile-github-connection-write.d.ts +24 -2
- package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
- package/dist/mount/relayfile-github-connection-write.js +90 -1
- package/dist/mount/relayfile-github-connection-write.js.map +1 -1
- package/dist/node/factory-node.d.ts.map +1 -1
- package/dist/node/factory-node.js +18 -4
- package/dist/node/factory-node.js.map +1 -1
- package/dist/observability/events.d.ts +52 -52
- package/dist/orchestrator/batch-tracker.d.ts +30 -0
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js +50 -2
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +36 -2
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +863 -98
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +2 -2
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +2 -2
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/reaper.d.ts +16 -2
- package/dist/orchestrator/reaper.d.ts.map +1 -1
- package/dist/orchestrator/reaper.js +79 -5
- package/dist/orchestrator/reaper.js.map +1 -1
- package/dist/ports/fleet.d.ts +6 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -1
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +39 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +35 -6
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/document-store.d.ts +32 -0
- package/dist/state/document-store.d.ts.map +1 -0
- package/dist/state/document-store.js +7 -0
- package/dist/state/document-store.js.map +1 -0
- package/dist/state/file-state-store.d.ts +20 -3
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +89 -245
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +2 -1
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +12 -3
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/state/watch-state-document.d.ts +5 -0
- package/dist/state/watch-state-document.d.ts.map +1 -0
- package/dist/state/watch-state-document.js +444 -0
- package/dist/state/watch-state-document.js.map +1 -0
- package/dist/testing/deadline.d.ts +11 -0
- package/dist/testing/deadline.d.ts.map +1 -0
- package/dist/testing/deadline.js +23 -0
- package/dist/testing/deadline.js.map +1 -0
- package/dist/testing/fakes.d.ts +2 -0
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js.map +1 -1
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +1 -0
- package/dist/testing/index.js.map +1 -1
- package/dist/triage/heuristic.d.ts +5 -0
- package/dist/triage/heuristic.d.ts.map +1 -1
- package/dist/triage/heuristic.js +42 -3
- package/dist/triage/heuristic.js.map +1 -1
- package/dist/triage/index.d.ts +1 -1
- package/dist/triage/index.d.ts.map +1 -1
- package/dist/triage/index.js +1 -1
- package/dist/triage/index.js.map +1 -1
- package/dist/triage/schema.d.ts +73 -55
- package/dist/triage/schema.d.ts.map +1 -1
- package/dist/triage/schema.js +2 -1
- package/dist/triage/schema.js.map +1 -1
- package/dist/types.d.ts +51 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/github.d.ts +22 -7
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +86 -16
- package/dist/writeback/github.js.map +1 -1
- package/dist/writeback/index.d.ts +1 -1
- package/dist/writeback/index.d.ts.map +1 -1
- package/dist/writeback/index.js +1 -1
- package/dist/writeback/index.js.map +1 -1
- package/package.json +13 -8
|
@@ -5,6 +5,7 @@ import { FactoryConfigSchema } from '../config/schema.js';
|
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
8
|
+
import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftName, isFactoryGithubOperationDraftName, } from '../github/writeback-paths.js';
|
|
8
9
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
10
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
10
11
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
@@ -12,13 +13,14 @@ import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsT
|
|
|
12
13
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
13
14
|
import { isInFactoryScope } from '../safety/factory-scope.js';
|
|
14
15
|
import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
|
|
16
|
+
import { dispatchAgentIdentityKey } from '../dispatch/work-unit-identity.js';
|
|
15
17
|
import { deriveDescriptorsFromMount, prescriptiveInstructions, } from '@agent-relay/integration-prompts';
|
|
16
18
|
import { parseGithubHumanInputRequest, renderAgentTask, } from '../dispatch/templates.js';
|
|
17
19
|
import { resolveTestGuidance } from '../dispatch/test-guidance.js';
|
|
18
|
-
import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels } from '../triage/index.js';
|
|
20
|
+
import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels, swarmChannel, swarmMemberSlugs, swarmTaskFor } from '../triage/index.js';
|
|
19
21
|
import { agentNameForRole, sanitizeAgentSlug } from '../triage/agent-names.js';
|
|
20
22
|
import { isResourceSubscriptionsUnavailable } from '../subscriptions/index.js';
|
|
21
|
-
import { GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
|
|
23
|
+
import { AppGithubWriteback, FACTORY_GITHUB_STATUS_LABELS, GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
|
|
22
24
|
import { parseSlackThreadReply, slackThreadReplyGlob } from '../subscriptions/slack-filter.js';
|
|
23
25
|
import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared.js';
|
|
24
26
|
import { issueKey, } from './batch-tracker.js';
|
|
@@ -29,6 +31,7 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
|
29
31
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
30
32
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
31
33
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
34
|
+
import { FleetControlPlaneCircuit, FleetControlPlaneCircuitOpenError, guardFleetControlPlane, } from '../fleet/control-plane-circuit.js';
|
|
32
35
|
class ClarificationWakeLeaseLostError extends Error {
|
|
33
36
|
}
|
|
34
37
|
class ClarificationQuestionDeliveryLeaseLostError extends Error {
|
|
@@ -137,6 +140,8 @@ const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
|
137
140
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
138
141
|
const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
|
|
139
142
|
const DISPATCH_WRITEBACK_RETRY_MS = 250;
|
|
143
|
+
const HELD_PAST_DEADLINE_RELEASE_REASON = 'held-past-deadline';
|
|
144
|
+
const HELD_DEADLINE_OVERDUE_RETRY_MS = 1_000;
|
|
140
145
|
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
141
146
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
142
147
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
@@ -152,6 +157,7 @@ const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
|
|
|
152
157
|
const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
|
|
153
158
|
const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
|
|
154
159
|
const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
160
|
+
const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
|
|
155
161
|
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
156
162
|
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
157
163
|
const GITHUB_FACTORY_LABEL = 'factory';
|
|
@@ -221,6 +227,7 @@ export class FactoryLoop {
|
|
|
221
227
|
#mount;
|
|
222
228
|
#states;
|
|
223
229
|
#fleet;
|
|
230
|
+
#fleetControlPlane;
|
|
224
231
|
#ticketDispatchDelivery;
|
|
225
232
|
#triage;
|
|
226
233
|
#linear;
|
|
@@ -310,6 +317,9 @@ export class FactoryLoop {
|
|
|
310
317
|
#dispatchClaimStatuses = new Map();
|
|
311
318
|
#localReleaseCheckpoints = new Map();
|
|
312
319
|
#dispatchLifecycleRenewTimer;
|
|
320
|
+
#heldAgentDeadlineTimer;
|
|
321
|
+
#heldAgentDeadlineDueAtMs;
|
|
322
|
+
#heldAgentDeadlineSweepInFlight;
|
|
313
323
|
#clarificationSweepTimer;
|
|
314
324
|
#clarificationSweepDueAtMs;
|
|
315
325
|
#clarificationSweepInFlight;
|
|
@@ -341,6 +351,12 @@ export class FactoryLoop {
|
|
|
341
351
|
#readinessReconcileTimer;
|
|
342
352
|
#readinessReconcileInFlight;
|
|
343
353
|
#readinessReconcileIntervalMs = 60_000;
|
|
354
|
+
#readinessReconcileConsecutiveFailures = 0;
|
|
355
|
+
#readinessReconcileLastDurationMs;
|
|
356
|
+
#readinessReconcileLastStartedAtMs;
|
|
357
|
+
#readinessReconcileLastCompletedAtMs;
|
|
358
|
+
#readinessReconcileLastFailureAtMs;
|
|
359
|
+
#readinessReconcileLastError;
|
|
344
360
|
#liveEventQueue = [];
|
|
345
361
|
#liveEventDrainScheduled = false;
|
|
346
362
|
#liveEventDrainActive = false;
|
|
@@ -419,7 +435,9 @@ export class FactoryLoop {
|
|
|
419
435
|
#runOnceInFlightDryRun;
|
|
420
436
|
#discoverySession;
|
|
421
437
|
#discoverySweepEpoch;
|
|
438
|
+
#discoverySweepStartedAtMs;
|
|
422
439
|
#discoverySweepRenewTimer;
|
|
440
|
+
#discoverySweepRenewalInFlight;
|
|
423
441
|
#discoverySweepLeaseLost = false;
|
|
424
442
|
#discoveryOverloadError;
|
|
425
443
|
#resolvedIssueSource;
|
|
@@ -438,7 +456,6 @@ export class FactoryLoop {
|
|
|
438
456
|
// synced records (state.name but no state.id) without the states catalog.
|
|
439
457
|
this.#states = ports.stateResolution ?? stateResolutionFromIds(config.stateIds, config.linear.states);
|
|
440
458
|
installFactoryDraftPredicate(this.#mount, config);
|
|
441
|
-
this.#fleet = ports.fleet;
|
|
442
459
|
this.#ticketDispatchDelivery = ports.ticketDispatchDelivery ?? createTicketDispatchDelivery({
|
|
443
460
|
mountRoot: config.localMountRoot,
|
|
444
461
|
});
|
|
@@ -447,7 +464,7 @@ export class FactoryLoop {
|
|
|
447
464
|
safety: config.safety,
|
|
448
465
|
});
|
|
449
466
|
this.#githubWritebackProvided = Boolean(ports.githubWriteback);
|
|
450
|
-
this.#githubWriteback = ports.githubWriteback ??
|
|
467
|
+
this.#githubWriteback = ports.githubWriteback ?? defaultGithubWriteback(config, ports.mount);
|
|
451
468
|
this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
|
|
452
469
|
this.#github = ports.github ?? MountGithubRead(ports.mount);
|
|
453
470
|
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
@@ -469,6 +486,13 @@ export class FactoryLoop {
|
|
|
469
486
|
this.#probePrResolver = ports.probePrResolver ?? ((issue) => this.#resolveIssuePr(issue));
|
|
470
487
|
this.#logger = normalizeLogger(ports.logger ?? console);
|
|
471
488
|
this.#clock = ports.clock ?? realClock;
|
|
489
|
+
this.#fleetControlPlane = new FleetControlPlaneCircuit({
|
|
490
|
+
timeoutMs: config.fleetHealth.rosterTimeoutMs,
|
|
491
|
+
failureThreshold: config.fleetHealth.failureThreshold,
|
|
492
|
+
resetTimeoutMs: config.fleetHealth.resetTimeoutMs,
|
|
493
|
+
now: () => this.#clock.now(),
|
|
494
|
+
});
|
|
495
|
+
this.#fleet = guardFleetControlPlane(ports.fleet, this.#fleetControlPlane);
|
|
472
496
|
this.#processIdentityReader = ports.processIdentityReader ?? readProcessIdentity;
|
|
473
497
|
this.#processFinder = ports.processFinder ?? ((agentName, opts) => findAgentProcessByName(agentName, {
|
|
474
498
|
readProcessIdentity: this.#processIdentityReader,
|
|
@@ -576,11 +600,32 @@ export class FactoryLoop {
|
|
|
576
600
|
this.#batchView = batch;
|
|
577
601
|
return batch;
|
|
578
602
|
}
|
|
603
|
+
/**
|
|
604
|
+
* Resolves once this issue's durable dispatch row reaches a terminal phase,
|
|
605
|
+
* and reports which one. A caller that turns the run into an exit code needs
|
|
606
|
+
* the phase: a dispatch that hit capacity returns an empty hold result and
|
|
607
|
+
* schedules a durable retry, so the pre-wait result says nothing about how
|
|
608
|
+
* the run actually ended.
|
|
609
|
+
*
|
|
610
|
+
* `undefined` means no terminal phase was observed — there was no lifecycle
|
|
611
|
+
* row to wait on, or the wait ended because Factory is stopping.
|
|
612
|
+
*/
|
|
579
613
|
async waitForDispatchTerminal(issue) {
|
|
580
614
|
const key = issueKey(issue);
|
|
581
615
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
582
|
-
|
|
583
|
-
|
|
616
|
+
// No durable row means this dispatch never claimed a lifecycle: a
|
|
617
|
+
// dependency park, a triage escalation, and a label refusal all return
|
|
618
|
+
// before the claim. Nothing can ever become terminal, so polling would
|
|
619
|
+
// never stop and the caller would never produce an exit code at all.
|
|
620
|
+
if (!lifecycle)
|
|
621
|
+
return undefined;
|
|
622
|
+
if (isTerminalDispatchLifecycle(lifecycle))
|
|
623
|
+
return lifecycle.phase;
|
|
624
|
+
// Capture the phase at the moment this waiter observes it. The waiters are
|
|
625
|
+
// shared across callers of one row and carry no payload, so a re-read after
|
|
626
|
+
// the fact can race lifecycle cleanup or a reopened dispatch for the same
|
|
627
|
+
// issue and report a different run — or none.
|
|
628
|
+
let observedPhase;
|
|
584
629
|
await new Promise((resolve) => {
|
|
585
630
|
let settled = false;
|
|
586
631
|
let timer;
|
|
@@ -589,10 +634,11 @@ export class FactoryLoop {
|
|
|
589
634
|
waiters = new Set();
|
|
590
635
|
this.#dispatchTerminalWaiters.set(key, waiters);
|
|
591
636
|
}
|
|
592
|
-
const finish = () => {
|
|
637
|
+
const finish = (phase) => {
|
|
593
638
|
if (settled)
|
|
594
639
|
return;
|
|
595
640
|
settled = true;
|
|
641
|
+
observedPhase = phase;
|
|
596
642
|
if (timer)
|
|
597
643
|
clearTimeout(timer);
|
|
598
644
|
const current = this.#dispatchTerminalWaiters.get(key);
|
|
@@ -615,7 +661,7 @@ export class FactoryLoop {
|
|
|
615
661
|
try {
|
|
616
662
|
const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
617
663
|
if (latest && isTerminalDispatchLifecycle(latest)) {
|
|
618
|
-
this.#resolveDispatchTerminalWaiters(issue);
|
|
664
|
+
this.#resolveDispatchTerminalWaiters(issue, latest.phase);
|
|
619
665
|
return;
|
|
620
666
|
}
|
|
621
667
|
if (latest?.phase === 'waiting-for-human' && this.#startMode === 'dispatch-owner') {
|
|
@@ -645,6 +691,10 @@ export class FactoryLoop {
|
|
|
645
691
|
};
|
|
646
692
|
void poll();
|
|
647
693
|
});
|
|
694
|
+
// `observedPhase` is whatever resolved THIS waiter. It stays undefined only
|
|
695
|
+
// when the wait ended without a terminal resolution at all — Factory is
|
|
696
|
+
// stopping — which is exactly what `undefined` reports.
|
|
697
|
+
return observedPhase;
|
|
648
698
|
}
|
|
649
699
|
async start(opts = {}) {
|
|
650
700
|
if (this.#started) {
|
|
@@ -785,6 +835,11 @@ export class FactoryLoop {
|
|
|
785
835
|
if (this.#dispatchLifecycleRenewTimer)
|
|
786
836
|
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
787
837
|
this.#dispatchLifecycleRenewTimer = undefined;
|
|
838
|
+
if (this.#heldAgentDeadlineTimer)
|
|
839
|
+
clearTimeout(this.#heldAgentDeadlineTimer);
|
|
840
|
+
this.#heldAgentDeadlineTimer = undefined;
|
|
841
|
+
this.#heldAgentDeadlineDueAtMs = undefined;
|
|
842
|
+
await this.#heldAgentDeadlineSweepInFlight;
|
|
788
843
|
for (const timer of this.#dispatchLifecycleRetryTimers.values())
|
|
789
844
|
clearTimeout(timer);
|
|
790
845
|
this.#dispatchLifecycleRetryTimers.clear();
|
|
@@ -1122,24 +1177,40 @@ export class FactoryLoop {
|
|
|
1122
1177
|
this.#readinessReconcileTimer.unref?.();
|
|
1123
1178
|
}
|
|
1124
1179
|
async #reconcileReadyIssues() {
|
|
1180
|
+
const startedAtMs = this.#clock.now();
|
|
1181
|
+
this.#readinessReconcileLastStartedAtMs = startedAtMs;
|
|
1125
1182
|
this.#increment('readinessReconcileSweeps');
|
|
1126
1183
|
this.#logger.info?.('[factory] periodic readiness reconciliation started', {
|
|
1127
1184
|
intervalMs: this.#readinessReconcileIntervalMs,
|
|
1128
1185
|
});
|
|
1129
1186
|
try {
|
|
1130
1187
|
const report = await this.runOnce();
|
|
1188
|
+
this.#readinessReconcileConsecutiveFailures = 0;
|
|
1189
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1190
|
+
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1191
|
+
this.#readinessReconcileLastError = undefined;
|
|
1131
1192
|
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1193
|
+
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1132
1194
|
candidates: report.pulled.length,
|
|
1133
1195
|
dispatched: report.dispatched.length,
|
|
1134
1196
|
skipped: report.skipped.length,
|
|
1135
1197
|
});
|
|
1136
1198
|
}
|
|
1137
1199
|
catch (error) {
|
|
1200
|
+
const errorMessage = describeError(error).errorMessage;
|
|
1201
|
+
this.#readinessReconcileConsecutiveFailures += 1;
|
|
1202
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1203
|
+
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1204
|
+
this.#readinessReconcileLastError = errorMessage;
|
|
1138
1205
|
this.#increment('readinessReconcileErrors');
|
|
1139
1206
|
this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
|
|
1140
|
-
error:
|
|
1207
|
+
error: errorMessage,
|
|
1208
|
+
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1209
|
+
consecutiveFailures: this.#readinessReconcileConsecutiveFailures,
|
|
1210
|
+
degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
1141
1211
|
});
|
|
1142
1212
|
}
|
|
1213
|
+
await this.#refreshLiveHeartbeat();
|
|
1143
1214
|
}
|
|
1144
1215
|
#scheduleLivePoll(delayMs, options) {
|
|
1145
1216
|
if (this.#livePollTimer || !this.#started)
|
|
@@ -1796,7 +1867,30 @@ export class FactoryLoop {
|
|
|
1796
1867
|
}
|
|
1797
1868
|
}
|
|
1798
1869
|
}
|
|
1870
|
+
async #assertFleetControlPlaneAvailable() {
|
|
1871
|
+
try {
|
|
1872
|
+
await this.#fleet.roster();
|
|
1873
|
+
this.#increment('fleetControlPlaneProbeSuccesses');
|
|
1874
|
+
}
|
|
1875
|
+
catch (error) {
|
|
1876
|
+
const health = this.#fleetControlPlane.status();
|
|
1877
|
+
this.#increment('fleetControlPlaneProbeFailures');
|
|
1878
|
+
if (health.state === 'open')
|
|
1879
|
+
this.#increment('fleetControlPlaneCircuitOpen');
|
|
1880
|
+
this.#logger.error?.('[factory] fleet control plane unavailable; dispatch paused', {
|
|
1881
|
+
state: health.state,
|
|
1882
|
+
consecutiveFailures: health.consecutiveFailures,
|
|
1883
|
+
retryAtMs: health.retryAtMs,
|
|
1884
|
+
error: health.lastError ?? 'unknown control-plane failure',
|
|
1885
|
+
});
|
|
1886
|
+
throw contextualError('Factory dispatch paused because the fleet control plane is unavailable', error);
|
|
1887
|
+
}
|
|
1888
|
+
}
|
|
1799
1889
|
async #runOnceWithDiscoveryFence(opts) {
|
|
1890
|
+
const sweepStartedAtMs = this.#clock.now();
|
|
1891
|
+
if (!(opts.dryRun ?? this.#config.dryRun)) {
|
|
1892
|
+
await this.#assertFleetControlPlaneAvailable();
|
|
1893
|
+
}
|
|
1800
1894
|
let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1801
1895
|
if (!claim.acquired && claim.reason === 'backoff') {
|
|
1802
1896
|
const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
|
|
@@ -1812,6 +1906,8 @@ export class FactoryLoop {
|
|
|
1812
1906
|
if (!claim.acquired || !claim.lease) {
|
|
1813
1907
|
this.#increment('discoverySweepsSkippedInFlight');
|
|
1814
1908
|
this.#logger.info?.('[factory] skipped discovery because another process owns the sweep lease', {
|
|
1909
|
+
owner: claim.state.lease?.owner,
|
|
1910
|
+
epoch: claim.state.lease?.epoch,
|
|
1815
1911
|
leaseUntilMs: claim.state.lease?.leaseUntilMs,
|
|
1816
1912
|
});
|
|
1817
1913
|
return {
|
|
@@ -1823,7 +1919,23 @@ export class FactoryLoop {
|
|
|
1823
1919
|
discoveryDeferred: 'sweep-in-flight',
|
|
1824
1920
|
};
|
|
1825
1921
|
}
|
|
1922
|
+
if (claim.reclaimedLease) {
|
|
1923
|
+
this.#increment('discoverySweepOrphanTakeovers');
|
|
1924
|
+
this.#logger.warn?.('[factory] reclaimed discovery sweep lease from a stopped process', {
|
|
1925
|
+
owner: claim.lease.owner,
|
|
1926
|
+
epoch: claim.lease.epoch,
|
|
1927
|
+
previousOwner: claim.reclaimedLease.owner,
|
|
1928
|
+
previousEpoch: claim.reclaimedLease.epoch,
|
|
1929
|
+
previousLeaseUntilMs: claim.reclaimedLease.leaseUntilMs,
|
|
1930
|
+
});
|
|
1931
|
+
}
|
|
1932
|
+
this.#logger.info?.('[factory] discovery sweep lease claimed', {
|
|
1933
|
+
owner: claim.lease.owner,
|
|
1934
|
+
epoch: claim.lease.epoch,
|
|
1935
|
+
leaseUntilMs: claim.lease.leaseUntilMs,
|
|
1936
|
+
});
|
|
1826
1937
|
this.#discoverySweepEpoch = claim.lease.epoch;
|
|
1938
|
+
this.#discoverySweepStartedAtMs = sweepStartedAtMs;
|
|
1827
1939
|
this.#discoverySweepLeaseLost = false;
|
|
1828
1940
|
this.#discoveryOverloadError = undefined;
|
|
1829
1941
|
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
@@ -1834,6 +1946,11 @@ export class FactoryLoop {
|
|
|
1834
1946
|
if (this.#discoveryOverloadError)
|
|
1835
1947
|
throw this.#discoveryOverloadError;
|
|
1836
1948
|
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
1949
|
+
// Do not clear the durable lease while a renewal can still be waiting on
|
|
1950
|
+
// the same state-file lock. A late renewal that observes the completed
|
|
1951
|
+
// (lease-less) checkpoint is a false lease-loss signal and can poison an
|
|
1952
|
+
// otherwise successful reconcile cycle.
|
|
1953
|
+
await this.#stopDiscoverySweepRenewal();
|
|
1837
1954
|
if (this.#discoverySweepLeaseLost) {
|
|
1838
1955
|
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
1839
1956
|
}
|
|
@@ -1841,9 +1958,16 @@ export class FactoryLoop {
|
|
|
1841
1958
|
leaseReleased = completed;
|
|
1842
1959
|
if (!completed)
|
|
1843
1960
|
throw new Error('discovery sweep lease was lost before completion');
|
|
1961
|
+
this.#logger.info?.('[factory] discovery sweep checkpoint committed', {
|
|
1962
|
+
owner: claim.lease.owner,
|
|
1963
|
+
epoch: claim.lease.epoch,
|
|
1964
|
+
durationMs: this.#elapsedSince(sweepStartedAtMs),
|
|
1965
|
+
checkpointUpdatedAtMs: checkpoint?.updatedAtMs,
|
|
1966
|
+
});
|
|
1844
1967
|
return report;
|
|
1845
1968
|
}
|
|
1846
1969
|
catch (error) {
|
|
1970
|
+
await this.#stopDiscoverySweepRenewal();
|
|
1847
1971
|
const overload = relayfileOverload(error);
|
|
1848
1972
|
if (overload) {
|
|
1849
1973
|
const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
|
|
@@ -1868,11 +1992,10 @@ export class FactoryLoop {
|
|
|
1868
1992
|
throw error;
|
|
1869
1993
|
}
|
|
1870
1994
|
finally {
|
|
1871
|
-
|
|
1872
|
-
clearInterval(this.#discoverySweepRenewTimer);
|
|
1873
|
-
this.#discoverySweepRenewTimer = undefined;
|
|
1995
|
+
await this.#stopDiscoverySweepRenewal();
|
|
1874
1996
|
this.#discoverySession = undefined;
|
|
1875
1997
|
this.#discoverySweepEpoch = undefined;
|
|
1998
|
+
this.#discoverySweepStartedAtMs = undefined;
|
|
1876
1999
|
this.#discoveryOverloadError = undefined;
|
|
1877
2000
|
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
1878
2001
|
// a stale `true` here would otherwise make every #listRelayfileTree
|
|
@@ -2080,27 +2203,69 @@ export class FactoryLoop {
|
|
|
2080
2203
|
}
|
|
2081
2204
|
}
|
|
2082
2205
|
#startDiscoverySweepRenewal(epoch) {
|
|
2083
|
-
let renewalInFlight = false;
|
|
2084
2206
|
this.#discoverySweepRenewTimer = setInterval(() => {
|
|
2085
|
-
if (
|
|
2207
|
+
if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
|
|
2086
2208
|
return;
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
this.#
|
|
2092
|
-
this.#logger.warn?.('[factory] discovery sweep lease renewal was rejected; fencing further tree requests');
|
|
2209
|
+
const renewal = this.#renewDiscoverySweepLease(epoch);
|
|
2210
|
+
this.#discoverySweepRenewalInFlight = renewal;
|
|
2211
|
+
void renewal.finally(() => {
|
|
2212
|
+
if (this.#discoverySweepRenewalInFlight === renewal) {
|
|
2213
|
+
this.#discoverySweepRenewalInFlight = undefined;
|
|
2093
2214
|
}
|
|
2094
|
-
}).catch((error) => {
|
|
2095
|
-
this.#logger.warn?.('[factory] discovery sweep lease renewal failed; retaining the current lease window', {
|
|
2096
|
-
error: describeError(error).errorMessage,
|
|
2097
|
-
});
|
|
2098
|
-
}).finally(() => {
|
|
2099
|
-
renewalInFlight = false;
|
|
2100
2215
|
});
|
|
2101
2216
|
}, DISCOVERY_SWEEP_RENEW_MS);
|
|
2102
2217
|
this.#discoverySweepRenewTimer.unref?.();
|
|
2103
2218
|
}
|
|
2219
|
+
async #renewDiscoverySweepLease(epoch) {
|
|
2220
|
+
const requestedAtMs = this.#clock.now();
|
|
2221
|
+
try {
|
|
2222
|
+
let renewal;
|
|
2223
|
+
if (this.#state.renewDiscoverySweepWithDetails) {
|
|
2224
|
+
renewal = await this.#state.renewDiscoverySweepWithDetails(this.#workspaceId, this.#discoverySweepOwner, epoch, requestedAtMs, DISCOVERY_SWEEP_LEASE_MS);
|
|
2225
|
+
}
|
|
2226
|
+
else {
|
|
2227
|
+
const renewed = await this.#state.renewDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, requestedAtMs, DISCOVERY_SWEEP_LEASE_MS);
|
|
2228
|
+
renewal = renewed
|
|
2229
|
+
? {
|
|
2230
|
+
renewed: true,
|
|
2231
|
+
lease: {
|
|
2232
|
+
owner: this.#discoverySweepOwner,
|
|
2233
|
+
epoch,
|
|
2234
|
+
leaseUntilMs: requestedAtMs + DISCOVERY_SWEEP_LEASE_MS,
|
|
2235
|
+
},
|
|
2236
|
+
}
|
|
2237
|
+
: { renewed: false, reason: 'unknown' };
|
|
2238
|
+
}
|
|
2239
|
+
if (!renewal.renewed && this.#discoverySweepEpoch === epoch) {
|
|
2240
|
+
this.#discoverySweepLeaseLost = true;
|
|
2241
|
+
this.#increment('discoverySweepLeaseLosses');
|
|
2242
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal was rejected; fencing further tree requests', {
|
|
2243
|
+
reason: renewal.reason,
|
|
2244
|
+
requestedOwner: this.#discoverySweepOwner,
|
|
2245
|
+
requestedEpoch: epoch,
|
|
2246
|
+
requestedAtMs,
|
|
2247
|
+
elapsedMs: this.#elapsedSince(this.#discoverySweepStartedAtMs ?? requestedAtMs),
|
|
2248
|
+
observedOwner: renewal.observedLease?.owner,
|
|
2249
|
+
observedEpoch: renewal.observedLease?.epoch,
|
|
2250
|
+
observedLeaseUntilMs: renewal.observedLease?.leaseUntilMs,
|
|
2251
|
+
});
|
|
2252
|
+
}
|
|
2253
|
+
}
|
|
2254
|
+
catch (error) {
|
|
2255
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal failed; retaining the current lease window', {
|
|
2256
|
+
owner: this.#discoverySweepOwner,
|
|
2257
|
+
epoch,
|
|
2258
|
+
requestedAtMs,
|
|
2259
|
+
error: describeError(error).errorMessage,
|
|
2260
|
+
});
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
async #stopDiscoverySweepRenewal() {
|
|
2264
|
+
if (this.#discoverySweepRenewTimer)
|
|
2265
|
+
clearInterval(this.#discoverySweepRenewTimer);
|
|
2266
|
+
this.#discoverySweepRenewTimer = undefined;
|
|
2267
|
+
await this.#discoverySweepRenewalInFlight;
|
|
2268
|
+
}
|
|
2104
2269
|
async #prepareDiscoverySession(claim) {
|
|
2105
2270
|
const checkpoint = structuredClone(claim.state.checkpoint ?? {
|
|
2106
2271
|
trees: {},
|
|
@@ -2592,7 +2757,7 @@ export class FactoryLoop {
|
|
|
2592
2757
|
await this.#state.clearBabysitterSession(this.#workspaceId, issueKey(lifecycle.issue));
|
|
2593
2758
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2594
2759
|
this.#abandonedDispatchReasons.delete(key);
|
|
2595
|
-
this.#resolveDispatchTerminalWaiters(lifecycle.issue);
|
|
2760
|
+
this.#resolveDispatchTerminalWaiters(lifecycle.issue, 'abandoned');
|
|
2596
2761
|
await this.#writeInFlightRegistry().catch((error) => {
|
|
2597
2762
|
this.#logger.warn?.('[factory] failed to rewrite registry after orphaned claim release', {
|
|
2598
2763
|
issue: issue.key,
|
|
@@ -2947,6 +3112,16 @@ export class FactoryLoop {
|
|
|
2947
3112
|
reports.push(failedIterationReport(error, opts.dryRun ?? this.#config.dryRun));
|
|
2948
3113
|
await this.#reapDispatchFailureHandoffsNow(heartbeatPath, registryPath);
|
|
2949
3114
|
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'running', iteration + 1, maxIterations);
|
|
3115
|
+
const fleetControlPlane = this.#fleetControlPlane.status();
|
|
3116
|
+
if (fleetControlPlane.state !== 'closed') {
|
|
3117
|
+
this.#increment('loopCircuitBreaks');
|
|
3118
|
+
this.#logger.error?.('[factory] stopping loop because dispatch is paused by the fleet control-plane circuit', {
|
|
3119
|
+
state: fleetControlPlane.state,
|
|
3120
|
+
consecutiveFailures: fleetControlPlane.consecutiveFailures,
|
|
3121
|
+
retryAtMs: fleetControlPlane.retryAtMs,
|
|
3122
|
+
});
|
|
3123
|
+
throw new FleetControlPlaneCircuitOpenError(fleetControlPlane.retryAtMs ?? this.#clock.now(), fleetControlPlane.state);
|
|
3124
|
+
}
|
|
2950
3125
|
if (consecutiveFailures >= maxConsecutiveFailures) {
|
|
2951
3126
|
this.#increment('loopCircuitBreaks');
|
|
2952
3127
|
this.#logger.error?.('[factory] stopping loop after consecutive iteration failures', {
|
|
@@ -3102,6 +3277,13 @@ export class FactoryLoop {
|
|
|
3102
3277
|
};
|
|
3103
3278
|
}
|
|
3104
3279
|
this.#clearDependencyPark(batch, dispatchDecision.issue);
|
|
3280
|
+
// Event-driven and direct dispatches do not necessarily pass through issue
|
|
3281
|
+
// discovery. Admit them before creating previews, claiming a lifecycle, or
|
|
3282
|
+
// consuming a dispatch attempt. The mutation proxy probes again at the
|
|
3283
|
+
// actual spawn/resume boundary so a later control-plane fault still fails
|
|
3284
|
+
// closed.
|
|
3285
|
+
if (!dryRun)
|
|
3286
|
+
await this.#assertFleetControlPlaneAvailable();
|
|
3105
3287
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
3106
3288
|
// Local dispatches need the same deterministic branch identity as remote
|
|
3107
3289
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -3282,7 +3464,7 @@ export class FactoryLoop {
|
|
|
3282
3464
|
await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
|
|
3283
3465
|
}
|
|
3284
3466
|
if (!dryRun) {
|
|
3285
|
-
await this.#ensureSlackDispatchThread(record, result);
|
|
3467
|
+
await this.#ensureSlackDispatchThread(record, result, liveIssue);
|
|
3286
3468
|
}
|
|
3287
3469
|
return result;
|
|
3288
3470
|
}
|
|
@@ -3375,6 +3557,7 @@ export class FactoryLoop {
|
|
|
3375
3557
|
}
|
|
3376
3558
|
status() {
|
|
3377
3559
|
const batch = this.#batchView;
|
|
3560
|
+
const nowMs = this.#clock.now();
|
|
3378
3561
|
const inFlightDispatches = batch?.inFlight
|
|
3379
3562
|
.filter((record) => !record.dryRun)
|
|
3380
3563
|
.map((record) => ({
|
|
@@ -3389,7 +3572,7 @@ export class FactoryLoop {
|
|
|
3389
3572
|
claim: {
|
|
3390
3573
|
...(record.dispatchClaim ?? this.#dispatchClaimStatuses.get(issueKey(record.issue)) ?? {
|
|
3391
3574
|
state: 'pending',
|
|
3392
|
-
updatedAtMs:
|
|
3575
|
+
updatedAtMs: nowMs,
|
|
3393
3576
|
}),
|
|
3394
3577
|
},
|
|
3395
3578
|
})) ?? [];
|
|
@@ -3404,9 +3587,12 @@ export class FactoryLoop {
|
|
|
3404
3587
|
capacityBlocked: parked.capacityBlocked,
|
|
3405
3588
|
})) ?? [],
|
|
3406
3589
|
counters: { ...this.#counters },
|
|
3590
|
+
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
3407
3591
|
slackDegraded: this.#slackDegraded,
|
|
3408
3592
|
slackDegradedReason: this.#slackDegradedReason,
|
|
3409
3593
|
eventListener: this.#eventListenerStatus(),
|
|
3594
|
+
readinessReconcile: this.#readinessReconcileStatus(),
|
|
3595
|
+
heldAgents: batch?.inFlight.flatMap((record) => heldAgentsForRecord(record, nowMs, this.#config.dispatch.agentHoldTimeoutMs, this.#config.terminalState)) ?? [],
|
|
3410
3596
|
};
|
|
3411
3597
|
}
|
|
3412
3598
|
#eventListenerStatus() {
|
|
@@ -3427,6 +3613,34 @@ export class FactoryLoop {
|
|
|
3427
3613
|
}
|
|
3428
3614
|
return { state: 'starting' };
|
|
3429
3615
|
}
|
|
3616
|
+
#readinessReconcileStatus() {
|
|
3617
|
+
const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
|
|
3618
|
+
const state = this.#startMode !== 'live'
|
|
3619
|
+
? 'not-running'
|
|
3620
|
+
: consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
|
|
3621
|
+
? 'degraded'
|
|
3622
|
+
: consecutiveFailures > 0
|
|
3623
|
+
? 'retrying'
|
|
3624
|
+
: 'healthy';
|
|
3625
|
+
return {
|
|
3626
|
+
state,
|
|
3627
|
+
consecutiveFailures,
|
|
3628
|
+
failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
3629
|
+
...(this.#readinessReconcileLastDurationMs !== undefined
|
|
3630
|
+
? { lastDurationMs: this.#readinessReconcileLastDurationMs }
|
|
3631
|
+
: {}),
|
|
3632
|
+
...(this.#readinessReconcileLastStartedAtMs !== undefined
|
|
3633
|
+
? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
|
|
3634
|
+
: {}),
|
|
3635
|
+
...(this.#readinessReconcileLastCompletedAtMs !== undefined
|
|
3636
|
+
? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
|
|
3637
|
+
: {}),
|
|
3638
|
+
...(this.#readinessReconcileLastFailureAtMs !== undefined
|
|
3639
|
+
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
3640
|
+
: {}),
|
|
3641
|
+
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
3642
|
+
};
|
|
3643
|
+
}
|
|
3430
3644
|
on(event, listener) {
|
|
3431
3645
|
let listeners = this.#listeners.get(event);
|
|
3432
3646
|
if (!listeners) {
|
|
@@ -3687,6 +3901,7 @@ export class FactoryLoop {
|
|
|
3687
3901
|
});
|
|
3688
3902
|
}
|
|
3689
3903
|
}
|
|
3904
|
+
this.#rescheduleHeldAgentDeadlineSweep();
|
|
3690
3905
|
}
|
|
3691
3906
|
catch (error) {
|
|
3692
3907
|
this.#logger.warn?.('[factory] failed to re-adopt durable in-flight agents', { error });
|
|
@@ -3799,6 +4014,93 @@ export class FactoryLoop {
|
|
|
3799
4014
|
}, DISPATCH_LIFECYCLE_RENEW_MS);
|
|
3800
4015
|
this.#dispatchLifecycleRenewTimer.unref?.();
|
|
3801
4016
|
}
|
|
4017
|
+
#scheduleHeldAgentDeadline(record) {
|
|
4018
|
+
if (this.#stopping || record.dryRun || record.heldSinceAtMs === undefined || record.agents.size === 0)
|
|
4019
|
+
return;
|
|
4020
|
+
const dueAtMs = record.heldSinceAtMs + this.#config.dispatch.agentHoldTimeoutMs;
|
|
4021
|
+
if (this.#heldAgentDeadlineTimer &&
|
|
4022
|
+
this.#heldAgentDeadlineDueAtMs !== undefined &&
|
|
4023
|
+
this.#heldAgentDeadlineDueAtMs <= dueAtMs)
|
|
4024
|
+
return;
|
|
4025
|
+
if (this.#heldAgentDeadlineTimer)
|
|
4026
|
+
clearTimeout(this.#heldAgentDeadlineTimer);
|
|
4027
|
+
this.#heldAgentDeadlineDueAtMs = dueAtMs;
|
|
4028
|
+
const remainingMs = dueAtMs - this.#clock.now();
|
|
4029
|
+
// An overdue lifecycle can temporarily be fenced by another owner. Avoid
|
|
4030
|
+
// a zero-delay reschedule loop while its lease is being reclaimed.
|
|
4031
|
+
const delayMs = remainingMs <= 0
|
|
4032
|
+
? HELD_DEADLINE_OVERDUE_RETRY_MS
|
|
4033
|
+
: Math.min(remainingMs, 2_147_483_647);
|
|
4034
|
+
this.#heldAgentDeadlineTimer = setTimeout(() => {
|
|
4035
|
+
this.#heldAgentDeadlineTimer = undefined;
|
|
4036
|
+
this.#heldAgentDeadlineDueAtMs = undefined;
|
|
4037
|
+
const sweep = this.#sweepHeldAgentDeadlines()
|
|
4038
|
+
.catch((error) => {
|
|
4039
|
+
this.#logger.warn?.('[factory] held-agent deadline sweep failed; retrying', {
|
|
4040
|
+
error: describeError(error).errorMessage,
|
|
4041
|
+
});
|
|
4042
|
+
})
|
|
4043
|
+
.finally(() => {
|
|
4044
|
+
if (this.#heldAgentDeadlineSweepInFlight === sweep)
|
|
4045
|
+
this.#heldAgentDeadlineSweepInFlight = undefined;
|
|
4046
|
+
this.#rescheduleHeldAgentDeadlineSweep();
|
|
4047
|
+
});
|
|
4048
|
+
this.#heldAgentDeadlineSweepInFlight = sweep;
|
|
4049
|
+
}, delayMs);
|
|
4050
|
+
this.#heldAgentDeadlineTimer.unref?.();
|
|
4051
|
+
}
|
|
4052
|
+
#rescheduleHeldAgentDeadlineSweep() {
|
|
4053
|
+
if (this.#stopping)
|
|
4054
|
+
return;
|
|
4055
|
+
for (const record of this.#batchView?.inFlight ?? [])
|
|
4056
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
4057
|
+
}
|
|
4058
|
+
async #sweepHeldAgentDeadlines() {
|
|
4059
|
+
const nowMs = this.#clock.now();
|
|
4060
|
+
const timeoutMs = this.#config.dispatch.agentHoldTimeoutMs;
|
|
4061
|
+
for (const record of [...(await this.#batch()).inFlight]) {
|
|
4062
|
+
const heldSinceAtMs = record.heldSinceAtMs;
|
|
4063
|
+
if (record.dryRun ||
|
|
4064
|
+
heldSinceAtMs === undefined ||
|
|
4065
|
+
record.agents.size === 0 ||
|
|
4066
|
+
nowMs - heldSinceAtMs < timeoutMs)
|
|
4067
|
+
continue;
|
|
4068
|
+
const key = issueKey(record.issue);
|
|
4069
|
+
if (this.#abandonedDispatchReasons.has(key))
|
|
4070
|
+
continue;
|
|
4071
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
4072
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
4073
|
+
if (!lifecycle || isTerminalDispatchLifecycle(lifecycle))
|
|
4074
|
+
continue;
|
|
4075
|
+
// Terminal writeback already won the race. Finish its normal release
|
|
4076
|
+
// reason instead of relabeling an acknowledged completion as a timeout.
|
|
4077
|
+
if (lifecycle.phase === 'releasing') {
|
|
4078
|
+
await this.#finishDurableRelease(record, lifecycle.releaseReason);
|
|
4079
|
+
continue;
|
|
4080
|
+
}
|
|
4081
|
+
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
4082
|
+
continue;
|
|
4083
|
+
}
|
|
4084
|
+
const heldForMs = Math.max(0, this.#clock.now() - heldSinceAtMs);
|
|
4085
|
+
const details = {
|
|
4086
|
+
issue: record.issue.key,
|
|
4087
|
+
heldForMs,
|
|
4088
|
+
holdTimeoutMs: timeoutMs,
|
|
4089
|
+
waitingForTerminalState: this.#config.terminalState,
|
|
4090
|
+
reason: HELD_PAST_DEADLINE_RELEASE_REASON,
|
|
4091
|
+
agents: [...record.agents.keys()].sort(),
|
|
4092
|
+
};
|
|
4093
|
+
this.#logger.warn?.('[factory] releasing agents held past deadline', details);
|
|
4094
|
+
await this.#abandonStuckDispatch(record, HELD_PAST_DEADLINE_RELEASE_REASON);
|
|
4095
|
+
const lifecycle = this.#usesDurableDispatchLifecycle()
|
|
4096
|
+
? await this.#state.getDispatchLifecycle(this.#workspaceId, key)
|
|
4097
|
+
: undefined;
|
|
4098
|
+
if (!lifecycle || isTerminalDispatchLifecycle(lifecycle)) {
|
|
4099
|
+
this.#increment('heldPastDeadlineReleases');
|
|
4100
|
+
this.#logger.warn?.('[factory] released agents held past deadline', details);
|
|
4101
|
+
}
|
|
4102
|
+
}
|
|
4103
|
+
}
|
|
3802
4104
|
async #renewDispatchLifecycles() {
|
|
3803
4105
|
for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
|
|
3804
4106
|
const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
@@ -4164,6 +4466,7 @@ export class FactoryLoop {
|
|
|
4164
4466
|
});
|
|
4165
4467
|
}
|
|
4166
4468
|
async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set(), telemetry = {}) {
|
|
4469
|
+
record.lifecyclePhase = phase;
|
|
4167
4470
|
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
4168
4471
|
return true;
|
|
4169
4472
|
if (isTerminalDispatchPhase(phase))
|
|
@@ -4188,12 +4491,25 @@ export class FactoryLoop {
|
|
|
4188
4491
|
}
|
|
4189
4492
|
for (const agent of lifecycle.agents) {
|
|
4190
4493
|
const previousAgent = previous?.agents.find((candidate) => candidate.name === agent.name);
|
|
4191
|
-
|
|
4192
|
-
|
|
4494
|
+
// The in-flight record is authoritative for release state: the stamp
|
|
4495
|
+
// now lives on the tracked agent and round-trips through the lifecycle,
|
|
4496
|
+
// so a still-released agent brings its own stamp here. Inheriting the
|
|
4497
|
+
// previous row's stamp instead would refile a respawned worker as
|
|
4498
|
+
// released — a respawn reuses both the agent name and the deterministic
|
|
4499
|
+
// invocation id, so neither can tell the generations apart — and every
|
|
4500
|
+
// consumer that filters on `releasedAtMs` would then treat a live
|
|
4501
|
+
// worker as gone.
|
|
4193
4502
|
if (previousAgent?.costUsage)
|
|
4194
4503
|
agent.costUsage = structuredClone(previousAgent.costUsage);
|
|
4195
|
-
if (releasedAgentNames.has(agent.name))
|
|
4196
|
-
agent.releasedAtMs
|
|
4504
|
+
if (releasedAgentNames.has(agent.name)) {
|
|
4505
|
+
const releasedAtMs = agent.releasedAtMs ?? this.#clock.now();
|
|
4506
|
+
agent.releasedAtMs = releasedAtMs;
|
|
4507
|
+
// Keep the record the single source of the stamp, so the next save
|
|
4508
|
+
// carries it without having to read it back off the durable row.
|
|
4509
|
+
const tracked = record.agents.get(agent.name);
|
|
4510
|
+
if (tracked)
|
|
4511
|
+
tracked.releasedAtMs ??= releasedAtMs;
|
|
4512
|
+
}
|
|
4197
4513
|
}
|
|
4198
4514
|
const saved = await this.#state.saveDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), lifecycle);
|
|
4199
4515
|
if (!saved) {
|
|
@@ -4221,10 +4537,14 @@ export class FactoryLoop {
|
|
|
4221
4537
|
return true;
|
|
4222
4538
|
});
|
|
4223
4539
|
}
|
|
4224
|
-
|
|
4540
|
+
// Every caller of this knows the phase the row settled in, and each waiter
|
|
4541
|
+
// must be handed it directly. A waiter that instead re-read the shared row
|
|
4542
|
+
// after release could see it cleared, or see the next dispatch for the same
|
|
4543
|
+
// issue, and classify the wrong run.
|
|
4544
|
+
#resolveDispatchTerminalWaiters(issue, phase) {
|
|
4225
4545
|
const key = issueKey(issue);
|
|
4226
4546
|
for (const resolve of this.#dispatchTerminalWaiters.get(key) ?? [])
|
|
4227
|
-
resolve();
|
|
4547
|
+
resolve(phase);
|
|
4228
4548
|
this.#dispatchTerminalWaiters.delete(key);
|
|
4229
4549
|
}
|
|
4230
4550
|
#scheduleDispatchLifecycleRetry(record) {
|
|
@@ -4311,7 +4631,7 @@ export class FactoryLoop {
|
|
|
4311
4631
|
if (!lifecycle)
|
|
4312
4632
|
return;
|
|
4313
4633
|
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
4314
|
-
this.#resolveDispatchTerminalWaiters(lifecycle.issue);
|
|
4634
|
+
this.#resolveDispatchTerminalWaiters(lifecycle.issue, lifecycle.phase);
|
|
4315
4635
|
return;
|
|
4316
4636
|
}
|
|
4317
4637
|
if (lifecycle.phase === 'waiting-for-human')
|
|
@@ -4367,6 +4687,7 @@ export class FactoryLoop {
|
|
|
4367
4687
|
const batch = await this.#batch();
|
|
4368
4688
|
const durableRecord = inFlightRecordFromLifecycle(lifecycle);
|
|
4369
4689
|
const record = lifecycle.phase === 'releasing' ? durableRecord : batch.restore(durableRecord);
|
|
4690
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
4370
4691
|
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
4371
4692
|
return;
|
|
4372
4693
|
if (acquiredNow && this.#config.babysitter.enabled)
|
|
@@ -4691,7 +5012,7 @@ export class FactoryLoop {
|
|
|
4691
5012
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
4692
5013
|
await this.#writeInFlightRegistry();
|
|
4693
5014
|
this.#increment('dispatchLifecycleStaleIssuesAbandoned');
|
|
4694
|
-
this.#resolveDispatchTerminalWaiters(record.issue);
|
|
5015
|
+
this.#resolveDispatchTerminalWaiters(record.issue, 'abandoned');
|
|
4695
5016
|
this.#logger.info?.('[factory] abandoned durable dispatch whose live issue is no longer ready', {
|
|
4696
5017
|
issue: record.issue.key,
|
|
4697
5018
|
reason,
|
|
@@ -4767,7 +5088,7 @@ export class FactoryLoop {
|
|
|
4767
5088
|
this.#increment(releaseReason === 'issue-human-review' ? 'humanReview' : 'done');
|
|
4768
5089
|
this.#emit('issue-done', { issue: record.issue });
|
|
4769
5090
|
await this.#writeInFlightRegistry();
|
|
4770
|
-
this.#resolveDispatchTerminalWaiters(record.issue);
|
|
5091
|
+
this.#resolveDispatchTerminalWaiters(record.issue, 'complete');
|
|
4771
5092
|
return true;
|
|
4772
5093
|
}
|
|
4773
5094
|
async #assertDispatchLifecycleOwner(record) {
|
|
@@ -5790,6 +6111,8 @@ export class FactoryLoop {
|
|
|
5790
6111
|
updatedAtMs,
|
|
5791
6112
|
registryPath,
|
|
5792
6113
|
eventListener: this.#eventListenerStatus(),
|
|
6114
|
+
readinessReconcile: this.#readinessReconcileStatus(),
|
|
6115
|
+
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
5793
6116
|
};
|
|
5794
6117
|
await mkdir(dirname(path), { recursive: true });
|
|
5795
6118
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
@@ -6048,6 +6371,22 @@ export class FactoryLoop {
|
|
|
6048
6371
|
}
|
|
6049
6372
|
try {
|
|
6050
6373
|
await this.#fleet.release(agentName, reason);
|
|
6374
|
+
// Invalidate the spawn memory the moment the release is confirmed, so a
|
|
6375
|
+
// retry for the same deterministic invocation spawns a real worker
|
|
6376
|
+
// instead of inheriting this one's claim. Shutdown is excluded on
|
|
6377
|
+
// purpose: nothing is being retried there, and a takeover must still be
|
|
6378
|
+
// free to adopt agents this process merely stopped supervising.
|
|
6379
|
+
if (record && batch && context !== 'stop') {
|
|
6380
|
+
const releasedInvocationId = batch.recordRelease(record, agentName, this.#clock.now());
|
|
6381
|
+
if (releasedInvocationId) {
|
|
6382
|
+
this.#logger.debug?.('[factory] released agent invocation is no longer dispatchable', {
|
|
6383
|
+
issue: record.issue.key,
|
|
6384
|
+
agentName,
|
|
6385
|
+
reason,
|
|
6386
|
+
invocationId: releasedInvocationId,
|
|
6387
|
+
});
|
|
6388
|
+
}
|
|
6389
|
+
}
|
|
6051
6390
|
if (record)
|
|
6052
6391
|
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
6053
6392
|
}
|
|
@@ -6214,7 +6553,7 @@ export class FactoryLoop {
|
|
|
6214
6553
|
const updatedAtMs = this.#clock.now();
|
|
6215
6554
|
const agents = [];
|
|
6216
6555
|
const seenAgents = new Set();
|
|
6217
|
-
const appendAgent = async (issue, agentName, tracked) => {
|
|
6556
|
+
const appendAgent = async (issue, agentName, tracked, hold) => {
|
|
6218
6557
|
const key = registryHandoffKey(issue, agentName);
|
|
6219
6558
|
if (seenAgents.has(key)) {
|
|
6220
6559
|
return;
|
|
@@ -6240,6 +6579,12 @@ export class FactoryLoop {
|
|
|
6240
6579
|
...(fleetTracked?.invocationId ? { invocationId: fleetTracked.invocationId } : {}),
|
|
6241
6580
|
...(fleetTracked?.node ? { node: fleetTracked.node } : {}),
|
|
6242
6581
|
...(dispatchClaim ? { dispatchClaim: { ...dispatchClaim } } : {}),
|
|
6582
|
+
...(hold?.heldSinceAtMs !== undefined ? {
|
|
6583
|
+
heldSinceAtMs: hold.heldSinceAtMs,
|
|
6584
|
+
holdDeadlineAtMs: hold.heldSinceAtMs + this.#config.dispatch.agentHoldTimeoutMs,
|
|
6585
|
+
waitingForTerminalState: this.#config.terminalState,
|
|
6586
|
+
...(hold.lifecyclePhase ? { lifecyclePhase: hold.lifecyclePhase } : {}),
|
|
6587
|
+
} : {}),
|
|
6243
6588
|
});
|
|
6244
6589
|
};
|
|
6245
6590
|
if (!empty) {
|
|
@@ -6250,7 +6595,7 @@ export class FactoryLoop {
|
|
|
6250
6595
|
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
6251
6596
|
}
|
|
6252
6597
|
for (const [agentName, tracked] of record.agents) {
|
|
6253
|
-
await appendAgent(record.issue, agentName, tracked);
|
|
6598
|
+
await appendAgent(record.issue, agentName, tracked, record);
|
|
6254
6599
|
}
|
|
6255
6600
|
}
|
|
6256
6601
|
}
|
|
@@ -6271,7 +6616,11 @@ export class FactoryLoop {
|
|
|
6271
6616
|
const batch = await this.#batch();
|
|
6272
6617
|
const invocationId = batch.invocationIdFor(record.issue, spec);
|
|
6273
6618
|
const existing = record.agents.get(spec.name);
|
|
6274
|
-
|
|
6619
|
+
// A released placement keeps its bookkeeping entry, but it is not a worker.
|
|
6620
|
+
// Answering with its old spawn result here would report a synthetic success
|
|
6621
|
+
// for a process that no longer exists.
|
|
6622
|
+
if (existing?.result && existing.releasedAtMs === undefined) {
|
|
6623
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
6275
6624
|
return { name: existing.result?.name ?? spec.name };
|
|
6276
6625
|
}
|
|
6277
6626
|
if (!batch.shouldSpawn(record, invocationId)) {
|
|
@@ -6298,6 +6647,7 @@ export class FactoryLoop {
|
|
|
6298
6647
|
const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
|
|
6299
6648
|
if (rosterAgent) {
|
|
6300
6649
|
const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
|
|
6650
|
+
record.heldSinceAtMs ??= this.#clock.now();
|
|
6301
6651
|
batch.recordSpawn(record, spec, invocationId, {
|
|
6302
6652
|
name: spec.name,
|
|
6303
6653
|
sessionRef: existing?.sessionRef ?? spec.sessionRef,
|
|
@@ -6307,6 +6657,7 @@ export class FactoryLoop {
|
|
|
6307
6657
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
6308
6658
|
throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`);
|
|
6309
6659
|
}
|
|
6660
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
6310
6661
|
const adopted = record.agents.get(spec.name);
|
|
6311
6662
|
if (adopted)
|
|
6312
6663
|
await this.#reportAgent(record, adopted, 'agent.adopted');
|
|
@@ -6318,6 +6669,7 @@ export class FactoryLoop {
|
|
|
6318
6669
|
result = await this.#fleet.spawn({
|
|
6319
6670
|
name: spec.name,
|
|
6320
6671
|
capability: spec.capability,
|
|
6672
|
+
identityKey: dispatchAgentIdentityKey(record.issue, spec.role),
|
|
6321
6673
|
node: spec.node ?? 'self',
|
|
6322
6674
|
repo: spec.repo,
|
|
6323
6675
|
task: spec.task,
|
|
@@ -6339,10 +6691,12 @@ export class FactoryLoop {
|
|
|
6339
6691
|
: 'agent_spawn_failed',
|
|
6340
6692
|
});
|
|
6341
6693
|
}
|
|
6694
|
+
record.heldSinceAtMs ??= this.#clock.now();
|
|
6342
6695
|
batch.recordSpawn(record, spec, invocationId, result);
|
|
6343
6696
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
6344
6697
|
throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`);
|
|
6345
6698
|
}
|
|
6699
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
6346
6700
|
const spawned = record.agents.get(result.name);
|
|
6347
6701
|
if (spawned)
|
|
6348
6702
|
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
@@ -6417,6 +6771,16 @@ export class FactoryLoop {
|
|
|
6417
6771
|
return;
|
|
6418
6772
|
}
|
|
6419
6773
|
}
|
|
6774
|
+
// Swarm workers share the lead's checkout and lifecycle branch. If a worker
|
|
6775
|
+
// exit reached the publication/completion paths below, whichever worker
|
|
6776
|
+
// finished first would publish whatever partial state was on the shared
|
|
6777
|
+
// branch and mark every swarm member "done" via the shared-branch PR probe,
|
|
6778
|
+
// releasing the still-working lead. The lead alone is authoritative for
|
|
6779
|
+
// publication and completion in a swarm.
|
|
6780
|
+
if (exiting?.spec.swarmRole === 'worker') {
|
|
6781
|
+
this.#increment('swarmWorkerExitsSuppressed');
|
|
6782
|
+
return;
|
|
6783
|
+
}
|
|
6420
6784
|
if (isCompletionReason(reason)) {
|
|
6421
6785
|
if (exiting?.spec.role === 'implementer' && await this.#issueHasCompletionPr(record, {
|
|
6422
6786
|
openOnly: this.#config.babysitter.enabled,
|
|
@@ -6661,6 +7025,7 @@ export class FactoryLoop {
|
|
|
6661
7025
|
const result = await this.#fleet.spawn({
|
|
6662
7026
|
name: tracked.spec.name,
|
|
6663
7027
|
capability: tracked.spec.capability,
|
|
7028
|
+
identityKey: dispatchAgentIdentityKey(record.issue, tracked.spec.role),
|
|
6664
7029
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
6665
7030
|
repo: tracked.spec.repo,
|
|
6666
7031
|
task: tracked.spec.task,
|
|
@@ -7301,7 +7666,10 @@ export class FactoryLoop {
|
|
|
7301
7666
|
try {
|
|
7302
7667
|
const thread = await this.#slackDispatchThreadFor(record);
|
|
7303
7668
|
if (thread && this.#slack) {
|
|
7304
|
-
await this.#
|
|
7669
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
7670
|
+
if (!issue)
|
|
7671
|
+
return;
|
|
7672
|
+
await this.#slack.reply(thread.threadId, `:warning: ${slackIssueSubject(issue, slackNotificationRepos(record.decision))}\nThe implementer exited without opening a PR after a retry; this needs a human look.`);
|
|
7305
7673
|
}
|
|
7306
7674
|
}
|
|
7307
7675
|
catch (error) {
|
|
@@ -7362,6 +7730,8 @@ export class FactoryLoop {
|
|
|
7362
7730
|
// record leaves the batch.
|
|
7363
7731
|
async #abandonStuckDispatch(record, reason) {
|
|
7364
7732
|
const key = issueKey(record.issue);
|
|
7733
|
+
const heldPastDeadline = reason === HELD_PAST_DEADLINE_RELEASE_REASON;
|
|
7734
|
+
const agentReleaseReason = heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : 'issue-abandoned';
|
|
7365
7735
|
this.#abandonedDispatchReasons.set(key, reason);
|
|
7366
7736
|
if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
7367
7737
|
this.#increment('abandonedDispatchReleaseRetries');
|
|
@@ -7387,9 +7757,9 @@ export class FactoryLoop {
|
|
|
7387
7757
|
}
|
|
7388
7758
|
const agents = [...record.agents];
|
|
7389
7759
|
for (const [agentName, tracked] of agents) {
|
|
7390
|
-
if (tracked.spec.role === 'implementer')
|
|
7760
|
+
if (!heldPastDeadline && tracked.spec.role === 'implementer')
|
|
7391
7761
|
continue;
|
|
7392
|
-
this.#fleet.markAgentTerminal?.(agentName, `implementer-terminal:${reason}`);
|
|
7762
|
+
this.#fleet.markAgentTerminal?.(agentName, heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : `implementer-terminal:${reason}`);
|
|
7393
7763
|
}
|
|
7394
7764
|
const worktreeHandoffs = this.#dispatchFailureHandoffs(record, []);
|
|
7395
7765
|
let cleanupComplete = true;
|
|
@@ -7402,13 +7772,13 @@ export class FactoryLoop {
|
|
|
7402
7772
|
const worktreeAgentNames = new Set(worktreeHandoffs.map((handoff) => handoff.name));
|
|
7403
7773
|
const nonWorktreeAgents = agents.filter(([name]) => !worktreeAgentNames.has(name));
|
|
7404
7774
|
if (nonWorktreeAgents.length > 0) {
|
|
7405
|
-
const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents,
|
|
7775
|
+
const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents, agentReleaseReason, 'completion');
|
|
7406
7776
|
cleanupComplete = failed.length === 0;
|
|
7407
7777
|
}
|
|
7408
|
-
cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs) && cleanupComplete;
|
|
7778
|
+
cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs, agentReleaseReason) && cleanupComplete;
|
|
7409
7779
|
}
|
|
7410
7780
|
else if (agents.length > 0) {
|
|
7411
|
-
const failed = await this.#releaseAndTerminateAgents(agents,
|
|
7781
|
+
const failed = await this.#releaseAndTerminateAgents(agents, agentReleaseReason, 'completion');
|
|
7412
7782
|
cleanupComplete = failed.length === 0;
|
|
7413
7783
|
}
|
|
7414
7784
|
if (!cleanupComplete) {
|
|
@@ -7537,6 +7907,7 @@ export class FactoryLoop {
|
|
|
7537
7907
|
const result = await this.#fleet.resume({
|
|
7538
7908
|
name,
|
|
7539
7909
|
sessionRef: tracked.sessionRef,
|
|
7910
|
+
identityKey: dispatchAgentIdentityKey(record.issue, tracked.spec.role),
|
|
7540
7911
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
7541
7912
|
capability: tracked.spec.capability,
|
|
7542
7913
|
repo: tracked.spec.repo,
|
|
@@ -7665,6 +8036,7 @@ export class FactoryLoop {
|
|
|
7665
8036
|
const result = await this.#fleet.spawn({
|
|
7666
8037
|
name: replacementSpec.name,
|
|
7667
8038
|
capability: replacementSpec.capability,
|
|
8039
|
+
identityKey: dispatchAgentIdentityKey(record.issue, replacementSpec.role),
|
|
7668
8040
|
node: tracked.result?.node ?? replacementSpec.node ?? 'self',
|
|
7669
8041
|
repo: replacementSpec.repo,
|
|
7670
8042
|
task: replacementSpec.task,
|
|
@@ -7997,7 +8369,11 @@ export class FactoryLoop {
|
|
|
7997
8369
|
return await this.#postAgentQuestionToGithub(record, question, 'no Slack dispatch thread exists');
|
|
7998
8370
|
}
|
|
7999
8371
|
try {
|
|
8000
|
-
await this.#
|
|
8372
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
8373
|
+
if (!issue) {
|
|
8374
|
+
throw new Error(`Unable to describe Slack question notification for unreadable issue ${record.issue.key}`);
|
|
8375
|
+
}
|
|
8376
|
+
await this.#slack.reply(threadId, agentQuestionSlackText(issue, question, this.#config.slack.stakeholderUserIds, slackNotificationRepos(record.decision)));
|
|
8001
8377
|
this.#increment('agentQuestionsPostedToSlack');
|
|
8002
8378
|
this.#recordSlackWritebackSuccess('agent-question');
|
|
8003
8379
|
return true;
|
|
@@ -8099,10 +8475,13 @@ export class FactoryLoop {
|
|
|
8099
8475
|
return await this.#deliverClarificationQuestionToGithub(key, claimed, `Slack writeback is degraded${this.#slackDegradedReason ? `: ${this.#slackDegradedReason}` : ''}`);
|
|
8100
8476
|
}
|
|
8101
8477
|
try {
|
|
8102
|
-
|
|
8478
|
+
if (!claimedIssue) {
|
|
8479
|
+
throw new Error(`Unable to describe Slack question notification for unreadable issue ${claimed.issue.key}`);
|
|
8480
|
+
}
|
|
8481
|
+
await this.#slack.reply(claimed.threadId, agentQuestionSlackText(claimedIssue, {
|
|
8103
8482
|
agentName: claimed.askerName,
|
|
8104
8483
|
question: claimed.question,
|
|
8105
|
-
}, this.#config.slack.stakeholderUserIds));
|
|
8484
|
+
}, this.#config.slack.stakeholderUserIds, slackNotificationRepos(claimed.decision)));
|
|
8106
8485
|
const completed = await this.#state.completeClarificationQuestionDelivery(this.#workspaceId, key, this.#clarificationWakeOwner, this.#clock.now());
|
|
8107
8486
|
if (!completed) {
|
|
8108
8487
|
this.#increment('clarificationQuestionDeliveryOwnershipLost');
|
|
@@ -8300,7 +8679,11 @@ export class FactoryLoop {
|
|
|
8300
8679
|
return;
|
|
8301
8680
|
}
|
|
8302
8681
|
try {
|
|
8303
|
-
await this.#
|
|
8682
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
8683
|
+
if (!issue) {
|
|
8684
|
+
throw new Error(`Unable to describe Slack question mirror for unreadable issue ${record.issue.key}`);
|
|
8685
|
+
}
|
|
8686
|
+
await this.#slack.reply(threadId, agentQuestionSlackText(issue, question, this.#config.slack.stakeholderUserIds, slackNotificationRepos(record.decision)));
|
|
8304
8687
|
this.#increment('agentQuestionsMirroredToSlack');
|
|
8305
8688
|
this.#recordSlackWritebackSuccess('agent-question-mirror');
|
|
8306
8689
|
}
|
|
@@ -9121,6 +9504,15 @@ export class FactoryLoop {
|
|
|
9121
9504
|
previewStartCommand: spec.preview?.startCommand,
|
|
9122
9505
|
} : {}),
|
|
9123
9506
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
9507
|
+
...(spec.swarmRole && spec.channel ? {
|
|
9508
|
+
swarm: {
|
|
9509
|
+
role: spec.swarmRole,
|
|
9510
|
+
channel: spec.channel,
|
|
9511
|
+
otherMemberNames: decision.implementers
|
|
9512
|
+
.filter((implementer) => implementer.channel === spec.channel && implementer.name !== spec.name)
|
|
9513
|
+
.map((implementer) => implementer.name),
|
|
9514
|
+
},
|
|
9515
|
+
} : {}),
|
|
9124
9516
|
}),
|
|
9125
9517
|
};
|
|
9126
9518
|
};
|
|
@@ -9395,7 +9787,7 @@ export class FactoryLoop {
|
|
|
9395
9787
|
prMetaShowsMerged(snapshot) &&
|
|
9396
9788
|
prSnapshotIssueMatchScore(snapshot, session.issue.key) >= 30) {
|
|
9397
9789
|
await this.#state.clearBabysitterSession(this.#workspaceId, persistedKey);
|
|
9398
|
-
await this.#advanceMergedPrToDone(snapshot, record);
|
|
9790
|
+
await this.#advanceMergedPrToDone(snapshot, session.repo, record);
|
|
9399
9791
|
this.#increment('babysitterOwnershipRestoreMerged');
|
|
9400
9792
|
this.#logger.info?.('[factory] completed restored lifecycle whose pull request was already merged', {
|
|
9401
9793
|
issue: session.issue.key,
|
|
@@ -10516,7 +10908,7 @@ export class FactoryLoop {
|
|
|
10516
10908
|
if (owned) {
|
|
10517
10909
|
if (prMetaShowsMerged(snapshot)) {
|
|
10518
10910
|
if (owned.record)
|
|
10519
|
-
await this.#advanceMergedPrToDone(snapshot, owned.record);
|
|
10911
|
+
await this.#advanceMergedPrToDone(snapshot, repo, owned.record);
|
|
10520
10912
|
else
|
|
10521
10913
|
await this.#cancelBabysitterWake(owned.key);
|
|
10522
10914
|
return;
|
|
@@ -10572,7 +10964,7 @@ export class FactoryLoop {
|
|
|
10572
10964
|
return;
|
|
10573
10965
|
}
|
|
10574
10966
|
if (prMetaShowsMerged(snapshot)) {
|
|
10575
|
-
await this.#advanceMergedPrToDone(snapshot, record);
|
|
10967
|
+
await this.#advanceMergedPrToDone(snapshot, repo, record);
|
|
10576
10968
|
return;
|
|
10577
10969
|
}
|
|
10578
10970
|
if (!this.#config.babysitter.enabled) {
|
|
@@ -10630,12 +11022,18 @@ export class FactoryLoop {
|
|
|
10630
11022
|
}
|
|
10631
11023
|
return best?.record;
|
|
10632
11024
|
}
|
|
10633
|
-
async #advanceMergedPrToDone(snapshot, record) {
|
|
11025
|
+
async #advanceMergedPrToDone(snapshot, repo, record) {
|
|
11026
|
+
const mergedPullRequest = { repo, number: snapshot.number, url: snapshot.url };
|
|
10634
11027
|
if (record) {
|
|
10635
|
-
await this.#completeIssue(record, {
|
|
11028
|
+
await this.#completeIssue(record, {
|
|
11029
|
+
targetState: 'done',
|
|
11030
|
+
runMergeGate: false,
|
|
11031
|
+
completionReason: 'pr-merged',
|
|
11032
|
+
mergedPullRequest,
|
|
11033
|
+
});
|
|
10636
11034
|
return;
|
|
10637
11035
|
}
|
|
10638
|
-
const issue = await this.#findMergeAdvanceIssueForPr(snapshot);
|
|
11036
|
+
const issue = await this.#findMergeAdvanceIssueForPr(snapshot, repo);
|
|
10639
11037
|
if (!issue) {
|
|
10640
11038
|
this.#increment('mergedPrAdvanceNoIssue');
|
|
10641
11039
|
return;
|
|
@@ -10670,11 +11068,13 @@ export class FactoryLoop {
|
|
|
10670
11068
|
const channel = await this.#slackChannelDir();
|
|
10671
11069
|
if (channel) {
|
|
10672
11070
|
const systemOfRecord = githubIssue ? 'GitHub issue closed' : 'Linear state set to Done';
|
|
11071
|
+
const subject = slackIssueSubject(issue, [repo]);
|
|
11072
|
+
const pullRequest = slackPullRequestLink(mergedPullRequest);
|
|
10673
11073
|
const root = await this.#slack.postThread({
|
|
10674
11074
|
channel,
|
|
10675
|
-
text: `${
|
|
11075
|
+
text: `${subject}\nPR merged · ${pullRequest} · ${systemOfRecord}`,
|
|
10676
11076
|
});
|
|
10677
|
-
await this.#slack.reply(root.threadId, `${
|
|
11077
|
+
await this.#slack.reply(root.threadId, `${subject}\n${systemOfRecord} · ${pullRequest}`);
|
|
10678
11078
|
this.#recordSlackWritebackSuccess('merge-done-thread');
|
|
10679
11079
|
}
|
|
10680
11080
|
}
|
|
@@ -10688,7 +11088,7 @@ export class FactoryLoop {
|
|
|
10688
11088
|
this.#error(error, issueRef(issue));
|
|
10689
11089
|
}
|
|
10690
11090
|
}
|
|
10691
|
-
async #findMergeAdvanceIssueForPr(snapshot) {
|
|
11091
|
+
async #findMergeAdvanceIssueForPr(snapshot, eventRepo) {
|
|
10692
11092
|
// An issue is "upstream" of a merge if it sits in the agent-implementing or
|
|
10693
11093
|
// human-review role for its team. UUIDs are globally unique, so the reverse
|
|
10694
11094
|
// role lookup covers every team without per-team scoping here.
|
|
@@ -10697,16 +11097,34 @@ export class FactoryLoop {
|
|
|
10697
11097
|
return role === 'agentImplementing' || role === 'humanReview';
|
|
10698
11098
|
};
|
|
10699
11099
|
let best;
|
|
11100
|
+
let ambiguous = false;
|
|
10700
11101
|
const scanStartedAtMs = this.#clock.now();
|
|
10701
11102
|
// This no-record path runs after agents are released, so there is no
|
|
10702
11103
|
// tracked PR identity left. Keep the scan simple and prefer branch identity
|
|
10703
11104
|
// over title/body references to avoid "related to AR-N" body false positives.
|
|
10704
11105
|
const githubSource = await this.#issueSource() === 'github';
|
|
11106
|
+
// A GitHub-native issue key is a bare number; the same number exists
|
|
11107
|
+
// independently in every configured repository. Only a same-repo candidate
|
|
11108
|
+
// may complete the merge, so a merged PR in one repo can never close an
|
|
11109
|
+
// unrelated issue that happens to share its number in another. #276.
|
|
11110
|
+
const eventRepoKey = validGithubRepo(eventRepo) ? eventRepo.toLowerCase() : undefined;
|
|
11111
|
+
if (githubSource && !eventRepoKey) {
|
|
11112
|
+
this.#logger.warn?.('[factory] merge advance skipped: event repo is not identifiable', {
|
|
11113
|
+
prNumber: snapshot.number,
|
|
11114
|
+
});
|
|
11115
|
+
return undefined;
|
|
11116
|
+
}
|
|
10705
11117
|
const paths = githubSource ? await this.#githubIssuePaths() : await this.#listRelayfileTree(ISSUE_ROOT, 'merge advance issue scan');
|
|
10706
11118
|
for (const path of paths) {
|
|
10707
11119
|
if (githubSource ? !isGithubIssueFilePath(path) : !isIssueFilePath(path)) {
|
|
10708
11120
|
continue;
|
|
10709
11121
|
}
|
|
11122
|
+
if (githubSource) {
|
|
11123
|
+
const issueParts = githubIssuePathParts(path);
|
|
11124
|
+
if (!issueParts || `${issueParts.owner}/${issueParts.repo}`.toLowerCase() !== eventRepoKey) {
|
|
11125
|
+
continue;
|
|
11126
|
+
}
|
|
11127
|
+
}
|
|
10710
11128
|
const issue = await this.#readIssue(path);
|
|
10711
11129
|
if (!issue || (githubSource
|
|
10712
11130
|
? issue.state?.name?.trim().toLowerCase() === 'closed'
|
|
@@ -10719,10 +11137,27 @@ export class FactoryLoop {
|
|
|
10719
11137
|
const score = prSnapshotIssueMatchScore(snapshot, issue.key);
|
|
10720
11138
|
if (score > 0 && (!best || score > best.score)) {
|
|
10721
11139
|
best = { issue, score };
|
|
11140
|
+
ambiguous = false;
|
|
11141
|
+
}
|
|
11142
|
+
else if (score > 0 && best && score === best.score) {
|
|
11143
|
+
ambiguous = true;
|
|
10722
11144
|
}
|
|
10723
11145
|
}
|
|
11146
|
+
// Fail-closed on a tie. A silent first-wins on unrelated same-score
|
|
11147
|
+
// candidates is the exact class of quiet corruption we are removing.
|
|
11148
|
+
if (ambiguous) {
|
|
11149
|
+
this.#increment('mergedPrAdvanceAmbiguous');
|
|
11150
|
+
this.#logger.warn?.('[factory] merge advance skipped: multiple candidates tied on match score', {
|
|
11151
|
+
prNumber: snapshot.number,
|
|
11152
|
+
eventRepo,
|
|
11153
|
+
matchedIssue: best?.issue.key,
|
|
11154
|
+
matchScore: best?.score,
|
|
11155
|
+
});
|
|
11156
|
+
return undefined;
|
|
11157
|
+
}
|
|
10724
11158
|
this.#logger.debug?.(`[factory] scanned ${githubSource ? 'GitHub' : 'Linear'} issues for merged PR advance`, {
|
|
10725
11159
|
prNumber: snapshot.number,
|
|
11160
|
+
eventRepo,
|
|
10726
11161
|
durationMs: this.#clock.now() - scanStartedAtMs,
|
|
10727
11162
|
matchedIssue: best?.issue.key,
|
|
10728
11163
|
matchScore: best?.score,
|
|
@@ -11280,20 +11715,26 @@ export class FactoryLoop {
|
|
|
11280
11715
|
}
|
|
11281
11716
|
if (!await this.#saveDispatchLifecycle(record, 'writeback-applied'))
|
|
11282
11717
|
return;
|
|
11283
|
-
if (this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
|
|
11718
|
+
if (issue && this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
|
|
11284
11719
|
try {
|
|
11285
11720
|
const channel = await this.#slackChannelDir();
|
|
11286
11721
|
if (channel) {
|
|
11287
11722
|
const merged = opts.completionReason === 'pr-merged';
|
|
11288
11723
|
const systemOfRecord = githubIssue ? 'GitHub status' : 'Linear state';
|
|
11289
|
-
const
|
|
11290
|
-
|
|
11291
|
-
|
|
11292
|
-
const
|
|
11293
|
-
?
|
|
11294
|
-
:
|
|
11295
|
-
|
|
11296
|
-
|
|
11724
|
+
const pullRequests = await this.#slackPullRequestRefs(record, opts.mergedPullRequest ? [opts.mergedPullRequest] : []);
|
|
11725
|
+
const pullRequestLinks = pullRequests.map(slackPullRequestLink).join(' · ');
|
|
11726
|
+
const subject = slackIssueSubject(issue, slackNotificationRepos(record.decision));
|
|
11727
|
+
const stateResult = githubIssue && merged && !humanReview
|
|
11728
|
+
? 'GitHub issue closed'
|
|
11729
|
+
: `${systemOfRecord} set to ${statusLabel}`;
|
|
11730
|
+
const completionText = [
|
|
11731
|
+
subject,
|
|
11732
|
+
merged
|
|
11733
|
+
? `PR merged${pullRequestLinks ? ` · ${pullRequestLinks}` : ''} · ${stateResult}`
|
|
11734
|
+
: `Completed${humanReview ? ' · awaiting human review' : ''}${pullRequestLinks ? ` · ${pullRequestLinks}` : ''}`,
|
|
11735
|
+
...(!merged ? [`Status: ${statusLabel} · Merge policy: ${this.#config.mergePolicy}`] : []),
|
|
11736
|
+
].join('\n');
|
|
11737
|
+
const stateText = `${subject}\n${stateResult}${pullRequestLinks ? ` · ${pullRequestLinks}` : ''}`;
|
|
11297
11738
|
const root = await this.#slack.postThread({
|
|
11298
11739
|
channel,
|
|
11299
11740
|
text: completionText,
|
|
@@ -11769,7 +12210,30 @@ export class FactoryLoop {
|
|
|
11769
12210
|
this.#logger.warn?.('[factory] cleared invalid persisted Slack thread id', { issue: key });
|
|
11770
12211
|
return undefined;
|
|
11771
12212
|
}
|
|
11772
|
-
async #
|
|
12213
|
+
async #slackPullRequestRefs(record, additional = []) {
|
|
12214
|
+
const lifecycle = await this.#state
|
|
12215
|
+
.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
|
|
12216
|
+
.catch(() => undefined);
|
|
12217
|
+
const lifecycleRefs = publishedPullRequests(lifecycle).map((receipt) => ({
|
|
12218
|
+
repo: receipt.repo,
|
|
12219
|
+
number: receipt.number,
|
|
12220
|
+
url: receipt.url,
|
|
12221
|
+
}));
|
|
12222
|
+
const trackedRefs = [...record.agents.values()]
|
|
12223
|
+
.map((tracked) => tracked.spec.ownedPullRequest)
|
|
12224
|
+
.filter((ref) => Boolean(ref))
|
|
12225
|
+
.map((ref) => ({ repo: ref.repo, number: ref.number }));
|
|
12226
|
+
const babysitterRefs = [...this.#babysitterPr.entries()]
|
|
12227
|
+
.filter(([key]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue))
|
|
12228
|
+
.map(([, ref]) => ({ repo: ref.repo, number: ref.prNumber }));
|
|
12229
|
+
return uniqueSlackPullRequestRefs([
|
|
12230
|
+
...additional,
|
|
12231
|
+
...lifecycleRefs,
|
|
12232
|
+
...babysitterRefs,
|
|
12233
|
+
...trackedRefs,
|
|
12234
|
+
]);
|
|
12235
|
+
}
|
|
12236
|
+
async #ensureSlackDispatchThread(record, result, sourceIssue) {
|
|
11773
12237
|
if (!this.#slack || !this.#config.slack || result.dryRun) {
|
|
11774
12238
|
return;
|
|
11775
12239
|
}
|
|
@@ -11805,7 +12269,7 @@ export class FactoryLoop {
|
|
|
11805
12269
|
}
|
|
11806
12270
|
return;
|
|
11807
12271
|
}
|
|
11808
|
-
const start = this.#postAndWatchSlackDispatchThread(record, result);
|
|
12272
|
+
const start = this.#postAndWatchSlackDispatchThread(record, result, sourceIssue);
|
|
11809
12273
|
this.#slackWatcherStarts.set(key, start);
|
|
11810
12274
|
try {
|
|
11811
12275
|
await start;
|
|
@@ -11818,19 +12282,24 @@ export class FactoryLoop {
|
|
|
11818
12282
|
this.#slackWatcherStarts.delete(key);
|
|
11819
12283
|
}
|
|
11820
12284
|
}
|
|
11821
|
-
async #postAndWatchSlackDispatchThread(record, result) {
|
|
12285
|
+
async #postAndWatchSlackDispatchThread(record, result, sourceIssue) {
|
|
11822
12286
|
if (!this.#slack || !this.#config.slack) {
|
|
11823
12287
|
return;
|
|
11824
12288
|
}
|
|
11825
|
-
const
|
|
12289
|
+
const issue = sourceIssue ?? await this.#readIssue(record.issue.path);
|
|
12290
|
+
if (!issue) {
|
|
12291
|
+
throw new Error(`Unable to describe Slack dispatch notification for unreadable issue ${record.issue.key}`);
|
|
12292
|
+
}
|
|
12293
|
+
const previews = uniquePreviewReferences(result.previews ?? this.#previewReferences.get(issueKey(record.issue)) ?? []);
|
|
12294
|
+
const repos = slackNotificationRepos(record.decision);
|
|
11826
12295
|
const root = await this.#slack.postThread({
|
|
11827
12296
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
11828
12297
|
text: [
|
|
11829
|
-
|
|
11830
|
-
`
|
|
12298
|
+
slackIssueSubject(issue, repos),
|
|
12299
|
+
`Dispatched · ${result.agents.map((agent) => agent.name).join(', ') || 'no agents'} · Repos: ${slackRepoList(repos)}`,
|
|
11831
12300
|
...(previews.length > 0
|
|
11832
12301
|
? [previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join(' · ')]
|
|
11833
|
-
: []),
|
|
12302
|
+
: [`State: ${result.stateId ?? 'dispatching'}`]),
|
|
11834
12303
|
].join('\n'),
|
|
11835
12304
|
});
|
|
11836
12305
|
await this.#state.setSlackThread(this.#workspaceId, issueKey(record.issue), root.threadId);
|
|
@@ -11859,6 +12328,7 @@ export class FactoryLoop {
|
|
|
11859
12328
|
const rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
|
|
11860
12329
|
name: agentName,
|
|
11861
12330
|
sessionRef,
|
|
12331
|
+
role: owned.tracked.spec.role,
|
|
11862
12332
|
node: owned.tracked.result?.node ?? owned.tracked.spec.node,
|
|
11863
12333
|
capability: owned.tracked.spec.capability,
|
|
11864
12334
|
repo: owned.tracked.spec.repo,
|
|
@@ -11891,6 +12361,7 @@ export class FactoryLoop {
|
|
|
11891
12361
|
agent: {
|
|
11892
12362
|
name: agentName,
|
|
11893
12363
|
sessionRef,
|
|
12364
|
+
role: owned.tracked.spec.role,
|
|
11894
12365
|
node: owned.tracked.result?.node ?? owned.tracked.spec.node,
|
|
11895
12366
|
capability: owned.tracked.spec.capability,
|
|
11896
12367
|
repo: owned.tracked.spec.repo,
|
|
@@ -11954,9 +12425,20 @@ export class FactoryLoop {
|
|
|
11954
12425
|
}, Math.max(1_000, Math.floor(SLACK_CONVERSATION_TURN_LEASE_MS / 3)));
|
|
11955
12426
|
heartbeat.unref?.();
|
|
11956
12427
|
try {
|
|
12428
|
+
// Sessions bound after this field was introduced carry their role
|
|
12429
|
+
// durably, so resume never depends on a live lookup by the agent's
|
|
12430
|
+
// current name — which a since-completed rename (e.g. babysitter
|
|
12431
|
+
// retarget) can otherwise leave unresolvable forever. Older
|
|
12432
|
+
// already-persisted sessions fall back to the live lookup.
|
|
12433
|
+
const conversationRole = claimed.agent.role
|
|
12434
|
+
?? (await this.#batch()).getIssueByAgent(claimed.agent.name)?.agents.get(claimed.agent.name)?.spec.role;
|
|
12435
|
+
if (!conversationRole) {
|
|
12436
|
+
throw new Error(`Cannot resume ${claimed.agent.name}: its dispatch role is unavailable for identity proof`);
|
|
12437
|
+
}
|
|
11957
12438
|
const result = await this.#fleet.resume({
|
|
11958
12439
|
name: claimed.agent.name,
|
|
11959
12440
|
sessionRef: claimed.agent.sessionRef,
|
|
12441
|
+
identityKey: dispatchAgentIdentityKey(claimed.issue, conversationRole),
|
|
11960
12442
|
node: claimed.agent.node ?? 'self',
|
|
11961
12443
|
capability: claimed.agent.capability,
|
|
11962
12444
|
repo: claimed.agent.repo,
|
|
@@ -12193,13 +12675,13 @@ export class FactoryLoop {
|
|
|
12193
12675
|
.filter((part) => Boolean(part))
|
|
12194
12676
|
.join(' ');
|
|
12195
12677
|
const replyInstruction = source?.url
|
|
12196
|
-
?
|
|
12678
|
+
? 'Reply on the linked GitHub issue so Factory can resume.'
|
|
12197
12679
|
: 'Reply on the source GitHub issue so Factory can resume.';
|
|
12198
12680
|
const root = await this.#slack.postThread({
|
|
12199
12681
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
12200
12682
|
text: [
|
|
12201
|
-
`${audience ? `${audience} ` : ''}${
|
|
12202
|
-
`Reason: ${reason}`,
|
|
12683
|
+
`${audience ? `${audience} ` : ''}${slackIssueSubject(issue, slackNotificationRepos(decision))}`,
|
|
12684
|
+
`Triage blocked · Reason: ${reason}`,
|
|
12203
12685
|
`Question: ${triageEscalationQuestion(decision, issue)} ${replyInstruction}`,
|
|
12204
12686
|
].join('\n'),
|
|
12205
12687
|
});
|
|
@@ -12323,12 +12805,15 @@ export class FactoryLoop {
|
|
|
12323
12805
|
return;
|
|
12324
12806
|
}
|
|
12325
12807
|
const issue = await this.#readIssue(decision.issue.path);
|
|
12808
|
+
if (!issue) {
|
|
12809
|
+
throw new Error(`Unable to describe Slack triage escalation for unreadable issue ${decision.issue.key}`);
|
|
12810
|
+
}
|
|
12326
12811
|
const stakeholderMentions = slackMentions(this.#config.slack.stakeholderUserIds);
|
|
12327
12812
|
const root = await this.#slack.postThread({
|
|
12328
12813
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
12329
12814
|
text: [
|
|
12330
|
-
`${stakeholderMentions ? `${stakeholderMentions} ` : ''}${
|
|
12331
|
-
`Reason: ${reason}`,
|
|
12815
|
+
`${stakeholderMentions ? `${stakeholderMentions} ` : ''}${slackIssueSubject(issue, slackNotificationRepos(decision))}`,
|
|
12816
|
+
`Triage blocked · Reason: ${reason}`,
|
|
12332
12817
|
`Question: ${triageEscalationQuestion(decision, issue)}`,
|
|
12333
12818
|
].join('\n'),
|
|
12334
12819
|
});
|
|
@@ -12671,7 +13156,11 @@ export class FactoryLoop {
|
|
|
12671
13156
|
waitingAgeMs,
|
|
12672
13157
|
});
|
|
12673
13158
|
try {
|
|
12674
|
-
await this.#
|
|
13159
|
+
const issue = await this.#readIssue(escalated.issue.path);
|
|
13160
|
+
if (!issue) {
|
|
13161
|
+
throw new Error(`Unable to describe stale Slack clarification for unreadable issue ${escalated.issue.key}`);
|
|
13162
|
+
}
|
|
13163
|
+
await this.#slack.reply(waiting.threadId, clarificationStaleSlackText(escalated, issue, this.#config.slack.stakeholderUserIds));
|
|
12675
13164
|
const completed = await this.#state.completeClarificationEscalation(this.#workspaceId, key, this.#clarificationWakeOwner, this.#clock.now());
|
|
12676
13165
|
if (!completed) {
|
|
12677
13166
|
this.#increment('clarificationEscalationOwnershipLost');
|
|
@@ -12935,6 +13424,9 @@ export class FactoryLoop {
|
|
|
12935
13424
|
this.#increment('clarificationWakesQueuedForCapacity');
|
|
12936
13425
|
return;
|
|
12937
13426
|
}
|
|
13427
|
+
// A human-answer wake starts a new agent-hold generation. Time spent
|
|
13428
|
+
// parked with the previous team released must not consume its deadline.
|
|
13429
|
+
record.heldSinceAtMs = undefined;
|
|
12938
13430
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
12939
13431
|
batch.complete(waiting.issue);
|
|
12940
13432
|
await this.#state.releaseClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner);
|
|
@@ -12964,8 +13456,10 @@ export class FactoryLoop {
|
|
|
12964
13456
|
}
|
|
12965
13457
|
: await this.#resumeOrColdStartClarificationAgent(parked.name, tracked, waiting);
|
|
12966
13458
|
const invocationId = batch.invocationIdFor(record.issue, tracked.spec);
|
|
13459
|
+
record.heldSinceAtMs ??= this.#clock.now();
|
|
12967
13460
|
batch.recordSpawn(record, tracked.spec, invocationId, result);
|
|
12968
13461
|
await this.#saveDispatchLifecycle(record, 'dispatching');
|
|
13462
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
12969
13463
|
const live = record.agents.get(result.name);
|
|
12970
13464
|
if (live) {
|
|
12971
13465
|
resumed.push([result.name, live]);
|
|
@@ -13104,6 +13598,7 @@ export class FactoryLoop {
|
|
|
13104
13598
|
const resumed = await this.#fleet.resume({
|
|
13105
13599
|
name,
|
|
13106
13600
|
sessionRef: tracked.sessionRef,
|
|
13601
|
+
identityKey: dispatchAgentIdentityKey(waiting.issue, tracked.spec.role),
|
|
13107
13602
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
13108
13603
|
capability: tracked.spec.capability,
|
|
13109
13604
|
repo: tracked.spec.repo,
|
|
@@ -13134,6 +13629,7 @@ export class FactoryLoop {
|
|
|
13134
13629
|
return await this.#fleet.spawn({
|
|
13135
13630
|
name,
|
|
13136
13631
|
capability: tracked.spec.capability,
|
|
13632
|
+
identityKey: dispatchAgentIdentityKey(waiting.issue, tracked.spec.role),
|
|
13137
13633
|
node: tracked.result?.node ?? tracked.spec.node ?? 'self',
|
|
13138
13634
|
task,
|
|
13139
13635
|
workflow: tracked.spec.workflow,
|
|
@@ -13447,6 +13943,15 @@ export class FactoryLoop {
|
|
|
13447
13943
|
return hasTitlePrefix(issue.title, FACTORY_E2E_MARKER);
|
|
13448
13944
|
}
|
|
13449
13945
|
}
|
|
13946
|
+
const defaultGithubWriteback = (config, mount) => {
|
|
13947
|
+
if (config.github.identity !== 'app') {
|
|
13948
|
+
return new GhCliGithubWriteback();
|
|
13949
|
+
}
|
|
13950
|
+
if (!mount.githubWrite) {
|
|
13951
|
+
throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
|
|
13952
|
+
}
|
|
13953
|
+
return new AppGithubWriteback(mount.githubWrite);
|
|
13954
|
+
};
|
|
13450
13955
|
export function parseLinearIssue(path, content) {
|
|
13451
13956
|
const parsed = parseJsonContent(content);
|
|
13452
13957
|
const payload = wrappedPayload(parsed);
|
|
@@ -13784,6 +14289,82 @@ function dispatchIssueUrl(issue) {
|
|
|
13784
14289
|
?? stringValue(source?.url)
|
|
13785
14290
|
?? issue.path;
|
|
13786
14291
|
}
|
|
14292
|
+
const SLACK_ISSUE_TITLE_MAX_LENGTH = 120;
|
|
14293
|
+
function slackEscapeText(value) {
|
|
14294
|
+
return value
|
|
14295
|
+
.replace(/&/gu, '&')
|
|
14296
|
+
.replace(/</gu, '<')
|
|
14297
|
+
.replace(/>/gu, '>');
|
|
14298
|
+
}
|
|
14299
|
+
function slackLinkUrl(value) {
|
|
14300
|
+
try {
|
|
14301
|
+
const url = new URL(value);
|
|
14302
|
+
if (url.protocol !== 'https:' && url.protocol !== 'http:')
|
|
14303
|
+
return undefined;
|
|
14304
|
+
return url.toString()
|
|
14305
|
+
.replace(/\|/gu, '%7C')
|
|
14306
|
+
.replace(/</gu, '%3C')
|
|
14307
|
+
.replace(/>/gu, '%3E');
|
|
14308
|
+
}
|
|
14309
|
+
catch {
|
|
14310
|
+
return undefined;
|
|
14311
|
+
}
|
|
14312
|
+
}
|
|
14313
|
+
function truncateSlackIssueTitle(title) {
|
|
14314
|
+
const normalized = title.replace(/\s+/gu, ' ').trim() || 'Untitled issue';
|
|
14315
|
+
const characters = Array.from(normalized);
|
|
14316
|
+
if (characters.length <= SLACK_ISSUE_TITLE_MAX_LENGTH)
|
|
14317
|
+
return normalized;
|
|
14318
|
+
return `${characters.slice(0, SLACK_ISSUE_TITLE_MAX_LENGTH - 1).join('').trimEnd()}…`;
|
|
14319
|
+
}
|
|
14320
|
+
function slackRepoName(repo) {
|
|
14321
|
+
const normalized = repo.trim().replace(/^\/+|\/+$/gu, '');
|
|
14322
|
+
return normalized.slice(normalized.lastIndexOf('/') + 1) || normalized || 'unknown repo';
|
|
14323
|
+
}
|
|
14324
|
+
function slackNotificationRepos(decision) {
|
|
14325
|
+
const repos = [
|
|
14326
|
+
...decision.routes.map((route) => route.repo),
|
|
14327
|
+
...decision.implementers.map((implementer) => implementer.repo),
|
|
14328
|
+
];
|
|
14329
|
+
return [...new Map(repos
|
|
14330
|
+
.filter((repo) => Boolean(repo.trim()))
|
|
14331
|
+
.map((repo) => [repo.trim().toLowerCase(), repo.trim()])).values()];
|
|
14332
|
+
}
|
|
14333
|
+
function slackRepoList(repos) {
|
|
14334
|
+
const names = [...new Set(repos.map(slackRepoName))];
|
|
14335
|
+
return slackEscapeText(names.join(', ') || 'unknown repo');
|
|
14336
|
+
}
|
|
14337
|
+
function slackIssueSubject(issue, repos = []) {
|
|
14338
|
+
const githubSource = githubIssueSourceRef(issue);
|
|
14339
|
+
const githubPath = githubIssuePathParts(issue.path);
|
|
14340
|
+
const githubIdentity = githubSource ?? githubPath;
|
|
14341
|
+
const payload = wrappedPayload(issue.raw);
|
|
14342
|
+
const source = asRecord(payload.source);
|
|
14343
|
+
const fallbackIssueUrl = githubIdentity
|
|
14344
|
+
? `https://github.com/${githubIdentity.owner}/${githubIdentity.repo}/issues/${githubIdentity.number}`
|
|
14345
|
+
: `https://linear.app/issue/${encodeURIComponent(issue.key)}`;
|
|
14346
|
+
const issueUrl = slackLinkUrl(githubSource?.url
|
|
14347
|
+
?? stringValue(payload.url)
|
|
14348
|
+
?? stringValue(payload.html_url)
|
|
14349
|
+
?? stringValue(source?.url)
|
|
14350
|
+
?? fallbackIssueUrl) ?? fallbackIssueUrl;
|
|
14351
|
+
const repoNames = [...new Set(repos.map(slackRepoName))];
|
|
14352
|
+
const label = githubIdentity
|
|
14353
|
+
? `${githubIdentity.repo}#${githubIdentity.number}`
|
|
14354
|
+
: `${repoNames.join(', ') || 'unknown repo'} · ${issue.key}`;
|
|
14355
|
+
return `<${issueUrl}|${slackEscapeText(label)}> — ${slackEscapeText(truncateSlackIssueTitle(issue.title))}`;
|
|
14356
|
+
}
|
|
14357
|
+
function slackPullRequestLink(pullRequest) {
|
|
14358
|
+
const fallbackUrl = `https://github.com/${pullRequest.repo}/pull/${pullRequest.number}`;
|
|
14359
|
+
const url = slackLinkUrl(pullRequest.url ?? fallbackUrl) ?? fallbackUrl;
|
|
14360
|
+
const label = `${slackRepoName(pullRequest.repo)}#${pullRequest.number}`;
|
|
14361
|
+
return `<${url}|${slackEscapeText(label)}>`;
|
|
14362
|
+
}
|
|
14363
|
+
function uniqueSlackPullRequestRefs(refs) {
|
|
14364
|
+
return [...new Map(refs
|
|
14365
|
+
.filter((ref) => Boolean(ref.repo.trim()) && Number.isSafeInteger(ref.number) && ref.number > 0)
|
|
14366
|
+
.map((ref) => [`${ref.repo.trim().toLowerCase()}#${ref.number}`, { ...ref, repo: ref.repo.trim() }])).values()];
|
|
14367
|
+
}
|
|
13787
14368
|
function ticketDispatchNotificationText(payload) {
|
|
13788
14369
|
return `${payload.summary}\n${JSON.stringify(payload)}`;
|
|
13789
14370
|
}
|
|
@@ -13966,13 +14547,19 @@ function labelDerivedDispatchDecision(liveIssue, decision, config) {
|
|
|
13966
14547
|
maxImplementers,
|
|
13967
14548
|
};
|
|
13968
14549
|
}
|
|
13969
|
-
|
|
13970
|
-
|
|
14550
|
+
// Swarm always shares one checkout (lead + workers collaborate live over a
|
|
14551
|
+
// shared relay channel), so — like single/workflow — only the first matched
|
|
14552
|
+
// label route is used; it is never fanned out across repo labels like team.
|
|
14553
|
+
const selectedRoutes = scope === 'single' || scope === 'swarm'
|
|
14554
|
+
? routesByLabel.routes.slice(0, 1)
|
|
14555
|
+
: routesByLabel.routes;
|
|
13971
14556
|
const selectedImplementers = scope === 'team'
|
|
13972
|
-
?
|
|
14557
|
+
? routesByLabel.routes.map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route))
|
|
13973
14558
|
: scope === 'single'
|
|
13974
|
-
?
|
|
13975
|
-
:
|
|
14559
|
+
? routesByLabel.routes.slice(0, 1).map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route))
|
|
14560
|
+
: scope === 'swarm'
|
|
14561
|
+
? routeSwarmImplementerSpecs(liveIssue, config, selectedRoutes[0]?.route, maxImplementers)
|
|
14562
|
+
: [];
|
|
13976
14563
|
const routes = selectedRoutes.map(({ route }) => route);
|
|
13977
14564
|
const workflow = scope === 'workflow'
|
|
13978
14565
|
? routeWorkflowSpec(liveIssue, config, selectedRoutes, decision.workflow)
|
|
@@ -14101,6 +14688,24 @@ function routeImplementerSpec(issue, config, slug, route) {
|
|
|
14101
14688
|
node: 'self',
|
|
14102
14689
|
};
|
|
14103
14690
|
}
|
|
14691
|
+
function routeSwarmImplementerSpecs(issue, config, route, maxImplementers) {
|
|
14692
|
+
if (!route) {
|
|
14693
|
+
return [];
|
|
14694
|
+
}
|
|
14695
|
+
const channel = swarmChannel(issue);
|
|
14696
|
+
return swarmMemberSlugs(maxImplementers).map((slug) => ({
|
|
14697
|
+
name: agentNameForRole(issue, 'impl', { repo: route.repo, discriminator: slug }),
|
|
14698
|
+
role: 'implementer',
|
|
14699
|
+
capability: config.agentCapabilities.implementer,
|
|
14700
|
+
model: config.models.implementer,
|
|
14701
|
+
task: swarmTaskFor(issue, route, slug, channel),
|
|
14702
|
+
repo: route.repo,
|
|
14703
|
+
clonePath: route.clonePath,
|
|
14704
|
+
channel,
|
|
14705
|
+
swarmRole: slug === 'lead' ? 'lead' : 'worker',
|
|
14706
|
+
node: 'self',
|
|
14707
|
+
}));
|
|
14708
|
+
}
|
|
14104
14709
|
function decisionWithLifecycleBranches(decision, runId, opts = {}) {
|
|
14105
14710
|
const implementerBranch = (spec) => {
|
|
14106
14711
|
const runSuffix = `-${runId.slice(0, 8)}`;
|
|
@@ -15247,12 +15852,123 @@ const isAllowedFactoryDraft = async (path, content, opts, mount, config) => {
|
|
|
15247
15852
|
if (/^\/slack\/channels\/[^/]+\/messages\/.+/u.test(path)) {
|
|
15248
15853
|
return true;
|
|
15249
15854
|
}
|
|
15250
|
-
if (
|
|
15855
|
+
if (await isAllowedFactoryGithubDraft(path, content, opts, mount, config))
|
|
15856
|
+
return true;
|
|
15857
|
+
return false;
|
|
15858
|
+
};
|
|
15859
|
+
const isFactoryGithubAuthoredArtifactPath = (path) => /^\/github\/repos\/[^/]+\/[^/]+\/(?:pull-requests\/factory-[^/]+\.json|refs\/(?:factory\.json|refs%2Fheads%2Ffactory%2F[^/]+\.json)|pulls\/[1-9]\d*\/close\.json)$/iu.test(path);
|
|
15860
|
+
export const isAllowedFactoryGithubArtifactDraft = (path, opts) => opts?.guarded === true && isFactoryGithubAuthoredArtifactPath(path);
|
|
15861
|
+
const factoryGithubIssueWriteTarget = (path) => {
|
|
15862
|
+
const match = /^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/([1-9]\d*)(?:\.json|\/(comments|labels)\/([^/]+))$/iu.exec(path);
|
|
15863
|
+
if (!match?.[1] || !match[2] || !match[3])
|
|
15864
|
+
return undefined;
|
|
15865
|
+
const child = match[4];
|
|
15866
|
+
const filename = match[5];
|
|
15867
|
+
if (child === 'comments' && (!filename || !isFactoryGithubIssueCommentDraftName(filename)))
|
|
15868
|
+
return undefined;
|
|
15869
|
+
if (child === 'labels' && (!filename || !isFactoryGithubOperationDraftName(filename)))
|
|
15870
|
+
return undefined;
|
|
15871
|
+
try {
|
|
15872
|
+
return {
|
|
15873
|
+
owner: decodeURIComponent(match[1]),
|
|
15874
|
+
repo: decodeURIComponent(match[2]),
|
|
15875
|
+
number: Number(match[3]),
|
|
15876
|
+
kind: child === 'comments' ? 'comment' : child === 'labels' ? 'label-operation' : 'issue-update',
|
|
15877
|
+
};
|
|
15878
|
+
}
|
|
15879
|
+
catch {
|
|
15880
|
+
return undefined;
|
|
15881
|
+
}
|
|
15882
|
+
};
|
|
15883
|
+
const factoryGithubRepositoryLabelWriteTarget = (path) => {
|
|
15884
|
+
const match = /^\/github\/repos\/([^/]+)\/([^/]+)\/labels\/([^/]+)$/iu.exec(path);
|
|
15885
|
+
if (!match?.[1] || !match[2] || !match[3] || !isFactoryGithubOperationDraftName(match[3]))
|
|
15886
|
+
return undefined;
|
|
15887
|
+
try {
|
|
15888
|
+
return { owner: decodeURIComponent(match[1]), repo: decodeURIComponent(match[2]) };
|
|
15889
|
+
}
|
|
15890
|
+
catch {
|
|
15891
|
+
return undefined;
|
|
15892
|
+
}
|
|
15893
|
+
};
|
|
15894
|
+
const githubLifecycleLabel = (name) => Object.values(FACTORY_GITHUB_STATUS_LABELS).find((label) => label.name === name);
|
|
15895
|
+
const hasExactKeys = (value, keys) => {
|
|
15896
|
+
const actual = Object.keys(value).sort();
|
|
15897
|
+
const expected = [...keys].sort();
|
|
15898
|
+
return actual.length === expected.length && actual.every((key, index) => key === expected[index]);
|
|
15899
|
+
};
|
|
15900
|
+
const isAllowedFactoryGithubIssueWriteContent = (kind, content) => {
|
|
15901
|
+
const value = asRecord(content);
|
|
15902
|
+
if (!value)
|
|
15903
|
+
return false;
|
|
15904
|
+
if (kind === 'issue-update') {
|
|
15905
|
+
return hasExactKeys(value, ['state']) && value.state === 'closed';
|
|
15906
|
+
}
|
|
15907
|
+
if (kind === 'comment') {
|
|
15908
|
+
return hasExactKeys(value, ['body']) && typeof value.body === 'string' && value.body.trim().length > 0;
|
|
15909
|
+
}
|
|
15910
|
+
if (value.operation === 'add') {
|
|
15911
|
+
return hasExactKeys(value, ['labels', 'operation']) &&
|
|
15912
|
+
Array.isArray(value.labels) && value.labels.length === 1 && Boolean(githubLifecycleLabel(value.labels[0]));
|
|
15913
|
+
}
|
|
15914
|
+
return value.operation === 'remove' && hasExactKeys(value, ['label', 'operation']) && Boolean(githubLifecycleLabel(value.label));
|
|
15915
|
+
};
|
|
15916
|
+
const isAllowedFactoryGithubRepositoryLabelContent = (content) => {
|
|
15917
|
+
const value = asRecord(content);
|
|
15918
|
+
if (!value || !hasExactKeys(value, ['color', 'description', 'name']))
|
|
15919
|
+
return false;
|
|
15920
|
+
const expected = githubLifecycleLabel(value.name);
|
|
15921
|
+
return Boolean(expected && expected.color === value.color && expected.description === value.description);
|
|
15922
|
+
};
|
|
15923
|
+
/**
|
|
15924
|
+
* Last-resort mount guard for Factory-authored GitHub drafts. PR/ref paths are
|
|
15925
|
+
* intrinsically Factory-owned. Issue mutations additionally require a current
|
|
15926
|
+
* in-scope issue projection in one of Relayfile's supported repository layouts.
|
|
15927
|
+
*/
|
|
15928
|
+
export const isAllowedFactoryGithubDraft = async (path, content, opts, mount, config) => {
|
|
15929
|
+
if (!opts?.guarded)
|
|
15930
|
+
return false;
|
|
15931
|
+
if (isAllowedFactoryGithubArtifactDraft(path, opts))
|
|
15251
15932
|
return true;
|
|
15933
|
+
const repositoryLabelTarget = factoryGithubRepositoryLabelWriteTarget(path);
|
|
15934
|
+
if (repositoryLabelTarget) {
|
|
15935
|
+
const repoPath = `/github/repos/${encodeURIComponent(repositoryLabelTarget.owner)}/${encodeURIComponent(repositoryLabelTarget.repo)}/`;
|
|
15936
|
+
return isConfiguredGithubRepoPath(repoPath, config) && isAllowedFactoryGithubRepositoryLabelContent(content);
|
|
15937
|
+
}
|
|
15938
|
+
const target = factoryGithubIssueWriteTarget(path);
|
|
15939
|
+
if (!target)
|
|
15940
|
+
return false;
|
|
15941
|
+
if (!isAllowedFactoryGithubIssueWriteContent(target.kind, content))
|
|
15942
|
+
return false;
|
|
15943
|
+
if (target.kind === 'comment') {
|
|
15944
|
+
const body = asRecord(content)?.body;
|
|
15945
|
+
const draftName = path.slice(path.lastIndexOf('/') + 1);
|
|
15946
|
+
if (typeof body !== 'string' || draftName !== factoryGithubIssueCommentDraftName(body))
|
|
15947
|
+
return false;
|
|
15948
|
+
}
|
|
15949
|
+
const repoPath = `/github/repos/${encodeURIComponent(target.owner)}/${encodeURIComponent(target.repo)}`;
|
|
15950
|
+
if (!isConfiguredGithubRepoPath(`${repoPath}/`, config))
|
|
15951
|
+
return false;
|
|
15952
|
+
const compactRepo = `${encodeURIComponent(target.owner)}__${encodeURIComponent(target.repo)}`;
|
|
15953
|
+
const candidates = [
|
|
15954
|
+
`${repoPath}/issues/by-id/${target.number}.json`,
|
|
15955
|
+
`/github/repos/${compactRepo}/issues/by-id/${target.number}.json`,
|
|
15956
|
+
`${repoPath}/issues/${target.number}/meta.json`,
|
|
15957
|
+
`/github/repos/${compactRepo}/issues/${target.number}/meta.json`,
|
|
15958
|
+
`${repoPath}/issues/${target.number}.json`,
|
|
15959
|
+
`/github/repos/${compactRepo}/issues/${target.number}.json`,
|
|
15960
|
+
];
|
|
15961
|
+
for (const candidate of candidates) {
|
|
15962
|
+
try {
|
|
15963
|
+
const issue = parseGithubFactoryIssue(candidate, (await mount.readFile(candidate)).content);
|
|
15964
|
+
return issue.state?.name === 'open' && isInFactoryScope(issue, config.safety);
|
|
15965
|
+
}
|
|
15966
|
+
catch {
|
|
15967
|
+
// Try the next canonical/alias shape. Any total miss fails closed.
|
|
15968
|
+
}
|
|
15252
15969
|
}
|
|
15253
15970
|
return false;
|
|
15254
15971
|
};
|
|
15255
|
-
const isFactoryGithubWritebackPath = (path) => /^\/github\/repos\/[^/]+\/[^/]+\/(?:pull-requests\/factory-[^/]+\.json|refs\/(?:factory\.json|refs%2Fheads%2Ffactory%2F[^/]+\.json)|pulls\/[1-9]\d*\/close\.json)$/iu.test(path);
|
|
15256
15972
|
const isIssuePathInFactoryScope = async (mount, path, config) => {
|
|
15257
15973
|
try {
|
|
15258
15974
|
return isInFactoryScope(parseLinearIssue(path, (await mount.readFile(path)).content), config.safety);
|
|
@@ -15705,6 +16421,7 @@ const cloneTrackedAgent = (tracked) => ({
|
|
|
15705
16421
|
result: tracked.result ? { ...tracked.result } : undefined,
|
|
15706
16422
|
sessionRef: tracked.sessionRef,
|
|
15707
16423
|
unreachableWakeResumedSessionRef: tracked.unreachableWakeResumedSessionRef,
|
|
16424
|
+
releasedAtMs: tracked.releasedAtMs,
|
|
15708
16425
|
});
|
|
15709
16426
|
const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') => ({
|
|
15710
16427
|
spec: {
|
|
@@ -15722,6 +16439,28 @@ const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') =>
|
|
|
15722
16439
|
});
|
|
15723
16440
|
const isTerminalDispatchLifecycle = (lifecycle) => lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned';
|
|
15724
16441
|
const isTerminalDispatchPhase = (phase) => phase === 'complete' || phase === 'abandoned';
|
|
16442
|
+
const heldAgentsForRecord = (record, nowMs, holdTimeoutMs, terminalState) => {
|
|
16443
|
+
if (record.dryRun || record.heldSinceAtMs === undefined)
|
|
16444
|
+
return [];
|
|
16445
|
+
const heldSinceAtMs = record.heldSinceAtMs;
|
|
16446
|
+
const holdDeadlineAtMs = heldSinceAtMs + holdTimeoutMs;
|
|
16447
|
+
const heldForMs = Math.max(0, nowMs - heldSinceAtMs);
|
|
16448
|
+
return [...record.agents]
|
|
16449
|
+
.filter(([, tracked]) => Boolean(tracked.result))
|
|
16450
|
+
.map(([name, tracked]) => ({
|
|
16451
|
+
name,
|
|
16452
|
+
role: tracked.spec.role,
|
|
16453
|
+
issue: { ...record.issue },
|
|
16454
|
+
...(record.lifecyclePhase ? { lifecyclePhase: record.lifecyclePhase } : {}),
|
|
16455
|
+
waitingForTerminalState: terminalState,
|
|
16456
|
+
heldSince: new Date(heldSinceAtMs).toISOString(),
|
|
16457
|
+
heldSinceAtMs,
|
|
16458
|
+
heldForMs,
|
|
16459
|
+
holdDeadline: new Date(holdDeadlineAtMs).toISOString(),
|
|
16460
|
+
holdDeadlineAtMs,
|
|
16461
|
+
pastDeadline: nowMs >= holdDeadlineAtMs,
|
|
16462
|
+
}));
|
|
16463
|
+
};
|
|
15725
16464
|
const costUsageGroupId = (runId, tracked) => JSON.stringify([runId, tracked.spec.invocationId ?? tracked.spec.name]);
|
|
15726
16465
|
const costEntryId = (groupId, model) => JSON.stringify([groupId, model]);
|
|
15727
16466
|
const publishedPullRequests = (lifecycle) => {
|
|
@@ -15759,7 +16498,11 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
15759
16498
|
decision: structuredClone(record.decision),
|
|
15760
16499
|
dryRun: record.dryRun,
|
|
15761
16500
|
phase,
|
|
15762
|
-
agents: [...record.agents].map(([name, tracked]) => ({
|
|
16501
|
+
agents: [...record.agents].map(([name, tracked]) => ({
|
|
16502
|
+
name,
|
|
16503
|
+
tracked: cloneTrackedAgent(tracked),
|
|
16504
|
+
...(tracked.releasedAtMs !== undefined ? { releasedAtMs: tracked.releasedAtMs } : {}),
|
|
16505
|
+
})),
|
|
15763
16506
|
invocationIds: [...record.invocationIds],
|
|
15764
16507
|
result: record.result ? structuredClone(record.result) : undefined,
|
|
15765
16508
|
...(record.dispatchClaim ? { dispatchClaim: { ...record.dispatchClaim } } : {}),
|
|
@@ -15767,16 +16510,27 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
15767
16510
|
...(pullRequest ? { pullRequest: { ...pullRequest } } : {}),
|
|
15768
16511
|
...(releaseReason ? { releaseReason } : {}),
|
|
15769
16512
|
...(cost ? { cost: structuredClone(cost) } : {}),
|
|
16513
|
+
...(record.heldSinceAtMs !== undefined ? { heldSinceAtMs: record.heldSinceAtMs } : {}),
|
|
15770
16514
|
updatedAtMs,
|
|
15771
16515
|
});
|
|
15772
16516
|
const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
15773
16517
|
issue: { ...lifecycle.issue },
|
|
15774
16518
|
decision: structuredClone(lifecycle.decision),
|
|
15775
16519
|
dryRun: lifecycle.dryRun,
|
|
15776
|
-
|
|
16520
|
+
// Release state has to survive takeover with the agents themselves. Without
|
|
16521
|
+
// it the rebuilt record reads a released placement as a live worker and the
|
|
16522
|
+
// spawn gate answers for a process that no longer exists.
|
|
16523
|
+
agents: new Map(lifecycle.agents.map((agent) => [agent.name, {
|
|
16524
|
+
...cloneTrackedAgent(agent.tracked),
|
|
16525
|
+
releasedAtMs: agent.releasedAtMs ?? agent.tracked.releasedAtMs,
|
|
16526
|
+
}])),
|
|
15777
16527
|
invocationIds: new Set(lifecycle.invocationIds),
|
|
15778
16528
|
result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
|
|
15779
16529
|
...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
|
|
16530
|
+
heldSinceAtMs: lifecycle.heldSinceAtMs ?? (lifecycle.agents.some((agent) => agent.releasedAtMs === undefined)
|
|
16531
|
+
? lifecycle.updatedAtMs
|
|
16532
|
+
: undefined),
|
|
16533
|
+
lifecyclePhase: lifecycle.phase,
|
|
15780
16534
|
});
|
|
15781
16535
|
const dispatchResultFromLifecycle = (lifecycle) => lifecycle.result ? structuredClone(lifecycle.result) : {
|
|
15782
16536
|
issue: { ...lifecycle.issue },
|
|
@@ -15796,7 +16550,14 @@ const triageEscalationReason = (decision) => {
|
|
|
15796
16550
|
}
|
|
15797
16551
|
return `${reasons.join(' and ')}${decision.rationale ? `: ${decision.rationale}` : ''}`;
|
|
15798
16552
|
};
|
|
15799
|
-
|
|
16553
|
+
/**
|
|
16554
|
+
* The dispatch claim race: another writer changed the issue's live state
|
|
16555
|
+
* between this process reading it and writing back, so the dispatch was
|
|
16556
|
+
* abandoned. Exported so callers outside this module — notably the CLI, which
|
|
16557
|
+
* turns it into a distinct exit code — can recognize it by type rather than by
|
|
16558
|
+
* matching on `error.name`, and so tests can construct a genuine instance.
|
|
16559
|
+
*/
|
|
16560
|
+
export class LiveDispatchStateChangedError extends Error {
|
|
15800
16561
|
issueKey;
|
|
15801
16562
|
constructor(issueKey) {
|
|
15802
16563
|
super(`Live state changed before writeback for ${issueKey}`);
|
|
@@ -15804,6 +16565,10 @@ class LiveDispatchStateChangedError extends Error {
|
|
|
15804
16565
|
this.issueKey = issueKey;
|
|
15805
16566
|
}
|
|
15806
16567
|
}
|
|
16568
|
+
/** Whether a thrown value is a {@link LiveDispatchStateChangedError}. */
|
|
16569
|
+
export function isLiveDispatchStateChangedError(error) {
|
|
16570
|
+
return error instanceof LiveDispatchStateChangedError;
|
|
16571
|
+
}
|
|
15807
16572
|
const triageEscalationQuestion = (decision, issue) => {
|
|
15808
16573
|
const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
|
|
15809
16574
|
const subject = issue?.title?.trim() || decision.issue.key;
|
|
@@ -15971,14 +16736,14 @@ const slackUserIdMatchingIdentity = (payload, identity) => {
|
|
|
15971
16736
|
? userId
|
|
15972
16737
|
: undefined;
|
|
15973
16738
|
};
|
|
15974
|
-
const agentQuestionSlackText = (issue, question, stakeholderUserIds = []) => [
|
|
15975
|
-
slackMentions(stakeholderUserIds),
|
|
15976
|
-
`${
|
|
16739
|
+
const agentQuestionSlackText = (issue, question, stakeholderUserIds = [], repos = []) => [
|
|
16740
|
+
`${slackMentions(stakeholderUserIds) ?? ''} ${slackIssueSubject(issue, repos)}`.trim(),
|
|
16741
|
+
`${question.agentName} needs input.`,
|
|
15977
16742
|
`Question: ${question.question}`,
|
|
15978
16743
|
].filter((line) => Boolean(line)).join('\n');
|
|
15979
|
-
const clarificationStaleSlackText = (waiting, stakeholderUserIds = []) => [
|
|
15980
|
-
slackMentions(stakeholderUserIds),
|
|
15981
|
-
|
|
16744
|
+
const clarificationStaleSlackText = (waiting, issue, stakeholderUserIds = []) => [
|
|
16745
|
+
`${slackMentions(stakeholderUserIds) ?? ''} ${slackIssueSubject(issue, slackNotificationRepos(waiting.decision))}`.trim(),
|
|
16746
|
+
'This issue has been parked for seven days without a reply.',
|
|
15982
16747
|
`Question from ${waiting.askerName}: ${waiting.question} Reply in this thread to wake the saved agent team, or move the issue out of Agent Implementing to cancel the wake.`,
|
|
15983
16748
|
].filter((line) => Boolean(line)).join('\n');
|
|
15984
16749
|
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|