@agent-relay/factory 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +2 -0
- package/dist/cli/fleet.js.map +1 -1
- package/dist/dispatch/templates.d.ts +5 -1
- package/dist/dispatch/templates.d.ts.map +1 -1
- package/dist/dispatch/templates.js +27 -12
- package/dist/dispatch/templates.js.map +1 -1
- package/dist/fleet/internal-fleet-client.js +1 -1
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/git/agent-worktree.d.ts +18 -0
- package/dist/git/agent-worktree.d.ts.map +1 -0
- package/dist/git/agent-worktree.js +172 -0
- package/dist/git/agent-worktree.js.map +1 -0
- package/dist/mount/local-mount-preflight.js +6 -8
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +1 -0
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +18 -2
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +510 -105
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +2 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -0
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +2 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +3 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/ports/worktree.d.ts +12 -0
- package/dist/ports/worktree.d.ts.map +1 -0
- package/dist/ports/worktree.js +2 -0
- package/dist/ports/worktree.js.map +1 -0
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/slack.d.ts.map +1 -1
- package/dist/writeback/slack.js +15 -8
- package/dist/writeback/slack.js.map +1 -1
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ import { FactoryConfigSchema } from '../config/schema.js';
|
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
8
|
+
import { factoryWorktreePath } from '../git/agent-worktree.js';
|
|
8
9
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
9
10
|
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
10
11
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
@@ -34,15 +35,6 @@ const GITHUB_ISSUE_ROOT = '/github/repos';
|
|
|
34
35
|
const READY_EVENTS_LIMIT = 100;
|
|
35
36
|
const LIVE_ISSUE_GLOB = `${ISSUE_ROOT}/**`;
|
|
36
37
|
const LIVE_RELAYFLOW_GLOB = '/**';
|
|
37
|
-
// Subscribe broadly under /github/repos and let isGithubIssueFilePath() /
|
|
38
|
-
// githubPullPathParts() re-validate the exact shape in the callback.
|
|
39
|
-
// globMatchesPath() treats a non-terminal `**` as a single-segment wildcard, so
|
|
40
|
-
// a more specific glob like `.../issues/**/*.json` would miss the two-segment
|
|
41
|
-
// <owner>/<repo> prefix and the nested <number>__<slug>/meta.json (and
|
|
42
|
-
// directory) event shapes this factory accepts. A terminal `**` prefix-matches
|
|
43
|
-
// every descendant, so all supported issue AND pull-request path variants reach
|
|
44
|
-
// the handler — including the PR change events the babysitter reacts to.
|
|
45
|
-
export const LIVE_GITHUB_ISSUE_GLOB = `${GITHUB_ISSUE_ROOT}/**`;
|
|
46
38
|
const LIVE_DEDUPE_LIMIT = 5_000;
|
|
47
39
|
const LIVE_EVENT_DRAIN_BATCH_SIZE = 5;
|
|
48
40
|
const COMPLETION_SWEEP_INTERVAL_MS = 15_000;
|
|
@@ -125,6 +117,7 @@ export class FactoryLoop {
|
|
|
125
117
|
#state;
|
|
126
118
|
#workspaceId;
|
|
127
119
|
#relayflows;
|
|
120
|
+
#worktrees;
|
|
128
121
|
#batchView;
|
|
129
122
|
#batchReady;
|
|
130
123
|
#listeners = new Map();
|
|
@@ -158,6 +151,7 @@ export class FactoryLoop {
|
|
|
158
151
|
#dispatchTerminalWaiters = new Map();
|
|
159
152
|
#dispatchLifecycleRetryTimers = new Map();
|
|
160
153
|
#dispatchLifecycleDrives = new Set();
|
|
154
|
+
#localReleaseCheckpoints = new Map();
|
|
161
155
|
#dispatchLifecycleRenewTimer;
|
|
162
156
|
#clarificationSweepTimer;
|
|
163
157
|
#clarificationSweepDueAtMs;
|
|
@@ -260,6 +254,7 @@ export class FactoryLoop {
|
|
|
260
254
|
this.#terminationGraceMs = ports.terminationGraceMs;
|
|
261
255
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
262
256
|
this.#relayflows = ports.relayflows;
|
|
257
|
+
this.#worktrees = ports.worktrees;
|
|
263
258
|
this.#state = ports.stateStore ?? new InMemoryStateStore({
|
|
264
259
|
batchSize: config.batchSize,
|
|
265
260
|
agentQuestionDedupeLimit: AGENT_QUESTION_DEDUPE_LIMIT,
|
|
@@ -453,7 +448,7 @@ export class FactoryLoop {
|
|
|
453
448
|
if ((opts.mode ?? 'live') === 'live') {
|
|
454
449
|
this.#started = true;
|
|
455
450
|
try {
|
|
456
|
-
await this.#startLiveSubscription(opts.liveSubscription);
|
|
451
|
+
await this.#startLiveSubscription(issueSource, opts.liveSubscription);
|
|
457
452
|
await this.#rearmSlackReplyWatchers();
|
|
458
453
|
await this.#drainReadyClarificationWake();
|
|
459
454
|
await this.#rearmGithubIssueCommentWatchers();
|
|
@@ -467,7 +462,7 @@ export class FactoryLoop {
|
|
|
467
462
|
}
|
|
468
463
|
}
|
|
469
464
|
await this.#backfillReadyIssues();
|
|
470
|
-
this.#subscription = this.#mount.subscribe(this.#subscriptionGlobs([`${ISSUE_ROOT}/**/*.json
|
|
465
|
+
this.#subscription = this.#mount.subscribe(this.#subscriptionGlobs(issueSource, [`${ISSUE_ROOT}/**/*.json`]), (event) => {
|
|
471
466
|
void this.#dispatchRelayflowEvent(event);
|
|
472
467
|
// The SDK types `resource` as always-present, but the polling fallback and
|
|
473
468
|
// degraded-sync paths can deliver events without it. Skip those rather
|
|
@@ -610,7 +605,7 @@ export class FactoryLoop {
|
|
|
610
605
|
async dispose() {
|
|
611
606
|
await this.stop();
|
|
612
607
|
}
|
|
613
|
-
async #startLiveSubscription(overrides = {}) {
|
|
608
|
+
async #startLiveSubscription(issueSource, overrides = {}) {
|
|
614
609
|
const options = this.#liveOptions(overrides);
|
|
615
610
|
await this.#startLiveHeartbeat();
|
|
616
611
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
@@ -624,41 +619,51 @@ export class FactoryLoop {
|
|
|
624
619
|
replaySkewMarginMs: this.#liveReplaySkewMarginMs,
|
|
625
620
|
highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
|
|
626
621
|
});
|
|
627
|
-
// Register the live subscription BEFORE the startup
|
|
622
|
+
// Register the live subscription BEFORE the startup backfill so an issue
|
|
628
623
|
// that becomes Ready *during* the pull is captured, not lost in the window
|
|
629
624
|
// between listTree and subscribe. Events buffer (deferred drain) until the
|
|
630
625
|
// pull finishes; batch dedupe then suppresses any overlap with what the
|
|
631
626
|
// pull already dispatched.
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
627
|
+
this.#deferLiveEventDrain = true;
|
|
628
|
+
try {
|
|
629
|
+
if (options.transport === 'poll') {
|
|
630
|
+
// Capture the cursor before the backfill. Events written while listTree
|
|
631
|
+
// is running are then picked up by the first poll instead of falling
|
|
632
|
+
// into a cursor-advance gap.
|
|
633
|
+
this.#liveEventCursor = await this.#currentEventCursor(options.eventLimit);
|
|
634
|
+
}
|
|
635
|
+
else {
|
|
636
|
+
this.#subscription = this.#mount.subscribe(this.#subscriptionGlobs(issueSource, [LIVE_ISSUE_GLOB]), (event) => {
|
|
637
|
+
this.#enqueueLiveEvent(event);
|
|
638
|
+
}, { from: 'now', coalesce: 'none' });
|
|
639
|
+
}
|
|
640
|
+
if (highWatermark.routeUnavailable) {
|
|
641
|
+
this.#increment('liveHighWatermarkFullPullFallbacks');
|
|
642
|
+
}
|
|
643
|
+
this.#increment('liveStartupBackfills');
|
|
644
|
+
this.#logger.info?.('[factory] running startup ready-issue backfill before draining buffered events', {
|
|
645
|
+
highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
|
|
646
|
+
});
|
|
644
647
|
try {
|
|
645
648
|
await this.runOnce();
|
|
646
649
|
}
|
|
647
650
|
catch (error) {
|
|
648
|
-
// A startup
|
|
649
|
-
// to the live event stream (plus any buffered events) instead of
|
|
650
|
-
// the factory down.
|
|
651
|
-
this.#increment('
|
|
651
|
+
// A startup backfill failure must not abort the daemon: log it and fall
|
|
652
|
+
// back to the live event stream (plus any buffered events) instead of
|
|
653
|
+
// leaving the factory down.
|
|
654
|
+
this.#increment('liveStartupBackfillErrors');
|
|
655
|
+
if (highWatermark.routeUnavailable) {
|
|
656
|
+
this.#increment('liveHighWatermarkFullPullErrors');
|
|
657
|
+
}
|
|
652
658
|
this.#error(error);
|
|
653
659
|
}
|
|
654
|
-
finally {
|
|
655
|
-
this.#deferLiveEventDrain = false;
|
|
656
|
-
this.#scheduleLiveEventDrain();
|
|
657
|
-
}
|
|
658
|
-
await this.#refreshLiveHeartbeatIfDue();
|
|
659
660
|
}
|
|
661
|
+
finally {
|
|
662
|
+
this.#deferLiveEventDrain = false;
|
|
663
|
+
this.#scheduleLiveEventDrain();
|
|
664
|
+
}
|
|
665
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
660
666
|
if (options.transport === 'poll') {
|
|
661
|
-
this.#liveEventCursor = await this.#currentEventCursor(options.eventLimit);
|
|
662
667
|
this.#scheduleLivePoll(0, options);
|
|
663
668
|
}
|
|
664
669
|
}
|
|
@@ -873,14 +878,25 @@ export class FactoryLoop {
|
|
|
873
878
|
return;
|
|
874
879
|
await this.#refreshLiveHeartbeat();
|
|
875
880
|
}
|
|
876
|
-
#subscriptionGlobs(
|
|
877
|
-
|
|
881
|
+
#subscriptionGlobs(issueSource, linearGlobs) {
|
|
882
|
+
if (this.#relayflows)
|
|
883
|
+
return [LIVE_RELAYFLOW_GLOB];
|
|
884
|
+
return [
|
|
885
|
+
...(issueSource === 'linear' ? linearGlobs : []),
|
|
886
|
+
...githubRepoSubscriptionGlobs(this.#config),
|
|
887
|
+
];
|
|
878
888
|
}
|
|
879
889
|
async #prepareLiveEventForDrain(event, seenIssueKeys) {
|
|
880
890
|
const path = changeEventPath(event);
|
|
881
891
|
if (!path) {
|
|
882
892
|
return { dispatchRelayflow: false };
|
|
883
893
|
}
|
|
894
|
+
if (!this.#relayflows &&
|
|
895
|
+
path.startsWith(`${GITHUB_ISSUE_ROOT}/`) &&
|
|
896
|
+
!isConfiguredGithubRepoPath(path, this.#config)) {
|
|
897
|
+
this.#increment('liveGithubEventsOutsideConfiguredRepos');
|
|
898
|
+
return { dispatchRelayflow: false };
|
|
899
|
+
}
|
|
884
900
|
const isPullPath = isGithubPullFilePath(path);
|
|
885
901
|
const babysitterEvent = this.#config.babysitter.enabled
|
|
886
902
|
? githubBabysitterEventPathParts(path)
|
|
@@ -1169,7 +1185,9 @@ export class FactoryLoop {
|
|
|
1169
1185
|
for (const path of paths) {
|
|
1170
1186
|
const issue = await this.#readIssue(path);
|
|
1171
1187
|
readyIssueReads += 1;
|
|
1172
|
-
lastReadyReadProgressAtMs = this.#logTimedProgress(
|
|
1188
|
+
lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
|
|
1189
|
+
? '[factory] GitHub ready issue read progress'
|
|
1190
|
+
: '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
|
|
1173
1191
|
if (issue && issueSource === 'linear') {
|
|
1174
1192
|
await this.#recordCanonicalIssueState(issue);
|
|
1175
1193
|
}
|
|
@@ -1487,9 +1505,15 @@ export class FactoryLoop {
|
|
|
1487
1505
|
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
1488
1506
|
// recover a persisted minimal triage task after a crash in this gap.
|
|
1489
1507
|
const durableRemoteDispatch = !dryRun && this.#fleet.placementLocality === 'remote';
|
|
1490
|
-
|
|
1508
|
+
// Local dispatches need the same deterministic branch identity as remote
|
|
1509
|
+
// ones. Without it, every worker starts in the configured shared checkout
|
|
1510
|
+
// and concurrent issues can switch each other back to the base branch.
|
|
1511
|
+
const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
|
|
1512
|
+
const lifecycleRunId = !dryRun && (durableRemoteDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
|
|
1491
1513
|
if (lifecycleRunId) {
|
|
1492
|
-
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId
|
|
1514
|
+
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
|
|
1515
|
+
isolateLocalWorktree,
|
|
1516
|
+
});
|
|
1493
1517
|
}
|
|
1494
1518
|
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
1495
1519
|
if (durableRemoteDispatch) {
|
|
@@ -1541,6 +1565,7 @@ export class FactoryLoop {
|
|
|
1541
1565
|
name: spawned.name,
|
|
1542
1566
|
tracked: cloneTrackedAgent(tracked),
|
|
1543
1567
|
persistedAtMs: this.#clock.now(),
|
|
1568
|
+
worktree: this.#agentWorktree(record, tracked.spec),
|
|
1544
1569
|
});
|
|
1545
1570
|
}
|
|
1546
1571
|
agents.push({ name: spawned.name, role: spec.role });
|
|
@@ -1585,7 +1610,12 @@ export class FactoryLoop {
|
|
|
1585
1610
|
return result;
|
|
1586
1611
|
}
|
|
1587
1612
|
catch (error) {
|
|
1588
|
-
|
|
1613
|
+
// A spawn can fail after the broker accepted it but before its ack
|
|
1614
|
+
// reached Factory. Include every planned worktree agent, not only the
|
|
1615
|
+
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
1616
|
+
const failureHandoffs = this.#dispatchFailureHandoffs(record, spawnedForReaperHandoff);
|
|
1617
|
+
await this.#persistDispatchFailureReaperHandoff(record, failureHandoffs);
|
|
1618
|
+
const worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs);
|
|
1589
1619
|
await this.#recordDispatchFailure(decision.issue);
|
|
1590
1620
|
const failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key);
|
|
1591
1621
|
await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable');
|
|
@@ -1593,6 +1623,20 @@ export class FactoryLoop {
|
|
|
1593
1623
|
if (!failedState?.terminal)
|
|
1594
1624
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
1595
1625
|
this.#error(error, decision.issue);
|
|
1626
|
+
// The teardown runs while the record still exists so it can safely
|
|
1627
|
+
// derive every shared checkout. Rewrite the registry only after abandon
|
|
1628
|
+
// removes those agents from the ordinary in-flight view.
|
|
1629
|
+
if (worktreesTornDown) {
|
|
1630
|
+
try {
|
|
1631
|
+
await this.#writeInFlightRegistry();
|
|
1632
|
+
}
|
|
1633
|
+
catch (registryError) {
|
|
1634
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after dispatch worktree teardown', {
|
|
1635
|
+
issue: record.issue,
|
|
1636
|
+
error: describeError(registryError).errorMessage,
|
|
1637
|
+
});
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1596
1640
|
throw error;
|
|
1597
1641
|
}
|
|
1598
1642
|
}
|
|
@@ -1803,6 +1847,31 @@ export class FactoryLoop {
|
|
|
1803
1847
|
}, DISPATCH_LIFECYCLE_RETRY_MS);
|
|
1804
1848
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
1805
1849
|
}
|
|
1850
|
+
#scheduleReleaseRetry(record, reason) {
|
|
1851
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
1852
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
1853
|
+
return;
|
|
1854
|
+
}
|
|
1855
|
+
const key = issueKey(record.issue);
|
|
1856
|
+
if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
|
|
1857
|
+
return;
|
|
1858
|
+
const timer = setTimeout(() => {
|
|
1859
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
1860
|
+
const drive = this.#finishDurableRelease(record, reason)
|
|
1861
|
+
.then(() => undefined)
|
|
1862
|
+
.catch((error) => {
|
|
1863
|
+
this.#logger.warn?.('[factory] local completion cleanup retry failed', {
|
|
1864
|
+
issue: record.issue.key,
|
|
1865
|
+
error: describeError(error).errorMessage,
|
|
1866
|
+
});
|
|
1867
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
1868
|
+
})
|
|
1869
|
+
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
1870
|
+
this.#dispatchLifecycleDrives.add(drive);
|
|
1871
|
+
}, DISPATCH_LIFECYCLE_RETRY_MS);
|
|
1872
|
+
timer.unref?.();
|
|
1873
|
+
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
1874
|
+
}
|
|
1806
1875
|
async #driveDispatchLifecycle(key) {
|
|
1807
1876
|
if (this.#stopping)
|
|
1808
1877
|
return;
|
|
@@ -1953,12 +2022,12 @@ export class FactoryLoop {
|
|
|
1953
2022
|
}
|
|
1954
2023
|
async #finishDurableRelease(record, releaseReason) {
|
|
1955
2024
|
const batch = await this.#batch();
|
|
1956
|
-
const next = this.#fleet.placementLocality === 'remote' ? undefined : batch.complete(record.issue);
|
|
1957
2025
|
const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done');
|
|
1958
|
-
const
|
|
2026
|
+
const releaseKey = issueKey(record.issue);
|
|
2027
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, releaseKey);
|
|
1959
2028
|
const released = new Set(lifecycle?.agents
|
|
1960
2029
|
.filter((agent) => agent.releasedAtMs !== undefined)
|
|
1961
|
-
.map((agent) => agent.name) ?? []);
|
|
2030
|
+
.map((agent) => agent.name) ?? this.#localReleaseCheckpoints.get(releaseKey) ?? []);
|
|
1962
2031
|
const failed = [];
|
|
1963
2032
|
for (const agent of record.agents) {
|
|
1964
2033
|
if (released.has(agent[0]))
|
|
@@ -1969,19 +2038,37 @@ export class FactoryLoop {
|
|
|
1969
2038
|
continue;
|
|
1970
2039
|
}
|
|
1971
2040
|
released.add(agent[0]);
|
|
2041
|
+
if (this.#fleet.placementLocality === 'local') {
|
|
2042
|
+
this.#localReleaseCheckpoints.set(releaseKey, new Set(released));
|
|
2043
|
+
}
|
|
1972
2044
|
// Persist each acknowledged release independently. A takeover retries
|
|
1973
2045
|
// only agents whose release did not reach a fenced durable checkpoint.
|
|
1974
2046
|
if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, reason, released))
|
|
1975
2047
|
return false;
|
|
1976
2048
|
}
|
|
1977
|
-
if (next)
|
|
1978
|
-
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
1979
2049
|
await this.#writeInFlightRegistry();
|
|
1980
2050
|
if (failed.length > 0) {
|
|
1981
2051
|
this.#increment('dispatchLifecycleReleaseRetries');
|
|
1982
|
-
this.#
|
|
2052
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
2053
|
+
return false;
|
|
2054
|
+
}
|
|
2055
|
+
// The PR branch is already pushed and the babysitter has declared the
|
|
2056
|
+
// current PR green with review feedback addressed. Release is now fenced,
|
|
2057
|
+
// so no agent can race cleanup of the shared per-issue worktree.
|
|
2058
|
+
try {
|
|
2059
|
+
await this.#cleanupAgentWorktrees(record);
|
|
2060
|
+
}
|
|
2061
|
+
catch {
|
|
2062
|
+
// Completion remains in-flight until the isolated checkout is gone.
|
|
2063
|
+
// Remote lifecycles retry from their durable `releasing` phase; local
|
|
2064
|
+
// lifecycles retain this record and retry directly from the same fence.
|
|
2065
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
1983
2066
|
return false;
|
|
1984
2067
|
}
|
|
2068
|
+
const next = this.#fleet.placementLocality === 'remote' ? undefined : batch.complete(record.issue);
|
|
2069
|
+
this.#localReleaseCheckpoints.delete(releaseKey);
|
|
2070
|
+
if (next)
|
|
2071
|
+
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
1985
2072
|
// Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
|
|
1986
2073
|
// the babysitter's durable ownership/wake/critical state while that epoch
|
|
1987
2074
|
// is still valid so a later reopened issue cannot inherit a stale PR owner.
|
|
@@ -2458,6 +2545,7 @@ export class FactoryLoop {
|
|
|
2458
2545
|
try {
|
|
2459
2546
|
const protectedPids = await this.#protectedPids();
|
|
2460
2547
|
let registryChanged = false;
|
|
2548
|
+
const readyToClear = new Set();
|
|
2461
2549
|
for (const [key, handoff] of handoffs) {
|
|
2462
2550
|
const roots = await this.#terminationRoots(handoff.name, handoff.tracked, protectedPids);
|
|
2463
2551
|
if (roots.pids.length === 0 && roots.status === 'unresolved') {
|
|
@@ -2470,8 +2558,6 @@ export class FactoryLoop {
|
|
|
2470
2558
|
unresolvedAgeMs,
|
|
2471
2559
|
});
|
|
2472
2560
|
if (unresolvedAgeMs >= DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS) {
|
|
2473
|
-
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2474
|
-
registryChanged = true;
|
|
2475
2561
|
this.#increment('dispatchFailureReaperHandoffsDroppedStaleUnresolved');
|
|
2476
2562
|
this.#logger.warn?.('[factory] dropped stale unresolved dispatch-failed handoff', {
|
|
2477
2563
|
agentName: handoff.name,
|
|
@@ -2481,6 +2567,7 @@ export class FactoryLoop {
|
|
|
2481
2567
|
});
|
|
2482
2568
|
try {
|
|
2483
2569
|
await this.#fleet.release(handoff.name, 'dispatch failed');
|
|
2570
|
+
readyToClear.add(key);
|
|
2484
2571
|
}
|
|
2485
2572
|
catch (error) {
|
|
2486
2573
|
this.#logger.warn?.('[factory] failed to release unresolved dispatch-failure handoff after pruning', {
|
|
@@ -2514,16 +2601,55 @@ export class FactoryLoop {
|
|
|
2514
2601
|
}
|
|
2515
2602
|
}
|
|
2516
2603
|
if (!blockingSkip) {
|
|
2517
|
-
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2518
|
-
registryChanged = true;
|
|
2519
2604
|
try {
|
|
2520
2605
|
await this.#fleet.release(handoff.name, 'dispatch failed');
|
|
2606
|
+
readyToClear.add(key);
|
|
2521
2607
|
}
|
|
2522
2608
|
catch (error) {
|
|
2523
2609
|
this.#logger.warn?.(`[factory] failed to release ${handoff.name} after dispatch-failure reap`, error);
|
|
2524
2610
|
}
|
|
2525
2611
|
}
|
|
2526
2612
|
}
|
|
2613
|
+
if (readyToClear.size > 0) {
|
|
2614
|
+
const worktreeGroups = new Map();
|
|
2615
|
+
for (const entry of handoffs) {
|
|
2616
|
+
const worktreePath = entry[1].worktree?.worktreePath;
|
|
2617
|
+
if (!worktreePath)
|
|
2618
|
+
continue;
|
|
2619
|
+
const group = worktreeGroups.get(worktreePath) ?? [];
|
|
2620
|
+
group.push(entry);
|
|
2621
|
+
worktreeGroups.set(worktreePath, group);
|
|
2622
|
+
}
|
|
2623
|
+
for (const group of worktreeGroups.values()) {
|
|
2624
|
+
if (!group.every(([key]) => readyToClear.has(key)))
|
|
2625
|
+
continue;
|
|
2626
|
+
try {
|
|
2627
|
+
await this.#cleanupFailureHandoffWorktrees(group.map(([, handoff]) => handoff));
|
|
2628
|
+
}
|
|
2629
|
+
catch (error) {
|
|
2630
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
2631
|
+
this.#logger.warn?.('[factory] retained dispatch-failure handoff after worktree cleanup failed', {
|
|
2632
|
+
issue: group[0]?.[1].issue,
|
|
2633
|
+
worktreePath: group[0]?.[1].worktree?.worktreePath,
|
|
2634
|
+
error: describeError(error).errorMessage,
|
|
2635
|
+
});
|
|
2636
|
+
continue;
|
|
2637
|
+
}
|
|
2638
|
+
for (const [key] of group) {
|
|
2639
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2640
|
+
readyToClear.delete(key);
|
|
2641
|
+
registryChanged = true;
|
|
2642
|
+
}
|
|
2643
|
+
}
|
|
2644
|
+
// Legacy and non-worktree handoffs can be cleared directly once their
|
|
2645
|
+
// process is gone and the broker accepted the release.
|
|
2646
|
+
for (const [key, handoff] of handoffs) {
|
|
2647
|
+
if (!readyToClear.has(key) || handoff.worktree)
|
|
2648
|
+
continue;
|
|
2649
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2650
|
+
registryChanged = true;
|
|
2651
|
+
}
|
|
2652
|
+
}
|
|
2527
2653
|
if (registryChanged) {
|
|
2528
2654
|
await this.#writeInFlightRegistry(registryPath, heartbeatPath);
|
|
2529
2655
|
}
|
|
@@ -2738,6 +2864,62 @@ export class FactoryLoop {
|
|
|
2738
2864
|
this.#error(error, record.issue);
|
|
2739
2865
|
}
|
|
2740
2866
|
}
|
|
2867
|
+
#dispatchFailureHandoffs(record, acknowledged) {
|
|
2868
|
+
const handoffs = new Map(acknowledged.map((handoff) => [handoff.name, handoff]));
|
|
2869
|
+
if (!this.#worktrees)
|
|
2870
|
+
return [...handoffs.values()];
|
|
2871
|
+
for (const [name, tracked] of record.agents) {
|
|
2872
|
+
const worktree = this.#agentWorktree(record, tracked.spec);
|
|
2873
|
+
if (!worktree)
|
|
2874
|
+
continue;
|
|
2875
|
+
const existing = handoffs.get(name);
|
|
2876
|
+
handoffs.set(name, {
|
|
2877
|
+
issue: record.issue,
|
|
2878
|
+
name,
|
|
2879
|
+
tracked: cloneTrackedAgent(tracked),
|
|
2880
|
+
persistedAtMs: existing?.persistedAtMs ?? this.#clock.now(),
|
|
2881
|
+
worktree,
|
|
2882
|
+
});
|
|
2883
|
+
}
|
|
2884
|
+
return [...handoffs.values()];
|
|
2885
|
+
}
|
|
2886
|
+
async #teardownFailedDispatchWorktrees(handoffs) {
|
|
2887
|
+
if (!this.#worktrees || !handoffs.some((handoff) => handoff.worktree))
|
|
2888
|
+
return false;
|
|
2889
|
+
const failed = await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'dispatch failed', 'completion');
|
|
2890
|
+
if (failed.length > 0)
|
|
2891
|
+
return false;
|
|
2892
|
+
try {
|
|
2893
|
+
await this.#cleanupFailureHandoffWorktrees(handoffs);
|
|
2894
|
+
for (const handoff of handoffs) {
|
|
2895
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
2896
|
+
}
|
|
2897
|
+
return true;
|
|
2898
|
+
}
|
|
2899
|
+
catch (error) {
|
|
2900
|
+
// Keep the durable handoffs. The loop reaper will retry cleanup only
|
|
2901
|
+
// after it has reconfirmed every agent sharing the checkout is gone.
|
|
2902
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
2903
|
+
this.#logger.warn?.('[factory] retained dispatch-failure handoffs after worktree cleanup failed', {
|
|
2904
|
+
issue: handoffs[0]?.issue,
|
|
2905
|
+
error: describeError(error).errorMessage,
|
|
2906
|
+
});
|
|
2907
|
+
return false;
|
|
2908
|
+
}
|
|
2909
|
+
}
|
|
2910
|
+
async #cleanupFailureHandoffWorktrees(handoffs) {
|
|
2911
|
+
if (!this.#worktrees)
|
|
2912
|
+
return;
|
|
2913
|
+
const unique = new Map();
|
|
2914
|
+
for (const handoff of handoffs) {
|
|
2915
|
+
if (handoff.worktree)
|
|
2916
|
+
unique.set(handoff.worktree.worktreePath, handoff.worktree);
|
|
2917
|
+
}
|
|
2918
|
+
for (const worktree of unique.values()) {
|
|
2919
|
+
await this.#worktrees.cleanup(worktree);
|
|
2920
|
+
this.#increment('agentWorktreesCleaned');
|
|
2921
|
+
}
|
|
2922
|
+
}
|
|
2741
2923
|
async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false) {
|
|
2742
2924
|
const updatedAtMs = this.#clock.now();
|
|
2743
2925
|
const agents = [];
|
|
@@ -2832,6 +3014,7 @@ export class FactoryLoop {
|
|
|
2832
3014
|
}
|
|
2833
3015
|
return { name: spec.name };
|
|
2834
3016
|
}
|
|
3017
|
+
await this.#prepareAgentWorktree(record, spec);
|
|
2835
3018
|
let result;
|
|
2836
3019
|
try {
|
|
2837
3020
|
result = await this.#fleet.spawn({
|
|
@@ -3049,6 +3232,7 @@ export class FactoryLoop {
|
|
|
3049
3232
|
else {
|
|
3050
3233
|
const invocationId = `${batch.invocationIdFor(record.issue, tracked.spec)}:restart:${this.#clock.now()}`;
|
|
3051
3234
|
try {
|
|
3235
|
+
await this.#prepareAgentWorktree(record, tracked.spec);
|
|
3052
3236
|
const result = await this.#fleet.spawn({
|
|
3053
3237
|
name: tracked.spec.name,
|
|
3054
3238
|
capability: tracked.spec.capability,
|
|
@@ -3178,6 +3362,66 @@ export class FactoryLoop {
|
|
|
3178
3362
|
});
|
|
3179
3363
|
return result;
|
|
3180
3364
|
}
|
|
3365
|
+
async #prepareAgentWorktree(record, spec) {
|
|
3366
|
+
const worktree = this.#agentWorktree(record, spec);
|
|
3367
|
+
if (!worktree || !this.#worktrees)
|
|
3368
|
+
return;
|
|
3369
|
+
try {
|
|
3370
|
+
await this.#worktrees.prepare(worktree);
|
|
3371
|
+
this.#increment('agentWorktreesPrepared');
|
|
3372
|
+
}
|
|
3373
|
+
catch (error) {
|
|
3374
|
+
throw contextualError(`Unable to prepare isolated worktree for ${record.issue.key}/${spec.repo} at ${worktree.worktreePath}`, error);
|
|
3375
|
+
}
|
|
3376
|
+
}
|
|
3377
|
+
#agentWorktree(record, spec) {
|
|
3378
|
+
if (!spec.baseClonePath || !spec.clonePath || spec.baseClonePath === spec.clonePath)
|
|
3379
|
+
return undefined;
|
|
3380
|
+
const implementer = record.decision.implementers.find((candidate) => candidate.repo === spec.repo && candidate.branch)
|
|
3381
|
+
?? [...record.agents.values()]
|
|
3382
|
+
.map((tracked) => tracked.spec)
|
|
3383
|
+
.find((candidate) => candidate.repo === spec.repo && candidate.role === 'implementer' && candidate.branch);
|
|
3384
|
+
const branch = spec.branch ?? implementer?.branch;
|
|
3385
|
+
if (!branch)
|
|
3386
|
+
return undefined;
|
|
3387
|
+
return {
|
|
3388
|
+
repo: spec.repo,
|
|
3389
|
+
issueKey: record.issue.key,
|
|
3390
|
+
baseClonePath: spec.baseClonePath,
|
|
3391
|
+
worktreePath: spec.clonePath,
|
|
3392
|
+
branch,
|
|
3393
|
+
};
|
|
3394
|
+
}
|
|
3395
|
+
async #cleanupAgentWorktrees(record) {
|
|
3396
|
+
if (!this.#worktrees)
|
|
3397
|
+
return;
|
|
3398
|
+
const unique = new Map();
|
|
3399
|
+
for (const tracked of record.agents.values()) {
|
|
3400
|
+
const worktree = this.#agentWorktree(record, tracked.spec);
|
|
3401
|
+
if (worktree)
|
|
3402
|
+
unique.set(worktree.worktreePath, worktree);
|
|
3403
|
+
}
|
|
3404
|
+
const failures = [];
|
|
3405
|
+
for (const worktree of unique.values()) {
|
|
3406
|
+
try {
|
|
3407
|
+
await this.#worktrees.cleanup(worktree);
|
|
3408
|
+
this.#increment('agentWorktreesCleaned');
|
|
3409
|
+
}
|
|
3410
|
+
catch (error) {
|
|
3411
|
+
failures.push(`${worktree.worktreePath}: ${describeError(error).errorMessage}`);
|
|
3412
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
3413
|
+
this.#logger.warn?.('[factory] failed to clean completed issue worktree', {
|
|
3414
|
+
issue: record.issue.key,
|
|
3415
|
+
repo: worktree.repo,
|
|
3416
|
+
worktreePath: worktree.worktreePath,
|
|
3417
|
+
error: describeError(error).errorMessage,
|
|
3418
|
+
});
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
if (failures.length > 0) {
|
|
3422
|
+
throw new Error(`Factory worktree cleanup incomplete for ${record.issue.key}: ${failures.join('; ')}`);
|
|
3423
|
+
}
|
|
3424
|
+
}
|
|
3181
3425
|
async #confirmPublishedRemotePullRequest(repo, result, expectedHeadRef) {
|
|
3182
3426
|
const parts = githubRepoParts(repo);
|
|
3183
3427
|
if (!parts)
|
|
@@ -3365,6 +3609,7 @@ export class FactoryLoop {
|
|
|
3365
3609
|
if (!tracked.sessionRef) {
|
|
3366
3610
|
return;
|
|
3367
3611
|
}
|
|
3612
|
+
await this.#prepareAgentWorktree(record, tracked.spec);
|
|
3368
3613
|
const result = await this.#fleet.resume({
|
|
3369
3614
|
name,
|
|
3370
3615
|
sessionRef: tracked.sessionRef,
|
|
@@ -3607,7 +3852,7 @@ export class FactoryLoop {
|
|
|
3607
3852
|
catch {
|
|
3608
3853
|
// The initiator logs Slack watcher startup failures.
|
|
3609
3854
|
}
|
|
3610
|
-
const threadId = await this.#
|
|
3855
|
+
const threadId = await this.#persistedSlackThread(key);
|
|
3611
3856
|
if (!threadId) {
|
|
3612
3857
|
this.#increment('agentQuestionsSkippedMissingThread');
|
|
3613
3858
|
this.#logger.warn?.('[factory] agent question has no Slack dispatch thread', {
|
|
@@ -3835,7 +4080,7 @@ export class FactoryLoop {
|
|
|
3835
4080
|
return undefined;
|
|
3836
4081
|
}
|
|
3837
4082
|
const key = issueKey(record.issue);
|
|
3838
|
-
const threadId = await this.#
|
|
4083
|
+
const threadId = await this.#persistedSlackThread(key);
|
|
3839
4084
|
if (!threadId) {
|
|
3840
4085
|
this.#increment('agentQuestionReleaseSkippedMissingThread');
|
|
3841
4086
|
return undefined;
|
|
@@ -3888,7 +4133,7 @@ export class FactoryLoop {
|
|
|
3888
4133
|
issue: { ...record.issue },
|
|
3889
4134
|
decision: structuredClone(record.decision),
|
|
3890
4135
|
dryRun: record.dryRun,
|
|
3891
|
-
threadId: await this.#
|
|
4136
|
+
threadId: await this.#persistedSlackThread(key),
|
|
3892
4137
|
questionSource: 'github',
|
|
3893
4138
|
askerName: question.agentName,
|
|
3894
4139
|
question: question.question,
|
|
@@ -3914,7 +4159,7 @@ export class FactoryLoop {
|
|
|
3914
4159
|
this.#increment('agentQuestionSlackMirrorsSkippedDegraded');
|
|
3915
4160
|
return;
|
|
3916
4161
|
}
|
|
3917
|
-
const threadId = await this.#
|
|
4162
|
+
const threadId = await this.#persistedSlackThread(issueKey(record.issue));
|
|
3918
4163
|
if (!threadId) {
|
|
3919
4164
|
this.#increment('agentQuestionSlackMirrorsSkippedMissingThread');
|
|
3920
4165
|
return;
|
|
@@ -4689,7 +4934,8 @@ export class FactoryLoop {
|
|
|
4689
4934
|
implementerNames,
|
|
4690
4935
|
integrationsMountRoot: this.#integrationsMountRoot(),
|
|
4691
4936
|
integrationInstructions,
|
|
4692
|
-
branchName: spec.branch,
|
|
4937
|
+
branchName: spec.branch ?? decision.implementers.find((candidate) => candidate.repo === spec.repo)?.branch,
|
|
4938
|
+
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
4693
4939
|
agentName: spec.name,
|
|
4694
4940
|
}),
|
|
4695
4941
|
});
|
|
@@ -5471,7 +5717,20 @@ export class FactoryLoop {
|
|
|
5471
5717
|
}
|
|
5472
5718
|
const route = record.decision.routes.find((candidate) => candidate.repo === prRef.repo)
|
|
5473
5719
|
?? record.decision.routes[0];
|
|
5474
|
-
const
|
|
5720
|
+
const initialSpec = babysitterSpec(issue, this.#config, route);
|
|
5721
|
+
const sharedCheckout = [...record.agents.values()]
|
|
5722
|
+
.map((agent) => agent.spec)
|
|
5723
|
+
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath);
|
|
5724
|
+
const implementerBranch = record.decision.implementers
|
|
5725
|
+
.find((candidate) => candidate.repo === initialSpec.repo && candidate.branch)?.branch;
|
|
5726
|
+
const spec = sharedCheckout
|
|
5727
|
+
? {
|
|
5728
|
+
...initialSpec,
|
|
5729
|
+
baseClonePath: sharedCheckout.baseClonePath,
|
|
5730
|
+
clonePath: sharedCheckout.clonePath,
|
|
5731
|
+
...(implementerBranch ? { branch: implementerBranch } : {}),
|
|
5732
|
+
}
|
|
5733
|
+
: initialSpec;
|
|
5475
5734
|
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
5476
5735
|
const reviewerName = reviewer?.result?.name ?? reviewer?.spec.name
|
|
5477
5736
|
?? agentNameForRole(issue, 'review', { repo: route?.repo ?? prRef.repo });
|
|
@@ -5481,7 +5740,7 @@ export class FactoryLoop {
|
|
|
5481
5740
|
const integrationInstructions = await this.#resolveIntegrationInstructions();
|
|
5482
5741
|
const task = renderAgentTask({
|
|
5483
5742
|
issue: templateIssueFromRecord(record, issue),
|
|
5484
|
-
route: route ?? { repo: prRef.repo },
|
|
5743
|
+
route: { ...(route ?? { repo: prRef.repo }), clonePath: spec.clonePath },
|
|
5485
5744
|
role: 'babysitter',
|
|
5486
5745
|
config: { mergePolicy: this.#config.mergePolicy, terminalState: this.#config.terminalState },
|
|
5487
5746
|
reviewerName,
|
|
@@ -5490,6 +5749,8 @@ export class FactoryLoop {
|
|
|
5490
5749
|
slackDispatchThread: await this.#slackDispatchThreadFor(record),
|
|
5491
5750
|
integrationsMountRoot: this.#integrationsMountRoot(),
|
|
5492
5751
|
integrationInstructions,
|
|
5752
|
+
branchName: spec.branch,
|
|
5753
|
+
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
5493
5754
|
agentName: spec.name,
|
|
5494
5755
|
});
|
|
5495
5756
|
const spawned = await this.#spawnAgent(record, {
|
|
@@ -5658,6 +5919,7 @@ export class FactoryLoop {
|
|
|
5658
5919
|
return;
|
|
5659
5920
|
}
|
|
5660
5921
|
this.#completionInFlight.add(completionKey);
|
|
5922
|
+
let releaseReasonForRetry;
|
|
5661
5923
|
try {
|
|
5662
5924
|
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
5663
5925
|
return;
|
|
@@ -5743,6 +6005,7 @@ export class FactoryLoop {
|
|
|
5743
6005
|
await this.#runCompletionMergeGate(issue);
|
|
5744
6006
|
}
|
|
5745
6007
|
const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
|
|
6008
|
+
releaseReasonForRetry = releaseReason;
|
|
5746
6009
|
if (this.#fleet.placementLocality === 'remote') {
|
|
5747
6010
|
// Durable capacity is released as soon as terminal writeback is
|
|
5748
6011
|
// acknowledged. Agent cleanup remains fenced/retryable in `releasing`.
|
|
@@ -5759,7 +6022,10 @@ export class FactoryLoop {
|
|
|
5759
6022
|
}
|
|
5760
6023
|
catch (error) {
|
|
5761
6024
|
this.#error(error, record.issue);
|
|
5762
|
-
|
|
6025
|
+
if (releaseReasonForRetry)
|
|
6026
|
+
this.#scheduleReleaseRetry(record, releaseReasonForRetry);
|
|
6027
|
+
else
|
|
6028
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
5763
6029
|
}
|
|
5764
6030
|
finally {
|
|
5765
6031
|
this.#completionInFlight.delete(completionKey);
|
|
@@ -6013,6 +6279,19 @@ export class FactoryLoop {
|
|
|
6013
6279
|
this.#lastObservedSlackEventAtMs = this.#clock.now();
|
|
6014
6280
|
this.#increment('slackWebhookEventsObserved');
|
|
6015
6281
|
}
|
|
6282
|
+
async #persistedSlackThread(key) {
|
|
6283
|
+
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
6284
|
+
if (!threadId || /^\d+[._]\d+$/u.test(threadId))
|
|
6285
|
+
return threadId;
|
|
6286
|
+
// Older Factory versions persisted the Relayfile draft client id when the
|
|
6287
|
+
// acknowledged file had not yet reconciled its provider payload. Slack
|
|
6288
|
+
// cannot use that value as thread_ts. Drop it so the caller establishes a
|
|
6289
|
+
// fresh provider-backed root instead of producing invalid_thread_ts.
|
|
6290
|
+
await this.#state.clearSlackThread(this.#workspaceId, key);
|
|
6291
|
+
this.#increment('invalidSlackThreadsCleared');
|
|
6292
|
+
this.#logger.warn?.('[factory] cleared invalid persisted Slack thread id', { issue: key });
|
|
6293
|
+
return undefined;
|
|
6294
|
+
}
|
|
6016
6295
|
async #ensureSlackDispatchThread(record, result) {
|
|
6017
6296
|
if (!this.#slack || !this.#config.slack || result.dryRun) {
|
|
6018
6297
|
return;
|
|
@@ -6021,7 +6300,7 @@ export class FactoryLoop {
|
|
|
6021
6300
|
return;
|
|
6022
6301
|
}
|
|
6023
6302
|
const key = issueKey(record.issue);
|
|
6024
|
-
const existingThread = await this.#
|
|
6303
|
+
const existingThread = await this.#persistedSlackThread(key);
|
|
6025
6304
|
const watcherStart = this.#slackWatcherStarts.get(key);
|
|
6026
6305
|
if (existingThread || watcherStart) {
|
|
6027
6306
|
try {
|
|
@@ -6074,6 +6353,16 @@ export class FactoryLoop {
|
|
|
6074
6353
|
if (dryRun) {
|
|
6075
6354
|
return;
|
|
6076
6355
|
}
|
|
6356
|
+
// A source GitHub issue is the durable stakeholder record. Keep both the
|
|
6357
|
+
// question and authorized response there, then mirror the escalation once
|
|
6358
|
+
// to Slack for stakeholder visibility without making Slack a competing
|
|
6359
|
+
// clarification workflow.
|
|
6360
|
+
const sourceIssue = await this.#readIssue(decision.issue.path);
|
|
6361
|
+
if (sourceIssue && githubIssueSourceRef(sourceIssue)) {
|
|
6362
|
+
const result = await this.#escalateTriageToGithub(decision, reason);
|
|
6363
|
+
await this.#mirrorGithubTriageEscalationToSlack(decision, sourceIssue, reason);
|
|
6364
|
+
return result;
|
|
6365
|
+
}
|
|
6077
6366
|
if (!this.#slack || !this.#config.slack) {
|
|
6078
6367
|
return await this.#escalateTriageToGithub(decision, reason);
|
|
6079
6368
|
}
|
|
@@ -6081,7 +6370,7 @@ export class FactoryLoop {
|
|
|
6081
6370
|
return;
|
|
6082
6371
|
}
|
|
6083
6372
|
const key = issueKey(decision.issue);
|
|
6084
|
-
const existingThread = await this.#
|
|
6373
|
+
const existingThread = await this.#persistedSlackThread(key);
|
|
6085
6374
|
const watcherStart = this.#slackWatcherStarts.get(key);
|
|
6086
6375
|
if (existingThread || watcherStart) {
|
|
6087
6376
|
try {
|
|
@@ -6134,7 +6423,7 @@ export class FactoryLoop {
|
|
|
6134
6423
|
}
|
|
6135
6424
|
try {
|
|
6136
6425
|
await this.#githubWriteback.postComment(issue, [
|
|
6137
|
-
|
|
6426
|
+
`@${authorizedAuthor}, Factory needs clarification before dispatching ${decision.issue.key}.`,
|
|
6138
6427
|
`Reason: ${reason}`,
|
|
6139
6428
|
`Question: ${question}`,
|
|
6140
6429
|
`Authorized responder: @${authorizedAuthor} (the issue reporter).`,
|
|
@@ -6149,15 +6438,79 @@ export class FactoryLoop {
|
|
|
6149
6438
|
this.#surfaceEscalationDeliveryFailure('triage', decision.issue, correlationId, 'GitHub issue comment writeback failed', error);
|
|
6150
6439
|
}
|
|
6151
6440
|
}
|
|
6441
|
+
async #mirrorGithubTriageEscalationToSlack(decision, issue, reason) {
|
|
6442
|
+
if (!this.#slack || !this.#config.slack)
|
|
6443
|
+
return;
|
|
6444
|
+
if (await this.#shouldSkipSlackWriteback('triage-escalation-mirror')) {
|
|
6445
|
+
this.#increment('triageEscalationSlackMirrorsSkippedDegraded');
|
|
6446
|
+
return;
|
|
6447
|
+
}
|
|
6448
|
+
const key = issueKey(decision.issue);
|
|
6449
|
+
const existingThread = await this.#persistedSlackThread(key);
|
|
6450
|
+
const inFlight = this.#slackWatcherStarts.get(key);
|
|
6451
|
+
if (existingThread || inFlight) {
|
|
6452
|
+
if (inFlight) {
|
|
6453
|
+
try {
|
|
6454
|
+
await inFlight;
|
|
6455
|
+
}
|
|
6456
|
+
catch {
|
|
6457
|
+
// The initiator records the optional mirror failure.
|
|
6458
|
+
}
|
|
6459
|
+
}
|
|
6460
|
+
this.#increment('triageEscalationSlackMirrorDuplicatesSuppressed');
|
|
6461
|
+
return;
|
|
6462
|
+
}
|
|
6463
|
+
const start = this.#postGithubTriageSlackMirror(decision, issue, reason);
|
|
6464
|
+
this.#slackWatcherStarts.set(key, start);
|
|
6465
|
+
try {
|
|
6466
|
+
await start;
|
|
6467
|
+
}
|
|
6468
|
+
catch (error) {
|
|
6469
|
+
this.#markSlackWritebackFailure('triage-escalation-mirror', error);
|
|
6470
|
+
this.#increment('triageEscalationSlackMirrorFailures');
|
|
6471
|
+
this.#logger.warn?.('[factory] optional GitHub triage Slack mirror failed', {
|
|
6472
|
+
issue: decision.issue.key,
|
|
6473
|
+
error: describeError(error).errorMessage,
|
|
6474
|
+
});
|
|
6475
|
+
}
|
|
6476
|
+
finally {
|
|
6477
|
+
this.#slackWatcherStarts.delete(key);
|
|
6478
|
+
}
|
|
6479
|
+
}
|
|
6480
|
+
async #postGithubTriageSlackMirror(decision, issue, reason) {
|
|
6481
|
+
if (!this.#slack || !this.#config.slack)
|
|
6482
|
+
return;
|
|
6483
|
+
const source = githubIssueSourceRef(issue);
|
|
6484
|
+
const stakeholderMentions = slackMentions(this.#config.slack.stakeholderUserIds);
|
|
6485
|
+
const reporter = githubIssueAuthor(issue);
|
|
6486
|
+
const audience = [stakeholderMentions, reporter ? `GitHub reporter: @${reporter}.` : undefined]
|
|
6487
|
+
.filter((part) => Boolean(part))
|
|
6488
|
+
.join(' ');
|
|
6489
|
+
const replyInstruction = source?.url
|
|
6490
|
+
? `Reply on the GitHub issue so Factory can resume: ${source.url}`
|
|
6491
|
+
: 'Reply on the source GitHub issue so Factory can resume.';
|
|
6492
|
+
const root = await this.#slack.postThread({
|
|
6493
|
+
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
6494
|
+
text: [
|
|
6495
|
+
`${audience ? `${audience} ` : ''}${decision.issue.key}: factory triage escalation for ${issue.title}`,
|
|
6496
|
+
`Reason: ${reason}`,
|
|
6497
|
+
`Question: ${triageEscalationQuestion(decision)} ${replyInstruction}`,
|
|
6498
|
+
].join('\n'),
|
|
6499
|
+
});
|
|
6500
|
+
await this.#state.setSlackThread(this.#workspaceId, issueKey(decision.issue), root.threadId);
|
|
6501
|
+
this.#increment('triageEscalationsMirroredToSlack');
|
|
6502
|
+
this.#recordSlackWritebackSuccess('triage-escalation-mirror');
|
|
6503
|
+
}
|
|
6152
6504
|
async #postAndWatchSlackEscalationThread(decision, reason) {
|
|
6153
6505
|
if (!this.#slack || !this.#config.slack) {
|
|
6154
6506
|
return;
|
|
6155
6507
|
}
|
|
6156
6508
|
const issue = await this.#readIssue(decision.issue.path);
|
|
6509
|
+
const stakeholderMentions = slackMentions(this.#config.slack.stakeholderUserIds);
|
|
6157
6510
|
const root = await this.#slack.postThread({
|
|
6158
6511
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
6159
6512
|
text: [
|
|
6160
|
-
`${decision.issue.key}: factory triage escalation for ${issue?.title ?? decision.issue.key}`,
|
|
6513
|
+
`${stakeholderMentions ? `${stakeholderMentions} ` : ''}${decision.issue.key}: factory triage escalation for ${issue?.title ?? decision.issue.key}`,
|
|
6161
6514
|
`Reason: ${reason}`,
|
|
6162
6515
|
`Question: ${triageEscalationQuestion(decision)}`,
|
|
6163
6516
|
].join('\n'),
|
|
@@ -6350,7 +6703,7 @@ export class FactoryLoop {
|
|
|
6350
6703
|
}
|
|
6351
6704
|
let threadId;
|
|
6352
6705
|
try {
|
|
6353
|
-
threadId = await this.#
|
|
6706
|
+
threadId = await this.#persistedSlackThread(key);
|
|
6354
6707
|
}
|
|
6355
6708
|
catch (error) {
|
|
6356
6709
|
this.#logger.warn?.('[factory] unable to read persisted Slack thread during watcher rehydration', { issue: record.issue.key, error });
|
|
@@ -6845,6 +7198,7 @@ export class FactoryLoop {
|
|
|
6845
7198
|
}
|
|
6846
7199
|
async #resumeOrColdStartClarificationAgent(name, tracked, waiting) {
|
|
6847
7200
|
const task = clarificationResumeTask(tracked.spec.task, waiting);
|
|
7201
|
+
await this.#prepareAgentWorktree(waitingRecord(waiting), tracked.spec);
|
|
6848
7202
|
if (tracked.sessionRef) {
|
|
6849
7203
|
try {
|
|
6850
7204
|
const resumed = await this.#fleet.resume({
|
|
@@ -7054,7 +7408,7 @@ export class FactoryLoop {
|
|
|
7054
7408
|
if (!this.#config.slack) {
|
|
7055
7409
|
return undefined;
|
|
7056
7410
|
}
|
|
7057
|
-
const threadId = await this.#
|
|
7411
|
+
const threadId = await this.#persistedSlackThread(issueKey(record.issue));
|
|
7058
7412
|
const channel = await this.#slackChannelDir() ?? this.#config.slack.channel;
|
|
7059
7413
|
return threadId
|
|
7060
7414
|
? { channel, threadId, mountRoot: this.#integrationsMountRoot() }
|
|
@@ -7595,36 +7949,35 @@ function routeImplementerSpec(issue, config, slug, route) {
|
|
|
7595
7949
|
node: 'self',
|
|
7596
7950
|
};
|
|
7597
7951
|
}
|
|
7598
|
-
function decisionWithLifecycleBranches(decision, runId) {
|
|
7599
|
-
const
|
|
7952
|
+
function decisionWithLifecycleBranches(decision, runId, opts = {}) {
|
|
7953
|
+
const implementerBranch = (spec) => {
|
|
7954
|
+
const runSuffix = `-${runId.slice(0, 8)}`;
|
|
7955
|
+
const stem = `${sanitizeAgentSlug(decision.issue.key)}-${sanitizeAgentSlug(spec.repo)}`
|
|
7956
|
+
.slice(0, 120 - 'factory/'.length - runSuffix.length);
|
|
7957
|
+
return `factory/${stem}${runSuffix}`;
|
|
7958
|
+
};
|
|
7959
|
+
const branchByRepo = new Map(decision.implementers.map((spec) => [spec.repo, implementerBranch(spec)]));
|
|
7960
|
+
const withBranch = (spec, branch) => {
|
|
7961
|
+
const baseClonePath = spec.baseClonePath ?? spec.clonePath;
|
|
7962
|
+
const clonePath = opts.isolateLocalWorktree && baseClonePath && branch
|
|
7963
|
+
? factoryWorktreePath(baseClonePath, decision.issue.key, spec.repo, runId)
|
|
7964
|
+
: spec.clonePath;
|
|
7600
7965
|
const lifecycleSpec = {
|
|
7601
7966
|
...spec,
|
|
7967
|
+
...(opts.isolateLocalWorktree && baseClonePath && branch ? { baseClonePath, clonePath } : {}),
|
|
7602
7968
|
// The same persisted lifecycle reuses this id after takeover, while a
|
|
7603
7969
|
// genuine reopen gets a new id and cannot replay an old placement ack.
|
|
7604
7970
|
invocationId: `factory:${decision.issue.key}:${runId}:${spec.role}:${sanitizeAgentSlug(spec.name)}`,
|
|
7605
7971
|
};
|
|
7606
|
-
|
|
7607
|
-
return lifecycleSpec;
|
|
7608
|
-
const runSuffix = `-${runId.slice(0, 8)}`;
|
|
7609
|
-
const stem = `${sanitizeAgentSlug(decision.issue.key)}-${sanitizeAgentSlug(spec.repo)}`
|
|
7610
|
-
.slice(0, 120 - 'factory/'.length - runSuffix.length);
|
|
7611
|
-
const branch = `factory/${stem}${runSuffix}`;
|
|
7612
|
-
return {
|
|
7613
|
-
...lifecycleSpec,
|
|
7614
|
-
branch,
|
|
7615
|
-
task: [
|
|
7616
|
-
spec.task,
|
|
7617
|
-
'',
|
|
7618
|
-
`Factory publication branch: ${branch}`,
|
|
7619
|
-
'Before editing, create or reset that exact branch from the repository default branch. Commit and push only that branch.',
|
|
7620
|
-
].join('\n'),
|
|
7621
|
-
};
|
|
7972
|
+
return branch ? { ...lifecycleSpec, branch } : lifecycleSpec;
|
|
7622
7973
|
};
|
|
7623
7974
|
return {
|
|
7624
7975
|
...structuredClone(decision),
|
|
7625
|
-
implementers: decision.implementers.map(withBranch),
|
|
7626
|
-
reviewer: withBranch(decision.reviewer),
|
|
7627
|
-
...(decision.workflow
|
|
7976
|
+
implementers: decision.implementers.map((spec) => withBranch(spec, branchByRepo.get(spec.repo))),
|
|
7977
|
+
reviewer: withBranch(decision.reviewer, branchByRepo.get(decision.reviewer.repo)),
|
|
7978
|
+
...(decision.workflow
|
|
7979
|
+
? { workflow: withBranch(decision.workflow, branchByRepo.get(decision.workflow.repo)) }
|
|
7980
|
+
: {}),
|
|
7628
7981
|
};
|
|
7629
7982
|
}
|
|
7630
7983
|
function routeReviewerSpec(issue, config, route, reviewer) {
|
|
@@ -7724,12 +8077,21 @@ function taskForDispatch(issue, route, role) {
|
|
|
7724
8077
|
issue.description,
|
|
7725
8078
|
].join('\n\n');
|
|
7726
8079
|
}
|
|
7727
|
-
const templateIssueFromRecord = (record, issue) =>
|
|
7728
|
-
|
|
7729
|
-
|
|
7730
|
-
|
|
7731
|
-
|
|
7732
|
-
|
|
8080
|
+
const templateIssueFromRecord = (record, issue) => {
|
|
8081
|
+
const github = issue ? githubIssueSourceRef(issue) : undefined;
|
|
8082
|
+
const reporter = issue ? githubIssueAuthor(issue) : undefined;
|
|
8083
|
+
return {
|
|
8084
|
+
key: issue?.key ?? record.issue.key,
|
|
8085
|
+
title: issue?.title ?? record.issue.key,
|
|
8086
|
+
description: issue?.description ?? '',
|
|
8087
|
+
github: github
|
|
8088
|
+
? {
|
|
8089
|
+
...github,
|
|
8090
|
+
...(reporter ? { reporter } : {}),
|
|
8091
|
+
}
|
|
8092
|
+
: undefined,
|
|
8093
|
+
};
|
|
8094
|
+
};
|
|
7733
8095
|
const routeForSpec = (decision, spec) => {
|
|
7734
8096
|
const route = decision.routes.find((candidate) => candidate.repo === spec.repo && candidate.clonePath === spec.clonePath) ?? decision.routes.find((candidate) => candidate.repo === spec.repo);
|
|
7735
8097
|
return {
|
|
@@ -8006,24 +8368,69 @@ const reposFromConfig = (config) => {
|
|
|
8006
8368
|
].filter((repo) => Boolean(repo)));
|
|
8007
8369
|
return [...repos];
|
|
8008
8370
|
};
|
|
8009
|
-
const
|
|
8010
|
-
const
|
|
8011
|
-
for (const
|
|
8012
|
-
|
|
8371
|
+
const configuredGithubRepoParts = (config) => {
|
|
8372
|
+
const repos = new Map();
|
|
8373
|
+
for (const configuredRepo of reposFromConfig(config)) {
|
|
8374
|
+
let parts = githubRepoParts(configuredRepo);
|
|
8375
|
+
if (!parts) {
|
|
8376
|
+
try {
|
|
8377
|
+
parts = githubRepoParts(normalizeGithubRepo(configuredRepo, config.repos.org));
|
|
8378
|
+
}
|
|
8379
|
+
catch {
|
|
8380
|
+
continue;
|
|
8381
|
+
}
|
|
8382
|
+
}
|
|
8013
8383
|
if (!parts)
|
|
8014
8384
|
continue;
|
|
8015
|
-
|
|
8385
|
+
repos.set(`${parts.owner.toLowerCase()}/${parts.repo.toLowerCase()}`, parts);
|
|
8386
|
+
}
|
|
8387
|
+
return [...repos.values()];
|
|
8388
|
+
};
|
|
8389
|
+
// A terminal `**` is intentional: relayfile's matcher treats a non-terminal
|
|
8390
|
+
// `**` as a single-segment wildcard. Scoping at the repository root still
|
|
8391
|
+
// covers every supported issue, PR, review, comment, and check path without
|
|
8392
|
+
// subscribing this factory to other repositories in the workspace.
|
|
8393
|
+
export const githubRepoSubscriptionGlobs = (config) => configuredGithubRepoParts(config).flatMap(({ owner, repo }) => [
|
|
8394
|
+
`${GITHUB_ISSUE_ROOT}/${owner}/${repo}/**`,
|
|
8395
|
+
`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/**`,
|
|
8396
|
+
]);
|
|
8397
|
+
const githubIssueScanRoots = (config) => {
|
|
8398
|
+
const roots = new Set();
|
|
8399
|
+
for (const { owner, repo } of configuredGithubRepoParts(config)) {
|
|
8400
|
+
roots.add(`${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues`);
|
|
8401
|
+
roots.add(`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`);
|
|
8016
8402
|
}
|
|
8017
8403
|
return [...roots];
|
|
8018
8404
|
};
|
|
8405
|
+
const githubRepoPathParts = (path) => {
|
|
8406
|
+
const compactSegment = path.match(/^\/github\/repos\/([^/]+)\//u)?.[1];
|
|
8407
|
+
const separator = compactSegment?.indexOf('__') ?? -1;
|
|
8408
|
+
if (compactSegment && separator > 0 && separator < compactSegment.length - 2) {
|
|
8409
|
+
return {
|
|
8410
|
+
owner: compactSegment.slice(0, separator),
|
|
8411
|
+
repo: compactSegment.slice(separator + 2),
|
|
8412
|
+
};
|
|
8413
|
+
}
|
|
8414
|
+
const nested = path.match(/^\/github\/repos\/([^/]+)\/([^/]+)\//u);
|
|
8415
|
+
if (nested)
|
|
8416
|
+
return { owner: nested[1], repo: nested[2] };
|
|
8417
|
+
return undefined;
|
|
8418
|
+
};
|
|
8419
|
+
const isConfiguredGithubRepoPath = (path, config) => {
|
|
8420
|
+
const pathParts = githubRepoPathParts(path);
|
|
8421
|
+
if (!pathParts)
|
|
8422
|
+
return false;
|
|
8423
|
+
const pathRepo = `${pathParts.owner.toLowerCase()}/${pathParts.repo.toLowerCase()}`;
|
|
8424
|
+
return configuredGithubRepoParts(config).some(({ owner, repo }) => `${owner.toLowerCase()}/${repo.toLowerCase()}` === pathRepo);
|
|
8425
|
+
};
|
|
8019
8426
|
const githubRepoParts = (repo) => {
|
|
8020
8427
|
const split = repo.match(/^([^/]+)\/([^/]+)$/u);
|
|
8021
8428
|
if (split) {
|
|
8022
8429
|
return { owner: split[1], repo: split[2] };
|
|
8023
8430
|
}
|
|
8024
|
-
const
|
|
8025
|
-
if (
|
|
8026
|
-
return { owner:
|
|
8431
|
+
const separator = repo.indexOf('__');
|
|
8432
|
+
if (separator > 0 && separator < repo.length - 2 && !repo.includes('/')) {
|
|
8433
|
+
return { owner: repo.slice(0, separator), repo: repo.slice(separator + 2) };
|
|
8027
8434
|
}
|
|
8028
8435
|
return undefined;
|
|
8029
8436
|
};
|
|
@@ -8660,12 +9067,10 @@ const isAgentAlreadyExistsError = (error) => {
|
|
|
8660
9067
|
return /already exists/iu.test(message);
|
|
8661
9068
|
};
|
|
8662
9069
|
const defaultRestartPolicy = (spec) =>
|
|
8663
|
-
//
|
|
8664
|
-
//
|
|
8665
|
-
// exit should resume its session rather than drop the PR. The reviewer is
|
|
8666
|
-
// short-lived and keeps the fleet default.
|
|
9070
|
+
// Factory owns durable resume/respawn decisions. Broker-level retries race
|
|
9071
|
+
// that lifecycle and can re-register the same name before Factory resumes it.
|
|
8667
9072
|
spec.role === 'implementer' || spec.role === 'babysitter'
|
|
8668
|
-
? { maxRestarts:
|
|
9073
|
+
? { maxRestarts: 0 }
|
|
8669
9074
|
: spec.restartPolicy;
|
|
8670
9075
|
const slackPayloadTs = (threadId) => threadId.replace(/_/g, '.');
|
|
8671
9076
|
const slackChannelMessagesPrefix = (channelDir) => `/slack/channels/${channelDir}/messages/`;
|