@agent-relay/factory 0.1.25 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +2 -0
- package/dist/cli/fleet.js.map +1 -1
- package/dist/dispatch/templates.d.ts +5 -1
- package/dist/dispatch/templates.d.ts.map +1 -1
- package/dist/dispatch/templates.js +27 -12
- package/dist/dispatch/templates.js.map +1 -1
- package/dist/fleet/internal-fleet-client.js +1 -1
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/git/agent-worktree.d.ts +18 -0
- package/dist/git/agent-worktree.d.ts.map +1 -0
- package/dist/git/agent-worktree.js +172 -0
- package/dist/git/agent-worktree.js.map +1 -0
- package/dist/mount/local-mount-preflight.js +6 -8
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +1 -0
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +18 -2
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +713 -94
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +2 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -0
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +2 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +3 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/ports/worktree.d.ts +12 -0
- package/dist/ports/worktree.d.ts.map +1 -0
- package/dist/ports/worktree.js +2 -0
- package/dist/ports/worktree.js.map +1 -0
- package/dist/ports/writeback.d.ts +5 -1
- package/dist/ports/writeback.d.ts.map +1 -1
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/github.d.ts +2 -0
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +36 -0
- package/dist/writeback/github.js.map +1 -1
- package/dist/writeback/slack.d.ts.map +1 -1
- package/dist/writeback/slack.js +15 -8
- package/dist/writeback/slack.js.map +1 -1
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ import { FactoryConfigSchema } from '../config/schema.js';
|
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
8
|
+
import { factoryWorktreePath } from '../git/agent-worktree.js';
|
|
8
9
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
9
10
|
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
10
11
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
@@ -116,6 +117,7 @@ export class FactoryLoop {
|
|
|
116
117
|
#state;
|
|
117
118
|
#workspaceId;
|
|
118
119
|
#relayflows;
|
|
120
|
+
#worktrees;
|
|
119
121
|
#batchView;
|
|
120
122
|
#batchReady;
|
|
121
123
|
#listeners = new Map();
|
|
@@ -127,6 +129,9 @@ export class FactoryLoop {
|
|
|
127
129
|
#githubIssueCommentWatchers = new Map();
|
|
128
130
|
#githubIssueCommentWatchStates = new Map();
|
|
129
131
|
#githubIssueCommentQueues = new Map();
|
|
132
|
+
#githubIssueAuthors = new Map();
|
|
133
|
+
#githubIssueAuthorLookups = new Map();
|
|
134
|
+
#reconciledGithubInProgress = new Set();
|
|
130
135
|
#resolvedSlackChannelDir;
|
|
131
136
|
#slackChannelDirRefresh;
|
|
132
137
|
// Agents we've already logged an ambiguous-PID-lookup warning for, so the
|
|
@@ -149,6 +154,7 @@ export class FactoryLoop {
|
|
|
149
154
|
#dispatchTerminalWaiters = new Map();
|
|
150
155
|
#dispatchLifecycleRetryTimers = new Map();
|
|
151
156
|
#dispatchLifecycleDrives = new Set();
|
|
157
|
+
#localReleaseCheckpoints = new Map();
|
|
152
158
|
#dispatchLifecycleRenewTimer;
|
|
153
159
|
#clarificationSweepTimer;
|
|
154
160
|
#clarificationSweepDueAtMs;
|
|
@@ -251,6 +257,7 @@ export class FactoryLoop {
|
|
|
251
257
|
this.#terminationGraceMs = ports.terminationGraceMs;
|
|
252
258
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
253
259
|
this.#relayflows = ports.relayflows;
|
|
260
|
+
this.#worktrees = ports.worktrees;
|
|
254
261
|
this.#state = ports.stateStore ?? new InMemoryStateStore({
|
|
255
262
|
batchSize: config.batchSize,
|
|
256
263
|
agentQuestionDedupeLimit: AGENT_QUESTION_DEDUPE_LIMIT,
|
|
@@ -943,12 +950,16 @@ export class FactoryLoop {
|
|
|
943
950
|
return { dispatchRelayflow: true };
|
|
944
951
|
}
|
|
945
952
|
if (isGithubIssueFilePath(path)) {
|
|
946
|
-
const
|
|
953
|
+
const parts = githubIssuePathParts(path);
|
|
954
|
+
const sourceKey = parts
|
|
955
|
+
? `github:${githubIssueIdentity(parts.owner, parts.repo, parts.number)}`
|
|
956
|
+
: `github:${path}`;
|
|
947
957
|
if (seenIssueKeys.has(sourceKey)) {
|
|
948
958
|
this.#increment('liveDuplicateIssueEventsSuppressed');
|
|
949
|
-
this.#logger.debug?.('[factory] suppressed duplicate live GitHub issue
|
|
959
|
+
this.#logger.debug?.('[factory] suppressed duplicate live GitHub issue alias in current drain', {
|
|
950
960
|
id: event.id,
|
|
951
961
|
path,
|
|
962
|
+
issue: parts ? sourceKey.slice('github:'.length) : undefined,
|
|
952
963
|
});
|
|
953
964
|
return { dispatchRelayflow: false };
|
|
954
965
|
}
|
|
@@ -1127,7 +1138,8 @@ export class FactoryLoop {
|
|
|
1127
1138
|
});
|
|
1128
1139
|
}
|
|
1129
1140
|
async #resolveIssuePr(issue, opts = {}) {
|
|
1130
|
-
const
|
|
1141
|
+
const issueKey = issueStateKey(issueRef(issue));
|
|
1142
|
+
const key = opts.openOnly ? `${issueKey}:open` : issueKey;
|
|
1131
1143
|
const now = this.#clock.now();
|
|
1132
1144
|
const cached = this.#probePrResolvedCache.get(key);
|
|
1133
1145
|
if (cached && cached.expiresAtMs > now) {
|
|
@@ -1172,6 +1184,9 @@ export class FactoryLoop {
|
|
|
1172
1184
|
await this.#ensureGithubIngestionReady();
|
|
1173
1185
|
}
|
|
1174
1186
|
const paths = await this.#readyIssuePaths();
|
|
1187
|
+
const orphanRecovery = issueSource === 'github'
|
|
1188
|
+
? await this.#githubOrphanRecoveryContext()
|
|
1189
|
+
: undefined;
|
|
1175
1190
|
const pulled = [];
|
|
1176
1191
|
const triaged = [];
|
|
1177
1192
|
const dispatched = [];
|
|
@@ -1181,7 +1196,9 @@ export class FactoryLoop {
|
|
|
1181
1196
|
for (const path of paths) {
|
|
1182
1197
|
const issue = await this.#readIssue(path);
|
|
1183
1198
|
readyIssueReads += 1;
|
|
1184
|
-
lastReadyReadProgressAtMs = this.#logTimedProgress(
|
|
1199
|
+
lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
|
|
1200
|
+
? '[factory] GitHub ready issue read progress'
|
|
1201
|
+
: '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
|
|
1185
1202
|
if (issue && issueSource === 'linear') {
|
|
1186
1203
|
await this.#recordCanonicalIssueState(issue);
|
|
1187
1204
|
}
|
|
@@ -1189,36 +1206,74 @@ export class FactoryLoop {
|
|
|
1189
1206
|
continue;
|
|
1190
1207
|
}
|
|
1191
1208
|
pulled.push(issueRef(issue));
|
|
1192
|
-
const
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1209
|
+
const wasReady = this.#isIssueReady(issue);
|
|
1210
|
+
const labels = isGithubIssue(issue)
|
|
1211
|
+
? new Set(issue.labels.map((label) => label.trim().toLowerCase()))
|
|
1212
|
+
: undefined;
|
|
1213
|
+
const requiredLabel = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
1214
|
+
const mayRecoverGithubOrphan = !wasReady &&
|
|
1215
|
+
!dryRun &&
|
|
1216
|
+
issueSource === 'github' &&
|
|
1217
|
+
Boolean(orphanRecovery) &&
|
|
1218
|
+
Boolean(requiredLabel) &&
|
|
1219
|
+
Boolean(labels?.has(requiredLabel)) &&
|
|
1220
|
+
Boolean(labels?.has('factory:in-progress')) &&
|
|
1221
|
+
!labels?.has('factory:human-review');
|
|
1222
|
+
if (!mayRecoverGithubOrphan) {
|
|
1223
|
+
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1224
|
+
if (dispatchBlock) {
|
|
1225
|
+
skipped.push({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1226
|
+
continue;
|
|
1227
|
+
}
|
|
1196
1228
|
}
|
|
1197
1229
|
const batch = await this.#batch();
|
|
1198
1230
|
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
1199
1231
|
skipped.push({ issue: issueRef(issue), reason: 'already tracked' });
|
|
1200
1232
|
continue;
|
|
1201
1233
|
}
|
|
1202
|
-
|
|
1234
|
+
const recoveredOrphan = mayRecoverGithubOrphan &&
|
|
1235
|
+
await this.#reconcileOrphanedGithubInProgress(issue, orphanRecovery, dryRun);
|
|
1236
|
+
if (!wasReady && !recoveredOrphan) {
|
|
1237
|
+
if (mayRecoverGithubOrphan) {
|
|
1238
|
+
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1239
|
+
if (dispatchBlock) {
|
|
1240
|
+
skipped.push({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1241
|
+
continue;
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1203
1244
|
skipped.push({ issue: issueRef(issue), reason: 'live state is not ready-for-agent' });
|
|
1204
1245
|
continue;
|
|
1205
1246
|
}
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1247
|
+
const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
|
|
1248
|
+
try {
|
|
1249
|
+
if (recoveredOrphan) {
|
|
1250
|
+
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1251
|
+
if (dispatchBlock) {
|
|
1252
|
+
skipped.push({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1253
|
+
continue;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
1257
|
+
skipped.push({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
|
|
1258
|
+
continue;
|
|
1259
|
+
}
|
|
1260
|
+
if (!isDispatchableIssue(issue)) {
|
|
1261
|
+
skipped.push({ issue: issueRef(issue), reason: 'not reconciled real Linear issue' });
|
|
1262
|
+
continue;
|
|
1263
|
+
}
|
|
1264
|
+
const decision = await this.triageIssue(issue);
|
|
1265
|
+
triaged.push(decision);
|
|
1266
|
+
const result = await this.dispatch(decision, { dryRun });
|
|
1267
|
+
if (result.agents.length === 0 && !dryRun) {
|
|
1268
|
+
skipped.push({ issue: decision.issue, reason: 'queued or escalated' });
|
|
1269
|
+
}
|
|
1270
|
+
else {
|
|
1271
|
+
dispatched.push(result);
|
|
1272
|
+
}
|
|
1219
1273
|
}
|
|
1220
|
-
|
|
1221
|
-
|
|
1274
|
+
finally {
|
|
1275
|
+
if (recoveredIdentity)
|
|
1276
|
+
this.#reconciledGithubInProgress.delete(recoveredIdentity);
|
|
1222
1277
|
}
|
|
1223
1278
|
}
|
|
1224
1279
|
report = { pulled, triaged, dispatched, skipped, dryRun, slackDegraded: this.#slackDegraded };
|
|
@@ -1249,6 +1304,140 @@ export class FactoryLoop {
|
|
|
1249
1304
|
}
|
|
1250
1305
|
}
|
|
1251
1306
|
}
|
|
1307
|
+
async #githubOrphanRecoveryContext() {
|
|
1308
|
+
try {
|
|
1309
|
+
const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
|
|
1310
|
+
readFactoryInFlightRegistry(this.#config.loop.registryPath),
|
|
1311
|
+
this.#fleet.roster(),
|
|
1312
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
1313
|
+
this.#state.listWaitingClarifications(this.#workspaceId),
|
|
1314
|
+
]);
|
|
1315
|
+
const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
|
|
1316
|
+
const activeIssueIdentities = new Set();
|
|
1317
|
+
for (const [, lifecycle] of lifecycles) {
|
|
1318
|
+
if (isTerminalDispatchLifecycle(lifecycle))
|
|
1319
|
+
continue;
|
|
1320
|
+
const identity = githubIssueRefIdentity(lifecycle.issue);
|
|
1321
|
+
if (identity)
|
|
1322
|
+
activeIssueIdentities.add(identity);
|
|
1323
|
+
}
|
|
1324
|
+
for (const [, waiting] of waitingClarifications) {
|
|
1325
|
+
const identity = githubIssueRefIdentity(waiting.issue);
|
|
1326
|
+
if (identity)
|
|
1327
|
+
activeIssueIdentities.add(identity);
|
|
1328
|
+
}
|
|
1329
|
+
for (const agent of registry?.agents ?? []) {
|
|
1330
|
+
if (!onlineAgents.has(agent.name) || !agent.issue)
|
|
1331
|
+
continue;
|
|
1332
|
+
const identity = githubIssueRefIdentity(agent.issue);
|
|
1333
|
+
if (identity)
|
|
1334
|
+
activeIssueIdentities.add(identity);
|
|
1335
|
+
}
|
|
1336
|
+
return {
|
|
1337
|
+
activeIssueIdentities,
|
|
1338
|
+
onlineAgentNames: onlineAgents,
|
|
1339
|
+
};
|
|
1340
|
+
}
|
|
1341
|
+
catch (error) {
|
|
1342
|
+
this.#increment('githubOrphanRecoveryContextFailures');
|
|
1343
|
+
this.#logger.warn?.('[factory] could not establish orphan-recovery safety context; preserving in-progress issues', {
|
|
1344
|
+
error: describeError(error).errorMessage,
|
|
1345
|
+
});
|
|
1346
|
+
return undefined;
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
async #reconcileOrphanedGithubInProgress(issue, context, dryRun) {
|
|
1350
|
+
if (dryRun || !context || !isGithubIssue(issue))
|
|
1351
|
+
return false;
|
|
1352
|
+
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
1353
|
+
const required = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
1354
|
+
if (!required ||
|
|
1355
|
+
!labels.has(required) ||
|
|
1356
|
+
!labels.has('factory:in-progress') ||
|
|
1357
|
+
labels.has('factory:human-review'))
|
|
1358
|
+
return false;
|
|
1359
|
+
const identity = githubIssueRefIdentity(issueRef(issue));
|
|
1360
|
+
if (!identity ||
|
|
1361
|
+
context.activeIssueIdentities.has(identity) ||
|
|
1362
|
+
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue))) {
|
|
1363
|
+
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1364
|
+
return false;
|
|
1365
|
+
}
|
|
1366
|
+
const getProviderStatus = this.#githubWriteback.getIssueStatus;
|
|
1367
|
+
if (!getProviderStatus) {
|
|
1368
|
+
this.#increment('githubOrphanRecoveryStatusLookupUnavailable');
|
|
1369
|
+
return false;
|
|
1370
|
+
}
|
|
1371
|
+
let providerStatus;
|
|
1372
|
+
try {
|
|
1373
|
+
providerStatus = await getProviderStatus.call(this.#githubWriteback, issue);
|
|
1374
|
+
}
|
|
1375
|
+
catch (error) {
|
|
1376
|
+
this.#increment('githubOrphanRecoveryStatusLookupFailures');
|
|
1377
|
+
this.#logger.warn?.('[factory] could not verify provider-authoritative GitHub issue status; preserving it', {
|
|
1378
|
+
issue: issue.key,
|
|
1379
|
+
error: describeError(error).errorMessage,
|
|
1380
|
+
});
|
|
1381
|
+
return false;
|
|
1382
|
+
}
|
|
1383
|
+
if (!providerStatus || providerStatus === 'human-review') {
|
|
1384
|
+
this.#increment('githubOrphanRecoveriesBlockedProviderStatus');
|
|
1385
|
+
return false;
|
|
1386
|
+
}
|
|
1387
|
+
let hasOpenPr;
|
|
1388
|
+
try {
|
|
1389
|
+
hasOpenPr = await this.#hasOpenCompletionPr(issue);
|
|
1390
|
+
}
|
|
1391
|
+
catch (error) {
|
|
1392
|
+
this.#increment('githubOrphanRecoveryPrProbeFailures');
|
|
1393
|
+
this.#logger.warn?.('[factory] could not prove an in-progress GitHub issue has no open PR; preserving it', {
|
|
1394
|
+
issue: issue.key,
|
|
1395
|
+
error: describeError(error).errorMessage,
|
|
1396
|
+
});
|
|
1397
|
+
return false;
|
|
1398
|
+
}
|
|
1399
|
+
if (hasOpenPr) {
|
|
1400
|
+
this.#increment('githubOrphanRecoveriesBlockedOpenPr');
|
|
1401
|
+
this.#logger.info?.('[factory] preserved in-progress GitHub issue because a matching open PR exists', {
|
|
1402
|
+
issue: issue.key,
|
|
1403
|
+
});
|
|
1404
|
+
return false;
|
|
1405
|
+
}
|
|
1406
|
+
try {
|
|
1407
|
+
if (providerStatus === 'in-progress') {
|
|
1408
|
+
await this.#githubWriteback.setStatus(issue, 'ready');
|
|
1409
|
+
}
|
|
1410
|
+
// A crashed dispatch may leave its durable attempt marked in-flight even
|
|
1411
|
+
// after every agent and lifecycle disappeared. Only clear that stale bit
|
|
1412
|
+
// after all provider, agent, lifecycle, and open-PR safety checks pass.
|
|
1413
|
+
await this.#clearDispatchInFlight(issue);
|
|
1414
|
+
this.#reconciledGithubInProgress.add(identity);
|
|
1415
|
+
this.#increment('githubOrphanedInProgressRecovered');
|
|
1416
|
+
this.#logger.warn?.('[factory] recovered orphaned GitHub in-progress issue for redispatch', {
|
|
1417
|
+
issue: issue.key,
|
|
1418
|
+
path: issue.path,
|
|
1419
|
+
});
|
|
1420
|
+
return true;
|
|
1421
|
+
}
|
|
1422
|
+
catch (error) {
|
|
1423
|
+
this.#increment('githubOrphanRecoveryWritebackFailures');
|
|
1424
|
+
this.#logger.warn?.('[factory] failed to clear orphaned GitHub lifecycle status; preserving in-progress issue', {
|
|
1425
|
+
issue: issue.key,
|
|
1426
|
+
error: describeError(error).errorMessage,
|
|
1427
|
+
});
|
|
1428
|
+
return false;
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
async #hasOpenCompletionPr(issue) {
|
|
1432
|
+
if (this.#customProbePrResolver) {
|
|
1433
|
+
return Boolean(await this.#probePrResolver(issue));
|
|
1434
|
+
}
|
|
1435
|
+
return Boolean(await this.#resolveIssuePr(issue, {
|
|
1436
|
+
titleMarker: FACTORY_E2E_MARKER,
|
|
1437
|
+
openOnly: true,
|
|
1438
|
+
failOnLookupError: true,
|
|
1439
|
+
}));
|
|
1440
|
+
}
|
|
1252
1441
|
async #listRelayfileTree(prefix, phase) {
|
|
1253
1442
|
return this.#withRelayfileOperation('listTree', { phase, prefix }, () => this.#mount.listTree(prefix), {
|
|
1254
1443
|
count: (paths) => paths.length,
|
|
@@ -1499,9 +1688,15 @@ export class FactoryLoop {
|
|
|
1499
1688
|
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
1500
1689
|
// recover a persisted minimal triage task after a crash in this gap.
|
|
1501
1690
|
const durableRemoteDispatch = !dryRun && this.#fleet.placementLocality === 'remote';
|
|
1502
|
-
|
|
1691
|
+
// Local dispatches need the same deterministic branch identity as remote
|
|
1692
|
+
// ones. Without it, every worker starts in the configured shared checkout
|
|
1693
|
+
// and concurrent issues can switch each other back to the base branch.
|
|
1694
|
+
const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
|
|
1695
|
+
const lifecycleRunId = !dryRun && (durableRemoteDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
|
|
1503
1696
|
if (lifecycleRunId) {
|
|
1504
|
-
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId
|
|
1697
|
+
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
|
|
1698
|
+
isolateLocalWorktree,
|
|
1699
|
+
});
|
|
1505
1700
|
}
|
|
1506
1701
|
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
1507
1702
|
if (durableRemoteDispatch) {
|
|
@@ -1553,6 +1748,7 @@ export class FactoryLoop {
|
|
|
1553
1748
|
name: spawned.name,
|
|
1554
1749
|
tracked: cloneTrackedAgent(tracked),
|
|
1555
1750
|
persistedAtMs: this.#clock.now(),
|
|
1751
|
+
worktree: this.#agentWorktree(record, tracked.spec),
|
|
1556
1752
|
});
|
|
1557
1753
|
}
|
|
1558
1754
|
agents.push({ name: spawned.name, role: spec.role });
|
|
@@ -1597,7 +1793,12 @@ export class FactoryLoop {
|
|
|
1597
1793
|
return result;
|
|
1598
1794
|
}
|
|
1599
1795
|
catch (error) {
|
|
1600
|
-
|
|
1796
|
+
// A spawn can fail after the broker accepted it but before its ack
|
|
1797
|
+
// reached Factory. Include every planned worktree agent, not only the
|
|
1798
|
+
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
1799
|
+
const failureHandoffs = this.#dispatchFailureHandoffs(record, spawnedForReaperHandoff);
|
|
1800
|
+
await this.#persistDispatchFailureReaperHandoff(record, failureHandoffs);
|
|
1801
|
+
const worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs);
|
|
1601
1802
|
await this.#recordDispatchFailure(decision.issue);
|
|
1602
1803
|
const failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key);
|
|
1603
1804
|
await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable');
|
|
@@ -1605,6 +1806,20 @@ export class FactoryLoop {
|
|
|
1605
1806
|
if (!failedState?.terminal)
|
|
1606
1807
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
1607
1808
|
this.#error(error, decision.issue);
|
|
1809
|
+
// The teardown runs while the record still exists so it can safely
|
|
1810
|
+
// derive every shared checkout. Rewrite the registry only after abandon
|
|
1811
|
+
// removes those agents from the ordinary in-flight view.
|
|
1812
|
+
if (worktreesTornDown) {
|
|
1813
|
+
try {
|
|
1814
|
+
await this.#writeInFlightRegistry();
|
|
1815
|
+
}
|
|
1816
|
+
catch (registryError) {
|
|
1817
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after dispatch worktree teardown', {
|
|
1818
|
+
issue: record.issue,
|
|
1819
|
+
error: describeError(registryError).errorMessage,
|
|
1820
|
+
});
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1608
1823
|
throw error;
|
|
1609
1824
|
}
|
|
1610
1825
|
}
|
|
@@ -1815,6 +2030,31 @@ export class FactoryLoop {
|
|
|
1815
2030
|
}, DISPATCH_LIFECYCLE_RETRY_MS);
|
|
1816
2031
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
1817
2032
|
}
|
|
2033
|
+
#scheduleReleaseRetry(record, reason) {
|
|
2034
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
2035
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
2036
|
+
return;
|
|
2037
|
+
}
|
|
2038
|
+
const key = issueKey(record.issue);
|
|
2039
|
+
if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
|
|
2040
|
+
return;
|
|
2041
|
+
const timer = setTimeout(() => {
|
|
2042
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
2043
|
+
const drive = this.#finishDurableRelease(record, reason)
|
|
2044
|
+
.then(() => undefined)
|
|
2045
|
+
.catch((error) => {
|
|
2046
|
+
this.#logger.warn?.('[factory] local completion cleanup retry failed', {
|
|
2047
|
+
issue: record.issue.key,
|
|
2048
|
+
error: describeError(error).errorMessage,
|
|
2049
|
+
});
|
|
2050
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
2051
|
+
})
|
|
2052
|
+
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
2053
|
+
this.#dispatchLifecycleDrives.add(drive);
|
|
2054
|
+
}, DISPATCH_LIFECYCLE_RETRY_MS);
|
|
2055
|
+
timer.unref?.();
|
|
2056
|
+
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
2057
|
+
}
|
|
1818
2058
|
async #driveDispatchLifecycle(key) {
|
|
1819
2059
|
if (this.#stopping)
|
|
1820
2060
|
return;
|
|
@@ -1965,12 +2205,12 @@ export class FactoryLoop {
|
|
|
1965
2205
|
}
|
|
1966
2206
|
async #finishDurableRelease(record, releaseReason) {
|
|
1967
2207
|
const batch = await this.#batch();
|
|
1968
|
-
const next = this.#fleet.placementLocality === 'remote' ? undefined : batch.complete(record.issue);
|
|
1969
2208
|
const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done');
|
|
1970
|
-
const
|
|
2209
|
+
const releaseKey = issueKey(record.issue);
|
|
2210
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, releaseKey);
|
|
1971
2211
|
const released = new Set(lifecycle?.agents
|
|
1972
2212
|
.filter((agent) => agent.releasedAtMs !== undefined)
|
|
1973
|
-
.map((agent) => agent.name) ?? []);
|
|
2213
|
+
.map((agent) => agent.name) ?? this.#localReleaseCheckpoints.get(releaseKey) ?? []);
|
|
1974
2214
|
const failed = [];
|
|
1975
2215
|
for (const agent of record.agents) {
|
|
1976
2216
|
if (released.has(agent[0]))
|
|
@@ -1981,19 +2221,37 @@ export class FactoryLoop {
|
|
|
1981
2221
|
continue;
|
|
1982
2222
|
}
|
|
1983
2223
|
released.add(agent[0]);
|
|
2224
|
+
if (this.#fleet.placementLocality === 'local') {
|
|
2225
|
+
this.#localReleaseCheckpoints.set(releaseKey, new Set(released));
|
|
2226
|
+
}
|
|
1984
2227
|
// Persist each acknowledged release independently. A takeover retries
|
|
1985
2228
|
// only agents whose release did not reach a fenced durable checkpoint.
|
|
1986
2229
|
if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, reason, released))
|
|
1987
2230
|
return false;
|
|
1988
2231
|
}
|
|
1989
|
-
if (next)
|
|
1990
|
-
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
1991
2232
|
await this.#writeInFlightRegistry();
|
|
1992
2233
|
if (failed.length > 0) {
|
|
1993
2234
|
this.#increment('dispatchLifecycleReleaseRetries');
|
|
1994
|
-
this.#
|
|
2235
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
2236
|
+
return false;
|
|
2237
|
+
}
|
|
2238
|
+
// The PR branch is already pushed and the babysitter has declared the
|
|
2239
|
+
// current PR green with review feedback addressed. Release is now fenced,
|
|
2240
|
+
// so no agent can race cleanup of the shared per-issue worktree.
|
|
2241
|
+
try {
|
|
2242
|
+
await this.#cleanupAgentWorktrees(record);
|
|
2243
|
+
}
|
|
2244
|
+
catch {
|
|
2245
|
+
// Completion remains in-flight until the isolated checkout is gone.
|
|
2246
|
+
// Remote lifecycles retry from their durable `releasing` phase; local
|
|
2247
|
+
// lifecycles retain this record and retry directly from the same fence.
|
|
2248
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
1995
2249
|
return false;
|
|
1996
2250
|
}
|
|
2251
|
+
const next = this.#fleet.placementLocality === 'remote' ? undefined : batch.complete(record.issue);
|
|
2252
|
+
this.#localReleaseCheckpoints.delete(releaseKey);
|
|
2253
|
+
if (next)
|
|
2254
|
+
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
1997
2255
|
// Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
|
|
1998
2256
|
// the babysitter's durable ownership/wake/critical state while that epoch
|
|
1999
2257
|
// is still valid so a later reopened issue cannot inherit a stale PR owner.
|
|
@@ -2110,11 +2368,14 @@ export class FactoryLoop {
|
|
|
2110
2368
|
if (githubState === 'closed') {
|
|
2111
2369
|
return false;
|
|
2112
2370
|
}
|
|
2113
|
-
|
|
2371
|
+
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
2372
|
+
if (labels.has('factory:human-review'))
|
|
2373
|
+
return false;
|
|
2374
|
+
const identity = githubIssueRefIdentity(issueRef(issue));
|
|
2375
|
+
if (labels.has('factory:in-progress') && (!identity || !this.#reconciledGithubInProgress.has(identity)))
|
|
2114
2376
|
return false;
|
|
2115
|
-
}
|
|
2116
2377
|
const required = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
2117
|
-
return Boolean(required) &&
|
|
2378
|
+
return Boolean(required) && labels.has(required);
|
|
2118
2379
|
}
|
|
2119
2380
|
async #postIssueComment(issue, body) {
|
|
2120
2381
|
if (isGithubIssue(issue)) {
|
|
@@ -2199,12 +2460,22 @@ export class FactoryLoop {
|
|
|
2199
2460
|
}
|
|
2200
2461
|
async #githubIssuePaths() {
|
|
2201
2462
|
try {
|
|
2202
|
-
const issuePaths = new
|
|
2463
|
+
const issuePaths = new Map();
|
|
2203
2464
|
for (const root of githubIssueScanRoots(this.#config)) {
|
|
2204
2465
|
const paths = await this.#listRelayfileTree(root, 'GitHub issue ingestion');
|
|
2205
2466
|
for (const path of paths) {
|
|
2206
|
-
|
|
2207
|
-
|
|
2467
|
+
const parts = githubIssuePathParts(path);
|
|
2468
|
+
if (parts) {
|
|
2469
|
+
const identity = githubIssueIdentity(parts.owner, parts.repo, parts.number);
|
|
2470
|
+
const existing = issuePaths.get(identity);
|
|
2471
|
+
if (!existing || githubIssuePathPreference(path) < githubIssuePathPreference(existing)) {
|
|
2472
|
+
if (existing)
|
|
2473
|
+
this.#increment('githubIssueAliasPathsSuppressed');
|
|
2474
|
+
issuePaths.set(identity, path);
|
|
2475
|
+
}
|
|
2476
|
+
else {
|
|
2477
|
+
this.#increment('githubIssueAliasPathsSuppressed');
|
|
2478
|
+
}
|
|
2208
2479
|
}
|
|
2209
2480
|
else if (githubIssueDirectoryPathParts(path) !== undefined) {
|
|
2210
2481
|
// listTree returns the issue directory entry alongside its
|
|
@@ -2220,7 +2491,7 @@ export class FactoryLoop {
|
|
|
2220
2491
|
}
|
|
2221
2492
|
}
|
|
2222
2493
|
}
|
|
2223
|
-
return [...issuePaths].sort();
|
|
2494
|
+
return [...issuePaths.values()].sort();
|
|
2224
2495
|
}
|
|
2225
2496
|
catch (error) {
|
|
2226
2497
|
this.#increment('githubIssueListFailures');
|
|
@@ -2470,6 +2741,7 @@ export class FactoryLoop {
|
|
|
2470
2741
|
try {
|
|
2471
2742
|
const protectedPids = await this.#protectedPids();
|
|
2472
2743
|
let registryChanged = false;
|
|
2744
|
+
const readyToClear = new Set();
|
|
2473
2745
|
for (const [key, handoff] of handoffs) {
|
|
2474
2746
|
const roots = await this.#terminationRoots(handoff.name, handoff.tracked, protectedPids);
|
|
2475
2747
|
if (roots.pids.length === 0 && roots.status === 'unresolved') {
|
|
@@ -2482,8 +2754,6 @@ export class FactoryLoop {
|
|
|
2482
2754
|
unresolvedAgeMs,
|
|
2483
2755
|
});
|
|
2484
2756
|
if (unresolvedAgeMs >= DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS) {
|
|
2485
|
-
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2486
|
-
registryChanged = true;
|
|
2487
2757
|
this.#increment('dispatchFailureReaperHandoffsDroppedStaleUnresolved');
|
|
2488
2758
|
this.#logger.warn?.('[factory] dropped stale unresolved dispatch-failed handoff', {
|
|
2489
2759
|
agentName: handoff.name,
|
|
@@ -2493,6 +2763,7 @@ export class FactoryLoop {
|
|
|
2493
2763
|
});
|
|
2494
2764
|
try {
|
|
2495
2765
|
await this.#fleet.release(handoff.name, 'dispatch failed');
|
|
2766
|
+
readyToClear.add(key);
|
|
2496
2767
|
}
|
|
2497
2768
|
catch (error) {
|
|
2498
2769
|
this.#logger.warn?.('[factory] failed to release unresolved dispatch-failure handoff after pruning', {
|
|
@@ -2526,16 +2797,55 @@ export class FactoryLoop {
|
|
|
2526
2797
|
}
|
|
2527
2798
|
}
|
|
2528
2799
|
if (!blockingSkip) {
|
|
2529
|
-
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2530
|
-
registryChanged = true;
|
|
2531
2800
|
try {
|
|
2532
2801
|
await this.#fleet.release(handoff.name, 'dispatch failed');
|
|
2802
|
+
readyToClear.add(key);
|
|
2533
2803
|
}
|
|
2534
2804
|
catch (error) {
|
|
2535
2805
|
this.#logger.warn?.(`[factory] failed to release ${handoff.name} after dispatch-failure reap`, error);
|
|
2536
2806
|
}
|
|
2537
2807
|
}
|
|
2538
2808
|
}
|
|
2809
|
+
if (readyToClear.size > 0) {
|
|
2810
|
+
const worktreeGroups = new Map();
|
|
2811
|
+
for (const entry of handoffs) {
|
|
2812
|
+
const worktreePath = entry[1].worktree?.worktreePath;
|
|
2813
|
+
if (!worktreePath)
|
|
2814
|
+
continue;
|
|
2815
|
+
const group = worktreeGroups.get(worktreePath) ?? [];
|
|
2816
|
+
group.push(entry);
|
|
2817
|
+
worktreeGroups.set(worktreePath, group);
|
|
2818
|
+
}
|
|
2819
|
+
for (const group of worktreeGroups.values()) {
|
|
2820
|
+
if (!group.every(([key]) => readyToClear.has(key)))
|
|
2821
|
+
continue;
|
|
2822
|
+
try {
|
|
2823
|
+
await this.#cleanupFailureHandoffWorktrees(group.map(([, handoff]) => handoff));
|
|
2824
|
+
}
|
|
2825
|
+
catch (error) {
|
|
2826
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
2827
|
+
this.#logger.warn?.('[factory] retained dispatch-failure handoff after worktree cleanup failed', {
|
|
2828
|
+
issue: group[0]?.[1].issue,
|
|
2829
|
+
worktreePath: group[0]?.[1].worktree?.worktreePath,
|
|
2830
|
+
error: describeError(error).errorMessage,
|
|
2831
|
+
});
|
|
2832
|
+
continue;
|
|
2833
|
+
}
|
|
2834
|
+
for (const [key] of group) {
|
|
2835
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2836
|
+
readyToClear.delete(key);
|
|
2837
|
+
registryChanged = true;
|
|
2838
|
+
}
|
|
2839
|
+
}
|
|
2840
|
+
// Legacy and non-worktree handoffs can be cleared directly once their
|
|
2841
|
+
// process is gone and the broker accepted the release.
|
|
2842
|
+
for (const [key, handoff] of handoffs) {
|
|
2843
|
+
if (!readyToClear.has(key) || handoff.worktree)
|
|
2844
|
+
continue;
|
|
2845
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
2846
|
+
registryChanged = true;
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2539
2849
|
if (registryChanged) {
|
|
2540
2850
|
await this.#writeInFlightRegistry(registryPath, heartbeatPath);
|
|
2541
2851
|
}
|
|
@@ -2750,6 +3060,62 @@ export class FactoryLoop {
|
|
|
2750
3060
|
this.#error(error, record.issue);
|
|
2751
3061
|
}
|
|
2752
3062
|
}
|
|
3063
|
+
#dispatchFailureHandoffs(record, acknowledged) {
|
|
3064
|
+
const handoffs = new Map(acknowledged.map((handoff) => [handoff.name, handoff]));
|
|
3065
|
+
if (!this.#worktrees)
|
|
3066
|
+
return [...handoffs.values()];
|
|
3067
|
+
for (const [name, tracked] of record.agents) {
|
|
3068
|
+
const worktree = this.#agentWorktree(record, tracked.spec);
|
|
3069
|
+
if (!worktree)
|
|
3070
|
+
continue;
|
|
3071
|
+
const existing = handoffs.get(name);
|
|
3072
|
+
handoffs.set(name, {
|
|
3073
|
+
issue: record.issue,
|
|
3074
|
+
name,
|
|
3075
|
+
tracked: cloneTrackedAgent(tracked),
|
|
3076
|
+
persistedAtMs: existing?.persistedAtMs ?? this.#clock.now(),
|
|
3077
|
+
worktree,
|
|
3078
|
+
});
|
|
3079
|
+
}
|
|
3080
|
+
return [...handoffs.values()];
|
|
3081
|
+
}
|
|
3082
|
+
async #teardownFailedDispatchWorktrees(handoffs) {
|
|
3083
|
+
if (!this.#worktrees || !handoffs.some((handoff) => handoff.worktree))
|
|
3084
|
+
return false;
|
|
3085
|
+
const failed = await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'dispatch failed', 'completion');
|
|
3086
|
+
if (failed.length > 0)
|
|
3087
|
+
return false;
|
|
3088
|
+
try {
|
|
3089
|
+
await this.#cleanupFailureHandoffWorktrees(handoffs);
|
|
3090
|
+
for (const handoff of handoffs) {
|
|
3091
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
3092
|
+
}
|
|
3093
|
+
return true;
|
|
3094
|
+
}
|
|
3095
|
+
catch (error) {
|
|
3096
|
+
// Keep the durable handoffs. The loop reaper will retry cleanup only
|
|
3097
|
+
// after it has reconfirmed every agent sharing the checkout is gone.
|
|
3098
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
3099
|
+
this.#logger.warn?.('[factory] retained dispatch-failure handoffs after worktree cleanup failed', {
|
|
3100
|
+
issue: handoffs[0]?.issue,
|
|
3101
|
+
error: describeError(error).errorMessage,
|
|
3102
|
+
});
|
|
3103
|
+
return false;
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
async #cleanupFailureHandoffWorktrees(handoffs) {
|
|
3107
|
+
if (!this.#worktrees)
|
|
3108
|
+
return;
|
|
3109
|
+
const unique = new Map();
|
|
3110
|
+
for (const handoff of handoffs) {
|
|
3111
|
+
if (handoff.worktree)
|
|
3112
|
+
unique.set(handoff.worktree.worktreePath, handoff.worktree);
|
|
3113
|
+
}
|
|
3114
|
+
for (const worktree of unique.values()) {
|
|
3115
|
+
await this.#worktrees.cleanup(worktree);
|
|
3116
|
+
this.#increment('agentWorktreesCleaned');
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
2753
3119
|
async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false) {
|
|
2754
3120
|
const updatedAtMs = this.#clock.now();
|
|
2755
3121
|
const agents = [];
|
|
@@ -2844,6 +3210,7 @@ export class FactoryLoop {
|
|
|
2844
3210
|
}
|
|
2845
3211
|
return { name: spec.name };
|
|
2846
3212
|
}
|
|
3213
|
+
await this.#prepareAgentWorktree(record, spec);
|
|
2847
3214
|
let result;
|
|
2848
3215
|
try {
|
|
2849
3216
|
result = await this.#fleet.spawn({
|
|
@@ -3061,6 +3428,7 @@ export class FactoryLoop {
|
|
|
3061
3428
|
else {
|
|
3062
3429
|
const invocationId = `${batch.invocationIdFor(record.issue, tracked.spec)}:restart:${this.#clock.now()}`;
|
|
3063
3430
|
try {
|
|
3431
|
+
await this.#prepareAgentWorktree(record, tracked.spec);
|
|
3064
3432
|
const result = await this.#fleet.spawn({
|
|
3065
3433
|
name: tracked.spec.name,
|
|
3066
3434
|
capability: tracked.spec.capability,
|
|
@@ -3190,6 +3558,66 @@ export class FactoryLoop {
|
|
|
3190
3558
|
});
|
|
3191
3559
|
return result;
|
|
3192
3560
|
}
|
|
3561
|
+
async #prepareAgentWorktree(record, spec) {
|
|
3562
|
+
const worktree = this.#agentWorktree(record, spec);
|
|
3563
|
+
if (!worktree || !this.#worktrees)
|
|
3564
|
+
return;
|
|
3565
|
+
try {
|
|
3566
|
+
await this.#worktrees.prepare(worktree);
|
|
3567
|
+
this.#increment('agentWorktreesPrepared');
|
|
3568
|
+
}
|
|
3569
|
+
catch (error) {
|
|
3570
|
+
throw contextualError(`Unable to prepare isolated worktree for ${record.issue.key}/${spec.repo} at ${worktree.worktreePath}`, error);
|
|
3571
|
+
}
|
|
3572
|
+
}
|
|
3573
|
+
#agentWorktree(record, spec) {
|
|
3574
|
+
if (!spec.baseClonePath || !spec.clonePath || spec.baseClonePath === spec.clonePath)
|
|
3575
|
+
return undefined;
|
|
3576
|
+
const implementer = record.decision.implementers.find((candidate) => candidate.repo === spec.repo && candidate.branch)
|
|
3577
|
+
?? [...record.agents.values()]
|
|
3578
|
+
.map((tracked) => tracked.spec)
|
|
3579
|
+
.find((candidate) => candidate.repo === spec.repo && candidate.role === 'implementer' && candidate.branch);
|
|
3580
|
+
const branch = spec.branch ?? implementer?.branch;
|
|
3581
|
+
if (!branch)
|
|
3582
|
+
return undefined;
|
|
3583
|
+
return {
|
|
3584
|
+
repo: spec.repo,
|
|
3585
|
+
issueKey: record.issue.key,
|
|
3586
|
+
baseClonePath: spec.baseClonePath,
|
|
3587
|
+
worktreePath: spec.clonePath,
|
|
3588
|
+
branch,
|
|
3589
|
+
};
|
|
3590
|
+
}
|
|
3591
|
+
async #cleanupAgentWorktrees(record) {
|
|
3592
|
+
if (!this.#worktrees)
|
|
3593
|
+
return;
|
|
3594
|
+
const unique = new Map();
|
|
3595
|
+
for (const tracked of record.agents.values()) {
|
|
3596
|
+
const worktree = this.#agentWorktree(record, tracked.spec);
|
|
3597
|
+
if (worktree)
|
|
3598
|
+
unique.set(worktree.worktreePath, worktree);
|
|
3599
|
+
}
|
|
3600
|
+
const failures = [];
|
|
3601
|
+
for (const worktree of unique.values()) {
|
|
3602
|
+
try {
|
|
3603
|
+
await this.#worktrees.cleanup(worktree);
|
|
3604
|
+
this.#increment('agentWorktreesCleaned');
|
|
3605
|
+
}
|
|
3606
|
+
catch (error) {
|
|
3607
|
+
failures.push(`${worktree.worktreePath}: ${describeError(error).errorMessage}`);
|
|
3608
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
3609
|
+
this.#logger.warn?.('[factory] failed to clean completed issue worktree', {
|
|
3610
|
+
issue: record.issue.key,
|
|
3611
|
+
repo: worktree.repo,
|
|
3612
|
+
worktreePath: worktree.worktreePath,
|
|
3613
|
+
error: describeError(error).errorMessage,
|
|
3614
|
+
});
|
|
3615
|
+
}
|
|
3616
|
+
}
|
|
3617
|
+
if (failures.length > 0) {
|
|
3618
|
+
throw new Error(`Factory worktree cleanup incomplete for ${record.issue.key}: ${failures.join('; ')}`);
|
|
3619
|
+
}
|
|
3620
|
+
}
|
|
3193
3621
|
async #confirmPublishedRemotePullRequest(repo, result, expectedHeadRef) {
|
|
3194
3622
|
const parts = githubRepoParts(repo);
|
|
3195
3623
|
if (!parts)
|
|
@@ -3377,6 +3805,7 @@ export class FactoryLoop {
|
|
|
3377
3805
|
if (!tracked.sessionRef) {
|
|
3378
3806
|
return;
|
|
3379
3807
|
}
|
|
3808
|
+
await this.#prepareAgentWorktree(record, tracked.spec);
|
|
3380
3809
|
const result = await this.#fleet.resume({
|
|
3381
3810
|
name,
|
|
3382
3811
|
sessionRef: tracked.sessionRef,
|
|
@@ -3619,7 +4048,7 @@ export class FactoryLoop {
|
|
|
3619
4048
|
catch {
|
|
3620
4049
|
// The initiator logs Slack watcher startup failures.
|
|
3621
4050
|
}
|
|
3622
|
-
const threadId = await this.#
|
|
4051
|
+
const threadId = await this.#persistedSlackThread(key);
|
|
3623
4052
|
if (!threadId) {
|
|
3624
4053
|
this.#increment('agentQuestionsSkippedMissingThread');
|
|
3625
4054
|
this.#logger.warn?.('[factory] agent question has no Slack dispatch thread', {
|
|
@@ -3847,7 +4276,7 @@ export class FactoryLoop {
|
|
|
3847
4276
|
return undefined;
|
|
3848
4277
|
}
|
|
3849
4278
|
const key = issueKey(record.issue);
|
|
3850
|
-
const threadId = await this.#
|
|
4279
|
+
const threadId = await this.#persistedSlackThread(key);
|
|
3851
4280
|
if (!threadId) {
|
|
3852
4281
|
this.#increment('agentQuestionReleaseSkippedMissingThread');
|
|
3853
4282
|
return undefined;
|
|
@@ -3900,7 +4329,7 @@ export class FactoryLoop {
|
|
|
3900
4329
|
issue: { ...record.issue },
|
|
3901
4330
|
decision: structuredClone(record.decision),
|
|
3902
4331
|
dryRun: record.dryRun,
|
|
3903
|
-
threadId: await this.#
|
|
4332
|
+
threadId: await this.#persistedSlackThread(key),
|
|
3904
4333
|
questionSource: 'github',
|
|
3905
4334
|
askerName: question.agentName,
|
|
3906
4335
|
question: question.question,
|
|
@@ -3926,7 +4355,7 @@ export class FactoryLoop {
|
|
|
3926
4355
|
this.#increment('agentQuestionSlackMirrorsSkippedDegraded');
|
|
3927
4356
|
return;
|
|
3928
4357
|
}
|
|
3929
|
-
const threadId = await this.#
|
|
4358
|
+
const threadId = await this.#persistedSlackThread(issueKey(record.issue));
|
|
3930
4359
|
if (!threadId) {
|
|
3931
4360
|
this.#increment('agentQuestionSlackMirrorsSkippedMissingThread');
|
|
3932
4361
|
return;
|
|
@@ -4041,7 +4470,7 @@ export class FactoryLoop {
|
|
|
4041
4470
|
const correlationId = githubEscalationCorrelationId('agent-question', record.issue, question.question);
|
|
4042
4471
|
const issue = await this.#readIssue(record.issue.path);
|
|
4043
4472
|
const source = issue ? githubIssueSourceRef(issue) : undefined;
|
|
4044
|
-
const authorizedAuthor = issue ?
|
|
4473
|
+
const authorizedAuthor = issue ? await this.#resolveGithubIssueAuthor(issue) : undefined;
|
|
4045
4474
|
if (!issue || !source || !authorizedAuthor) {
|
|
4046
4475
|
this.#surfaceEscalationDeliveryFailure('agent-question', record.issue, correlationId, fallbackReason
|
|
4047
4476
|
? `${fallbackReason}; no GitHub issue write path with an identifiable issue reporter is available`
|
|
@@ -4537,7 +4966,7 @@ export class FactoryLoop {
|
|
|
4537
4966
|
};
|
|
4538
4967
|
const correlationId = githubEscalationCorrelationId('agent-question', record.issue, `${comment.commentId}:${question.question}`);
|
|
4539
4968
|
const issue = await this.#readIssue(record.issue.path);
|
|
4540
|
-
const authorizedAuthor = issue ?
|
|
4969
|
+
const authorizedAuthor = issue ? await this.#resolveGithubIssueAuthor(issue) : undefined;
|
|
4541
4970
|
if (!authorizedAuthor) {
|
|
4542
4971
|
this.#increment('githubAgentQuestionsIgnoredMissingAuthorizedAuthor');
|
|
4543
4972
|
this.#surfaceEscalationDeliveryFailure('agent-question', record.issue, correlationId, 'source GitHub issue has no identifiable reporter authorized to answer the durable question');
|
|
@@ -4701,7 +5130,8 @@ export class FactoryLoop {
|
|
|
4701
5130
|
implementerNames,
|
|
4702
5131
|
integrationsMountRoot: this.#integrationsMountRoot(),
|
|
4703
5132
|
integrationInstructions,
|
|
4704
|
-
branchName: spec.branch,
|
|
5133
|
+
branchName: spec.branch ?? decision.implementers.find((candidate) => candidate.repo === spec.repo)?.branch,
|
|
5134
|
+
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
4705
5135
|
agentName: spec.name,
|
|
4706
5136
|
}),
|
|
4707
5137
|
});
|
|
@@ -5483,7 +5913,20 @@ export class FactoryLoop {
|
|
|
5483
5913
|
}
|
|
5484
5914
|
const route = record.decision.routes.find((candidate) => candidate.repo === prRef.repo)
|
|
5485
5915
|
?? record.decision.routes[0];
|
|
5486
|
-
const
|
|
5916
|
+
const initialSpec = babysitterSpec(issue, this.#config, route);
|
|
5917
|
+
const sharedCheckout = [...record.agents.values()]
|
|
5918
|
+
.map((agent) => agent.spec)
|
|
5919
|
+
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath);
|
|
5920
|
+
const implementerBranch = record.decision.implementers
|
|
5921
|
+
.find((candidate) => candidate.repo === initialSpec.repo && candidate.branch)?.branch;
|
|
5922
|
+
const spec = sharedCheckout
|
|
5923
|
+
? {
|
|
5924
|
+
...initialSpec,
|
|
5925
|
+
baseClonePath: sharedCheckout.baseClonePath,
|
|
5926
|
+
clonePath: sharedCheckout.clonePath,
|
|
5927
|
+
...(implementerBranch ? { branch: implementerBranch } : {}),
|
|
5928
|
+
}
|
|
5929
|
+
: initialSpec;
|
|
5487
5930
|
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
5488
5931
|
const reviewerName = reviewer?.result?.name ?? reviewer?.spec.name
|
|
5489
5932
|
?? agentNameForRole(issue, 'review', { repo: route?.repo ?? prRef.repo });
|
|
@@ -5493,7 +5936,7 @@ export class FactoryLoop {
|
|
|
5493
5936
|
const integrationInstructions = await this.#resolveIntegrationInstructions();
|
|
5494
5937
|
const task = renderAgentTask({
|
|
5495
5938
|
issue: templateIssueFromRecord(record, issue),
|
|
5496
|
-
route: route ?? { repo: prRef.repo },
|
|
5939
|
+
route: { ...(route ?? { repo: prRef.repo }), clonePath: spec.clonePath },
|
|
5497
5940
|
role: 'babysitter',
|
|
5498
5941
|
config: { mergePolicy: this.#config.mergePolicy, terminalState: this.#config.terminalState },
|
|
5499
5942
|
reviewerName,
|
|
@@ -5502,6 +5945,8 @@ export class FactoryLoop {
|
|
|
5502
5945
|
slackDispatchThread: await this.#slackDispatchThreadFor(record),
|
|
5503
5946
|
integrationsMountRoot: this.#integrationsMountRoot(),
|
|
5504
5947
|
integrationInstructions,
|
|
5948
|
+
branchName: spec.branch,
|
|
5949
|
+
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
5505
5950
|
agentName: spec.name,
|
|
5506
5951
|
});
|
|
5507
5952
|
const spawned = await this.#spawnAgent(record, {
|
|
@@ -5670,6 +6115,7 @@ export class FactoryLoop {
|
|
|
5670
6115
|
return;
|
|
5671
6116
|
}
|
|
5672
6117
|
this.#completionInFlight.add(completionKey);
|
|
6118
|
+
let releaseReasonForRetry;
|
|
5673
6119
|
try {
|
|
5674
6120
|
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
5675
6121
|
return;
|
|
@@ -5755,6 +6201,7 @@ export class FactoryLoop {
|
|
|
5755
6201
|
await this.#runCompletionMergeGate(issue);
|
|
5756
6202
|
}
|
|
5757
6203
|
const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
|
|
6204
|
+
releaseReasonForRetry = releaseReason;
|
|
5758
6205
|
if (this.#fleet.placementLocality === 'remote') {
|
|
5759
6206
|
// Durable capacity is released as soon as terminal writeback is
|
|
5760
6207
|
// acknowledged. Agent cleanup remains fenced/retryable in `releasing`.
|
|
@@ -5771,7 +6218,10 @@ export class FactoryLoop {
|
|
|
5771
6218
|
}
|
|
5772
6219
|
catch (error) {
|
|
5773
6220
|
this.#error(error, record.issue);
|
|
5774
|
-
|
|
6221
|
+
if (releaseReasonForRetry)
|
|
6222
|
+
this.#scheduleReleaseRetry(record, releaseReasonForRetry);
|
|
6223
|
+
else
|
|
6224
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
5775
6225
|
}
|
|
5776
6226
|
finally {
|
|
5777
6227
|
this.#completionInFlight.delete(completionKey);
|
|
@@ -6025,6 +6475,19 @@ export class FactoryLoop {
|
|
|
6025
6475
|
this.#lastObservedSlackEventAtMs = this.#clock.now();
|
|
6026
6476
|
this.#increment('slackWebhookEventsObserved');
|
|
6027
6477
|
}
|
|
6478
|
+
async #persistedSlackThread(key) {
|
|
6479
|
+
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
6480
|
+
if (!threadId || /^\d+[._]\d+$/u.test(threadId))
|
|
6481
|
+
return threadId;
|
|
6482
|
+
// Older Factory versions persisted the Relayfile draft client id when the
|
|
6483
|
+
// acknowledged file had not yet reconciled its provider payload. Slack
|
|
6484
|
+
// cannot use that value as thread_ts. Drop it so the caller establishes a
|
|
6485
|
+
// fresh provider-backed root instead of producing invalid_thread_ts.
|
|
6486
|
+
await this.#state.clearSlackThread(this.#workspaceId, key);
|
|
6487
|
+
this.#increment('invalidSlackThreadsCleared');
|
|
6488
|
+
this.#logger.warn?.('[factory] cleared invalid persisted Slack thread id', { issue: key });
|
|
6489
|
+
return undefined;
|
|
6490
|
+
}
|
|
6028
6491
|
async #ensureSlackDispatchThread(record, result) {
|
|
6029
6492
|
if (!this.#slack || !this.#config.slack || result.dryRun) {
|
|
6030
6493
|
return;
|
|
@@ -6033,7 +6496,7 @@ export class FactoryLoop {
|
|
|
6033
6496
|
return;
|
|
6034
6497
|
}
|
|
6035
6498
|
const key = issueKey(record.issue);
|
|
6036
|
-
const existingThread = await this.#
|
|
6499
|
+
const existingThread = await this.#persistedSlackThread(key);
|
|
6037
6500
|
const watcherStart = this.#slackWatcherStarts.get(key);
|
|
6038
6501
|
if (existingThread || watcherStart) {
|
|
6039
6502
|
try {
|
|
@@ -6086,6 +6549,16 @@ export class FactoryLoop {
|
|
|
6086
6549
|
if (dryRun) {
|
|
6087
6550
|
return;
|
|
6088
6551
|
}
|
|
6552
|
+
// A source GitHub issue is the durable stakeholder record. Keep both the
|
|
6553
|
+
// question and authorized response there, then mirror the escalation once
|
|
6554
|
+
// to Slack for stakeholder visibility without making Slack a competing
|
|
6555
|
+
// clarification workflow.
|
|
6556
|
+
const sourceIssue = await this.#readIssue(decision.issue.path);
|
|
6557
|
+
if (sourceIssue && githubIssueSourceRef(sourceIssue)) {
|
|
6558
|
+
const result = await this.#escalateTriageToGithub(decision, reason);
|
|
6559
|
+
await this.#mirrorGithubTriageEscalationToSlack(decision, sourceIssue, reason);
|
|
6560
|
+
return result;
|
|
6561
|
+
}
|
|
6089
6562
|
if (!this.#slack || !this.#config.slack) {
|
|
6090
6563
|
return await this.#escalateTriageToGithub(decision, reason);
|
|
6091
6564
|
}
|
|
@@ -6093,7 +6566,7 @@ export class FactoryLoop {
|
|
|
6093
6566
|
return;
|
|
6094
6567
|
}
|
|
6095
6568
|
const key = issueKey(decision.issue);
|
|
6096
|
-
const existingThread = await this.#
|
|
6569
|
+
const existingThread = await this.#persistedSlackThread(key);
|
|
6097
6570
|
const watcherStart = this.#slackWatcherStarts.get(key);
|
|
6098
6571
|
if (existingThread || watcherStart) {
|
|
6099
6572
|
try {
|
|
@@ -6130,7 +6603,7 @@ export class FactoryLoop {
|
|
|
6130
6603
|
const correlationId = githubEscalationCorrelationId('triage', decision.issue, question);
|
|
6131
6604
|
const issue = await this.#readIssue(decision.issue.path);
|
|
6132
6605
|
const source = issue ? githubIssueSourceRef(issue) : undefined;
|
|
6133
|
-
const authorizedAuthor = issue ?
|
|
6606
|
+
const authorizedAuthor = issue ? await this.#resolveGithubIssueAuthor(issue) : undefined;
|
|
6134
6607
|
if (!issue || !source || !authorizedAuthor) {
|
|
6135
6608
|
this.#surfaceEscalationDeliveryFailure('triage', decision.issue, correlationId, 'no Slack channel or GitHub issue write path with an identifiable issue reporter is available');
|
|
6136
6609
|
return;
|
|
@@ -6146,7 +6619,7 @@ export class FactoryLoop {
|
|
|
6146
6619
|
}
|
|
6147
6620
|
try {
|
|
6148
6621
|
await this.#githubWriteback.postComment(issue, [
|
|
6149
|
-
|
|
6622
|
+
`@${authorizedAuthor}, Factory needs clarification before dispatching ${decision.issue.key}.`,
|
|
6150
6623
|
`Reason: ${reason}`,
|
|
6151
6624
|
`Question: ${question}`,
|
|
6152
6625
|
`Authorized responder: @${authorizedAuthor} (the issue reporter).`,
|
|
@@ -6161,15 +6634,119 @@ export class FactoryLoop {
|
|
|
6161
6634
|
this.#surfaceEscalationDeliveryFailure('triage', decision.issue, correlationId, 'GitHub issue comment writeback failed', error);
|
|
6162
6635
|
}
|
|
6163
6636
|
}
|
|
6637
|
+
async #mirrorGithubTriageEscalationToSlack(decision, issue, reason) {
|
|
6638
|
+
if (!this.#slack || !this.#config.slack)
|
|
6639
|
+
return;
|
|
6640
|
+
if (await this.#shouldSkipSlackWriteback('triage-escalation-mirror')) {
|
|
6641
|
+
this.#increment('triageEscalationSlackMirrorsSkippedDegraded');
|
|
6642
|
+
return;
|
|
6643
|
+
}
|
|
6644
|
+
const key = issueKey(decision.issue);
|
|
6645
|
+
const existingThread = await this.#persistedSlackThread(key);
|
|
6646
|
+
const inFlight = this.#slackWatcherStarts.get(key);
|
|
6647
|
+
if (existingThread || inFlight) {
|
|
6648
|
+
if (inFlight) {
|
|
6649
|
+
try {
|
|
6650
|
+
await inFlight;
|
|
6651
|
+
}
|
|
6652
|
+
catch {
|
|
6653
|
+
// The initiator records the optional mirror failure.
|
|
6654
|
+
}
|
|
6655
|
+
}
|
|
6656
|
+
this.#increment('triageEscalationSlackMirrorDuplicatesSuppressed');
|
|
6657
|
+
return;
|
|
6658
|
+
}
|
|
6659
|
+
const start = this.#postGithubTriageSlackMirror(decision, issue, reason);
|
|
6660
|
+
this.#slackWatcherStarts.set(key, start);
|
|
6661
|
+
try {
|
|
6662
|
+
await start;
|
|
6663
|
+
}
|
|
6664
|
+
catch (error) {
|
|
6665
|
+
this.#markSlackWritebackFailure('triage-escalation-mirror', error);
|
|
6666
|
+
this.#increment('triageEscalationSlackMirrorFailures');
|
|
6667
|
+
this.#logger.warn?.('[factory] optional GitHub triage Slack mirror failed', {
|
|
6668
|
+
issue: decision.issue.key,
|
|
6669
|
+
error: describeError(error).errorMessage,
|
|
6670
|
+
});
|
|
6671
|
+
}
|
|
6672
|
+
finally {
|
|
6673
|
+
this.#slackWatcherStarts.delete(key);
|
|
6674
|
+
}
|
|
6675
|
+
}
|
|
6676
|
+
async #postGithubTriageSlackMirror(decision, issue, reason) {
|
|
6677
|
+
if (!this.#slack || !this.#config.slack)
|
|
6678
|
+
return;
|
|
6679
|
+
const source = githubIssueSourceRef(issue);
|
|
6680
|
+
const stakeholderMentions = slackMentions(this.#config.slack.stakeholderUserIds);
|
|
6681
|
+
const reporter = await this.#resolveGithubIssueAuthor(issue);
|
|
6682
|
+
const audience = [stakeholderMentions, reporter ? `GitHub reporter: @${reporter}.` : undefined]
|
|
6683
|
+
.filter((part) => Boolean(part))
|
|
6684
|
+
.join(' ');
|
|
6685
|
+
const replyInstruction = source?.url
|
|
6686
|
+
? `Reply on the GitHub issue so Factory can resume: ${source.url}`
|
|
6687
|
+
: 'Reply on the source GitHub issue so Factory can resume.';
|
|
6688
|
+
const root = await this.#slack.postThread({
|
|
6689
|
+
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
6690
|
+
text: [
|
|
6691
|
+
`${audience ? `${audience} ` : ''}${decision.issue.key}: factory triage escalation for ${issue.title}`,
|
|
6692
|
+
`Reason: ${reason}`,
|
|
6693
|
+
`Question: ${triageEscalationQuestion(decision)} ${replyInstruction}`,
|
|
6694
|
+
].join('\n'),
|
|
6695
|
+
});
|
|
6696
|
+
await this.#state.setSlackThread(this.#workspaceId, issueKey(decision.issue), root.threadId);
|
|
6697
|
+
this.#increment('triageEscalationsMirroredToSlack');
|
|
6698
|
+
this.#recordSlackWritebackSuccess('triage-escalation-mirror');
|
|
6699
|
+
}
|
|
6700
|
+
async #resolveGithubIssueAuthor(issue) {
|
|
6701
|
+
const embeddedAuthor = githubIssueAuthor(issue);
|
|
6702
|
+
if (embeddedAuthor)
|
|
6703
|
+
return embeddedAuthor;
|
|
6704
|
+
const source = githubIssueSourceRef(issue);
|
|
6705
|
+
const lookup = this.#githubWriteback.getIssueAuthor;
|
|
6706
|
+
if (!source || !lookup)
|
|
6707
|
+
return undefined;
|
|
6708
|
+
const key = githubIssueSourceKey(source);
|
|
6709
|
+
if (this.#githubIssueAuthors.has(key)) {
|
|
6710
|
+
return this.#githubIssueAuthors.get(key);
|
|
6711
|
+
}
|
|
6712
|
+
const existing = this.#githubIssueAuthorLookups.get(key);
|
|
6713
|
+
if (existing)
|
|
6714
|
+
return existing;
|
|
6715
|
+
const pending = lookup.call(this.#githubWriteback, issue)
|
|
6716
|
+
.then((author) => {
|
|
6717
|
+
const normalized = author?.trim() || undefined;
|
|
6718
|
+
this.#githubIssueAuthors.set(key, normalized);
|
|
6719
|
+
if (normalized) {
|
|
6720
|
+
this.#increment('githubIssueAuthorsResolvedFromProvider');
|
|
6721
|
+
}
|
|
6722
|
+
return normalized;
|
|
6723
|
+
})
|
|
6724
|
+
.catch((error) => {
|
|
6725
|
+
this.#increment('githubIssueAuthorLookupFailures');
|
|
6726
|
+
this.#logger.warn?.('[factory] provider-authoritative GitHub issue author lookup failed', {
|
|
6727
|
+
owner: source.owner,
|
|
6728
|
+
repo: source.repo,
|
|
6729
|
+
issue: source.number,
|
|
6730
|
+
error: describeError(error).errorMessage,
|
|
6731
|
+
});
|
|
6732
|
+
return undefined;
|
|
6733
|
+
})
|
|
6734
|
+
.finally(() => {
|
|
6735
|
+
this.#githubIssueAuthorLookups.delete(key);
|
|
6736
|
+
});
|
|
6737
|
+
this.#githubIssueAuthorLookups.set(key, pending);
|
|
6738
|
+
return pending;
|
|
6739
|
+
}
|
|
6164
6740
|
async #postAndWatchSlackEscalationThread(decision, reason) {
|
|
6165
6741
|
if (!this.#slack || !this.#config.slack) {
|
|
6166
6742
|
return;
|
|
6167
6743
|
}
|
|
6168
6744
|
const issue = await this.#readIssue(decision.issue.path);
|
|
6745
|
+
const stakeholderMentions = slackMentions(this.#config.slack.stakeholderUserIds);
|
|
6169
6746
|
const root = await this.#slack.postThread({
|
|
6170
6747
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
6171
6748
|
text: [
|
|
6172
|
-
`${decision.issue.key}: factory triage escalation for ${issue?.title ?? decision.issue.key}`,
|
|
6749
|
+
`${stakeholderMentions ? `${stakeholderMentions} ` : ''}${decision.issue.key}: factory triage escalation for ${issue?.title ?? decision.issue.key}`,
|
|
6173
6750
|
`Reason: ${reason}`,
|
|
6174
6751
|
`Question: ${triageEscalationQuestion(decision)}`,
|
|
6175
6752
|
].join('\n'),
|
|
@@ -6362,7 +6939,7 @@ export class FactoryLoop {
|
|
|
6362
6939
|
}
|
|
6363
6940
|
let threadId;
|
|
6364
6941
|
try {
|
|
6365
|
-
threadId = await this.#
|
|
6942
|
+
threadId = await this.#persistedSlackThread(key);
|
|
6366
6943
|
}
|
|
6367
6944
|
catch (error) {
|
|
6368
6945
|
this.#logger.warn?.('[factory] unable to read persisted Slack thread during watcher rehydration', { issue: record.issue.key, error });
|
|
@@ -6857,6 +7434,7 @@ export class FactoryLoop {
|
|
|
6857
7434
|
}
|
|
6858
7435
|
async #resumeOrColdStartClarificationAgent(name, tracked, waiting) {
|
|
6859
7436
|
const task = clarificationResumeTask(tracked.spec.task, waiting);
|
|
7437
|
+
await this.#prepareAgentWorktree(waitingRecord(waiting), tracked.spec);
|
|
6860
7438
|
if (tracked.sessionRef) {
|
|
6861
7439
|
try {
|
|
6862
7440
|
const resumed = await this.#fleet.resume({
|
|
@@ -7066,7 +7644,7 @@ export class FactoryLoop {
|
|
|
7066
7644
|
if (!this.#config.slack) {
|
|
7067
7645
|
return undefined;
|
|
7068
7646
|
}
|
|
7069
|
-
const threadId = await this.#
|
|
7647
|
+
const threadId = await this.#persistedSlackThread(issueKey(record.issue));
|
|
7070
7648
|
const channel = await this.#slackChannelDir() ?? this.#config.slack.channel;
|
|
7071
7649
|
return threadId
|
|
7072
7650
|
? { channel, threadId, mountRoot: this.#integrationsMountRoot() }
|
|
@@ -7607,36 +8185,35 @@ function routeImplementerSpec(issue, config, slug, route) {
|
|
|
7607
8185
|
node: 'self',
|
|
7608
8186
|
};
|
|
7609
8187
|
}
|
|
7610
|
-
function decisionWithLifecycleBranches(decision, runId) {
|
|
7611
|
-
const
|
|
8188
|
+
function decisionWithLifecycleBranches(decision, runId, opts = {}) {
|
|
8189
|
+
const implementerBranch = (spec) => {
|
|
8190
|
+
const runSuffix = `-${runId.slice(0, 8)}`;
|
|
8191
|
+
const stem = `${sanitizeAgentSlug(decision.issue.key)}-${sanitizeAgentSlug(spec.repo)}`
|
|
8192
|
+
.slice(0, 120 - 'factory/'.length - runSuffix.length);
|
|
8193
|
+
return `factory/${stem}${runSuffix}`;
|
|
8194
|
+
};
|
|
8195
|
+
const branchByRepo = new Map(decision.implementers.map((spec) => [spec.repo, implementerBranch(spec)]));
|
|
8196
|
+
const withBranch = (spec, branch) => {
|
|
8197
|
+
const baseClonePath = spec.baseClonePath ?? spec.clonePath;
|
|
8198
|
+
const clonePath = opts.isolateLocalWorktree && baseClonePath && branch
|
|
8199
|
+
? factoryWorktreePath(baseClonePath, decision.issue.key, spec.repo, runId)
|
|
8200
|
+
: spec.clonePath;
|
|
7612
8201
|
const lifecycleSpec = {
|
|
7613
8202
|
...spec,
|
|
8203
|
+
...(opts.isolateLocalWorktree && baseClonePath && branch ? { baseClonePath, clonePath } : {}),
|
|
7614
8204
|
// The same persisted lifecycle reuses this id after takeover, while a
|
|
7615
8205
|
// genuine reopen gets a new id and cannot replay an old placement ack.
|
|
7616
8206
|
invocationId: `factory:${decision.issue.key}:${runId}:${spec.role}:${sanitizeAgentSlug(spec.name)}`,
|
|
7617
8207
|
};
|
|
7618
|
-
|
|
7619
|
-
return lifecycleSpec;
|
|
7620
|
-
const runSuffix = `-${runId.slice(0, 8)}`;
|
|
7621
|
-
const stem = `${sanitizeAgentSlug(decision.issue.key)}-${sanitizeAgentSlug(spec.repo)}`
|
|
7622
|
-
.slice(0, 120 - 'factory/'.length - runSuffix.length);
|
|
7623
|
-
const branch = `factory/${stem}${runSuffix}`;
|
|
7624
|
-
return {
|
|
7625
|
-
...lifecycleSpec,
|
|
7626
|
-
branch,
|
|
7627
|
-
task: [
|
|
7628
|
-
spec.task,
|
|
7629
|
-
'',
|
|
7630
|
-
`Factory publication branch: ${branch}`,
|
|
7631
|
-
'Before editing, create or reset that exact branch from the repository default branch. Commit and push only that branch.',
|
|
7632
|
-
].join('\n'),
|
|
7633
|
-
};
|
|
8208
|
+
return branch ? { ...lifecycleSpec, branch } : lifecycleSpec;
|
|
7634
8209
|
};
|
|
7635
8210
|
return {
|
|
7636
8211
|
...structuredClone(decision),
|
|
7637
|
-
implementers: decision.implementers.map(withBranch),
|
|
7638
|
-
reviewer: withBranch(decision.reviewer),
|
|
7639
|
-
...(decision.workflow
|
|
8212
|
+
implementers: decision.implementers.map((spec) => withBranch(spec, branchByRepo.get(spec.repo))),
|
|
8213
|
+
reviewer: withBranch(decision.reviewer, branchByRepo.get(decision.reviewer.repo)),
|
|
8214
|
+
...(decision.workflow
|
|
8215
|
+
? { workflow: withBranch(decision.workflow, branchByRepo.get(decision.workflow.repo)) }
|
|
8216
|
+
: {}),
|
|
7640
8217
|
};
|
|
7641
8218
|
}
|
|
7642
8219
|
function routeReviewerSpec(issue, config, route, reviewer) {
|
|
@@ -7736,12 +8313,21 @@ function taskForDispatch(issue, route, role) {
|
|
|
7736
8313
|
issue.description,
|
|
7737
8314
|
].join('\n\n');
|
|
7738
8315
|
}
|
|
7739
|
-
const templateIssueFromRecord = (record, issue) =>
|
|
7740
|
-
|
|
7741
|
-
|
|
7742
|
-
|
|
7743
|
-
|
|
7744
|
-
|
|
8316
|
+
const templateIssueFromRecord = (record, issue) => {
|
|
8317
|
+
const github = issue ? githubIssueSourceRef(issue) : undefined;
|
|
8318
|
+
const reporter = issue ? githubIssueAuthor(issue) : undefined;
|
|
8319
|
+
return {
|
|
8320
|
+
key: issue?.key ?? record.issue.key,
|
|
8321
|
+
title: issue?.title ?? record.issue.key,
|
|
8322
|
+
description: issue?.description ?? '',
|
|
8323
|
+
github: github
|
|
8324
|
+
? {
|
|
8325
|
+
...github,
|
|
8326
|
+
...(reporter ? { reporter } : {}),
|
|
8327
|
+
}
|
|
8328
|
+
: undefined,
|
|
8329
|
+
};
|
|
8330
|
+
};
|
|
7745
8331
|
const routeForSpec = (decision, spec) => {
|
|
7746
8332
|
const route = decision.routes.find((candidate) => candidate.repo === spec.repo && candidate.clonePath === spec.clonePath) ?? decision.routes.find((candidate) => candidate.repo === spec.repo);
|
|
7747
8333
|
return {
|
|
@@ -7786,6 +8372,26 @@ export const githubIssuePathParts = (path) => {
|
|
|
7786
8372
|
slug: match[7],
|
|
7787
8373
|
};
|
|
7788
8374
|
};
|
|
8375
|
+
const githubIssueIdentity = (owner, repo, number) => `${owner.toLowerCase()}/${repo.toLowerCase()}#${number}`;
|
|
8376
|
+
const githubIssueRefIdentity = (issue) => {
|
|
8377
|
+
const parts = githubIssuePathParts(issue.path);
|
|
8378
|
+
return parts ? githubIssueIdentity(parts.owner, parts.repo, parts.number) : undefined;
|
|
8379
|
+
};
|
|
8380
|
+
const githubIssuePathPreference = (path) => {
|
|
8381
|
+
if (path.includes('/issues/by-id/'))
|
|
8382
|
+
return 0;
|
|
8383
|
+
if (path.endsWith('/meta.json'))
|
|
8384
|
+
return 1;
|
|
8385
|
+
if (path.endsWith('.json'))
|
|
8386
|
+
return 2;
|
|
8387
|
+
return 3;
|
|
8388
|
+
};
|
|
8389
|
+
const githubAgentNameMatchesIssue = (name, issue) => {
|
|
8390
|
+
const parts = githubIssuePathParts(issue.path);
|
|
8391
|
+
if (!parts)
|
|
8392
|
+
return false;
|
|
8393
|
+
return name.startsWith(`ar-${parts.number}-`) && name.endsWith(`-${sanitizeAgentSlug(parts.repo)}`);
|
|
8394
|
+
};
|
|
7789
8395
|
const githubIssueCommentPathParts = (path) => {
|
|
7790
8396
|
const match = path.match(/^\/github\/repos\/(?:([^/]+)\/([^/]+)|([A-Za-z0-9-]+)__([^/]+))\/issues\/(\d+)(?:__[^/]*)?\/comments\/([^/]+?)(?:\.json|\/(?:meta|metadata)\.json)$/u);
|
|
7791
8397
|
const owner = match?.[1] ?? match?.[3];
|
|
@@ -7945,6 +8551,8 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}) => {
|
|
|
7945
8551
|
if (!path.endsWith('.json'))
|
|
7946
8552
|
continue;
|
|
7947
8553
|
const pr = await readProbePrCandidate(mount, path);
|
|
8554
|
+
if (opts.openOnly && normalizePrState(pr?.state) !== 'OPEN')
|
|
8555
|
+
continue;
|
|
7948
8556
|
const score = pr
|
|
7949
8557
|
? issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts)
|
|
7950
8558
|
: 0;
|
|
@@ -7957,6 +8565,7 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}) => {
|
|
|
7957
8565
|
};
|
|
7958
8566
|
const resolveIssuePrFromGh = async (run, config, issue, opts = {}, logger) => {
|
|
7959
8567
|
const candidates = [];
|
|
8568
|
+
let lookupFailures = 0;
|
|
7960
8569
|
for (const repo of reposFromConfig(config)) {
|
|
7961
8570
|
let payload;
|
|
7962
8571
|
try {
|
|
@@ -7973,26 +8582,33 @@ const resolveIssuePrFromGh = async (run, config, issue, opts = {}, logger) => {
|
|
|
7973
8582
|
String(PROBE_PR_GH_CANDIDATE_LIMIT),
|
|
7974
8583
|
]);
|
|
7975
8584
|
if (!result.stdout.trim()) {
|
|
8585
|
+
lookupFailures += 1;
|
|
7976
8586
|
logger?.warn?.('[factory] gh PR resolver returned empty output', { issue: issue.key, repo });
|
|
7977
8587
|
continue;
|
|
7978
8588
|
}
|
|
7979
8589
|
payload = parseJsonContent(result.stdout);
|
|
7980
8590
|
}
|
|
7981
8591
|
catch (error) {
|
|
8592
|
+
lookupFailures += 1;
|
|
7982
8593
|
logger?.warn?.('[factory] gh PR resolver failed', { issue: issue.key, repo, error });
|
|
7983
8594
|
continue;
|
|
7984
8595
|
}
|
|
7985
8596
|
if (!Array.isArray(payload)) {
|
|
8597
|
+
lookupFailures += 1;
|
|
7986
8598
|
logger?.warn?.('[factory] gh PR resolver returned non-array payload', { issue: issue.key, repo });
|
|
7987
8599
|
continue;
|
|
7988
8600
|
}
|
|
7989
8601
|
if (payload.length >= PROBE_PR_GH_CANDIDATE_LIMIT) {
|
|
7990
8602
|
logger?.warn?.('[factory] gh PR resolver hit candidate limit', { issue: issue.key, repo, limit: PROBE_PR_GH_CANDIDATE_LIMIT });
|
|
8603
|
+
if (opts.failOnLookupError)
|
|
8604
|
+
lookupFailures += 1;
|
|
7991
8605
|
}
|
|
7992
8606
|
for (const entry of payload) {
|
|
7993
8607
|
const pr = ghProbePrCandidate(entry);
|
|
7994
8608
|
if (!pr || !containsIssueKey(pr.headRef, issue.key))
|
|
7995
8609
|
continue;
|
|
8610
|
+
if (opts.openOnly && normalizePrState(pr.state) !== 'OPEN')
|
|
8611
|
+
continue;
|
|
7996
8612
|
const score = issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts);
|
|
7997
8613
|
if (score <= 0)
|
|
7998
8614
|
continue;
|
|
@@ -8005,9 +8621,13 @@ const resolveIssuePrFromGh = async (run, config, issue, opts = {}, logger) => {
|
|
|
8005
8621
|
});
|
|
8006
8622
|
}
|
|
8007
8623
|
}
|
|
8008
|
-
|
|
8624
|
+
const resolved = candidates.sort((a, b) => b.score - a.score ||
|
|
8009
8625
|
Number(b.open) - Number(a.open) ||
|
|
8010
8626
|
b.prNumber - a.prNumber)[0];
|
|
8627
|
+
if (!resolved && opts.failOnLookupError && lookupFailures > 0) {
|
|
8628
|
+
throw new Error(`Unable to confirm open pull request state for ${issue.key} in ${lookupFailures} configured repository lookup(s)`);
|
|
8629
|
+
}
|
|
8630
|
+
return resolved;
|
|
8011
8631
|
};
|
|
8012
8632
|
const reposFromConfig = (config) => {
|
|
8013
8633
|
const repos = new Set([
|
|
@@ -8102,6 +8722,7 @@ const readProbePrCandidate = async (mount, path) => {
|
|
|
8102
8722
|
body: stringValue(payload.body) ?? '',
|
|
8103
8723
|
headRef: refName(payload.headRef) ?? refName(payload.head) ?? stringValue(payload.head_ref) ?? '',
|
|
8104
8724
|
draft: booleanValue(payload.isDraft) ?? booleanValue(payload.draft),
|
|
8725
|
+
state: stringValue(payload.state),
|
|
8105
8726
|
};
|
|
8106
8727
|
}
|
|
8107
8728
|
catch {
|
|
@@ -8717,12 +9338,10 @@ const isAgentAlreadyExistsError = (error) => {
|
|
|
8717
9338
|
return /already exists/iu.test(message);
|
|
8718
9339
|
};
|
|
8719
9340
|
const defaultRestartPolicy = (spec) =>
|
|
8720
|
-
//
|
|
8721
|
-
//
|
|
8722
|
-
// exit should resume its session rather than drop the PR. The reviewer is
|
|
8723
|
-
// short-lived and keeps the fleet default.
|
|
9341
|
+
// Factory owns durable resume/respawn decisions. Broker-level retries race
|
|
9342
|
+
// that lifecycle and can re-register the same name before Factory resumes it.
|
|
8724
9343
|
spec.role === 'implementer' || spec.role === 'babysitter'
|
|
8725
|
-
? { maxRestarts:
|
|
9344
|
+
? { maxRestarts: 0 }
|
|
8726
9345
|
: spec.restartPolicy;
|
|
8727
9346
|
const slackPayloadTs = (threadId) => threadId.replace(/_/g, '.');
|
|
8728
9347
|
const slackChannelMessagesPrefix = (channelDir) => `/slack/channels/${channelDir}/messages/`;
|