@agent-relay/factory 0.1.32 → 0.1.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/dist/cli/fleet.d.ts +4 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +115 -6
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +103 -8
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +12 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +9 -2
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +39 -1
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +1 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +1 -0
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/observability/cloud-reporter.d.ts +46 -0
- package/dist/observability/cloud-reporter.d.ts.map +1 -0
- package/dist/observability/cloud-reporter.js +371 -0
- package/dist/observability/cloud-reporter.js.map +1 -0
- package/dist/observability/events.d.ts +1315 -0
- package/dist/observability/events.d.ts.map +1 -0
- package/dist/observability/events.js +242 -0
- package/dist/observability/events.js.map +1 -0
- package/dist/observability/index.d.ts +5 -0
- package/dist/observability/index.d.ts.map +1 -0
- package/dist/observability/index.js +5 -0
- package/dist/observability/index.js.map +1 -0
- package/dist/observability/instance-identity.d.ts +21 -0
- package/dist/observability/instance-identity.d.ts.map +1 -0
- package/dist/observability/instance-identity.js +77 -0
- package/dist/observability/instance-identity.js.map +1 -0
- package/dist/observability/outbox.d.ts +42 -0
- package/dist/observability/outbox.d.ts.map +1 -0
- package/dist/observability/outbox.js +305 -0
- package/dist/observability/outbox.js.map +1 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +495 -60
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +3 -4
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -0
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/observability.d.ts +19 -0
- package/dist/ports/observability.d.ts.map +1 -0
- package/dist/ports/observability.js +2 -0
- package/dist/ports/observability.js.map +1 -0
- package/dist/triage/schema.d.ts +14 -14
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -1
|
@@ -20,6 +20,7 @@ import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../write
|
|
|
20
20
|
import { issueKey } from './batch-tracker.js';
|
|
21
21
|
import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
|
|
22
22
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
23
|
+
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
23
24
|
class ClarificationWakeLeaseLostError extends Error {
|
|
24
25
|
}
|
|
25
26
|
class ClarificationQuestionDeliveryLeaseLostError extends Error {
|
|
@@ -119,6 +120,7 @@ export class FactoryLoop {
|
|
|
119
120
|
#workspaceId;
|
|
120
121
|
#relayflows;
|
|
121
122
|
#worktrees;
|
|
123
|
+
#reporter;
|
|
122
124
|
#batchView;
|
|
123
125
|
#batchReady;
|
|
124
126
|
#listeners = new Map();
|
|
@@ -132,6 +134,8 @@ export class FactoryLoop {
|
|
|
132
134
|
#githubIssueCommentQueues = new Map();
|
|
133
135
|
#githubIssueAuthors = new Map();
|
|
134
136
|
#githubIssueAuthorLookups = new Map();
|
|
137
|
+
#githubIssuePreferredPaths = new Map();
|
|
138
|
+
#githubIssuePathIndexReady = false;
|
|
135
139
|
#slackReporterUserIds = new Map();
|
|
136
140
|
#slackReporterUserIdLookups = new Map();
|
|
137
141
|
#reconciledGithubInProgress = new Set();
|
|
@@ -263,6 +267,7 @@ export class FactoryLoop {
|
|
|
263
267
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
264
268
|
this.#relayflows = ports.relayflows;
|
|
265
269
|
this.#worktrees = ports.worktrees;
|
|
270
|
+
this.#reporter = ports.reporter;
|
|
266
271
|
this.#state = ports.stateStore ?? new InMemoryStateStore({
|
|
267
272
|
batchSize: config.batchSize,
|
|
268
273
|
agentQuestionDedupeLimit: AGENT_QUESTION_DEDUPE_LIMIT,
|
|
@@ -1362,6 +1367,7 @@ export class FactoryLoop {
|
|
|
1362
1367
|
]);
|
|
1363
1368
|
const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
|
|
1364
1369
|
const activeIssueIdentities = new Set();
|
|
1370
|
+
const legacyUnownedAgentsByIssue = new Map();
|
|
1365
1371
|
for (const [, lifecycle] of lifecycles) {
|
|
1366
1372
|
if (isTerminalDispatchLifecycle(lifecycle))
|
|
1367
1373
|
continue;
|
|
@@ -1378,12 +1384,26 @@ export class FactoryLoop {
|
|
|
1378
1384
|
if (!onlineAgents.has(agent.name) || !agent.issue)
|
|
1379
1385
|
continue;
|
|
1380
1386
|
const identity = githubIssueRefIdentity(agent.issue);
|
|
1381
|
-
if (identity)
|
|
1387
|
+
if (!identity)
|
|
1388
|
+
continue;
|
|
1389
|
+
const isLegacyLocalWorker = this.#usesDurableDispatchLifecycle() &&
|
|
1390
|
+
this.#fleet.placementLocality === 'local' &&
|
|
1391
|
+
!agent.node &&
|
|
1392
|
+
!agent.invocationId &&
|
|
1393
|
+
!activeIssueIdentities.has(identity);
|
|
1394
|
+
if (isLegacyLocalWorker) {
|
|
1395
|
+
const agents = legacyUnownedAgentsByIssue.get(identity) ?? [];
|
|
1396
|
+
agents.push(agent);
|
|
1397
|
+
legacyUnownedAgentsByIssue.set(identity, agents);
|
|
1398
|
+
}
|
|
1399
|
+
else {
|
|
1382
1400
|
activeIssueIdentities.add(identity);
|
|
1401
|
+
}
|
|
1383
1402
|
}
|
|
1384
1403
|
return {
|
|
1385
1404
|
activeIssueIdentities,
|
|
1386
1405
|
onlineAgentNames: onlineAgents,
|
|
1406
|
+
legacyUnownedAgentsByIssue,
|
|
1387
1407
|
};
|
|
1388
1408
|
}
|
|
1389
1409
|
catch (error) {
|
|
@@ -1405,9 +1425,14 @@ export class FactoryLoop {
|
|
|
1405
1425
|
labels.has('factory:human-review'))
|
|
1406
1426
|
return false;
|
|
1407
1427
|
const identity = githubIssueRefIdentity(issueRef(issue));
|
|
1428
|
+
const legacyUnownedAgents = identity
|
|
1429
|
+
? (context.legacyUnownedAgentsByIssue.get(identity) ?? [])
|
|
1430
|
+
.filter((agent) => githubAgentNameMatchesIssue(agent.name, issue))
|
|
1431
|
+
: [];
|
|
1432
|
+
const legacyUnownedAgentNames = new Set(legacyUnownedAgents.map((agent) => agent.name));
|
|
1408
1433
|
if (!identity ||
|
|
1409
1434
|
context.activeIssueIdentities.has(identity) ||
|
|
1410
|
-
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue))) {
|
|
1435
|
+
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue) && !legacyUnownedAgentNames.has(name))) {
|
|
1411
1436
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1412
1437
|
return false;
|
|
1413
1438
|
}
|
|
@@ -1447,7 +1472,7 @@ export class FactoryLoop {
|
|
|
1447
1472
|
if (openPr) {
|
|
1448
1473
|
let adopted = false;
|
|
1449
1474
|
try {
|
|
1450
|
-
adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr);
|
|
1475
|
+
adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr, legacyUnownedAgents);
|
|
1451
1476
|
}
|
|
1452
1477
|
catch (error) {
|
|
1453
1478
|
this.#increment('githubOrphanedPullRequestAdoptionFailures');
|
|
@@ -1468,6 +1493,14 @@ export class FactoryLoop {
|
|
|
1468
1493
|
});
|
|
1469
1494
|
return false;
|
|
1470
1495
|
}
|
|
1496
|
+
// A pre-durable local Factory may have left live, registry-proven workers
|
|
1497
|
+
// without a lifecycle record. They are safe to adopt only once their open
|
|
1498
|
+
// PR proves which dispatch they own. Without that proof, preserve the issue
|
|
1499
|
+
// and workers instead of redispatching duplicate agents.
|
|
1500
|
+
if (legacyUnownedAgents.length > 0) {
|
|
1501
|
+
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1502
|
+
return false;
|
|
1503
|
+
}
|
|
1471
1504
|
try {
|
|
1472
1505
|
if (providerStatus === 'in-progress') {
|
|
1473
1506
|
await this.#githubWriteback.setStatus(issue, 'ready');
|
|
@@ -1504,7 +1537,7 @@ export class FactoryLoop {
|
|
|
1504
1537
|
allowLegacyGithubBranch: true,
|
|
1505
1538
|
});
|
|
1506
1539
|
}
|
|
1507
|
-
async #adoptOrphanedGithubPullRequest(issue, pr) {
|
|
1540
|
+
async #adoptOrphanedGithubPullRequest(issue, pr, legacyUnownedAgents = []) {
|
|
1508
1541
|
const headRef = pr.headRef;
|
|
1509
1542
|
if (!headRef)
|
|
1510
1543
|
return false;
|
|
@@ -1543,14 +1576,14 @@ export class FactoryLoop {
|
|
|
1543
1576
|
: spec),
|
|
1544
1577
|
};
|
|
1545
1578
|
}
|
|
1546
|
-
const
|
|
1579
|
+
const durableAdoption = this.#usesDurableDispatchLifecycle();
|
|
1547
1580
|
const publishedPr = {
|
|
1548
1581
|
repo: pr.repo,
|
|
1549
1582
|
number: pr.prNumber,
|
|
1550
1583
|
url: pr.url ?? `https://github.com/${pr.repo}/pull/${pr.prNumber}`,
|
|
1551
1584
|
headRef,
|
|
1552
1585
|
};
|
|
1553
|
-
if (
|
|
1586
|
+
if (durableAdoption) {
|
|
1554
1587
|
const claim = await this.#claimDispatchLifecycle(decision, false, randomUUID(), {
|
|
1555
1588
|
phase: 'published',
|
|
1556
1589
|
pullRequest: publishedPr,
|
|
@@ -1566,7 +1599,7 @@ export class FactoryLoop {
|
|
|
1566
1599
|
const batch = await this.#batch();
|
|
1567
1600
|
const record = batch.start(decision, false);
|
|
1568
1601
|
if (!record) {
|
|
1569
|
-
if (
|
|
1602
|
+
if (durableAdoption) {
|
|
1570
1603
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
|
|
1571
1604
|
if (durable) {
|
|
1572
1605
|
this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(durable));
|
|
@@ -1576,6 +1609,48 @@ export class FactoryLoop {
|
|
|
1576
1609
|
}
|
|
1577
1610
|
return false;
|
|
1578
1611
|
}
|
|
1612
|
+
if (durableAdoption && legacyUnownedAgents.length > 0) {
|
|
1613
|
+
const specsByName = new Map(dispatchSpecs(record.decision).map((spec) => [spec.name, spec]));
|
|
1614
|
+
const initialBabysitter = babysitterSpec(issue, this.#config, route);
|
|
1615
|
+
const sharedCheckout = record.decision.implementers.find((candidate) => candidate.repo === initialBabysitter.repo && candidate.baseClonePath && candidate.clonePath);
|
|
1616
|
+
const legacyBabysitter = {
|
|
1617
|
+
...initialBabysitter,
|
|
1618
|
+
...(sharedCheckout
|
|
1619
|
+
? {
|
|
1620
|
+
baseClonePath: sharedCheckout.baseClonePath,
|
|
1621
|
+
clonePath: sharedCheckout.clonePath,
|
|
1622
|
+
...(headRef ? { branch: headRef } : {}),
|
|
1623
|
+
...(sharedCheckout.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
1624
|
+
}
|
|
1625
|
+
: {}),
|
|
1626
|
+
ownedPullRequest: { repo: pr.repo, number: pr.prNumber, path: pr.path },
|
|
1627
|
+
};
|
|
1628
|
+
specsByName.set(legacyBabysitter.name, legacyBabysitter);
|
|
1629
|
+
const adopted = [];
|
|
1630
|
+
for (const agent of legacyUnownedAgents) {
|
|
1631
|
+
const spec = specsByName.get(agent.name);
|
|
1632
|
+
if (!spec)
|
|
1633
|
+
continue;
|
|
1634
|
+
const invocationId = batch.invocationIdFor(record.issue, spec);
|
|
1635
|
+
batch.recordSpawn(record, spec, invocationId, {
|
|
1636
|
+
name: agent.name,
|
|
1637
|
+
sessionRef: agent.sessionRef,
|
|
1638
|
+
pids: agent.pids,
|
|
1639
|
+
locality: 'local',
|
|
1640
|
+
});
|
|
1641
|
+
adopted.push({ name: agent.name, invocationId });
|
|
1642
|
+
}
|
|
1643
|
+
if (adopted.length > 0) {
|
|
1644
|
+
this.#fleet.hydrateTracked?.(adopted);
|
|
1645
|
+
await this.#saveDispatchLifecycle(record, 'published', publishedPr);
|
|
1646
|
+
for (const agent of adopted) {
|
|
1647
|
+
this.#increment('legacyLocalWorkersAdopted');
|
|
1648
|
+
const tracked = record.agents.get(agent.name);
|
|
1649
|
+
if (tracked)
|
|
1650
|
+
await this.#reportAgent(record, tracked, 'agent.adopted');
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1579
1654
|
await this.#ensureBabysitter(record, {
|
|
1580
1655
|
repo: pr.repo,
|
|
1581
1656
|
prNumber: pr.prNumber,
|
|
@@ -1585,7 +1660,7 @@ export class FactoryLoop {
|
|
|
1585
1660
|
});
|
|
1586
1661
|
const babysitter = [...record.agents.values()].find((tracked) => tracked.spec.role === 'babysitter');
|
|
1587
1662
|
if (!babysitter) {
|
|
1588
|
-
if (
|
|
1663
|
+
if (durableAdoption)
|
|
1589
1664
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
1590
1665
|
else
|
|
1591
1666
|
batch.abandon(record.issue);
|
|
@@ -1593,10 +1668,15 @@ export class FactoryLoop {
|
|
|
1593
1668
|
}
|
|
1594
1669
|
record.result = {
|
|
1595
1670
|
issue: record.issue,
|
|
1596
|
-
agents: [
|
|
1671
|
+
agents: [...record.agents.values()].map((tracked) => ({
|
|
1672
|
+
name: tracked.result?.name ?? tracked.spec.name,
|
|
1673
|
+
role: tracked.spec.role,
|
|
1674
|
+
})),
|
|
1597
1675
|
dryRun: false,
|
|
1598
1676
|
};
|
|
1599
1677
|
await this.#writeInFlightRegistry();
|
|
1678
|
+
if (durableAdoption)
|
|
1679
|
+
await this.#saveDispatchLifecycle(record, 'running', publishedPr);
|
|
1600
1680
|
this.#increment('githubOrphanedPullRequestsAdopted');
|
|
1601
1681
|
return true;
|
|
1602
1682
|
}
|
|
@@ -1778,7 +1858,7 @@ export class FactoryLoop {
|
|
|
1778
1858
|
if (existingRecord?.result) {
|
|
1779
1859
|
return existingRecord.result;
|
|
1780
1860
|
}
|
|
1781
|
-
if (!dryRun && this.#
|
|
1861
|
+
if (!dryRun && this.#usesDurableDispatchLifecycle()) {
|
|
1782
1862
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
|
|
1783
1863
|
if (durable && !isTerminalDispatchLifecycle(durable)) {
|
|
1784
1864
|
if (durable.result && this.#dispatchLifecycleEpochs.has(issueKey(decision.issue))) {
|
|
@@ -1803,6 +1883,9 @@ export class FactoryLoop {
|
|
|
1803
1883
|
this.#error(error, decision.issue);
|
|
1804
1884
|
throw error;
|
|
1805
1885
|
}
|
|
1886
|
+
if (!this.#isIssueReady(liveIssue)) {
|
|
1887
|
+
throw new LiveDispatchStateChangedError(decision.issue.key);
|
|
1888
|
+
}
|
|
1806
1889
|
if (!isDispatchableIssue(liveIssue)) {
|
|
1807
1890
|
const error = new Error(`Refusing to dispatch ${decision.issue.key}: not reconciled real Linear issue`);
|
|
1808
1891
|
this.#error(error, decision.issue);
|
|
@@ -1849,19 +1932,19 @@ export class FactoryLoop {
|
|
|
1849
1932
|
// Full task rendering is part of the durable spawn specification. It must
|
|
1850
1933
|
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
1851
1934
|
// recover a persisted minimal triage task after a crash in this gap.
|
|
1852
|
-
const
|
|
1935
|
+
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
1853
1936
|
// Local dispatches need the same deterministic branch identity as remote
|
|
1854
1937
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
1855
1938
|
// and concurrent issues can switch each other back to the base branch.
|
|
1856
1939
|
const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
|
|
1857
|
-
const lifecycleRunId = !dryRun && (
|
|
1940
|
+
const lifecycleRunId = !dryRun && (durableDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
|
|
1858
1941
|
if (lifecycleRunId) {
|
|
1859
1942
|
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
|
|
1860
1943
|
isolateLocalWorktree,
|
|
1861
1944
|
});
|
|
1862
1945
|
}
|
|
1863
1946
|
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
1864
|
-
if (
|
|
1947
|
+
if (durableDispatch) {
|
|
1865
1948
|
const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun, lifecycleRunId);
|
|
1866
1949
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
1867
1950
|
dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision);
|
|
@@ -1881,7 +1964,7 @@ export class FactoryLoop {
|
|
|
1881
1964
|
return restored.result;
|
|
1882
1965
|
}
|
|
1883
1966
|
}
|
|
1884
|
-
if (!
|
|
1967
|
+
if (!durableDispatch)
|
|
1885
1968
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
1886
1969
|
await this.#recordDispatchAttempt(dispatchDecision.issue);
|
|
1887
1970
|
const record = batch.start(dispatchDecision, dryRun);
|
|
@@ -1899,6 +1982,12 @@ export class FactoryLoop {
|
|
|
1899
1982
|
await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
|
|
1900
1983
|
const spawnedForReaperHandoff = [];
|
|
1901
1984
|
try {
|
|
1985
|
+
if (!dryRun) {
|
|
1986
|
+
const issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
1987
|
+
if (!issue || !this.#isIssueReady(issue)) {
|
|
1988
|
+
throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1902
1991
|
const specs = dispatchSpecs(dispatchDecision);
|
|
1903
1992
|
const agents = [];
|
|
1904
1993
|
for (const spec of specs) {
|
|
@@ -1960,8 +2049,10 @@ export class FactoryLoop {
|
|
|
1960
2049
|
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
1961
2050
|
const failureHandoffs = this.#dispatchFailureHandoffs(record, spawnedForReaperHandoff);
|
|
1962
2051
|
await this.#persistDispatchFailureReaperHandoff(record, failureHandoffs);
|
|
1963
|
-
let worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs);
|
|
1964
2052
|
const liveStateChanged = error instanceof LiveDispatchStateChangedError;
|
|
2053
|
+
const cancellationReason = factoryCloudDispatchCancellationReason(error);
|
|
2054
|
+
const cleanupReason = liveStateChanged ? 'live dispatch state changed' : 'dispatch failed';
|
|
2055
|
+
let worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs, cleanupReason);
|
|
1965
2056
|
if (liveStateChanged && !failureHandoffs.some((handoff) => handoff.worktree)) {
|
|
1966
2057
|
const failed = await this.#releaseAndTerminateAgents(failureHandoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion');
|
|
1967
2058
|
if (failed.length === 0) {
|
|
@@ -1974,13 +2065,13 @@ export class FactoryLoop {
|
|
|
1974
2065
|
let failedState;
|
|
1975
2066
|
if (liveStateChanged) {
|
|
1976
2067
|
await this.#clearDispatchInFlight(decision.issue);
|
|
1977
|
-
await this.#saveDispatchLifecycle(record, 'abandoned');
|
|
2068
|
+
await this.#saveDispatchLifecycle(record, 'abandoned', undefined, undefined, new Set(), { cancellationReason });
|
|
1978
2069
|
this.#increment('dispatchLiveStateRaces');
|
|
1979
2070
|
}
|
|
1980
2071
|
else {
|
|
1981
2072
|
await this.#recordDispatchFailure(decision.issue);
|
|
1982
2073
|
failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key);
|
|
1983
|
-
await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable');
|
|
2074
|
+
await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable', undefined, undefined, new Set(), { cancellationReason: failedState?.terminal ? cancellationReason : undefined });
|
|
1984
2075
|
}
|
|
1985
2076
|
batch.abandon(decision.issue);
|
|
1986
2077
|
if (!liveStateChanged && !failedState?.terminal)
|
|
@@ -2057,7 +2148,7 @@ export class FactoryLoop {
|
|
|
2057
2148
|
});
|
|
2058
2149
|
}
|
|
2059
2150
|
}
|
|
2060
|
-
//
|
|
2151
|
+
// Durable backends survive orchestrator restarts: re-adopt the agents recorded
|
|
2061
2152
|
// in the durable lifecycle store, restore their full batch/spec association,
|
|
2062
2153
|
// then reconcile once so exits that happened while this process was down are
|
|
2063
2154
|
// handled instead of being dropped as unknown agents.
|
|
@@ -2097,8 +2188,9 @@ export class FactoryLoop {
|
|
|
2097
2188
|
for (const agent of claim.lifecycle.agents) {
|
|
2098
2189
|
const invocationId = agent.tracked.spec.invocationId;
|
|
2099
2190
|
const node = agent.tracked.result?.node;
|
|
2100
|
-
if (invocationId || node)
|
|
2191
|
+
if (invocationId || node || agent.tracked.result) {
|
|
2101
2192
|
agents.push({ name: agent.name, invocationId, node });
|
|
2193
|
+
}
|
|
2102
2194
|
}
|
|
2103
2195
|
}
|
|
2104
2196
|
// Migration fallback for registries written before durable lifecycle
|
|
@@ -2136,6 +2228,12 @@ export class FactoryLoop {
|
|
|
2136
2228
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2137
2229
|
this.#increment('dispatchLifecycleLeasesLost');
|
|
2138
2230
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2231
|
+
if (lifecycle) {
|
|
2232
|
+
await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
|
|
2233
|
+
level: 'error',
|
|
2234
|
+
errorCode: 'lease_lost',
|
|
2235
|
+
});
|
|
2236
|
+
}
|
|
2139
2237
|
if (lifecycle && !isTerminalDispatchLifecycle(lifecycle) && lifecycle.phase !== 'waiting-for-human') {
|
|
2140
2238
|
this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(lifecycle));
|
|
2141
2239
|
}
|
|
@@ -2171,10 +2269,87 @@ export class FactoryLoop {
|
|
|
2171
2269
|
}
|
|
2172
2270
|
this.#dispatchLifecycleEpochs.set(key, claim.lease.epoch);
|
|
2173
2271
|
this.#scheduleDispatchLifecycleRenewal();
|
|
2272
|
+
if (claim.created) {
|
|
2273
|
+
await this.#reportLifecycle(claim.lifecycle, 'run.started');
|
|
2274
|
+
}
|
|
2174
2275
|
return { created: claim.created, lifecycle: claim.lifecycle };
|
|
2175
2276
|
}
|
|
2176
|
-
|
|
2177
|
-
|
|
2277
|
+
#usesDurableDispatchLifecycle() {
|
|
2278
|
+
return this.#fleet.durableOwnership ?? this.#fleet.placementLocality === 'remote';
|
|
2279
|
+
}
|
|
2280
|
+
async #report(input) {
|
|
2281
|
+
if (!this.#reporter)
|
|
2282
|
+
return;
|
|
2283
|
+
try {
|
|
2284
|
+
await this.#reporter.report(createFactoryCloudEventV1(input, {
|
|
2285
|
+
now: () => new Date(this.#clock.now()),
|
|
2286
|
+
}));
|
|
2287
|
+
}
|
|
2288
|
+
catch (error) {
|
|
2289
|
+
// A custom reporter is allowed through the public port, so defend the
|
|
2290
|
+
// orchestration path even if it violates the port's no-reject contract.
|
|
2291
|
+
this.#increment('factoryEventReportingFailures');
|
|
2292
|
+
this.#logger.warn?.('[factory] progress reporter rejected an event', {
|
|
2293
|
+
eventType: input.type,
|
|
2294
|
+
errorClass: telemetryErrorClass(error),
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
}
|
|
2298
|
+
async #reportLifecycle(lifecycle, type, options = {}) {
|
|
2299
|
+
await this.#report({
|
|
2300
|
+
type,
|
|
2301
|
+
level: options.level ?? (type === 'run.failed' || type === 'factory.anomaly' ? 'error' : 'info'),
|
|
2302
|
+
runId: lifecycle.runId,
|
|
2303
|
+
phase: lifecycle.phase,
|
|
2304
|
+
status: telemetryRunStatus(lifecycle.phase),
|
|
2305
|
+
run: {
|
|
2306
|
+
source: githubIssuePathParts(lifecycle.issue.path) ? 'github' : 'linear',
|
|
2307
|
+
repository: lifecycle.decision.routes[0]?.repo,
|
|
2308
|
+
issueKey: lifecycle.issue.key,
|
|
2309
|
+
recipe: lifecycle.decision.scope,
|
|
2310
|
+
},
|
|
2311
|
+
attributes: {
|
|
2312
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
2313
|
+
component: 'orchestrator',
|
|
2314
|
+
operation: 'save_lifecycle',
|
|
2315
|
+
previousPhase: options.previousPhase,
|
|
2316
|
+
errorCode: options.errorCode,
|
|
2317
|
+
cancellationReason: options.cancellationReason,
|
|
2318
|
+
dryRun: lifecycle.dryRun,
|
|
2319
|
+
trackedAgents: lifecycle.agents.length,
|
|
2320
|
+
},
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2323
|
+
async #reportAgent(record, tracked, type, options = {}) {
|
|
2324
|
+
const lifecycle = await this.#state
|
|
2325
|
+
.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
|
|
2326
|
+
.catch(() => undefined);
|
|
2327
|
+
if (!lifecycle)
|
|
2328
|
+
return;
|
|
2329
|
+
await this.#report({
|
|
2330
|
+
type,
|
|
2331
|
+
runId: lifecycle.runId,
|
|
2332
|
+
phase: lifecycle.phase,
|
|
2333
|
+
status: telemetryRunStatus(lifecycle.phase),
|
|
2334
|
+
run: {
|
|
2335
|
+
source: githubIssuePathParts(record.issue.path) ? 'github' : 'linear',
|
|
2336
|
+
repository: tracked.spec.repo,
|
|
2337
|
+
issueKey: record.issue.key,
|
|
2338
|
+
recipe: record.decision.scope,
|
|
2339
|
+
},
|
|
2340
|
+
attributes: {
|
|
2341
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
2342
|
+
component: 'fleet',
|
|
2343
|
+
operation: type.slice('agent.'.length),
|
|
2344
|
+
agentRole: tracked.spec.role,
|
|
2345
|
+
invocationId: tracked.spec.invocationId,
|
|
2346
|
+
locality: tracked.result?.locality ?? this.#fleet.placementLocality,
|
|
2347
|
+
releaseReason: factoryCloudReleaseReasonV1(options.releaseReason),
|
|
2348
|
+
},
|
|
2349
|
+
});
|
|
2350
|
+
}
|
|
2351
|
+
async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set(), telemetry = {}) {
|
|
2352
|
+
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
2178
2353
|
return true;
|
|
2179
2354
|
const key = issueKey(record.issue);
|
|
2180
2355
|
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
@@ -2195,9 +2370,20 @@ export class FactoryLoop {
|
|
|
2195
2370
|
if (!saved) {
|
|
2196
2371
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2197
2372
|
this.#increment('dispatchLifecycleFencesRejected');
|
|
2373
|
+
await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
|
|
2374
|
+
level: 'error',
|
|
2375
|
+
errorCode: 'fence_rejected',
|
|
2376
|
+
});
|
|
2198
2377
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
2199
2378
|
return false;
|
|
2200
2379
|
}
|
|
2380
|
+
if (previous?.phase !== lifecycle.phase) {
|
|
2381
|
+
await this.#reportLifecycle(lifecycle, lifecycle.phase === 'complete'
|
|
2382
|
+
? 'run.succeeded'
|
|
2383
|
+
: lifecycle.phase === 'abandoned'
|
|
2384
|
+
? 'run.cancelled'
|
|
2385
|
+
: 'run.phase_changed', { previousPhase: previous?.phase, cancellationReason: telemetry.cancellationReason });
|
|
2386
|
+
}
|
|
2201
2387
|
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
2202
2388
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2203
2389
|
}
|
|
@@ -2229,7 +2415,7 @@ export class FactoryLoop {
|
|
|
2229
2415
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
2230
2416
|
}
|
|
2231
2417
|
#scheduleReleaseRetry(record, reason) {
|
|
2232
|
-
if (this.#
|
|
2418
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
2233
2419
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
2234
2420
|
return;
|
|
2235
2421
|
}
|
|
@@ -2366,6 +2552,16 @@ export class FactoryLoop {
|
|
|
2366
2552
|
}
|
|
2367
2553
|
}
|
|
2368
2554
|
async #resumeDurableDispatch(record) {
|
|
2555
|
+
if (!record.dryRun) {
|
|
2556
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
2557
|
+
if (!issue) {
|
|
2558
|
+
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is not currently readable`);
|
|
2559
|
+
}
|
|
2560
|
+
if (isGithubIssue(issue) && !this.#isGithubIssueResumable(issue)) {
|
|
2561
|
+
await this.#abandonDurableResume(record, 'live GitHub issue is closed or no longer ready-for-agent');
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
}
|
|
2369
2565
|
const agents = [];
|
|
2370
2566
|
const specs = dispatchSpecs(record.decision);
|
|
2371
2567
|
const plannedNames = new Set(specs.map((spec) => spec.name));
|
|
@@ -2413,6 +2609,55 @@ export class FactoryLoop {
|
|
|
2413
2609
|
}
|
|
2414
2610
|
}
|
|
2415
2611
|
}
|
|
2612
|
+
#isGithubIssueResumable(issue) {
|
|
2613
|
+
if (this.#isIssueReady(issue))
|
|
2614
|
+
return true;
|
|
2615
|
+
if (githubFactoryIssueIsClosed(issue))
|
|
2616
|
+
return false;
|
|
2617
|
+
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
2618
|
+
const required = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
2619
|
+
return Boolean(required) &&
|
|
2620
|
+
labels.has(required) &&
|
|
2621
|
+
labels.has('factory:in-progress') &&
|
|
2622
|
+
!labels.has('factory:human-review');
|
|
2623
|
+
}
|
|
2624
|
+
async #abandonDurableResume(record, reason) {
|
|
2625
|
+
const handoffs = this.#dispatchFailureHandoffs(record, [...record.agents].map(([name, tracked]) => ({
|
|
2626
|
+
issue: record.issue,
|
|
2627
|
+
name,
|
|
2628
|
+
tracked: cloneTrackedAgent(tracked),
|
|
2629
|
+
persistedAtMs: this.#clock.now(),
|
|
2630
|
+
})));
|
|
2631
|
+
await this.#persistDispatchFailureReaperHandoff(record, handoffs);
|
|
2632
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'source_state_changed' }))
|
|
2633
|
+
return;
|
|
2634
|
+
await this.#clearDispatchInFlight(record.issue);
|
|
2635
|
+
const batch = await this.#batch();
|
|
2636
|
+
batch.abandon(record.issue);
|
|
2637
|
+
for (const [name] of record.agents) {
|
|
2638
|
+
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
2639
|
+
}
|
|
2640
|
+
if (handoffs.some((handoff) => handoff.worktree)) {
|
|
2641
|
+
await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed');
|
|
2642
|
+
}
|
|
2643
|
+
else if (handoffs.length > 0) {
|
|
2644
|
+
const failed = new Set(await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion'));
|
|
2645
|
+
for (const handoff of handoffs) {
|
|
2646
|
+
if (failed.has(handoff.name))
|
|
2647
|
+
continue;
|
|
2648
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
2649
|
+
}
|
|
2650
|
+
}
|
|
2651
|
+
await this.#stopSlackWatcher(record.issue);
|
|
2652
|
+
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
2653
|
+
await this.#writeInFlightRegistry();
|
|
2654
|
+
this.#increment('dispatchLifecycleStaleIssuesAbandoned');
|
|
2655
|
+
this.#resolveDispatchTerminalWaiters(record.issue);
|
|
2656
|
+
this.#logger.info?.('[factory] abandoned durable dispatch whose live issue is no longer ready', {
|
|
2657
|
+
issue: record.issue.key,
|
|
2658
|
+
reason,
|
|
2659
|
+
});
|
|
2660
|
+
}
|
|
2416
2661
|
async #finishDurableRelease(record, releaseReason) {
|
|
2417
2662
|
const batch = await this.#batch();
|
|
2418
2663
|
const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done');
|
|
@@ -2458,14 +2703,14 @@ export class FactoryLoop {
|
|
|
2458
2703
|
this.#scheduleReleaseRetry(record, reason);
|
|
2459
2704
|
return false;
|
|
2460
2705
|
}
|
|
2461
|
-
const next = this.#
|
|
2706
|
+
const next = this.#usesDurableDispatchLifecycle() ? undefined : batch.complete(record.issue);
|
|
2462
2707
|
this.#localReleaseCheckpoints.delete(releaseKey);
|
|
2463
2708
|
if (next)
|
|
2464
2709
|
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
2465
2710
|
// Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
|
|
2466
2711
|
// the babysitter's durable ownership/wake/critical state while that epoch
|
|
2467
2712
|
// is still valid so a later reopened issue cannot inherit a stale PR owner.
|
|
2468
|
-
if (this.#
|
|
2713
|
+
if (this.#usesDurableDispatchLifecycle() && this.#config.babysitter.enabled) {
|
|
2469
2714
|
await this.#cancelBabysitterWake(issueKey(record.issue));
|
|
2470
2715
|
}
|
|
2471
2716
|
if (!await this.#saveDispatchLifecycle(record, 'complete'))
|
|
@@ -2480,7 +2725,7 @@ export class FactoryLoop {
|
|
|
2480
2725
|
return await this.#assertIssueDispatchLifecycleOwner(record.issue);
|
|
2481
2726
|
}
|
|
2482
2727
|
async #assertIssueDispatchLifecycleOwner(issue) {
|
|
2483
|
-
if (this.#
|
|
2728
|
+
if (!this.#usesDurableDispatchLifecycle())
|
|
2484
2729
|
return true;
|
|
2485
2730
|
const key = issueKey(issue);
|
|
2486
2731
|
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
@@ -2580,8 +2825,7 @@ export class FactoryLoop {
|
|
|
2580
2825
|
if (!isGithubIssue(issue)) {
|
|
2581
2826
|
return this.#states.isRole(issue.stateId, 'readyForAgent');
|
|
2582
2827
|
}
|
|
2583
|
-
|
|
2584
|
-
if (githubState === 'closed') {
|
|
2828
|
+
if (githubFactoryIssueIsClosed(issue)) {
|
|
2585
2829
|
return false;
|
|
2586
2830
|
}
|
|
2587
2831
|
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
@@ -2706,9 +2950,14 @@ export class FactoryLoop {
|
|
|
2706
2950
|
}
|
|
2707
2951
|
}
|
|
2708
2952
|
}
|
|
2953
|
+
for (const [identity, path] of issuePaths) {
|
|
2954
|
+
this.#githubIssuePreferredPaths.set(identity, path);
|
|
2955
|
+
}
|
|
2956
|
+
this.#githubIssuePathIndexReady = true;
|
|
2709
2957
|
return [...issuePaths.values()].sort();
|
|
2710
2958
|
}
|
|
2711
2959
|
catch (error) {
|
|
2960
|
+
this.#githubIssuePathIndexReady = false;
|
|
2712
2961
|
this.#increment('githubIssueListFailures');
|
|
2713
2962
|
this.#logger.warn?.('[factory] failed to list GitHub issue source tree', error);
|
|
2714
2963
|
return [];
|
|
@@ -2773,7 +3022,11 @@ export class FactoryLoop {
|
|
|
2773
3022
|
}
|
|
2774
3023
|
}
|
|
2775
3024
|
async #readGithubIssue(path) {
|
|
2776
|
-
const
|
|
3025
|
+
const preferredPath = await this.#preferredGithubIssuePath(path);
|
|
3026
|
+
const candidatePaths = [...new Set([
|
|
3027
|
+
...githubIssueReadCandidatePaths(preferredPath),
|
|
3028
|
+
...githubIssueReadCandidatePaths(path),
|
|
3029
|
+
])];
|
|
2777
3030
|
try {
|
|
2778
3031
|
for (const candidatePath of candidatePaths) {
|
|
2779
3032
|
try {
|
|
@@ -2797,6 +3050,30 @@ export class FactoryLoop {
|
|
|
2797
3050
|
throw error;
|
|
2798
3051
|
}
|
|
2799
3052
|
}
|
|
3053
|
+
async #preferredGithubIssuePath(path) {
|
|
3054
|
+
const parts = githubIssuePathParts(path) ?? githubIssueDirectoryPathParts(path);
|
|
3055
|
+
if (!parts)
|
|
3056
|
+
return path;
|
|
3057
|
+
const identity = githubIssueIdentity(parts.owner, parts.repo, parts.number);
|
|
3058
|
+
const cached = this.#githubIssuePreferredPaths.get(identity);
|
|
3059
|
+
if (cached && githubIssuePathPreference(cached) <= githubIssuePathPreference(path))
|
|
3060
|
+
return cached;
|
|
3061
|
+
if (githubIssuePathPreference(path) === 0) {
|
|
3062
|
+
this.#githubIssuePreferredPaths.set(identity, path);
|
|
3063
|
+
return path;
|
|
3064
|
+
}
|
|
3065
|
+
// Normal discovery has already indexed every configured GitHub issue path.
|
|
3066
|
+
// A dispatch-only replacement owner can reach this method before that
|
|
3067
|
+
// backfill, so build the same shared index once instead of traversing both
|
|
3068
|
+
// repository trees separately for every durable issue it recovers.
|
|
3069
|
+
if (!this.#githubIssuePathIndexReady) {
|
|
3070
|
+
await this.#githubIssuePaths();
|
|
3071
|
+
}
|
|
3072
|
+
const indexed = this.#githubIssuePreferredPaths.get(identity);
|
|
3073
|
+
return indexed && githubIssuePathPreference(indexed) < githubIssuePathPreference(path)
|
|
3074
|
+
? indexed
|
|
3075
|
+
: path;
|
|
3076
|
+
}
|
|
2800
3077
|
async #findGithubIssueMirror(ghIssue, candidates) {
|
|
2801
3078
|
const draftPath = githubIssueMirrorDraftPath(ghIssue);
|
|
2802
3079
|
try {
|
|
@@ -2947,6 +3224,19 @@ export class FactoryLoop {
|
|
|
2947
3224
|
await mkdir(dirname(path), { recursive: true });
|
|
2948
3225
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
2949
3226
|
await this.#writeInFlightRegistry(registryPath, path);
|
|
3227
|
+
const batch = this.#batchView;
|
|
3228
|
+
await this.#report({
|
|
3229
|
+
type: 'instance.heartbeat',
|
|
3230
|
+
attributes: {
|
|
3231
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
3232
|
+
mode: status,
|
|
3233
|
+
component: 'orchestrator',
|
|
3234
|
+
operation: 'heartbeat',
|
|
3235
|
+
activeRuns: batch?.inFlight.length,
|
|
3236
|
+
queuedRuns: batch?.queued.length,
|
|
3237
|
+
trackedAgents: batch?.inFlight.reduce((count, record) => count + record.agents.size, 0),
|
|
3238
|
+
},
|
|
3239
|
+
});
|
|
2950
3240
|
}
|
|
2951
3241
|
async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
|
|
2952
3242
|
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
@@ -3150,7 +3440,9 @@ export class FactoryLoop {
|
|
|
3150
3440
|
async #releaseAndTerminateAgents(agents, reason, context) {
|
|
3151
3441
|
const failed = [];
|
|
3152
3442
|
const protectedPids = await this.#protectedPids();
|
|
3443
|
+
const batch = this.#batchView;
|
|
3153
3444
|
for (const [agentName, tracked] of agents) {
|
|
3445
|
+
const record = batch?.getIssueByAgent(agentName);
|
|
3154
3446
|
if (context === 'stop') {
|
|
3155
3447
|
await this.#refreshStoppingHeartbeat();
|
|
3156
3448
|
}
|
|
@@ -3182,10 +3474,23 @@ export class FactoryLoop {
|
|
|
3182
3474
|
}
|
|
3183
3475
|
try {
|
|
3184
3476
|
await this.#fleet.release(agentName, reason);
|
|
3477
|
+
if (record)
|
|
3478
|
+
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
3185
3479
|
}
|
|
3186
3480
|
catch (error) {
|
|
3187
3481
|
failed.push(agentName);
|
|
3188
3482
|
this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
|
|
3483
|
+
if (record) {
|
|
3484
|
+
const lifecycle = await this.#state
|
|
3485
|
+
.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
|
|
3486
|
+
.catch(() => undefined);
|
|
3487
|
+
if (lifecycle) {
|
|
3488
|
+
await this.#reportLifecycle(lifecycle, 'factory.failure', {
|
|
3489
|
+
level: 'error',
|
|
3490
|
+
errorCode: 'release_failed',
|
|
3491
|
+
});
|
|
3492
|
+
}
|
|
3493
|
+
}
|
|
3189
3494
|
}
|
|
3190
3495
|
if (context === 'stop') {
|
|
3191
3496
|
await this.#refreshStoppingHeartbeat();
|
|
@@ -3294,10 +3599,10 @@ export class FactoryLoop {
|
|
|
3294
3599
|
}
|
|
3295
3600
|
return [...handoffs.values()];
|
|
3296
3601
|
}
|
|
3297
|
-
async #teardownFailedDispatchWorktrees(handoffs) {
|
|
3602
|
+
async #teardownFailedDispatchWorktrees(handoffs, releaseReason = 'dispatch failed') {
|
|
3298
3603
|
if (!this.#worktrees || !handoffs.some((handoff) => handoff.worktree))
|
|
3299
3604
|
return false;
|
|
3300
|
-
const failed = await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]),
|
|
3605
|
+
const failed = await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), releaseReason, 'completion');
|
|
3301
3606
|
if (failed.length > 0)
|
|
3302
3607
|
return false;
|
|
3303
3608
|
try {
|
|
@@ -3423,6 +3728,9 @@ export class FactoryLoop {
|
|
|
3423
3728
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
3424
3729
|
throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`);
|
|
3425
3730
|
}
|
|
3731
|
+
const adopted = record.agents.get(spec.name);
|
|
3732
|
+
if (adopted)
|
|
3733
|
+
await this.#reportAgent(record, adopted, 'agent.adopted');
|
|
3426
3734
|
return { name: spec.name };
|
|
3427
3735
|
}
|
|
3428
3736
|
await this.#prepareAgentWorktree(record, spec);
|
|
@@ -3445,12 +3753,20 @@ export class FactoryLoop {
|
|
|
3445
3753
|
});
|
|
3446
3754
|
}
|
|
3447
3755
|
catch (error) {
|
|
3448
|
-
|
|
3756
|
+
const wrapped = contextualError(`Dispatch spawn failed for ${record.issue.key}/${spec.name} (${spec.capability}) cwd=${spec.clonePath ?? 'default'}`, error);
|
|
3757
|
+
throw Object.assign(wrapped, {
|
|
3758
|
+
factoryCancellationReason: isDispatchDeliveryError(error)
|
|
3759
|
+
? 'agent_delivery_failed'
|
|
3760
|
+
: 'agent_spawn_failed',
|
|
3761
|
+
});
|
|
3449
3762
|
}
|
|
3450
3763
|
batch.recordSpawn(record, spec, invocationId, result);
|
|
3451
3764
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
3452
3765
|
throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`);
|
|
3453
3766
|
}
|
|
3767
|
+
const spawned = record.agents.get(result.name);
|
|
3768
|
+
if (spawned)
|
|
3769
|
+
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
3454
3770
|
return { name: result.name };
|
|
3455
3771
|
}
|
|
3456
3772
|
async #handleAgentExit(name, reason) {
|
|
@@ -3469,6 +3785,19 @@ export class FactoryLoop {
|
|
|
3469
3785
|
const batch = await this.#batch();
|
|
3470
3786
|
const record = batch.getIssueByAgent(name);
|
|
3471
3787
|
if (!record) {
|
|
3788
|
+
if (/^ar-\d+-/u.test(name)) {
|
|
3789
|
+
await this.#report({
|
|
3790
|
+
type: 'factory.anomaly',
|
|
3791
|
+
level: 'error',
|
|
3792
|
+
attributes: {
|
|
3793
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
3794
|
+
component: 'fleet',
|
|
3795
|
+
operation: 'agent_exit',
|
|
3796
|
+
errorCode: 'unowned_agent',
|
|
3797
|
+
count: 1,
|
|
3798
|
+
},
|
|
3799
|
+
});
|
|
3800
|
+
}
|
|
3472
3801
|
return;
|
|
3473
3802
|
}
|
|
3474
3803
|
if (!await this.#assertDispatchLifecycleOwner(record)) {
|
|
@@ -3487,7 +3816,9 @@ export class FactoryLoop {
|
|
|
3487
3816
|
return;
|
|
3488
3817
|
}
|
|
3489
3818
|
const exiting = record.agents.get(name);
|
|
3490
|
-
if (
|
|
3819
|
+
if (exiting)
|
|
3820
|
+
await this.#reportAgent(record, exiting, 'agent.exited', { releaseReason: reason });
|
|
3821
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
3491
3822
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
3492
3823
|
if (lifecycle?.phase === 'parking') {
|
|
3493
3824
|
this.#increment('clarificationParkingExitsSuppressed');
|
|
@@ -4176,6 +4507,7 @@ export class FactoryLoop {
|
|
|
4176
4507
|
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
4177
4508
|
}
|
|
4178
4509
|
}
|
|
4510
|
+
await this.#reportAgent(record, tracked, 'agent.resumed');
|
|
4179
4511
|
}
|
|
4180
4512
|
async #handleDeliveryFailed(info) {
|
|
4181
4513
|
const critical = await this.#state.consumeCritical(this.#workspaceId, info.msgId ?? '');
|
|
@@ -4185,7 +4517,7 @@ export class FactoryLoop {
|
|
|
4185
4517
|
}
|
|
4186
4518
|
const record = (await this.#batch()).getIssueByAgent(info.to);
|
|
4187
4519
|
const issue = critical.issue ?? record?.issue;
|
|
4188
|
-
const error = new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`);
|
|
4520
|
+
const error = Object.assign(new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`), { code: 'fleet_delivery_failed' });
|
|
4189
4521
|
this.#error(error, issue);
|
|
4190
4522
|
if (isTerminalDeliveryFailure(info.reason)) {
|
|
4191
4523
|
this.#increment('criticalDeliveryTerminalFailures');
|
|
@@ -5661,7 +5993,7 @@ export class FactoryLoop {
|
|
|
5661
5993
|
}
|
|
5662
5994
|
async #cancelBabysitterWake(issueIdentity) {
|
|
5663
5995
|
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
5664
|
-
const mayClearDurable = this.#
|
|
5996
|
+
const mayClearDurable = !this.#usesDurableDispatchLifecycle()
|
|
5665
5997
|
|| Boolean(issue && await this.#assertIssueDispatchLifecycleOwner(issue));
|
|
5666
5998
|
for (const [key, state] of this.#babysitterWakeStates) {
|
|
5667
5999
|
if (issueKey(state.issue) !== issueIdentity)
|
|
@@ -5922,19 +6254,7 @@ export class FactoryLoop {
|
|
|
5922
6254
|
},
|
|
5923
6255
|
};
|
|
5924
6256
|
let targets;
|
|
5925
|
-
if (this.#fleet.
|
|
5926
|
-
if (!this.#fleet.sendInput) {
|
|
5927
|
-
throw new Error('Fleet client advertises PTY prompt delivery without raw input support');
|
|
5928
|
-
}
|
|
5929
|
-
// Internal agents are harness-owned PTYs, not Relaycast identities.
|
|
5930
|
-
// Stage the validated metadata-only wake directly through the broker so
|
|
5931
|
-
// a missing Relaycast registration cannot strand a live babysitter.
|
|
5932
|
-
// The CR remains a separate write: a critical-section begin can arrive
|
|
5933
|
-
// while this write is in flight and must be able to defer submission.
|
|
5934
|
-
await this.#fleet.sendInput(input.to, input.text);
|
|
5935
|
-
targets = [input.to];
|
|
5936
|
-
}
|
|
5937
|
-
else if (!this.#fleet.waitForInjected) {
|
|
6257
|
+
if (!this.#fleet.waitForInjected) {
|
|
5938
6258
|
await this.#fleet.sendMessage(input);
|
|
5939
6259
|
if (this.#stopping || state.cancelled) {
|
|
5940
6260
|
state.deliveringKinds = undefined;
|
|
@@ -6403,7 +6723,12 @@ export class FactoryLoop {
|
|
|
6403
6723
|
prNumber: prRef.prNumber,
|
|
6404
6724
|
babysitter: spawned.name,
|
|
6405
6725
|
});
|
|
6406
|
-
|
|
6726
|
+
// Internal PTY spawns receive `task` atomically in spawnPty. Re-sending
|
|
6727
|
+
// the same task through Relaycast before the worker has registered its
|
|
6728
|
+
// messaging identity can fail an otherwise successful spawn and erase
|
|
6729
|
+
// valid babysitter ownership. Remote placement still needs the explicit,
|
|
6730
|
+
// confirmed follow-up injection used by its spawn protocol.
|
|
6731
|
+
if (this.#fleet.waitForInjected && this.#fleet.placementLocality !== 'local') {
|
|
6407
6732
|
const input = {
|
|
6408
6733
|
to: tracked?.result?.name ?? spawned.name,
|
|
6409
6734
|
text: task,
|
|
@@ -6637,7 +6962,7 @@ export class FactoryLoop {
|
|
|
6637
6962
|
}
|
|
6638
6963
|
const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
|
|
6639
6964
|
releaseReasonForRetry = releaseReason;
|
|
6640
|
-
if (this.#
|
|
6965
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
6641
6966
|
// Durable capacity is released as soon as terminal writeback is
|
|
6642
6967
|
// acknowledged. Agent cleanup remains fenced/retryable in `releasing`.
|
|
6643
6968
|
const batch = await this.#batch();
|
|
@@ -6667,7 +6992,7 @@ export class FactoryLoop {
|
|
|
6667
6992
|
this.#babysitterPr.delete(completionKey);
|
|
6668
6993
|
await this.#cancelBabysitterWake(completionKey);
|
|
6669
6994
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
|
|
6670
|
-
if (this.#
|
|
6995
|
+
if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
|
|
6671
6996
|
for (const publishedKey of this.#publishedPullRequests.keys()) {
|
|
6672
6997
|
if (publishedKey.startsWith(`${completionKey}:`))
|
|
6673
6998
|
this.#publishedPullRequests.delete(publishedKey);
|
|
@@ -6677,7 +7002,16 @@ export class FactoryLoop {
|
|
|
6677
7002
|
}
|
|
6678
7003
|
#emit(event, payload) {
|
|
6679
7004
|
for (const listener of this.#listeners.get(event) ?? []) {
|
|
6680
|
-
|
|
7005
|
+
try {
|
|
7006
|
+
listener(payload);
|
|
7007
|
+
}
|
|
7008
|
+
catch (error) {
|
|
7009
|
+
this.#increment('factoryEventListenerFailures');
|
|
7010
|
+
this.#logger.warn?.('[factory] event listener failed', {
|
|
7011
|
+
event,
|
|
7012
|
+
errorClass: error instanceof Error ? error.name : 'Error',
|
|
7013
|
+
});
|
|
7014
|
+
}
|
|
6681
7015
|
}
|
|
6682
7016
|
}
|
|
6683
7017
|
#error(error, issue) {
|
|
@@ -6692,6 +7026,48 @@ export class FactoryLoop {
|
|
|
6692
7026
|
...details,
|
|
6693
7027
|
...(issue ? { issue: issue.key } : {}),
|
|
6694
7028
|
});
|
|
7029
|
+
const failureCode = telemetryCategory(error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
|
|
7030
|
+
? error.code
|
|
7031
|
+
: 'factory_error');
|
|
7032
|
+
const failureClass = telemetryErrorClass(error);
|
|
7033
|
+
void (async () => {
|
|
7034
|
+
try {
|
|
7035
|
+
const lifecycle = issue
|
|
7036
|
+
? await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(issue)).catch(() => undefined)
|
|
7037
|
+
: undefined;
|
|
7038
|
+
if (lifecycle) {
|
|
7039
|
+
await this.#reportLifecycle(lifecycle, 'factory.failure', {
|
|
7040
|
+
level: 'error',
|
|
7041
|
+
errorCode: failureCode,
|
|
7042
|
+
});
|
|
7043
|
+
return;
|
|
7044
|
+
}
|
|
7045
|
+
await this.#report({
|
|
7046
|
+
type: 'factory.failure',
|
|
7047
|
+
level: 'error',
|
|
7048
|
+
attributes: {
|
|
7049
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
7050
|
+
component: 'orchestrator',
|
|
7051
|
+
operation: 'error',
|
|
7052
|
+
errorClass: failureClass,
|
|
7053
|
+
errorCode: failureCode,
|
|
7054
|
+
},
|
|
7055
|
+
});
|
|
7056
|
+
}
|
|
7057
|
+
catch (telemetryError) {
|
|
7058
|
+
// This is the terminal guard for the intentionally floating telemetry
|
|
7059
|
+
// task. Never log raw messages or allow a custom logger to turn this
|
|
7060
|
+
// best-effort path into an unhandled rejection.
|
|
7061
|
+
try {
|
|
7062
|
+
this.#logger.warn?.('[factory] failed to report failure telemetry', {
|
|
7063
|
+
errorClass: telemetryErrorClass(telemetryError),
|
|
7064
|
+
});
|
|
7065
|
+
}
|
|
7066
|
+
catch {
|
|
7067
|
+
// Reporting and logging are both non-critical to orchestration.
|
|
7068
|
+
}
|
|
7069
|
+
}
|
|
7070
|
+
})();
|
|
6695
7071
|
this.#emit('error', { error, ...details, issue });
|
|
6696
7072
|
}
|
|
6697
7073
|
#surfaceEscalationDeliveryFailure(kind, issue, correlationId, reason, cause) {
|
|
@@ -7755,7 +8131,7 @@ export class FactoryLoop {
|
|
|
7755
8131
|
}
|
|
7756
8132
|
let lifecycleDecision = waiting.decision;
|
|
7757
8133
|
let promotedLifecycle;
|
|
7758
|
-
if (!waiting.dryRun && this.#
|
|
8134
|
+
if (!waiting.dryRun && this.#usesDurableDispatchLifecycle()) {
|
|
7759
8135
|
try {
|
|
7760
8136
|
const claim = await this.#claimDispatchLifecycle(waiting.decision, false);
|
|
7761
8137
|
const lifecycleKey = issueKey(waiting.issue);
|
|
@@ -8444,6 +8820,9 @@ export function isDispatchableIssue(issue) {
|
|
|
8444
8820
|
if (!isGithubIssue(issue)) {
|
|
8445
8821
|
return false;
|
|
8446
8822
|
}
|
|
8823
|
+
if (githubFactoryIssueIsClosed(issue)) {
|
|
8824
|
+
return false;
|
|
8825
|
+
}
|
|
8447
8826
|
const payload = wrappedPayload(issue.raw);
|
|
8448
8827
|
const source = asRecord(payload.source);
|
|
8449
8828
|
const sourceId = stringValue(source?.id);
|
|
@@ -8458,6 +8837,7 @@ const isGithubIssue = (issue) => {
|
|
|
8458
8837
|
return stringValue(source?.provider)?.toLowerCase() === 'github' &&
|
|
8459
8838
|
(stringValue(wrapper?.provider)?.toLowerCase() === 'github' || isGithubIssueFilePath(issue.path));
|
|
8460
8839
|
};
|
|
8840
|
+
const githubFactoryIssueIsClosed = (issue) => (issue.state?.name ?? stringValue(wrappedPayload(issue.raw).state) ?? '').trim().toLowerCase() === 'closed';
|
|
8461
8841
|
const githubIssueSourceRef = (issue) => {
|
|
8462
8842
|
const source = asRecord(wrappedPayload(issue.raw).source);
|
|
8463
8843
|
if (stringValue(source?.provider)?.toLowerCase() !== 'github') {
|
|
@@ -8906,13 +9286,15 @@ const githubIssueRefIdentity = (issue) => {
|
|
|
8906
9286
|
return parts ? githubIssueIdentity(parts.owner, parts.repo, parts.number) : undefined;
|
|
8907
9287
|
};
|
|
8908
9288
|
const githubIssuePathPreference = (path) => {
|
|
8909
|
-
if (path.includes('/issues/by-id/'))
|
|
8910
|
-
return 0;
|
|
8911
9289
|
if (path.endsWith('/meta.json'))
|
|
9290
|
+
return 0;
|
|
9291
|
+
if (path.endsWith('/metadata.json'))
|
|
8912
9292
|
return 1;
|
|
8913
|
-
if (path.
|
|
9293
|
+
if (path.includes('/issues/by-id/'))
|
|
8914
9294
|
return 2;
|
|
8915
|
-
|
|
9295
|
+
if (path.endsWith('.json'))
|
|
9296
|
+
return 3;
|
|
9297
|
+
return 4;
|
|
8916
9298
|
};
|
|
8917
9299
|
const githubAgentNameMatchesIssue = (name, issue) => {
|
|
8918
9300
|
const parts = githubIssuePathParts(issue.path);
|
|
@@ -9228,11 +9610,15 @@ export const githubRepoSubscriptionGlobs = (config) => configuredGithubRepoParts
|
|
|
9228
9610
|
const githubIssueScanRoots = (config) => {
|
|
9229
9611
|
const roots = new Set();
|
|
9230
9612
|
for (const { owner, repo } of configuredGithubRepoParts(config)) {
|
|
9231
|
-
|
|
9232
|
-
|
|
9613
|
+
for (const root of githubIssueRepoRoots(owner, repo))
|
|
9614
|
+
roots.add(root);
|
|
9233
9615
|
}
|
|
9234
9616
|
return [...roots];
|
|
9235
9617
|
};
|
|
9618
|
+
const githubIssueRepoRoots = (owner, repo) => [
|
|
9619
|
+
`${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues`,
|
|
9620
|
+
`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`,
|
|
9621
|
+
];
|
|
9236
9622
|
const githubRepoPathParts = (path) => {
|
|
9237
9623
|
const compactSegment = path.match(/^\/github\/repos\/([^/]+)\//u)?.[1];
|
|
9238
9624
|
const separator = compactSegment?.indexOf('__') ?? -1;
|
|
@@ -9981,7 +10367,13 @@ const isAgentAlreadyExistsError = (error) => {
|
|
|
9981
10367
|
const defaultRestartPolicy = (spec) =>
|
|
9982
10368
|
// Factory owns durable resume/respawn decisions. Broker-level retries race
|
|
9983
10369
|
// that lifecycle and can re-register the same name before Factory resumes it.
|
|
9984
|
-
|
|
10370
|
+
// The reviewer shares the implementer's dispatch lifecycle: it is spawned in
|
|
10371
|
+
// the same batch, torn down by the same dispatch-failure/teardown paths, and
|
|
10372
|
+
// resumed through the same durable #resumeDurableDispatch flow. Without this
|
|
10373
|
+
// opt-out the broker's default restart policy re-registers a torn-down
|
|
10374
|
+
// reviewer's name as an orphan (relay#1116-family) while the dashboard only
|
|
10375
|
+
// reports dispatch_failed — the exact orphan/restart sequence being audited.
|
|
10376
|
+
spec.role === 'implementer' || spec.role === 'reviewer' || spec.role === 'babysitter'
|
|
9985
10377
|
? { maxRestarts: 0 }
|
|
9986
10378
|
: spec.restartPolicy;
|
|
9987
10379
|
const slackPayloadTs = (threadId) => threadId.replace(/_/g, '.');
|
|
@@ -10046,9 +10438,52 @@ const failedIterationReport = (error, dryRun) => {
|
|
|
10046
10438
|
};
|
|
10047
10439
|
const isRegistrationLagInjectionError = (error) => {
|
|
10048
10440
|
const { errorMessage } = describeError(error);
|
|
10049
|
-
return /recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
|
|
10441
|
+
return /agent_not_found|recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
|
|
10050
10442
|
.test(errorMessage);
|
|
10051
10443
|
};
|
|
10444
|
+
const isDispatchDeliveryError = (error) => {
|
|
10445
|
+
if (isRegistrationLagInjectionError(error))
|
|
10446
|
+
return true;
|
|
10447
|
+
const { errorMessage } = describeError(error);
|
|
10448
|
+
return /delivery[_ -]failed|dead-lettered|max delivery retries exceeded/iu.test(errorMessage);
|
|
10449
|
+
};
|
|
10450
|
+
const factoryCloudDispatchCancellationReason = (error) => {
|
|
10451
|
+
if (error instanceof LiveDispatchStateChangedError)
|
|
10452
|
+
return 'source_state_changed';
|
|
10453
|
+
if (error && typeof error === 'object' && 'factoryCancellationReason' in error) {
|
|
10454
|
+
const reason = error.factoryCancellationReason;
|
|
10455
|
+
if (reason === 'agent_spawn_failed' || reason === 'agent_delivery_failed')
|
|
10456
|
+
return reason;
|
|
10457
|
+
}
|
|
10458
|
+
return 'dispatch_failed';
|
|
10459
|
+
};
|
|
10460
|
+
const telemetryRunStatus = (phase) => {
|
|
10461
|
+
switch (phase) {
|
|
10462
|
+
case 'queued':
|
|
10463
|
+
return 'queued';
|
|
10464
|
+
case 'retryable':
|
|
10465
|
+
return 'blocked';
|
|
10466
|
+
case 'parking':
|
|
10467
|
+
case 'waiting-for-human':
|
|
10468
|
+
return 'waiting';
|
|
10469
|
+
case 'complete':
|
|
10470
|
+
return 'succeeded';
|
|
10471
|
+
case 'abandoned':
|
|
10472
|
+
return 'cancelled';
|
|
10473
|
+
default:
|
|
10474
|
+
return 'running';
|
|
10475
|
+
}
|
|
10476
|
+
};
|
|
10477
|
+
const telemetryCategory = (value) => {
|
|
10478
|
+
if (!value)
|
|
10479
|
+
return undefined;
|
|
10480
|
+
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
|
|
10481
|
+
return normalized.slice(0, 120) || undefined;
|
|
10482
|
+
};
|
|
10483
|
+
const telemetryErrorClass = (error) => {
|
|
10484
|
+
const name = error instanceof Error ? error.name : '';
|
|
10485
|
+
return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
|
|
10486
|
+
};
|
|
10052
10487
|
const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
|
|
10053
10488
|
const retryOnTimeout = async (fn, opts) => {
|
|
10054
10489
|
let lastError;
|