@agent-relay/factory 0.1.32 → 0.1.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +36 -0
  2. package/dist/cli/fleet.d.ts +4 -1
  3. package/dist/cli/fleet.d.ts.map +1 -1
  4. package/dist/cli/fleet.js +115 -6
  5. package/dist/cli/fleet.js.map +1 -1
  6. package/dist/config/schema.d.ts +103 -8
  7. package/dist/config/schema.d.ts.map +1 -1
  8. package/dist/config/schema.js +12 -0
  9. package/dist/config/schema.js.map +1 -1
  10. package/dist/fleet/internal-fleet-client.d.ts +9 -2
  11. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  12. package/dist/fleet/internal-fleet-client.js +39 -1
  13. package/dist/fleet/internal-fleet-client.js.map +1 -1
  14. package/dist/fleet/relay-fleet-client.d.ts +1 -0
  15. package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
  16. package/dist/fleet/relay-fleet-client.js +1 -0
  17. package/dist/fleet/relay-fleet-client.js.map +1 -1
  18. package/dist/index.d.ts +7 -1
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +3 -0
  21. package/dist/index.js.map +1 -1
  22. package/dist/observability/cloud-reporter.d.ts +46 -0
  23. package/dist/observability/cloud-reporter.d.ts.map +1 -0
  24. package/dist/observability/cloud-reporter.js +371 -0
  25. package/dist/observability/cloud-reporter.js.map +1 -0
  26. package/dist/observability/events.d.ts +1315 -0
  27. package/dist/observability/events.d.ts.map +1 -0
  28. package/dist/observability/events.js +242 -0
  29. package/dist/observability/events.js.map +1 -0
  30. package/dist/observability/index.d.ts +5 -0
  31. package/dist/observability/index.d.ts.map +1 -0
  32. package/dist/observability/index.js +5 -0
  33. package/dist/observability/index.js.map +1 -0
  34. package/dist/observability/instance-identity.d.ts +21 -0
  35. package/dist/observability/instance-identity.d.ts.map +1 -0
  36. package/dist/observability/instance-identity.js +77 -0
  37. package/dist/observability/instance-identity.js.map +1 -0
  38. package/dist/observability/outbox.d.ts +42 -0
  39. package/dist/observability/outbox.d.ts.map +1 -0
  40. package/dist/observability/outbox.js +305 -0
  41. package/dist/observability/outbox.js.map +1 -0
  42. package/dist/orchestrator/factory.d.ts.map +1 -1
  43. package/dist/orchestrator/factory.js +495 -60
  44. package/dist/orchestrator/factory.js.map +1 -1
  45. package/dist/ports/fleet.d.ts +3 -4
  46. package/dist/ports/fleet.d.ts.map +1 -1
  47. package/dist/ports/index.d.ts +1 -0
  48. package/dist/ports/index.d.ts.map +1 -1
  49. package/dist/ports/observability.d.ts +19 -0
  50. package/dist/ports/observability.d.ts.map +1 -0
  51. package/dist/ports/observability.js +2 -0
  52. package/dist/ports/observability.js.map +1 -0
  53. package/dist/triage/schema.d.ts +14 -14
  54. package/dist/types.d.ts +3 -0
  55. package/dist/types.d.ts.map +1 -1
  56. package/package.json +5 -1
@@ -20,6 +20,7 @@ import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../write
20
20
  import { issueKey } from './batch-tracker.js';
21
21
  import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
22
22
  import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
23
+ import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
23
24
  class ClarificationWakeLeaseLostError extends Error {
24
25
  }
25
26
  class ClarificationQuestionDeliveryLeaseLostError extends Error {
@@ -119,6 +120,7 @@ export class FactoryLoop {
119
120
  #workspaceId;
120
121
  #relayflows;
121
122
  #worktrees;
123
+ #reporter;
122
124
  #batchView;
123
125
  #batchReady;
124
126
  #listeners = new Map();
@@ -132,6 +134,8 @@ export class FactoryLoop {
132
134
  #githubIssueCommentQueues = new Map();
133
135
  #githubIssueAuthors = new Map();
134
136
  #githubIssueAuthorLookups = new Map();
137
+ #githubIssuePreferredPaths = new Map();
138
+ #githubIssuePathIndexReady = false;
135
139
  #slackReporterUserIds = new Map();
136
140
  #slackReporterUserIdLookups = new Map();
137
141
  #reconciledGithubInProgress = new Set();
@@ -263,6 +267,7 @@ export class FactoryLoop {
263
267
  this.#workspaceId = config.workspaceId ?? 'default';
264
268
  this.#relayflows = ports.relayflows;
265
269
  this.#worktrees = ports.worktrees;
270
+ this.#reporter = ports.reporter;
266
271
  this.#state = ports.stateStore ?? new InMemoryStateStore({
267
272
  batchSize: config.batchSize,
268
273
  agentQuestionDedupeLimit: AGENT_QUESTION_DEDUPE_LIMIT,
@@ -1362,6 +1367,7 @@ export class FactoryLoop {
1362
1367
  ]);
1363
1368
  const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
1364
1369
  const activeIssueIdentities = new Set();
1370
+ const legacyUnownedAgentsByIssue = new Map();
1365
1371
  for (const [, lifecycle] of lifecycles) {
1366
1372
  if (isTerminalDispatchLifecycle(lifecycle))
1367
1373
  continue;
@@ -1378,12 +1384,26 @@ export class FactoryLoop {
1378
1384
  if (!onlineAgents.has(agent.name) || !agent.issue)
1379
1385
  continue;
1380
1386
  const identity = githubIssueRefIdentity(agent.issue);
1381
- if (identity)
1387
+ if (!identity)
1388
+ continue;
1389
+ const isLegacyLocalWorker = this.#usesDurableDispatchLifecycle() &&
1390
+ this.#fleet.placementLocality === 'local' &&
1391
+ !agent.node &&
1392
+ !agent.invocationId &&
1393
+ !activeIssueIdentities.has(identity);
1394
+ if (isLegacyLocalWorker) {
1395
+ const agents = legacyUnownedAgentsByIssue.get(identity) ?? [];
1396
+ agents.push(agent);
1397
+ legacyUnownedAgentsByIssue.set(identity, agents);
1398
+ }
1399
+ else {
1382
1400
  activeIssueIdentities.add(identity);
1401
+ }
1383
1402
  }
1384
1403
  return {
1385
1404
  activeIssueIdentities,
1386
1405
  onlineAgentNames: onlineAgents,
1406
+ legacyUnownedAgentsByIssue,
1387
1407
  };
1388
1408
  }
1389
1409
  catch (error) {
@@ -1405,9 +1425,14 @@ export class FactoryLoop {
1405
1425
  labels.has('factory:human-review'))
1406
1426
  return false;
1407
1427
  const identity = githubIssueRefIdentity(issueRef(issue));
1428
+ const legacyUnownedAgents = identity
1429
+ ? (context.legacyUnownedAgentsByIssue.get(identity) ?? [])
1430
+ .filter((agent) => githubAgentNameMatchesIssue(agent.name, issue))
1431
+ : [];
1432
+ const legacyUnownedAgentNames = new Set(legacyUnownedAgents.map((agent) => agent.name));
1408
1433
  if (!identity ||
1409
1434
  context.activeIssueIdentities.has(identity) ||
1410
- [...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue))) {
1435
+ [...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue) && !legacyUnownedAgentNames.has(name))) {
1411
1436
  this.#increment('githubOrphanRecoveriesBlockedActive');
1412
1437
  return false;
1413
1438
  }
@@ -1447,7 +1472,7 @@ export class FactoryLoop {
1447
1472
  if (openPr) {
1448
1473
  let adopted = false;
1449
1474
  try {
1450
- adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr);
1475
+ adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr, legacyUnownedAgents);
1451
1476
  }
1452
1477
  catch (error) {
1453
1478
  this.#increment('githubOrphanedPullRequestAdoptionFailures');
@@ -1468,6 +1493,14 @@ export class FactoryLoop {
1468
1493
  });
1469
1494
  return false;
1470
1495
  }
1496
+ // A pre-durable local Factory may have left live, registry-proven workers
1497
+ // without a lifecycle record. They are safe to adopt only once their open
1498
+ // PR proves which dispatch they own. Without that proof, preserve the issue
1499
+ // and workers instead of redispatching duplicate agents.
1500
+ if (legacyUnownedAgents.length > 0) {
1501
+ this.#increment('githubOrphanRecoveriesBlockedActive');
1502
+ return false;
1503
+ }
1471
1504
  try {
1472
1505
  if (providerStatus === 'in-progress') {
1473
1506
  await this.#githubWriteback.setStatus(issue, 'ready');
@@ -1504,7 +1537,7 @@ export class FactoryLoop {
1504
1537
  allowLegacyGithubBranch: true,
1505
1538
  });
1506
1539
  }
1507
- async #adoptOrphanedGithubPullRequest(issue, pr) {
1540
+ async #adoptOrphanedGithubPullRequest(issue, pr, legacyUnownedAgents = []) {
1508
1541
  const headRef = pr.headRef;
1509
1542
  if (!headRef)
1510
1543
  return false;
@@ -1543,14 +1576,14 @@ export class FactoryLoop {
1543
1576
  : spec),
1544
1577
  };
1545
1578
  }
1546
- const durableRemoteAdoption = this.#fleet.placementLocality === 'remote';
1579
+ const durableAdoption = this.#usesDurableDispatchLifecycle();
1547
1580
  const publishedPr = {
1548
1581
  repo: pr.repo,
1549
1582
  number: pr.prNumber,
1550
1583
  url: pr.url ?? `https://github.com/${pr.repo}/pull/${pr.prNumber}`,
1551
1584
  headRef,
1552
1585
  };
1553
- if (durableRemoteAdoption) {
1586
+ if (durableAdoption) {
1554
1587
  const claim = await this.#claimDispatchLifecycle(decision, false, randomUUID(), {
1555
1588
  phase: 'published',
1556
1589
  pullRequest: publishedPr,
@@ -1566,7 +1599,7 @@ export class FactoryLoop {
1566
1599
  const batch = await this.#batch();
1567
1600
  const record = batch.start(decision, false);
1568
1601
  if (!record) {
1569
- if (durableRemoteAdoption) {
1602
+ if (durableAdoption) {
1570
1603
  const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
1571
1604
  if (durable) {
1572
1605
  this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(durable));
@@ -1576,6 +1609,48 @@ export class FactoryLoop {
1576
1609
  }
1577
1610
  return false;
1578
1611
  }
1612
+ if (durableAdoption && legacyUnownedAgents.length > 0) {
1613
+ const specsByName = new Map(dispatchSpecs(record.decision).map((spec) => [spec.name, spec]));
1614
+ const initialBabysitter = babysitterSpec(issue, this.#config, route);
1615
+ const sharedCheckout = record.decision.implementers.find((candidate) => candidate.repo === initialBabysitter.repo && candidate.baseClonePath && candidate.clonePath);
1616
+ const legacyBabysitter = {
1617
+ ...initialBabysitter,
1618
+ ...(sharedCheckout
1619
+ ? {
1620
+ baseClonePath: sharedCheckout.baseClonePath,
1621
+ clonePath: sharedCheckout.clonePath,
1622
+ ...(headRef ? { branch: headRef } : {}),
1623
+ ...(sharedCheckout.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
1624
+ }
1625
+ : {}),
1626
+ ownedPullRequest: { repo: pr.repo, number: pr.prNumber, path: pr.path },
1627
+ };
1628
+ specsByName.set(legacyBabysitter.name, legacyBabysitter);
1629
+ const adopted = [];
1630
+ for (const agent of legacyUnownedAgents) {
1631
+ const spec = specsByName.get(agent.name);
1632
+ if (!spec)
1633
+ continue;
1634
+ const invocationId = batch.invocationIdFor(record.issue, spec);
1635
+ batch.recordSpawn(record, spec, invocationId, {
1636
+ name: agent.name,
1637
+ sessionRef: agent.sessionRef,
1638
+ pids: agent.pids,
1639
+ locality: 'local',
1640
+ });
1641
+ adopted.push({ name: agent.name, invocationId });
1642
+ }
1643
+ if (adopted.length > 0) {
1644
+ this.#fleet.hydrateTracked?.(adopted);
1645
+ await this.#saveDispatchLifecycle(record, 'published', publishedPr);
1646
+ for (const agent of adopted) {
1647
+ this.#increment('legacyLocalWorkersAdopted');
1648
+ const tracked = record.agents.get(agent.name);
1649
+ if (tracked)
1650
+ await this.#reportAgent(record, tracked, 'agent.adopted');
1651
+ }
1652
+ }
1653
+ }
1579
1654
  await this.#ensureBabysitter(record, {
1580
1655
  repo: pr.repo,
1581
1656
  prNumber: pr.prNumber,
@@ -1585,7 +1660,7 @@ export class FactoryLoop {
1585
1660
  });
1586
1661
  const babysitter = [...record.agents.values()].find((tracked) => tracked.spec.role === 'babysitter');
1587
1662
  if (!babysitter) {
1588
- if (durableRemoteAdoption)
1663
+ if (durableAdoption)
1589
1664
  this.#scheduleDispatchLifecycleRetry(record);
1590
1665
  else
1591
1666
  batch.abandon(record.issue);
@@ -1593,10 +1668,15 @@ export class FactoryLoop {
1593
1668
  }
1594
1669
  record.result = {
1595
1670
  issue: record.issue,
1596
- agents: [{ name: babysitter.result?.name ?? babysitter.spec.name, role: 'babysitter' }],
1671
+ agents: [...record.agents.values()].map((tracked) => ({
1672
+ name: tracked.result?.name ?? tracked.spec.name,
1673
+ role: tracked.spec.role,
1674
+ })),
1597
1675
  dryRun: false,
1598
1676
  };
1599
1677
  await this.#writeInFlightRegistry();
1678
+ if (durableAdoption)
1679
+ await this.#saveDispatchLifecycle(record, 'running', publishedPr);
1600
1680
  this.#increment('githubOrphanedPullRequestsAdopted');
1601
1681
  return true;
1602
1682
  }
@@ -1778,7 +1858,7 @@ export class FactoryLoop {
1778
1858
  if (existingRecord?.result) {
1779
1859
  return existingRecord.result;
1780
1860
  }
1781
- if (!dryRun && this.#fleet.placementLocality === 'remote') {
1861
+ if (!dryRun && this.#usesDurableDispatchLifecycle()) {
1782
1862
  const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
1783
1863
  if (durable && !isTerminalDispatchLifecycle(durable)) {
1784
1864
  if (durable.result && this.#dispatchLifecycleEpochs.has(issueKey(decision.issue))) {
@@ -1803,6 +1883,9 @@ export class FactoryLoop {
1803
1883
  this.#error(error, decision.issue);
1804
1884
  throw error;
1805
1885
  }
1886
+ if (!this.#isIssueReady(liveIssue)) {
1887
+ throw new LiveDispatchStateChangedError(decision.issue.key);
1888
+ }
1806
1889
  if (!isDispatchableIssue(liveIssue)) {
1807
1890
  const error = new Error(`Refusing to dispatch ${decision.issue.key}: not reconciled real Linear issue`);
1808
1891
  this.#error(error, decision.issue);
@@ -1849,19 +1932,19 @@ export class FactoryLoop {
1849
1932
  // Full task rendering is part of the durable spawn specification. It must
1850
1933
  // happen before a remote lifecycle is first claimed so takeover cannot
1851
1934
  // recover a persisted minimal triage task after a crash in this gap.
1852
- const durableRemoteDispatch = !dryRun && this.#fleet.placementLocality === 'remote';
1935
+ const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
1853
1936
  // Local dispatches need the same deterministic branch identity as remote
1854
1937
  // ones. Without it, every worker starts in the configured shared checkout
1855
1938
  // and concurrent issues can switch each other back to the base branch.
1856
1939
  const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
1857
- const lifecycleRunId = !dryRun && (durableRemoteDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
1940
+ const lifecycleRunId = !dryRun && (durableDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
1858
1941
  if (lifecycleRunId) {
1859
1942
  dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
1860
1943
  isolateLocalWorktree,
1861
1944
  });
1862
1945
  }
1863
1946
  dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
1864
- if (durableRemoteDispatch) {
1947
+ if (durableDispatch) {
1865
1948
  const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun, lifecycleRunId);
1866
1949
  this.#consumePendingDispatchClarifications(dispatchDecision.issue);
1867
1950
  dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision);
@@ -1881,7 +1964,7 @@ export class FactoryLoop {
1881
1964
  return restored.result;
1882
1965
  }
1883
1966
  }
1884
- if (!durableRemoteDispatch)
1967
+ if (!durableDispatch)
1885
1968
  this.#consumePendingDispatchClarifications(dispatchDecision.issue);
1886
1969
  await this.#recordDispatchAttempt(dispatchDecision.issue);
1887
1970
  const record = batch.start(dispatchDecision, dryRun);
@@ -1899,6 +1982,12 @@ export class FactoryLoop {
1899
1982
  await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
1900
1983
  const spawnedForReaperHandoff = [];
1901
1984
  try {
1985
+ if (!dryRun) {
1986
+ const issue = await this.#readIssue(dispatchDecision.issue.path);
1987
+ if (!issue || !this.#isIssueReady(issue)) {
1988
+ throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
1989
+ }
1990
+ }
1902
1991
  const specs = dispatchSpecs(dispatchDecision);
1903
1992
  const agents = [];
1904
1993
  for (const spec of specs) {
@@ -1960,8 +2049,10 @@ export class FactoryLoop {
1960
2049
  // acknowledged spawns, so cleanup never races a name-only survivor.
1961
2050
  const failureHandoffs = this.#dispatchFailureHandoffs(record, spawnedForReaperHandoff);
1962
2051
  await this.#persistDispatchFailureReaperHandoff(record, failureHandoffs);
1963
- let worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs);
1964
2052
  const liveStateChanged = error instanceof LiveDispatchStateChangedError;
2053
+ const cancellationReason = factoryCloudDispatchCancellationReason(error);
2054
+ const cleanupReason = liveStateChanged ? 'live dispatch state changed' : 'dispatch failed';
2055
+ let worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs, cleanupReason);
1965
2056
  if (liveStateChanged && !failureHandoffs.some((handoff) => handoff.worktree)) {
1966
2057
  const failed = await this.#releaseAndTerminateAgents(failureHandoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion');
1967
2058
  if (failed.length === 0) {
@@ -1974,13 +2065,13 @@ export class FactoryLoop {
1974
2065
  let failedState;
1975
2066
  if (liveStateChanged) {
1976
2067
  await this.#clearDispatchInFlight(decision.issue);
1977
- await this.#saveDispatchLifecycle(record, 'abandoned');
2068
+ await this.#saveDispatchLifecycle(record, 'abandoned', undefined, undefined, new Set(), { cancellationReason });
1978
2069
  this.#increment('dispatchLiveStateRaces');
1979
2070
  }
1980
2071
  else {
1981
2072
  await this.#recordDispatchFailure(decision.issue);
1982
2073
  failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key);
1983
- await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable');
2074
+ await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable', undefined, undefined, new Set(), { cancellationReason: failedState?.terminal ? cancellationReason : undefined });
1984
2075
  }
1985
2076
  batch.abandon(decision.issue);
1986
2077
  if (!liveStateChanged && !failedState?.terminal)
@@ -2057,7 +2148,7 @@ export class FactoryLoop {
2057
2148
  });
2058
2149
  }
2059
2150
  }
2060
- // Remote backends survive orchestrator restarts: re-adopt the agents recorded
2151
+ // Durable backends survive orchestrator restarts: re-adopt the agents recorded
2061
2152
  // in the durable lifecycle store, restore their full batch/spec association,
2062
2153
  // then reconcile once so exits that happened while this process was down are
2063
2154
  // handled instead of being dropped as unknown agents.
@@ -2097,8 +2188,9 @@ export class FactoryLoop {
2097
2188
  for (const agent of claim.lifecycle.agents) {
2098
2189
  const invocationId = agent.tracked.spec.invocationId;
2099
2190
  const node = agent.tracked.result?.node;
2100
- if (invocationId || node)
2191
+ if (invocationId || node || agent.tracked.result) {
2101
2192
  agents.push({ name: agent.name, invocationId, node });
2193
+ }
2102
2194
  }
2103
2195
  }
2104
2196
  // Migration fallback for registries written before durable lifecycle
@@ -2136,6 +2228,12 @@ export class FactoryLoop {
2136
2228
  this.#dispatchLifecycleEpochs.delete(key);
2137
2229
  this.#increment('dispatchLifecycleLeasesLost');
2138
2230
  const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
2231
+ if (lifecycle) {
2232
+ await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
2233
+ level: 'error',
2234
+ errorCode: 'lease_lost',
2235
+ });
2236
+ }
2139
2237
  if (lifecycle && !isTerminalDispatchLifecycle(lifecycle) && lifecycle.phase !== 'waiting-for-human') {
2140
2238
  this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(lifecycle));
2141
2239
  }
@@ -2171,10 +2269,87 @@ export class FactoryLoop {
2171
2269
  }
2172
2270
  this.#dispatchLifecycleEpochs.set(key, claim.lease.epoch);
2173
2271
  this.#scheduleDispatchLifecycleRenewal();
2272
+ if (claim.created) {
2273
+ await this.#reportLifecycle(claim.lifecycle, 'run.started');
2274
+ }
2174
2275
  return { created: claim.created, lifecycle: claim.lifecycle };
2175
2276
  }
2176
- async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set()) {
2177
- if (record.dryRun || this.#fleet.placementLocality !== 'remote')
2277
+ #usesDurableDispatchLifecycle() {
2278
+ return this.#fleet.durableOwnership ?? this.#fleet.placementLocality === 'remote';
2279
+ }
2280
+ async #report(input) {
2281
+ if (!this.#reporter)
2282
+ return;
2283
+ try {
2284
+ await this.#reporter.report(createFactoryCloudEventV1(input, {
2285
+ now: () => new Date(this.#clock.now()),
2286
+ }));
2287
+ }
2288
+ catch (error) {
2289
+ // A custom reporter is allowed through the public port, so defend the
2290
+ // orchestration path even if it violates the port's no-reject contract.
2291
+ this.#increment('factoryEventReportingFailures');
2292
+ this.#logger.warn?.('[factory] progress reporter rejected an event', {
2293
+ eventType: input.type,
2294
+ errorClass: telemetryErrorClass(error),
2295
+ });
2296
+ }
2297
+ }
2298
+ async #reportLifecycle(lifecycle, type, options = {}) {
2299
+ await this.#report({
2300
+ type,
2301
+ level: options.level ?? (type === 'run.failed' || type === 'factory.anomaly' ? 'error' : 'info'),
2302
+ runId: lifecycle.runId,
2303
+ phase: lifecycle.phase,
2304
+ status: telemetryRunStatus(lifecycle.phase),
2305
+ run: {
2306
+ source: githubIssuePathParts(lifecycle.issue.path) ? 'github' : 'linear',
2307
+ repository: lifecycle.decision.routes[0]?.repo,
2308
+ issueKey: lifecycle.issue.key,
2309
+ recipe: lifecycle.decision.scope,
2310
+ },
2311
+ attributes: {
2312
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
2313
+ component: 'orchestrator',
2314
+ operation: 'save_lifecycle',
2315
+ previousPhase: options.previousPhase,
2316
+ errorCode: options.errorCode,
2317
+ cancellationReason: options.cancellationReason,
2318
+ dryRun: lifecycle.dryRun,
2319
+ trackedAgents: lifecycle.agents.length,
2320
+ },
2321
+ });
2322
+ }
2323
+ async #reportAgent(record, tracked, type, options = {}) {
2324
+ const lifecycle = await this.#state
2325
+ .getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
2326
+ .catch(() => undefined);
2327
+ if (!lifecycle)
2328
+ return;
2329
+ await this.#report({
2330
+ type,
2331
+ runId: lifecycle.runId,
2332
+ phase: lifecycle.phase,
2333
+ status: telemetryRunStatus(lifecycle.phase),
2334
+ run: {
2335
+ source: githubIssuePathParts(record.issue.path) ? 'github' : 'linear',
2336
+ repository: tracked.spec.repo,
2337
+ issueKey: record.issue.key,
2338
+ recipe: record.decision.scope,
2339
+ },
2340
+ attributes: {
2341
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
2342
+ component: 'fleet',
2343
+ operation: type.slice('agent.'.length),
2344
+ agentRole: tracked.spec.role,
2345
+ invocationId: tracked.spec.invocationId,
2346
+ locality: tracked.result?.locality ?? this.#fleet.placementLocality,
2347
+ releaseReason: factoryCloudReleaseReasonV1(options.releaseReason),
2348
+ },
2349
+ });
2350
+ }
2351
+ async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set(), telemetry = {}) {
2352
+ if (record.dryRun || !this.#usesDurableDispatchLifecycle())
2178
2353
  return true;
2179
2354
  const key = issueKey(record.issue);
2180
2355
  const epoch = this.#dispatchLifecycleEpochs.get(key);
@@ -2195,9 +2370,20 @@ export class FactoryLoop {
2195
2370
  if (!saved) {
2196
2371
  this.#dispatchLifecycleEpochs.delete(key);
2197
2372
  this.#increment('dispatchLifecycleFencesRejected');
2373
+ await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
2374
+ level: 'error',
2375
+ errorCode: 'fence_rejected',
2376
+ });
2198
2377
  this.#scheduleDispatchLifecycleRetry(record);
2199
2378
  return false;
2200
2379
  }
2380
+ if (previous?.phase !== lifecycle.phase) {
2381
+ await this.#reportLifecycle(lifecycle, lifecycle.phase === 'complete'
2382
+ ? 'run.succeeded'
2383
+ : lifecycle.phase === 'abandoned'
2384
+ ? 'run.cancelled'
2385
+ : 'run.phase_changed', { previousPhase: previous?.phase, cancellationReason: telemetry.cancellationReason });
2386
+ }
2201
2387
  if (isTerminalDispatchLifecycle(lifecycle)) {
2202
2388
  this.#dispatchLifecycleEpochs.delete(key);
2203
2389
  }
@@ -2229,7 +2415,7 @@ export class FactoryLoop {
2229
2415
  this.#dispatchLifecycleRetryTimers.set(key, timer);
2230
2416
  }
2231
2417
  #scheduleReleaseRetry(record, reason) {
2232
- if (this.#fleet.placementLocality === 'remote') {
2418
+ if (this.#usesDurableDispatchLifecycle()) {
2233
2419
  this.#scheduleDispatchLifecycleRetry(record);
2234
2420
  return;
2235
2421
  }
@@ -2366,6 +2552,16 @@ export class FactoryLoop {
2366
2552
  }
2367
2553
  }
2368
2554
  async #resumeDurableDispatch(record) {
2555
+ if (!record.dryRun) {
2556
+ const issue = await this.#readIssue(record.issue.path);
2557
+ if (!issue) {
2558
+ throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is not currently readable`);
2559
+ }
2560
+ if (isGithubIssue(issue) && !this.#isGithubIssueResumable(issue)) {
2561
+ await this.#abandonDurableResume(record, 'live GitHub issue is closed or no longer ready-for-agent');
2562
+ return;
2563
+ }
2564
+ }
2369
2565
  const agents = [];
2370
2566
  const specs = dispatchSpecs(record.decision);
2371
2567
  const plannedNames = new Set(specs.map((spec) => spec.name));
@@ -2413,6 +2609,55 @@ export class FactoryLoop {
2413
2609
  }
2414
2610
  }
2415
2611
  }
2612
+ #isGithubIssueResumable(issue) {
2613
+ if (this.#isIssueReady(issue))
2614
+ return true;
2615
+ if (githubFactoryIssueIsClosed(issue))
2616
+ return false;
2617
+ const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
2618
+ const required = this.#config.safety.requireLabel.trim().toLowerCase();
2619
+ return Boolean(required) &&
2620
+ labels.has(required) &&
2621
+ labels.has('factory:in-progress') &&
2622
+ !labels.has('factory:human-review');
2623
+ }
2624
+ async #abandonDurableResume(record, reason) {
2625
+ const handoffs = this.#dispatchFailureHandoffs(record, [...record.agents].map(([name, tracked]) => ({
2626
+ issue: record.issue,
2627
+ name,
2628
+ tracked: cloneTrackedAgent(tracked),
2629
+ persistedAtMs: this.#clock.now(),
2630
+ })));
2631
+ await this.#persistDispatchFailureReaperHandoff(record, handoffs);
2632
+ if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'source_state_changed' }))
2633
+ return;
2634
+ await this.#clearDispatchInFlight(record.issue);
2635
+ const batch = await this.#batch();
2636
+ batch.abandon(record.issue);
2637
+ for (const [name] of record.agents) {
2638
+ this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
2639
+ }
2640
+ if (handoffs.some((handoff) => handoff.worktree)) {
2641
+ await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed');
2642
+ }
2643
+ else if (handoffs.length > 0) {
2644
+ const failed = new Set(await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion'));
2645
+ for (const handoff of handoffs) {
2646
+ if (failed.has(handoff.name))
2647
+ continue;
2648
+ await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
2649
+ }
2650
+ }
2651
+ await this.#stopSlackWatcher(record.issue);
2652
+ await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
2653
+ await this.#writeInFlightRegistry();
2654
+ this.#increment('dispatchLifecycleStaleIssuesAbandoned');
2655
+ this.#resolveDispatchTerminalWaiters(record.issue);
2656
+ this.#logger.info?.('[factory] abandoned durable dispatch whose live issue is no longer ready', {
2657
+ issue: record.issue.key,
2658
+ reason,
2659
+ });
2660
+ }
2416
2661
  async #finishDurableRelease(record, releaseReason) {
2417
2662
  const batch = await this.#batch();
2418
2663
  const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done');
@@ -2458,14 +2703,14 @@ export class FactoryLoop {
2458
2703
  this.#scheduleReleaseRetry(record, reason);
2459
2704
  return false;
2460
2705
  }
2461
- const next = this.#fleet.placementLocality === 'remote' ? undefined : batch.complete(record.issue);
2706
+ const next = this.#usesDurableDispatchLifecycle() ? undefined : batch.complete(record.issue);
2462
2707
  this.#localReleaseCheckpoints.delete(releaseKey);
2463
2708
  if (next)
2464
2709
  await this.dispatch(next.decision, { dryRun: next.dryRun });
2465
2710
  // Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
2466
2711
  // the babysitter's durable ownership/wake/critical state while that epoch
2467
2712
  // is still valid so a later reopened issue cannot inherit a stale PR owner.
2468
- if (this.#fleet.placementLocality === 'remote' && this.#config.babysitter.enabled) {
2713
+ if (this.#usesDurableDispatchLifecycle() && this.#config.babysitter.enabled) {
2469
2714
  await this.#cancelBabysitterWake(issueKey(record.issue));
2470
2715
  }
2471
2716
  if (!await this.#saveDispatchLifecycle(record, 'complete'))
@@ -2480,7 +2725,7 @@ export class FactoryLoop {
2480
2725
  return await this.#assertIssueDispatchLifecycleOwner(record.issue);
2481
2726
  }
2482
2727
  async #assertIssueDispatchLifecycleOwner(issue) {
2483
- if (this.#fleet.placementLocality !== 'remote')
2728
+ if (!this.#usesDurableDispatchLifecycle())
2484
2729
  return true;
2485
2730
  const key = issueKey(issue);
2486
2731
  const epoch = this.#dispatchLifecycleEpochs.get(key);
@@ -2580,8 +2825,7 @@ export class FactoryLoop {
2580
2825
  if (!isGithubIssue(issue)) {
2581
2826
  return this.#states.isRole(issue.stateId, 'readyForAgent');
2582
2827
  }
2583
- const githubState = (issue.state?.name ?? stringValue(wrappedPayload(issue.raw).state) ?? '').trim().toLowerCase();
2584
- if (githubState === 'closed') {
2828
+ if (githubFactoryIssueIsClosed(issue)) {
2585
2829
  return false;
2586
2830
  }
2587
2831
  const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
@@ -2706,9 +2950,14 @@ export class FactoryLoop {
2706
2950
  }
2707
2951
  }
2708
2952
  }
2953
+ for (const [identity, path] of issuePaths) {
2954
+ this.#githubIssuePreferredPaths.set(identity, path);
2955
+ }
2956
+ this.#githubIssuePathIndexReady = true;
2709
2957
  return [...issuePaths.values()].sort();
2710
2958
  }
2711
2959
  catch (error) {
2960
+ this.#githubIssuePathIndexReady = false;
2712
2961
  this.#increment('githubIssueListFailures');
2713
2962
  this.#logger.warn?.('[factory] failed to list GitHub issue source tree', error);
2714
2963
  return [];
@@ -2773,7 +3022,11 @@ export class FactoryLoop {
2773
3022
  }
2774
3023
  }
2775
3024
  async #readGithubIssue(path) {
2776
- const candidatePaths = githubIssueReadCandidatePaths(path);
3025
+ const preferredPath = await this.#preferredGithubIssuePath(path);
3026
+ const candidatePaths = [...new Set([
3027
+ ...githubIssueReadCandidatePaths(preferredPath),
3028
+ ...githubIssueReadCandidatePaths(path),
3029
+ ])];
2777
3030
  try {
2778
3031
  for (const candidatePath of candidatePaths) {
2779
3032
  try {
@@ -2797,6 +3050,30 @@ export class FactoryLoop {
2797
3050
  throw error;
2798
3051
  }
2799
3052
  }
3053
+ async #preferredGithubIssuePath(path) {
3054
+ const parts = githubIssuePathParts(path) ?? githubIssueDirectoryPathParts(path);
3055
+ if (!parts)
3056
+ return path;
3057
+ const identity = githubIssueIdentity(parts.owner, parts.repo, parts.number);
3058
+ const cached = this.#githubIssuePreferredPaths.get(identity);
3059
+ if (cached && githubIssuePathPreference(cached) <= githubIssuePathPreference(path))
3060
+ return cached;
3061
+ if (githubIssuePathPreference(path) === 0) {
3062
+ this.#githubIssuePreferredPaths.set(identity, path);
3063
+ return path;
3064
+ }
3065
+ // Normal discovery has already indexed every configured GitHub issue path.
3066
+ // A dispatch-only replacement owner can reach this method before that
3067
+ // backfill, so build the same shared index once instead of traversing both
3068
+ // repository trees separately for every durable issue it recovers.
3069
+ if (!this.#githubIssuePathIndexReady) {
3070
+ await this.#githubIssuePaths();
3071
+ }
3072
+ const indexed = this.#githubIssuePreferredPaths.get(identity);
3073
+ return indexed && githubIssuePathPreference(indexed) < githubIssuePathPreference(path)
3074
+ ? indexed
3075
+ : path;
3076
+ }
2800
3077
  async #findGithubIssueMirror(ghIssue, candidates) {
2801
3078
  const draftPath = githubIssueMirrorDraftPath(ghIssue);
2802
3079
  try {
@@ -2947,6 +3224,19 @@ export class FactoryLoop {
2947
3224
  await mkdir(dirname(path), { recursive: true });
2948
3225
  await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
2949
3226
  await this.#writeInFlightRegistry(registryPath, path);
3227
+ const batch = this.#batchView;
3228
+ await this.#report({
3229
+ type: 'instance.heartbeat',
3230
+ attributes: {
3231
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
3232
+ mode: status,
3233
+ component: 'orchestrator',
3234
+ operation: 'heartbeat',
3235
+ activeRuns: batch?.inFlight.length,
3236
+ queuedRuns: batch?.queued.length,
3237
+ trackedAgents: batch?.inFlight.reduce((count, record) => count + record.agents.size, 0),
3238
+ },
3239
+ });
2950
3240
  }
2951
3241
  async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
2952
3242
  const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
@@ -3150,7 +3440,9 @@ export class FactoryLoop {
3150
3440
  async #releaseAndTerminateAgents(agents, reason, context) {
3151
3441
  const failed = [];
3152
3442
  const protectedPids = await this.#protectedPids();
3443
+ const batch = this.#batchView;
3153
3444
  for (const [agentName, tracked] of agents) {
3445
+ const record = batch?.getIssueByAgent(agentName);
3154
3446
  if (context === 'stop') {
3155
3447
  await this.#refreshStoppingHeartbeat();
3156
3448
  }
@@ -3182,10 +3474,23 @@ export class FactoryLoop {
3182
3474
  }
3183
3475
  try {
3184
3476
  await this.#fleet.release(agentName, reason);
3477
+ if (record)
3478
+ await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
3185
3479
  }
3186
3480
  catch (error) {
3187
3481
  failed.push(agentName);
3188
3482
  this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
3483
+ if (record) {
3484
+ const lifecycle = await this.#state
3485
+ .getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
3486
+ .catch(() => undefined);
3487
+ if (lifecycle) {
3488
+ await this.#reportLifecycle(lifecycle, 'factory.failure', {
3489
+ level: 'error',
3490
+ errorCode: 'release_failed',
3491
+ });
3492
+ }
3493
+ }
3189
3494
  }
3190
3495
  if (context === 'stop') {
3191
3496
  await this.#refreshStoppingHeartbeat();
@@ -3294,10 +3599,10 @@ export class FactoryLoop {
3294
3599
  }
3295
3600
  return [...handoffs.values()];
3296
3601
  }
3297
- async #teardownFailedDispatchWorktrees(handoffs) {
3602
+ async #teardownFailedDispatchWorktrees(handoffs, releaseReason = 'dispatch failed') {
3298
3603
  if (!this.#worktrees || !handoffs.some((handoff) => handoff.worktree))
3299
3604
  return false;
3300
- const failed = await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'dispatch failed', 'completion');
3605
+ const failed = await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), releaseReason, 'completion');
3301
3606
  if (failed.length > 0)
3302
3607
  return false;
3303
3608
  try {
@@ -3423,6 +3728,9 @@ export class FactoryLoop {
3423
3728
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
3424
3729
  throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`);
3425
3730
  }
3731
+ const adopted = record.agents.get(spec.name);
3732
+ if (adopted)
3733
+ await this.#reportAgent(record, adopted, 'agent.adopted');
3426
3734
  return { name: spec.name };
3427
3735
  }
3428
3736
  await this.#prepareAgentWorktree(record, spec);
@@ -3445,12 +3753,20 @@ export class FactoryLoop {
3445
3753
  });
3446
3754
  }
3447
3755
  catch (error) {
3448
- throw contextualError(`Dispatch spawn failed for ${record.issue.key}/${spec.name} (${spec.capability}) cwd=${spec.clonePath ?? 'default'}`, error);
3756
+ const wrapped = contextualError(`Dispatch spawn failed for ${record.issue.key}/${spec.name} (${spec.capability}) cwd=${spec.clonePath ?? 'default'}`, error);
3757
+ throw Object.assign(wrapped, {
3758
+ factoryCancellationReason: isDispatchDeliveryError(error)
3759
+ ? 'agent_delivery_failed'
3760
+ : 'agent_spawn_failed',
3761
+ });
3449
3762
  }
3450
3763
  batch.recordSpawn(record, spec, invocationId, result);
3451
3764
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
3452
3765
  throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`);
3453
3766
  }
3767
+ const spawned = record.agents.get(result.name);
3768
+ if (spawned)
3769
+ await this.#reportAgent(record, spawned, 'agent.spawned');
3454
3770
  return { name: result.name };
3455
3771
  }
3456
3772
  async #handleAgentExit(name, reason) {
@@ -3469,6 +3785,19 @@ export class FactoryLoop {
3469
3785
  const batch = await this.#batch();
3470
3786
  const record = batch.getIssueByAgent(name);
3471
3787
  if (!record) {
3788
+ if (/^ar-\d+-/u.test(name)) {
3789
+ await this.#report({
3790
+ type: 'factory.anomaly',
3791
+ level: 'error',
3792
+ attributes: {
3793
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
3794
+ component: 'fleet',
3795
+ operation: 'agent_exit',
3796
+ errorCode: 'unowned_agent',
3797
+ count: 1,
3798
+ },
3799
+ });
3800
+ }
3472
3801
  return;
3473
3802
  }
3474
3803
  if (!await this.#assertDispatchLifecycleOwner(record)) {
@@ -3487,7 +3816,9 @@ export class FactoryLoop {
3487
3816
  return;
3488
3817
  }
3489
3818
  const exiting = record.agents.get(name);
3490
- if (this.#fleet.placementLocality === 'remote') {
3819
+ if (exiting)
3820
+ await this.#reportAgent(record, exiting, 'agent.exited', { releaseReason: reason });
3821
+ if (this.#usesDurableDispatchLifecycle()) {
3491
3822
  const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
3492
3823
  if (lifecycle?.phase === 'parking') {
3493
3824
  this.#increment('clarificationParkingExitsSuppressed');
@@ -4176,6 +4507,7 @@ export class FactoryLoop {
4176
4507
  await this.#persistBabysitterSession(record.issue, ref, tracked);
4177
4508
  }
4178
4509
  }
4510
+ await this.#reportAgent(record, tracked, 'agent.resumed');
4179
4511
  }
4180
4512
  async #handleDeliveryFailed(info) {
4181
4513
  const critical = await this.#state.consumeCritical(this.#workspaceId, info.msgId ?? '');
@@ -4185,7 +4517,7 @@ export class FactoryLoop {
4185
4517
  }
4186
4518
  const record = (await this.#batch()).getIssueByAgent(info.to);
4187
4519
  const issue = critical.issue ?? record?.issue;
4188
- const error = new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`);
4520
+ const error = Object.assign(new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`), { code: 'fleet_delivery_failed' });
4189
4521
  this.#error(error, issue);
4190
4522
  if (isTerminalDeliveryFailure(info.reason)) {
4191
4523
  this.#increment('criticalDeliveryTerminalFailures');
@@ -5661,7 +5993,7 @@ export class FactoryLoop {
5661
5993
  }
5662
5994
  async #cancelBabysitterWake(issueIdentity) {
5663
5995
  const issue = this.#babysitterIssueRefs.get(issueIdentity);
5664
- const mayClearDurable = this.#fleet.placementLocality !== 'remote'
5996
+ const mayClearDurable = !this.#usesDurableDispatchLifecycle()
5665
5997
  || Boolean(issue && await this.#assertIssueDispatchLifecycleOwner(issue));
5666
5998
  for (const [key, state] of this.#babysitterWakeStates) {
5667
5999
  if (issueKey(state.issue) !== issueIdentity)
@@ -5922,19 +6254,7 @@ export class FactoryLoop {
5922
6254
  },
5923
6255
  };
5924
6256
  let targets;
5925
- if (this.#fleet.promptDelivery === 'pty') {
5926
- if (!this.#fleet.sendInput) {
5927
- throw new Error('Fleet client advertises PTY prompt delivery without raw input support');
5928
- }
5929
- // Internal agents are harness-owned PTYs, not Relaycast identities.
5930
- // Stage the validated metadata-only wake directly through the broker so
5931
- // a missing Relaycast registration cannot strand a live babysitter.
5932
- // The CR remains a separate write: a critical-section begin can arrive
5933
- // while this write is in flight and must be able to defer submission.
5934
- await this.#fleet.sendInput(input.to, input.text);
5935
- targets = [input.to];
5936
- }
5937
- else if (!this.#fleet.waitForInjected) {
6257
+ if (!this.#fleet.waitForInjected) {
5938
6258
  await this.#fleet.sendMessage(input);
5939
6259
  if (this.#stopping || state.cancelled) {
5940
6260
  state.deliveringKinds = undefined;
@@ -6403,7 +6723,12 @@ export class FactoryLoop {
6403
6723
  prNumber: prRef.prNumber,
6404
6724
  babysitter: spawned.name,
6405
6725
  });
6406
- if (this.#fleet.waitForInjected) {
6726
+ // Internal PTY spawns receive `task` atomically in spawnPty. Re-sending
6727
+ // the same task through Relaycast before the worker has registered its
6728
+ // messaging identity can fail an otherwise successful spawn and erase
6729
+ // valid babysitter ownership. Remote placement still needs the explicit,
6730
+ // confirmed follow-up injection used by its spawn protocol.
6731
+ if (this.#fleet.waitForInjected && this.#fleet.placementLocality !== 'local') {
6407
6732
  const input = {
6408
6733
  to: tracked?.result?.name ?? spawned.name,
6409
6734
  text: task,
@@ -6637,7 +6962,7 @@ export class FactoryLoop {
6637
6962
  }
6638
6963
  const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
6639
6964
  releaseReasonForRetry = releaseReason;
6640
- if (this.#fleet.placementLocality === 'remote') {
6965
+ if (this.#usesDurableDispatchLifecycle()) {
6641
6966
  // Durable capacity is released as soon as terminal writeback is
6642
6967
  // acknowledged. Agent cleanup remains fenced/retryable in `releasing`.
6643
6968
  const batch = await this.#batch();
@@ -6667,7 +6992,7 @@ export class FactoryLoop {
6667
6992
  this.#babysitterPr.delete(completionKey);
6668
6993
  await this.#cancelBabysitterWake(completionKey);
6669
6994
  const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
6670
- if (this.#fleet.placementLocality !== 'remote' || (durable && isTerminalDispatchLifecycle(durable))) {
6995
+ if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
6671
6996
  for (const publishedKey of this.#publishedPullRequests.keys()) {
6672
6997
  if (publishedKey.startsWith(`${completionKey}:`))
6673
6998
  this.#publishedPullRequests.delete(publishedKey);
@@ -6677,7 +7002,16 @@ export class FactoryLoop {
6677
7002
  }
6678
7003
  #emit(event, payload) {
6679
7004
  for (const listener of this.#listeners.get(event) ?? []) {
6680
- listener(payload);
7005
+ try {
7006
+ listener(payload);
7007
+ }
7008
+ catch (error) {
7009
+ this.#increment('factoryEventListenerFailures');
7010
+ this.#logger.warn?.('[factory] event listener failed', {
7011
+ event,
7012
+ errorClass: error instanceof Error ? error.name : 'Error',
7013
+ });
7014
+ }
6681
7015
  }
6682
7016
  }
6683
7017
  #error(error, issue) {
@@ -6692,6 +7026,48 @@ export class FactoryLoop {
6692
7026
  ...details,
6693
7027
  ...(issue ? { issue: issue.key } : {}),
6694
7028
  });
7029
+ const failureCode = telemetryCategory(error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
7030
+ ? error.code
7031
+ : 'factory_error');
7032
+ const failureClass = telemetryErrorClass(error);
7033
+ void (async () => {
7034
+ try {
7035
+ const lifecycle = issue
7036
+ ? await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(issue)).catch(() => undefined)
7037
+ : undefined;
7038
+ if (lifecycle) {
7039
+ await this.#reportLifecycle(lifecycle, 'factory.failure', {
7040
+ level: 'error',
7041
+ errorCode: failureCode,
7042
+ });
7043
+ return;
7044
+ }
7045
+ await this.#report({
7046
+ type: 'factory.failure',
7047
+ level: 'error',
7048
+ attributes: {
7049
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
7050
+ component: 'orchestrator',
7051
+ operation: 'error',
7052
+ errorClass: failureClass,
7053
+ errorCode: failureCode,
7054
+ },
7055
+ });
7056
+ }
7057
+ catch (telemetryError) {
7058
+ // This is the terminal guard for the intentionally floating telemetry
7059
+ // task. Never log raw messages or allow a custom logger to turn this
7060
+ // best-effort path into an unhandled rejection.
7061
+ try {
7062
+ this.#logger.warn?.('[factory] failed to report failure telemetry', {
7063
+ errorClass: telemetryErrorClass(telemetryError),
7064
+ });
7065
+ }
7066
+ catch {
7067
+ // Reporting and logging are both non-critical to orchestration.
7068
+ }
7069
+ }
7070
+ })();
6695
7071
  this.#emit('error', { error, ...details, issue });
6696
7072
  }
6697
7073
  #surfaceEscalationDeliveryFailure(kind, issue, correlationId, reason, cause) {
@@ -7755,7 +8131,7 @@ export class FactoryLoop {
7755
8131
  }
7756
8132
  let lifecycleDecision = waiting.decision;
7757
8133
  let promotedLifecycle;
7758
- if (!waiting.dryRun && this.#fleet.placementLocality === 'remote') {
8134
+ if (!waiting.dryRun && this.#usesDurableDispatchLifecycle()) {
7759
8135
  try {
7760
8136
  const claim = await this.#claimDispatchLifecycle(waiting.decision, false);
7761
8137
  const lifecycleKey = issueKey(waiting.issue);
@@ -8444,6 +8820,9 @@ export function isDispatchableIssue(issue) {
8444
8820
  if (!isGithubIssue(issue)) {
8445
8821
  return false;
8446
8822
  }
8823
+ if (githubFactoryIssueIsClosed(issue)) {
8824
+ return false;
8825
+ }
8447
8826
  const payload = wrappedPayload(issue.raw);
8448
8827
  const source = asRecord(payload.source);
8449
8828
  const sourceId = stringValue(source?.id);
@@ -8458,6 +8837,7 @@ const isGithubIssue = (issue) => {
8458
8837
  return stringValue(source?.provider)?.toLowerCase() === 'github' &&
8459
8838
  (stringValue(wrapper?.provider)?.toLowerCase() === 'github' || isGithubIssueFilePath(issue.path));
8460
8839
  };
8840
+ const githubFactoryIssueIsClosed = (issue) => (issue.state?.name ?? stringValue(wrappedPayload(issue.raw).state) ?? '').trim().toLowerCase() === 'closed';
8461
8841
  const githubIssueSourceRef = (issue) => {
8462
8842
  const source = asRecord(wrappedPayload(issue.raw).source);
8463
8843
  if (stringValue(source?.provider)?.toLowerCase() !== 'github') {
@@ -8906,13 +9286,15 @@ const githubIssueRefIdentity = (issue) => {
8906
9286
  return parts ? githubIssueIdentity(parts.owner, parts.repo, parts.number) : undefined;
8907
9287
  };
8908
9288
  const githubIssuePathPreference = (path) => {
8909
- if (path.includes('/issues/by-id/'))
8910
- return 0;
8911
9289
  if (path.endsWith('/meta.json'))
9290
+ return 0;
9291
+ if (path.endsWith('/metadata.json'))
8912
9292
  return 1;
8913
- if (path.endsWith('.json'))
9293
+ if (path.includes('/issues/by-id/'))
8914
9294
  return 2;
8915
- return 3;
9295
+ if (path.endsWith('.json'))
9296
+ return 3;
9297
+ return 4;
8916
9298
  };
8917
9299
  const githubAgentNameMatchesIssue = (name, issue) => {
8918
9300
  const parts = githubIssuePathParts(issue.path);
@@ -9228,11 +9610,15 @@ export const githubRepoSubscriptionGlobs = (config) => configuredGithubRepoParts
9228
9610
  const githubIssueScanRoots = (config) => {
9229
9611
  const roots = new Set();
9230
9612
  for (const { owner, repo } of configuredGithubRepoParts(config)) {
9231
- roots.add(`${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues`);
9232
- roots.add(`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`);
9613
+ for (const root of githubIssueRepoRoots(owner, repo))
9614
+ roots.add(root);
9233
9615
  }
9234
9616
  return [...roots];
9235
9617
  };
9618
+ const githubIssueRepoRoots = (owner, repo) => [
9619
+ `${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues`,
9620
+ `${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`,
9621
+ ];
9236
9622
  const githubRepoPathParts = (path) => {
9237
9623
  const compactSegment = path.match(/^\/github\/repos\/([^/]+)\//u)?.[1];
9238
9624
  const separator = compactSegment?.indexOf('__') ?? -1;
@@ -9981,7 +10367,13 @@ const isAgentAlreadyExistsError = (error) => {
9981
10367
  const defaultRestartPolicy = (spec) =>
9982
10368
  // Factory owns durable resume/respawn decisions. Broker-level retries race
9983
10369
  // that lifecycle and can re-register the same name before Factory resumes it.
9984
- spec.role === 'implementer' || spec.role === 'babysitter'
10370
+ // The reviewer shares the implementer's dispatch lifecycle: it is spawned in
10371
+ // the same batch, torn down by the same dispatch-failure/teardown paths, and
10372
+ // resumed through the same durable #resumeDurableDispatch flow. Without this
10373
+ // opt-out the broker's default restart policy re-registers a torn-down
10374
+ // reviewer's name as an orphan (relay#1116-family) while the dashboard only
10375
+ // reports dispatch_failed — the exact orphan/restart sequence being audited.
10376
+ spec.role === 'implementer' || spec.role === 'reviewer' || spec.role === 'babysitter'
9985
10377
  ? { maxRestarts: 0 }
9986
10378
  : spec.restartPolicy;
9987
10379
  const slackPayloadTs = (threadId) => threadId.replace(/_/g, '.');
@@ -10046,9 +10438,52 @@ const failedIterationReport = (error, dryRun) => {
10046
10438
  };
10047
10439
  const isRegistrationLagInjectionError = (error) => {
10048
10440
  const { errorMessage } = describeError(error);
10049
- return /recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
10441
+ return /agent_not_found|recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
10050
10442
  .test(errorMessage);
10051
10443
  };
10444
+ const isDispatchDeliveryError = (error) => {
10445
+ if (isRegistrationLagInjectionError(error))
10446
+ return true;
10447
+ const { errorMessage } = describeError(error);
10448
+ return /delivery[_ -]failed|dead-lettered|max delivery retries exceeded/iu.test(errorMessage);
10449
+ };
10450
+ const factoryCloudDispatchCancellationReason = (error) => {
10451
+ if (error instanceof LiveDispatchStateChangedError)
10452
+ return 'source_state_changed';
10453
+ if (error && typeof error === 'object' && 'factoryCancellationReason' in error) {
10454
+ const reason = error.factoryCancellationReason;
10455
+ if (reason === 'agent_spawn_failed' || reason === 'agent_delivery_failed')
10456
+ return reason;
10457
+ }
10458
+ return 'dispatch_failed';
10459
+ };
10460
+ const telemetryRunStatus = (phase) => {
10461
+ switch (phase) {
10462
+ case 'queued':
10463
+ return 'queued';
10464
+ case 'retryable':
10465
+ return 'blocked';
10466
+ case 'parking':
10467
+ case 'waiting-for-human':
10468
+ return 'waiting';
10469
+ case 'complete':
10470
+ return 'succeeded';
10471
+ case 'abandoned':
10472
+ return 'cancelled';
10473
+ default:
10474
+ return 'running';
10475
+ }
10476
+ };
10477
+ const telemetryCategory = (value) => {
10478
+ if (!value)
10479
+ return undefined;
10480
+ const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
10481
+ return normalized.slice(0, 120) || undefined;
10482
+ };
10483
+ const telemetryErrorClass = (error) => {
10484
+ const name = error instanceof Error ? error.name : '';
10485
+ return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
10486
+ };
10052
10487
  const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
10053
10488
  const retryOnTimeout = async (fn, opts) => {
10054
10489
  let lastError;