@agent-relay/factory 0.1.32 → 0.1.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +36 -0
  2. package/dist/cli/fleet.d.ts +4 -1
  3. package/dist/cli/fleet.d.ts.map +1 -1
  4. package/dist/cli/fleet.js +113 -6
  5. package/dist/cli/fleet.js.map +1 -1
  6. package/dist/config/schema.d.ts +88 -8
  7. package/dist/config/schema.d.ts.map +1 -1
  8. package/dist/config/schema.js +10 -0
  9. package/dist/config/schema.js.map +1 -1
  10. package/dist/fleet/internal-fleet-client.d.ts +9 -2
  11. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  12. package/dist/fleet/internal-fleet-client.js +39 -1
  13. package/dist/fleet/internal-fleet-client.js.map +1 -1
  14. package/dist/fleet/relay-fleet-client.d.ts +1 -0
  15. package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
  16. package/dist/fleet/relay-fleet-client.js +1 -0
  17. package/dist/fleet/relay-fleet-client.js.map +1 -1
  18. package/dist/index.d.ts +7 -1
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +3 -0
  21. package/dist/index.js.map +1 -1
  22. package/dist/observability/cloud-reporter.d.ts +46 -0
  23. package/dist/observability/cloud-reporter.d.ts.map +1 -0
  24. package/dist/observability/cloud-reporter.js +371 -0
  25. package/dist/observability/cloud-reporter.js.map +1 -0
  26. package/dist/observability/events.d.ts +1275 -0
  27. package/dist/observability/events.d.ts.map +1 -0
  28. package/dist/observability/events.js +229 -0
  29. package/dist/observability/events.js.map +1 -0
  30. package/dist/observability/index.d.ts +5 -0
  31. package/dist/observability/index.d.ts.map +1 -0
  32. package/dist/observability/index.js +5 -0
  33. package/dist/observability/index.js.map +1 -0
  34. package/dist/observability/instance-identity.d.ts +7 -0
  35. package/dist/observability/instance-identity.d.ts.map +1 -0
  36. package/dist/observability/instance-identity.js +59 -0
  37. package/dist/observability/instance-identity.js.map +1 -0
  38. package/dist/observability/outbox.d.ts +42 -0
  39. package/dist/observability/outbox.d.ts.map +1 -0
  40. package/dist/observability/outbox.js +305 -0
  41. package/dist/observability/outbox.js.map +1 -0
  42. package/dist/orchestrator/factory.d.ts.map +1 -1
  43. package/dist/orchestrator/factory.js +336 -43
  44. package/dist/orchestrator/factory.js.map +1 -1
  45. package/dist/ports/fleet.d.ts +3 -4
  46. package/dist/ports/fleet.d.ts.map +1 -1
  47. package/dist/ports/index.d.ts +1 -0
  48. package/dist/ports/index.d.ts.map +1 -1
  49. package/dist/ports/observability.d.ts +19 -0
  50. package/dist/ports/observability.d.ts.map +1 -0
  51. package/dist/ports/observability.js +2 -0
  52. package/dist/ports/observability.js.map +1 -0
  53. package/dist/triage/schema.d.ts +14 -14
  54. package/dist/types.d.ts +3 -0
  55. package/dist/types.d.ts.map +1 -1
  56. package/package.json +5 -1
@@ -20,6 +20,7 @@ import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../write
20
20
  import { issueKey } from './batch-tracker.js';
21
21
  import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
22
22
  import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
23
+ import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
23
24
  class ClarificationWakeLeaseLostError extends Error {
24
25
  }
25
26
  class ClarificationQuestionDeliveryLeaseLostError extends Error {
@@ -119,6 +120,7 @@ export class FactoryLoop {
119
120
  #workspaceId;
120
121
  #relayflows;
121
122
  #worktrees;
123
+ #reporter;
122
124
  #batchView;
123
125
  #batchReady;
124
126
  #listeners = new Map();
@@ -263,6 +265,7 @@ export class FactoryLoop {
263
265
  this.#workspaceId = config.workspaceId ?? 'default';
264
266
  this.#relayflows = ports.relayflows;
265
267
  this.#worktrees = ports.worktrees;
268
+ this.#reporter = ports.reporter;
266
269
  this.#state = ports.stateStore ?? new InMemoryStateStore({
267
270
  batchSize: config.batchSize,
268
271
  agentQuestionDedupeLimit: AGENT_QUESTION_DEDUPE_LIMIT,
@@ -1362,6 +1365,7 @@ export class FactoryLoop {
1362
1365
  ]);
1363
1366
  const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
1364
1367
  const activeIssueIdentities = new Set();
1368
+ const legacyUnownedAgentsByIssue = new Map();
1365
1369
  for (const [, lifecycle] of lifecycles) {
1366
1370
  if (isTerminalDispatchLifecycle(lifecycle))
1367
1371
  continue;
@@ -1378,12 +1382,26 @@ export class FactoryLoop {
1378
1382
  if (!onlineAgents.has(agent.name) || !agent.issue)
1379
1383
  continue;
1380
1384
  const identity = githubIssueRefIdentity(agent.issue);
1381
- if (identity)
1385
+ if (!identity)
1386
+ continue;
1387
+ const isLegacyLocalWorker = this.#usesDurableDispatchLifecycle() &&
1388
+ this.#fleet.placementLocality === 'local' &&
1389
+ !agent.node &&
1390
+ !agent.invocationId &&
1391
+ !activeIssueIdentities.has(identity);
1392
+ if (isLegacyLocalWorker) {
1393
+ const agents = legacyUnownedAgentsByIssue.get(identity) ?? [];
1394
+ agents.push(agent);
1395
+ legacyUnownedAgentsByIssue.set(identity, agents);
1396
+ }
1397
+ else {
1382
1398
  activeIssueIdentities.add(identity);
1399
+ }
1383
1400
  }
1384
1401
  return {
1385
1402
  activeIssueIdentities,
1386
1403
  onlineAgentNames: onlineAgents,
1404
+ legacyUnownedAgentsByIssue,
1387
1405
  };
1388
1406
  }
1389
1407
  catch (error) {
@@ -1405,9 +1423,14 @@ export class FactoryLoop {
1405
1423
  labels.has('factory:human-review'))
1406
1424
  return false;
1407
1425
  const identity = githubIssueRefIdentity(issueRef(issue));
1426
+ const legacyUnownedAgents = identity
1427
+ ? (context.legacyUnownedAgentsByIssue.get(identity) ?? [])
1428
+ .filter((agent) => githubAgentNameMatchesIssue(agent.name, issue))
1429
+ : [];
1430
+ const legacyUnownedAgentNames = new Set(legacyUnownedAgents.map((agent) => agent.name));
1408
1431
  if (!identity ||
1409
1432
  context.activeIssueIdentities.has(identity) ||
1410
- [...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue))) {
1433
+ [...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue) && !legacyUnownedAgentNames.has(name))) {
1411
1434
  this.#increment('githubOrphanRecoveriesBlockedActive');
1412
1435
  return false;
1413
1436
  }
@@ -1447,7 +1470,7 @@ export class FactoryLoop {
1447
1470
  if (openPr) {
1448
1471
  let adopted = false;
1449
1472
  try {
1450
- adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr);
1473
+ adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr, legacyUnownedAgents);
1451
1474
  }
1452
1475
  catch (error) {
1453
1476
  this.#increment('githubOrphanedPullRequestAdoptionFailures');
@@ -1468,6 +1491,14 @@ export class FactoryLoop {
1468
1491
  });
1469
1492
  return false;
1470
1493
  }
1494
+ // A pre-durable local Factory may have left live, registry-proven workers
1495
+ // without a lifecycle record. They are safe to adopt only once their open
1496
+ // PR proves which dispatch they own. Without that proof, preserve the issue
1497
+ // and workers instead of redispatching duplicate agents.
1498
+ if (legacyUnownedAgents.length > 0) {
1499
+ this.#increment('githubOrphanRecoveriesBlockedActive');
1500
+ return false;
1501
+ }
1471
1502
  try {
1472
1503
  if (providerStatus === 'in-progress') {
1473
1504
  await this.#githubWriteback.setStatus(issue, 'ready');
@@ -1504,7 +1535,7 @@ export class FactoryLoop {
1504
1535
  allowLegacyGithubBranch: true,
1505
1536
  });
1506
1537
  }
1507
- async #adoptOrphanedGithubPullRequest(issue, pr) {
1538
+ async #adoptOrphanedGithubPullRequest(issue, pr, legacyUnownedAgents = []) {
1508
1539
  const headRef = pr.headRef;
1509
1540
  if (!headRef)
1510
1541
  return false;
@@ -1543,14 +1574,14 @@ export class FactoryLoop {
1543
1574
  : spec),
1544
1575
  };
1545
1576
  }
1546
- const durableRemoteAdoption = this.#fleet.placementLocality === 'remote';
1577
+ const durableAdoption = this.#usesDurableDispatchLifecycle();
1547
1578
  const publishedPr = {
1548
1579
  repo: pr.repo,
1549
1580
  number: pr.prNumber,
1550
1581
  url: pr.url ?? `https://github.com/${pr.repo}/pull/${pr.prNumber}`,
1551
1582
  headRef,
1552
1583
  };
1553
- if (durableRemoteAdoption) {
1584
+ if (durableAdoption) {
1554
1585
  const claim = await this.#claimDispatchLifecycle(decision, false, randomUUID(), {
1555
1586
  phase: 'published',
1556
1587
  pullRequest: publishedPr,
@@ -1566,7 +1597,7 @@ export class FactoryLoop {
1566
1597
  const batch = await this.#batch();
1567
1598
  const record = batch.start(decision, false);
1568
1599
  if (!record) {
1569
- if (durableRemoteAdoption) {
1600
+ if (durableAdoption) {
1570
1601
  const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
1571
1602
  if (durable) {
1572
1603
  this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(durable));
@@ -1576,6 +1607,48 @@ export class FactoryLoop {
1576
1607
  }
1577
1608
  return false;
1578
1609
  }
1610
+ if (durableAdoption && legacyUnownedAgents.length > 0) {
1611
+ const specsByName = new Map(dispatchSpecs(record.decision).map((spec) => [spec.name, spec]));
1612
+ const initialBabysitter = babysitterSpec(issue, this.#config, route);
1613
+ const sharedCheckout = record.decision.implementers.find((candidate) => candidate.repo === initialBabysitter.repo && candidate.baseClonePath && candidate.clonePath);
1614
+ const legacyBabysitter = {
1615
+ ...initialBabysitter,
1616
+ ...(sharedCheckout
1617
+ ? {
1618
+ baseClonePath: sharedCheckout.baseClonePath,
1619
+ clonePath: sharedCheckout.clonePath,
1620
+ ...(headRef ? { branch: headRef } : {}),
1621
+ ...(sharedCheckout.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
1622
+ }
1623
+ : {}),
1624
+ ownedPullRequest: { repo: pr.repo, number: pr.prNumber, path: pr.path },
1625
+ };
1626
+ specsByName.set(legacyBabysitter.name, legacyBabysitter);
1627
+ const adopted = [];
1628
+ for (const agent of legacyUnownedAgents) {
1629
+ const spec = specsByName.get(agent.name);
1630
+ if (!spec)
1631
+ continue;
1632
+ const invocationId = batch.invocationIdFor(record.issue, spec);
1633
+ batch.recordSpawn(record, spec, invocationId, {
1634
+ name: agent.name,
1635
+ sessionRef: agent.sessionRef,
1636
+ pids: agent.pids,
1637
+ locality: 'local',
1638
+ });
1639
+ adopted.push({ name: agent.name, invocationId });
1640
+ }
1641
+ if (adopted.length > 0) {
1642
+ this.#fleet.hydrateTracked?.(adopted);
1643
+ await this.#saveDispatchLifecycle(record, 'published', publishedPr);
1644
+ for (const agent of adopted) {
1645
+ this.#increment('legacyLocalWorkersAdopted');
1646
+ const tracked = record.agents.get(agent.name);
1647
+ if (tracked)
1648
+ await this.#reportAgent(record, tracked, 'agent.adopted');
1649
+ }
1650
+ }
1651
+ }
1579
1652
  await this.#ensureBabysitter(record, {
1580
1653
  repo: pr.repo,
1581
1654
  prNumber: pr.prNumber,
@@ -1585,7 +1658,7 @@ export class FactoryLoop {
1585
1658
  });
1586
1659
  const babysitter = [...record.agents.values()].find((tracked) => tracked.spec.role === 'babysitter');
1587
1660
  if (!babysitter) {
1588
- if (durableRemoteAdoption)
1661
+ if (durableAdoption)
1589
1662
  this.#scheduleDispatchLifecycleRetry(record);
1590
1663
  else
1591
1664
  batch.abandon(record.issue);
@@ -1593,10 +1666,15 @@ export class FactoryLoop {
1593
1666
  }
1594
1667
  record.result = {
1595
1668
  issue: record.issue,
1596
- agents: [{ name: babysitter.result?.name ?? babysitter.spec.name, role: 'babysitter' }],
1669
+ agents: [...record.agents.values()].map((tracked) => ({
1670
+ name: tracked.result?.name ?? tracked.spec.name,
1671
+ role: tracked.spec.role,
1672
+ })),
1597
1673
  dryRun: false,
1598
1674
  };
1599
1675
  await this.#writeInFlightRegistry();
1676
+ if (durableAdoption)
1677
+ await this.#saveDispatchLifecycle(record, 'running', publishedPr);
1600
1678
  this.#increment('githubOrphanedPullRequestsAdopted');
1601
1679
  return true;
1602
1680
  }
@@ -1778,7 +1856,7 @@ export class FactoryLoop {
1778
1856
  if (existingRecord?.result) {
1779
1857
  return existingRecord.result;
1780
1858
  }
1781
- if (!dryRun && this.#fleet.placementLocality === 'remote') {
1859
+ if (!dryRun && this.#usesDurableDispatchLifecycle()) {
1782
1860
  const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
1783
1861
  if (durable && !isTerminalDispatchLifecycle(durable)) {
1784
1862
  if (durable.result && this.#dispatchLifecycleEpochs.has(issueKey(decision.issue))) {
@@ -1849,19 +1927,19 @@ export class FactoryLoop {
1849
1927
  // Full task rendering is part of the durable spawn specification. It must
1850
1928
  // happen before a remote lifecycle is first claimed so takeover cannot
1851
1929
  // recover a persisted minimal triage task after a crash in this gap.
1852
- const durableRemoteDispatch = !dryRun && this.#fleet.placementLocality === 'remote';
1930
+ const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
1853
1931
  // Local dispatches need the same deterministic branch identity as remote
1854
1932
  // ones. Without it, every worker starts in the configured shared checkout
1855
1933
  // and concurrent issues can switch each other back to the base branch.
1856
1934
  const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
1857
- const lifecycleRunId = !dryRun && (durableRemoteDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
1935
+ const lifecycleRunId = !dryRun && (durableDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
1858
1936
  if (lifecycleRunId) {
1859
1937
  dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
1860
1938
  isolateLocalWorktree,
1861
1939
  });
1862
1940
  }
1863
1941
  dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
1864
- if (durableRemoteDispatch) {
1942
+ if (durableDispatch) {
1865
1943
  const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun, lifecycleRunId);
1866
1944
  this.#consumePendingDispatchClarifications(dispatchDecision.issue);
1867
1945
  dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision);
@@ -1881,7 +1959,7 @@ export class FactoryLoop {
1881
1959
  return restored.result;
1882
1960
  }
1883
1961
  }
1884
- if (!durableRemoteDispatch)
1962
+ if (!durableDispatch)
1885
1963
  this.#consumePendingDispatchClarifications(dispatchDecision.issue);
1886
1964
  await this.#recordDispatchAttempt(dispatchDecision.issue);
1887
1965
  const record = batch.start(dispatchDecision, dryRun);
@@ -2057,7 +2135,7 @@ export class FactoryLoop {
2057
2135
  });
2058
2136
  }
2059
2137
  }
2060
- // Remote backends survive orchestrator restarts: re-adopt the agents recorded
2138
+ // Durable backends survive orchestrator restarts: re-adopt the agents recorded
2061
2139
  // in the durable lifecycle store, restore their full batch/spec association,
2062
2140
  // then reconcile once so exits that happened while this process was down are
2063
2141
  // handled instead of being dropped as unknown agents.
@@ -2097,8 +2175,9 @@ export class FactoryLoop {
2097
2175
  for (const agent of claim.lifecycle.agents) {
2098
2176
  const invocationId = agent.tracked.spec.invocationId;
2099
2177
  const node = agent.tracked.result?.node;
2100
- if (invocationId || node)
2178
+ if (invocationId || node || agent.tracked.result) {
2101
2179
  agents.push({ name: agent.name, invocationId, node });
2180
+ }
2102
2181
  }
2103
2182
  }
2104
2183
  // Migration fallback for registries written before durable lifecycle
@@ -2136,6 +2215,12 @@ export class FactoryLoop {
2136
2215
  this.#dispatchLifecycleEpochs.delete(key);
2137
2216
  this.#increment('dispatchLifecycleLeasesLost');
2138
2217
  const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
2218
+ if (lifecycle) {
2219
+ await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
2220
+ level: 'error',
2221
+ errorCode: 'lease_lost',
2222
+ });
2223
+ }
2139
2224
  if (lifecycle && !isTerminalDispatchLifecycle(lifecycle) && lifecycle.phase !== 'waiting-for-human') {
2140
2225
  this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(lifecycle));
2141
2226
  }
@@ -2171,10 +2256,86 @@ export class FactoryLoop {
2171
2256
  }
2172
2257
  this.#dispatchLifecycleEpochs.set(key, claim.lease.epoch);
2173
2258
  this.#scheduleDispatchLifecycleRenewal();
2259
+ if (claim.created) {
2260
+ await this.#reportLifecycle(claim.lifecycle, 'run.started');
2261
+ }
2174
2262
  return { created: claim.created, lifecycle: claim.lifecycle };
2175
2263
  }
2264
+ #usesDurableDispatchLifecycle() {
2265
+ return this.#fleet.durableOwnership ?? this.#fleet.placementLocality === 'remote';
2266
+ }
2267
+ async #report(input) {
2268
+ if (!this.#reporter)
2269
+ return;
2270
+ try {
2271
+ await this.#reporter.report(createFactoryCloudEventV1(input, {
2272
+ now: () => new Date(this.#clock.now()),
2273
+ }));
2274
+ }
2275
+ catch (error) {
2276
+ // A custom reporter is allowed through the public port, so defend the
2277
+ // orchestration path even if it violates the port's no-reject contract.
2278
+ this.#increment('factoryEventReportingFailures');
2279
+ this.#logger.warn?.('[factory] progress reporter rejected an event', {
2280
+ eventType: input.type,
2281
+ errorClass: telemetryErrorClass(error),
2282
+ });
2283
+ }
2284
+ }
2285
+ async #reportLifecycle(lifecycle, type, options = {}) {
2286
+ await this.#report({
2287
+ type,
2288
+ level: options.level ?? (type === 'run.failed' || type === 'factory.anomaly' ? 'error' : 'info'),
2289
+ runId: lifecycle.runId,
2290
+ phase: lifecycle.phase,
2291
+ status: telemetryRunStatus(lifecycle.phase),
2292
+ run: {
2293
+ source: githubIssuePathParts(lifecycle.issue.path) ? 'github' : 'linear',
2294
+ repository: lifecycle.decision.routes[0]?.repo,
2295
+ issueKey: lifecycle.issue.key,
2296
+ recipe: lifecycle.decision.scope,
2297
+ },
2298
+ attributes: {
2299
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
2300
+ component: 'orchestrator',
2301
+ operation: 'save_lifecycle',
2302
+ previousPhase: options.previousPhase,
2303
+ errorCode: options.errorCode,
2304
+ dryRun: lifecycle.dryRun,
2305
+ trackedAgents: lifecycle.agents.length,
2306
+ },
2307
+ });
2308
+ }
2309
+ async #reportAgent(record, tracked, type, options = {}) {
2310
+ const lifecycle = await this.#state
2311
+ .getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
2312
+ .catch(() => undefined);
2313
+ if (!lifecycle)
2314
+ return;
2315
+ await this.#report({
2316
+ type,
2317
+ runId: lifecycle.runId,
2318
+ phase: lifecycle.phase,
2319
+ status: telemetryRunStatus(lifecycle.phase),
2320
+ run: {
2321
+ source: githubIssuePathParts(record.issue.path) ? 'github' : 'linear',
2322
+ repository: tracked.spec.repo,
2323
+ issueKey: record.issue.key,
2324
+ recipe: record.decision.scope,
2325
+ },
2326
+ attributes: {
2327
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
2328
+ component: 'fleet',
2329
+ operation: type.slice('agent.'.length),
2330
+ agentRole: tracked.spec.role,
2331
+ invocationId: tracked.spec.invocationId,
2332
+ locality: tracked.result?.locality ?? this.#fleet.placementLocality,
2333
+ releaseReason: factoryCloudReleaseReasonV1(options.releaseReason),
2334
+ },
2335
+ });
2336
+ }
2176
2337
  async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set()) {
2177
- if (record.dryRun || this.#fleet.placementLocality !== 'remote')
2338
+ if (record.dryRun || !this.#usesDurableDispatchLifecycle())
2178
2339
  return true;
2179
2340
  const key = issueKey(record.issue);
2180
2341
  const epoch = this.#dispatchLifecycleEpochs.get(key);
@@ -2195,9 +2356,20 @@ export class FactoryLoop {
2195
2356
  if (!saved) {
2196
2357
  this.#dispatchLifecycleEpochs.delete(key);
2197
2358
  this.#increment('dispatchLifecycleFencesRejected');
2359
+ await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
2360
+ level: 'error',
2361
+ errorCode: 'fence_rejected',
2362
+ });
2198
2363
  this.#scheduleDispatchLifecycleRetry(record);
2199
2364
  return false;
2200
2365
  }
2366
+ if (previous?.phase !== lifecycle.phase) {
2367
+ await this.#reportLifecycle(lifecycle, lifecycle.phase === 'complete'
2368
+ ? 'run.succeeded'
2369
+ : lifecycle.phase === 'abandoned'
2370
+ ? 'run.cancelled'
2371
+ : 'run.phase_changed', { previousPhase: previous?.phase });
2372
+ }
2201
2373
  if (isTerminalDispatchLifecycle(lifecycle)) {
2202
2374
  this.#dispatchLifecycleEpochs.delete(key);
2203
2375
  }
@@ -2229,7 +2401,7 @@ export class FactoryLoop {
2229
2401
  this.#dispatchLifecycleRetryTimers.set(key, timer);
2230
2402
  }
2231
2403
  #scheduleReleaseRetry(record, reason) {
2232
- if (this.#fleet.placementLocality === 'remote') {
2404
+ if (this.#usesDurableDispatchLifecycle()) {
2233
2405
  this.#scheduleDispatchLifecycleRetry(record);
2234
2406
  return;
2235
2407
  }
@@ -2458,14 +2630,14 @@ export class FactoryLoop {
2458
2630
  this.#scheduleReleaseRetry(record, reason);
2459
2631
  return false;
2460
2632
  }
2461
- const next = this.#fleet.placementLocality === 'remote' ? undefined : batch.complete(record.issue);
2633
+ const next = this.#usesDurableDispatchLifecycle() ? undefined : batch.complete(record.issue);
2462
2634
  this.#localReleaseCheckpoints.delete(releaseKey);
2463
2635
  if (next)
2464
2636
  await this.dispatch(next.decision, { dryRun: next.dryRun });
2465
2637
  // Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
2466
2638
  // the babysitter's durable ownership/wake/critical state while that epoch
2467
2639
  // is still valid so a later reopened issue cannot inherit a stale PR owner.
2468
- if (this.#fleet.placementLocality === 'remote' && this.#config.babysitter.enabled) {
2640
+ if (this.#usesDurableDispatchLifecycle() && this.#config.babysitter.enabled) {
2469
2641
  await this.#cancelBabysitterWake(issueKey(record.issue));
2470
2642
  }
2471
2643
  if (!await this.#saveDispatchLifecycle(record, 'complete'))
@@ -2480,7 +2652,7 @@ export class FactoryLoop {
2480
2652
  return await this.#assertIssueDispatchLifecycleOwner(record.issue);
2481
2653
  }
2482
2654
  async #assertIssueDispatchLifecycleOwner(issue) {
2483
- if (this.#fleet.placementLocality !== 'remote')
2655
+ if (!this.#usesDurableDispatchLifecycle())
2484
2656
  return true;
2485
2657
  const key = issueKey(issue);
2486
2658
  const epoch = this.#dispatchLifecycleEpochs.get(key);
@@ -2947,6 +3119,19 @@ export class FactoryLoop {
2947
3119
  await mkdir(dirname(path), { recursive: true });
2948
3120
  await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
2949
3121
  await this.#writeInFlightRegistry(registryPath, path);
3122
+ const batch = this.#batchView;
3123
+ await this.#report({
3124
+ type: 'instance.heartbeat',
3125
+ attributes: {
3126
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
3127
+ mode: status,
3128
+ component: 'orchestrator',
3129
+ operation: 'heartbeat',
3130
+ activeRuns: batch?.inFlight.length,
3131
+ queuedRuns: batch?.queued.length,
3132
+ trackedAgents: batch?.inFlight.reduce((count, record) => count + record.agents.size, 0),
3133
+ },
3134
+ });
2950
3135
  }
2951
3136
  async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
2952
3137
  const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
@@ -3150,7 +3335,9 @@ export class FactoryLoop {
3150
3335
  async #releaseAndTerminateAgents(agents, reason, context) {
3151
3336
  const failed = [];
3152
3337
  const protectedPids = await this.#protectedPids();
3338
+ const batch = this.#batchView;
3153
3339
  for (const [agentName, tracked] of agents) {
3340
+ const record = batch?.getIssueByAgent(agentName);
3154
3341
  if (context === 'stop') {
3155
3342
  await this.#refreshStoppingHeartbeat();
3156
3343
  }
@@ -3182,10 +3369,23 @@ export class FactoryLoop {
3182
3369
  }
3183
3370
  try {
3184
3371
  await this.#fleet.release(agentName, reason);
3372
+ if (record)
3373
+ await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
3185
3374
  }
3186
3375
  catch (error) {
3187
3376
  failed.push(agentName);
3188
3377
  this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
3378
+ if (record) {
3379
+ const lifecycle = await this.#state
3380
+ .getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
3381
+ .catch(() => undefined);
3382
+ if (lifecycle) {
3383
+ await this.#reportLifecycle(lifecycle, 'factory.failure', {
3384
+ level: 'error',
3385
+ errorCode: 'release_failed',
3386
+ });
3387
+ }
3388
+ }
3189
3389
  }
3190
3390
  if (context === 'stop') {
3191
3391
  await this.#refreshStoppingHeartbeat();
@@ -3423,6 +3623,9 @@ export class FactoryLoop {
3423
3623
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
3424
3624
  throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`);
3425
3625
  }
3626
+ const adopted = record.agents.get(spec.name);
3627
+ if (adopted)
3628
+ await this.#reportAgent(record, adopted, 'agent.adopted');
3426
3629
  return { name: spec.name };
3427
3630
  }
3428
3631
  await this.#prepareAgentWorktree(record, spec);
@@ -3451,6 +3654,9 @@ export class FactoryLoop {
3451
3654
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
3452
3655
  throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`);
3453
3656
  }
3657
+ const spawned = record.agents.get(result.name);
3658
+ if (spawned)
3659
+ await this.#reportAgent(record, spawned, 'agent.spawned');
3454
3660
  return { name: result.name };
3455
3661
  }
3456
3662
  async #handleAgentExit(name, reason) {
@@ -3469,6 +3675,19 @@ export class FactoryLoop {
3469
3675
  const batch = await this.#batch();
3470
3676
  const record = batch.getIssueByAgent(name);
3471
3677
  if (!record) {
3678
+ if (/^ar-\d+-/u.test(name)) {
3679
+ await this.#report({
3680
+ type: 'factory.anomaly',
3681
+ level: 'error',
3682
+ attributes: {
3683
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
3684
+ component: 'fleet',
3685
+ operation: 'agent_exit',
3686
+ errorCode: 'unowned_agent',
3687
+ count: 1,
3688
+ },
3689
+ });
3690
+ }
3472
3691
  return;
3473
3692
  }
3474
3693
  if (!await this.#assertDispatchLifecycleOwner(record)) {
@@ -3487,7 +3706,9 @@ export class FactoryLoop {
3487
3706
  return;
3488
3707
  }
3489
3708
  const exiting = record.agents.get(name);
3490
- if (this.#fleet.placementLocality === 'remote') {
3709
+ if (exiting)
3710
+ await this.#reportAgent(record, exiting, 'agent.exited', { releaseReason: reason });
3711
+ if (this.#usesDurableDispatchLifecycle()) {
3491
3712
  const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
3492
3713
  if (lifecycle?.phase === 'parking') {
3493
3714
  this.#increment('clarificationParkingExitsSuppressed');
@@ -4176,6 +4397,7 @@ export class FactoryLoop {
4176
4397
  await this.#persistBabysitterSession(record.issue, ref, tracked);
4177
4398
  }
4178
4399
  }
4400
+ await this.#reportAgent(record, tracked, 'agent.resumed');
4179
4401
  }
4180
4402
  async #handleDeliveryFailed(info) {
4181
4403
  const critical = await this.#state.consumeCritical(this.#workspaceId, info.msgId ?? '');
@@ -4185,7 +4407,7 @@ export class FactoryLoop {
4185
4407
  }
4186
4408
  const record = (await this.#batch()).getIssueByAgent(info.to);
4187
4409
  const issue = critical.issue ?? record?.issue;
4188
- const error = new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`);
4410
+ const error = Object.assign(new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`), { code: 'fleet_delivery_failed' });
4189
4411
  this.#error(error, issue);
4190
4412
  if (isTerminalDeliveryFailure(info.reason)) {
4191
4413
  this.#increment('criticalDeliveryTerminalFailures');
@@ -5661,7 +5883,7 @@ export class FactoryLoop {
5661
5883
  }
5662
5884
  async #cancelBabysitterWake(issueIdentity) {
5663
5885
  const issue = this.#babysitterIssueRefs.get(issueIdentity);
5664
- const mayClearDurable = this.#fleet.placementLocality !== 'remote'
5886
+ const mayClearDurable = !this.#usesDurableDispatchLifecycle()
5665
5887
  || Boolean(issue && await this.#assertIssueDispatchLifecycleOwner(issue));
5666
5888
  for (const [key, state] of this.#babysitterWakeStates) {
5667
5889
  if (issueKey(state.issue) !== issueIdentity)
@@ -5922,19 +6144,7 @@ export class FactoryLoop {
5922
6144
  },
5923
6145
  };
5924
6146
  let targets;
5925
- if (this.#fleet.promptDelivery === 'pty') {
5926
- if (!this.#fleet.sendInput) {
5927
- throw new Error('Fleet client advertises PTY prompt delivery without raw input support');
5928
- }
5929
- // Internal agents are harness-owned PTYs, not Relaycast identities.
5930
- // Stage the validated metadata-only wake directly through the broker so
5931
- // a missing Relaycast registration cannot strand a live babysitter.
5932
- // The CR remains a separate write: a critical-section begin can arrive
5933
- // while this write is in flight and must be able to defer submission.
5934
- await this.#fleet.sendInput(input.to, input.text);
5935
- targets = [input.to];
5936
- }
5937
- else if (!this.#fleet.waitForInjected) {
6147
+ if (!this.#fleet.waitForInjected) {
5938
6148
  await this.#fleet.sendMessage(input);
5939
6149
  if (this.#stopping || state.cancelled) {
5940
6150
  state.deliveringKinds = undefined;
@@ -6403,7 +6613,12 @@ export class FactoryLoop {
6403
6613
  prNumber: prRef.prNumber,
6404
6614
  babysitter: spawned.name,
6405
6615
  });
6406
- if (this.#fleet.waitForInjected) {
6616
+ // Internal PTY spawns receive `task` atomically in spawnPty. Re-sending
6617
+ // the same task through Relaycast before the worker has registered its
6618
+ // messaging identity can fail an otherwise successful spawn and erase
6619
+ // valid babysitter ownership. Remote placement still needs the explicit,
6620
+ // confirmed follow-up injection used by its spawn protocol.
6621
+ if (this.#fleet.waitForInjected && this.#fleet.placementLocality !== 'local') {
6407
6622
  const input = {
6408
6623
  to: tracked?.result?.name ?? spawned.name,
6409
6624
  text: task,
@@ -6637,7 +6852,7 @@ export class FactoryLoop {
6637
6852
  }
6638
6853
  const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
6639
6854
  releaseReasonForRetry = releaseReason;
6640
- if (this.#fleet.placementLocality === 'remote') {
6855
+ if (this.#usesDurableDispatchLifecycle()) {
6641
6856
  // Durable capacity is released as soon as terminal writeback is
6642
6857
  // acknowledged. Agent cleanup remains fenced/retryable in `releasing`.
6643
6858
  const batch = await this.#batch();
@@ -6667,7 +6882,7 @@ export class FactoryLoop {
6667
6882
  this.#babysitterPr.delete(completionKey);
6668
6883
  await this.#cancelBabysitterWake(completionKey);
6669
6884
  const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
6670
- if (this.#fleet.placementLocality !== 'remote' || (durable && isTerminalDispatchLifecycle(durable))) {
6885
+ if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
6671
6886
  for (const publishedKey of this.#publishedPullRequests.keys()) {
6672
6887
  if (publishedKey.startsWith(`${completionKey}:`))
6673
6888
  this.#publishedPullRequests.delete(publishedKey);
@@ -6677,7 +6892,16 @@ export class FactoryLoop {
6677
6892
  }
6678
6893
  #emit(event, payload) {
6679
6894
  for (const listener of this.#listeners.get(event) ?? []) {
6680
- listener(payload);
6895
+ try {
6896
+ listener(payload);
6897
+ }
6898
+ catch (error) {
6899
+ this.#increment('factoryEventListenerFailures');
6900
+ this.#logger.warn?.('[factory] event listener failed', {
6901
+ event,
6902
+ errorClass: error instanceof Error ? error.name : 'Error',
6903
+ });
6904
+ }
6681
6905
  }
6682
6906
  }
6683
6907
  #error(error, issue) {
@@ -6692,6 +6916,48 @@ export class FactoryLoop {
6692
6916
  ...details,
6693
6917
  ...(issue ? { issue: issue.key } : {}),
6694
6918
  });
6919
+ const failureCode = telemetryCategory(error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
6920
+ ? error.code
6921
+ : 'factory_error');
6922
+ const failureClass = telemetryErrorClass(error);
6923
+ void (async () => {
6924
+ try {
6925
+ const lifecycle = issue
6926
+ ? await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(issue)).catch(() => undefined)
6927
+ : undefined;
6928
+ if (lifecycle) {
6929
+ await this.#reportLifecycle(lifecycle, 'factory.failure', {
6930
+ level: 'error',
6931
+ errorCode: failureCode,
6932
+ });
6933
+ return;
6934
+ }
6935
+ await this.#report({
6936
+ type: 'factory.failure',
6937
+ level: 'error',
6938
+ attributes: {
6939
+ backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
6940
+ component: 'orchestrator',
6941
+ operation: 'error',
6942
+ errorClass: failureClass,
6943
+ errorCode: failureCode,
6944
+ },
6945
+ });
6946
+ }
6947
+ catch (telemetryError) {
6948
+ // This is the terminal guard for the intentionally floating telemetry
6949
+ // task. Never log raw messages or allow a custom logger to turn this
6950
+ // best-effort path into an unhandled rejection.
6951
+ try {
6952
+ this.#logger.warn?.('[factory] failed to report failure telemetry', {
6953
+ errorClass: telemetryErrorClass(telemetryError),
6954
+ });
6955
+ }
6956
+ catch {
6957
+ // Reporting and logging are both non-critical to orchestration.
6958
+ }
6959
+ }
6960
+ })();
6695
6961
  this.#emit('error', { error, ...details, issue });
6696
6962
  }
6697
6963
  #surfaceEscalationDeliveryFailure(kind, issue, correlationId, reason, cause) {
@@ -7755,7 +8021,7 @@ export class FactoryLoop {
7755
8021
  }
7756
8022
  let lifecycleDecision = waiting.decision;
7757
8023
  let promotedLifecycle;
7758
- if (!waiting.dryRun && this.#fleet.placementLocality === 'remote') {
8024
+ if (!waiting.dryRun && this.#usesDurableDispatchLifecycle()) {
7759
8025
  try {
7760
8026
  const claim = await this.#claimDispatchLifecycle(waiting.decision, false);
7761
8027
  const lifecycleKey = issueKey(waiting.issue);
@@ -10046,9 +10312,36 @@ const failedIterationReport = (error, dryRun) => {
10046
10312
  };
10047
10313
  const isRegistrationLagInjectionError = (error) => {
10048
10314
  const { errorMessage } = describeError(error);
10049
- return /recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
10315
+ return /agent_not_found|recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
10050
10316
  .test(errorMessage);
10051
10317
  };
10318
+ const telemetryRunStatus = (phase) => {
10319
+ switch (phase) {
10320
+ case 'queued':
10321
+ return 'queued';
10322
+ case 'retryable':
10323
+ return 'blocked';
10324
+ case 'parking':
10325
+ case 'waiting-for-human':
10326
+ return 'waiting';
10327
+ case 'complete':
10328
+ return 'succeeded';
10329
+ case 'abandoned':
10330
+ return 'cancelled';
10331
+ default:
10332
+ return 'running';
10333
+ }
10334
+ };
10335
+ const telemetryCategory = (value) => {
10336
+ if (!value)
10337
+ return undefined;
10338
+ const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
10339
+ return normalized.slice(0, 120) || undefined;
10340
+ };
10341
+ const telemetryErrorClass = (error) => {
10342
+ const name = error instanceof Error ? error.name : '';
10343
+ return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
10344
+ };
10052
10345
  const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
10053
10346
  const retryOnTimeout = async (fn, opts) => {
10054
10347
  let lastError;