@agent-relay/factory 0.1.32 → 0.1.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/dist/cli/fleet.d.ts +4 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +113 -6
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +88 -8
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +10 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +9 -2
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +39 -1
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +1 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +1 -0
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/observability/cloud-reporter.d.ts +46 -0
- package/dist/observability/cloud-reporter.d.ts.map +1 -0
- package/dist/observability/cloud-reporter.js +371 -0
- package/dist/observability/cloud-reporter.js.map +1 -0
- package/dist/observability/events.d.ts +1275 -0
- package/dist/observability/events.d.ts.map +1 -0
- package/dist/observability/events.js +229 -0
- package/dist/observability/events.js.map +1 -0
- package/dist/observability/index.d.ts +5 -0
- package/dist/observability/index.d.ts.map +1 -0
- package/dist/observability/index.js +5 -0
- package/dist/observability/index.js.map +1 -0
- package/dist/observability/instance-identity.d.ts +7 -0
- package/dist/observability/instance-identity.d.ts.map +1 -0
- package/dist/observability/instance-identity.js +59 -0
- package/dist/observability/instance-identity.js.map +1 -0
- package/dist/observability/outbox.d.ts +42 -0
- package/dist/observability/outbox.d.ts.map +1 -0
- package/dist/observability/outbox.js +305 -0
- package/dist/observability/outbox.js.map +1 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +336 -43
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +3 -4
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -0
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/observability.d.ts +19 -0
- package/dist/ports/observability.d.ts.map +1 -0
- package/dist/ports/observability.js +2 -0
- package/dist/ports/observability.js.map +1 -0
- package/dist/triage/schema.d.ts +14 -14
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -1
|
@@ -20,6 +20,7 @@ import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../write
|
|
|
20
20
|
import { issueKey } from './batch-tracker.js';
|
|
21
21
|
import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
|
|
22
22
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
23
|
+
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
23
24
|
class ClarificationWakeLeaseLostError extends Error {
|
|
24
25
|
}
|
|
25
26
|
class ClarificationQuestionDeliveryLeaseLostError extends Error {
|
|
@@ -119,6 +120,7 @@ export class FactoryLoop {
|
|
|
119
120
|
#workspaceId;
|
|
120
121
|
#relayflows;
|
|
121
122
|
#worktrees;
|
|
123
|
+
#reporter;
|
|
122
124
|
#batchView;
|
|
123
125
|
#batchReady;
|
|
124
126
|
#listeners = new Map();
|
|
@@ -263,6 +265,7 @@ export class FactoryLoop {
|
|
|
263
265
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
264
266
|
this.#relayflows = ports.relayflows;
|
|
265
267
|
this.#worktrees = ports.worktrees;
|
|
268
|
+
this.#reporter = ports.reporter;
|
|
266
269
|
this.#state = ports.stateStore ?? new InMemoryStateStore({
|
|
267
270
|
batchSize: config.batchSize,
|
|
268
271
|
agentQuestionDedupeLimit: AGENT_QUESTION_DEDUPE_LIMIT,
|
|
@@ -1362,6 +1365,7 @@ export class FactoryLoop {
|
|
|
1362
1365
|
]);
|
|
1363
1366
|
const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
|
|
1364
1367
|
const activeIssueIdentities = new Set();
|
|
1368
|
+
const legacyUnownedAgentsByIssue = new Map();
|
|
1365
1369
|
for (const [, lifecycle] of lifecycles) {
|
|
1366
1370
|
if (isTerminalDispatchLifecycle(lifecycle))
|
|
1367
1371
|
continue;
|
|
@@ -1378,12 +1382,26 @@ export class FactoryLoop {
|
|
|
1378
1382
|
if (!onlineAgents.has(agent.name) || !agent.issue)
|
|
1379
1383
|
continue;
|
|
1380
1384
|
const identity = githubIssueRefIdentity(agent.issue);
|
|
1381
|
-
if (identity)
|
|
1385
|
+
if (!identity)
|
|
1386
|
+
continue;
|
|
1387
|
+
const isLegacyLocalWorker = this.#usesDurableDispatchLifecycle() &&
|
|
1388
|
+
this.#fleet.placementLocality === 'local' &&
|
|
1389
|
+
!agent.node &&
|
|
1390
|
+
!agent.invocationId &&
|
|
1391
|
+
!activeIssueIdentities.has(identity);
|
|
1392
|
+
if (isLegacyLocalWorker) {
|
|
1393
|
+
const agents = legacyUnownedAgentsByIssue.get(identity) ?? [];
|
|
1394
|
+
agents.push(agent);
|
|
1395
|
+
legacyUnownedAgentsByIssue.set(identity, agents);
|
|
1396
|
+
}
|
|
1397
|
+
else {
|
|
1382
1398
|
activeIssueIdentities.add(identity);
|
|
1399
|
+
}
|
|
1383
1400
|
}
|
|
1384
1401
|
return {
|
|
1385
1402
|
activeIssueIdentities,
|
|
1386
1403
|
onlineAgentNames: onlineAgents,
|
|
1404
|
+
legacyUnownedAgentsByIssue,
|
|
1387
1405
|
};
|
|
1388
1406
|
}
|
|
1389
1407
|
catch (error) {
|
|
@@ -1405,9 +1423,14 @@ export class FactoryLoop {
|
|
|
1405
1423
|
labels.has('factory:human-review'))
|
|
1406
1424
|
return false;
|
|
1407
1425
|
const identity = githubIssueRefIdentity(issueRef(issue));
|
|
1426
|
+
const legacyUnownedAgents = identity
|
|
1427
|
+
? (context.legacyUnownedAgentsByIssue.get(identity) ?? [])
|
|
1428
|
+
.filter((agent) => githubAgentNameMatchesIssue(agent.name, issue))
|
|
1429
|
+
: [];
|
|
1430
|
+
const legacyUnownedAgentNames = new Set(legacyUnownedAgents.map((agent) => agent.name));
|
|
1408
1431
|
if (!identity ||
|
|
1409
1432
|
context.activeIssueIdentities.has(identity) ||
|
|
1410
|
-
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue))) {
|
|
1433
|
+
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue) && !legacyUnownedAgentNames.has(name))) {
|
|
1411
1434
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1412
1435
|
return false;
|
|
1413
1436
|
}
|
|
@@ -1447,7 +1470,7 @@ export class FactoryLoop {
|
|
|
1447
1470
|
if (openPr) {
|
|
1448
1471
|
let adopted = false;
|
|
1449
1472
|
try {
|
|
1450
|
-
adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr);
|
|
1473
|
+
adopted = await this.#adoptOrphanedGithubPullRequest(issue, openPr, legacyUnownedAgents);
|
|
1451
1474
|
}
|
|
1452
1475
|
catch (error) {
|
|
1453
1476
|
this.#increment('githubOrphanedPullRequestAdoptionFailures');
|
|
@@ -1468,6 +1491,14 @@ export class FactoryLoop {
|
|
|
1468
1491
|
});
|
|
1469
1492
|
return false;
|
|
1470
1493
|
}
|
|
1494
|
+
// A pre-durable local Factory may have left live, registry-proven workers
|
|
1495
|
+
// without a lifecycle record. They are safe to adopt only once their open
|
|
1496
|
+
// PR proves which dispatch they own. Without that proof, preserve the issue
|
|
1497
|
+
// and workers instead of redispatching duplicate agents.
|
|
1498
|
+
if (legacyUnownedAgents.length > 0) {
|
|
1499
|
+
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1500
|
+
return false;
|
|
1501
|
+
}
|
|
1471
1502
|
try {
|
|
1472
1503
|
if (providerStatus === 'in-progress') {
|
|
1473
1504
|
await this.#githubWriteback.setStatus(issue, 'ready');
|
|
@@ -1504,7 +1535,7 @@ export class FactoryLoop {
|
|
|
1504
1535
|
allowLegacyGithubBranch: true,
|
|
1505
1536
|
});
|
|
1506
1537
|
}
|
|
1507
|
-
async #adoptOrphanedGithubPullRequest(issue, pr) {
|
|
1538
|
+
async #adoptOrphanedGithubPullRequest(issue, pr, legacyUnownedAgents = []) {
|
|
1508
1539
|
const headRef = pr.headRef;
|
|
1509
1540
|
if (!headRef)
|
|
1510
1541
|
return false;
|
|
@@ -1543,14 +1574,14 @@ export class FactoryLoop {
|
|
|
1543
1574
|
: spec),
|
|
1544
1575
|
};
|
|
1545
1576
|
}
|
|
1546
|
-
const
|
|
1577
|
+
const durableAdoption = this.#usesDurableDispatchLifecycle();
|
|
1547
1578
|
const publishedPr = {
|
|
1548
1579
|
repo: pr.repo,
|
|
1549
1580
|
number: pr.prNumber,
|
|
1550
1581
|
url: pr.url ?? `https://github.com/${pr.repo}/pull/${pr.prNumber}`,
|
|
1551
1582
|
headRef,
|
|
1552
1583
|
};
|
|
1553
|
-
if (
|
|
1584
|
+
if (durableAdoption) {
|
|
1554
1585
|
const claim = await this.#claimDispatchLifecycle(decision, false, randomUUID(), {
|
|
1555
1586
|
phase: 'published',
|
|
1556
1587
|
pullRequest: publishedPr,
|
|
@@ -1566,7 +1597,7 @@ export class FactoryLoop {
|
|
|
1566
1597
|
const batch = await this.#batch();
|
|
1567
1598
|
const record = batch.start(decision, false);
|
|
1568
1599
|
if (!record) {
|
|
1569
|
-
if (
|
|
1600
|
+
if (durableAdoption) {
|
|
1570
1601
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
|
|
1571
1602
|
if (durable) {
|
|
1572
1603
|
this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(durable));
|
|
@@ -1576,6 +1607,48 @@ export class FactoryLoop {
|
|
|
1576
1607
|
}
|
|
1577
1608
|
return false;
|
|
1578
1609
|
}
|
|
1610
|
+
if (durableAdoption && legacyUnownedAgents.length > 0) {
|
|
1611
|
+
const specsByName = new Map(dispatchSpecs(record.decision).map((spec) => [spec.name, spec]));
|
|
1612
|
+
const initialBabysitter = babysitterSpec(issue, this.#config, route);
|
|
1613
|
+
const sharedCheckout = record.decision.implementers.find((candidate) => candidate.repo === initialBabysitter.repo && candidate.baseClonePath && candidate.clonePath);
|
|
1614
|
+
const legacyBabysitter = {
|
|
1615
|
+
...initialBabysitter,
|
|
1616
|
+
...(sharedCheckout
|
|
1617
|
+
? {
|
|
1618
|
+
baseClonePath: sharedCheckout.baseClonePath,
|
|
1619
|
+
clonePath: sharedCheckout.clonePath,
|
|
1620
|
+
...(headRef ? { branch: headRef } : {}),
|
|
1621
|
+
...(sharedCheckout.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
1622
|
+
}
|
|
1623
|
+
: {}),
|
|
1624
|
+
ownedPullRequest: { repo: pr.repo, number: pr.prNumber, path: pr.path },
|
|
1625
|
+
};
|
|
1626
|
+
specsByName.set(legacyBabysitter.name, legacyBabysitter);
|
|
1627
|
+
const adopted = [];
|
|
1628
|
+
for (const agent of legacyUnownedAgents) {
|
|
1629
|
+
const spec = specsByName.get(agent.name);
|
|
1630
|
+
if (!spec)
|
|
1631
|
+
continue;
|
|
1632
|
+
const invocationId = batch.invocationIdFor(record.issue, spec);
|
|
1633
|
+
batch.recordSpawn(record, spec, invocationId, {
|
|
1634
|
+
name: agent.name,
|
|
1635
|
+
sessionRef: agent.sessionRef,
|
|
1636
|
+
pids: agent.pids,
|
|
1637
|
+
locality: 'local',
|
|
1638
|
+
});
|
|
1639
|
+
adopted.push({ name: agent.name, invocationId });
|
|
1640
|
+
}
|
|
1641
|
+
if (adopted.length > 0) {
|
|
1642
|
+
this.#fleet.hydrateTracked?.(adopted);
|
|
1643
|
+
await this.#saveDispatchLifecycle(record, 'published', publishedPr);
|
|
1644
|
+
for (const agent of adopted) {
|
|
1645
|
+
this.#increment('legacyLocalWorkersAdopted');
|
|
1646
|
+
const tracked = record.agents.get(agent.name);
|
|
1647
|
+
if (tracked)
|
|
1648
|
+
await this.#reportAgent(record, tracked, 'agent.adopted');
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1579
1652
|
await this.#ensureBabysitter(record, {
|
|
1580
1653
|
repo: pr.repo,
|
|
1581
1654
|
prNumber: pr.prNumber,
|
|
@@ -1585,7 +1658,7 @@ export class FactoryLoop {
|
|
|
1585
1658
|
});
|
|
1586
1659
|
const babysitter = [...record.agents.values()].find((tracked) => tracked.spec.role === 'babysitter');
|
|
1587
1660
|
if (!babysitter) {
|
|
1588
|
-
if (
|
|
1661
|
+
if (durableAdoption)
|
|
1589
1662
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
1590
1663
|
else
|
|
1591
1664
|
batch.abandon(record.issue);
|
|
@@ -1593,10 +1666,15 @@ export class FactoryLoop {
|
|
|
1593
1666
|
}
|
|
1594
1667
|
record.result = {
|
|
1595
1668
|
issue: record.issue,
|
|
1596
|
-
agents: [
|
|
1669
|
+
agents: [...record.agents.values()].map((tracked) => ({
|
|
1670
|
+
name: tracked.result?.name ?? tracked.spec.name,
|
|
1671
|
+
role: tracked.spec.role,
|
|
1672
|
+
})),
|
|
1597
1673
|
dryRun: false,
|
|
1598
1674
|
};
|
|
1599
1675
|
await this.#writeInFlightRegistry();
|
|
1676
|
+
if (durableAdoption)
|
|
1677
|
+
await this.#saveDispatchLifecycle(record, 'running', publishedPr);
|
|
1600
1678
|
this.#increment('githubOrphanedPullRequestsAdopted');
|
|
1601
1679
|
return true;
|
|
1602
1680
|
}
|
|
@@ -1778,7 +1856,7 @@ export class FactoryLoop {
|
|
|
1778
1856
|
if (existingRecord?.result) {
|
|
1779
1857
|
return existingRecord.result;
|
|
1780
1858
|
}
|
|
1781
|
-
if (!dryRun && this.#
|
|
1859
|
+
if (!dryRun && this.#usesDurableDispatchLifecycle()) {
|
|
1782
1860
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue));
|
|
1783
1861
|
if (durable && !isTerminalDispatchLifecycle(durable)) {
|
|
1784
1862
|
if (durable.result && this.#dispatchLifecycleEpochs.has(issueKey(decision.issue))) {
|
|
@@ -1849,19 +1927,19 @@ export class FactoryLoop {
|
|
|
1849
1927
|
// Full task rendering is part of the durable spawn specification. It must
|
|
1850
1928
|
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
1851
1929
|
// recover a persisted minimal triage task after a crash in this gap.
|
|
1852
|
-
const
|
|
1930
|
+
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
1853
1931
|
// Local dispatches need the same deterministic branch identity as remote
|
|
1854
1932
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
1855
1933
|
// and concurrent issues can switch each other back to the base branch.
|
|
1856
1934
|
const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
|
|
1857
|
-
const lifecycleRunId = !dryRun && (
|
|
1935
|
+
const lifecycleRunId = !dryRun && (durableDispatch || isolateLocalWorktree) ? randomUUID() : undefined;
|
|
1858
1936
|
if (lifecycleRunId) {
|
|
1859
1937
|
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
|
|
1860
1938
|
isolateLocalWorktree,
|
|
1861
1939
|
});
|
|
1862
1940
|
}
|
|
1863
1941
|
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
1864
|
-
if (
|
|
1942
|
+
if (durableDispatch) {
|
|
1865
1943
|
const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun, lifecycleRunId);
|
|
1866
1944
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
1867
1945
|
dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision);
|
|
@@ -1881,7 +1959,7 @@ export class FactoryLoop {
|
|
|
1881
1959
|
return restored.result;
|
|
1882
1960
|
}
|
|
1883
1961
|
}
|
|
1884
|
-
if (!
|
|
1962
|
+
if (!durableDispatch)
|
|
1885
1963
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
1886
1964
|
await this.#recordDispatchAttempt(dispatchDecision.issue);
|
|
1887
1965
|
const record = batch.start(dispatchDecision, dryRun);
|
|
@@ -2057,7 +2135,7 @@ export class FactoryLoop {
|
|
|
2057
2135
|
});
|
|
2058
2136
|
}
|
|
2059
2137
|
}
|
|
2060
|
-
//
|
|
2138
|
+
// Durable backends survive orchestrator restarts: re-adopt the agents recorded
|
|
2061
2139
|
// in the durable lifecycle store, restore their full batch/spec association,
|
|
2062
2140
|
// then reconcile once so exits that happened while this process was down are
|
|
2063
2141
|
// handled instead of being dropped as unknown agents.
|
|
@@ -2097,8 +2175,9 @@ export class FactoryLoop {
|
|
|
2097
2175
|
for (const agent of claim.lifecycle.agents) {
|
|
2098
2176
|
const invocationId = agent.tracked.spec.invocationId;
|
|
2099
2177
|
const node = agent.tracked.result?.node;
|
|
2100
|
-
if (invocationId || node)
|
|
2178
|
+
if (invocationId || node || agent.tracked.result) {
|
|
2101
2179
|
agents.push({ name: agent.name, invocationId, node });
|
|
2180
|
+
}
|
|
2102
2181
|
}
|
|
2103
2182
|
}
|
|
2104
2183
|
// Migration fallback for registries written before durable lifecycle
|
|
@@ -2136,6 +2215,12 @@ export class FactoryLoop {
|
|
|
2136
2215
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2137
2216
|
this.#increment('dispatchLifecycleLeasesLost');
|
|
2138
2217
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2218
|
+
if (lifecycle) {
|
|
2219
|
+
await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
|
|
2220
|
+
level: 'error',
|
|
2221
|
+
errorCode: 'lease_lost',
|
|
2222
|
+
});
|
|
2223
|
+
}
|
|
2139
2224
|
if (lifecycle && !isTerminalDispatchLifecycle(lifecycle) && lifecycle.phase !== 'waiting-for-human') {
|
|
2140
2225
|
this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(lifecycle));
|
|
2141
2226
|
}
|
|
@@ -2171,10 +2256,86 @@ export class FactoryLoop {
|
|
|
2171
2256
|
}
|
|
2172
2257
|
this.#dispatchLifecycleEpochs.set(key, claim.lease.epoch);
|
|
2173
2258
|
this.#scheduleDispatchLifecycleRenewal();
|
|
2259
|
+
if (claim.created) {
|
|
2260
|
+
await this.#reportLifecycle(claim.lifecycle, 'run.started');
|
|
2261
|
+
}
|
|
2174
2262
|
return { created: claim.created, lifecycle: claim.lifecycle };
|
|
2175
2263
|
}
|
|
2264
|
+
#usesDurableDispatchLifecycle() {
|
|
2265
|
+
return this.#fleet.durableOwnership ?? this.#fleet.placementLocality === 'remote';
|
|
2266
|
+
}
|
|
2267
|
+
async #report(input) {
|
|
2268
|
+
if (!this.#reporter)
|
|
2269
|
+
return;
|
|
2270
|
+
try {
|
|
2271
|
+
await this.#reporter.report(createFactoryCloudEventV1(input, {
|
|
2272
|
+
now: () => new Date(this.#clock.now()),
|
|
2273
|
+
}));
|
|
2274
|
+
}
|
|
2275
|
+
catch (error) {
|
|
2276
|
+
// A custom reporter is allowed through the public port, so defend the
|
|
2277
|
+
// orchestration path even if it violates the port's no-reject contract.
|
|
2278
|
+
this.#increment('factoryEventReportingFailures');
|
|
2279
|
+
this.#logger.warn?.('[factory] progress reporter rejected an event', {
|
|
2280
|
+
eventType: input.type,
|
|
2281
|
+
errorClass: telemetryErrorClass(error),
|
|
2282
|
+
});
|
|
2283
|
+
}
|
|
2284
|
+
}
|
|
2285
|
+
async #reportLifecycle(lifecycle, type, options = {}) {
|
|
2286
|
+
await this.#report({
|
|
2287
|
+
type,
|
|
2288
|
+
level: options.level ?? (type === 'run.failed' || type === 'factory.anomaly' ? 'error' : 'info'),
|
|
2289
|
+
runId: lifecycle.runId,
|
|
2290
|
+
phase: lifecycle.phase,
|
|
2291
|
+
status: telemetryRunStatus(lifecycle.phase),
|
|
2292
|
+
run: {
|
|
2293
|
+
source: githubIssuePathParts(lifecycle.issue.path) ? 'github' : 'linear',
|
|
2294
|
+
repository: lifecycle.decision.routes[0]?.repo,
|
|
2295
|
+
issueKey: lifecycle.issue.key,
|
|
2296
|
+
recipe: lifecycle.decision.scope,
|
|
2297
|
+
},
|
|
2298
|
+
attributes: {
|
|
2299
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
2300
|
+
component: 'orchestrator',
|
|
2301
|
+
operation: 'save_lifecycle',
|
|
2302
|
+
previousPhase: options.previousPhase,
|
|
2303
|
+
errorCode: options.errorCode,
|
|
2304
|
+
dryRun: lifecycle.dryRun,
|
|
2305
|
+
trackedAgents: lifecycle.agents.length,
|
|
2306
|
+
},
|
|
2307
|
+
});
|
|
2308
|
+
}
|
|
2309
|
+
async #reportAgent(record, tracked, type, options = {}) {
|
|
2310
|
+
const lifecycle = await this.#state
|
|
2311
|
+
.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
|
|
2312
|
+
.catch(() => undefined);
|
|
2313
|
+
if (!lifecycle)
|
|
2314
|
+
return;
|
|
2315
|
+
await this.#report({
|
|
2316
|
+
type,
|
|
2317
|
+
runId: lifecycle.runId,
|
|
2318
|
+
phase: lifecycle.phase,
|
|
2319
|
+
status: telemetryRunStatus(lifecycle.phase),
|
|
2320
|
+
run: {
|
|
2321
|
+
source: githubIssuePathParts(record.issue.path) ? 'github' : 'linear',
|
|
2322
|
+
repository: tracked.spec.repo,
|
|
2323
|
+
issueKey: record.issue.key,
|
|
2324
|
+
recipe: record.decision.scope,
|
|
2325
|
+
},
|
|
2326
|
+
attributes: {
|
|
2327
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
2328
|
+
component: 'fleet',
|
|
2329
|
+
operation: type.slice('agent.'.length),
|
|
2330
|
+
agentRole: tracked.spec.role,
|
|
2331
|
+
invocationId: tracked.spec.invocationId,
|
|
2332
|
+
locality: tracked.result?.locality ?? this.#fleet.placementLocality,
|
|
2333
|
+
releaseReason: factoryCloudReleaseReasonV1(options.releaseReason),
|
|
2334
|
+
},
|
|
2335
|
+
});
|
|
2336
|
+
}
|
|
2176
2337
|
async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set()) {
|
|
2177
|
-
if (record.dryRun || this.#
|
|
2338
|
+
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
2178
2339
|
return true;
|
|
2179
2340
|
const key = issueKey(record.issue);
|
|
2180
2341
|
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
@@ -2195,9 +2356,20 @@ export class FactoryLoop {
|
|
|
2195
2356
|
if (!saved) {
|
|
2196
2357
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2197
2358
|
this.#increment('dispatchLifecycleFencesRejected');
|
|
2359
|
+
await this.#reportLifecycle(lifecycle, 'factory.anomaly', {
|
|
2360
|
+
level: 'error',
|
|
2361
|
+
errorCode: 'fence_rejected',
|
|
2362
|
+
});
|
|
2198
2363
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
2199
2364
|
return false;
|
|
2200
2365
|
}
|
|
2366
|
+
if (previous?.phase !== lifecycle.phase) {
|
|
2367
|
+
await this.#reportLifecycle(lifecycle, lifecycle.phase === 'complete'
|
|
2368
|
+
? 'run.succeeded'
|
|
2369
|
+
: lifecycle.phase === 'abandoned'
|
|
2370
|
+
? 'run.cancelled'
|
|
2371
|
+
: 'run.phase_changed', { previousPhase: previous?.phase });
|
|
2372
|
+
}
|
|
2201
2373
|
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
2202
2374
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
2203
2375
|
}
|
|
@@ -2229,7 +2401,7 @@ export class FactoryLoop {
|
|
|
2229
2401
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
2230
2402
|
}
|
|
2231
2403
|
#scheduleReleaseRetry(record, reason) {
|
|
2232
|
-
if (this.#
|
|
2404
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
2233
2405
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
2234
2406
|
return;
|
|
2235
2407
|
}
|
|
@@ -2458,14 +2630,14 @@ export class FactoryLoop {
|
|
|
2458
2630
|
this.#scheduleReleaseRetry(record, reason);
|
|
2459
2631
|
return false;
|
|
2460
2632
|
}
|
|
2461
|
-
const next = this.#
|
|
2633
|
+
const next = this.#usesDurableDispatchLifecycle() ? undefined : batch.complete(record.issue);
|
|
2462
2634
|
this.#localReleaseCheckpoints.delete(releaseKey);
|
|
2463
2635
|
if (next)
|
|
2464
2636
|
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
2465
2637
|
// Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
|
|
2466
2638
|
// the babysitter's durable ownership/wake/critical state while that epoch
|
|
2467
2639
|
// is still valid so a later reopened issue cannot inherit a stale PR owner.
|
|
2468
|
-
if (this.#
|
|
2640
|
+
if (this.#usesDurableDispatchLifecycle() && this.#config.babysitter.enabled) {
|
|
2469
2641
|
await this.#cancelBabysitterWake(issueKey(record.issue));
|
|
2470
2642
|
}
|
|
2471
2643
|
if (!await this.#saveDispatchLifecycle(record, 'complete'))
|
|
@@ -2480,7 +2652,7 @@ export class FactoryLoop {
|
|
|
2480
2652
|
return await this.#assertIssueDispatchLifecycleOwner(record.issue);
|
|
2481
2653
|
}
|
|
2482
2654
|
async #assertIssueDispatchLifecycleOwner(issue) {
|
|
2483
|
-
if (this.#
|
|
2655
|
+
if (!this.#usesDurableDispatchLifecycle())
|
|
2484
2656
|
return true;
|
|
2485
2657
|
const key = issueKey(issue);
|
|
2486
2658
|
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
@@ -2947,6 +3119,19 @@ export class FactoryLoop {
|
|
|
2947
3119
|
await mkdir(dirname(path), { recursive: true });
|
|
2948
3120
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
2949
3121
|
await this.#writeInFlightRegistry(registryPath, path);
|
|
3122
|
+
const batch = this.#batchView;
|
|
3123
|
+
await this.#report({
|
|
3124
|
+
type: 'instance.heartbeat',
|
|
3125
|
+
attributes: {
|
|
3126
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
3127
|
+
mode: status,
|
|
3128
|
+
component: 'orchestrator',
|
|
3129
|
+
operation: 'heartbeat',
|
|
3130
|
+
activeRuns: batch?.inFlight.length,
|
|
3131
|
+
queuedRuns: batch?.queued.length,
|
|
3132
|
+
trackedAgents: batch?.inFlight.reduce((count, record) => count + record.agents.size, 0),
|
|
3133
|
+
},
|
|
3134
|
+
});
|
|
2950
3135
|
}
|
|
2951
3136
|
async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
|
|
2952
3137
|
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
@@ -3150,7 +3335,9 @@ export class FactoryLoop {
|
|
|
3150
3335
|
async #releaseAndTerminateAgents(agents, reason, context) {
|
|
3151
3336
|
const failed = [];
|
|
3152
3337
|
const protectedPids = await this.#protectedPids();
|
|
3338
|
+
const batch = this.#batchView;
|
|
3153
3339
|
for (const [agentName, tracked] of agents) {
|
|
3340
|
+
const record = batch?.getIssueByAgent(agentName);
|
|
3154
3341
|
if (context === 'stop') {
|
|
3155
3342
|
await this.#refreshStoppingHeartbeat();
|
|
3156
3343
|
}
|
|
@@ -3182,10 +3369,23 @@ export class FactoryLoop {
|
|
|
3182
3369
|
}
|
|
3183
3370
|
try {
|
|
3184
3371
|
await this.#fleet.release(agentName, reason);
|
|
3372
|
+
if (record)
|
|
3373
|
+
await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
|
|
3185
3374
|
}
|
|
3186
3375
|
catch (error) {
|
|
3187
3376
|
failed.push(agentName);
|
|
3188
3377
|
this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
|
|
3378
|
+
if (record) {
|
|
3379
|
+
const lifecycle = await this.#state
|
|
3380
|
+
.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
|
|
3381
|
+
.catch(() => undefined);
|
|
3382
|
+
if (lifecycle) {
|
|
3383
|
+
await this.#reportLifecycle(lifecycle, 'factory.failure', {
|
|
3384
|
+
level: 'error',
|
|
3385
|
+
errorCode: 'release_failed',
|
|
3386
|
+
});
|
|
3387
|
+
}
|
|
3388
|
+
}
|
|
3189
3389
|
}
|
|
3190
3390
|
if (context === 'stop') {
|
|
3191
3391
|
await this.#refreshStoppingHeartbeat();
|
|
@@ -3423,6 +3623,9 @@ export class FactoryLoop {
|
|
|
3423
3623
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
3424
3624
|
throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`);
|
|
3425
3625
|
}
|
|
3626
|
+
const adopted = record.agents.get(spec.name);
|
|
3627
|
+
if (adopted)
|
|
3628
|
+
await this.#reportAgent(record, adopted, 'agent.adopted');
|
|
3426
3629
|
return { name: spec.name };
|
|
3427
3630
|
}
|
|
3428
3631
|
await this.#prepareAgentWorktree(record, spec);
|
|
@@ -3451,6 +3654,9 @@ export class FactoryLoop {
|
|
|
3451
3654
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
3452
3655
|
throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`);
|
|
3453
3656
|
}
|
|
3657
|
+
const spawned = record.agents.get(result.name);
|
|
3658
|
+
if (spawned)
|
|
3659
|
+
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
3454
3660
|
return { name: result.name };
|
|
3455
3661
|
}
|
|
3456
3662
|
async #handleAgentExit(name, reason) {
|
|
@@ -3469,6 +3675,19 @@ export class FactoryLoop {
|
|
|
3469
3675
|
const batch = await this.#batch();
|
|
3470
3676
|
const record = batch.getIssueByAgent(name);
|
|
3471
3677
|
if (!record) {
|
|
3678
|
+
if (/^ar-\d+-/u.test(name)) {
|
|
3679
|
+
await this.#report({
|
|
3680
|
+
type: 'factory.anomaly',
|
|
3681
|
+
level: 'error',
|
|
3682
|
+
attributes: {
|
|
3683
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
3684
|
+
component: 'fleet',
|
|
3685
|
+
operation: 'agent_exit',
|
|
3686
|
+
errorCode: 'unowned_agent',
|
|
3687
|
+
count: 1,
|
|
3688
|
+
},
|
|
3689
|
+
});
|
|
3690
|
+
}
|
|
3472
3691
|
return;
|
|
3473
3692
|
}
|
|
3474
3693
|
if (!await this.#assertDispatchLifecycleOwner(record)) {
|
|
@@ -3487,7 +3706,9 @@ export class FactoryLoop {
|
|
|
3487
3706
|
return;
|
|
3488
3707
|
}
|
|
3489
3708
|
const exiting = record.agents.get(name);
|
|
3490
|
-
if (
|
|
3709
|
+
if (exiting)
|
|
3710
|
+
await this.#reportAgent(record, exiting, 'agent.exited', { releaseReason: reason });
|
|
3711
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
3491
3712
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
3492
3713
|
if (lifecycle?.phase === 'parking') {
|
|
3493
3714
|
this.#increment('clarificationParkingExitsSuppressed');
|
|
@@ -4176,6 +4397,7 @@ export class FactoryLoop {
|
|
|
4176
4397
|
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
4177
4398
|
}
|
|
4178
4399
|
}
|
|
4400
|
+
await this.#reportAgent(record, tracked, 'agent.resumed');
|
|
4179
4401
|
}
|
|
4180
4402
|
async #handleDeliveryFailed(info) {
|
|
4181
4403
|
const critical = await this.#state.consumeCritical(this.#workspaceId, info.msgId ?? '');
|
|
@@ -4185,7 +4407,7 @@ export class FactoryLoop {
|
|
|
4185
4407
|
}
|
|
4186
4408
|
const record = (await this.#batch()).getIssueByAgent(info.to);
|
|
4187
4409
|
const issue = critical.issue ?? record?.issue;
|
|
4188
|
-
const error = new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`);
|
|
4410
|
+
const error = Object.assign(new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`), { code: 'fleet_delivery_failed' });
|
|
4189
4411
|
this.#error(error, issue);
|
|
4190
4412
|
if (isTerminalDeliveryFailure(info.reason)) {
|
|
4191
4413
|
this.#increment('criticalDeliveryTerminalFailures');
|
|
@@ -5661,7 +5883,7 @@ export class FactoryLoop {
|
|
|
5661
5883
|
}
|
|
5662
5884
|
async #cancelBabysitterWake(issueIdentity) {
|
|
5663
5885
|
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
5664
|
-
const mayClearDurable = this.#
|
|
5886
|
+
const mayClearDurable = !this.#usesDurableDispatchLifecycle()
|
|
5665
5887
|
|| Boolean(issue && await this.#assertIssueDispatchLifecycleOwner(issue));
|
|
5666
5888
|
for (const [key, state] of this.#babysitterWakeStates) {
|
|
5667
5889
|
if (issueKey(state.issue) !== issueIdentity)
|
|
@@ -5922,19 +6144,7 @@ export class FactoryLoop {
|
|
|
5922
6144
|
},
|
|
5923
6145
|
};
|
|
5924
6146
|
let targets;
|
|
5925
|
-
if (this.#fleet.
|
|
5926
|
-
if (!this.#fleet.sendInput) {
|
|
5927
|
-
throw new Error('Fleet client advertises PTY prompt delivery without raw input support');
|
|
5928
|
-
}
|
|
5929
|
-
// Internal agents are harness-owned PTYs, not Relaycast identities.
|
|
5930
|
-
// Stage the validated metadata-only wake directly through the broker so
|
|
5931
|
-
// a missing Relaycast registration cannot strand a live babysitter.
|
|
5932
|
-
// The CR remains a separate write: a critical-section begin can arrive
|
|
5933
|
-
// while this write is in flight and must be able to defer submission.
|
|
5934
|
-
await this.#fleet.sendInput(input.to, input.text);
|
|
5935
|
-
targets = [input.to];
|
|
5936
|
-
}
|
|
5937
|
-
else if (!this.#fleet.waitForInjected) {
|
|
6147
|
+
if (!this.#fleet.waitForInjected) {
|
|
5938
6148
|
await this.#fleet.sendMessage(input);
|
|
5939
6149
|
if (this.#stopping || state.cancelled) {
|
|
5940
6150
|
state.deliveringKinds = undefined;
|
|
@@ -6403,7 +6613,12 @@ export class FactoryLoop {
|
|
|
6403
6613
|
prNumber: prRef.prNumber,
|
|
6404
6614
|
babysitter: spawned.name,
|
|
6405
6615
|
});
|
|
6406
|
-
|
|
6616
|
+
// Internal PTY spawns receive `task` atomically in spawnPty. Re-sending
|
|
6617
|
+
// the same task through Relaycast before the worker has registered its
|
|
6618
|
+
// messaging identity can fail an otherwise successful spawn and erase
|
|
6619
|
+
// valid babysitter ownership. Remote placement still needs the explicit,
|
|
6620
|
+
// confirmed follow-up injection used by its spawn protocol.
|
|
6621
|
+
if (this.#fleet.waitForInjected && this.#fleet.placementLocality !== 'local') {
|
|
6407
6622
|
const input = {
|
|
6408
6623
|
to: tracked?.result?.name ?? spawned.name,
|
|
6409
6624
|
text: task,
|
|
@@ -6637,7 +6852,7 @@ export class FactoryLoop {
|
|
|
6637
6852
|
}
|
|
6638
6853
|
const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
|
|
6639
6854
|
releaseReasonForRetry = releaseReason;
|
|
6640
|
-
if (this.#
|
|
6855
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
6641
6856
|
// Durable capacity is released as soon as terminal writeback is
|
|
6642
6857
|
// acknowledged. Agent cleanup remains fenced/retryable in `releasing`.
|
|
6643
6858
|
const batch = await this.#batch();
|
|
@@ -6667,7 +6882,7 @@ export class FactoryLoop {
|
|
|
6667
6882
|
this.#babysitterPr.delete(completionKey);
|
|
6668
6883
|
await this.#cancelBabysitterWake(completionKey);
|
|
6669
6884
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
|
|
6670
|
-
if (this.#
|
|
6885
|
+
if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
|
|
6671
6886
|
for (const publishedKey of this.#publishedPullRequests.keys()) {
|
|
6672
6887
|
if (publishedKey.startsWith(`${completionKey}:`))
|
|
6673
6888
|
this.#publishedPullRequests.delete(publishedKey);
|
|
@@ -6677,7 +6892,16 @@ export class FactoryLoop {
|
|
|
6677
6892
|
}
|
|
6678
6893
|
#emit(event, payload) {
|
|
6679
6894
|
for (const listener of this.#listeners.get(event) ?? []) {
|
|
6680
|
-
|
|
6895
|
+
try {
|
|
6896
|
+
listener(payload);
|
|
6897
|
+
}
|
|
6898
|
+
catch (error) {
|
|
6899
|
+
this.#increment('factoryEventListenerFailures');
|
|
6900
|
+
this.#logger.warn?.('[factory] event listener failed', {
|
|
6901
|
+
event,
|
|
6902
|
+
errorClass: error instanceof Error ? error.name : 'Error',
|
|
6903
|
+
});
|
|
6904
|
+
}
|
|
6681
6905
|
}
|
|
6682
6906
|
}
|
|
6683
6907
|
#error(error, issue) {
|
|
@@ -6692,6 +6916,48 @@ export class FactoryLoop {
|
|
|
6692
6916
|
...details,
|
|
6693
6917
|
...(issue ? { issue: issue.key } : {}),
|
|
6694
6918
|
});
|
|
6919
|
+
const failureCode = telemetryCategory(error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
|
|
6920
|
+
? error.code
|
|
6921
|
+
: 'factory_error');
|
|
6922
|
+
const failureClass = telemetryErrorClass(error);
|
|
6923
|
+
void (async () => {
|
|
6924
|
+
try {
|
|
6925
|
+
const lifecycle = issue
|
|
6926
|
+
? await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(issue)).catch(() => undefined)
|
|
6927
|
+
: undefined;
|
|
6928
|
+
if (lifecycle) {
|
|
6929
|
+
await this.#reportLifecycle(lifecycle, 'factory.failure', {
|
|
6930
|
+
level: 'error',
|
|
6931
|
+
errorCode: failureCode,
|
|
6932
|
+
});
|
|
6933
|
+
return;
|
|
6934
|
+
}
|
|
6935
|
+
await this.#report({
|
|
6936
|
+
type: 'factory.failure',
|
|
6937
|
+
level: 'error',
|
|
6938
|
+
attributes: {
|
|
6939
|
+
backend: this.#fleet.placementLocality === 'remote' ? 'relay' : 'internal',
|
|
6940
|
+
component: 'orchestrator',
|
|
6941
|
+
operation: 'error',
|
|
6942
|
+
errorClass: failureClass,
|
|
6943
|
+
errorCode: failureCode,
|
|
6944
|
+
},
|
|
6945
|
+
});
|
|
6946
|
+
}
|
|
6947
|
+
catch (telemetryError) {
|
|
6948
|
+
// This is the terminal guard for the intentionally floating telemetry
|
|
6949
|
+
// task. Never log raw messages or allow a custom logger to turn this
|
|
6950
|
+
// best-effort path into an unhandled rejection.
|
|
6951
|
+
try {
|
|
6952
|
+
this.#logger.warn?.('[factory] failed to report failure telemetry', {
|
|
6953
|
+
errorClass: telemetryErrorClass(telemetryError),
|
|
6954
|
+
});
|
|
6955
|
+
}
|
|
6956
|
+
catch {
|
|
6957
|
+
// Reporting and logging are both non-critical to orchestration.
|
|
6958
|
+
}
|
|
6959
|
+
}
|
|
6960
|
+
})();
|
|
6695
6961
|
this.#emit('error', { error, ...details, issue });
|
|
6696
6962
|
}
|
|
6697
6963
|
#surfaceEscalationDeliveryFailure(kind, issue, correlationId, reason, cause) {
|
|
@@ -7755,7 +8021,7 @@ export class FactoryLoop {
|
|
|
7755
8021
|
}
|
|
7756
8022
|
let lifecycleDecision = waiting.decision;
|
|
7757
8023
|
let promotedLifecycle;
|
|
7758
|
-
if (!waiting.dryRun && this.#
|
|
8024
|
+
if (!waiting.dryRun && this.#usesDurableDispatchLifecycle()) {
|
|
7759
8025
|
try {
|
|
7760
8026
|
const claim = await this.#claimDispatchLifecycle(waiting.decision, false);
|
|
7761
8027
|
const lifecycleKey = issueKey(waiting.issue);
|
|
@@ -10046,9 +10312,36 @@ const failedIterationReport = (error, dryRun) => {
|
|
|
10046
10312
|
};
|
|
10047
10313
|
const isRegistrationLagInjectionError = (error) => {
|
|
10048
10314
|
const { errorMessage } = describeError(error);
|
|
10049
|
-
return /recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
|
|
10315
|
+
return /agent_not_found|recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
|
|
10050
10316
|
.test(errorMessage);
|
|
10051
10317
|
};
|
|
10318
|
+
const telemetryRunStatus = (phase) => {
|
|
10319
|
+
switch (phase) {
|
|
10320
|
+
case 'queued':
|
|
10321
|
+
return 'queued';
|
|
10322
|
+
case 'retryable':
|
|
10323
|
+
return 'blocked';
|
|
10324
|
+
case 'parking':
|
|
10325
|
+
case 'waiting-for-human':
|
|
10326
|
+
return 'waiting';
|
|
10327
|
+
case 'complete':
|
|
10328
|
+
return 'succeeded';
|
|
10329
|
+
case 'abandoned':
|
|
10330
|
+
return 'cancelled';
|
|
10331
|
+
default:
|
|
10332
|
+
return 'running';
|
|
10333
|
+
}
|
|
10334
|
+
};
|
|
10335
|
+
const telemetryCategory = (value) => {
|
|
10336
|
+
if (!value)
|
|
10337
|
+
return undefined;
|
|
10338
|
+
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
|
|
10339
|
+
return normalized.slice(0, 120) || undefined;
|
|
10340
|
+
};
|
|
10341
|
+
const telemetryErrorClass = (error) => {
|
|
10342
|
+
const name = error instanceof Error ? error.name : '';
|
|
10343
|
+
return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
|
|
10344
|
+
};
|
|
10052
10345
|
const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
|
|
10053
10346
|
const retryOnTimeout = async (fn, opts) => {
|
|
10054
10347
|
let lastError;
|