@gh-symphony/cli 0.0.17 → 0.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -9
- package/dist/{chunk-EFMFGOWM.js → chunk-6CI3UUMH.js} +282 -57
- package/dist/chunk-C7G7RJ4G.js +146 -0
- package/dist/{chunk-MHIWAIVD.js → chunk-GKENCODJ.js} +141 -53
- package/dist/{project-557FE2GD.js → chunk-H2YXSYOZ.js} +108 -92
- package/dist/{chunk-TF3QNWNC.js → chunk-M3IFVLQS.js} +246 -212
- package/dist/{chunk-IWR4UQEJ.js → chunk-RN2PACNV.js} +350 -523
- package/dist/chunk-TILHWBP6.js +638 -0
- package/dist/{chunk-6HBZC3BE.js → chunk-XN5ABWZ6.js} +23 -5
- package/dist/{chunk-76QPITKI.js → chunk-Y6TYJMNT.js} +1 -1
- package/dist/{config-cmd-AZ7POMAA.js → config-cmd-DNXNL26Z.js} +3 -1
- package/dist/doctor-IYHCFXOZ.js +1126 -0
- package/dist/index.js +157 -19
- package/dist/init-KZT6YNOH.js +33 -0
- package/dist/{logs-6LNGT2GF.js → logs-6JKKYDGJ.js} +1 -1
- package/dist/project-DNALEWO3.js +22 -0
- package/dist/{recover-LVBI2TGH.js → recover-C3V2QAUB.js} +3 -3
- package/dist/repo-HDDE7OUI.js +321 -0
- package/dist/{run-WITYAYFZ.js → run-XI2S5Y4V.js} +3 -3
- package/dist/setup-K4CYYJBF.js +431 -0
- package/dist/{start-JUFKNL3N.js → start-M6IQGRFO.js} +5 -5
- package/dist/{status-3WK5BWRZ.js → status-QSCFVGRQ.js} +2 -2
- package/dist/{stop-AA3AP5M6.js → stop-7MFCBQVW.js} +2 -2
- package/dist/upgrade-F4VE4XBS.js +165 -0
- package/dist/{version-YVM2A25J.js → version-Y5RYNWMF.js} +1 -1
- package/dist/worker-entry.js +39 -11
- package/dist/workflow-TBIFY5MO.js +497 -0
- package/package.json +4 -4
- package/dist/chunk-JO3AXHQI.js +0 -130
- package/dist/chunk-TH5QPO3Y.js +0 -67
- package/dist/init-EZXQAXZM.js +0 -17
- package/dist/repo-R3XBIVAX.js +0 -121
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
|
+
DEFAULT_MAX_FAILURE_RETRIES,
|
|
3
4
|
DEFAULT_WORKFLOW_LIFECYCLE,
|
|
4
5
|
WorkflowConfigStore,
|
|
5
6
|
buildHookEnv,
|
|
@@ -25,7 +26,7 @@ import {
|
|
|
25
26
|
resolveIssueWorkspaceDirectory,
|
|
26
27
|
safeReadDir,
|
|
27
28
|
scheduleRetryAt
|
|
28
|
-
} from "./chunk-
|
|
29
|
+
} from "./chunk-M3IFVLQS.js";
|
|
29
30
|
|
|
30
31
|
// ../orchestrator/dist/service.js
|
|
31
32
|
import { mkdir as mkdir3, readFile as readFile3, rm as rm3, writeFile as writeFile3 } from "fs/promises";
|
|
@@ -286,7 +287,8 @@ var OrchestratorFsStore = class {
|
|
|
286
287
|
if (issues) {
|
|
287
288
|
return issues.map((issue) => ({
|
|
288
289
|
...issue,
|
|
289
|
-
completedOnce: issue.completedOnce ?? false
|
|
290
|
+
completedOnce: issue.completedOnce ?? false,
|
|
291
|
+
failureRetryCount: issue.failureRetryCount ?? 0
|
|
290
292
|
}));
|
|
291
293
|
}
|
|
292
294
|
const legacyLeases = await readJsonFile(join2(this.projectDir(projectId), "leases.json")) ?? [];
|
|
@@ -298,6 +300,7 @@ var OrchestratorFsStore = class {
|
|
|
298
300
|
identifier: lease.issueIdentifier,
|
|
299
301
|
workspaceKey: deriveIssueWorkspaceKeyFromIdentifier(lease.issueIdentifier),
|
|
300
302
|
completedOnce: false,
|
|
303
|
+
failureRetryCount: 0,
|
|
301
304
|
state: lease.status === "active" ? "claimed" : "released",
|
|
302
305
|
currentRunId: lease.status === "active" ? lease.runId : null,
|
|
303
306
|
retryEntry: null,
|
|
@@ -464,9 +467,12 @@ async function pathExists(path) {
|
|
|
464
467
|
}
|
|
465
468
|
|
|
466
469
|
// ../tracker-github/dist/adapter.js
|
|
470
|
+
import { createHash } from "crypto";
|
|
467
471
|
var DEFAULT_API_URL = "https://api.github.com/graphql";
|
|
468
472
|
var DEFAULT_PAGE_SIZE = 25;
|
|
469
473
|
var DEFAULT_NETWORK_TIMEOUT_MS = 3e4;
|
|
474
|
+
var RATE_LIMIT_THRESHOLD = 100;
|
|
475
|
+
var MAX_RATE_LIMIT_WAIT_MS = 6e4;
|
|
470
476
|
var GitHubTrackerError = class extends Error {
|
|
471
477
|
};
|
|
472
478
|
var GitHubTrackerHttpError = class extends GitHubTrackerError {
|
|
@@ -480,6 +486,7 @@ var GitHubTrackerHttpError = class extends GitHubTrackerError {
|
|
|
480
486
|
};
|
|
481
487
|
var GitHubTrackerQueryError = class extends GitHubTrackerError {
|
|
482
488
|
};
|
|
489
|
+
var cachedGitHubGraphQLRateLimits = /* @__PURE__ */ new Map();
|
|
483
490
|
function normalizeProjectItem(projectId, item, lifecycle = DEFAULT_WORKFLOW_LIFECYCLE, priority = {}, rateLimits = null) {
|
|
484
491
|
if (item.content?.__typename !== "Issue") {
|
|
485
492
|
return null;
|
|
@@ -494,6 +501,9 @@ function normalizeProjectItem(projectId, item, lifecycle = DEFAULT_WORKFLOW_LIFE
|
|
|
494
501
|
state: normalizeBlockerState(node.state, lifecycle)
|
|
495
502
|
}
|
|
496
503
|
] : []);
|
|
504
|
+
const issueUpdatedAtMs = parseTimestampMs(item.content.updatedAt);
|
|
505
|
+
const itemUpdatedAtMs = parseTimestampMs(item.updatedAt);
|
|
506
|
+
const trackedUpdatedAt = itemUpdatedAtMs !== null && (issueUpdatedAtMs === null || itemUpdatedAtMs > issueUpdatedAtMs) ? item.updatedAt : item.content.updatedAt ?? item.updatedAt;
|
|
497
507
|
return {
|
|
498
508
|
id: item.content.id,
|
|
499
509
|
identifier: `${repository.owner.login}/${repository.name}#${item.content.number}`,
|
|
@@ -507,7 +517,7 @@ function normalizeProjectItem(projectId, item, lifecycle = DEFAULT_WORKFLOW_LIFE
|
|
|
507
517
|
labels: (item.content.labels?.nodes ?? []).flatMap((label) => label?.name ? [label.name.toLowerCase()] : []).sort(),
|
|
508
518
|
blockedBy,
|
|
509
519
|
createdAt: item.content.createdAt,
|
|
510
|
-
updatedAt:
|
|
520
|
+
updatedAt: trackedUpdatedAt,
|
|
511
521
|
repository: {
|
|
512
522
|
owner: repository.owner.login,
|
|
513
523
|
name: repository.name,
|
|
@@ -814,6 +824,8 @@ async function executeGraphQLQuery(config, query, variables, fetchImpl) {
|
|
|
814
824
|
return result.data;
|
|
815
825
|
}
|
|
816
826
|
async function executeGraphQLQueryWithMetadata(config, query, variables, fetchImpl) {
|
|
827
|
+
const tokenFingerprint = fingerprintToken(config.token);
|
|
828
|
+
await guardGraphQLRateLimit(tokenFingerprint);
|
|
817
829
|
const response = await fetchImpl(config.apiUrl ?? DEFAULT_API_URL, {
|
|
818
830
|
method: "POST",
|
|
819
831
|
headers: {
|
|
@@ -838,11 +850,38 @@ async function executeGraphQLQueryWithMetadata(config, query, variables, fetchIm
|
|
|
838
850
|
throw new GitHubTrackerQueryError("GitHub GraphQL response did not include data.");
|
|
839
851
|
}
|
|
840
852
|
const data = payload.data;
|
|
853
|
+
const rateLimits = extractGitHubRateLimits(response.headers);
|
|
854
|
+
cachedGitHubGraphQLRateLimits.set(tokenFingerprint, rateLimits);
|
|
841
855
|
return {
|
|
842
856
|
data,
|
|
843
|
-
rateLimits
|
|
857
|
+
rateLimits
|
|
844
858
|
};
|
|
845
859
|
}
|
|
860
|
+
async function guardGraphQLRateLimit(tokenFingerprint) {
|
|
861
|
+
const rateLimit = cachedGitHubGraphQLRateLimits.get(tokenFingerprint) ?? null;
|
|
862
|
+
if (!rateLimit) {
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
const remaining = rateLimit.remaining;
|
|
866
|
+
if (remaining === null || remaining > RATE_LIMIT_THRESHOLD) {
|
|
867
|
+
return;
|
|
868
|
+
}
|
|
869
|
+
const resetAtMs = parseTimestampMs(rateLimit.resetAt);
|
|
870
|
+
if (resetAtMs === null) {
|
|
871
|
+
throw new GitHubTrackerError("Rate limit near exhaustion");
|
|
872
|
+
}
|
|
873
|
+
const waitMs = Math.max(0, resetAtMs - Date.now());
|
|
874
|
+
if (waitMs > MAX_RATE_LIMIT_WAIT_MS) {
|
|
875
|
+
throw new GitHubTrackerError("Rate limit near exhaustion");
|
|
876
|
+
}
|
|
877
|
+
cachedGitHubGraphQLRateLimits.delete(tokenFingerprint);
|
|
878
|
+
if (waitMs > 0) {
|
|
879
|
+
await sleep(waitMs);
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
function fingerprintToken(token) {
|
|
883
|
+
return createHash("sha256").update(token).digest("hex");
|
|
884
|
+
}
|
|
846
885
|
function extractGitHubRateLimits(headers) {
|
|
847
886
|
if (!headers || typeof headers.get !== "function") {
|
|
848
887
|
return null;
|
|
@@ -872,6 +911,18 @@ function parseIntegerHeader(value) {
|
|
|
872
911
|
const parsed = Number.parseInt(value, 10);
|
|
873
912
|
return Number.isFinite(parsed) ? parsed : null;
|
|
874
913
|
}
|
|
914
|
+
function parseTimestampMs(value) {
|
|
915
|
+
if (!value) {
|
|
916
|
+
return null;
|
|
917
|
+
}
|
|
918
|
+
const timestampMs = Date.parse(value);
|
|
919
|
+
return Number.isFinite(timestampMs) ? timestampMs : null;
|
|
920
|
+
}
|
|
921
|
+
function sleep(ms) {
|
|
922
|
+
return new Promise((resolve4) => {
|
|
923
|
+
setTimeout(resolve4, ms);
|
|
924
|
+
});
|
|
925
|
+
}
|
|
875
926
|
var PROJECT_ITEMS_QUERY = `
|
|
876
927
|
query ProjectItems($projectId: ID!, $cursor: String, $pageSize: Int!) {
|
|
877
928
|
node(id: $projectId) {
|
|
@@ -1086,7 +1137,7 @@ var ISSUE_PROJECT_ITEMS_PAGE_QUERY = `
|
|
|
1086
1137
|
`;
|
|
1087
1138
|
|
|
1088
1139
|
// ../tracker-github/dist/orchestrator-adapter.js
|
|
1089
|
-
import { createHash } from "crypto";
|
|
1140
|
+
import { createHash as createHash2 } from "crypto";
|
|
1090
1141
|
var githubProjectTrackerAdapter = {
|
|
1091
1142
|
async listIssues(project, dependencies = {}) {
|
|
1092
1143
|
return listProjectIssues(project, dependencies);
|
|
@@ -1174,7 +1225,7 @@ function hashToken(token) {
|
|
|
1174
1225
|
if (!token) {
|
|
1175
1226
|
return null;
|
|
1176
1227
|
}
|
|
1177
|
-
return
|
|
1228
|
+
return createHash2("sha256").update(token).digest("hex");
|
|
1178
1229
|
}
|
|
1179
1230
|
var trackerAdapters = {
|
|
1180
1231
|
"github-project": githubProjectTrackerAdapter
|
|
@@ -1333,19 +1384,33 @@ var DEFAULT_POLL_INTERVAL_MS = 3e4;
|
|
|
1333
1384
|
var DEFAULT_CONCURRENCY = 3;
|
|
1334
1385
|
var DEFAULT_RETRY_BACKOFF_MS = 3e4;
|
|
1335
1386
|
var CONTINUATION_RETRY_DELAY_MS = 1e3;
|
|
1387
|
+
var DEFAULT_GLOBAL_MAX_TURNS = 100;
|
|
1388
|
+
var DEFAULT_MAX_TOKENS = 256e3;
|
|
1336
1389
|
var DEFAULT_WORKER_COMMAND = "node packages/worker/dist/index.js";
|
|
1337
1390
|
var DEFAULT_MAX_NONPRODUCTIVE_TURNS = 3;
|
|
1391
|
+
var LOW_RATE_LIMIT_WARNING_THRESHOLD = 0.05;
|
|
1392
|
+
var MAX_FAILURE_RETRIES_EXCEEDED_REASON = "max_failure_retries_exceeded";
|
|
1338
1393
|
var STUCK_WORKER_TIMEOUT_MS = 30 * 60 * 1e3;
|
|
1339
1394
|
function isUsableWorkflowResolution(resolution) {
|
|
1340
1395
|
return resolution.isValid || resolution.usedLastKnownGood;
|
|
1341
1396
|
}
|
|
1342
|
-
function
|
|
1397
|
+
function parseTimestampMs2(value) {
|
|
1343
1398
|
if (!value) {
|
|
1344
1399
|
return null;
|
|
1345
1400
|
}
|
|
1346
1401
|
const parsed = new Date(value).getTime();
|
|
1347
1402
|
return Number.isFinite(parsed) ? parsed : null;
|
|
1348
1403
|
}
|
|
1404
|
+
function parseFiniteNumber(value) {
|
|
1405
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
1406
|
+
return value;
|
|
1407
|
+
}
|
|
1408
|
+
if (typeof value === "string" && value.trim()) {
|
|
1409
|
+
const parsed = Number(value);
|
|
1410
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
1411
|
+
}
|
|
1412
|
+
return null;
|
|
1413
|
+
}
|
|
1349
1414
|
var OrchestratorService = class {
|
|
1350
1415
|
store;
|
|
1351
1416
|
projectConfig;
|
|
@@ -1513,11 +1578,12 @@ var OrchestratorService = class {
|
|
|
1513
1578
|
let recovered = 0;
|
|
1514
1579
|
let pollIntervalMs = DEFAULT_POLL_INTERVAL_MS;
|
|
1515
1580
|
let rateLimits = null;
|
|
1581
|
+
let trackerRateLimits = null;
|
|
1516
1582
|
let issueRecords = await this.store.loadProjectIssueOrchestrations(tenant.projectId);
|
|
1517
1583
|
const allRuns = (await this.store.loadAllRuns()).filter((run) => run.projectId === tenant.projectId);
|
|
1518
1584
|
const activeRuns = allRuns.filter((run) => isActiveRunStatus(run.status));
|
|
1519
1585
|
for (const run of activeRuns) {
|
|
1520
|
-
const outcome = await this.reconcileRun(tenant, run, issueRecords);
|
|
1586
|
+
const outcome = await this.reconcileRun(tenant, run, issueRecords, trackerDependencies);
|
|
1521
1587
|
issueRecords = outcome.issueRecords;
|
|
1522
1588
|
if (outcome.recovered) {
|
|
1523
1589
|
recovered += 1;
|
|
@@ -1555,11 +1621,13 @@ var OrchestratorService = class {
|
|
|
1555
1621
|
});
|
|
1556
1622
|
}
|
|
1557
1623
|
rateLimits = resolveProjectRateLimits(syncedActiveRuns, trackedIssuesByIdentifier.values());
|
|
1624
|
+
trackerRateLimits = resolveTrackerRateLimits(trackedIssuesByIdentifier.values());
|
|
1558
1625
|
const concurrency = await this.getProjectConcurrency(tenant);
|
|
1559
1626
|
const currentlyActive = issueRecords.filter((record) => isIssueOrchestrationClaimed(record.state)).length;
|
|
1560
1627
|
const availableSlots = Math.max(0, concurrency - currentlyActive);
|
|
1628
|
+
const latestRunsByIssueId = buildLatestRunMapByIssueId(projectRunsAfterReconcile);
|
|
1561
1629
|
const unscheduledCandidates = actionableCandidates.filter((issue) => {
|
|
1562
|
-
if (hasConvergenceLockedRun(projectRunsAfterReconcile, issue.id, issue.state)) {
|
|
1630
|
+
if (hasConvergenceLockedRun(projectRunsAfterReconcile, issue.id, issue.state, issue.updatedAt)) {
|
|
1563
1631
|
return false;
|
|
1564
1632
|
}
|
|
1565
1633
|
return !issueRecords.some((record) => record.issueId === issue.id && isIssueOrchestrationClaimed(record.state));
|
|
@@ -1579,6 +1647,9 @@ var OrchestratorService = class {
|
|
|
1579
1647
|
}
|
|
1580
1648
|
if (slotsRemaining <= 0)
|
|
1581
1649
|
break;
|
|
1650
|
+
if (await this.isFailureRetrySuppressedIssue(tenant, issue, issueRecords, latestRunsByIssueId.get(issue.id) ?? null)) {
|
|
1651
|
+
continue;
|
|
1652
|
+
}
|
|
1582
1653
|
if (isIssueBudgetExceeded(resolveIssueBudgetSnapshot(projectRunsAfterReconcile, issue.id), now)) {
|
|
1583
1654
|
continue;
|
|
1584
1655
|
}
|
|
@@ -1599,6 +1670,7 @@ var OrchestratorService = class {
|
|
|
1599
1670
|
identifier: issue.identifier,
|
|
1600
1671
|
workspaceKey: preferredWorkspaceKey,
|
|
1601
1672
|
state: "claimed",
|
|
1673
|
+
failureRetryCount: 0,
|
|
1602
1674
|
currentRunId: null,
|
|
1603
1675
|
retryEntry: null,
|
|
1604
1676
|
updatedAt: now.toISOString()
|
|
@@ -1680,7 +1752,11 @@ var OrchestratorService = class {
|
|
|
1680
1752
|
} catch (error) {
|
|
1681
1753
|
lastError = error instanceof Error ? error.message : "Unknown orchestration error";
|
|
1682
1754
|
}
|
|
1683
|
-
|
|
1755
|
+
const effectivePollIntervalMs = resolveAdaptivePollIntervalMs(pollIntervalMs, trackerRateLimits);
|
|
1756
|
+
if (effectivePollIntervalMs > pollIntervalMs && isLowRateLimit(trackerRateLimits, LOW_RATE_LIMIT_WARNING_THRESHOLD)) {
|
|
1757
|
+
this.writeStderr(`[orchestrator] low GitHub rate limit for ${tenant.projectId}: interval=${effectivePollIntervalMs}ms rateLimits=${JSON.stringify(trackerRateLimits)}`);
|
|
1758
|
+
}
|
|
1759
|
+
this.projectPollIntervals.set(tenant.projectId, effectivePollIntervalMs);
|
|
1684
1760
|
await this.store.saveProjectIssueOrchestrations(tenant.projectId, issueRecords);
|
|
1685
1761
|
const allTenantRuns = (await this.store.loadAllRuns()).filter((run) => run.projectId === tenant.projectId);
|
|
1686
1762
|
const latestRuns = allTenantRuns.filter((run) => isActiveRunStatus(run.status));
|
|
@@ -1874,26 +1950,9 @@ var OrchestratorService = class {
|
|
|
1874
1950
|
if (!lifecycle) {
|
|
1875
1951
|
lifecycle = resolution.lifecycle;
|
|
1876
1952
|
}
|
|
1877
|
-
if (!
|
|
1953
|
+
if (!this.isIssueCandidateEligible(issue, resolution.lifecycle, issues)) {
|
|
1878
1954
|
continue;
|
|
1879
1955
|
}
|
|
1880
|
-
if (matchesWorkflowState(issue.state, resolution.lifecycle.blockerCheckStates) && issue.blockedBy.length > 0) {
|
|
1881
|
-
const hasNonTerminalBlocker = issue.blockedBy.some((blockerRef) => {
|
|
1882
|
-
if (blockerRef.state && isStateTerminal(blockerRef.state, resolution.lifecycle)) {
|
|
1883
|
-
return false;
|
|
1884
|
-
}
|
|
1885
|
-
if (blockerRef.identifier) {
|
|
1886
|
-
const blockerIssue = issues.find((candidate) => candidate.identifier === blockerRef.identifier);
|
|
1887
|
-
if (blockerIssue?.state) {
|
|
1888
|
-
return !isStateTerminal(blockerIssue.state, resolution.lifecycle);
|
|
1889
|
-
}
|
|
1890
|
-
}
|
|
1891
|
-
return true;
|
|
1892
|
-
});
|
|
1893
|
-
if (hasNonTerminalBlocker) {
|
|
1894
|
-
continue;
|
|
1895
|
-
}
|
|
1896
|
-
}
|
|
1897
1956
|
candidates.push(issue);
|
|
1898
1957
|
}
|
|
1899
1958
|
if (!lifecycle && tenant.repositories.length > 0) {
|
|
@@ -1912,6 +1971,26 @@ var OrchestratorService = class {
|
|
|
1912
1971
|
}
|
|
1913
1972
|
};
|
|
1914
1973
|
}
|
|
1974
|
+
isIssueCandidateEligible(issue, lifecycle, issues) {
|
|
1975
|
+
if (!isStateActive(issue.state, lifecycle)) {
|
|
1976
|
+
return false;
|
|
1977
|
+
}
|
|
1978
|
+
if (!matchesWorkflowState(issue.state, lifecycle.blockerCheckStates) || issue.blockedBy.length === 0) {
|
|
1979
|
+
return true;
|
|
1980
|
+
}
|
|
1981
|
+
return !issue.blockedBy.some((blockerRef) => {
|
|
1982
|
+
if (blockerRef.state && isStateTerminal(blockerRef.state, lifecycle)) {
|
|
1983
|
+
return false;
|
|
1984
|
+
}
|
|
1985
|
+
if (blockerRef.identifier) {
|
|
1986
|
+
const blockerIssue = issues.find((candidate) => candidate.identifier === blockerRef.identifier);
|
|
1987
|
+
if (blockerIssue?.state) {
|
|
1988
|
+
return !isStateTerminal(blockerIssue.state, lifecycle);
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
return true;
|
|
1992
|
+
});
|
|
1993
|
+
}
|
|
1915
1994
|
async loadProjectWorkflow(tenant, repository) {
|
|
1916
1995
|
const cacheKey = this.workflowCacheKey(repository);
|
|
1917
1996
|
const pendingCache = this.workflowResolutionCache;
|
|
@@ -2233,13 +2312,13 @@ var OrchestratorService = class {
|
|
|
2233
2312
|
issuesByIdentifier
|
|
2234
2313
|
};
|
|
2235
2314
|
}
|
|
2236
|
-
async reconcileRun(tenant, run, issueRecords) {
|
|
2315
|
+
async reconcileRun(tenant, run, issueRecords, trackerDependencies = {}) {
|
|
2237
2316
|
const now = this.now();
|
|
2238
2317
|
if (run.processId && this.isProcessRunning(run.processId)) {
|
|
2239
2318
|
const retryPolicy = await this.loadRetryPolicy(tenant, run.repository);
|
|
2240
2319
|
const configuredStallTimeoutMs = retryPolicy?.stallTimeoutMs ?? null;
|
|
2241
|
-
const lastActivityAtMs =
|
|
2242
|
-
const startedAtMs =
|
|
2320
|
+
const lastActivityAtMs = parseTimestampMs2(run.lastEventAt ?? run.startedAt);
|
|
2321
|
+
const startedAtMs = parseTimestampMs2(run.startedAt);
|
|
2243
2322
|
const elapsedSinceLastActivityMs = lastActivityAtMs === null ? null : now.getTime() - lastActivityAtMs;
|
|
2244
2323
|
const runningSinceMs = startedAtMs === null ? null : now.getTime() - startedAtMs;
|
|
2245
2324
|
const isStalledByWorkflowTimeout = configuredStallTimeoutMs !== null && configuredStallTimeoutMs > 0 && elapsedSinceLastActivityMs !== null && elapsedSinceLastActivityMs > configuredStallTimeoutMs;
|
|
@@ -2318,7 +2397,7 @@ var OrchestratorService = class {
|
|
|
2318
2397
|
recovered: false
|
|
2319
2398
|
};
|
|
2320
2399
|
}
|
|
2321
|
-
if (await this.resolveRetryRestartAction(tenant, run) === "release") {
|
|
2400
|
+
if (await this.resolveRetryRestartAction(tenant, run, trackerDependencies) === "release") {
|
|
2322
2401
|
return this.releaseRetryingRun(runWithTokens, issueRecords, now);
|
|
2323
2402
|
}
|
|
2324
2403
|
return this.restartRun(tenant, run, issueRecords, now, workerSessionId);
|
|
@@ -2355,7 +2434,54 @@ var OrchestratorService = class {
|
|
|
2355
2434
|
state: run.issueState
|
|
2356
2435
|
});
|
|
2357
2436
|
}
|
|
2358
|
-
const retryKind = await this.classifyRetryKind(tenant, run);
|
|
2437
|
+
const retryKind = await this.classifyRetryKind(tenant, run, trackerDependencies);
|
|
2438
|
+
const failureRetryCount = retryKind === "failure" ? (this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0) + 1 : this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0;
|
|
2439
|
+
const maxFailureRetries = await this.loadMaxFailureRetries(tenant, run.repository);
|
|
2440
|
+
if (retryKind === "failure" && failureRetryCount >= maxFailureRetries) {
|
|
2441
|
+
const lastError = [
|
|
2442
|
+
`Run suppressed: ${MAX_FAILURE_RETRIES_EXCEEDED_REASON}.`,
|
|
2443
|
+
`failureRetryCount=${failureRetryCount}.`,
|
|
2444
|
+
`maxFailureRetries=${maxFailureRetries}.`
|
|
2445
|
+
].join(" ");
|
|
2446
|
+
const suppressedRun = {
|
|
2447
|
+
...runWithTokens,
|
|
2448
|
+
status: "suppressed",
|
|
2449
|
+
processId: null,
|
|
2450
|
+
updatedAt: now.toISOString(),
|
|
2451
|
+
completedAt: now.toISOString(),
|
|
2452
|
+
nextRetryAt: null,
|
|
2453
|
+
retryKind: null,
|
|
2454
|
+
runPhase: runWithTokens.runPhase ?? "failed",
|
|
2455
|
+
lastError
|
|
2456
|
+
};
|
|
2457
|
+
await this.store.saveRun(suppressedRun);
|
|
2458
|
+
await this.store.appendRunEvent(run.runId, {
|
|
2459
|
+
at: now.toISOString(),
|
|
2460
|
+
event: "run-suppressed",
|
|
2461
|
+
projectId: run.projectId,
|
|
2462
|
+
issueIdentifier: run.issueIdentifier,
|
|
2463
|
+
issueId: run.issueId,
|
|
2464
|
+
reason: MAX_FAILURE_RETRIES_EXCEEDED_REASON
|
|
2465
|
+
});
|
|
2466
|
+
this.logVerbose(`[run-completed] ${suppressedRun.runId} status=${suppressedRun.status}`);
|
|
2467
|
+
return {
|
|
2468
|
+
issueRecords: upsertIssueOrchestration(issueRecords, {
|
|
2469
|
+
issueId: run.issueId,
|
|
2470
|
+
identifier: run.issueIdentifier,
|
|
2471
|
+
workspaceKey: run.issueWorkspaceKey ?? deriveIssueWorkspaceKey({
|
|
2472
|
+
projectId: tenant.projectId,
|
|
2473
|
+
adapter: tenant.tracker.adapter,
|
|
2474
|
+
issueSubjectId: run.issueSubjectId
|
|
2475
|
+
}, run.issueIdentifier),
|
|
2476
|
+
state: "released",
|
|
2477
|
+
failureRetryCount,
|
|
2478
|
+
currentRunId: null,
|
|
2479
|
+
retryEntry: null,
|
|
2480
|
+
updatedAt: now.toISOString()
|
|
2481
|
+
}),
|
|
2482
|
+
recovered: false
|
|
2483
|
+
};
|
|
2484
|
+
}
|
|
2359
2485
|
let nextRetryAt;
|
|
2360
2486
|
if (retryKind === "continuation") {
|
|
2361
2487
|
nextRetryAt = new Date(now.getTime() + CONTINUATION_RETRY_DELAY_MS).toISOString();
|
|
@@ -2391,6 +2517,7 @@ var OrchestratorService = class {
|
|
|
2391
2517
|
}, run.issueIdentifier),
|
|
2392
2518
|
state: "retry_queued",
|
|
2393
2519
|
completedOnce: retryKind === "continuation" ? true : void 0,
|
|
2520
|
+
failureRetryCount,
|
|
2394
2521
|
currentRunId: run.runId,
|
|
2395
2522
|
retryEntry: {
|
|
2396
2523
|
attempt: retryRecord.attempt,
|
|
@@ -2605,49 +2732,47 @@ var OrchestratorService = class {
|
|
|
2605
2732
|
* — the worker completed its session and the issue hasn't transitioned away.
|
|
2606
2733
|
* Failure applies when we cannot confirm the issue is still actionable.
|
|
2607
2734
|
*/
|
|
2608
|
-
async classifyRetryKind(tenant, run) {
|
|
2735
|
+
async classifyRetryKind(tenant, run, trackerDependencies = {}) {
|
|
2609
2736
|
try {
|
|
2610
|
-
const
|
|
2611
|
-
|
|
2612
|
-
fetchImpl: this.dependencies.fetchImpl
|
|
2613
|
-
});
|
|
2614
|
-
const runIssue = issues.find((issue) => issue.identifier === run.issueIdentifier);
|
|
2615
|
-
if (!runIssue) {
|
|
2737
|
+
const eligibleContext = await this.fetchTrackedIssueEligibilityContext(tenant, run.issueIdentifier, trackerDependencies);
|
|
2738
|
+
if (!eligibleContext) {
|
|
2616
2739
|
return "failure";
|
|
2617
2740
|
}
|
|
2618
2741
|
const resolution = await this.loadProjectWorkflow(tenant, run.repository);
|
|
2619
2742
|
if (!isUsableWorkflowResolution(resolution)) {
|
|
2620
2743
|
return "failure";
|
|
2621
2744
|
}
|
|
2622
|
-
return
|
|
2745
|
+
return this.isIssueCandidateEligible(eligibleContext.issue, resolution.lifecycle, eligibleContext.issues) ? "continuation" : "failure";
|
|
2623
2746
|
} catch {
|
|
2624
2747
|
return "failure";
|
|
2625
2748
|
}
|
|
2626
2749
|
}
|
|
2627
|
-
async resolveRetryRestartAction(tenant, run) {
|
|
2750
|
+
async resolveRetryRestartAction(tenant, run, trackerDependencies = {}) {
|
|
2628
2751
|
try {
|
|
2629
2752
|
if (isIssueBudgetExceeded(resolveIssueBudgetSnapshot((await this.store.loadAllRuns()).filter((candidate) => candidate.projectId === tenant.projectId), run.issueId), this.now())) {
|
|
2630
2753
|
return "release";
|
|
2631
2754
|
}
|
|
2632
|
-
const
|
|
2633
|
-
if (!
|
|
2755
|
+
const eligibleContext = await this.fetchTrackedIssueEligibilityContext(tenant, run.issueIdentifier, trackerDependencies);
|
|
2756
|
+
if (!eligibleContext) {
|
|
2634
2757
|
return "release";
|
|
2635
2758
|
}
|
|
2636
2759
|
const resolution = await this.loadProjectWorkflow(tenant, run.repository);
|
|
2637
2760
|
if (!isUsableWorkflowResolution(resolution)) {
|
|
2638
2761
|
return "restart";
|
|
2639
2762
|
}
|
|
2640
|
-
return
|
|
2763
|
+
return this.isIssueCandidateEligible(eligibleContext.issue, resolution.lifecycle, eligibleContext.issues) ? "restart" : "release";
|
|
2641
2764
|
} catch {
|
|
2642
2765
|
return "restart";
|
|
2643
2766
|
}
|
|
2644
2767
|
}
|
|
2645
|
-
async
|
|
2768
|
+
async fetchTrackedIssueEligibilityContext(tenant, issueIdentifier, trackerDependencies = {}) {
|
|
2646
2769
|
const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
|
|
2647
|
-
const issues = await trackerAdapter.
|
|
2648
|
-
fetchImpl: this.dependencies.fetchImpl
|
|
2770
|
+
const issues = await trackerAdapter.listIssues(tenant, {
|
|
2771
|
+
fetchImpl: this.dependencies.fetchImpl,
|
|
2772
|
+
...trackerDependencies
|
|
2649
2773
|
});
|
|
2650
|
-
|
|
2774
|
+
const issue = issues.find((candidate) => candidate.identifier === issueIdentifier);
|
|
2775
|
+
return issue ? { issue, issues } : null;
|
|
2651
2776
|
}
|
|
2652
2777
|
async fetchWorkerRunInfo(run) {
|
|
2653
2778
|
const latestRun = await this.store.loadRun(run.runId, run.projectId) ?? run;
|
|
@@ -3020,6 +3145,36 @@ var OrchestratorService = class {
|
|
|
3020
3145
|
};
|
|
3021
3146
|
await this.store.saveIssueWorkspace(removedRecord);
|
|
3022
3147
|
}
|
|
3148
|
+
resolveFailureRetryCount(issueRecords, issueId) {
|
|
3149
|
+
return issueRecords.find((record) => record.issueId === issueId)?.failureRetryCount ?? null;
|
|
3150
|
+
}
|
|
3151
|
+
async isFailureRetrySuppressedIssue(tenant, issue, issueRecords, latestRun) {
|
|
3152
|
+
const issueRecord = issueRecords.find((record) => record.issueId === issue.id || record.identifier === issue.identifier) ?? null;
|
|
3153
|
+
if (!issueRecord || issueRecord.failureRetryCount <= 0) {
|
|
3154
|
+
return false;
|
|
3155
|
+
}
|
|
3156
|
+
const maxFailureRetries = await this.loadMaxFailureRetries(tenant, issue.repository);
|
|
3157
|
+
if (issueRecord.failureRetryCount < maxFailureRetries) {
|
|
3158
|
+
return false;
|
|
3159
|
+
}
|
|
3160
|
+
if (!latestRun || latestRun.status !== "suppressed" || latestRun.issueState !== issue.state || !latestRun.lastError?.includes(MAX_FAILURE_RETRIES_EXCEEDED_REASON)) {
|
|
3161
|
+
return false;
|
|
3162
|
+
}
|
|
3163
|
+
const issueUpdatedAtMs = parseTimestampMs2(issue.updatedAt);
|
|
3164
|
+
const suppressedAtMs = parseTimestampMs2(latestRun.completedAt ?? latestRun.updatedAt);
|
|
3165
|
+
if (issueUpdatedAtMs === null || suppressedAtMs === null) {
|
|
3166
|
+
return true;
|
|
3167
|
+
}
|
|
3168
|
+
return issueUpdatedAtMs <= suppressedAtMs;
|
|
3169
|
+
}
|
|
3170
|
+
async loadMaxFailureRetries(tenant, repository) {
|
|
3171
|
+
try {
|
|
3172
|
+
const resolution = await this.loadProjectWorkflow(tenant, repository);
|
|
3173
|
+
return isUsableWorkflowResolution(resolution) ? resolution.workflow.agent.maxFailureRetries : DEFAULT_MAX_FAILURE_RETRIES;
|
|
3174
|
+
} catch {
|
|
3175
|
+
return DEFAULT_MAX_FAILURE_RETRIES;
|
|
3176
|
+
}
|
|
3177
|
+
}
|
|
3023
3178
|
};
|
|
3024
3179
|
function hasTokenUsage(tokenUsage) {
|
|
3025
3180
|
return Boolean(tokenUsage && (tokenUsage.inputTokens > 0 || tokenUsage.outputTokens > 0 || tokenUsage.totalTokens > 0));
|
|
@@ -3034,7 +3189,7 @@ function resolveProjectRateLimits(runs, issues) {
|
|
|
3034
3189
|
if (!isRecord(run.rateLimits)) {
|
|
3035
3190
|
continue;
|
|
3036
3191
|
}
|
|
3037
|
-
const timestamp =
|
|
3192
|
+
const timestamp = parseTimestampMs2(run.lastEventAt ?? run.updatedAt ?? run.startedAt);
|
|
3038
3193
|
const sortableTimestamp = timestamp ?? -Infinity;
|
|
3039
3194
|
if (sortableTimestamp >= latestRunTimestamp) {
|
|
3040
3195
|
latestRunTimestamp = sortableTimestamp;
|
|
@@ -3051,6 +3206,51 @@ function resolveProjectRateLimits(runs, issues) {
|
|
|
3051
3206
|
}
|
|
3052
3207
|
return null;
|
|
3053
3208
|
}
|
|
3209
|
+
function resolveTrackerRateLimits(issues) {
|
|
3210
|
+
for (const issue of issues) {
|
|
3211
|
+
if (isGitHubTrackerRateLimits(issue.rateLimits)) {
|
|
3212
|
+
return issue.rateLimits;
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
return null;
|
|
3216
|
+
}
|
|
3217
|
+
function resolveAdaptivePollIntervalMs(basePollIntervalMs, rateLimits) {
|
|
3218
|
+
if (!Number.isFinite(basePollIntervalMs) || basePollIntervalMs <= 0) {
|
|
3219
|
+
return DEFAULT_POLL_INTERVAL_MS;
|
|
3220
|
+
}
|
|
3221
|
+
const ratio = extractRateLimitRatio(rateLimits);
|
|
3222
|
+
if (ratio === null || ratio > 0.5) {
|
|
3223
|
+
return basePollIntervalMs;
|
|
3224
|
+
}
|
|
3225
|
+
if (ratio >= 0.2) {
|
|
3226
|
+
return basePollIntervalMs * 2;
|
|
3227
|
+
}
|
|
3228
|
+
if (ratio >= LOW_RATE_LIMIT_WARNING_THRESHOLD) {
|
|
3229
|
+
return basePollIntervalMs * 4;
|
|
3230
|
+
}
|
|
3231
|
+
return basePollIntervalMs * 10;
|
|
3232
|
+
}
|
|
3233
|
+
function extractRateLimitRatio(rateLimits) {
|
|
3234
|
+
if (!isRecord(rateLimits)) {
|
|
3235
|
+
return null;
|
|
3236
|
+
}
|
|
3237
|
+
const limit = parseFiniteNumber(rateLimits.limit);
|
|
3238
|
+
const remaining = parseFiniteNumber(rateLimits.remaining);
|
|
3239
|
+
if (limit === null || remaining === null || limit <= 0 || remaining < 0) {
|
|
3240
|
+
return null;
|
|
3241
|
+
}
|
|
3242
|
+
return remaining / limit;
|
|
3243
|
+
}
|
|
3244
|
+
function isGitHubTrackerRateLimits(rateLimits) {
|
|
3245
|
+
if (!isRecord(rateLimits) || rateLimits.source !== "github") {
|
|
3246
|
+
return false;
|
|
3247
|
+
}
|
|
3248
|
+
return rateLimits.resource === void 0 || rateLimits.resource === null || rateLimits.resource === "graphql";
|
|
3249
|
+
}
|
|
3250
|
+
function isLowRateLimit(rateLimits, threshold) {
|
|
3251
|
+
const ratio = extractRateLimitRatio(rateLimits);
|
|
3252
|
+
return ratio !== null && ratio < threshold;
|
|
3253
|
+
}
|
|
3054
3254
|
function buildRuntimeSession(existing, sessionId, threadId, status, startedAt, updatedAt, exitClassification = void 0) {
|
|
3055
3255
|
if (existing === void 0 && sessionId === null && threadId === null && status === null && (exitClassification === void 0 || exitClassification === null)) {
|
|
3056
3256
|
return void 0;
|
|
@@ -3067,9 +3267,17 @@ function buildRuntimeSession(existing, sessionId, threadId, status, startedAt, u
|
|
|
3067
3267
|
function resolvePersistedCumulativeTurnCount(run) {
|
|
3068
3268
|
return run.cumulativeTurnCount ?? run.turnCount ?? 0;
|
|
3069
3269
|
}
|
|
3070
|
-
function hasConvergenceLockedRun(runs, issueId, issueState) {
|
|
3270
|
+
function hasConvergenceLockedRun(runs, issueId, issueState, issueUpdatedAt) {
|
|
3071
3271
|
const latestRun = runs.filter((run) => run.issueId === issueId).sort((left, right) => new Date(right.updatedAt).getTime() - new Date(left.updatedAt).getTime())[0];
|
|
3072
|
-
|
|
3272
|
+
if (latestRun?.runtimeSession?.exitClassification !== "convergence-detected" || latestRun.issueState !== issueState) {
|
|
3273
|
+
return false;
|
|
3274
|
+
}
|
|
3275
|
+
const convergedAtMs = parseTimestampMs2(latestRun.completedAt ?? latestRun.updatedAt);
|
|
3276
|
+
const issueUpdatedAtMs = parseTimestampMs2(issueUpdatedAt);
|
|
3277
|
+
if (convergedAtMs === null || issueUpdatedAtMs === null) {
|
|
3278
|
+
return true;
|
|
3279
|
+
}
|
|
3280
|
+
return issueUpdatedAtMs <= convergedAtMs;
|
|
3073
3281
|
}
|
|
3074
3282
|
function resolveIssueBudgetSnapshot(runs, issueId) {
|
|
3075
3283
|
const issueRuns = runs.filter((run) => run.issueId === issueId);
|
|
@@ -3089,12 +3297,12 @@ function resolveIssueBudgetSnapshot(runs, issueId) {
|
|
|
3089
3297
|
};
|
|
3090
3298
|
}
|
|
3091
3299
|
function isIssueBudgetExceeded(snapshot, now, env = process.env) {
|
|
3092
|
-
const globalMaxTurns = parsePositiveInteger(env.SYMPHONY_GLOBAL_MAX_TURNS ?? "");
|
|
3093
|
-
if (
|
|
3300
|
+
const globalMaxTurns = parsePositiveInteger(env.SYMPHONY_GLOBAL_MAX_TURNS ?? "") ?? DEFAULT_GLOBAL_MAX_TURNS;
|
|
3301
|
+
if (snapshot.cumulativeTurnCount >= globalMaxTurns) {
|
|
3094
3302
|
return true;
|
|
3095
3303
|
}
|
|
3096
|
-
const maxTokens = parsePositiveInteger(env.SYMPHONY_MAX_TOKENS ?? "");
|
|
3097
|
-
if (
|
|
3304
|
+
const maxTokens = parsePositiveInteger(env.SYMPHONY_MAX_TOKENS ?? "") ?? DEFAULT_MAX_TOKENS;
|
|
3305
|
+
if (snapshot.tokenUsage.totalTokens >= maxTokens) {
|
|
3098
3306
|
return true;
|
|
3099
3307
|
}
|
|
3100
3308
|
const sessionTimeoutMs = parsePositiveInteger(env.SYMPHONY_SESSION_TIMEOUT_MS ?? "");
|
|
@@ -3208,6 +3416,22 @@ function createRunId(now, projectId, issueIdentifier) {
|
|
|
3208
3416
|
now.getTime().toString(36)
|
|
3209
3417
|
].join("-");
|
|
3210
3418
|
}
|
|
3419
|
+
function buildLatestRunMapByIssueId(runs) {
|
|
3420
|
+
const latestRuns = /* @__PURE__ */ new Map();
|
|
3421
|
+
for (const run of runs) {
|
|
3422
|
+
const existing = latestRuns.get(run.issueId);
|
|
3423
|
+
if (!existing) {
|
|
3424
|
+
latestRuns.set(run.issueId, run);
|
|
3425
|
+
continue;
|
|
3426
|
+
}
|
|
3427
|
+
const runUpdatedAtMs = parseTimestampMs2(run.updatedAt) ?? -Infinity;
|
|
3428
|
+
const existingUpdatedAtMs = parseTimestampMs2(existing.updatedAt) ?? -Infinity;
|
|
3429
|
+
if (runUpdatedAtMs > existingUpdatedAtMs) {
|
|
3430
|
+
latestRuns.set(run.issueId, run);
|
|
3431
|
+
}
|
|
3432
|
+
}
|
|
3433
|
+
return latestRuns;
|
|
3434
|
+
}
|
|
3211
3435
|
function isIssueOrchestrationClaimed(state) {
|
|
3212
3436
|
return state === "claimed" || state === "running" || state === "retry_queued";
|
|
3213
3437
|
}
|
|
@@ -3218,7 +3442,8 @@ function upsertIssueOrchestration(issueRecords, nextRecord) {
|
|
|
3218
3442
|
...remaining,
|
|
3219
3443
|
{
|
|
3220
3444
|
...nextRecord,
|
|
3221
|
-
completedOnce: nextRecord.completedOnce ?? existingRecord?.completedOnce ?? false
|
|
3445
|
+
completedOnce: nextRecord.completedOnce ?? existingRecord?.completedOnce ?? false,
|
|
3446
|
+
failureRetryCount: nextRecord.failureRetryCount ?? existingRecord?.failureRetryCount ?? 0
|
|
3222
3447
|
}
|
|
3223
3448
|
];
|
|
3224
3449
|
}
|