@gh-symphony/cli 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -0
- package/dist/{chunk-IWR4UQEJ.js → chunk-5YLETHMR.js} +9 -358
- package/dist/chunk-62L6QQE6.js +362 -0
- package/dist/{chunk-JO3AXHQI.js → chunk-7UBUBSMH.js} +6 -2
- package/dist/chunk-C7G7RJ4G.js +146 -0
- package/dist/{chunk-EFMFGOWM.js → chunk-LZE6YUSB.js} +267 -53
- package/dist/{chunk-TF3QNWNC.js → chunk-OL73UN2X.js} +246 -212
- package/dist/{chunk-6HBZC3BE.js → chunk-XN5ABWZ6.js} +23 -5
- package/dist/{chunk-76QPITKI.js → chunk-Y6TYJMNT.js} +1 -1
- package/dist/{chunk-MHIWAIVD.js → chunk-ZYYY55WB.js} +70 -33
- package/dist/doctor-3QT5CZN4.js +532 -0
- package/dist/index.js +21 -9
- package/dist/{init-EZXQAXZM.js → init-E432UZ32.js} +3 -2
- package/dist/{logs-6LNGT2GF.js → logs-6JKKYDGJ.js} +1 -1
- package/dist/{project-557FE2GD.js → project-O57C32WF.js} +14 -12
- package/dist/{recover-LVBI2TGH.js → recover-UGUTQTWA.js} +3 -3
- package/dist/{run-WITYAYFZ.js → run-5H2R6CHB.js} +3 -3
- package/dist/{start-JUFKNL3N.js → start-5JGGJIMC.js} +5 -5
- package/dist/{status-3WK5BWRZ.js → status-QSCFVGRQ.js} +2 -2
- package/dist/{stop-AA3AP5M6.js → stop-7MFCBQVW.js} +2 -2
- package/dist/version-N7YXKG6V.js +16 -0
- package/dist/worker-entry.js +16 -10
- package/package.json +4 -4
- package/dist/chunk-TH5QPO3Y.js +0 -67
- package/dist/version-VBB62JWI.js +0 -30
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
|
+
DEFAULT_MAX_FAILURE_RETRIES,
|
|
3
4
|
DEFAULT_WORKFLOW_LIFECYCLE,
|
|
4
5
|
WorkflowConfigStore,
|
|
5
6
|
buildHookEnv,
|
|
@@ -25,7 +26,7 @@ import {
|
|
|
25
26
|
resolveIssueWorkspaceDirectory,
|
|
26
27
|
safeReadDir,
|
|
27
28
|
scheduleRetryAt
|
|
28
|
-
} from "./chunk-
|
|
29
|
+
} from "./chunk-OL73UN2X.js";
|
|
29
30
|
|
|
30
31
|
// ../orchestrator/dist/service.js
|
|
31
32
|
import { mkdir as mkdir3, readFile as readFile3, rm as rm3, writeFile as writeFile3 } from "fs/promises";
|
|
@@ -286,7 +287,8 @@ var OrchestratorFsStore = class {
|
|
|
286
287
|
if (issues) {
|
|
287
288
|
return issues.map((issue) => ({
|
|
288
289
|
...issue,
|
|
289
|
-
completedOnce: issue.completedOnce ?? false
|
|
290
|
+
completedOnce: issue.completedOnce ?? false,
|
|
291
|
+
failureRetryCount: issue.failureRetryCount ?? 0
|
|
290
292
|
}));
|
|
291
293
|
}
|
|
292
294
|
const legacyLeases = await readJsonFile(join2(this.projectDir(projectId), "leases.json")) ?? [];
|
|
@@ -298,6 +300,7 @@ var OrchestratorFsStore = class {
|
|
|
298
300
|
identifier: lease.issueIdentifier,
|
|
299
301
|
workspaceKey: deriveIssueWorkspaceKeyFromIdentifier(lease.issueIdentifier),
|
|
300
302
|
completedOnce: false,
|
|
303
|
+
failureRetryCount: 0,
|
|
301
304
|
state: lease.status === "active" ? "claimed" : "released",
|
|
302
305
|
currentRunId: lease.status === "active" ? lease.runId : null,
|
|
303
306
|
retryEntry: null,
|
|
@@ -464,9 +467,12 @@ async function pathExists(path) {
|
|
|
464
467
|
}
|
|
465
468
|
|
|
466
469
|
// ../tracker-github/dist/adapter.js
|
|
470
|
+
import { createHash } from "crypto";
|
|
467
471
|
var DEFAULT_API_URL = "https://api.github.com/graphql";
|
|
468
472
|
var DEFAULT_PAGE_SIZE = 25;
|
|
469
473
|
var DEFAULT_NETWORK_TIMEOUT_MS = 3e4;
|
|
474
|
+
var RATE_LIMIT_THRESHOLD = 100;
|
|
475
|
+
var MAX_RATE_LIMIT_WAIT_MS = 6e4;
|
|
470
476
|
var GitHubTrackerError = class extends Error {
|
|
471
477
|
};
|
|
472
478
|
var GitHubTrackerHttpError = class extends GitHubTrackerError {
|
|
@@ -480,6 +486,7 @@ var GitHubTrackerHttpError = class extends GitHubTrackerError {
|
|
|
480
486
|
};
|
|
481
487
|
var GitHubTrackerQueryError = class extends GitHubTrackerError {
|
|
482
488
|
};
|
|
489
|
+
var cachedGitHubGraphQLRateLimits = /* @__PURE__ */ new Map();
|
|
483
490
|
function normalizeProjectItem(projectId, item, lifecycle = DEFAULT_WORKFLOW_LIFECYCLE, priority = {}, rateLimits = null) {
|
|
484
491
|
if (item.content?.__typename !== "Issue") {
|
|
485
492
|
return null;
|
|
@@ -814,6 +821,8 @@ async function executeGraphQLQuery(config, query, variables, fetchImpl) {
|
|
|
814
821
|
return result.data;
|
|
815
822
|
}
|
|
816
823
|
async function executeGraphQLQueryWithMetadata(config, query, variables, fetchImpl) {
|
|
824
|
+
const tokenFingerprint = fingerprintToken(config.token);
|
|
825
|
+
await guardGraphQLRateLimit(tokenFingerprint);
|
|
817
826
|
const response = await fetchImpl(config.apiUrl ?? DEFAULT_API_URL, {
|
|
818
827
|
method: "POST",
|
|
819
828
|
headers: {
|
|
@@ -838,11 +847,38 @@ async function executeGraphQLQueryWithMetadata(config, query, variables, fetchIm
|
|
|
838
847
|
throw new GitHubTrackerQueryError("GitHub GraphQL response did not include data.");
|
|
839
848
|
}
|
|
840
849
|
const data = payload.data;
|
|
850
|
+
const rateLimits = extractGitHubRateLimits(response.headers);
|
|
851
|
+
cachedGitHubGraphQLRateLimits.set(tokenFingerprint, rateLimits);
|
|
841
852
|
return {
|
|
842
853
|
data,
|
|
843
|
-
rateLimits
|
|
854
|
+
rateLimits
|
|
844
855
|
};
|
|
845
856
|
}
|
|
857
|
+
async function guardGraphQLRateLimit(tokenFingerprint) {
|
|
858
|
+
const rateLimit = cachedGitHubGraphQLRateLimits.get(tokenFingerprint) ?? null;
|
|
859
|
+
if (!rateLimit) {
|
|
860
|
+
return;
|
|
861
|
+
}
|
|
862
|
+
const remaining = rateLimit.remaining;
|
|
863
|
+
if (remaining === null || remaining > RATE_LIMIT_THRESHOLD) {
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
const resetAtMs = parseTimestampMs(rateLimit.resetAt);
|
|
867
|
+
if (resetAtMs === null) {
|
|
868
|
+
throw new GitHubTrackerError("Rate limit near exhaustion");
|
|
869
|
+
}
|
|
870
|
+
const waitMs = Math.max(0, resetAtMs - Date.now());
|
|
871
|
+
if (waitMs > MAX_RATE_LIMIT_WAIT_MS) {
|
|
872
|
+
throw new GitHubTrackerError("Rate limit near exhaustion");
|
|
873
|
+
}
|
|
874
|
+
cachedGitHubGraphQLRateLimits.delete(tokenFingerprint);
|
|
875
|
+
if (waitMs > 0) {
|
|
876
|
+
await sleep(waitMs);
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
function fingerprintToken(token) {
|
|
880
|
+
return createHash("sha256").update(token).digest("hex");
|
|
881
|
+
}
|
|
846
882
|
function extractGitHubRateLimits(headers) {
|
|
847
883
|
if (!headers || typeof headers.get !== "function") {
|
|
848
884
|
return null;
|
|
@@ -872,6 +908,18 @@ function parseIntegerHeader(value) {
|
|
|
872
908
|
const parsed = Number.parseInt(value, 10);
|
|
873
909
|
return Number.isFinite(parsed) ? parsed : null;
|
|
874
910
|
}
|
|
911
|
+
function parseTimestampMs(value) {
|
|
912
|
+
if (!value) {
|
|
913
|
+
return null;
|
|
914
|
+
}
|
|
915
|
+
const timestampMs = Date.parse(value);
|
|
916
|
+
return Number.isFinite(timestampMs) ? timestampMs : null;
|
|
917
|
+
}
|
|
918
|
+
function sleep(ms) {
|
|
919
|
+
return new Promise((resolve4) => {
|
|
920
|
+
setTimeout(resolve4, ms);
|
|
921
|
+
});
|
|
922
|
+
}
|
|
875
923
|
var PROJECT_ITEMS_QUERY = `
|
|
876
924
|
query ProjectItems($projectId: ID!, $cursor: String, $pageSize: Int!) {
|
|
877
925
|
node(id: $projectId) {
|
|
@@ -1086,7 +1134,7 @@ var ISSUE_PROJECT_ITEMS_PAGE_QUERY = `
|
|
|
1086
1134
|
`;
|
|
1087
1135
|
|
|
1088
1136
|
// ../tracker-github/dist/orchestrator-adapter.js
|
|
1089
|
-
import { createHash } from "crypto";
|
|
1137
|
+
import { createHash as createHash2 } from "crypto";
|
|
1090
1138
|
var githubProjectTrackerAdapter = {
|
|
1091
1139
|
async listIssues(project, dependencies = {}) {
|
|
1092
1140
|
return listProjectIssues(project, dependencies);
|
|
@@ -1174,7 +1222,7 @@ function hashToken(token) {
|
|
|
1174
1222
|
if (!token) {
|
|
1175
1223
|
return null;
|
|
1176
1224
|
}
|
|
1177
|
-
return
|
|
1225
|
+
return createHash2("sha256").update(token).digest("hex");
|
|
1178
1226
|
}
|
|
1179
1227
|
var trackerAdapters = {
|
|
1180
1228
|
"github-project": githubProjectTrackerAdapter
|
|
@@ -1333,19 +1381,33 @@ var DEFAULT_POLL_INTERVAL_MS = 3e4;
|
|
|
1333
1381
|
var DEFAULT_CONCURRENCY = 3;
|
|
1334
1382
|
var DEFAULT_RETRY_BACKOFF_MS = 3e4;
|
|
1335
1383
|
var CONTINUATION_RETRY_DELAY_MS = 1e3;
|
|
1384
|
+
var DEFAULT_GLOBAL_MAX_TURNS = 100;
|
|
1385
|
+
var DEFAULT_MAX_TOKENS = 256e3;
|
|
1336
1386
|
var DEFAULT_WORKER_COMMAND = "node packages/worker/dist/index.js";
|
|
1337
1387
|
var DEFAULT_MAX_NONPRODUCTIVE_TURNS = 3;
|
|
1388
|
+
var LOW_RATE_LIMIT_WARNING_THRESHOLD = 0.05;
|
|
1389
|
+
var MAX_FAILURE_RETRIES_EXCEEDED_REASON = "max_failure_retries_exceeded";
|
|
1338
1390
|
var STUCK_WORKER_TIMEOUT_MS = 30 * 60 * 1e3;
|
|
1339
1391
|
function isUsableWorkflowResolution(resolution) {
|
|
1340
1392
|
return resolution.isValid || resolution.usedLastKnownGood;
|
|
1341
1393
|
}
|
|
1342
|
-
function
|
|
1394
|
+
function parseTimestampMs2(value) {
|
|
1343
1395
|
if (!value) {
|
|
1344
1396
|
return null;
|
|
1345
1397
|
}
|
|
1346
1398
|
const parsed = new Date(value).getTime();
|
|
1347
1399
|
return Number.isFinite(parsed) ? parsed : null;
|
|
1348
1400
|
}
|
|
1401
|
+
function parseFiniteNumber(value) {
|
|
1402
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
1403
|
+
return value;
|
|
1404
|
+
}
|
|
1405
|
+
if (typeof value === "string" && value.trim()) {
|
|
1406
|
+
const parsed = Number(value);
|
|
1407
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
1408
|
+
}
|
|
1409
|
+
return null;
|
|
1410
|
+
}
|
|
1349
1411
|
var OrchestratorService = class {
|
|
1350
1412
|
store;
|
|
1351
1413
|
projectConfig;
|
|
@@ -1513,11 +1575,12 @@ var OrchestratorService = class {
|
|
|
1513
1575
|
let recovered = 0;
|
|
1514
1576
|
let pollIntervalMs = DEFAULT_POLL_INTERVAL_MS;
|
|
1515
1577
|
let rateLimits = null;
|
|
1578
|
+
let trackerRateLimits = null;
|
|
1516
1579
|
let issueRecords = await this.store.loadProjectIssueOrchestrations(tenant.projectId);
|
|
1517
1580
|
const allRuns = (await this.store.loadAllRuns()).filter((run) => run.projectId === tenant.projectId);
|
|
1518
1581
|
const activeRuns = allRuns.filter((run) => isActiveRunStatus(run.status));
|
|
1519
1582
|
for (const run of activeRuns) {
|
|
1520
|
-
const outcome = await this.reconcileRun(tenant, run, issueRecords);
|
|
1583
|
+
const outcome = await this.reconcileRun(tenant, run, issueRecords, trackerDependencies);
|
|
1521
1584
|
issueRecords = outcome.issueRecords;
|
|
1522
1585
|
if (outcome.recovered) {
|
|
1523
1586
|
recovered += 1;
|
|
@@ -1555,9 +1618,11 @@ var OrchestratorService = class {
|
|
|
1555
1618
|
});
|
|
1556
1619
|
}
|
|
1557
1620
|
rateLimits = resolveProjectRateLimits(syncedActiveRuns, trackedIssuesByIdentifier.values());
|
|
1621
|
+
trackerRateLimits = resolveTrackerRateLimits(trackedIssuesByIdentifier.values());
|
|
1558
1622
|
const concurrency = await this.getProjectConcurrency(tenant);
|
|
1559
1623
|
const currentlyActive = issueRecords.filter((record) => isIssueOrchestrationClaimed(record.state)).length;
|
|
1560
1624
|
const availableSlots = Math.max(0, concurrency - currentlyActive);
|
|
1625
|
+
const latestRunsByIssueId = buildLatestRunMapByIssueId(projectRunsAfterReconcile);
|
|
1561
1626
|
const unscheduledCandidates = actionableCandidates.filter((issue) => {
|
|
1562
1627
|
if (hasConvergenceLockedRun(projectRunsAfterReconcile, issue.id, issue.state)) {
|
|
1563
1628
|
return false;
|
|
@@ -1579,6 +1644,9 @@ var OrchestratorService = class {
|
|
|
1579
1644
|
}
|
|
1580
1645
|
if (slotsRemaining <= 0)
|
|
1581
1646
|
break;
|
|
1647
|
+
if (await this.isFailureRetrySuppressedIssue(tenant, issue, issueRecords, latestRunsByIssueId.get(issue.id) ?? null)) {
|
|
1648
|
+
continue;
|
|
1649
|
+
}
|
|
1582
1650
|
if (isIssueBudgetExceeded(resolveIssueBudgetSnapshot(projectRunsAfterReconcile, issue.id), now)) {
|
|
1583
1651
|
continue;
|
|
1584
1652
|
}
|
|
@@ -1599,6 +1667,7 @@ var OrchestratorService = class {
|
|
|
1599
1667
|
identifier: issue.identifier,
|
|
1600
1668
|
workspaceKey: preferredWorkspaceKey,
|
|
1601
1669
|
state: "claimed",
|
|
1670
|
+
failureRetryCount: 0,
|
|
1602
1671
|
currentRunId: null,
|
|
1603
1672
|
retryEntry: null,
|
|
1604
1673
|
updatedAt: now.toISOString()
|
|
@@ -1680,7 +1749,11 @@ var OrchestratorService = class {
|
|
|
1680
1749
|
} catch (error) {
|
|
1681
1750
|
lastError = error instanceof Error ? error.message : "Unknown orchestration error";
|
|
1682
1751
|
}
|
|
1683
|
-
|
|
1752
|
+
const effectivePollIntervalMs = resolveAdaptivePollIntervalMs(pollIntervalMs, trackerRateLimits);
|
|
1753
|
+
if (effectivePollIntervalMs > pollIntervalMs && isLowRateLimit(trackerRateLimits, LOW_RATE_LIMIT_WARNING_THRESHOLD)) {
|
|
1754
|
+
this.writeStderr(`[orchestrator] low GitHub rate limit for ${tenant.projectId}: interval=${effectivePollIntervalMs}ms rateLimits=${JSON.stringify(trackerRateLimits)}`);
|
|
1755
|
+
}
|
|
1756
|
+
this.projectPollIntervals.set(tenant.projectId, effectivePollIntervalMs);
|
|
1684
1757
|
await this.store.saveProjectIssueOrchestrations(tenant.projectId, issueRecords);
|
|
1685
1758
|
const allTenantRuns = (await this.store.loadAllRuns()).filter((run) => run.projectId === tenant.projectId);
|
|
1686
1759
|
const latestRuns = allTenantRuns.filter((run) => isActiveRunStatus(run.status));
|
|
@@ -1874,26 +1947,9 @@ var OrchestratorService = class {
|
|
|
1874
1947
|
if (!lifecycle) {
|
|
1875
1948
|
lifecycle = resolution.lifecycle;
|
|
1876
1949
|
}
|
|
1877
|
-
if (!
|
|
1950
|
+
if (!this.isIssueCandidateEligible(issue, resolution.lifecycle, issues)) {
|
|
1878
1951
|
continue;
|
|
1879
1952
|
}
|
|
1880
|
-
if (matchesWorkflowState(issue.state, resolution.lifecycle.blockerCheckStates) && issue.blockedBy.length > 0) {
|
|
1881
|
-
const hasNonTerminalBlocker = issue.blockedBy.some((blockerRef) => {
|
|
1882
|
-
if (blockerRef.state && isStateTerminal(blockerRef.state, resolution.lifecycle)) {
|
|
1883
|
-
return false;
|
|
1884
|
-
}
|
|
1885
|
-
if (blockerRef.identifier) {
|
|
1886
|
-
const blockerIssue = issues.find((candidate) => candidate.identifier === blockerRef.identifier);
|
|
1887
|
-
if (blockerIssue?.state) {
|
|
1888
|
-
return !isStateTerminal(blockerIssue.state, resolution.lifecycle);
|
|
1889
|
-
}
|
|
1890
|
-
}
|
|
1891
|
-
return true;
|
|
1892
|
-
});
|
|
1893
|
-
if (hasNonTerminalBlocker) {
|
|
1894
|
-
continue;
|
|
1895
|
-
}
|
|
1896
|
-
}
|
|
1897
1953
|
candidates.push(issue);
|
|
1898
1954
|
}
|
|
1899
1955
|
if (!lifecycle && tenant.repositories.length > 0) {
|
|
@@ -1912,6 +1968,26 @@ var OrchestratorService = class {
|
|
|
1912
1968
|
}
|
|
1913
1969
|
};
|
|
1914
1970
|
}
|
|
1971
|
+
isIssueCandidateEligible(issue, lifecycle, issues) {
|
|
1972
|
+
if (!isStateActive(issue.state, lifecycle)) {
|
|
1973
|
+
return false;
|
|
1974
|
+
}
|
|
1975
|
+
if (!matchesWorkflowState(issue.state, lifecycle.blockerCheckStates) || issue.blockedBy.length === 0) {
|
|
1976
|
+
return true;
|
|
1977
|
+
}
|
|
1978
|
+
return !issue.blockedBy.some((blockerRef) => {
|
|
1979
|
+
if (blockerRef.state && isStateTerminal(blockerRef.state, lifecycle)) {
|
|
1980
|
+
return false;
|
|
1981
|
+
}
|
|
1982
|
+
if (blockerRef.identifier) {
|
|
1983
|
+
const blockerIssue = issues.find((candidate) => candidate.identifier === blockerRef.identifier);
|
|
1984
|
+
if (blockerIssue?.state) {
|
|
1985
|
+
return !isStateTerminal(blockerIssue.state, lifecycle);
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
return true;
|
|
1989
|
+
});
|
|
1990
|
+
}
|
|
1915
1991
|
async loadProjectWorkflow(tenant, repository) {
|
|
1916
1992
|
const cacheKey = this.workflowCacheKey(repository);
|
|
1917
1993
|
const pendingCache = this.workflowResolutionCache;
|
|
@@ -2233,13 +2309,13 @@ var OrchestratorService = class {
|
|
|
2233
2309
|
issuesByIdentifier
|
|
2234
2310
|
};
|
|
2235
2311
|
}
|
|
2236
|
-
async reconcileRun(tenant, run, issueRecords) {
|
|
2312
|
+
async reconcileRun(tenant, run, issueRecords, trackerDependencies = {}) {
|
|
2237
2313
|
const now = this.now();
|
|
2238
2314
|
if (run.processId && this.isProcessRunning(run.processId)) {
|
|
2239
2315
|
const retryPolicy = await this.loadRetryPolicy(tenant, run.repository);
|
|
2240
2316
|
const configuredStallTimeoutMs = retryPolicy?.stallTimeoutMs ?? null;
|
|
2241
|
-
const lastActivityAtMs =
|
|
2242
|
-
const startedAtMs =
|
|
2317
|
+
const lastActivityAtMs = parseTimestampMs2(run.lastEventAt ?? run.startedAt);
|
|
2318
|
+
const startedAtMs = parseTimestampMs2(run.startedAt);
|
|
2243
2319
|
const elapsedSinceLastActivityMs = lastActivityAtMs === null ? null : now.getTime() - lastActivityAtMs;
|
|
2244
2320
|
const runningSinceMs = startedAtMs === null ? null : now.getTime() - startedAtMs;
|
|
2245
2321
|
const isStalledByWorkflowTimeout = configuredStallTimeoutMs !== null && configuredStallTimeoutMs > 0 && elapsedSinceLastActivityMs !== null && elapsedSinceLastActivityMs > configuredStallTimeoutMs;
|
|
@@ -2318,7 +2394,7 @@ var OrchestratorService = class {
|
|
|
2318
2394
|
recovered: false
|
|
2319
2395
|
};
|
|
2320
2396
|
}
|
|
2321
|
-
if (await this.resolveRetryRestartAction(tenant, run) === "release") {
|
|
2397
|
+
if (await this.resolveRetryRestartAction(tenant, run, trackerDependencies) === "release") {
|
|
2322
2398
|
return this.releaseRetryingRun(runWithTokens, issueRecords, now);
|
|
2323
2399
|
}
|
|
2324
2400
|
return this.restartRun(tenant, run, issueRecords, now, workerSessionId);
|
|
@@ -2355,7 +2431,54 @@ var OrchestratorService = class {
|
|
|
2355
2431
|
state: run.issueState
|
|
2356
2432
|
});
|
|
2357
2433
|
}
|
|
2358
|
-
const retryKind = await this.classifyRetryKind(tenant, run);
|
|
2434
|
+
const retryKind = await this.classifyRetryKind(tenant, run, trackerDependencies);
|
|
2435
|
+
const failureRetryCount = retryKind === "failure" ? (this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0) + 1 : this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0;
|
|
2436
|
+
const maxFailureRetries = await this.loadMaxFailureRetries(tenant, run.repository);
|
|
2437
|
+
if (retryKind === "failure" && failureRetryCount >= maxFailureRetries) {
|
|
2438
|
+
const lastError = [
|
|
2439
|
+
`Run suppressed: ${MAX_FAILURE_RETRIES_EXCEEDED_REASON}.`,
|
|
2440
|
+
`failureRetryCount=${failureRetryCount}.`,
|
|
2441
|
+
`maxFailureRetries=${maxFailureRetries}.`
|
|
2442
|
+
].join(" ");
|
|
2443
|
+
const suppressedRun = {
|
|
2444
|
+
...runWithTokens,
|
|
2445
|
+
status: "suppressed",
|
|
2446
|
+
processId: null,
|
|
2447
|
+
updatedAt: now.toISOString(),
|
|
2448
|
+
completedAt: now.toISOString(),
|
|
2449
|
+
nextRetryAt: null,
|
|
2450
|
+
retryKind: null,
|
|
2451
|
+
runPhase: runWithTokens.runPhase ?? "failed",
|
|
2452
|
+
lastError
|
|
2453
|
+
};
|
|
2454
|
+
await this.store.saveRun(suppressedRun);
|
|
2455
|
+
await this.store.appendRunEvent(run.runId, {
|
|
2456
|
+
at: now.toISOString(),
|
|
2457
|
+
event: "run-suppressed",
|
|
2458
|
+
projectId: run.projectId,
|
|
2459
|
+
issueIdentifier: run.issueIdentifier,
|
|
2460
|
+
issueId: run.issueId,
|
|
2461
|
+
reason: MAX_FAILURE_RETRIES_EXCEEDED_REASON
|
|
2462
|
+
});
|
|
2463
|
+
this.logVerbose(`[run-completed] ${suppressedRun.runId} status=${suppressedRun.status}`);
|
|
2464
|
+
return {
|
|
2465
|
+
issueRecords: upsertIssueOrchestration(issueRecords, {
|
|
2466
|
+
issueId: run.issueId,
|
|
2467
|
+
identifier: run.issueIdentifier,
|
|
2468
|
+
workspaceKey: run.issueWorkspaceKey ?? deriveIssueWorkspaceKey({
|
|
2469
|
+
projectId: tenant.projectId,
|
|
2470
|
+
adapter: tenant.tracker.adapter,
|
|
2471
|
+
issueSubjectId: run.issueSubjectId
|
|
2472
|
+
}, run.issueIdentifier),
|
|
2473
|
+
state: "released",
|
|
2474
|
+
failureRetryCount,
|
|
2475
|
+
currentRunId: null,
|
|
2476
|
+
retryEntry: null,
|
|
2477
|
+
updatedAt: now.toISOString()
|
|
2478
|
+
}),
|
|
2479
|
+
recovered: false
|
|
2480
|
+
};
|
|
2481
|
+
}
|
|
2359
2482
|
let nextRetryAt;
|
|
2360
2483
|
if (retryKind === "continuation") {
|
|
2361
2484
|
nextRetryAt = new Date(now.getTime() + CONTINUATION_RETRY_DELAY_MS).toISOString();
|
|
@@ -2391,6 +2514,7 @@ var OrchestratorService = class {
|
|
|
2391
2514
|
}, run.issueIdentifier),
|
|
2392
2515
|
state: "retry_queued",
|
|
2393
2516
|
completedOnce: retryKind === "continuation" ? true : void 0,
|
|
2517
|
+
failureRetryCount,
|
|
2394
2518
|
currentRunId: run.runId,
|
|
2395
2519
|
retryEntry: {
|
|
2396
2520
|
attempt: retryRecord.attempt,
|
|
@@ -2605,49 +2729,47 @@ var OrchestratorService = class {
|
|
|
2605
2729
|
* — the worker completed its session and the issue hasn't transitioned away.
|
|
2606
2730
|
* Failure applies when we cannot confirm the issue is still actionable.
|
|
2607
2731
|
*/
|
|
2608
|
-
async classifyRetryKind(tenant, run) {
|
|
2732
|
+
async classifyRetryKind(tenant, run, trackerDependencies = {}) {
|
|
2609
2733
|
try {
|
|
2610
|
-
const
|
|
2611
|
-
|
|
2612
|
-
fetchImpl: this.dependencies.fetchImpl
|
|
2613
|
-
});
|
|
2614
|
-
const runIssue = issues.find((issue) => issue.identifier === run.issueIdentifier);
|
|
2615
|
-
if (!runIssue) {
|
|
2734
|
+
const eligibleContext = await this.fetchTrackedIssueEligibilityContext(tenant, run.issueIdentifier, trackerDependencies);
|
|
2735
|
+
if (!eligibleContext) {
|
|
2616
2736
|
return "failure";
|
|
2617
2737
|
}
|
|
2618
2738
|
const resolution = await this.loadProjectWorkflow(tenant, run.repository);
|
|
2619
2739
|
if (!isUsableWorkflowResolution(resolution)) {
|
|
2620
2740
|
return "failure";
|
|
2621
2741
|
}
|
|
2622
|
-
return
|
|
2742
|
+
return this.isIssueCandidateEligible(eligibleContext.issue, resolution.lifecycle, eligibleContext.issues) ? "continuation" : "failure";
|
|
2623
2743
|
} catch {
|
|
2624
2744
|
return "failure";
|
|
2625
2745
|
}
|
|
2626
2746
|
}
|
|
2627
|
-
async resolveRetryRestartAction(tenant, run) {
|
|
2747
|
+
async resolveRetryRestartAction(tenant, run, trackerDependencies = {}) {
|
|
2628
2748
|
try {
|
|
2629
2749
|
if (isIssueBudgetExceeded(resolveIssueBudgetSnapshot((await this.store.loadAllRuns()).filter((candidate) => candidate.projectId === tenant.projectId), run.issueId), this.now())) {
|
|
2630
2750
|
return "release";
|
|
2631
2751
|
}
|
|
2632
|
-
const
|
|
2633
|
-
if (!
|
|
2752
|
+
const eligibleContext = await this.fetchTrackedIssueEligibilityContext(tenant, run.issueIdentifier, trackerDependencies);
|
|
2753
|
+
if (!eligibleContext) {
|
|
2634
2754
|
return "release";
|
|
2635
2755
|
}
|
|
2636
2756
|
const resolution = await this.loadProjectWorkflow(tenant, run.repository);
|
|
2637
2757
|
if (!isUsableWorkflowResolution(resolution)) {
|
|
2638
2758
|
return "restart";
|
|
2639
2759
|
}
|
|
2640
|
-
return
|
|
2760
|
+
return this.isIssueCandidateEligible(eligibleContext.issue, resolution.lifecycle, eligibleContext.issues) ? "restart" : "release";
|
|
2641
2761
|
} catch {
|
|
2642
2762
|
return "restart";
|
|
2643
2763
|
}
|
|
2644
2764
|
}
|
|
2645
|
-
async
|
|
2765
|
+
async fetchTrackedIssueEligibilityContext(tenant, issueIdentifier, trackerDependencies = {}) {
|
|
2646
2766
|
const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
|
|
2647
|
-
const issues = await trackerAdapter.
|
|
2648
|
-
fetchImpl: this.dependencies.fetchImpl
|
|
2767
|
+
const issues = await trackerAdapter.listIssues(tenant, {
|
|
2768
|
+
fetchImpl: this.dependencies.fetchImpl,
|
|
2769
|
+
...trackerDependencies
|
|
2649
2770
|
});
|
|
2650
|
-
|
|
2771
|
+
const issue = issues.find((candidate) => candidate.identifier === issueIdentifier);
|
|
2772
|
+
return issue ? { issue, issues } : null;
|
|
2651
2773
|
}
|
|
2652
2774
|
async fetchWorkerRunInfo(run) {
|
|
2653
2775
|
const latestRun = await this.store.loadRun(run.runId, run.projectId) ?? run;
|
|
@@ -3020,6 +3142,36 @@ var OrchestratorService = class {
|
|
|
3020
3142
|
};
|
|
3021
3143
|
await this.store.saveIssueWorkspace(removedRecord);
|
|
3022
3144
|
}
|
|
3145
|
+
resolveFailureRetryCount(issueRecords, issueId) {
|
|
3146
|
+
return issueRecords.find((record) => record.issueId === issueId)?.failureRetryCount ?? null;
|
|
3147
|
+
}
|
|
3148
|
+
async isFailureRetrySuppressedIssue(tenant, issue, issueRecords, latestRun) {
|
|
3149
|
+
const issueRecord = issueRecords.find((record) => record.issueId === issue.id || record.identifier === issue.identifier) ?? null;
|
|
3150
|
+
if (!issueRecord || issueRecord.failureRetryCount <= 0) {
|
|
3151
|
+
return false;
|
|
3152
|
+
}
|
|
3153
|
+
const maxFailureRetries = await this.loadMaxFailureRetries(tenant, issue.repository);
|
|
3154
|
+
if (issueRecord.failureRetryCount < maxFailureRetries) {
|
|
3155
|
+
return false;
|
|
3156
|
+
}
|
|
3157
|
+
if (!latestRun || latestRun.status !== "suppressed" || latestRun.issueState !== issue.state || !latestRun.lastError?.includes(MAX_FAILURE_RETRIES_EXCEEDED_REASON)) {
|
|
3158
|
+
return false;
|
|
3159
|
+
}
|
|
3160
|
+
const issueUpdatedAtMs = parseTimestampMs2(issue.updatedAt);
|
|
3161
|
+
const suppressedAtMs = parseTimestampMs2(latestRun.completedAt ?? latestRun.updatedAt);
|
|
3162
|
+
if (issueUpdatedAtMs === null || suppressedAtMs === null) {
|
|
3163
|
+
return true;
|
|
3164
|
+
}
|
|
3165
|
+
return issueUpdatedAtMs <= suppressedAtMs;
|
|
3166
|
+
}
|
|
3167
|
+
async loadMaxFailureRetries(tenant, repository) {
|
|
3168
|
+
try {
|
|
3169
|
+
const resolution = await this.loadProjectWorkflow(tenant, repository);
|
|
3170
|
+
return isUsableWorkflowResolution(resolution) ? resolution.workflow.agent.maxFailureRetries : DEFAULT_MAX_FAILURE_RETRIES;
|
|
3171
|
+
} catch {
|
|
3172
|
+
return DEFAULT_MAX_FAILURE_RETRIES;
|
|
3173
|
+
}
|
|
3174
|
+
}
|
|
3023
3175
|
};
|
|
3024
3176
|
function hasTokenUsage(tokenUsage) {
|
|
3025
3177
|
return Boolean(tokenUsage && (tokenUsage.inputTokens > 0 || tokenUsage.outputTokens > 0 || tokenUsage.totalTokens > 0));
|
|
@@ -3034,7 +3186,7 @@ function resolveProjectRateLimits(runs, issues) {
|
|
|
3034
3186
|
if (!isRecord(run.rateLimits)) {
|
|
3035
3187
|
continue;
|
|
3036
3188
|
}
|
|
3037
|
-
const timestamp =
|
|
3189
|
+
const timestamp = parseTimestampMs2(run.lastEventAt ?? run.updatedAt ?? run.startedAt);
|
|
3038
3190
|
const sortableTimestamp = timestamp ?? -Infinity;
|
|
3039
3191
|
if (sortableTimestamp >= latestRunTimestamp) {
|
|
3040
3192
|
latestRunTimestamp = sortableTimestamp;
|
|
@@ -3051,6 +3203,51 @@ function resolveProjectRateLimits(runs, issues) {
|
|
|
3051
3203
|
}
|
|
3052
3204
|
return null;
|
|
3053
3205
|
}
|
|
3206
|
+
function resolveTrackerRateLimits(issues) {
|
|
3207
|
+
for (const issue of issues) {
|
|
3208
|
+
if (isGitHubTrackerRateLimits(issue.rateLimits)) {
|
|
3209
|
+
return issue.rateLimits;
|
|
3210
|
+
}
|
|
3211
|
+
}
|
|
3212
|
+
return null;
|
|
3213
|
+
}
|
|
3214
|
+
function resolveAdaptivePollIntervalMs(basePollIntervalMs, rateLimits) {
|
|
3215
|
+
if (!Number.isFinite(basePollIntervalMs) || basePollIntervalMs <= 0) {
|
|
3216
|
+
return DEFAULT_POLL_INTERVAL_MS;
|
|
3217
|
+
}
|
|
3218
|
+
const ratio = extractRateLimitRatio(rateLimits);
|
|
3219
|
+
if (ratio === null || ratio > 0.5) {
|
|
3220
|
+
return basePollIntervalMs;
|
|
3221
|
+
}
|
|
3222
|
+
if (ratio >= 0.2) {
|
|
3223
|
+
return basePollIntervalMs * 2;
|
|
3224
|
+
}
|
|
3225
|
+
if (ratio >= LOW_RATE_LIMIT_WARNING_THRESHOLD) {
|
|
3226
|
+
return basePollIntervalMs * 4;
|
|
3227
|
+
}
|
|
3228
|
+
return basePollIntervalMs * 10;
|
|
3229
|
+
}
|
|
3230
|
+
function extractRateLimitRatio(rateLimits) {
|
|
3231
|
+
if (!isRecord(rateLimits)) {
|
|
3232
|
+
return null;
|
|
3233
|
+
}
|
|
3234
|
+
const limit = parseFiniteNumber(rateLimits.limit);
|
|
3235
|
+
const remaining = parseFiniteNumber(rateLimits.remaining);
|
|
3236
|
+
if (limit === null || remaining === null || limit <= 0 || remaining < 0) {
|
|
3237
|
+
return null;
|
|
3238
|
+
}
|
|
3239
|
+
return remaining / limit;
|
|
3240
|
+
}
|
|
3241
|
+
function isGitHubTrackerRateLimits(rateLimits) {
|
|
3242
|
+
if (!isRecord(rateLimits) || rateLimits.source !== "github") {
|
|
3243
|
+
return false;
|
|
3244
|
+
}
|
|
3245
|
+
return rateLimits.resource === void 0 || rateLimits.resource === null || rateLimits.resource === "graphql";
|
|
3246
|
+
}
|
|
3247
|
+
function isLowRateLimit(rateLimits, threshold) {
|
|
3248
|
+
const ratio = extractRateLimitRatio(rateLimits);
|
|
3249
|
+
return ratio !== null && ratio < threshold;
|
|
3250
|
+
}
|
|
3054
3251
|
function buildRuntimeSession(existing, sessionId, threadId, status, startedAt, updatedAt, exitClassification = void 0) {
|
|
3055
3252
|
if (existing === void 0 && sessionId === null && threadId === null && status === null && (exitClassification === void 0 || exitClassification === null)) {
|
|
3056
3253
|
return void 0;
|
|
@@ -3089,12 +3286,12 @@ function resolveIssueBudgetSnapshot(runs, issueId) {
|
|
|
3089
3286
|
};
|
|
3090
3287
|
}
|
|
3091
3288
|
function isIssueBudgetExceeded(snapshot, now, env = process.env) {
|
|
3092
|
-
const globalMaxTurns = parsePositiveInteger(env.SYMPHONY_GLOBAL_MAX_TURNS ?? "");
|
|
3093
|
-
if (
|
|
3289
|
+
const globalMaxTurns = parsePositiveInteger(env.SYMPHONY_GLOBAL_MAX_TURNS ?? "") ?? DEFAULT_GLOBAL_MAX_TURNS;
|
|
3290
|
+
if (snapshot.cumulativeTurnCount >= globalMaxTurns) {
|
|
3094
3291
|
return true;
|
|
3095
3292
|
}
|
|
3096
|
-
const maxTokens = parsePositiveInteger(env.SYMPHONY_MAX_TOKENS ?? "");
|
|
3097
|
-
if (
|
|
3293
|
+
const maxTokens = parsePositiveInteger(env.SYMPHONY_MAX_TOKENS ?? "") ?? DEFAULT_MAX_TOKENS;
|
|
3294
|
+
if (snapshot.tokenUsage.totalTokens >= maxTokens) {
|
|
3098
3295
|
return true;
|
|
3099
3296
|
}
|
|
3100
3297
|
const sessionTimeoutMs = parsePositiveInteger(env.SYMPHONY_SESSION_TIMEOUT_MS ?? "");
|
|
@@ -3208,6 +3405,22 @@ function createRunId(now, projectId, issueIdentifier) {
|
|
|
3208
3405
|
now.getTime().toString(36)
|
|
3209
3406
|
].join("-");
|
|
3210
3407
|
}
|
|
3408
|
+
function buildLatestRunMapByIssueId(runs) {
|
|
3409
|
+
const latestRuns = /* @__PURE__ */ new Map();
|
|
3410
|
+
for (const run of runs) {
|
|
3411
|
+
const existing = latestRuns.get(run.issueId);
|
|
3412
|
+
if (!existing) {
|
|
3413
|
+
latestRuns.set(run.issueId, run);
|
|
3414
|
+
continue;
|
|
3415
|
+
}
|
|
3416
|
+
const runUpdatedAtMs = parseTimestampMs2(run.updatedAt) ?? -Infinity;
|
|
3417
|
+
const existingUpdatedAtMs = parseTimestampMs2(existing.updatedAt) ?? -Infinity;
|
|
3418
|
+
if (runUpdatedAtMs > existingUpdatedAtMs) {
|
|
3419
|
+
latestRuns.set(run.issueId, run);
|
|
3420
|
+
}
|
|
3421
|
+
}
|
|
3422
|
+
return latestRuns;
|
|
3423
|
+
}
|
|
3211
3424
|
function isIssueOrchestrationClaimed(state) {
|
|
3212
3425
|
return state === "claimed" || state === "running" || state === "retry_queued";
|
|
3213
3426
|
}
|
|
@@ -3218,7 +3431,8 @@ function upsertIssueOrchestration(issueRecords, nextRecord) {
|
|
|
3218
3431
|
...remaining,
|
|
3219
3432
|
{
|
|
3220
3433
|
...nextRecord,
|
|
3221
|
-
completedOnce: nextRecord.completedOnce ?? existingRecord?.completedOnce ?? false
|
|
3434
|
+
completedOnce: nextRecord.completedOnce ?? existingRecord?.completedOnce ?? false,
|
|
3435
|
+
failureRetryCount: nextRecord.failureRetryCount ?? existingRecord?.failureRetryCount ?? 0
|
|
3222
3436
|
}
|
|
3223
3437
|
];
|
|
3224
3438
|
}
|