@gh-symphony/cli 0.0.17 → 0.0.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +105 -9
  2. package/dist/{chunk-EFMFGOWM.js → chunk-6CI3UUMH.js} +282 -57
  3. package/dist/chunk-C7G7RJ4G.js +146 -0
  4. package/dist/{chunk-MHIWAIVD.js → chunk-GKENCODJ.js} +141 -53
  5. package/dist/{project-557FE2GD.js → chunk-H2YXSYOZ.js} +108 -92
  6. package/dist/{chunk-TF3QNWNC.js → chunk-M3IFVLQS.js} +246 -212
  7. package/dist/{chunk-IWR4UQEJ.js → chunk-RN2PACNV.js} +350 -523
  8. package/dist/chunk-TILHWBP6.js +638 -0
  9. package/dist/{chunk-6HBZC3BE.js → chunk-XN5ABWZ6.js} +23 -5
  10. package/dist/{chunk-76QPITKI.js → chunk-Y6TYJMNT.js} +1 -1
  11. package/dist/{config-cmd-AZ7POMAA.js → config-cmd-DNXNL26Z.js} +3 -1
  12. package/dist/doctor-IYHCFXOZ.js +1126 -0
  13. package/dist/index.js +157 -19
  14. package/dist/init-KZT6YNOH.js +33 -0
  15. package/dist/{logs-6LNGT2GF.js → logs-6JKKYDGJ.js} +1 -1
  16. package/dist/project-DNALEWO3.js +22 -0
  17. package/dist/{recover-LVBI2TGH.js → recover-C3V2QAUB.js} +3 -3
  18. package/dist/repo-HDDE7OUI.js +321 -0
  19. package/dist/{run-WITYAYFZ.js → run-XI2S5Y4V.js} +3 -3
  20. package/dist/setup-K4CYYJBF.js +431 -0
  21. package/dist/{start-JUFKNL3N.js → start-M6IQGRFO.js} +5 -5
  22. package/dist/{status-3WK5BWRZ.js → status-QSCFVGRQ.js} +2 -2
  23. package/dist/{stop-AA3AP5M6.js → stop-7MFCBQVW.js} +2 -2
  24. package/dist/upgrade-F4VE4XBS.js +165 -0
  25. package/dist/{version-YVM2A25J.js → version-Y5RYNWMF.js} +1 -1
  26. package/dist/worker-entry.js +39 -11
  27. package/dist/workflow-TBIFY5MO.js +497 -0
  28. package/package.json +4 -4
  29. package/dist/chunk-JO3AXHQI.js +0 -130
  30. package/dist/chunk-TH5QPO3Y.js +0 -67
  31. package/dist/init-EZXQAXZM.js +0 -17
  32. package/dist/repo-R3XBIVAX.js +0 -121
@@ -1,5 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
+ DEFAULT_MAX_FAILURE_RETRIES,
3
4
  DEFAULT_WORKFLOW_LIFECYCLE,
4
5
  WorkflowConfigStore,
5
6
  buildHookEnv,
@@ -25,7 +26,7 @@ import {
25
26
  resolveIssueWorkspaceDirectory,
26
27
  safeReadDir,
27
28
  scheduleRetryAt
28
- } from "./chunk-TF3QNWNC.js";
29
+ } from "./chunk-M3IFVLQS.js";
29
30
 
30
31
  // ../orchestrator/dist/service.js
31
32
  import { mkdir as mkdir3, readFile as readFile3, rm as rm3, writeFile as writeFile3 } from "fs/promises";
@@ -286,7 +287,8 @@ var OrchestratorFsStore = class {
286
287
  if (issues) {
287
288
  return issues.map((issue) => ({
288
289
  ...issue,
289
- completedOnce: issue.completedOnce ?? false
290
+ completedOnce: issue.completedOnce ?? false,
291
+ failureRetryCount: issue.failureRetryCount ?? 0
290
292
  }));
291
293
  }
292
294
  const legacyLeases = await readJsonFile(join2(this.projectDir(projectId), "leases.json")) ?? [];
@@ -298,6 +300,7 @@ var OrchestratorFsStore = class {
298
300
  identifier: lease.issueIdentifier,
299
301
  workspaceKey: deriveIssueWorkspaceKeyFromIdentifier(lease.issueIdentifier),
300
302
  completedOnce: false,
303
+ failureRetryCount: 0,
301
304
  state: lease.status === "active" ? "claimed" : "released",
302
305
  currentRunId: lease.status === "active" ? lease.runId : null,
303
306
  retryEntry: null,
@@ -464,9 +467,12 @@ async function pathExists(path) {
464
467
  }
465
468
 
466
469
  // ../tracker-github/dist/adapter.js
470
+ import { createHash } from "crypto";
467
471
  var DEFAULT_API_URL = "https://api.github.com/graphql";
468
472
  var DEFAULT_PAGE_SIZE = 25;
469
473
  var DEFAULT_NETWORK_TIMEOUT_MS = 3e4;
474
+ var RATE_LIMIT_THRESHOLD = 100;
475
+ var MAX_RATE_LIMIT_WAIT_MS = 6e4;
470
476
  var GitHubTrackerError = class extends Error {
471
477
  };
472
478
  var GitHubTrackerHttpError = class extends GitHubTrackerError {
@@ -480,6 +486,7 @@ var GitHubTrackerHttpError = class extends GitHubTrackerError {
480
486
  };
481
487
  var GitHubTrackerQueryError = class extends GitHubTrackerError {
482
488
  };
489
+ var cachedGitHubGraphQLRateLimits = /* @__PURE__ */ new Map();
483
490
  function normalizeProjectItem(projectId, item, lifecycle = DEFAULT_WORKFLOW_LIFECYCLE, priority = {}, rateLimits = null) {
484
491
  if (item.content?.__typename !== "Issue") {
485
492
  return null;
@@ -494,6 +501,9 @@ function normalizeProjectItem(projectId, item, lifecycle = DEFAULT_WORKFLOW_LIFE
494
501
  state: normalizeBlockerState(node.state, lifecycle)
495
502
  }
496
503
  ] : []);
504
+ const issueUpdatedAtMs = parseTimestampMs(item.content.updatedAt);
505
+ const itemUpdatedAtMs = parseTimestampMs(item.updatedAt);
506
+ const trackedUpdatedAt = itemUpdatedAtMs !== null && (issueUpdatedAtMs === null || itemUpdatedAtMs > issueUpdatedAtMs) ? item.updatedAt : item.content.updatedAt ?? item.updatedAt;
497
507
  return {
498
508
  id: item.content.id,
499
509
  identifier: `${repository.owner.login}/${repository.name}#${item.content.number}`,
@@ -507,7 +517,7 @@ function normalizeProjectItem(projectId, item, lifecycle = DEFAULT_WORKFLOW_LIFE
507
517
  labels: (item.content.labels?.nodes ?? []).flatMap((label) => label?.name ? [label.name.toLowerCase()] : []).sort(),
508
518
  blockedBy,
509
519
  createdAt: item.content.createdAt,
510
- updatedAt: item.content.updatedAt ?? item.updatedAt,
520
+ updatedAt: trackedUpdatedAt,
511
521
  repository: {
512
522
  owner: repository.owner.login,
513
523
  name: repository.name,
@@ -814,6 +824,8 @@ async function executeGraphQLQuery(config, query, variables, fetchImpl) {
814
824
  return result.data;
815
825
  }
816
826
  async function executeGraphQLQueryWithMetadata(config, query, variables, fetchImpl) {
827
+ const tokenFingerprint = fingerprintToken(config.token);
828
+ await guardGraphQLRateLimit(tokenFingerprint);
817
829
  const response = await fetchImpl(config.apiUrl ?? DEFAULT_API_URL, {
818
830
  method: "POST",
819
831
  headers: {
@@ -838,11 +850,38 @@ async function executeGraphQLQueryWithMetadata(config, query, variables, fetchIm
838
850
  throw new GitHubTrackerQueryError("GitHub GraphQL response did not include data.");
839
851
  }
840
852
  const data = payload.data;
853
+ const rateLimits = extractGitHubRateLimits(response.headers);
854
+ cachedGitHubGraphQLRateLimits.set(tokenFingerprint, rateLimits);
841
855
  return {
842
856
  data,
843
- rateLimits: extractGitHubRateLimits(response.headers)
857
+ rateLimits
844
858
  };
845
859
  }
860
+ async function guardGraphQLRateLimit(tokenFingerprint) {
861
+ const rateLimit = cachedGitHubGraphQLRateLimits.get(tokenFingerprint) ?? null;
862
+ if (!rateLimit) {
863
+ return;
864
+ }
865
+ const remaining = rateLimit.remaining;
866
+ if (remaining === null || remaining > RATE_LIMIT_THRESHOLD) {
867
+ return;
868
+ }
869
+ const resetAtMs = parseTimestampMs(rateLimit.resetAt);
870
+ if (resetAtMs === null) {
871
+ throw new GitHubTrackerError("Rate limit near exhaustion");
872
+ }
873
+ const waitMs = Math.max(0, resetAtMs - Date.now());
874
+ if (waitMs > MAX_RATE_LIMIT_WAIT_MS) {
875
+ throw new GitHubTrackerError("Rate limit near exhaustion");
876
+ }
877
+ cachedGitHubGraphQLRateLimits.delete(tokenFingerprint);
878
+ if (waitMs > 0) {
879
+ await sleep(waitMs);
880
+ }
881
+ }
882
+ function fingerprintToken(token) {
883
+ return createHash("sha256").update(token).digest("hex");
884
+ }
846
885
  function extractGitHubRateLimits(headers) {
847
886
  if (!headers || typeof headers.get !== "function") {
848
887
  return null;
@@ -872,6 +911,18 @@ function parseIntegerHeader(value) {
872
911
  const parsed = Number.parseInt(value, 10);
873
912
  return Number.isFinite(parsed) ? parsed : null;
874
913
  }
914
+ function parseTimestampMs(value) {
915
+ if (!value) {
916
+ return null;
917
+ }
918
+ const timestampMs = Date.parse(value);
919
+ return Number.isFinite(timestampMs) ? timestampMs : null;
920
+ }
921
+ function sleep(ms) {
922
+ return new Promise((resolve4) => {
923
+ setTimeout(resolve4, ms);
924
+ });
925
+ }
875
926
  var PROJECT_ITEMS_QUERY = `
876
927
  query ProjectItems($projectId: ID!, $cursor: String, $pageSize: Int!) {
877
928
  node(id: $projectId) {
@@ -1086,7 +1137,7 @@ var ISSUE_PROJECT_ITEMS_PAGE_QUERY = `
1086
1137
  `;
1087
1138
 
1088
1139
  // ../tracker-github/dist/orchestrator-adapter.js
1089
- import { createHash } from "crypto";
1140
+ import { createHash as createHash2 } from "crypto";
1090
1141
  var githubProjectTrackerAdapter = {
1091
1142
  async listIssues(project, dependencies = {}) {
1092
1143
  return listProjectIssues(project, dependencies);
@@ -1174,7 +1225,7 @@ function hashToken(token) {
1174
1225
  if (!token) {
1175
1226
  return null;
1176
1227
  }
1177
- return createHash("sha256").update(token).digest("hex");
1228
+ return createHash2("sha256").update(token).digest("hex");
1178
1229
  }
1179
1230
  var trackerAdapters = {
1180
1231
  "github-project": githubProjectTrackerAdapter
@@ -1333,19 +1384,33 @@ var DEFAULT_POLL_INTERVAL_MS = 3e4;
1333
1384
  var DEFAULT_CONCURRENCY = 3;
1334
1385
  var DEFAULT_RETRY_BACKOFF_MS = 3e4;
1335
1386
  var CONTINUATION_RETRY_DELAY_MS = 1e3;
1387
+ var DEFAULT_GLOBAL_MAX_TURNS = 100;
1388
+ var DEFAULT_MAX_TOKENS = 256e3;
1336
1389
  var DEFAULT_WORKER_COMMAND = "node packages/worker/dist/index.js";
1337
1390
  var DEFAULT_MAX_NONPRODUCTIVE_TURNS = 3;
1391
+ var LOW_RATE_LIMIT_WARNING_THRESHOLD = 0.05;
1392
+ var MAX_FAILURE_RETRIES_EXCEEDED_REASON = "max_failure_retries_exceeded";
1338
1393
  var STUCK_WORKER_TIMEOUT_MS = 30 * 60 * 1e3;
1339
1394
  function isUsableWorkflowResolution(resolution) {
1340
1395
  return resolution.isValid || resolution.usedLastKnownGood;
1341
1396
  }
1342
- function parseTimestampMs(value) {
1397
+ function parseTimestampMs2(value) {
1343
1398
  if (!value) {
1344
1399
  return null;
1345
1400
  }
1346
1401
  const parsed = new Date(value).getTime();
1347
1402
  return Number.isFinite(parsed) ? parsed : null;
1348
1403
  }
1404
+ function parseFiniteNumber(value) {
1405
+ if (typeof value === "number" && Number.isFinite(value)) {
1406
+ return value;
1407
+ }
1408
+ if (typeof value === "string" && value.trim()) {
1409
+ const parsed = Number(value);
1410
+ return Number.isFinite(parsed) ? parsed : null;
1411
+ }
1412
+ return null;
1413
+ }
1349
1414
  var OrchestratorService = class {
1350
1415
  store;
1351
1416
  projectConfig;
@@ -1513,11 +1578,12 @@ var OrchestratorService = class {
1513
1578
  let recovered = 0;
1514
1579
  let pollIntervalMs = DEFAULT_POLL_INTERVAL_MS;
1515
1580
  let rateLimits = null;
1581
+ let trackerRateLimits = null;
1516
1582
  let issueRecords = await this.store.loadProjectIssueOrchestrations(tenant.projectId);
1517
1583
  const allRuns = (await this.store.loadAllRuns()).filter((run) => run.projectId === tenant.projectId);
1518
1584
  const activeRuns = allRuns.filter((run) => isActiveRunStatus(run.status));
1519
1585
  for (const run of activeRuns) {
1520
- const outcome = await this.reconcileRun(tenant, run, issueRecords);
1586
+ const outcome = await this.reconcileRun(tenant, run, issueRecords, trackerDependencies);
1521
1587
  issueRecords = outcome.issueRecords;
1522
1588
  if (outcome.recovered) {
1523
1589
  recovered += 1;
@@ -1555,11 +1621,13 @@ var OrchestratorService = class {
1555
1621
  });
1556
1622
  }
1557
1623
  rateLimits = resolveProjectRateLimits(syncedActiveRuns, trackedIssuesByIdentifier.values());
1624
+ trackerRateLimits = resolveTrackerRateLimits(trackedIssuesByIdentifier.values());
1558
1625
  const concurrency = await this.getProjectConcurrency(tenant);
1559
1626
  const currentlyActive = issueRecords.filter((record) => isIssueOrchestrationClaimed(record.state)).length;
1560
1627
  const availableSlots = Math.max(0, concurrency - currentlyActive);
1628
+ const latestRunsByIssueId = buildLatestRunMapByIssueId(projectRunsAfterReconcile);
1561
1629
  const unscheduledCandidates = actionableCandidates.filter((issue) => {
1562
- if (hasConvergenceLockedRun(projectRunsAfterReconcile, issue.id, issue.state)) {
1630
+ if (hasConvergenceLockedRun(projectRunsAfterReconcile, issue.id, issue.state, issue.updatedAt)) {
1563
1631
  return false;
1564
1632
  }
1565
1633
  return !issueRecords.some((record) => record.issueId === issue.id && isIssueOrchestrationClaimed(record.state));
@@ -1579,6 +1647,9 @@ var OrchestratorService = class {
1579
1647
  }
1580
1648
  if (slotsRemaining <= 0)
1581
1649
  break;
1650
+ if (await this.isFailureRetrySuppressedIssue(tenant, issue, issueRecords, latestRunsByIssueId.get(issue.id) ?? null)) {
1651
+ continue;
1652
+ }
1582
1653
  if (isIssueBudgetExceeded(resolveIssueBudgetSnapshot(projectRunsAfterReconcile, issue.id), now)) {
1583
1654
  continue;
1584
1655
  }
@@ -1599,6 +1670,7 @@ var OrchestratorService = class {
1599
1670
  identifier: issue.identifier,
1600
1671
  workspaceKey: preferredWorkspaceKey,
1601
1672
  state: "claimed",
1673
+ failureRetryCount: 0,
1602
1674
  currentRunId: null,
1603
1675
  retryEntry: null,
1604
1676
  updatedAt: now.toISOString()
@@ -1680,7 +1752,11 @@ var OrchestratorService = class {
1680
1752
  } catch (error) {
1681
1753
  lastError = error instanceof Error ? error.message : "Unknown orchestration error";
1682
1754
  }
1683
- this.projectPollIntervals.set(tenant.projectId, pollIntervalMs);
1755
+ const effectivePollIntervalMs = resolveAdaptivePollIntervalMs(pollIntervalMs, trackerRateLimits);
1756
+ if (effectivePollIntervalMs > pollIntervalMs && isLowRateLimit(trackerRateLimits, LOW_RATE_LIMIT_WARNING_THRESHOLD)) {
1757
+ this.writeStderr(`[orchestrator] low GitHub rate limit for ${tenant.projectId}: interval=${effectivePollIntervalMs}ms rateLimits=${JSON.stringify(trackerRateLimits)}`);
1758
+ }
1759
+ this.projectPollIntervals.set(tenant.projectId, effectivePollIntervalMs);
1684
1760
  await this.store.saveProjectIssueOrchestrations(tenant.projectId, issueRecords);
1685
1761
  const allTenantRuns = (await this.store.loadAllRuns()).filter((run) => run.projectId === tenant.projectId);
1686
1762
  const latestRuns = allTenantRuns.filter((run) => isActiveRunStatus(run.status));
@@ -1874,26 +1950,9 @@ var OrchestratorService = class {
1874
1950
  if (!lifecycle) {
1875
1951
  lifecycle = resolution.lifecycle;
1876
1952
  }
1877
- if (!isStateActive(issue.state, resolution.lifecycle)) {
1953
+ if (!this.isIssueCandidateEligible(issue, resolution.lifecycle, issues)) {
1878
1954
  continue;
1879
1955
  }
1880
- if (matchesWorkflowState(issue.state, resolution.lifecycle.blockerCheckStates) && issue.blockedBy.length > 0) {
1881
- const hasNonTerminalBlocker = issue.blockedBy.some((blockerRef) => {
1882
- if (blockerRef.state && isStateTerminal(blockerRef.state, resolution.lifecycle)) {
1883
- return false;
1884
- }
1885
- if (blockerRef.identifier) {
1886
- const blockerIssue = issues.find((candidate) => candidate.identifier === blockerRef.identifier);
1887
- if (blockerIssue?.state) {
1888
- return !isStateTerminal(blockerIssue.state, resolution.lifecycle);
1889
- }
1890
- }
1891
- return true;
1892
- });
1893
- if (hasNonTerminalBlocker) {
1894
- continue;
1895
- }
1896
- }
1897
1956
  candidates.push(issue);
1898
1957
  }
1899
1958
  if (!lifecycle && tenant.repositories.length > 0) {
@@ -1912,6 +1971,26 @@ var OrchestratorService = class {
1912
1971
  }
1913
1972
  };
1914
1973
  }
1974
+ isIssueCandidateEligible(issue, lifecycle, issues) {
1975
+ if (!isStateActive(issue.state, lifecycle)) {
1976
+ return false;
1977
+ }
1978
+ if (!matchesWorkflowState(issue.state, lifecycle.blockerCheckStates) || issue.blockedBy.length === 0) {
1979
+ return true;
1980
+ }
1981
+ return !issue.blockedBy.some((blockerRef) => {
1982
+ if (blockerRef.state && isStateTerminal(blockerRef.state, lifecycle)) {
1983
+ return false;
1984
+ }
1985
+ if (blockerRef.identifier) {
1986
+ const blockerIssue = issues.find((candidate) => candidate.identifier === blockerRef.identifier);
1987
+ if (blockerIssue?.state) {
1988
+ return !isStateTerminal(blockerIssue.state, lifecycle);
1989
+ }
1990
+ }
1991
+ return true;
1992
+ });
1993
+ }
1915
1994
  async loadProjectWorkflow(tenant, repository) {
1916
1995
  const cacheKey = this.workflowCacheKey(repository);
1917
1996
  const pendingCache = this.workflowResolutionCache;
@@ -2233,13 +2312,13 @@ var OrchestratorService = class {
2233
2312
  issuesByIdentifier
2234
2313
  };
2235
2314
  }
2236
- async reconcileRun(tenant, run, issueRecords) {
2315
+ async reconcileRun(tenant, run, issueRecords, trackerDependencies = {}) {
2237
2316
  const now = this.now();
2238
2317
  if (run.processId && this.isProcessRunning(run.processId)) {
2239
2318
  const retryPolicy = await this.loadRetryPolicy(tenant, run.repository);
2240
2319
  const configuredStallTimeoutMs = retryPolicy?.stallTimeoutMs ?? null;
2241
- const lastActivityAtMs = parseTimestampMs(run.lastEventAt ?? run.startedAt);
2242
- const startedAtMs = parseTimestampMs(run.startedAt);
2320
+ const lastActivityAtMs = parseTimestampMs2(run.lastEventAt ?? run.startedAt);
2321
+ const startedAtMs = parseTimestampMs2(run.startedAt);
2243
2322
  const elapsedSinceLastActivityMs = lastActivityAtMs === null ? null : now.getTime() - lastActivityAtMs;
2244
2323
  const runningSinceMs = startedAtMs === null ? null : now.getTime() - startedAtMs;
2245
2324
  const isStalledByWorkflowTimeout = configuredStallTimeoutMs !== null && configuredStallTimeoutMs > 0 && elapsedSinceLastActivityMs !== null && elapsedSinceLastActivityMs > configuredStallTimeoutMs;
@@ -2318,7 +2397,7 @@ var OrchestratorService = class {
2318
2397
  recovered: false
2319
2398
  };
2320
2399
  }
2321
- if (await this.resolveRetryRestartAction(tenant, run) === "release") {
2400
+ if (await this.resolveRetryRestartAction(tenant, run, trackerDependencies) === "release") {
2322
2401
  return this.releaseRetryingRun(runWithTokens, issueRecords, now);
2323
2402
  }
2324
2403
  return this.restartRun(tenant, run, issueRecords, now, workerSessionId);
@@ -2355,7 +2434,54 @@ var OrchestratorService = class {
2355
2434
  state: run.issueState
2356
2435
  });
2357
2436
  }
2358
- const retryKind = await this.classifyRetryKind(tenant, run);
2437
+ const retryKind = await this.classifyRetryKind(tenant, run, trackerDependencies);
2438
+ const failureRetryCount = retryKind === "failure" ? (this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0) + 1 : this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0;
2439
+ const maxFailureRetries = await this.loadMaxFailureRetries(tenant, run.repository);
2440
+ if (retryKind === "failure" && failureRetryCount >= maxFailureRetries) {
2441
+ const lastError = [
2442
+ `Run suppressed: ${MAX_FAILURE_RETRIES_EXCEEDED_REASON}.`,
2443
+ `failureRetryCount=${failureRetryCount}.`,
2444
+ `maxFailureRetries=${maxFailureRetries}.`
2445
+ ].join(" ");
2446
+ const suppressedRun = {
2447
+ ...runWithTokens,
2448
+ status: "suppressed",
2449
+ processId: null,
2450
+ updatedAt: now.toISOString(),
2451
+ completedAt: now.toISOString(),
2452
+ nextRetryAt: null,
2453
+ retryKind: null,
2454
+ runPhase: runWithTokens.runPhase ?? "failed",
2455
+ lastError
2456
+ };
2457
+ await this.store.saveRun(suppressedRun);
2458
+ await this.store.appendRunEvent(run.runId, {
2459
+ at: now.toISOString(),
2460
+ event: "run-suppressed",
2461
+ projectId: run.projectId,
2462
+ issueIdentifier: run.issueIdentifier,
2463
+ issueId: run.issueId,
2464
+ reason: MAX_FAILURE_RETRIES_EXCEEDED_REASON
2465
+ });
2466
+ this.logVerbose(`[run-completed] ${suppressedRun.runId} status=${suppressedRun.status}`);
2467
+ return {
2468
+ issueRecords: upsertIssueOrchestration(issueRecords, {
2469
+ issueId: run.issueId,
2470
+ identifier: run.issueIdentifier,
2471
+ workspaceKey: run.issueWorkspaceKey ?? deriveIssueWorkspaceKey({
2472
+ projectId: tenant.projectId,
2473
+ adapter: tenant.tracker.adapter,
2474
+ issueSubjectId: run.issueSubjectId
2475
+ }, run.issueIdentifier),
2476
+ state: "released",
2477
+ failureRetryCount,
2478
+ currentRunId: null,
2479
+ retryEntry: null,
2480
+ updatedAt: now.toISOString()
2481
+ }),
2482
+ recovered: false
2483
+ };
2484
+ }
2359
2485
  let nextRetryAt;
2360
2486
  if (retryKind === "continuation") {
2361
2487
  nextRetryAt = new Date(now.getTime() + CONTINUATION_RETRY_DELAY_MS).toISOString();
@@ -2391,6 +2517,7 @@ var OrchestratorService = class {
2391
2517
  }, run.issueIdentifier),
2392
2518
  state: "retry_queued",
2393
2519
  completedOnce: retryKind === "continuation" ? true : void 0,
2520
+ failureRetryCount,
2394
2521
  currentRunId: run.runId,
2395
2522
  retryEntry: {
2396
2523
  attempt: retryRecord.attempt,
@@ -2605,49 +2732,47 @@ var OrchestratorService = class {
2605
2732
  * — the worker completed its session and the issue hasn't transitioned away.
2606
2733
  * Failure applies when we cannot confirm the issue is still actionable.
2607
2734
  */
2608
- async classifyRetryKind(tenant, run) {
2735
+ async classifyRetryKind(tenant, run, trackerDependencies = {}) {
2609
2736
  try {
2610
- const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
2611
- const issues = await trackerAdapter.listIssues(tenant, {
2612
- fetchImpl: this.dependencies.fetchImpl
2613
- });
2614
- const runIssue = issues.find((issue) => issue.identifier === run.issueIdentifier);
2615
- if (!runIssue) {
2737
+ const eligibleContext = await this.fetchTrackedIssueEligibilityContext(tenant, run.issueIdentifier, trackerDependencies);
2738
+ if (!eligibleContext) {
2616
2739
  return "failure";
2617
2740
  }
2618
2741
  const resolution = await this.loadProjectWorkflow(tenant, run.repository);
2619
2742
  if (!isUsableWorkflowResolution(resolution)) {
2620
2743
  return "failure";
2621
2744
  }
2622
- return isStateActive(runIssue.state, resolution.lifecycle) ? "continuation" : "failure";
2745
+ return this.isIssueCandidateEligible(eligibleContext.issue, resolution.lifecycle, eligibleContext.issues) ? "continuation" : "failure";
2623
2746
  } catch {
2624
2747
  return "failure";
2625
2748
  }
2626
2749
  }
2627
- async resolveRetryRestartAction(tenant, run) {
2750
+ async resolveRetryRestartAction(tenant, run, trackerDependencies = {}) {
2628
2751
  try {
2629
2752
  if (isIssueBudgetExceeded(resolveIssueBudgetSnapshot((await this.store.loadAllRuns()).filter((candidate) => candidate.projectId === tenant.projectId), run.issueId), this.now())) {
2630
2753
  return "release";
2631
2754
  }
2632
- const runIssue = await this.fetchTrackedIssueById(tenant, run.issueId);
2633
- if (!runIssue) {
2755
+ const eligibleContext = await this.fetchTrackedIssueEligibilityContext(tenant, run.issueIdentifier, trackerDependencies);
2756
+ if (!eligibleContext) {
2634
2757
  return "release";
2635
2758
  }
2636
2759
  const resolution = await this.loadProjectWorkflow(tenant, run.repository);
2637
2760
  if (!isUsableWorkflowResolution(resolution)) {
2638
2761
  return "restart";
2639
2762
  }
2640
- return isStateActive(runIssue.state, resolution.lifecycle) ? "restart" : "release";
2763
+ return this.isIssueCandidateEligible(eligibleContext.issue, resolution.lifecycle, eligibleContext.issues) ? "restart" : "release";
2641
2764
  } catch {
2642
2765
  return "restart";
2643
2766
  }
2644
2767
  }
2645
- async fetchTrackedIssueById(tenant, issueId) {
2768
+ async fetchTrackedIssueEligibilityContext(tenant, issueIdentifier, trackerDependencies = {}) {
2646
2769
  const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
2647
- const issues = await trackerAdapter.fetchIssueStatesByIds(tenant, [issueId], {
2648
- fetchImpl: this.dependencies.fetchImpl
2770
+ const issues = await trackerAdapter.listIssues(tenant, {
2771
+ fetchImpl: this.dependencies.fetchImpl,
2772
+ ...trackerDependencies
2649
2773
  });
2650
- return issues[0] ?? null;
2774
+ const issue = issues.find((candidate) => candidate.identifier === issueIdentifier);
2775
+ return issue ? { issue, issues } : null;
2651
2776
  }
2652
2777
  async fetchWorkerRunInfo(run) {
2653
2778
  const latestRun = await this.store.loadRun(run.runId, run.projectId) ?? run;
@@ -3020,6 +3145,36 @@ var OrchestratorService = class {
3020
3145
  };
3021
3146
  await this.store.saveIssueWorkspace(removedRecord);
3022
3147
  }
3148
+ resolveFailureRetryCount(issueRecords, issueId) {
3149
+ return issueRecords.find((record) => record.issueId === issueId)?.failureRetryCount ?? null;
3150
+ }
3151
+ async isFailureRetrySuppressedIssue(tenant, issue, issueRecords, latestRun) {
3152
+ const issueRecord = issueRecords.find((record) => record.issueId === issue.id || record.identifier === issue.identifier) ?? null;
3153
+ if (!issueRecord || issueRecord.failureRetryCount <= 0) {
3154
+ return false;
3155
+ }
3156
+ const maxFailureRetries = await this.loadMaxFailureRetries(tenant, issue.repository);
3157
+ if (issueRecord.failureRetryCount < maxFailureRetries) {
3158
+ return false;
3159
+ }
3160
+ if (!latestRun || latestRun.status !== "suppressed" || latestRun.issueState !== issue.state || !latestRun.lastError?.includes(MAX_FAILURE_RETRIES_EXCEEDED_REASON)) {
3161
+ return false;
3162
+ }
3163
+ const issueUpdatedAtMs = parseTimestampMs2(issue.updatedAt);
3164
+ const suppressedAtMs = parseTimestampMs2(latestRun.completedAt ?? latestRun.updatedAt);
3165
+ if (issueUpdatedAtMs === null || suppressedAtMs === null) {
3166
+ return true;
3167
+ }
3168
+ return issueUpdatedAtMs <= suppressedAtMs;
3169
+ }
3170
+ async loadMaxFailureRetries(tenant, repository) {
3171
+ try {
3172
+ const resolution = await this.loadProjectWorkflow(tenant, repository);
3173
+ return isUsableWorkflowResolution(resolution) ? resolution.workflow.agent.maxFailureRetries : DEFAULT_MAX_FAILURE_RETRIES;
3174
+ } catch {
3175
+ return DEFAULT_MAX_FAILURE_RETRIES;
3176
+ }
3177
+ }
3023
3178
  };
3024
3179
  function hasTokenUsage(tokenUsage) {
3025
3180
  return Boolean(tokenUsage && (tokenUsage.inputTokens > 0 || tokenUsage.outputTokens > 0 || tokenUsage.totalTokens > 0));
@@ -3034,7 +3189,7 @@ function resolveProjectRateLimits(runs, issues) {
3034
3189
  if (!isRecord(run.rateLimits)) {
3035
3190
  continue;
3036
3191
  }
3037
- const timestamp = parseTimestampMs(run.lastEventAt ?? run.updatedAt ?? run.startedAt);
3192
+ const timestamp = parseTimestampMs2(run.lastEventAt ?? run.updatedAt ?? run.startedAt);
3038
3193
  const sortableTimestamp = timestamp ?? -Infinity;
3039
3194
  if (sortableTimestamp >= latestRunTimestamp) {
3040
3195
  latestRunTimestamp = sortableTimestamp;
@@ -3051,6 +3206,51 @@ function resolveProjectRateLimits(runs, issues) {
3051
3206
  }
3052
3207
  return null;
3053
3208
  }
3209
+ function resolveTrackerRateLimits(issues) {
3210
+ for (const issue of issues) {
3211
+ if (isGitHubTrackerRateLimits(issue.rateLimits)) {
3212
+ return issue.rateLimits;
3213
+ }
3214
+ }
3215
+ return null;
3216
+ }
3217
+ function resolveAdaptivePollIntervalMs(basePollIntervalMs, rateLimits) {
3218
+ if (!Number.isFinite(basePollIntervalMs) || basePollIntervalMs <= 0) {
3219
+ return DEFAULT_POLL_INTERVAL_MS;
3220
+ }
3221
+ const ratio = extractRateLimitRatio(rateLimits);
3222
+ if (ratio === null || ratio > 0.5) {
3223
+ return basePollIntervalMs;
3224
+ }
3225
+ if (ratio >= 0.2) {
3226
+ return basePollIntervalMs * 2;
3227
+ }
3228
+ if (ratio >= LOW_RATE_LIMIT_WARNING_THRESHOLD) {
3229
+ return basePollIntervalMs * 4;
3230
+ }
3231
+ return basePollIntervalMs * 10;
3232
+ }
3233
+ function extractRateLimitRatio(rateLimits) {
3234
+ if (!isRecord(rateLimits)) {
3235
+ return null;
3236
+ }
3237
+ const limit = parseFiniteNumber(rateLimits.limit);
3238
+ const remaining = parseFiniteNumber(rateLimits.remaining);
3239
+ if (limit === null || remaining === null || limit <= 0 || remaining < 0) {
3240
+ return null;
3241
+ }
3242
+ return remaining / limit;
3243
+ }
3244
+ function isGitHubTrackerRateLimits(rateLimits) {
3245
+ if (!isRecord(rateLimits) || rateLimits.source !== "github") {
3246
+ return false;
3247
+ }
3248
+ return rateLimits.resource === void 0 || rateLimits.resource === null || rateLimits.resource === "graphql";
3249
+ }
3250
+ function isLowRateLimit(rateLimits, threshold) {
3251
+ const ratio = extractRateLimitRatio(rateLimits);
3252
+ return ratio !== null && ratio < threshold;
3253
+ }
3054
3254
  function buildRuntimeSession(existing, sessionId, threadId, status, startedAt, updatedAt, exitClassification = void 0) {
3055
3255
  if (existing === void 0 && sessionId === null && threadId === null && status === null && (exitClassification === void 0 || exitClassification === null)) {
3056
3256
  return void 0;
@@ -3067,9 +3267,17 @@ function buildRuntimeSession(existing, sessionId, threadId, status, startedAt, u
3067
3267
  function resolvePersistedCumulativeTurnCount(run) {
3068
3268
  return run.cumulativeTurnCount ?? run.turnCount ?? 0;
3069
3269
  }
3070
- function hasConvergenceLockedRun(runs, issueId, issueState) {
3270
+ function hasConvergenceLockedRun(runs, issueId, issueState, issueUpdatedAt) {
3071
3271
  const latestRun = runs.filter((run) => run.issueId === issueId).sort((left, right) => new Date(right.updatedAt).getTime() - new Date(left.updatedAt).getTime())[0];
3072
- return latestRun?.runtimeSession?.exitClassification === "convergence-detected" && latestRun.issueState === issueState;
3272
+ if (latestRun?.runtimeSession?.exitClassification !== "convergence-detected" || latestRun.issueState !== issueState) {
3273
+ return false;
3274
+ }
3275
+ const convergedAtMs = parseTimestampMs2(latestRun.completedAt ?? latestRun.updatedAt);
3276
+ const issueUpdatedAtMs = parseTimestampMs2(issueUpdatedAt);
3277
+ if (convergedAtMs === null || issueUpdatedAtMs === null) {
3278
+ return true;
3279
+ }
3280
+ return issueUpdatedAtMs <= convergedAtMs;
3073
3281
  }
3074
3282
  function resolveIssueBudgetSnapshot(runs, issueId) {
3075
3283
  const issueRuns = runs.filter((run) => run.issueId === issueId);
@@ -3089,12 +3297,12 @@ function resolveIssueBudgetSnapshot(runs, issueId) {
3089
3297
  };
3090
3298
  }
3091
3299
  function isIssueBudgetExceeded(snapshot, now, env = process.env) {
3092
- const globalMaxTurns = parsePositiveInteger(env.SYMPHONY_GLOBAL_MAX_TURNS ?? "");
3093
- if (globalMaxTurns !== null && snapshot.cumulativeTurnCount >= globalMaxTurns) {
3300
+ const globalMaxTurns = parsePositiveInteger(env.SYMPHONY_GLOBAL_MAX_TURNS ?? "") ?? DEFAULT_GLOBAL_MAX_TURNS;
3301
+ if (snapshot.cumulativeTurnCount >= globalMaxTurns) {
3094
3302
  return true;
3095
3303
  }
3096
- const maxTokens = parsePositiveInteger(env.SYMPHONY_MAX_TOKENS ?? "");
3097
- if (maxTokens !== null && snapshot.tokenUsage.totalTokens >= maxTokens) {
3304
+ const maxTokens = parsePositiveInteger(env.SYMPHONY_MAX_TOKENS ?? "") ?? DEFAULT_MAX_TOKENS;
3305
+ if (snapshot.tokenUsage.totalTokens >= maxTokens) {
3098
3306
  return true;
3099
3307
  }
3100
3308
  const sessionTimeoutMs = parsePositiveInteger(env.SYMPHONY_SESSION_TIMEOUT_MS ?? "");
@@ -3208,6 +3416,22 @@ function createRunId(now, projectId, issueIdentifier) {
3208
3416
  now.getTime().toString(36)
3209
3417
  ].join("-");
3210
3418
  }
3419
+ function buildLatestRunMapByIssueId(runs) {
3420
+ const latestRuns = /* @__PURE__ */ new Map();
3421
+ for (const run of runs) {
3422
+ const existing = latestRuns.get(run.issueId);
3423
+ if (!existing) {
3424
+ latestRuns.set(run.issueId, run);
3425
+ continue;
3426
+ }
3427
+ const runUpdatedAtMs = parseTimestampMs2(run.updatedAt) ?? -Infinity;
3428
+ const existingUpdatedAtMs = parseTimestampMs2(existing.updatedAt) ?? -Infinity;
3429
+ if (runUpdatedAtMs > existingUpdatedAtMs) {
3430
+ latestRuns.set(run.issueId, run);
3431
+ }
3432
+ }
3433
+ return latestRuns;
3434
+ }
3211
3435
  function isIssueOrchestrationClaimed(state) {
3212
3436
  return state === "claimed" || state === "running" || state === "retry_queued";
3213
3437
  }
@@ -3218,7 +3442,8 @@ function upsertIssueOrchestration(issueRecords, nextRecord) {
3218
3442
  ...remaining,
3219
3443
  {
3220
3444
  ...nextRecord,
3221
- completedOnce: nextRecord.completedOnce ?? existingRecord?.completedOnce ?? false
3445
+ completedOnce: nextRecord.completedOnce ?? existingRecord?.completedOnce ?? false,
3446
+ failureRetryCount: nextRecord.failureRetryCount ?? existingRecord?.failureRetryCount ?? 0
3222
3447
  }
3223
3448
  ];
3224
3449
  }