@link-assistant/hive-mind 1.59.4 → 1.59.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +242 -0
- package/package.json +1 -1
- package/src/bidirectional-interactive.lib.mjs +1 -0
- package/src/contributing-guidelines.lib.mjs +3 -2
- package/src/github-error-reporter.lib.mjs +3 -2
- package/src/github-merge-ci-signals.lib.mjs +8 -2
- package/src/github-merge-ci.lib.mjs +8 -2
- package/src/github-merge-ready-sync.lib.mjs +7 -1
- package/src/github-merge-repo-actions.lib.mjs +59 -15
- package/src/github-merge.lib.mjs +100 -58
- package/src/github-rate-limit.lib.mjs +276 -0
- package/src/github.batch.lib.mjs +1 -0
- package/src/hive.mjs +2 -2
- package/src/hive.recheck.lib.mjs +1 -0
- package/src/lib.mjs +30 -4
- package/src/limits.lib.mjs +1 -0
- package/src/protect-branch.mjs +3 -2
- package/src/queue-config.lib.mjs +7 -3
- package/src/review.mjs +3 -2
- package/src/reviewers-hive.mjs +3 -2
- package/src/solve.accept-invite.lib.mjs +7 -1
- package/src/solve.auto-continue.lib.mjs +3 -2
- package/src/solve.auto-ensure.lib.mjs +3 -2
- package/src/solve.auto-merge-helpers.lib.mjs +3 -2
- package/src/solve.auto-merge.lib.mjs +3 -2
- package/src/solve.auto-pr.lib.mjs +1 -0
- package/src/solve.branch-errors.lib.mjs +1 -0
- package/src/solve.error-handlers.lib.mjs +1 -0
- package/src/solve.execution.lib.mjs +3 -2
- package/src/solve.feedback.lib.mjs +1 -0
- package/src/solve.mjs +3 -1
- package/src/solve.preparation.lib.mjs +1 -0
- package/src/solve.progress-monitoring.lib.mjs +1 -0
- package/src/solve.repository.lib.mjs +3 -3
- package/src/solve.restart-shared.lib.mjs +3 -2
- package/src/solve.results.lib.mjs +3 -2
- package/src/solve.session.lib.mjs +1 -0
- package/src/solve.watch.lib.mjs +3 -2
- package/src/telegram-accept-invitations.lib.mjs +7 -1
- package/src/token-sanitization.lib.mjs +1 -0
- package/src/youtrack/youtrack-sync.mjs +1 -0
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* GitHub API rate-limit detection and retry utilities.
|
|
5
|
+
*
|
|
6
|
+
* Issue #1726: Hosted runners hit GitHub's 5,000/hr core API quota and bubble
|
|
7
|
+
* the failure up as a generic 403/HTTP error. The wrappers in lib.mjs only
|
|
8
|
+
* recognise transient TCP/TLS faults; rate-limit responses fell through and
|
|
9
|
+
* crashed callers (or worse, were silently swallowed in the merge subsystem
|
|
10
|
+
* making it look like "no workflows / no checks" — see
|
|
11
|
+
* src/github-merge.lib.mjs:getActiveRepoWorkflows in the original log).
|
|
12
|
+
*
|
|
13
|
+
* The retry policy required by the issue:
|
|
14
|
+
* wait = (resetTimestamp - now) + bufferMs (10 min) + random(jitterMs) (0-5 min)
|
|
15
|
+
*
|
|
16
|
+
* `bufferMs` and `jitterMs` already exist in src/config.lib.mjs#limitReset
|
|
17
|
+
* (added in #1236 for Claude limit waits) so we re-use them rather than
|
|
18
|
+
* duplicate constants.
|
|
19
|
+
*/
|
|
20
|
+
import { promisify } from 'node:util';
|
|
21
|
+
import { exec as execCb } from 'node:child_process';
|
|
22
|
+
|
|
23
|
+
import { limitReset, retryLimits } from './config.lib.mjs';
|
|
24
|
+
|
|
25
|
+
const exec = promisify(execCb);
|
|
26
|
+
|
|
27
|
+
const RATE_LIMIT_PATTERNS = ['api rate limit exceeded', 'rate limit exceeded', 'you have exceeded a secondary rate limit', 'secondary rate limit', 'abuse detection', 'was submitted too quickly'];
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Pull every plausible string out of a thrown error/result so pattern matches
|
|
31
|
+
* survive whatever shape the upstream caller gave us (Error, exec result with
|
|
32
|
+
* stdout/stderr, command-stream result, plain string, etc.).
|
|
33
|
+
*/
|
|
34
|
+
const collectErrorText = error => {
|
|
35
|
+
if (!error) return '';
|
|
36
|
+
if (typeof error === 'string') return error;
|
|
37
|
+
const parts = [];
|
|
38
|
+
if (typeof error.message === 'string') parts.push(error.message);
|
|
39
|
+
if (typeof error.stderr === 'string') parts.push(error.stderr);
|
|
40
|
+
else if (error.stderr && typeof error.stderr.toString === 'function') parts.push(error.stderr.toString());
|
|
41
|
+
if (typeof error.stdout === 'string') parts.push(error.stdout);
|
|
42
|
+
else if (error.stdout && typeof error.stdout.toString === 'function') parts.push(error.stdout.toString());
|
|
43
|
+
if (error.cause) parts.push(collectErrorText(error.cause));
|
|
44
|
+
return parts.join('\n');
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Detect whether `error` represents a GitHub rate-limit response.
|
|
49
|
+
* Recognises both primary (5,000/hr) and secondary (abuse-detection) forms.
|
|
50
|
+
*
|
|
51
|
+
* @param {unknown} error
|
|
52
|
+
* @returns {boolean}
|
|
53
|
+
*/
|
|
54
|
+
export const isRateLimitError = error => {
|
|
55
|
+
const text = collectErrorText(error).toLowerCase();
|
|
56
|
+
if (!text) return false;
|
|
57
|
+
return RATE_LIMIT_PATTERNS.some(pattern => text.includes(pattern));
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Extract a `Date` for when the rate-limit window resets, in priority order:
|
|
62
|
+
* 1. `X-RateLimit-Reset` header value (Unix epoch seconds) embedded in the
|
|
63
|
+
* error text — `gh` prints headers when --include is used and graphql
|
|
64
|
+
* surfaces them in the error body.
|
|
65
|
+
* 2. `Retry-After` header (seconds from now).
|
|
66
|
+
* 3. None — caller falls back to a polled `gh api rate_limit` lookup.
|
|
67
|
+
*
|
|
68
|
+
* @param {unknown} error
|
|
69
|
+
* @returns {Date|null}
|
|
70
|
+
*/
|
|
71
|
+
export const parseRateLimitReset = error => {
|
|
72
|
+
const text = collectErrorText(error);
|
|
73
|
+
if (!text) return null;
|
|
74
|
+
|
|
75
|
+
const resetMatch = text.match(/x-ratelimit-reset:\s*(\d+)/i);
|
|
76
|
+
if (resetMatch) {
|
|
77
|
+
const epochSeconds = Number(resetMatch[1]);
|
|
78
|
+
if (Number.isFinite(epochSeconds) && epochSeconds > 0) {
|
|
79
|
+
return new Date(epochSeconds * 1000);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const retryAfterMatch = text.match(/retry-after:\s*(\d+)/i);
|
|
84
|
+
if (retryAfterMatch) {
|
|
85
|
+
const seconds = Number(retryAfterMatch[1]);
|
|
86
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
87
|
+
return new Date(Date.now() + seconds * 1000);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return null;
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Ask `gh api rate_limit` directly when the error didn't carry a reset header.
|
|
96
|
+
* Returns the most-restrictive (soonest) reset time across the resources we
|
|
97
|
+
* touch (core, search, graphql) so we don't resume into a still-throttled
|
|
98
|
+
* bucket.
|
|
99
|
+
*
|
|
100
|
+
* @returns {Promise<Date|null>}
|
|
101
|
+
*/
|
|
102
|
+
export const fetchNextRateLimitReset = async () => {
|
|
103
|
+
try {
|
|
104
|
+
// eslint-disable-next-line gh-rate-limit/no-direct-gh-exec -- this IS the rate-limit helper; calling itself recursively would loop.
|
|
105
|
+
const { stdout } = await exec('gh api rate_limit');
|
|
106
|
+
const data = JSON.parse(stdout);
|
|
107
|
+
const resources = data?.resources || {};
|
|
108
|
+
const candidates = [];
|
|
109
|
+
for (const key of ['core', 'graphql', 'search']) {
|
|
110
|
+
const r = resources[key];
|
|
111
|
+
if (r && Number.isFinite(r.reset) && r.remaining === 0) {
|
|
112
|
+
candidates.push(r.reset);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (candidates.length === 0) return null;
|
|
116
|
+
const soonestEpoch = Math.min(...candidates);
|
|
117
|
+
return new Date(soonestEpoch * 1000);
|
|
118
|
+
} catch {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Compute the absolute wait deadline that satisfies issue #1726:
|
|
125
|
+
* reset + bufferMs (default 10 min) + random(0..jitterMs) (default 0-5 min)
|
|
126
|
+
*
|
|
127
|
+
* @param {Date|null} reset
|
|
128
|
+
* @returns {{ waitMs: number, deadline: Date, reset: Date|null, bufferMs: number, jitterMs: number }}
|
|
129
|
+
*/
|
|
130
|
+
export const computeRateLimitWait = (reset, now = Date.now()) => {
|
|
131
|
+
const bufferMs = limitReset.bufferMs;
|
|
132
|
+
const jitterMs = Math.floor(Math.random() * (limitReset.jitterMs + 1));
|
|
133
|
+
const resetTime = reset instanceof Date ? reset.getTime() : null;
|
|
134
|
+
const baselineWait = resetTime && resetTime > now ? resetTime - now : 0;
|
|
135
|
+
const waitMs = baselineWait + bufferMs + jitterMs;
|
|
136
|
+
return {
|
|
137
|
+
waitMs,
|
|
138
|
+
deadline: new Date(now + waitMs),
|
|
139
|
+
reset: reset || null,
|
|
140
|
+
bufferMs,
|
|
141
|
+
jitterMs,
|
|
142
|
+
};
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Sleep with optional periodic countdown notifications.
|
|
147
|
+
*
|
|
148
|
+
* @param {number} ms
|
|
149
|
+
* @param {(msg: string) => Promise<void>|void} [log]
|
|
150
|
+
*/
|
|
151
|
+
const sleepWithCountdown = async (ms, log) => {
|
|
152
|
+
if (ms <= 0) return;
|
|
153
|
+
if (!log || ms <= 60_000) {
|
|
154
|
+
await new Promise(resolve => setTimeout(resolve, ms));
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
let remaining = ms;
|
|
158
|
+
const timer = setInterval(() => {
|
|
159
|
+
remaining -= 60_000;
|
|
160
|
+
if (remaining > 0) {
|
|
161
|
+
const minutes = Math.round(remaining / 60_000);
|
|
162
|
+
Promise.resolve(log(`⏳ Rate-limit wait: ${minutes} min remaining...`)).catch(() => {});
|
|
163
|
+
}
|
|
164
|
+
}, 60_000);
|
|
165
|
+
try {
|
|
166
|
+
await new Promise(resolve => setTimeout(resolve, ms));
|
|
167
|
+
} finally {
|
|
168
|
+
clearInterval(timer);
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Wrap `fn` so that GitHub rate-limit errors are converted into a sleep until
|
|
174
|
+
* (resetTime + bufferMs + jitterMs) followed by a retry. Non-rate-limit errors
|
|
175
|
+
* are rethrown immediately so we don't mask programming bugs or 404s.
|
|
176
|
+
*
|
|
177
|
+
* @template T
|
|
178
|
+
* @param {() => Promise<T>} fn
|
|
179
|
+
* @param {object} [options]
|
|
180
|
+
* @param {number} [options.maxAttempts] - hard cap on rate-limit retries (default `retryLimits.maxApiRetries`).
|
|
181
|
+
* @param {string} [options.label] - prefix for log messages.
|
|
182
|
+
* @param {(msg: string) => Promise<void>|void} [options.log] - logger. Defaults to console.warn.
|
|
183
|
+
* @returns {Promise<T>}
|
|
184
|
+
*/
|
|
185
|
+
export const ghWithRateLimitRetry = async (fn, options = {}) => {
|
|
186
|
+
const maxAttempts = options.maxAttempts ?? retryLimits.maxApiRetries;
|
|
187
|
+
const label = options.label || 'gh';
|
|
188
|
+
const log = options.log || (msg => console.warn(msg));
|
|
189
|
+
|
|
190
|
+
let lastError;
|
|
191
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
192
|
+
try {
|
|
193
|
+
return await fn();
|
|
194
|
+
} catch (error) {
|
|
195
|
+
lastError = error;
|
|
196
|
+
if (!isRateLimitError(error)) throw error;
|
|
197
|
+
|
|
198
|
+
if (attempt === maxAttempts) {
|
|
199
|
+
await Promise.resolve(log(`❌ ${label}: rate limit still active after ${attempt} attempts; giving up.`));
|
|
200
|
+
throw error;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const reset = parseRateLimitReset(error) || (await fetchNextRateLimitReset());
|
|
204
|
+
const { waitMs, deadline, bufferMs, jitterMs } = computeRateLimitWait(reset);
|
|
205
|
+
const waitMinutes = Math.round(waitMs / 60_000);
|
|
206
|
+
const resetSummary = reset ? `reset at ${reset.toISOString()}` : 'reset time unknown (using buffer + jitter only)';
|
|
207
|
+
await Promise.resolve(log(`⏳ ${label}: GitHub API rate limit hit (attempt ${attempt}/${maxAttempts}). Waiting ${waitMinutes} min (${resetSummary}; buffer ${Math.round(bufferMs / 60_000)} min + jitter ${Math.round(jitterMs / 1000)}s) until ${deadline.toISOString()}.`));
|
|
208
|
+
await sleepWithCountdown(waitMs, log);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
// Unreachable — loop either returns or throws.
|
|
212
|
+
throw lastError;
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Convenience wrapper around child_process.exec that retries on rate-limit
|
|
217
|
+
* errors. Use it for callers that build a `gh` command string and want the
|
|
218
|
+
* existing exec-based ergonomics.
|
|
219
|
+
*
|
|
220
|
+
* @param {string} command
|
|
221
|
+
* @param {object} [options] - forwarded to ghWithRateLimitRetry, plus `execOptions`.
|
|
222
|
+
* @returns {Promise<{stdout: string, stderr: string}>}
|
|
223
|
+
*/
|
|
224
|
+
export const execGhWithRetry = async (command, options = {}) => {
|
|
225
|
+
const { execOptions, ...retryOptions } = options;
|
|
226
|
+
return ghWithRateLimitRetry(() => exec(command, execOptions), {
|
|
227
|
+
label: retryOptions.label || `gh exec (${command.split(/\s+/).slice(0, 3).join(' ')})`,
|
|
228
|
+
...retryOptions,
|
|
229
|
+
});
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Wrap a command-stream `$` tagged-template so every `$gh ...` it issues is
|
|
234
|
+
* retried on rate-limit errors. Returns a callable that delegates to the
|
|
235
|
+
* underlying `$` for non-`gh` commands and through `ghWithRateLimitRetry` for
|
|
236
|
+
* `gh ...` commands.
|
|
237
|
+
*
|
|
238
|
+
* Usage at the top of a file:
|
|
239
|
+
* const { $: rawDollar } = await use('command-stream');
|
|
240
|
+
* const $ = wrapDollarWithGhRetry(rawDollar);
|
|
241
|
+
*
|
|
242
|
+
* @template T
|
|
243
|
+
* @param {(strings: TemplateStringsArray, ...values: unknown[]) => Promise<T>} dollar
|
|
244
|
+
* @param {object} [options] - forwarded to ghWithRateLimitRetry per call.
|
|
245
|
+
* @returns {(strings: TemplateStringsArray, ...values: unknown[]) => Promise<T>}
|
|
246
|
+
*/
|
|
247
|
+
export const wrapDollarWithGhRetry = (dollar, options = {}) => {
|
|
248
|
+
const wrapped = (strings, ...values) => {
|
|
249
|
+
// Reconstruct the literal command for inspection (sufficient — leading
|
|
250
|
+
// `gh ` is what we care about).
|
|
251
|
+
let preview = '';
|
|
252
|
+
for (let i = 0; i < strings.length; i++) {
|
|
253
|
+
preview += strings[i];
|
|
254
|
+
if (i < values.length) preview += String(values[i] ?? '');
|
|
255
|
+
}
|
|
256
|
+
const isGh = /^\s*gh(?:\s|$)/.test(preview);
|
|
257
|
+
if (!isGh) return dollar(strings, ...values);
|
|
258
|
+
return ghWithRateLimitRetry(() => dollar(strings, ...values), {
|
|
259
|
+
label: `$gh (${preview.trim().split(/\s+/).slice(0, 3).join(' ')})`,
|
|
260
|
+
...options,
|
|
261
|
+
});
|
|
262
|
+
};
|
|
263
|
+
// Preserve a reference to the underlying $ for consumers that need it.
|
|
264
|
+
wrapped.raw = dollar;
|
|
265
|
+
return wrapped;
|
|
266
|
+
};
|
|
267
|
+
|
|
268
|
+
export default {
|
|
269
|
+
isRateLimitError,
|
|
270
|
+
parseRateLimitReset,
|
|
271
|
+
fetchNextRateLimitReset,
|
|
272
|
+
computeRateLimitWait,
|
|
273
|
+
ghWithRateLimitRetry,
|
|
274
|
+
execGhWithRetry,
|
|
275
|
+
wrapDollarWithGhRetry,
|
|
276
|
+
};
|
package/src/github.batch.lib.mjs
CHANGED
|
@@ -11,6 +11,7 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
11
11
|
import { log, cleanErrorMessage } from './lib.mjs';
|
|
12
12
|
import { githubLimits, timeouts } from './config.lib.mjs';
|
|
13
13
|
|
|
14
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
14
15
|
/**
|
|
15
16
|
* Check if a PR body/title indicates it fixes/closes/resolves a specific issue number
|
|
16
17
|
* GitHub auto-closes issues when PR body contains keywords like "fixes #123", "closes #123", "resolves #123"
|
package/src/hive.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Import Sentry instrumentation first (must be before other imports)
|
|
3
3
|
import './instrument.mjs';
|
|
4
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
4
5
|
const earlyArgs = process.argv.slice(2);
|
|
5
6
|
if (earlyArgs.includes('--version')) {
|
|
6
7
|
const { getVersion } = await import('./version.lib.mjs');
|
|
@@ -28,7 +29,6 @@ if (earlyArgs.includes('--help') || earlyArgs.includes('-h')) {
|
|
|
28
29
|
// Reuse createYargsConfig from shared module to avoid duplication
|
|
29
30
|
const { createYargsConfig } = await import('./hive.config.lib.mjs');
|
|
30
31
|
const helpYargs = createYargsConfig(yargs(rawArgs)).version(false);
|
|
31
|
-
// Show help and exit
|
|
32
32
|
helpYargs.showHelp();
|
|
33
33
|
process.exit(0);
|
|
34
34
|
} catch (error) {
|
|
@@ -1497,4 +1497,4 @@ if (isRunningDirectly) {
|
|
|
1497
1497
|
console.error('\nPlease report this issue at: https://github.com/link-assistant/hive-mind/issues');
|
|
1498
1498
|
process.exit(1);
|
|
1499
1499
|
}
|
|
1500
|
-
}
|
|
1500
|
+
}
|
package/src/hive.recheck.lib.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { log, cleanErrorMessage } from './lib.mjs';
|
|
|
5
5
|
import { batchCheckPullRequestsForIssues, batchCheckArchivedRepositories } from './github.lib.mjs';
|
|
6
6
|
import { reportError } from './sentry.lib.mjs';
|
|
7
7
|
|
|
8
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
8
9
|
/**
|
|
9
10
|
* Recheck conditions for an issue right before processing
|
|
10
11
|
* This ensures the issue should still be processed even if conditions changed since queuing
|
package/src/lib.mjs
CHANGED
|
@@ -478,12 +478,16 @@ export const isTransientNetworkError = error => {
|
|
|
478
478
|
/**
|
|
479
479
|
* Retry a GitHub CLI / API operation with exponential backoff on transient network errors.
|
|
480
480
|
* Unlike the generic `retry()`, this function:
|
|
481
|
-
* -
|
|
482
|
-
* -
|
|
481
|
+
* - Retries on transient network errors (TCP reset, TLS timeout, etc.)
|
|
482
|
+
* - Retries on GitHub API rate-limit errors, sleeping until reset + buffer + jitter
|
|
483
|
+
* (issue #1726 — see src/github-rate-limit.lib.mjs)
|
|
484
|
+
* - Immediately rethrows non-transient errors (404, 403 non-rate-limit, auth failures)
|
|
483
485
|
* - Logs stderr to the log file when a command fails (fixing terminal/log parity)
|
|
484
486
|
*
|
|
485
487
|
* Issue #1536: Most gh commands had no retry logic, causing solve to abort on
|
|
486
488
|
* intermittent network issues.
|
|
489
|
+
* Issue #1726: Rate limit errors silently surfaced as command failure with no retry,
|
|
490
|
+
* causing the merge subsystem to swallow them as "no workflows found".
|
|
487
491
|
*
|
|
488
492
|
* @param {Function} fn - Async function to execute (should call gh CLI or GitHub API)
|
|
489
493
|
* @param {Object} [options] - Options
|
|
@@ -496,11 +500,20 @@ export const isTransientNetworkError = error => {
|
|
|
496
500
|
*/
|
|
497
501
|
export const ghRetry = async (fn, options = {}) => {
|
|
498
502
|
const { maxAttempts = 3, delay = 1000, backoff = 2, label = 'gh command' } = options;
|
|
503
|
+
const { isRateLimitError, parseRateLimitReset, fetchNextRateLimitReset, computeRateLimitWait } = await import('./github-rate-limit.lib.mjs');
|
|
499
504
|
|
|
500
505
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
501
506
|
try {
|
|
502
507
|
return await fn();
|
|
503
508
|
} catch (error) {
|
|
509
|
+
if (isRateLimitError(error) && attempt < maxAttempts) {
|
|
510
|
+
const reset = parseRateLimitReset(error) || (await fetchNextRateLimitReset());
|
|
511
|
+
const { waitMs, deadline, bufferMs, jitterMs } = computeRateLimitWait(reset);
|
|
512
|
+
const resetSummary = reset ? `reset at ${reset.toISOString()}` : 'reset time unknown';
|
|
513
|
+
await log(`⏳ ${label}: GitHub API rate limit hit (attempt ${attempt}/${maxAttempts}). Waiting ${Math.round(waitMs / 60000)} min (${resetSummary}; buffer ${Math.round(bufferMs / 60000)} min + jitter ${Math.round(jitterMs / 1000)}s) until ${deadline.toISOString()}.`, { level: 'warn' });
|
|
514
|
+
await sleep(waitMs);
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
504
517
|
if (isTransientNetworkError(error) && attempt < maxAttempts) {
|
|
505
518
|
const waitTime = delay * Math.pow(backoff, attempt - 1);
|
|
506
519
|
await log(`⚠️ ${label}: Network error (attempt ${attempt}/${maxAttempts}), retrying in ${waitTime / 1000}s...`, { level: 'warn' });
|
|
@@ -527,6 +540,7 @@ export const ghRetry = async (fn, options = {}) => {
|
|
|
527
540
|
*/
|
|
528
541
|
export const ghCmdRetry = async (cmdFn, options = {}) => {
|
|
529
542
|
const { maxAttempts = 3, delay = 1000, backoff = 2, label = 'gh command' } = options;
|
|
543
|
+
const { isRateLimitError, parseRateLimitReset, fetchNextRateLimitReset, computeRateLimitWait } = await import('./github-rate-limit.lib.mjs');
|
|
530
544
|
|
|
531
545
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
532
546
|
const result = await cmdFn();
|
|
@@ -541,9 +555,21 @@ export const ghCmdRetry = async (cmdFn, options = {}) => {
|
|
|
541
555
|
return result;
|
|
542
556
|
}
|
|
543
557
|
|
|
544
|
-
// Check if this is a transient network error worth retrying
|
|
545
558
|
const combinedOutput = (result.stdout?.toString() || '') + ' ' + (result.stderr?.toString() || '');
|
|
546
|
-
|
|
559
|
+
const errorLike = { message: combinedOutput, stdout: result.stdout, stderr: result.stderr };
|
|
560
|
+
|
|
561
|
+
// Issue #1726: rate-limit errors deserve a long, deterministic wait.
|
|
562
|
+
if (isRateLimitError(errorLike) && attempt < maxAttempts) {
|
|
563
|
+
const reset = parseRateLimitReset(errorLike) || (await fetchNextRateLimitReset());
|
|
564
|
+
const { waitMs, deadline, bufferMs, jitterMs } = computeRateLimitWait(reset);
|
|
565
|
+
const resetSummary = reset ? `reset at ${reset.toISOString()}` : 'reset time unknown';
|
|
566
|
+
await log(`⏳ ${label}: GitHub API rate limit hit (attempt ${attempt}/${maxAttempts}). Waiting ${Math.round(waitMs / 60000)} min (${resetSummary}; buffer ${Math.round(bufferMs / 60000)} min + jitter ${Math.round(jitterMs / 1000)}s) until ${deadline.toISOString()}.`, { level: 'warn' });
|
|
567
|
+
await sleep(waitMs);
|
|
568
|
+
continue;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
// Check if this is a transient network error worth retrying
|
|
572
|
+
if (isTransientNetworkError(errorLike) && attempt < maxAttempts) {
|
|
547
573
|
const waitTime = delay * Math.pow(backoff, attempt - 1);
|
|
548
574
|
await log(`⚠️ ${label}: Network error (attempt ${attempt}/${maxAttempts}), retrying in ${waitTime / 1000}s...`, { level: 'warn' });
|
|
549
575
|
await sleep(waitTime);
|
package/src/limits.lib.mjs
CHANGED
|
@@ -12,6 +12,7 @@ import { promisify } from 'node:util';
|
|
|
12
12
|
import dayjs from 'dayjs';
|
|
13
13
|
import utc from 'dayjs/plugin/utc.js';
|
|
14
14
|
|
|
15
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
15
16
|
// Initialize dayjs plugins
|
|
16
17
|
dayjs.extend(utc);
|
|
17
18
|
|
package/src/protect-branch.mjs
CHANGED
|
@@ -18,8 +18,9 @@
|
|
|
18
18
|
const { use } = eval(await (await fetch('https://unpkg.com/use-m/use.js')).text());
|
|
19
19
|
|
|
20
20
|
// Use command-stream for consistent $ behavior across runtimes
|
|
21
|
-
const { $ } = await use('command-stream');
|
|
22
|
-
|
|
21
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
22
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
23
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
23
24
|
// Parse command line arguments
|
|
24
25
|
const args = process.argv.slice(2);
|
|
25
26
|
|
package/src/queue-config.lib.mjs
CHANGED
|
@@ -88,7 +88,7 @@ function normalizeMetricName(name) {
|
|
|
88
88
|
* (cpu (65% enqueue))
|
|
89
89
|
* (claude-5-hour (65% dequeue-one-at-a-time))
|
|
90
90
|
* (claude-weekly (97% dequeue-one-at-a-time))
|
|
91
|
-
* (github-api (
|
|
91
|
+
* (github-api (50% enqueue))
|
|
92
92
|
* )
|
|
93
93
|
* ```
|
|
94
94
|
*
|
|
@@ -253,7 +253,11 @@ export const QUEUE_CONFIG = {
|
|
|
253
253
|
claudeWeekly: getThresholdConfig('claudeWeekly', 'HIVE_MIND_CLAUDE_WEEKLY_THRESHOLD', 'HIVE_MIND_CLAUDE_WEEKLY_STRATEGY', 0.97, 'dequeue-one-at-a-time'),
|
|
254
254
|
codex5Hour: getThresholdConfig('codex5Hour', 'HIVE_MIND_CODEX_5_HOUR_SESSION_THRESHOLD', 'HIVE_MIND_CODEX_5_HOUR_SESSION_STRATEGY', 0.65, 'dequeue-one-at-a-time'),
|
|
255
255
|
codexWeekly: getThresholdConfig('codexWeekly', 'HIVE_MIND_CODEX_WEEKLY_THRESHOLD', 'HIVE_MIND_CODEX_WEEKLY_STRATEGY', 0.97, 'dequeue-one-at-a-time'),
|
|
256
|
-
|
|
256
|
+
// Issue #1726: lowered default from 0.75 to 0.50 to start backing off earlier
|
|
257
|
+
// and leave a wider safety margin before the hard 5,000/hr ceiling. Hosted
|
|
258
|
+
// runners hit the ceiling repeatedly with the 75% setting (see
|
|
259
|
+
// docs/case-studies/issue-1726).
|
|
260
|
+
githubApi: getThresholdConfig('githubApi', 'HIVE_MIND_GITHUB_API_THRESHOLD', 'HIVE_MIND_GITHUB_API_STRATEGY', 0.5, 'enqueue'),
|
|
257
261
|
},
|
|
258
262
|
|
|
259
263
|
// Legacy flat threshold values for backward compatibility
|
|
@@ -265,7 +269,7 @@ export const QUEUE_CONFIG = {
|
|
|
265
269
|
CLAUDE_WEEKLY_THRESHOLD: getThresholdConfig('claudeWeekly', 'HIVE_MIND_CLAUDE_WEEKLY_THRESHOLD', 'HIVE_MIND_CLAUDE_WEEKLY_STRATEGY', 0.97, 'dequeue-one-at-a-time').value,
|
|
266
270
|
CODEX_5_HOUR_SESSION_THRESHOLD: getThresholdConfig('codex5Hour', 'HIVE_MIND_CODEX_5_HOUR_SESSION_THRESHOLD', 'HIVE_MIND_CODEX_5_HOUR_SESSION_STRATEGY', 0.65, 'dequeue-one-at-a-time').value,
|
|
267
271
|
CODEX_WEEKLY_THRESHOLD: getThresholdConfig('codexWeekly', 'HIVE_MIND_CODEX_WEEKLY_THRESHOLD', 'HIVE_MIND_CODEX_WEEKLY_STRATEGY', 0.97, 'dequeue-one-at-a-time').value,
|
|
268
|
-
GITHUB_API_THRESHOLD: getThresholdConfig('githubApi', 'HIVE_MIND_GITHUB_API_THRESHOLD', 'HIVE_MIND_GITHUB_API_STRATEGY', 0.
|
|
272
|
+
GITHUB_API_THRESHOLD: getThresholdConfig('githubApi', 'HIVE_MIND_GITHUB_API_THRESHOLD', 'HIVE_MIND_GITHUB_API_STRATEGY', 0.5, 'enqueue').value,
|
|
269
273
|
|
|
270
274
|
// Timing
|
|
271
275
|
// MIN_START_INTERVAL_MS: Time to allow solve command to start actual claude process
|
package/src/review.mjs
CHANGED
|
@@ -35,8 +35,9 @@ if (earlyArgs.includes('--help') || earlyArgs.includes('-h')) {
|
|
|
35
35
|
const { use } = eval(await (await fetch('https://unpkg.com/use-m/use.js')).text());
|
|
36
36
|
|
|
37
37
|
// Use command-stream for consistent $ behavior across runtimes
|
|
38
|
-
const { $ } = await use('command-stream');
|
|
39
|
-
|
|
38
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
39
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
40
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
40
41
|
const yargs = (await use('yargs@latest')).default;
|
|
41
42
|
const os = (await use('os')).default;
|
|
42
43
|
const path = (await use('path')).default;
|
package/src/reviewers-hive.mjs
CHANGED
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
const { use } = eval(await (await fetch('https://unpkg.com/use-m/use.js')).text());
|
|
5
5
|
|
|
6
6
|
// Use command-stream for consistent $ behavior across runtimes
|
|
7
|
-
const { $ } = await use('command-stream');
|
|
8
|
-
|
|
7
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
8
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
9
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
9
10
|
const yargs = (await use('yargs@latest')).default;
|
|
10
11
|
const path = (await use('path')).default;
|
|
11
12
|
const fs = (await use('fs')).promises;
|
|
@@ -16,8 +16,14 @@
|
|
|
16
16
|
|
|
17
17
|
import { promisify } from 'util';
|
|
18
18
|
import { exec as execCallback } from 'child_process';
|
|
19
|
+
import { ghWithRateLimitRetry } from './github-rate-limit.lib.mjs';
|
|
19
20
|
|
|
20
|
-
const
|
|
21
|
+
const execRaw = promisify(execCallback);
|
|
22
|
+
// Issue #1726: rate-limit safe gh wrapper.
|
|
23
|
+
const exec = (cmd, opts) =>
|
|
24
|
+
ghWithRateLimitRetry(() => execRaw(cmd, opts), {
|
|
25
|
+
label: `gh exec (${cmd.split(/\s+/).slice(0, 3).join(' ')})`,
|
|
26
|
+
});
|
|
21
27
|
|
|
22
28
|
// Import retry utility (issue #1536)
|
|
23
29
|
const lib = await import('./lib.mjs');
|
|
@@ -13,8 +13,9 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
13
13
|
const use = globalThis.use;
|
|
14
14
|
|
|
15
15
|
// Use command-stream for consistent $ behavior across runtimes
|
|
16
|
-
const { $ } = await use('command-stream');
|
|
17
|
-
|
|
16
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
17
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
18
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
18
19
|
// Import shared library functions
|
|
19
20
|
const lib = await import('./lib.mjs');
|
|
20
21
|
const { log, cleanErrorMessage } = lib;
|
|
@@ -18,8 +18,9 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
18
18
|
const use = globalThis.use;
|
|
19
19
|
|
|
20
20
|
// Use command-stream for consistent $ behavior across runtimes
|
|
21
|
-
const { $ } = await use('command-stream');
|
|
22
|
-
|
|
21
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
22
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
23
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
23
24
|
// Import shared library functions
|
|
24
25
|
const lib = await import('./lib.mjs');
|
|
25
26
|
const { log } = lib;
|
|
@@ -21,8 +21,9 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
21
21
|
const use = globalThis.use;
|
|
22
22
|
|
|
23
23
|
// Use command-stream for consistent $ behavior across runtimes
|
|
24
|
-
const { $ } = await use('command-stream');
|
|
25
|
-
|
|
24
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
25
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
26
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
26
27
|
// Import shared library functions
|
|
27
28
|
const lib = await import('./lib.mjs');
|
|
28
29
|
const { log, formatAligned } = lib;
|
|
@@ -17,8 +17,9 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
17
17
|
const use = globalThis.use;
|
|
18
18
|
|
|
19
19
|
// Use command-stream for consistent $ behavior across runtimes
|
|
20
|
-
const { $ } = await use('command-stream');
|
|
21
|
-
|
|
20
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
21
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
22
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
22
23
|
// Import shared library functions
|
|
23
24
|
const lib = await import('./lib.mjs');
|
|
24
25
|
const { log, cleanErrorMessage, formatAligned, getLogFile } = lib;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { closingIssueNumbersContain, parseClosingIssueNumbers } from './pr-issue-linking.lib.mjs';
|
|
7
7
|
|
|
8
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
8
9
|
export async function handleAutoPrCreation({ argv, tempDir, branchName, issueNumber, owner, repo, defaultBranch, forkedRepo, isContinueMode, prNumber, log, formatAligned, $, reportError, path, fs }) {
|
|
9
10
|
// Skip auto-PR creation if:
|
|
10
11
|
// 1. Auto-PR creation is disabled AND we're not in continue mode with no PR
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
// Import Sentry integration
|
|
10
10
|
import { reportError } from './sentry.lib.mjs';
|
|
11
11
|
|
|
12
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
12
13
|
export async function handleBranchCheckoutError({ branchName, prNumber, errorOutput, issueUrl, owner, repo, tempDir, argv, formatAligned, log, $ }) {
|
|
13
14
|
// Check if this is a PR from a fork
|
|
14
15
|
let isForkPR = false;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// Import exit handler
|
|
6
6
|
import { safeExit } from './exit-handler.lib.mjs';
|
|
7
7
|
|
|
8
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
8
9
|
// Import Sentry integration
|
|
9
10
|
import { reportError } from './sentry.lib.mjs';
|
|
10
11
|
|
|
@@ -12,8 +12,9 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
12
12
|
const use = globalThis.use;
|
|
13
13
|
|
|
14
14
|
// Use command-stream for consistent $ behavior across runtimes
|
|
15
|
-
const { $ } = await use('command-stream');
|
|
16
|
-
|
|
15
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
16
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
17
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
17
18
|
const os = (await use('os')).default;
|
|
18
19
|
const path = (await use('path')).default;
|
|
19
20
|
const fs = (await use('fs')).promises;
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
// Import Sentry integration
|
|
7
7
|
import { reportError } from './sentry.lib.mjs';
|
|
8
8
|
|
|
9
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
9
10
|
export const detectAndCountFeedback = async params => {
|
|
10
11
|
const { prNumber, branchName, owner, repo, issueNumber, isContinueMode, argv, mergeStateStatus, prState, workStartTime, log, formatAligned, cleanErrorMessage, $ } = params;
|
|
11
12
|
|
package/src/solve.mjs
CHANGED
|
@@ -7,7 +7,9 @@ await handleSolveEarlyExit(earlyArgs);
|
|
|
7
7
|
|
|
8
8
|
const { use } = eval(await (await fetch('https://unpkg.com/use-m/use.js')).text());
|
|
9
9
|
globalThis.use = use;
|
|
10
|
-
const { $ } = await use('command-stream');
|
|
10
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
11
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
12
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
11
13
|
const config = await import('./solve.config.lib.mjs');
|
|
12
14
|
const { initializeConfig, parseArguments } = config;
|
|
13
15
|
// Import Sentry integration
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Handles timestamp collection, feedback detection, and pre-execution checks
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
6
7
|
// Import feedback detection functionality
|
|
7
8
|
const feedback = await import('./solve.feedback.lib.mjs');
|
|
8
9
|
const { detectAndCountFeedback } = feedback;
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
// comment is excluded from --auto-attach-solution-summary's AI-comment check.
|
|
29
29
|
import { LIVE_PROGRESS_SECTION_START_MARKER, LIVE_PROGRESS_SECTION_END_MARKER, postTrackedCommentFromFile, trackToolCommentId } from './tool-comments.lib.mjs';
|
|
30
30
|
|
|
31
|
+
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
31
32
|
/**
|
|
32
33
|
* Configuration constants for progress monitoring
|
|
33
34
|
*/
|
|
@@ -11,9 +11,9 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
11
11
|
}
|
|
12
12
|
const use = globalThis.use;
|
|
13
13
|
|
|
14
|
-
// Use command-stream for consistent $ behavior
|
|
15
|
-
const {
|
|
16
|
-
|
|
14
|
+
// Use command-stream for consistent $ behavior; wrap with rate-limit retry (#1726)
|
|
15
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
16
|
+
const $ = wrapDollarWithGhRetry((await use('command-stream')).$);
|
|
17
17
|
const os = (await use('os')).default;
|
|
18
18
|
const path = (await use('path')).default;
|
|
19
19
|
const fs = (await use('fs')).promises;
|
|
@@ -20,8 +20,9 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
20
20
|
const use = globalThis.use;
|
|
21
21
|
|
|
22
22
|
// Use command-stream for consistent $ behavior across runtimes
|
|
23
|
-
const { $ } = await use('command-stream');
|
|
24
|
-
|
|
23
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
24
|
+
const { wrapDollarWithGhRetry } = await import('./github-rate-limit.lib.mjs');
|
|
25
|
+
const $ = wrapDollarWithGhRetry(__rawDollar$);
|
|
25
26
|
// Import path and fs for cleanup operations
|
|
26
27
|
const path = (await use('path')).default;
|
|
27
28
|
const fs = (await use('fs')).promises;
|