agent-relay-runner 0.77.0 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/plugins/claude/.claude-plugin/plugin.json +1 -1
- package/src/quota.ts +28 -586
- package/src/runner-core.ts +3 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-relay-runner",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.78.0",
|
|
4
4
|
"description": "Unified provider lifecycle runner for Agent Relay",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"directory": "runner"
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"agent-relay-sdk": "0.2.
|
|
23
|
+
"agent-relay-sdk": "0.2.53"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@types/bun": "latest",
|
package/src/quota.ts
CHANGED
|
@@ -1,586 +1,28 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
type RunnerQuotaSample = {
|
|
31
|
-
quota?: QuotaState;
|
|
32
|
-
accountKey?: string;
|
|
33
|
-
};
|
|
34
|
-
|
|
35
|
-
type SharedQuotaEntry = {
|
|
36
|
-
provider: string;
|
|
37
|
-
accountKey: string;
|
|
38
|
-
lastAttemptAt?: number;
|
|
39
|
-
nextAttemptAt?: number;
|
|
40
|
-
quota?: QuotaState;
|
|
41
|
-
lastError?: ProviderQuotaError | null;
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
export class RunnerQuotaPoller {
|
|
45
|
-
private quotaState?: QuotaState;
|
|
46
|
-
private timer?: Timer;
|
|
47
|
-
private inFlight = false;
|
|
48
|
-
private active = false;
|
|
49
|
-
private lastLog?: { key: string; at: number };
|
|
50
|
-
|
|
51
|
-
constructor(private readonly options: {
|
|
52
|
-
provider: string;
|
|
53
|
-
agentId: string;
|
|
54
|
-
getProcess: () => ManagedProcess | undefined;
|
|
55
|
-
onUpdate: () => void;
|
|
56
|
-
log: (message: string) => void;
|
|
57
|
-
reportProviderQuota?: (input: ProviderQuotaUpdateInput) => Promise<unknown>;
|
|
58
|
-
quotaStateDir?: string;
|
|
59
|
-
fetchImpl?: typeof fetch;
|
|
60
|
-
}) {}
|
|
61
|
-
|
|
62
|
-
get current(): QuotaState | undefined {
|
|
63
|
-
return this.quotaState;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
start(): void {
|
|
67
|
-
this.stop();
|
|
68
|
-
if (!quotaProviderSupported(this.options.provider)) return;
|
|
69
|
-
this.active = true;
|
|
70
|
-
void this.refresh();
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
stop(): void {
|
|
74
|
-
this.active = false;
|
|
75
|
-
if (this.timer) clearTimeout(this.timer);
|
|
76
|
-
this.timer = undefined;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
clear(): void {
|
|
80
|
-
this.quotaState = undefined;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
private schedule(delayMs = QUOTA_POLL_INTERVAL_MS): void {
|
|
84
|
-
if (this.timer) clearTimeout(this.timer);
|
|
85
|
-
this.timer = undefined;
|
|
86
|
-
if (!this.active || !quotaProviderSupported(this.options.provider)) return;
|
|
87
|
-
this.timer = setTimeout(() => {
|
|
88
|
-
this.timer = undefined;
|
|
89
|
-
void this.refresh();
|
|
90
|
-
}, Math.max(1_000, delayMs));
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
private async refresh(): Promise<void> {
|
|
94
|
-
if (this.inFlight) return;
|
|
95
|
-
this.inFlight = true;
|
|
96
|
-
const now = Date.now();
|
|
97
|
-
let identity: QuotaIdentity | undefined;
|
|
98
|
-
let lockDir: string | undefined;
|
|
99
|
-
try {
|
|
100
|
-
const process = this.options.getProcess();
|
|
101
|
-
identity = await resolveRunnerQuotaIdentity({
|
|
102
|
-
provider: this.options.provider,
|
|
103
|
-
process,
|
|
104
|
-
});
|
|
105
|
-
const attempt = beginSharedQuotaAttempt(identity, this.options.quotaStateDir, now);
|
|
106
|
-
if (!attempt.poll) {
|
|
107
|
-
if (attempt.entry?.quota && this.active) {
|
|
108
|
-
this.quotaState = attempt.entry.quota;
|
|
109
|
-
this.options.onUpdate();
|
|
110
|
-
}
|
|
111
|
-
this.schedule(attempt.retryInMs);
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
lockDir = attempt.lockDir;
|
|
115
|
-
const sample = await collectRunnerQuotaSample({
|
|
116
|
-
provider: this.options.provider,
|
|
117
|
-
agentId: this.options.agentId,
|
|
118
|
-
process,
|
|
119
|
-
credentialsPath: identity.credentialsPath,
|
|
120
|
-
fetchImpl: this.options.fetchImpl,
|
|
121
|
-
});
|
|
122
|
-
const quota = sample.quota;
|
|
123
|
-
if (!quota) throw new QuotaCollectionError("creds_not_ready", `${this.options.provider} quota source is not ready`);
|
|
124
|
-
if (this.active) {
|
|
125
|
-
const update: ProviderQuotaUpdateInput = {
|
|
126
|
-
provider: identity.provider,
|
|
127
|
-
accountKey: sample.accountKey ?? identity.accountKey,
|
|
128
|
-
quota,
|
|
129
|
-
lastAttemptAt: quota.updatedAt,
|
|
130
|
-
sourceAgentId: this.options.agentId,
|
|
131
|
-
};
|
|
132
|
-
await this.reportProviderQuota(update);
|
|
133
|
-
finishSharedQuotaAttempt(identity, {
|
|
134
|
-
provider: update.provider,
|
|
135
|
-
accountKey: update.accountKey,
|
|
136
|
-
quota,
|
|
137
|
-
lastAttemptAt: update.lastAttemptAt,
|
|
138
|
-
nextAttemptAt: now + QUOTA_POLL_INTERVAL_MS,
|
|
139
|
-
lastError: null,
|
|
140
|
-
}, this.options.quotaStateDir);
|
|
141
|
-
this.quotaState = quota;
|
|
142
|
-
this.options.onUpdate();
|
|
143
|
-
}
|
|
144
|
-
this.schedule();
|
|
145
|
-
} catch (error) {
|
|
146
|
-
const retryAfterMs = quotaRetryAfterMs(error);
|
|
147
|
-
const lastError = providerQuotaErrorFromCollectorError(error, retryAfterMs);
|
|
148
|
-
const lastAttemptAt = Date.now();
|
|
149
|
-
const retryDelayMs = retryAfterMs ?? (identity && !readSharedQuotaEntry(identity, this.options.quotaStateDir)?.lastAttemptAt ? QUOTA_FAST_RETRY_MS : QUOTA_POLL_INTERVAL_MS);
|
|
150
|
-
if (identity) {
|
|
151
|
-
await this.reportProviderQuota({
|
|
152
|
-
provider: identity.provider,
|
|
153
|
-
accountKey: identity.accountKey,
|
|
154
|
-
lastAttemptAt,
|
|
155
|
-
lastError,
|
|
156
|
-
sourceAgentId: this.options.agentId,
|
|
157
|
-
}).catch((publishError) => {
|
|
158
|
-
this.options.log(`quota status publish failed: ${errMessage(publishError)}`);
|
|
159
|
-
});
|
|
160
|
-
finishSharedQuotaAttempt(identity, {
|
|
161
|
-
provider: identity.provider,
|
|
162
|
-
accountKey: identity.accountKey,
|
|
163
|
-
lastAttemptAt,
|
|
164
|
-
nextAttemptAt: lastAttemptAt + retryDelayMs,
|
|
165
|
-
lastError,
|
|
166
|
-
}, this.options.quotaStateDir);
|
|
167
|
-
}
|
|
168
|
-
if (this.active) this.logFailure(error, retryAfterMs);
|
|
169
|
-
this.schedule(retryDelayMs);
|
|
170
|
-
} finally {
|
|
171
|
-
if (lockDir) releaseQuotaLock(lockDir);
|
|
172
|
-
this.inFlight = false;
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
private async reportProviderQuota(input: ProviderQuotaUpdateInput): Promise<void> {
|
|
177
|
-
if (!this.options.reportProviderQuota) return;
|
|
178
|
-
await this.options.reportProviderQuota(input);
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
private logFailure(error: unknown, retryAfterMs: number | undefined): void {
|
|
182
|
-
const key = retryAfterMs !== undefined ? `retry-after:${retryAfterMs}` : errMessage(error);
|
|
183
|
-
const now = Date.now();
|
|
184
|
-
if (this.lastLog?.key === key && now - this.lastLog.at < QUOTA_FAILURE_LOG_INTERVAL_MS) return;
|
|
185
|
-
this.lastLog = { key, at: now };
|
|
186
|
-
const suffix = retryAfterMs !== undefined ? `; retrying in ${Math.round(retryAfterMs / 1000)}s` : "";
|
|
187
|
-
this.options.log(`quota refresh failed${suffix}: ${errMessage(error)}`);
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
export class QuotaRetryAfterError extends Error {
|
|
192
|
-
constructor(readonly retryAfterMs: number, message = "quota source requested retry backoff") {
|
|
193
|
-
super(message);
|
|
194
|
-
this.name = "QuotaRetryAfterError";
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
class QuotaCollectionError extends Error {
|
|
199
|
-
constructor(readonly type: ProviderQuotaError["type"], message: string) {
|
|
200
|
-
super(message);
|
|
201
|
-
this.name = "QuotaCollectionError";
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
function quotaRetryAfterMs(error: unknown): number | undefined {
|
|
206
|
-
return error instanceof QuotaRetryAfterError ? error.retryAfterMs : undefined;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
function providerQuotaErrorFromCollectorError(error: unknown, retryAfterMs: number | undefined): ProviderQuotaError {
|
|
210
|
-
if (retryAfterMs !== undefined) {
|
|
211
|
-
return { type: "rate_limited", message: errMessage(error), retryAfterMs };
|
|
212
|
-
}
|
|
213
|
-
if (error instanceof QuotaCollectionError) {
|
|
214
|
-
return { type: error.type, message: error.message };
|
|
215
|
-
}
|
|
216
|
-
return { type: "other", message: errMessage(error) };
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
export async function resolveRunnerQuotaIdentity(input: {
|
|
220
|
-
provider: string;
|
|
221
|
-
process?: ManagedProcess;
|
|
222
|
-
}): Promise<QuotaIdentity> {
|
|
223
|
-
const provider = input.provider.trim().toLowerCase();
|
|
224
|
-
if (provider === "claude") {
|
|
225
|
-
const credentialsPath = claudeCredentialsPathFromProcess(input.process);
|
|
226
|
-
const token = await readClaudeOAuthAccessToken(credentialsPath);
|
|
227
|
-
return {
|
|
228
|
-
provider,
|
|
229
|
-
accountKey: token ? `credential:${shortHash(token)}` : hostProviderAccountKey(provider),
|
|
230
|
-
credentialsPath,
|
|
231
|
-
};
|
|
232
|
-
}
|
|
233
|
-
return {
|
|
234
|
-
provider,
|
|
235
|
-
accountKey: codexAccountKeyFromProcess(input.process) ?? hostProviderAccountKey(provider),
|
|
236
|
-
};
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
async function collectRunnerQuotaSample(input: {
|
|
240
|
-
provider: string;
|
|
241
|
-
agentId: string;
|
|
242
|
-
process?: ManagedProcess;
|
|
243
|
-
credentialsPath?: string;
|
|
244
|
-
fetchImpl?: typeof fetch;
|
|
245
|
-
}): Promise<RunnerQuotaSample> {
|
|
246
|
-
if (input.provider === "claude") return collectClaudeQuotaSample({ agentId: input.agentId, credentialsPath: input.credentialsPath, fetchImpl: input.fetchImpl });
|
|
247
|
-
if (input.provider !== "codex") return {};
|
|
248
|
-
const client = input.process?.meta?.client as CodexQuotaClient | undefined;
|
|
249
|
-
if (!client?.isConnected?.() || typeof client.accountRateLimitsRead !== "function") return {};
|
|
250
|
-
return collectCodexQuotaSample({
|
|
251
|
-
agentId: codexProcessAgentId(input.process, input.agentId),
|
|
252
|
-
rateLimitsRead: () => client.accountRateLimitsRead!(),
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
async function collectClaudeQuotaSample(options: {
|
|
257
|
-
agentId: string;
|
|
258
|
-
credentialsPath?: string;
|
|
259
|
-
fetchImpl?: typeof fetch;
|
|
260
|
-
now?: number;
|
|
261
|
-
}): Promise<RunnerQuotaSample> {
|
|
262
|
-
const payload = await fetchClaudeUsagePayload(options);
|
|
263
|
-
if (payload === undefined) return {};
|
|
264
|
-
return {
|
|
265
|
-
quota: claudeUsageQuotaState(payload, options.agentId, options.now ?? Date.now()),
|
|
266
|
-
accountKey: providerPayloadAccountKey(payload),
|
|
267
|
-
};
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
export async function collectClaudeQuotaState(options: {
|
|
271
|
-
agentId: string;
|
|
272
|
-
credentialsPath?: string;
|
|
273
|
-
fetchImpl?: typeof fetch;
|
|
274
|
-
now?: number;
|
|
275
|
-
}): Promise<QuotaState | undefined> {
|
|
276
|
-
return (await collectClaudeQuotaSample(options)).quota;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
async function fetchClaudeUsagePayload(options: {
|
|
280
|
-
credentialsPath?: string;
|
|
281
|
-
fetchImpl?: typeof fetch;
|
|
282
|
-
}): Promise<unknown | undefined> {
|
|
283
|
-
const accessToken = await readClaudeOAuthAccessToken(options.credentialsPath ?? defaultClaudeCredentialsPath());
|
|
284
|
-
if (!accessToken) return undefined;
|
|
285
|
-
const fetchImpl = options.fetchImpl ?? fetch;
|
|
286
|
-
const response = await fetchImpl(CLAUDE_USAGE_URL, {
|
|
287
|
-
method: "GET",
|
|
288
|
-
headers: {
|
|
289
|
-
authorization: `Bearer ${accessToken}`,
|
|
290
|
-
"anthropic-beta": CLAUDE_OAUTH_BETA,
|
|
291
|
-
},
|
|
292
|
-
signal: AbortSignal.timeout(10_000),
|
|
293
|
-
});
|
|
294
|
-
if (response.status === 429) {
|
|
295
|
-
throw new QuotaRetryAfterError(retryAfterHeaderMs(response.headers.get("retry-after")) ?? 60_000, "Claude quota source returned 429");
|
|
296
|
-
}
|
|
297
|
-
if (response.status === 401 || response.status === 403) throw new QuotaCollectionError("auth", `Claude quota source returned ${response.status}`);
|
|
298
|
-
if (!response.ok) throw new QuotaCollectionError("other", `Claude quota source returned ${response.status}`);
|
|
299
|
-
return response.json();
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
export function claudeUsageQuotaState(payload: unknown, agentId: string, now = Date.now()): QuotaState | undefined {
|
|
303
|
-
return quotaStateFromProbeMetrics({
|
|
304
|
-
agentId,
|
|
305
|
-
contextPercent: 0,
|
|
306
|
-
quotaWindows: [
|
|
307
|
-
quotaWindowFromAnthropicUsage("five_hour", payload),
|
|
308
|
-
quotaWindowFromAnthropicUsage("seven_day", payload),
|
|
309
|
-
].filter((window): window is NonNullable<ContextProbeMetrics["quotaWindows"]>[number] => Boolean(window)),
|
|
310
|
-
source: "api",
|
|
311
|
-
confidence: "reported",
|
|
312
|
-
timestamp: now,
|
|
313
|
-
});
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
export async function collectCodexQuotaState(input: {
|
|
317
|
-
agentId: string;
|
|
318
|
-
rateLimitsRead: () => Promise<unknown>;
|
|
319
|
-
now?: number;
|
|
320
|
-
}): Promise<QuotaState | undefined> {
|
|
321
|
-
return (await collectCodexQuotaSample(input)).quota;
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
async function collectCodexQuotaSample(input: {
|
|
325
|
-
agentId: string;
|
|
326
|
-
rateLimitsRead: () => Promise<unknown>;
|
|
327
|
-
now?: number;
|
|
328
|
-
}): Promise<RunnerQuotaSample> {
|
|
329
|
-
try {
|
|
330
|
-
const payload = await input.rateLimitsRead();
|
|
331
|
-
return {
|
|
332
|
-
quota: codexRateLimitsQuotaState(payload, input.agentId, input.now ?? Date.now()),
|
|
333
|
-
accountKey: providerPayloadAccountKey(payload),
|
|
334
|
-
};
|
|
335
|
-
} catch (error) {
|
|
336
|
-
const retryAfter = codexRetryAfterMs(error);
|
|
337
|
-
if (retryAfter !== undefined) throw new QuotaRetryAfterError(retryAfter, "Codex quota source returned 429");
|
|
338
|
-
throw error;
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
export function codexRateLimitsQuotaState(payload: unknown, agentId: string, now = Date.now()): QuotaState | undefined {
|
|
343
|
-
const rateLimits = isRecord(payload) && isRecord(payload.rateLimits) ? payload.rateLimits
|
|
344
|
-
: isRecord(payload) && isRecord(payload.rate_limits) ? payload.rate_limits
|
|
345
|
-
: payload;
|
|
346
|
-
return quotaStateFromProbeMetrics({
|
|
347
|
-
agentId,
|
|
348
|
-
contextPercent: 0,
|
|
349
|
-
quotaWindows: [
|
|
350
|
-
quotaWindowFromCodexRateLimit("primary", rateLimits),
|
|
351
|
-
quotaWindowFromCodexRateLimit("secondary", rateLimits),
|
|
352
|
-
].filter((window): window is NonNullable<ContextProbeMetrics["quotaWindows"]>[number] => Boolean(window)),
|
|
353
|
-
source: "api",
|
|
354
|
-
confidence: "reported",
|
|
355
|
-
timestamp: now,
|
|
356
|
-
});
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
function defaultClaudeCredentialsPath(): string {
|
|
360
|
-
return join(defaultClaudeConfigDir(), ".credentials.json");
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
export function defaultClaudeConfigDir(): string {
|
|
364
|
-
return process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude");
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
function claudeCredentialsPathFromProcess(proc: ManagedProcess | undefined): string {
|
|
368
|
-
const env = processEnvFromMeta(proc);
|
|
369
|
-
const configDir = stringValue(env?.CLAUDE_CONFIG_DIR) ?? process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude");
|
|
370
|
-
return join(configDir, ".credentials.json");
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
function codexAccountKeyFromProcess(proc: ManagedProcess | undefined): string | undefined {
|
|
374
|
-
const env = processEnvFromMeta(proc);
|
|
375
|
-
const codexHome = stringValue(env?.CODEX_HOME) ?? process.env.CODEX_HOME;
|
|
376
|
-
return codexHome ? `home:${shortHash(codexHome)}` : undefined;
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
function processEnvFromMeta(proc: ManagedProcess | undefined): Record<string, unknown> | undefined {
|
|
380
|
-
const direct = isRecord(proc?.meta?.env) ? proc.meta.env : undefined;
|
|
381
|
-
if (direct) return direct;
|
|
382
|
-
const config = isRecord(proc?.meta?.config) ? proc.meta.config : undefined;
|
|
383
|
-
return isRecord(config?.env) ? config.env : undefined;
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
function hostProviderAccountKey(provider: string): string {
|
|
387
|
-
return `host:${hostname()}:${provider}`;
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
export function beginSharedQuotaAttempt(identity: QuotaIdentity, stateDir?: string, now = Date.now()): {
|
|
391
|
-
poll: boolean;
|
|
392
|
-
retryInMs: number;
|
|
393
|
-
entry?: SharedQuotaEntry;
|
|
394
|
-
lockDir?: string;
|
|
395
|
-
} {
|
|
396
|
-
const entry = readSharedQuotaEntry(identity, stateDir);
|
|
397
|
-
const nextAttemptAt = entry?.nextAttemptAt;
|
|
398
|
-
if (nextAttemptAt !== undefined && nextAttemptAt > now) {
|
|
399
|
-
return { poll: false, retryInMs: nextAttemptAt - now, entry };
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
const lockDir = sharedQuotaLockDir(identity, stateDir);
|
|
403
|
-
if (!acquireQuotaLock(lockDir, now)) {
|
|
404
|
-
return { poll: false, retryInMs: QUOTA_FAST_RETRY_MS, entry };
|
|
405
|
-
}
|
|
406
|
-
return { poll: true, retryInMs: 0, entry, lockDir };
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
function finishSharedQuotaAttempt(identity: QuotaIdentity, entry: SharedQuotaEntry, stateDir?: string): void {
|
|
410
|
-
const path = sharedQuotaStatePath(identity, stateDir);
|
|
411
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
412
|
-
writeFileSync(path, JSON.stringify(entry, null, 2));
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
function readSharedQuotaEntry(identity: QuotaIdentity, stateDir?: string): SharedQuotaEntry | undefined {
|
|
416
|
-
try {
|
|
417
|
-
const parsed = JSON.parse(readFileSync(sharedQuotaStatePath(identity, stateDir), "utf8")) as unknown;
|
|
418
|
-
if (!isRecord(parsed)) return undefined;
|
|
419
|
-
const provider = stringValue(parsed.provider);
|
|
420
|
-
const accountKey = stringValue(parsed.accountKey);
|
|
421
|
-
if (!provider || !accountKey) return undefined;
|
|
422
|
-
return {
|
|
423
|
-
provider,
|
|
424
|
-
accountKey,
|
|
425
|
-
...(typeof parsed.lastAttemptAt === "number" && Number.isFinite(parsed.lastAttemptAt) ? { lastAttemptAt: parsed.lastAttemptAt } : {}),
|
|
426
|
-
...(typeof parsed.nextAttemptAt === "number" && Number.isFinite(parsed.nextAttemptAt) ? { nextAttemptAt: parsed.nextAttemptAt } : {}),
|
|
427
|
-
...(isQuotaState(parsed.quota) ? { quota: parsed.quota } : {}),
|
|
428
|
-
...(parsed.lastError === null || isRecord(parsed.lastError) ? { lastError: parsed.lastError as ProviderQuotaError | null } : {}),
|
|
429
|
-
};
|
|
430
|
-
} catch {
|
|
431
|
-
return undefined;
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
function sharedQuotaStatePath(identity: QuotaIdentity, stateDir?: string): string {
|
|
436
|
-
return join(sharedQuotaStateDir(stateDir), `${sharedQuotaKey(identity)}.json`);
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
function sharedQuotaLockDir(identity: QuotaIdentity, stateDir?: string): string {
|
|
440
|
-
return join(sharedQuotaStateDir(stateDir), `${sharedQuotaKey(identity)}.lock`);
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
function sharedQuotaStateDir(stateDir?: string): string {
|
|
444
|
-
return stateDir ?? join(runnerOutboxDirWithInfoFallback() ?? agentRelayHome(), "quota");
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
function sharedQuotaKey(identity: QuotaIdentity): string {
|
|
448
|
-
return sanitizeFsName(`${identity.provider}-${identity.accountKey}`);
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
function acquireQuotaLock(lockDir: string, now = Date.now()): boolean {
|
|
452
|
-
try {
|
|
453
|
-
mkdirSync(lockDir, { recursive: false });
|
|
454
|
-
writeFileSync(join(lockDir, "owner"), String(now));
|
|
455
|
-
return true;
|
|
456
|
-
} catch {
|
|
457
|
-
const ownerPath = join(lockDir, "owner");
|
|
458
|
-
try {
|
|
459
|
-
const lockedAt = Number(readFileSync(ownerPath, "utf8"));
|
|
460
|
-
if (Number.isFinite(lockedAt) && now - lockedAt <= QUOTA_LOCK_STALE_MS) return false;
|
|
461
|
-
rmSync(lockDir, { recursive: true, force: true });
|
|
462
|
-
mkdirSync(lockDir, { recursive: false });
|
|
463
|
-
writeFileSync(ownerPath, String(now));
|
|
464
|
-
return true;
|
|
465
|
-
} catch {
|
|
466
|
-
return false;
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
function releaseQuotaLock(lockDir: string): void {
|
|
472
|
-
rmSync(lockDir, { recursive: true, force: true });
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
function isQuotaState(value: unknown): value is QuotaState {
|
|
476
|
-
return isRecord(value) && Array.isArray(value.windows) && typeof value.updatedAt === "number";
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
export function providerPayloadAccountKey(payload: unknown): string | undefined {
|
|
480
|
-
if (!isRecord(payload)) return undefined;
|
|
481
|
-
const direct = stringValue(payload.accountKey) ??
|
|
482
|
-
stringValue(payload.account_key) ??
|
|
483
|
-
stringValue(payload.accountId) ??
|
|
484
|
-
stringValue(payload.account_id) ??
|
|
485
|
-
stringValue(payload.orgId) ??
|
|
486
|
-
stringValue(payload.org_id) ??
|
|
487
|
-
stringValue(payload.organizationId) ??
|
|
488
|
-
stringValue(payload.organization_id);
|
|
489
|
-
if (direct) return direct;
|
|
490
|
-
const account = isRecord(payload.account) ? payload.account : undefined;
|
|
491
|
-
const org = isRecord(payload.org) ? payload.org : isRecord(payload.organization) ? payload.organization : undefined;
|
|
492
|
-
return stringValue(account?.id) ?? stringValue(org?.id);
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
function shortHash(value: string): string {
|
|
496
|
-
return createHash("sha256").update(value).digest("hex").slice(0, 16);
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
function quotaProviderSupported(provider: string): boolean {
|
|
500
|
-
return provider === "claude" || provider === "codex";
|
|
501
|
-
}
|
|
502
|
-
|
|
503
|
-
function codexProcessAgentId(process: ManagedProcess | undefined, fallback: string): string {
|
|
504
|
-
const config = isRecord(process?.meta?.config) ? process.meta.config : undefined;
|
|
505
|
-
return stringValue(config?.agentId) ?? fallback;
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
async function readClaudeOAuthAccessToken(path: string): Promise<string | undefined> {
|
|
509
|
-
try {
|
|
510
|
-
const parsed = JSON.parse(await readFile(path, "utf8")) as unknown;
|
|
511
|
-
const oauth = isRecord(parsed) && isRecord(parsed.claudeAiOauth) ? parsed.claudeAiOauth : undefined;
|
|
512
|
-
const token = oauth?.accessToken;
|
|
513
|
-
return typeof token === "string" && token.trim() ? token : undefined;
|
|
514
|
-
} catch {
|
|
515
|
-
return undefined;
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
function quotaWindowFromAnthropicUsage(name: QuotaWindowName, payload: unknown): NonNullable<ContextProbeMetrics["quotaWindows"]>[number] | undefined {
|
|
520
|
-
if (!isRecord(payload)) return undefined;
|
|
521
|
-
const raw = isRecord(payload[name]) ? payload[name] : undefined;
|
|
522
|
-
const utilization = normalizePercentLike(raw?.utilization);
|
|
523
|
-
if (utilization === undefined) return undefined;
|
|
524
|
-
return {
|
|
525
|
-
name,
|
|
526
|
-
utilization,
|
|
527
|
-
...(resetValueMs(raw?.resets_at) !== undefined ? { resetsAt: resetValueMs(raw?.resets_at)! } : {}),
|
|
528
|
-
unit: "%",
|
|
529
|
-
};
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
function quotaWindowFromCodexRateLimit(name: QuotaWindowName, payload: unknown): NonNullable<ContextProbeMetrics["quotaWindows"]>[number] | undefined {
|
|
533
|
-
if (!isRecord(payload)) return undefined;
|
|
534
|
-
const raw = isRecord(payload[name]) ? payload[name] : undefined;
|
|
535
|
-
const utilization = normalizePercentLike(raw?.usedPercent ?? raw?.used_percent ?? raw?.utilization);
|
|
536
|
-
if (utilization === undefined) return undefined;
|
|
537
|
-
return {
|
|
538
|
-
name,
|
|
539
|
-
utilization,
|
|
540
|
-
...(resetValueMs(raw?.resetsAt ?? raw?.resets_at) !== undefined ? { resetsAt: resetValueMs(raw?.resetsAt ?? raw?.resets_at)! } : {}),
|
|
541
|
-
unit: "%",
|
|
542
|
-
};
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
function normalizePercentLike(value: unknown): number | undefined {
|
|
546
|
-
const number = numberValue(value);
|
|
547
|
-
if (number === undefined) return undefined;
|
|
548
|
-
const fraction = number > 1 ? number / 100 : number;
|
|
549
|
-
return Math.max(0, Math.min(1, fraction));
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
function resetValueMs(value: unknown): number | undefined {
|
|
553
|
-
const number = numberValue(value);
|
|
554
|
-
if (number !== undefined) return number < 10_000_000_000 ? number * 1000 : number;
|
|
555
|
-
if (typeof value !== "string" || !value.trim()) return undefined;
|
|
556
|
-
const parsed = Date.parse(value);
|
|
557
|
-
return Number.isFinite(parsed) ? parsed : undefined;
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
function retryAfterHeaderMs(value: string | null): number | undefined {
|
|
561
|
-
if (!value) return undefined;
|
|
562
|
-
const seconds = Number(value);
|
|
563
|
-
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000;
|
|
564
|
-
const date = Date.parse(value);
|
|
565
|
-
if (!Number.isFinite(date)) return undefined;
|
|
566
|
-
return Math.max(0, date - Date.now());
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
function codexRetryAfterMs(error: unknown): number | undefined {
|
|
570
|
-
const message = errMessage(error);
|
|
571
|
-
if (!/\b429\b/.test(message)) return undefined;
|
|
572
|
-
return retryAfterHeaderMs(retryAfterFromText(message)) ?? 60_000;
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
function retryAfterFromText(message: string): string | null {
|
|
576
|
-
return /retry-after[:=]\s*([^,;\s]+)/i.exec(message)?.[1] ?? null;
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
function numberValue(value: unknown): number | undefined {
|
|
580
|
-
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
581
|
-
if (typeof value === "string" && value.trim()) {
|
|
582
|
-
const parsed = Number(value);
|
|
583
|
-
return Number.isFinite(parsed) ? parsed : undefined;
|
|
584
|
-
}
|
|
585
|
-
return undefined;
|
|
586
|
-
}
|
|
1
|
+
export {
|
|
2
|
+
CLAUDE_OAUTH_BETA,
|
|
3
|
+
CLAUDE_USAGE_URL,
|
|
4
|
+
QUOTA_FAILURE_LOG_INTERVAL_MS,
|
|
5
|
+
QUOTA_FAST_RETRY_MS,
|
|
6
|
+
QUOTA_POLL_INTERVAL_MS,
|
|
7
|
+
QuotaCollectionError,
|
|
8
|
+
QuotaRetryAfterError,
|
|
9
|
+
claudeUsageQuotaState,
|
|
10
|
+
collectClaudeQuotaSample,
|
|
11
|
+
collectClaudeQuotaState,
|
|
12
|
+
collectCodexQuotaSample,
|
|
13
|
+
collectCodexQuotaState,
|
|
14
|
+
codexRateLimitsQuotaState,
|
|
15
|
+
credentialAccountKey,
|
|
16
|
+
defaultClaudeConfigDir,
|
|
17
|
+
defaultClaudeCredentialsPath,
|
|
18
|
+
hostProviderAccountKey,
|
|
19
|
+
providerPayloadAccountKey,
|
|
20
|
+
providerQuotaErrorFromCollectorError,
|
|
21
|
+
quotaRetryAfterMs,
|
|
22
|
+
readClaudeOAuthAccessToken,
|
|
23
|
+
resolveClaudeQuotaIdentity,
|
|
24
|
+
resolveCodexQuotaIdentity,
|
|
25
|
+
resolveCodexQuotaIdentityFromHome,
|
|
26
|
+
type ProviderQuotaIdentity,
|
|
27
|
+
type ProviderQuotaSample,
|
|
28
|
+
} from "agent-relay-sdk/provider-quota";
|
package/src/runner-core.ts
CHANGED
|
@@ -24,7 +24,6 @@ import { RunnerInsights } from "./runner-insights";
|
|
|
24
24
|
import { BusyReconciler } from "./busy-reconciler";
|
|
25
25
|
import { publishCapturedResponse } from "./response-capture-report";
|
|
26
26
|
import { runnerLaunchInjectionSessionEvents, type RunnerRelayInjectionEvent } from "./relay-injection-events";
|
|
27
|
-
import { RunnerQuotaPoller } from "./quota";
|
|
28
27
|
import { capsFromEnv, contextStateDirFromEnv, logLevelFromEnv, mcpProxyEnabledFromEnv, orchestratorBaseDirFromEnv, orchestratorUrlFromEnv, registrationTimeoutMsFromEnv, runnerInfoFileFromEnv, runnerLogFileFromEnv, runnerOutboxDirWithInfoFallback, sessionDebugEnabled, tagsFromEnv, workspaceModeFromEnv } from "./config";
|
|
29
28
|
import {
|
|
30
29
|
appliedAgentProfileMetadata,
|
|
@@ -205,7 +204,6 @@ export class AgentRunner {
|
|
|
205
204
|
private readonly sessionOutbox: Outbox;
|
|
206
205
|
private readonly insights: RunnerInsights;
|
|
207
206
|
private readonly busyReconciler: BusyReconciler;
|
|
208
|
-
private readonly quotaPoller: RunnerQuotaPoller;
|
|
209
207
|
private currentToken?: string;
|
|
210
208
|
private currentTokenJti?: string;
|
|
211
209
|
private currentTokenProfileId?: string;
|
|
@@ -353,7 +351,6 @@ export class AgentRunner {
|
|
|
353
351
|
clearProviderTurn: (reason) => this.forceClearProviderTurn(reason),
|
|
354
352
|
sessionDebug: (message) => this.sessionDebug(message),
|
|
355
353
|
});
|
|
356
|
-
this.quotaPoller = new RunnerQuotaPoller({ provider: options.provider, agentId: this.agentId, getProcess: () => this.process, onUpdate: () => this.publishStatus(), log: (message) => this.logRunnerDiagnostic(message), reportProviderQuota: (input) => this.http.upsertProviderQuota(input).then(() => {}) });
|
|
357
354
|
this.bus = new RelayBusClient({
|
|
358
355
|
url: relayBusUrl(options.relayUrl),
|
|
359
356
|
role: "provider",
|
|
@@ -464,7 +461,7 @@ export class AgentRunner {
|
|
|
464
461
|
void this.sweepStaleScratch();
|
|
465
462
|
this.process = await this.spawnProvider();
|
|
466
463
|
this.writeRunnerInfoFile();
|
|
467
|
-
this.processStartedAt = Date.now();
|
|
464
|
+
this.processStartedAt = Date.now();
|
|
468
465
|
this.publishStatus();
|
|
469
466
|
await this.deliverInitialPrompt();
|
|
470
467
|
await this.bootstrapUnreadMessages();
|
|
@@ -496,7 +493,7 @@ export class AgentRunner {
|
|
|
496
493
|
if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
|
|
497
494
|
this.httpLivenessTimer = undefined;
|
|
498
495
|
if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
|
|
499
|
-
this.tokenRenewTimer = undefined;
|
|
496
|
+
this.tokenRenewTimer = undefined;
|
|
500
497
|
this.busyReconciler.disarm();
|
|
501
498
|
this.stopReasoningTail();
|
|
502
499
|
this.obligationCache.stop();
|
|
@@ -937,7 +934,6 @@ export class AgentRunner {
|
|
|
937
934
|
if (this.stopped) return;
|
|
938
935
|
this.process = await this.spawnProvider();
|
|
939
936
|
this.processStartedAt = Date.now();
|
|
940
|
-
this.quotaPoller.clear(); this.quotaPoller.start();
|
|
941
937
|
} finally {
|
|
942
938
|
this.restartInProgress = false;
|
|
943
939
|
}
|
|
@@ -2114,7 +2110,7 @@ export class AgentRunner {
|
|
|
2114
2110
|
const probeMetrics = this.latestProbeMetrics();
|
|
2115
2111
|
const probeContext = probeMetrics ? contextStateFromProbeMetrics(probeMetrics) : undefined;
|
|
2116
2112
|
const context = processContext ?? probeContext;
|
|
2117
|
-
const quota =
|
|
2113
|
+
const quota = probeMetrics ? quotaStateFromProbeMetrics(probeMetrics) : undefined;
|
|
2118
2114
|
const terminalSession = this.providerTerminalSession();
|
|
2119
2115
|
const terminalSocket = this.providerTerminalSocket();
|
|
2120
2116
|
const probeModel: ProbeModelInfo | undefined = probeMetrics?.model || probeMetrics?.effort
|