@cartanova/qgrid-cli 2.3.5 → 2.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/dist/application/qgrid/oauth.js +15 -12
- package/bundle/dist/application/qgrid/qgrid-run-lifecycle.js +2 -1
- package/bundle/dist/application/qgrid/qgrid.dispatcher.js +2 -1
- package/bundle/dist/application/qgrid/qgrid.frame.js +40 -20
- package/bundle/dist/application/qgrid/qgrid.types.js +8 -2
- package/bundle/dist/application/qgrid/token-subscriber.js +16 -8
- package/bundle/dist/application/qgrid/token-trigger-setup.js +3 -2
- package/bundle/dist/application/request-log/request-log.model.js +12 -7
- package/bundle/dist/application/sonamu.generated.js +11 -3
- package/bundle/dist/application/sonamu.generated.sso.js +7 -2
- package/bundle/dist/application/token/token.model.js +10 -2
- package/bundle/dist/application/token/token.types.js +5 -3
- package/bundle/dist/i18n/sd.generated.js +4 -1
- package/bundle/dist/migrations/20260630223950_alter_tokens_add1_alter1.js +16 -0
- package/bundle/dist/migrations/20260701105327_alter_request_logs_add1_alter1.js +16 -0
- package/bundle/dist/migrations/20260701105328_alter_request_log_steps_add1_alter1.js +16 -0
- package/bundle/dist/sonamu.config.js +13 -6
- package/bundle/dist/utils/providers/anthropic/anthropic-dispatcher.js +99 -12
- package/bundle/dist/utils/providers/anthropic/anthropic-quota.js +31 -0
- package/bundle/dist/utils/providers/anthropic/claude-session.js +8 -3
- package/bundle/dist/utils/providers/common/ttft.js +25 -0
- package/bundle/dist/utils/providers/openai/codex-worker.js +35 -7
- package/bundle/dist/utils/providers/openai/openai-dispatcher.js +237 -39
- package/bundle/dist/utils/providers/openai/openai-quota.js +30 -0
- package/bundle/src/application/qgrid/oauth.ts +17 -7
- package/bundle/src/application/qgrid/qgrid-run-lifecycle.test.ts +96 -0
- package/bundle/src/application/qgrid/qgrid-run-lifecycle.ts +1 -0
- package/bundle/src/application/qgrid/qgrid.dispatcher.test.ts +16 -0
- package/bundle/src/application/qgrid/qgrid.dispatcher.ts +1 -0
- package/bundle/src/application/qgrid/qgrid.frame.test.ts +172 -0
- package/bundle/src/application/qgrid/qgrid.frame.ts +45 -20
- package/bundle/src/application/qgrid/qgrid.types.ts +6 -0
- package/bundle/src/application/qgrid/token-subscriber.test.ts +89 -0
- package/bundle/src/application/qgrid/token-subscriber.ts +40 -11
- package/bundle/src/application/qgrid/token-trigger-setup.ts +2 -1
- package/bundle/src/application/qgrid/tool-emulation.test.ts +7 -3
- package/bundle/src/application/request-log/request-log.entity.json +3 -0
- package/bundle/src/application/request-log/request-log.model.test.ts +92 -0
- package/bundle/src/application/request-log/request-log.model.ts +20 -6
- package/bundle/src/application/request-log-step/request-log-step.entity.json +3 -0
- package/bundle/src/application/sonamu.generated.http +7 -2
- package/bundle/src/application/sonamu.generated.sso.ts +5 -0
- package/bundle/src/application/sonamu.generated.ts +11 -1
- package/bundle/src/application/token/token.entity.json +8 -2
- package/bundle/src/application/token/token.model.test.ts +69 -0
- package/bundle/src/application/token/token.model.ts +8 -1
- package/bundle/src/application/token/token.types.test.ts +34 -0
- package/bundle/src/application/token/token.types.ts +5 -0
- package/bundle/src/i18n/sd.generated.ts +3 -0
- package/bundle/src/migrations/20260630223950_alter_tokens_add1_alter1.ts +15 -0
- package/bundle/src/migrations/20260701105327_alter_request_logs_add1_alter1.ts +15 -0
- package/bundle/src/migrations/20260701105328_alter_request_log_steps_add1_alter1.ts +15 -0
- package/bundle/src/sonamu.config.ts +8 -1
- package/bundle/src/utils/providers/anthropic/anthropic-dispatcher.test.ts +183 -15
- package/bundle/src/utils/providers/anthropic/anthropic-dispatcher.ts +142 -12
- package/bundle/src/utils/providers/anthropic/anthropic-quota.test.ts +89 -0
- package/bundle/src/utils/providers/anthropic/anthropic-quota.ts +27 -0
- package/bundle/src/utils/providers/anthropic/claude-session.ts +7 -2
- package/bundle/src/utils/providers/common/provider-dispatcher.ts +1 -0
- package/bundle/src/utils/providers/common/ttft.test.ts +37 -0
- package/bundle/src/utils/providers/common/ttft.ts +26 -0
- package/bundle/src/utils/providers/openai/codex-worker.test.ts +148 -0
- package/bundle/src/utils/providers/openai/codex-worker.ts +50 -8
- package/bundle/src/utils/providers/openai/openai-dispatcher.test.ts +378 -6
- package/bundle/src/utils/providers/openai/openai-dispatcher.ts +332 -37
- package/bundle/src/utils/providers/openai/openai-quota.test.ts +85 -0
- package/bundle/src/utils/providers/openai/openai-quota.ts +46 -0
- package/bundle/web-dist/client/assets/index-5WdFEqZT.css +1 -0
- package/bundle/web-dist/client/assets/{index-DeThumA3.js → index-jnUY7nJ-.js} +3 -3
- package/bundle/web-dist/client/assets/logs-B0pWpDrg.js +1 -0
- package/bundle/web-dist/client/assets/routes-7XhVuG7o.js +5 -0
- package/bundle/web-dist/client/assets/{sd.generated-7jzAQL2_.js → sd.generated-DXhjjL2f.js} +1 -1
- package/bundle/web-dist/client/assets/{services.generated-Bx1Eh7ot.js → services.generated-B7u0U1wC.js} +1 -1
- package/bundle/web-dist/client/assets/{show-C6OofoAL.js → show-CcjZ9nr7.js} +3 -3
- package/bundle/web-dist/client/assets/tokens-DlwAm0v4.js +1 -0
- package/bundle/web-dist/client/index.html +3 -3
- package/bundle/web-dist/server/assets/{lazyRouteComponent-BEaI4mOq.js → lazyRouteComponent-B0LkbiYf.js} +1 -2
- package/bundle/web-dist/server/assets/{logs-CHM0fhfO.js → logs-dLx6gMXT.js} +17 -5
- package/bundle/web-dist/server/assets/{routes-C9xFMJ_r.js → routes-DD3Hb9xO.js} +201 -13
- package/bundle/web-dist/server/assets/{sd.generated-BeUHLWKH.js → sd.generated-DdfpD_iN.js} +131 -1
- package/bundle/web-dist/server/assets/{services.generated-C39hdSv0.js → services.generated-D3ZCvIkt.js} +5 -4
- package/bundle/web-dist/server/assets/{show-BZWqH09U.js → show-BQjeyv6A.js} +76 -25
- package/bundle/web-dist/server/assets/{tokens-IPwv3pNB.js → tokens-CSOsBpP7.js} +4 -3
- package/bundle/web-dist/server/entry-server.generated.js +6 -7
- package/dist/cli.js +67 -15
- package/package.json +1 -1
- package/bundle/web-dist/client/assets/index-B5DSo5uH.css +0 -1
- package/bundle/web-dist/client/assets/logs-CtmOVH58.js +0 -1
- package/bundle/web-dist/client/assets/routes-Cfoe9J0I.js +0 -5
- package/bundle/web-dist/client/assets/tokens-BAZq8sTo.js +0 -1
- package/bundle/web-dist/server/assets/shim-B9ZrFt3Z.js +0 -129
- /package/bundle/web-dist/server/assets/{cost-DzjfVtUr.js → cost-CSH_t17I.js} +0 -0
|
@@ -8,8 +8,10 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { getLogger } from "@logtape/logtape";
|
|
10
10
|
|
|
11
|
+
import { QuotaThresholdExceededError } from "../../../application/qgrid/qgrid.types";
|
|
11
12
|
import { TokenModel } from "../../../application/token/token.model";
|
|
12
13
|
import { type OpenAICredentials } from "../../../application/token/token.types";
|
|
14
|
+
import { type GetAccountRateLimitsResponse } from "../../../codex-protocol/v2/GetAccountRateLimitsResponse";
|
|
13
15
|
import {
|
|
14
16
|
type GenerateRequest,
|
|
15
17
|
type GenerateResult,
|
|
@@ -17,6 +19,11 @@ import {
|
|
|
17
19
|
type ProviderDispatcher,
|
|
18
20
|
} from "../common/provider-dispatcher";
|
|
19
21
|
import { CodexAppServerWorker, type StreamCallbacks, type TurnRequest } from "./codex-worker";
|
|
22
|
+
import {
|
|
23
|
+
readOpenAIQuotaUsage,
|
|
24
|
+
type OpenAIQuotaUsageResult,
|
|
25
|
+
type OpenAIRateLimitsWithMeta,
|
|
26
|
+
} from "./openai-quota";
|
|
20
27
|
import { handleChatgptAuthTokensRefresh } from "./openai-refresh";
|
|
21
28
|
|
|
22
29
|
const logger = getLogger(["qgrid", "openai-dispatcher"]);
|
|
@@ -46,17 +53,45 @@ function sleep(ms: number): Promise<void> {
|
|
|
46
53
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
47
54
|
}
|
|
48
55
|
|
|
56
|
+
function quotaThresholdExceededMessage(
|
|
57
|
+
provider: string,
|
|
58
|
+
tokens: Array<{ tokenName: string; threshold: number }>,
|
|
59
|
+
): string {
|
|
60
|
+
const details = tokens
|
|
61
|
+
.map((token) => `${token.tokenName} (threshold ${token.threshold}%)`)
|
|
62
|
+
.join(", ");
|
|
63
|
+
return details.length > 0
|
|
64
|
+
? `All ${provider} tokens exceeded quota threshold: ${details}`
|
|
65
|
+
: `All ${provider} tokens exceeded quota threshold`;
|
|
66
|
+
}
|
|
67
|
+
|
|
49
68
|
type QueueItem = {
|
|
50
69
|
resolve: (worker: CodexAppServerWorker) => void;
|
|
51
70
|
reject: (error: Error) => void;
|
|
52
71
|
timer: ReturnType<typeof setTimeout>;
|
|
53
72
|
abortCleanup?: () => void;
|
|
73
|
+
excludedTokenIds: Set<number>;
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
type TokenMetadata = {
|
|
77
|
+
name: string;
|
|
78
|
+
quotaThreshold?: number | null;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
type TokenEligibility = {
|
|
82
|
+
readyActiveTokenIds: Set<number>;
|
|
83
|
+
eligibleTokenIds: Set<number>;
|
|
84
|
+
overThresholdTokens: Array<{ tokenName: string; threshold: number }>;
|
|
85
|
+
thresholdedTokenCount: number;
|
|
54
86
|
};
|
|
55
87
|
|
|
56
88
|
export class OpenAIDispatcher implements ProviderDispatcher {
|
|
57
89
|
workerPool = new Map<number, CodexAppServerWorker[]>();
|
|
90
|
+
tokenMetadata = new Map<number, TokenMetadata>();
|
|
58
91
|
rrCursor = 0;
|
|
59
92
|
queue: QueueItem[] = [];
|
|
93
|
+
private draining = false;
|
|
94
|
+
private drainAgain = false;
|
|
60
95
|
|
|
61
96
|
async start(): Promise<void> {
|
|
62
97
|
const { rows } = await TokenModel.findMany("A");
|
|
@@ -67,7 +102,12 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
67
102
|
|
|
68
103
|
await Promise.allSettled(
|
|
69
104
|
openaiTokens.map(async (t) => {
|
|
70
|
-
await this.spawnWorkers(
|
|
105
|
+
await this.spawnWorkers(
|
|
106
|
+
t.id,
|
|
107
|
+
t.name,
|
|
108
|
+
t.credentials as OpenAICredentials,
|
|
109
|
+
t.quota_threshold,
|
|
110
|
+
);
|
|
71
111
|
if (!t.active) this.onTokenDeactivated(t.id);
|
|
72
112
|
}),
|
|
73
113
|
);
|
|
@@ -78,19 +118,29 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
78
118
|
const kills = [...this.workerPool.values()].flat().map((w) => w.kill());
|
|
79
119
|
await Promise.allSettled(kills);
|
|
80
120
|
this.workerPool.clear();
|
|
121
|
+
this.tokenMetadata.clear();
|
|
122
|
+
this.rateLimitsCache.clear();
|
|
81
123
|
}
|
|
82
124
|
|
|
83
125
|
// ── Token events (from TokenSubscriber) ─────────────────────────
|
|
84
126
|
|
|
85
|
-
async onTokenAdded(
|
|
127
|
+
async onTokenAdded(
|
|
128
|
+
id: number,
|
|
129
|
+
name: string,
|
|
130
|
+
credentials: OpenAICredentials,
|
|
131
|
+
quotaThreshold?: number | null,
|
|
132
|
+
): Promise<void> {
|
|
133
|
+
this.setTokenMetadata(id, name, quotaThreshold);
|
|
86
134
|
if (this.workerPool.has(id)) return;
|
|
87
|
-
await this.spawnWorkers(id, name, credentials);
|
|
135
|
+
await this.spawnWorkers(id, name, credentials, quotaThreshold);
|
|
88
136
|
}
|
|
89
137
|
|
|
90
138
|
async onTokenRemoved(id: number): Promise<void> {
|
|
91
139
|
const workers = this.workerPool.get(id) ?? [];
|
|
92
140
|
const tokenName = workers[0]?.tokenName;
|
|
93
141
|
this.workerPool.delete(id);
|
|
142
|
+
this.tokenMetadata.delete(id);
|
|
143
|
+
this.rateLimitsCache.delete(id);
|
|
94
144
|
await Promise.allSettled(workers.map((w) => w.kill()));
|
|
95
145
|
if (this.getAllReadyActiveWorkers().length === 0) {
|
|
96
146
|
this.rejectAllQueued("NO_OPENAI_WORKERS");
|
|
@@ -101,23 +151,33 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
101
151
|
}
|
|
102
152
|
}
|
|
103
153
|
|
|
104
|
-
async onTokenUpdated(
|
|
154
|
+
async onTokenUpdated(
|
|
155
|
+
id: number,
|
|
156
|
+
name: string,
|
|
157
|
+
credentials: OpenAICredentials,
|
|
158
|
+
quotaThreshold?: number | null,
|
|
159
|
+
): Promise<void> {
|
|
105
160
|
const existing = this.workerPool.get(id) ?? [];
|
|
161
|
+
const renamed = existing.some((w) => w.tokenName !== name);
|
|
162
|
+
this.setTokenMetadata(id, name, quotaThreshold);
|
|
163
|
+
if (renamed) this.rateLimitsCache.delete(id);
|
|
164
|
+
|
|
106
165
|
if (existing.length === 0) {
|
|
107
|
-
await this.spawnWorkers(id, name, credentials);
|
|
166
|
+
await this.spawnWorkers(id, name, credentials, quotaThreshold);
|
|
108
167
|
return;
|
|
109
168
|
}
|
|
110
169
|
|
|
111
170
|
if (existing.every((w) => w.canReuseForToken(name, credentials))) {
|
|
112
171
|
existing.forEach((w) => w.updateTokenState(name, credentials));
|
|
113
172
|
logger.info(`workers updated in-place: ${name}`);
|
|
114
|
-
this.
|
|
173
|
+
this.requestDrainQueue();
|
|
115
174
|
return;
|
|
116
175
|
}
|
|
117
176
|
|
|
118
177
|
this.workerPool.delete(id);
|
|
178
|
+
this.rateLimitsCache.delete(id);
|
|
119
179
|
await Promise.allSettled(existing.map((w) => w.kill()));
|
|
120
|
-
await this.spawnWorkers(id, name, credentials);
|
|
180
|
+
await this.spawnWorkers(id, name, credentials, quotaThreshold);
|
|
121
181
|
}
|
|
122
182
|
|
|
123
183
|
onTokenDeactivated(id: number): void {
|
|
@@ -139,16 +199,40 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
139
199
|
workers.forEach((w) => {
|
|
140
200
|
w.active = true;
|
|
141
201
|
});
|
|
142
|
-
this.
|
|
202
|
+
this.requestDrainQueue();
|
|
143
203
|
const label = tokenName ?? `token ${id}`;
|
|
144
204
|
logger.info(`workers activated: ${label}`);
|
|
145
205
|
}
|
|
146
206
|
|
|
207
|
+
async replaceTokens(
|
|
208
|
+
rows: Array<{
|
|
209
|
+
id: number;
|
|
210
|
+
name: string;
|
|
211
|
+
credentials: OpenAICredentials;
|
|
212
|
+
quotaThreshold?: number | null;
|
|
213
|
+
}>,
|
|
214
|
+
): Promise<void> {
|
|
215
|
+
const next = new Set(rows.map((r) => r.id));
|
|
216
|
+
for (const id of Array.from(this.workerPool.keys())) {
|
|
217
|
+
if (!next.has(id)) await this.onTokenRemoved(id);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
for (const row of rows) {
|
|
221
|
+
if (this.workerPool.has(row.id)) {
|
|
222
|
+
await this.onTokenUpdated(row.id, row.name, row.credentials, row.quotaThreshold);
|
|
223
|
+
} else {
|
|
224
|
+
await this.onTokenAdded(row.id, row.name, row.credentials, row.quotaThreshold);
|
|
225
|
+
}
|
|
226
|
+
this.onTokenActivated(row.id);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
147
230
|
// ── Generate ────────────────────────────────────────────────────
|
|
148
231
|
|
|
149
232
|
async generate(req: GenerateRequest): Promise<GenerateResult> {
|
|
233
|
+
const excludedTokenIds = new Set<number>();
|
|
150
234
|
// thread 재사용 경로: 좌표 worker 를 점유해 기존 thread 에 turn 만 실행.
|
|
151
|
-
const reuseWorker = await this.acquireReuseWorker(req);
|
|
235
|
+
const reuseWorker = await this.acquireReuseWorker(req, excludedTokenIds);
|
|
152
236
|
if (reuseWorker) {
|
|
153
237
|
logger.info(
|
|
154
238
|
`↻ ${reuseWorker.tokenName}[${reuseWorker.tokenId}] (reuse thread, ${req.model})`,
|
|
@@ -158,7 +242,7 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
158
242
|
);
|
|
159
243
|
}
|
|
160
244
|
|
|
161
|
-
const worker = this.acquireIdleWorker();
|
|
245
|
+
const worker = await this.acquireIdleWorker(excludedTokenIds);
|
|
162
246
|
if (worker) {
|
|
163
247
|
logger.info(`→ ${worker.tokenName}[${worker.tokenId}] (model: ${req.model})`);
|
|
164
248
|
return this.executeAndRelease(worker, (w) => this.executeTurn(w, req));
|
|
@@ -166,11 +250,12 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
166
250
|
return this.enqueue((w) => {
|
|
167
251
|
logger.info(`→ ${w.tokenName}[${w.tokenId}] (model: ${req.model}, queued)`);
|
|
168
252
|
return this.executeTurn(w, req);
|
|
169
|
-
});
|
|
253
|
+
}, excludedTokenIds);
|
|
170
254
|
}
|
|
171
255
|
|
|
172
256
|
async generateStream(req: GenerateRequest, cb: GenerateStreamCallbacks): Promise<void> {
|
|
173
|
-
const
|
|
257
|
+
const excludedTokenIds = new Set<number>();
|
|
258
|
+
const reuseWorker = await this.acquireReuseWorker(req, excludedTokenIds);
|
|
174
259
|
if (reuseWorker) {
|
|
175
260
|
logger.info(`↻ ${reuseWorker.tokenName}[${reuseWorker.tokenId}] (reuse thread, [stream])`);
|
|
176
261
|
return this.executeAndRelease(reuseWorker, (w) =>
|
|
@@ -178,7 +263,7 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
178
263
|
);
|
|
179
264
|
}
|
|
180
265
|
|
|
181
|
-
const worker = this.acquireIdleWorker();
|
|
266
|
+
const worker = await this.acquireIdleWorker(excludedTokenIds);
|
|
182
267
|
if (worker) {
|
|
183
268
|
logger.info(`→ ${worker.tokenName}[${worker.tokenId}] (model: ${req.model}, [stream])`);
|
|
184
269
|
return this.executeAndRelease(worker, (w) => this.executeStreamTurn(w, req, cb));
|
|
@@ -186,13 +271,16 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
186
271
|
return this.enqueue((w) => {
|
|
187
272
|
logger.info(`→ ${w.tokenName}[${w.tokenId}] (model: ${req.model}, [stream], queued)`);
|
|
188
273
|
return this.executeStreamTurn(w, req, cb);
|
|
189
|
-
});
|
|
274
|
+
}, excludedTokenIds);
|
|
190
275
|
}
|
|
191
276
|
|
|
192
277
|
// reuse 좌표가 유효하면 그 worker 를 busy 점유해 반환. 없거나 폴백 대상이면 null.
|
|
193
278
|
// 폴백 사유: 기능 off / 좌표 없음 / worker 부재·죽음 / epoch 불일치(restart) / thread 소멸 /
|
|
194
279
|
// busy 대기 타임아웃. null 반환 시 호출부는 새 thread 경로로 진행.
|
|
195
|
-
async acquireReuseWorker(
|
|
280
|
+
async acquireReuseWorker(
|
|
281
|
+
req: GenerateRequest,
|
|
282
|
+
excludedTokenIds = new Set<number>(),
|
|
283
|
+
): Promise<CodexAppServerWorker | null> {
|
|
196
284
|
if (!THREAD_REUSE_ENABLED || !req.reuse) return null;
|
|
197
285
|
const { workerId, threadId, epoch } = req.reuse;
|
|
198
286
|
|
|
@@ -200,6 +288,9 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
200
288
|
if (!worker || !worker.isReady || !worker.active) return null;
|
|
201
289
|
if (worker.epoch !== epoch) return null;
|
|
202
290
|
if (!worker.hasThread(threadId)) return null;
|
|
291
|
+
if (!(await this.isTokenQuotaEligible(worker.tokenId, worker.tokenName, excludedTokenIds))) {
|
|
292
|
+
return null;
|
|
293
|
+
}
|
|
203
294
|
|
|
204
295
|
const deadline = Date.now() + REUSE_WORKER_WAIT_MS;
|
|
205
296
|
for (;;) {
|
|
@@ -228,8 +319,16 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
228
319
|
// ── Worker selection ────────────────────────────────────────────
|
|
229
320
|
// 기본 round-robin
|
|
230
321
|
// TODO: 추후 다른 알고리즘으로 변경 가능하게 지원
|
|
231
|
-
acquireIdleWorker(
|
|
232
|
-
|
|
322
|
+
async acquireIdleWorker(
|
|
323
|
+
excludedTokenIds = new Set<number>(),
|
|
324
|
+
): Promise<CodexAppServerWorker | null> {
|
|
325
|
+
const eligibility = await this.resolveTokenEligibility(excludedTokenIds);
|
|
326
|
+
if (eligibility.readyActiveTokenIds.size === 0) return null;
|
|
327
|
+
if (eligibility.eligibleTokenIds.size === 0) this.throwQuotaThresholdExceeded(eligibility);
|
|
328
|
+
|
|
329
|
+
const allWorkers = this.getAllReadyActiveWorkers().filter((w) =>
|
|
330
|
+
eligibility.eligibleTokenIds.has(w.tokenId),
|
|
331
|
+
);
|
|
233
332
|
if (allWorkers.length === 0) return null;
|
|
234
333
|
for (let i = 0; i < allWorkers.length; i++) {
|
|
235
334
|
const w = allWorkers[(this.rrCursor + i) % allWorkers.length]!;
|
|
@@ -247,12 +346,15 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
247
346
|
|
|
248
347
|
// ── Queue ───────────────────────────────────────────────────────
|
|
249
348
|
|
|
250
|
-
enqueue<T>(
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
349
|
+
async enqueue<T>(
|
|
350
|
+
execute: (worker: CodexAppServerWorker) => Promise<T>,
|
|
351
|
+
excludedTokenIds = new Set<number>(),
|
|
352
|
+
): Promise<T> {
|
|
353
|
+
const eligibility = await this.resolveTokenEligibility(excludedTokenIds);
|
|
354
|
+
if (eligibility.readyActiveTokenIds.size === 0) throw new Error("NO_OPENAI_WORKERS");
|
|
355
|
+
if (eligibility.eligibleTokenIds.size === 0) this.throwQuotaThresholdExceeded(eligibility);
|
|
254
356
|
if (this.queue.length >= MAX_QUEUE_SIZE) {
|
|
255
|
-
|
|
357
|
+
throw new Error("SERVER_BUSY");
|
|
256
358
|
}
|
|
257
359
|
|
|
258
360
|
return new Promise<T>((resolve, reject) => {
|
|
@@ -273,6 +375,7 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
273
375
|
this.removeFromQueue(item);
|
|
274
376
|
reject(new Error("SERVER_BUSY"));
|
|
275
377
|
}, QUEUE_TIMEOUT_MS),
|
|
378
|
+
excludedTokenIds: new Set(excludedTokenIds),
|
|
276
379
|
};
|
|
277
380
|
|
|
278
381
|
this.queue.push(item);
|
|
@@ -296,12 +399,34 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
296
399
|
this.queue = [];
|
|
297
400
|
}
|
|
298
401
|
|
|
299
|
-
drainQueue(releasedWorker?: CodexAppServerWorker): void {
|
|
402
|
+
async drainQueue(releasedWorker?: CodexAppServerWorker): Promise<void> {
|
|
403
|
+
if (this.draining) {
|
|
404
|
+
this.drainAgain = true;
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
this.draining = true;
|
|
409
|
+
let worker = releasedWorker;
|
|
410
|
+
try {
|
|
411
|
+
do {
|
|
412
|
+
this.drainAgain = false;
|
|
413
|
+
await this.drainQueueOnce(worker);
|
|
414
|
+
worker = undefined;
|
|
415
|
+
} while (this.drainAgain);
|
|
416
|
+
} catch (e) {
|
|
417
|
+
logger.warn(`drainQueue failed: ${(e as Error).message}`);
|
|
418
|
+
} finally {
|
|
419
|
+
this.draining = false;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
private async drainQueueOnce(releasedWorker?: CodexAppServerWorker): Promise<void> {
|
|
300
424
|
if (
|
|
301
425
|
releasedWorker &&
|
|
302
426
|
this.queue.length > 0 &&
|
|
303
427
|
releasedWorker.isReady &&
|
|
304
428
|
releasedWorker.active &&
|
|
429
|
+
(await this.isWorkerEligibleForQueueItem(releasedWorker, this.queue[0]!)) &&
|
|
305
430
|
releasedWorker.tryAcquireTurn()
|
|
306
431
|
) {
|
|
307
432
|
const next = this.queue.shift()!;
|
|
@@ -310,9 +435,20 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
310
435
|
}
|
|
311
436
|
|
|
312
437
|
while (this.queue.length > 0) {
|
|
313
|
-
const
|
|
438
|
+
const next = this.queue[0]!;
|
|
439
|
+
let worker: CodexAppServerWorker | null;
|
|
440
|
+
try {
|
|
441
|
+
worker = await this.acquireIdleWorker(next.excludedTokenIds);
|
|
442
|
+
} catch (e) {
|
|
443
|
+
if (e instanceof QuotaThresholdExceededError) {
|
|
444
|
+
this.queue.shift();
|
|
445
|
+
next.reject(e);
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
throw e;
|
|
449
|
+
}
|
|
314
450
|
if (!worker) break;
|
|
315
|
-
|
|
451
|
+
this.queue.shift();
|
|
316
452
|
next.resolve(worker);
|
|
317
453
|
}
|
|
318
454
|
}
|
|
@@ -327,7 +463,7 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
327
463
|
return await execute(worker);
|
|
328
464
|
} finally {
|
|
329
465
|
worker.releaseTurn();
|
|
330
|
-
this.
|
|
466
|
+
this.requestDrainQueue(worker);
|
|
331
467
|
}
|
|
332
468
|
}
|
|
333
469
|
|
|
@@ -360,6 +496,7 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
360
496
|
tokenName: worker.tokenName,
|
|
361
497
|
usage: result.usage,
|
|
362
498
|
durationMs: result.durationMs,
|
|
499
|
+
ttftMs: result.ttftMs,
|
|
363
500
|
model: result.model,
|
|
364
501
|
threadCoord: {
|
|
365
502
|
workerId: makeWorkerId(worker.tokenId, worker.workerIndex),
|
|
@@ -406,7 +543,9 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
406
543
|
tokenId: number,
|
|
407
544
|
tokenName: string,
|
|
408
545
|
credentials: OpenAICredentials,
|
|
546
|
+
quotaThreshold?: number | null,
|
|
409
547
|
): Promise<void> {
|
|
548
|
+
this.setTokenMetadata(tokenId, tokenName, quotaThreshold);
|
|
410
549
|
const workers: CodexAppServerWorker[] = [];
|
|
411
550
|
for (let i = 0; i < WORKERS_PER_TOKEN; i++) {
|
|
412
551
|
if (i > 0) await sleep(SPAWN_INTERVAL_MS);
|
|
@@ -431,7 +570,7 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
431
570
|
workerIndex,
|
|
432
571
|
});
|
|
433
572
|
|
|
434
|
-
worker.onReady = () => this.
|
|
573
|
+
worker.onReady = () => this.requestDrainQueue();
|
|
435
574
|
|
|
436
575
|
worker.setServerRequestHandler(async (method) => {
|
|
437
576
|
if (method === "account/chatgptAuthTokens/refresh") {
|
|
@@ -453,22 +592,178 @@ export class OpenAIDispatcher implements ProviderDispatcher {
|
|
|
453
592
|
|
|
454
593
|
// ── Rate limits ─────────────────────────────────────────────────
|
|
455
594
|
|
|
456
|
-
rateLimitsCache = new Map<
|
|
595
|
+
rateLimitsCache = new Map<number, { data: GetAccountRateLimitsResponse; cachedAt: number }>();
|
|
457
596
|
static readonly RATE_LIMITS_CACHE_TTL = 60_000;
|
|
458
597
|
|
|
459
|
-
async
|
|
460
|
-
const
|
|
461
|
-
const cached = this.rateLimitsCache.get(cacheKey);
|
|
598
|
+
async getRateLimitsByTokenId(tokenId: number): Promise<OpenAIRateLimitsWithMeta> {
|
|
599
|
+
const cached = this.rateLimitsCache.get(tokenId);
|
|
462
600
|
if (cached && Date.now() - cached.cachedAt < OpenAIDispatcher.RATE_LIMITS_CACHE_TTL) {
|
|
463
|
-
return cached
|
|
601
|
+
return cached;
|
|
464
602
|
}
|
|
465
603
|
|
|
466
|
-
const
|
|
467
|
-
const worker = tokenName ? allWorkers.find((w) => w.tokenName === tokenName) : allWorkers[0];
|
|
604
|
+
const worker = (this.workerPool.get(tokenId) ?? []).find((w) => w.isReady);
|
|
468
605
|
if (!worker) throw new Error("no ready openai workers");
|
|
469
|
-
const data = await worker.getRateLimits();
|
|
470
|
-
|
|
471
|
-
|
|
606
|
+
const data = (await worker.getRateLimits()) as GetAccountRateLimitsResponse;
|
|
607
|
+
const entry = { data, cachedAt: Date.now() };
|
|
608
|
+
this.rateLimitsCache.set(tokenId, entry);
|
|
609
|
+
return entry;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
private requestDrainQueue(releasedWorker?: CodexAppServerWorker): void {
|
|
613
|
+
void this.drainQueue(releasedWorker).catch((e) =>
|
|
614
|
+
logger.warn(`drainQueue failed: ${(e as Error).message}`),
|
|
615
|
+
);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
private setTokenMetadata(id: number, name: string, quotaThreshold?: number | null): void {
|
|
619
|
+
this.tokenMetadata.set(id, { name, quotaThreshold });
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
private getQuotaThreshold(tokenId: number): number | null | undefined {
|
|
623
|
+
return this.tokenMetadata.get(tokenId)?.quotaThreshold;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
private hasQuotaThreshold(tokenId: number): boolean {
|
|
627
|
+
const threshold = this.getQuotaThreshold(tokenId);
|
|
628
|
+
return threshold !== undefined && threshold !== null;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
private async resolveTokenEligibility(excludedTokenIds: Set<number>): Promise<TokenEligibility> {
|
|
632
|
+
const workers = this.getAllReadyActiveWorkers();
|
|
633
|
+
const byToken = new Map<number, string>();
|
|
634
|
+
for (const worker of workers) {
|
|
635
|
+
if (!byToken.has(worker.tokenId)) byToken.set(worker.tokenId, worker.tokenName);
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
const readyActiveTokenIds = new Set(byToken.keys());
|
|
639
|
+
const eligibleTokenIds = new Set<number>();
|
|
640
|
+
const overThresholdTokens: Array<{ tokenName: string; threshold: number }> = [];
|
|
641
|
+
let thresholdedTokenCount = 0;
|
|
642
|
+
|
|
643
|
+
const entries = Array.from(byToken.entries());
|
|
644
|
+
const checks = await Promise.allSettled(
|
|
645
|
+
entries.map(async ([tokenId, tokenName]) => {
|
|
646
|
+
if (excludedTokenIds.has(tokenId)) {
|
|
647
|
+
thresholdedTokenCount++;
|
|
648
|
+
return { tokenId, tokenName, eligible: false };
|
|
649
|
+
}
|
|
650
|
+
if (!this.hasQuotaThreshold(tokenId)) {
|
|
651
|
+
return { tokenId, tokenName, eligible: true };
|
|
652
|
+
}
|
|
653
|
+
thresholdedTokenCount++;
|
|
654
|
+
return {
|
|
655
|
+
tokenId,
|
|
656
|
+
tokenName,
|
|
657
|
+
eligible: await this.isTokenQuotaEligible(tokenId, tokenName),
|
|
658
|
+
};
|
|
659
|
+
}),
|
|
660
|
+
);
|
|
661
|
+
|
|
662
|
+
checks.forEach((check, index) => {
|
|
663
|
+
if (check.status === "fulfilled") {
|
|
664
|
+
if (check.value.eligible) {
|
|
665
|
+
eligibleTokenIds.add(check.value.tokenId);
|
|
666
|
+
} else {
|
|
667
|
+
const threshold = this.getQuotaThreshold(check.value.tokenId);
|
|
668
|
+
if (threshold !== undefined && threshold !== null) {
|
|
669
|
+
overThresholdTokens.push({ tokenName: check.value.tokenName, threshold });
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
const entry = entries[index];
|
|
676
|
+
if (entry) {
|
|
677
|
+
const [tokenId, tokenName] = entry;
|
|
678
|
+
this.logQuotaLookupFailOpen(tokenId, tokenName, String(check.reason));
|
|
679
|
+
eligibleTokenIds.add(tokenId);
|
|
680
|
+
}
|
|
681
|
+
});
|
|
682
|
+
|
|
683
|
+
return { readyActiveTokenIds, eligibleTokenIds, overThresholdTokens, thresholdedTokenCount };
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
private async isWorkerEligibleForQueueItem(
|
|
687
|
+
worker: CodexAppServerWorker,
|
|
688
|
+
item: QueueItem,
|
|
689
|
+
): Promise<boolean> {
|
|
690
|
+
if (item.excludedTokenIds.has(worker.tokenId)) return false;
|
|
691
|
+
return this.isTokenQuotaEligible(worker.tokenId, worker.tokenName);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
private async isTokenQuotaEligible(
|
|
695
|
+
tokenId: number,
|
|
696
|
+
tokenName: string,
|
|
697
|
+
excludedTokenIds?: Set<number>,
|
|
698
|
+
): Promise<boolean> {
|
|
699
|
+
const threshold = this.getQuotaThreshold(tokenId);
|
|
700
|
+
if (threshold === undefined || threshold === null) return true;
|
|
701
|
+
|
|
702
|
+
let result: OpenAIQuotaUsageResult;
|
|
703
|
+
try {
|
|
704
|
+
result = await readOpenAIQuotaUsage(() => this.getRateLimitsByTokenId(tokenId));
|
|
705
|
+
} catch (e) {
|
|
706
|
+
this.logQuotaLookupFailOpen(tokenId, tokenName, (e as Error).message);
|
|
707
|
+
return true;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
if (result.kind === "lookup_failed") {
|
|
711
|
+
this.logQuotaLookupFailOpen(tokenId, tokenName, result.reason);
|
|
712
|
+
return true;
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
if (result.utilizationPct >= threshold) {
|
|
716
|
+
this.logQuotaOverThreshold(tokenId, tokenName, threshold, result);
|
|
717
|
+
excludedTokenIds?.add(tokenId);
|
|
718
|
+
return false;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
return true;
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
private logQuotaOverThreshold(
|
|
725
|
+
tokenId: number,
|
|
726
|
+
tokenName: string,
|
|
727
|
+
threshold: number,
|
|
728
|
+
result: Extract<OpenAIQuotaUsageResult, { kind: "ok" }>,
|
|
729
|
+
): void {
|
|
730
|
+
logger.info("quota_threshold gate: over_threshold", {
|
|
731
|
+
tokenId,
|
|
732
|
+
tokenName,
|
|
733
|
+
provider: "openai",
|
|
734
|
+
threshold,
|
|
735
|
+
cachedUtilization: result.utilizationPct,
|
|
736
|
+
cacheAge: result.cacheAgeMs,
|
|
737
|
+
windowDurationMins: result.windowDurationMins,
|
|
738
|
+
resetsAt: result.resetsAt,
|
|
739
|
+
limitId: result.limitId,
|
|
740
|
+
reason: "over_threshold",
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
private logQuotaLookupFailOpen(tokenId: number, tokenName: string, reason: string): void {
|
|
745
|
+
const threshold = this.getQuotaThreshold(tokenId);
|
|
746
|
+
logger.warn("quota_threshold gate: lookup_fail_open", {
|
|
747
|
+
tokenId,
|
|
748
|
+
tokenName,
|
|
749
|
+
provider: "openai",
|
|
750
|
+
threshold,
|
|
751
|
+
reason: "lookup_fail_open",
|
|
752
|
+
lookupReason: reason,
|
|
753
|
+
});
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
private throwQuotaThresholdExceeded(eligibility: TokenEligibility): never {
|
|
757
|
+
logger.warn("quota_threshold gate: all_exceeded", {
|
|
758
|
+
provider: "openai",
|
|
759
|
+
tokenCount: eligibility.readyActiveTokenIds.size,
|
|
760
|
+
thresholdedTokenCount: eligibility.thresholdedTokenCount,
|
|
761
|
+
overThresholdTokens: eligibility.overThresholdTokens,
|
|
762
|
+
reason: "all_exceeded",
|
|
763
|
+
});
|
|
764
|
+
throw new QuotaThresholdExceededError(
|
|
765
|
+
quotaThresholdExceededMessage("openai", eligibility.overThresholdTokens),
|
|
766
|
+
);
|
|
472
767
|
}
|
|
473
768
|
|
|
474
769
|
// ── Browser login flow ───────────────────────────────────────────
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
|
|
3
|
+
import { type RateLimitSnapshot } from "../../../codex-protocol/v2/RateLimitSnapshot";
|
|
4
|
+
import { readOpenAIQuotaUsage, type OpenAIRateLimitsWithMeta } from "./openai-quota";
|
|
5
|
+
|
|
6
|
+
function snapshot(overrides: Partial<RateLimitSnapshot> = {}): RateLimitSnapshot {
|
|
7
|
+
return {
|
|
8
|
+
limitId: "codex-primary",
|
|
9
|
+
limitName: "Codex Primary",
|
|
10
|
+
primary: { usedPercent: 17, windowDurationMins: 300, resetsAt: 1_782_912_345 },
|
|
11
|
+
secondary: null,
|
|
12
|
+
credits: null,
|
|
13
|
+
planType: null,
|
|
14
|
+
rateLimitReachedType: null,
|
|
15
|
+
...overrides,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function payload(rateLimits: RateLimitSnapshot): OpenAIRateLimitsWithMeta {
|
|
20
|
+
return { data: { rateLimits, rateLimitsByLimitId: null }, cachedAt: Date.now() - 1_250 };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
describe("readOpenAIQuotaUsage", () => {
|
|
24
|
+
beforeEach(() => {
|
|
25
|
+
vi.useFakeTimers();
|
|
26
|
+
vi.setSystemTime(new Date("2026-06-30T12:00:05.000Z"));
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
afterEach(() => {
|
|
30
|
+
vi.useRealTimers();
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("uses primary.usedPercent as 0..100 percent", async () => {
|
|
34
|
+
await expect(readOpenAIQuotaUsage(async () => payload(snapshot()))).resolves.toEqual({
|
|
35
|
+
kind: "ok",
|
|
36
|
+
utilizationPct: 17,
|
|
37
|
+
cacheAgeMs: 1_250,
|
|
38
|
+
windowDurationMins: 300,
|
|
39
|
+
resetsAt: 1_782_912_345,
|
|
40
|
+
limitId: "codex-primary",
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it("keeps low percent utilization unchanged", async () => {
|
|
45
|
+
await expect(
|
|
46
|
+
readOpenAIQuotaUsage(async () =>
|
|
47
|
+
payload(snapshot({ primary: { usedPercent: 1, windowDurationMins: 300, resetsAt: null } })),
|
|
48
|
+
),
|
|
49
|
+
).resolves.toMatchObject({
|
|
50
|
+
kind: "ok",
|
|
51
|
+
utilizationPct: 1,
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("treats utilization 0 as a successful lookup", async () => {
|
|
56
|
+
await expect(
|
|
57
|
+
readOpenAIQuotaUsage(async () =>
|
|
58
|
+
payload(
|
|
59
|
+
snapshot({ primary: { usedPercent: 0, windowDurationMins: null, resetsAt: null } }),
|
|
60
|
+
),
|
|
61
|
+
),
|
|
62
|
+
).resolves.toMatchObject({
|
|
63
|
+
kind: "ok",
|
|
64
|
+
utilizationPct: 0,
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it("converts missing primary data and rejected RPCs into lookup_failed", async () => {
|
|
69
|
+
await expect(
|
|
70
|
+
readOpenAIQuotaUsage(async () => payload(snapshot({ primary: null }))),
|
|
71
|
+
).resolves.toMatchObject({
|
|
72
|
+
kind: "lookup_failed",
|
|
73
|
+
reason: "primary usedPercent unavailable",
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
await expect(
|
|
77
|
+
readOpenAIQuotaUsage(async () => {
|
|
78
|
+
throw new Error("rpc timeout");
|
|
79
|
+
}),
|
|
80
|
+
).resolves.toMatchObject({
|
|
81
|
+
kind: "lookup_failed",
|
|
82
|
+
reason: "rpc timeout",
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
});
|