codex-copilot-dx 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -18
- package/bin/codex-copilot-dx.mjs +9 -1
- package/package.json +1 -1
- package/scripts/benchmark.mjs +61 -0
- package/src/adapter.mjs +21 -2
- package/src/anthropic-handler.mjs +8 -4
- package/src/cli-options.mjs +5 -11
- package/src/cli-status.mjs +5 -1
- package/src/copilot-client.mjs +5 -0
- package/src/http-transport.mjs +211 -27
- package/src/legacy-command-warning.mjs +37 -0
- package/src/observability.mjs +2 -0
- package/src/pm-studio-relay.mjs +8 -2
- package/src/response-history.mjs +15 -2
- package/src/responses-bridge.mjs +37 -16
- package/src/responses-byte-budget.mjs +63 -5
- package/src/responses-handler.mjs +154 -28
- package/src/responses-image-pressure.mjs +180 -0
- package/src/responses-proxy.mjs +8 -5
- package/src/responses-request.mjs +17 -1
- package/src/runtime-config.mjs +2 -0
package/src/http-transport.mjs
CHANGED
|
@@ -47,6 +47,83 @@ function admissionAbortError(signal) {
|
|
|
47
47
|
return error;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
function createSupplementalAdmission({ maxWeight, maxQueued, waitTimeoutMs, label }) {
|
|
51
|
+
const queue = [];
|
|
52
|
+
let activeWeight = 0;
|
|
53
|
+
let activeRequests = 0;
|
|
54
|
+
|
|
55
|
+
const activate = (entry) => {
|
|
56
|
+
entry.cleanup();
|
|
57
|
+
activeWeight += entry.weight;
|
|
58
|
+
activeRequests += 1;
|
|
59
|
+
let released = false;
|
|
60
|
+
entry.resolve(() => {
|
|
61
|
+
if (released) return;
|
|
62
|
+
released = true;
|
|
63
|
+
activeWeight = Math.max(0, activeWeight - entry.weight);
|
|
64
|
+
activeRequests = Math.max(0, activeRequests - 1);
|
|
65
|
+
drain();
|
|
66
|
+
});
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const drain = () => {
|
|
70
|
+
for (let index = 0; index < queue.length;) {
|
|
71
|
+
const entry = queue[index];
|
|
72
|
+
if (activeWeight + entry.weight > maxWeight) {
|
|
73
|
+
index += 1;
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
queue.splice(index, 1);
|
|
77
|
+
activate(entry);
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const acquire = (requestedWeight, { signal } = {}) => {
|
|
82
|
+
if (signal?.aborted) return Promise.reject(admissionAbortError(signal));
|
|
83
|
+
const parsedWeight = Number.isFinite(requestedWeight) && requestedWeight > 0
|
|
84
|
+
? Math.ceil(requestedWeight)
|
|
85
|
+
: maxWeight;
|
|
86
|
+
const weight = Math.max(1, Math.min(parsedWeight, maxWeight));
|
|
87
|
+
if (queue.length >= maxQueued && activeWeight + weight > maxWeight) {
|
|
88
|
+
return Promise.reject(httpError(`${label} queue is full (${maxQueued} waiting)`, 503));
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return new Promise((resolve, reject) => {
|
|
92
|
+
let timer;
|
|
93
|
+
const entry = {
|
|
94
|
+
weight,
|
|
95
|
+
resolve,
|
|
96
|
+
cleanup: () => {
|
|
97
|
+
if (timer) clearTimeout(timer);
|
|
98
|
+
signal?.removeEventListener("abort", onAbort);
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
const remove = () => {
|
|
102
|
+
const index = queue.indexOf(entry);
|
|
103
|
+
if (index >= 0) queue.splice(index, 1);
|
|
104
|
+
};
|
|
105
|
+
const fail = (error) => {
|
|
106
|
+
entry.cleanup();
|
|
107
|
+
remove();
|
|
108
|
+
reject(error);
|
|
109
|
+
drain();
|
|
110
|
+
};
|
|
111
|
+
const onAbort = () => fail(admissionAbortError(signal));
|
|
112
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
113
|
+
timer = setTimeout(() => {
|
|
114
|
+
fail(httpError(`${label} admission timed out after ${waitTimeoutMs}ms`, 503));
|
|
115
|
+
}, waitTimeoutMs);
|
|
116
|
+
timer.unref?.();
|
|
117
|
+
queue.push(entry);
|
|
118
|
+
if (signal?.aborted) onAbort();
|
|
119
|
+
else drain();
|
|
120
|
+
});
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
acquire.stats = () => ({ activeWeight, activeRequests, queued: queue.length, maxWeight });
|
|
124
|
+
return acquire;
|
|
125
|
+
}
|
|
126
|
+
|
|
50
127
|
export function createRequestAdmission({
|
|
51
128
|
maxBytes = MAX_INFLIGHT_BODY_BYTES,
|
|
52
129
|
maxQueued = MAX_QUEUED_REQUESTS,
|
|
@@ -55,6 +132,24 @@ export function createRequestAdmission({
|
|
|
55
132
|
const byteLimit = parsePositiveInteger(maxBytes, MAX_INFLIGHT_BODY_BYTES);
|
|
56
133
|
const queueLimit = parsePositiveInteger(maxQueued, MAX_QUEUED_REQUESTS);
|
|
57
134
|
const timeoutMs = parsePositiveInteger(waitTimeoutMs, REQUEST_QUEUE_TIMEOUT_MS);
|
|
135
|
+
const acquireDecompression = createSupplementalAdmission({
|
|
136
|
+
maxWeight: 1,
|
|
137
|
+
maxQueued: queueLimit,
|
|
138
|
+
waitTimeoutMs: timeoutMs,
|
|
139
|
+
label: "Request decompression",
|
|
140
|
+
});
|
|
141
|
+
const acquireDecodedBody = createSupplementalAdmission({
|
|
142
|
+
maxWeight: byteLimit,
|
|
143
|
+
maxQueued: queueLimit,
|
|
144
|
+
waitTimeoutMs: timeoutMs,
|
|
145
|
+
label: "Decoded request body",
|
|
146
|
+
});
|
|
147
|
+
const acquireResponseHistory = createSupplementalAdmission({
|
|
148
|
+
maxWeight: byteLimit,
|
|
149
|
+
maxQueued: queueLimit,
|
|
150
|
+
waitTimeoutMs: timeoutMs,
|
|
151
|
+
label: "Response history",
|
|
152
|
+
});
|
|
58
153
|
const queue = [];
|
|
59
154
|
let activeBytes = 0;
|
|
60
155
|
let activeRequests = 0;
|
|
@@ -83,13 +178,28 @@ export function createRequestAdmission({
|
|
|
83
178
|
counters.waitMsTotal += waitMs;
|
|
84
179
|
counters.waitMsMax = Math.max(counters.waitMsMax, waitMs);
|
|
85
180
|
let released = false;
|
|
86
|
-
|
|
181
|
+
const supplementalReleases = new Set();
|
|
182
|
+
const release = () => {
|
|
87
183
|
if (released) return;
|
|
88
184
|
released = true;
|
|
185
|
+
for (const releaseSupplemental of supplementalReleases) releaseSupplemental();
|
|
186
|
+
supplementalReleases.clear();
|
|
89
187
|
activeBytes = Math.max(0, activeBytes - entry.weight);
|
|
90
188
|
activeRequests = Math.max(0, activeRequests - 1);
|
|
91
189
|
drain();
|
|
92
|
-
}
|
|
190
|
+
};
|
|
191
|
+
const reserve = async (gate, weight, options) => {
|
|
192
|
+
const releaseSupplemental = await gate(weight, options);
|
|
193
|
+
if (released) {
|
|
194
|
+
releaseSupplemental();
|
|
195
|
+
throw admissionAbortError(options?.signal);
|
|
196
|
+
}
|
|
197
|
+
supplementalReleases.add(releaseSupplemental);
|
|
198
|
+
};
|
|
199
|
+
release.acquireDecompression = (options) => acquireDecompression(1, options);
|
|
200
|
+
release.reserveDecodedBody = (bytes, options) => reserve(acquireDecodedBody, bytes, options);
|
|
201
|
+
release.reserveResponseHistory = (bytes, options) => reserve(acquireResponseHistory, bytes, options);
|
|
202
|
+
entry.resolve(release);
|
|
93
203
|
};
|
|
94
204
|
|
|
95
205
|
const drain = () => {
|
|
@@ -173,11 +283,64 @@ export function createRequestAdmission({
|
|
|
173
283
|
? Number((counters.waitMsTotal / counters.activated).toFixed(1))
|
|
174
284
|
: 0,
|
|
175
285
|
waitMsMax: counters.waitMsMax,
|
|
286
|
+
decompressionsActive: acquireDecompression.stats().activeRequests,
|
|
287
|
+
decompressionsQueued: acquireDecompression.stats().queued,
|
|
288
|
+
decodedBodyBytes: acquireDecodedBody.stats().activeWeight,
|
|
289
|
+
decodedBodiesActive: acquireDecodedBody.stats().activeRequests,
|
|
290
|
+
decodedBodiesQueued: acquireDecodedBody.stats().queued,
|
|
291
|
+
responseHistoryBytes: acquireResponseHistory.stats().activeWeight,
|
|
292
|
+
responseHistoriesActive: acquireResponseHistory.stats().activeRequests,
|
|
293
|
+
responseHistoriesQueued: acquireResponseHistory.stats().queued,
|
|
176
294
|
});
|
|
177
295
|
return acquire;
|
|
178
296
|
}
|
|
179
297
|
|
|
180
|
-
|
|
298
|
+
function consumeRequestChunks(req, { signal, onChunk }) {
|
|
299
|
+
return new Promise((resolve, reject) => {
|
|
300
|
+
let settled = false;
|
|
301
|
+
const cleanup = () => {
|
|
302
|
+
req.off("data", handleData);
|
|
303
|
+
req.off("end", handleEnd);
|
|
304
|
+
req.off("error", handleError);
|
|
305
|
+
req.off("aborted", handleAborted);
|
|
306
|
+
signal?.removeEventListener("abort", handleSignalAbort);
|
|
307
|
+
};
|
|
308
|
+
const finish = (error) => {
|
|
309
|
+
if (settled) return;
|
|
310
|
+
settled = true;
|
|
311
|
+
cleanup();
|
|
312
|
+
if (error) reject(error);
|
|
313
|
+
else resolve();
|
|
314
|
+
};
|
|
315
|
+
const stopReading = (error) => {
|
|
316
|
+
req.pause?.();
|
|
317
|
+
finish(error);
|
|
318
|
+
};
|
|
319
|
+
const handleData = (chunk) => {
|
|
320
|
+
try {
|
|
321
|
+
onChunk(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
322
|
+
} catch (error) {
|
|
323
|
+
stopReading(error);
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
const handleEnd = () => finish();
|
|
327
|
+
const handleError = (error) => finish(error);
|
|
328
|
+
const handleAborted = () => finish(admissionAbortError(signal));
|
|
329
|
+
const handleSignalAbort = () => stopReading(admissionAbortError(signal));
|
|
330
|
+
|
|
331
|
+
req.once("end", handleEnd);
|
|
332
|
+
req.once("error", handleError);
|
|
333
|
+
req.once("aborted", handleAborted);
|
|
334
|
+
signal?.addEventListener("abort", handleSignalAbort, { once: true });
|
|
335
|
+
if (signal?.aborted) handleSignalAbort();
|
|
336
|
+
else {
|
|
337
|
+
req.on("data", handleData);
|
|
338
|
+
req.resume?.();
|
|
339
|
+
}
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
async function readRequestBuffer(req, maxBytes, signal) {
|
|
181
344
|
const contentLength = requestContentLength(req);
|
|
182
345
|
if (Number.isFinite(contentLength) && contentLength > maxBytes) {
|
|
183
346
|
throw payloadTooLarge("Raw", maxBytes);
|
|
@@ -185,16 +348,18 @@ async function readRequestBuffer(req, maxBytes) {
|
|
|
185
348
|
|
|
186
349
|
const chunks = [];
|
|
187
350
|
let total = 0;
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
351
|
+
await consumeRequestChunks(req, {
|
|
352
|
+
signal,
|
|
353
|
+
onChunk(buffer) {
|
|
354
|
+
total += buffer.length;
|
|
355
|
+
if (total > maxBytes) throw payloadTooLarge("Raw", maxBytes);
|
|
356
|
+
chunks.push(buffer);
|
|
357
|
+
},
|
|
358
|
+
});
|
|
194
359
|
return Buffer.concat(chunks);
|
|
195
360
|
}
|
|
196
361
|
|
|
197
|
-
async function readIdentityText(req, maxBodyBytes, maxDecodedBodyBytes) {
|
|
362
|
+
async function readIdentityText(req, maxBodyBytes, maxDecodedBodyBytes, signal) {
|
|
198
363
|
const contentLength = requestContentLength(req);
|
|
199
364
|
if (Number.isFinite(contentLength) && contentLength > maxBodyBytes) {
|
|
200
365
|
throw payloadTooLarge("Raw", maxBodyBytes);
|
|
@@ -206,13 +371,15 @@ async function readIdentityText(req, maxBodyBytes, maxDecodedBodyBytes) {
|
|
|
206
371
|
const decoder = new TextDecoder();
|
|
207
372
|
let text = "";
|
|
208
373
|
let total = 0;
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
374
|
+
await consumeRequestChunks(req, {
|
|
375
|
+
signal,
|
|
376
|
+
onChunk(buffer) {
|
|
377
|
+
total += buffer.length;
|
|
378
|
+
if (total > maxBodyBytes) throw payloadTooLarge("Raw", maxBodyBytes);
|
|
379
|
+
if (total > maxDecodedBodyBytes) throw payloadTooLarge("Decoded", maxDecodedBodyBytes);
|
|
380
|
+
text += decoder.decode(buffer, { stream: true });
|
|
381
|
+
},
|
|
382
|
+
});
|
|
216
383
|
return text + decoder.decode();
|
|
217
384
|
}
|
|
218
385
|
|
|
@@ -267,22 +434,34 @@ function parseRequestJson(text) {
|
|
|
267
434
|
}
|
|
268
435
|
|
|
269
436
|
export async function readJsonBody(req, {
|
|
437
|
+
admission,
|
|
270
438
|
maxBodyBytes = MAX_BODY_BYTES,
|
|
271
439
|
maxDecodedBodyBytes = MAX_DECODED_BODY_BYTES,
|
|
440
|
+
signal,
|
|
272
441
|
} = {}) {
|
|
442
|
+
if (signal?.aborted) throw admissionAbortError(signal);
|
|
273
443
|
const encodings = contentEncodings(req.headers?.["content-encoding"]);
|
|
274
444
|
if (encodings.every((encoding) => encoding === "identity")) {
|
|
275
|
-
return parseRequestJson(await readIdentityText(req, maxBodyBytes, maxDecodedBodyBytes));
|
|
445
|
+
return parseRequestJson(await readIdentityText(req, maxBodyBytes, maxDecodedBodyBytes, signal));
|
|
276
446
|
}
|
|
277
|
-
const buffer = await readRequestBuffer(req, maxBodyBytes);
|
|
278
|
-
|
|
447
|
+
const buffer = await readRequestBuffer(req, maxBodyBytes, signal);
|
|
448
|
+
const releaseDecompression = await admission?.acquireDecompression?.({ signal }) || (() => {});
|
|
279
449
|
try {
|
|
280
|
-
decoded
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
450
|
+
let decoded;
|
|
451
|
+
try {
|
|
452
|
+
decoded = await decodeRequestBuffer(buffer, req.headers?.["content-encoding"], maxDecodedBodyBytes);
|
|
453
|
+
} catch (e) {
|
|
454
|
+
if (e?.statusCode) throw e;
|
|
455
|
+
throw httpError(`Invalid compressed request body: ${e.message}`, 400);
|
|
456
|
+
}
|
|
457
|
+
if (signal?.aborted) throw admissionAbortError(signal);
|
|
458
|
+
if (decoded.length > 0) {
|
|
459
|
+
await admission?.reserveDecodedBody?.(decoded.length, { signal });
|
|
460
|
+
}
|
|
461
|
+
return parseRequestJson(decoded.toString("utf8"));
|
|
462
|
+
} finally {
|
|
463
|
+
releaseDecompression();
|
|
284
464
|
}
|
|
285
|
-
return parseRequestJson(decoded.toString("utf8"));
|
|
286
465
|
}
|
|
287
466
|
|
|
288
467
|
export function sendJsonError(res, err, fallbackStatus = 400) {
|
|
@@ -291,7 +470,10 @@ export function sendJsonError(res, err, fallbackStatus = 400) {
|
|
|
291
470
|
res.end();
|
|
292
471
|
return;
|
|
293
472
|
}
|
|
294
|
-
|
|
473
|
+
const statusCode = err?.statusCode || fallbackStatus;
|
|
474
|
+
const headers = { "Content-Type": "application/json" };
|
|
475
|
+
if (statusCode === 408 || statusCode === 413) headers.Connection = "close";
|
|
476
|
+
res.writeHead(statusCode, headers);
|
|
295
477
|
res.end(JSON.stringify(err?.jsonBody || { error: err?.message || "Request failed" }));
|
|
296
478
|
}
|
|
297
479
|
|
|
@@ -327,7 +509,8 @@ export function isAbortLikeError(err) {
|
|
|
327
509
|
}
|
|
328
510
|
|
|
329
511
|
export function abortErrorStatusCode(reason) {
|
|
330
|
-
if (
|
|
512
|
+
if (reason === "request_body_timeout") return 408;
|
|
513
|
+
if (["responses_prepare_timeout", "upstream_timeout", "stream_handshake_timeout", "stream_idle_timeout"].includes(reason)) return 504;
|
|
331
514
|
if (reason === "client_aborted" || reason === "client_closed") return 499;
|
|
332
515
|
return 502;
|
|
333
516
|
}
|
|
@@ -352,6 +535,7 @@ export function createRequestAbort(req, res) {
|
|
|
352
535
|
return {
|
|
353
536
|
signal: controller.signal,
|
|
354
537
|
get reason() { return reason; },
|
|
538
|
+
abort,
|
|
355
539
|
setTimeout(ms, nextReason = "upstream_timeout") {
|
|
356
540
|
if (timer) clearTimeout(timer);
|
|
357
541
|
timer = null;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { atomicWriteFileSync } from "./atomic-file.mjs";
|
|
5
|
+
|
|
6
|
+
export const LEGACY_COMMAND_WARNING_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
7
|
+
export const LEGACY_COMMAND_WARNING = "codex-copilot-dx is deprecated; use ccdx instead. The compatibility alias will be removed in a future breaking release.";
|
|
8
|
+
|
|
9
|
+
export function legacyCommandWarningPath({ env = process.env, home = os.homedir() } = {}) {
|
|
10
|
+
const xdgCacheHome = String(env.XDG_CACHE_HOME || "").trim();
|
|
11
|
+
const cacheRoot = xdgCacheHome || path.join(home, ".cache");
|
|
12
|
+
return path.join(cacheRoot, "codex-copilot-dx", "legacy-command-warning");
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function shouldShowLegacyCommandWarning({
|
|
16
|
+
env = process.env,
|
|
17
|
+
home = os.homedir(),
|
|
18
|
+
interactive = process.stderr.isTTY === true,
|
|
19
|
+
now = Date.now(),
|
|
20
|
+
} = {}) {
|
|
21
|
+
if (!interactive) return false;
|
|
22
|
+
const filePath = legacyCommandWarningPath({ env, home });
|
|
23
|
+
try {
|
|
24
|
+
const previous = Number.parseInt(fs.readFileSync(filePath, "utf8"), 10);
|
|
25
|
+
if (Number.isFinite(previous) && Math.abs(now - previous) < LEGACY_COMMAND_WARNING_INTERVAL_MS) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
} catch {
|
|
29
|
+
// A missing or unreadable cache must not prevent the compatibility command.
|
|
30
|
+
}
|
|
31
|
+
try {
|
|
32
|
+
atomicWriteFileSync(filePath, `${now}\n`, { mode: 0o600 });
|
|
33
|
+
} catch {
|
|
34
|
+
// The warning remains best effort when the cache directory is read-only.
|
|
35
|
+
}
|
|
36
|
+
return true;
|
|
37
|
+
}
|
package/src/observability.mjs
CHANGED
|
@@ -154,6 +154,7 @@ export function runtimeStatusPayload({
|
|
|
154
154
|
metrics,
|
|
155
155
|
streamPerformance,
|
|
156
156
|
admission,
|
|
157
|
+
imagePressure,
|
|
157
158
|
modelRegistry,
|
|
158
159
|
codexClient,
|
|
159
160
|
claudeClient,
|
|
@@ -186,6 +187,7 @@ export function runtimeStatusPayload({
|
|
|
186
187
|
admission: admission?.diagnostics?.() || admission?.stats?.() || null,
|
|
187
188
|
response_history: responseHistoryStats(),
|
|
188
189
|
image_optimization: imageOptimizationStats(),
|
|
190
|
+
image_history_pressure: imagePressure?.snapshot?.() || null,
|
|
189
191
|
copilot: codexRuntime,
|
|
190
192
|
models: codexModels,
|
|
191
193
|
profiles: {
|
package/src/pm-studio-relay.mjs
CHANGED
|
@@ -252,6 +252,9 @@ async function relayUpstreamResponse(response, reqBody, res, abort, streamIdleTi
|
|
|
252
252
|
function safeFailure(error, abort) {
|
|
253
253
|
if (error?.statusCode && !isAbortLikeError(error)) return error;
|
|
254
254
|
if (abort?.signal.aborted || isAbortLikeError(error)) {
|
|
255
|
+
if (abort?.reason === "request_body_timeout") {
|
|
256
|
+
return localError(408, "request_timeout", "PM Studio relay request body timed out");
|
|
257
|
+
}
|
|
255
258
|
return localError(
|
|
256
259
|
abortErrorStatusCode(abort?.reason),
|
|
257
260
|
"upstream_timeout",
|
|
@@ -277,6 +280,7 @@ export function createPmStudioRelayHandler(options = {}) {
|
|
|
277
280
|
const upstreamTimeoutMs = parsePositiveInteger(options.upstreamTimeoutMs, runtime.upstreamTimeoutMs);
|
|
278
281
|
const streamHandshakeTimeoutMs = parsePositiveInteger(options.streamHandshakeTimeoutMs, runtime.streamHandshakeTimeoutMs);
|
|
279
282
|
const streamIdleTimeoutMs = parsePositiveInteger(options.streamIdleTimeoutMs, runtime.streamIdleTimeoutMs);
|
|
283
|
+
const requestBodyTimeoutMs = parsePositiveInteger(options.requestBodyTimeoutMs, runtime.requestBodyTimeoutMs);
|
|
280
284
|
const validationTtlMs = parsePositiveInteger(options.validationTtlMs, DEFAULT_VALIDATION_TTL_MS);
|
|
281
285
|
const validationCacheSize = parsePositiveInteger(options.validationCacheSize, DEFAULT_VALIDATION_CACHE_SIZE, 4096);
|
|
282
286
|
const validationConcurrency = parsePositiveInteger(
|
|
@@ -430,8 +434,10 @@ export function createPmStudioRelayHandler(options = {}) {
|
|
|
430
434
|
async function handlePost(req, res, pathname, token, abort) {
|
|
431
435
|
let releaseRequest = releaseOnce();
|
|
432
436
|
try {
|
|
433
|
-
|
|
434
|
-
|
|
437
|
+
const admission = await acquireRequest(req, { signal: abort.signal });
|
|
438
|
+
releaseRequest = releaseOnce(admission);
|
|
439
|
+
abort.setTimeout(requestBodyTimeoutMs, "request_body_timeout");
|
|
440
|
+
const body = await readJsonBody(req, { admission, signal: abort.signal });
|
|
435
441
|
const supportsStream = pathname === "/chat/completions" || pathname === "/responses";
|
|
436
442
|
const streaming = supportsStream && body?.stream === true;
|
|
437
443
|
const requestTimeoutMs = streaming ? streamHandshakeTimeoutMs : upstreamTimeoutMs;
|
package/src/response-history.mjs
CHANGED
|
@@ -174,7 +174,11 @@ export function responseHistoryStats() {
|
|
|
174
174
|
return { entries: histories.size, bytes: totalBytes, evicted: evictedIds.size };
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
export function
|
|
177
|
+
export function responseHistoryRootId(responseId) {
|
|
178
|
+
return histories.get(responseId)?.rootId || null;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function responseHistoryChain(responseId) {
|
|
178
182
|
const chain = [];
|
|
179
183
|
const seen = new Set();
|
|
180
184
|
let currentId = responseId;
|
|
@@ -189,7 +193,16 @@ export function materializeResponseHistory(responseId) {
|
|
|
189
193
|
chain.push(entry);
|
|
190
194
|
currentId = entry.parentId;
|
|
191
195
|
}
|
|
192
|
-
|
|
196
|
+
return { chain, rootId: chain[0]?.rootId };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export function responseHistoryMaterializedBytes(responseId) {
|
|
200
|
+
const { chain } = responseHistoryChain(responseId);
|
|
201
|
+
return chain.reduce((bytes, entry) => bytes + entry.bytes, 0);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export function materializeResponseHistory(responseId) {
|
|
205
|
+
const { chain, rootId } = responseHistoryChain(responseId);
|
|
193
206
|
const items = [];
|
|
194
207
|
for (const entry of chain.reverse()) items.push(...entry.inputItems, ...entry.outputItems);
|
|
195
208
|
const materialized = cloneJson(items);
|
package/src/responses-bridge.mjs
CHANGED
|
@@ -143,35 +143,55 @@ export function chatToResponses(chatResp, model) {
|
|
|
143
143
|
|
|
144
144
|
export async function forwardToChat(chatReq, emitEvent, onDone, onError, options = {}) {
|
|
145
145
|
delete chatReq.max_tokens;
|
|
146
|
-
const
|
|
147
|
-
|
|
146
|
+
const requestModel = chatReq.model || "unknown";
|
|
147
|
+
let upstreamReq = withChatStreamUsage(chatReq);
|
|
148
|
+
let {
|
|
149
|
+
abort,
|
|
150
|
+
bodyText,
|
|
151
|
+
chatCompletionsFn: configuredChatCompletionsFn,
|
|
152
|
+
onUpstreamResponse,
|
|
153
|
+
onUpstreamStart,
|
|
154
|
+
releaseRequest,
|
|
155
|
+
signal,
|
|
156
|
+
streamIdleTimeoutMs,
|
|
157
|
+
} = options;
|
|
158
|
+
options = undefined;
|
|
159
|
+
if (upstreamReq !== chatReq) bodyText = undefined;
|
|
148
160
|
let resp;
|
|
149
161
|
try {
|
|
150
|
-
const chatCompletionsFn =
|
|
162
|
+
const chatCompletionsFn = configuredChatCompletionsFn || chatCompletions;
|
|
151
163
|
try {
|
|
152
|
-
|
|
164
|
+
onUpstreamStart?.();
|
|
165
|
+
resp = await chatCompletionsFn(upstreamReq, {
|
|
166
|
+
signal,
|
|
167
|
+
bodyText,
|
|
168
|
+
onUpstreamStart,
|
|
169
|
+
});
|
|
153
170
|
} finally {
|
|
154
|
-
|
|
171
|
+
bodyText = undefined;
|
|
172
|
+
chatReq = undefined;
|
|
173
|
+
upstreamReq = undefined;
|
|
174
|
+
releaseRequest?.();
|
|
155
175
|
}
|
|
156
176
|
} catch (e) {
|
|
157
|
-
const statusCode = isAbortLikeError(e) ? abortErrorStatusCode(
|
|
177
|
+
const statusCode = isAbortLikeError(e) ? abortErrorStatusCode(abort?.reason) : 502;
|
|
158
178
|
await onError(statusCode, e.message);
|
|
159
|
-
return;
|
|
179
|
+
return false;
|
|
160
180
|
}
|
|
161
181
|
if (!resp.ok) {
|
|
162
182
|
await onError(resp.status, await resp.text(), undefined, resp);
|
|
163
|
-
return;
|
|
183
|
+
return false;
|
|
164
184
|
}
|
|
165
185
|
try {
|
|
166
186
|
await requireUpstreamEventStream(resp);
|
|
167
187
|
} catch (error) {
|
|
168
188
|
await onError(error.statusCode, error.message, error);
|
|
169
|
-
return;
|
|
189
|
+
return false;
|
|
170
190
|
}
|
|
171
|
-
|
|
172
|
-
|
|
191
|
+
onUpstreamResponse?.(resp);
|
|
192
|
+
abort?.setTimeout(streamIdleTimeoutMs, "stream_idle_timeout");
|
|
173
193
|
const respId = `resp_${uid()}`;
|
|
174
|
-
let actualModel =
|
|
194
|
+
let actualModel = requestModel;
|
|
175
195
|
let fullText = "";
|
|
176
196
|
let messageItem = null;
|
|
177
197
|
let nextOutputIndex = 0;
|
|
@@ -275,11 +295,11 @@ export async function forwardToChat(chatReq, emitEvent, onDone, onError, options
|
|
|
275
295
|
|
|
276
296
|
try {
|
|
277
297
|
for await (const line of webStreamLines(resp, {
|
|
278
|
-
onChunk: () =>
|
|
298
|
+
onChunk: () => abort?.setTimeout(streamIdleTimeoutMs, "stream_idle_timeout"),
|
|
279
299
|
})) {
|
|
280
300
|
if (!line.startsWith("data: ")) continue;
|
|
281
301
|
const data = line.slice(6).trim();
|
|
282
|
-
if (data === "[DONE]") { await emitCompleted(); onDone(); return; }
|
|
302
|
+
if (data === "[DONE]") { await emitCompleted(); onDone(); return true; }
|
|
283
303
|
let parsed;
|
|
284
304
|
try {
|
|
285
305
|
parsed = JSON.parse(data);
|
|
@@ -329,10 +349,11 @@ export async function forwardToChat(chatReq, emitEvent, onDone, onError, options
|
|
|
329
349
|
}
|
|
330
350
|
}
|
|
331
351
|
} catch (e) {
|
|
332
|
-
const statusCode = isAbortLikeError(e) ? abortErrorStatusCode(
|
|
352
|
+
const statusCode = isAbortLikeError(e) ? abortErrorStatusCode(abort?.reason) : (e?.statusCode || 502);
|
|
333
353
|
await onError(statusCode, e?.message || "upstream stream error", e);
|
|
334
|
-
return;
|
|
354
|
+
return false;
|
|
335
355
|
}
|
|
336
356
|
const error = incompleteUpstreamStream("[DONE]");
|
|
337
357
|
await onError(error.statusCode, error.message, error);
|
|
358
|
+
return false;
|
|
338
359
|
}
|
|
@@ -159,12 +159,14 @@ function recordBytesAfterDrop(record, state, partIndex) {
|
|
|
159
159
|
return state.baseBytes + remainingElementBytes + remainingCount - 1;
|
|
160
160
|
}
|
|
161
161
|
|
|
162
|
-
function selectImageDrops(refs, bodyBytes, targetBytes) {
|
|
162
|
+
function selectImageDrops(refs, bodyBytes, targetBytes, maxHistoricalImages = Number.POSITIVE_INFINITY) {
|
|
163
163
|
const candidates = orderedHistoricalImageCandidates(refs);
|
|
164
164
|
const selected = new Set();
|
|
165
165
|
const byRecord = new Map();
|
|
166
166
|
let projectedBytes = bodyBytes;
|
|
167
|
+
let historicalImages = refs.reduce((count, ref) => count + (ref.historical ? 1 : 0), 0);
|
|
167
168
|
for (const index of candidates) {
|
|
169
|
+
if (projectedBytes <= targetBytes && historicalImages <= maxHistoricalImages) break;
|
|
168
170
|
const ref = refs[index];
|
|
169
171
|
let state = byRecord.get(ref.record);
|
|
170
172
|
if (!state) {
|
|
@@ -173,7 +175,7 @@ function selectImageDrops(refs, bodyBytes, targetBytes) {
|
|
|
173
175
|
}
|
|
174
176
|
const afterBytes = recordBytesAfterDrop(ref.record, state, ref.partIndex);
|
|
175
177
|
const savings = state.currentBytes - afterBytes;
|
|
176
|
-
if (savings <= 0) continue;
|
|
178
|
+
if (savings <= 0 && historicalImages <= maxHistoricalImages) continue;
|
|
177
179
|
state.currentBytes = afterBytes;
|
|
178
180
|
if (ref.record.kind !== "top-level") {
|
|
179
181
|
state.remainingCount -= 1;
|
|
@@ -181,9 +183,9 @@ function selectImageDrops(refs, bodyBytes, targetBytes) {
|
|
|
181
183
|
}
|
|
182
184
|
selected.add(index);
|
|
183
185
|
projectedBytes -= savings;
|
|
184
|
-
|
|
186
|
+
historicalImages -= 1;
|
|
185
187
|
}
|
|
186
|
-
return selected;
|
|
188
|
+
return { selected, historicalImages, projectedBytes };
|
|
187
189
|
}
|
|
188
190
|
|
|
189
191
|
function applyImageDrops(records, refs, selected) {
|
|
@@ -253,7 +255,7 @@ export function trimResponsesHistoryToByteBudget(reqBody, {
|
|
|
253
255
|
while (bodyBytes > limit) {
|
|
254
256
|
const { records, refs } = collectResponseImages(inputItems, currentInputStart);
|
|
255
257
|
if (!refs.length) break;
|
|
256
|
-
const selected = selectImageDrops(refs, bodyBytes, limit);
|
|
258
|
+
const { selected } = selectImageDrops(refs, bodyBytes, limit);
|
|
257
259
|
if (!selected.size) break;
|
|
258
260
|
applyImageDrops(records, refs, selected);
|
|
259
261
|
imagesOmitted += selected.size;
|
|
@@ -278,6 +280,62 @@ export function trimResponsesHistoryToByteBudget(reqBody, {
|
|
|
278
280
|
};
|
|
279
281
|
}
|
|
280
282
|
|
|
283
|
+
export function responsesHistoricalImageStats(inputItems, currentInputStart = 0) {
|
|
284
|
+
const items = Array.isArray(inputItems) ? inputItems : [];
|
|
285
|
+
const { refs } = collectResponseImages(items, currentInputStart);
|
|
286
|
+
const historicalImages = refs.reduce((count, ref) => count + (ref.historical ? 1 : 0), 0);
|
|
287
|
+
return {
|
|
288
|
+
totalImages: refs.length,
|
|
289
|
+
historicalImages,
|
|
290
|
+
currentImages: refs.length - historicalImages,
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export function trimResponsesHistoricalImages(reqBody, {
|
|
295
|
+
currentInputStart = 0,
|
|
296
|
+
maxHistoricalImages = Number.POSITIVE_INFINITY,
|
|
297
|
+
targetBytes = Number.POSITIVE_INFINITY,
|
|
298
|
+
initialBodyText,
|
|
299
|
+
initialBodyBytes,
|
|
300
|
+
} = {}) {
|
|
301
|
+
let bodyText = initialBodyText ?? JSON.stringify(reqBody);
|
|
302
|
+
let bodyBytes = initialBodyBytes ?? Buffer.byteLength(bodyText);
|
|
303
|
+
const byteLimit = Number.isFinite(targetBytes) && targetBytes > 0
|
|
304
|
+
? Math.floor(targetBytes)
|
|
305
|
+
: Number.POSITIVE_INFINITY;
|
|
306
|
+
const imageLimit = Number.isFinite(maxHistoricalImages) && maxHistoricalImages >= 0
|
|
307
|
+
? Math.floor(maxHistoricalImages)
|
|
308
|
+
: Number.POSITIVE_INFINITY;
|
|
309
|
+
const inputItems = Array.isArray(reqBody?.input) ? reqBody.input : [];
|
|
310
|
+
let imagesOmitted = 0;
|
|
311
|
+
|
|
312
|
+
while (true) {
|
|
313
|
+
const { records, refs } = collectResponseImages(inputItems, currentInputStart);
|
|
314
|
+
const historicalImages = refs.reduce((count, ref) => count + (ref.historical ? 1 : 0), 0);
|
|
315
|
+
if (bodyBytes <= byteLimit && historicalImages <= imageLimit) break;
|
|
316
|
+
if (!historicalImages) break;
|
|
317
|
+
const { selected } = selectImageDrops(refs, bodyBytes, byteLimit, imageLimit);
|
|
318
|
+
if (!selected.size) break;
|
|
319
|
+
applyImageDrops(records, refs, selected);
|
|
320
|
+
imagesOmitted += selected.size;
|
|
321
|
+
({ bodyText, bodyBytes } = bodyPayload(reqBody));
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
const stats = responsesHistoricalImageStats(inputItems, currentInputStart);
|
|
325
|
+
return {
|
|
326
|
+
bodyText,
|
|
327
|
+
bodyBytes,
|
|
328
|
+
targetBytes: byteLimit,
|
|
329
|
+
maxHistoricalImages: imageLimit,
|
|
330
|
+
historicalImages: stats.historicalImages,
|
|
331
|
+
currentImages: stats.currentImages,
|
|
332
|
+
imagesOmitted,
|
|
333
|
+
overBudget: bodyBytes > byteLimit,
|
|
334
|
+
overImageBudget: stats.historicalImages > imageLimit,
|
|
335
|
+
adapted: imagesOmitted > 0,
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
|
|
281
339
|
export function enforceResponsesPayloadByteBudget(reqBody, payload) {
|
|
282
340
|
let finalized = payload;
|
|
283
341
|
if (payload?.overBudget) {
|