ghc-proxy 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +568 -529
- package/dist/main.mjs +2083 -773
- package/package.json +1 -1
package/dist/main.mjs
CHANGED
|
@@ -8,7 +8,7 @@ import fs, { mkdir, readdir, unlink, writeFile } from "node:fs/promises";
|
|
|
8
8
|
import os from "node:os";
|
|
9
9
|
import { randomUUID } from "node:crypto";
|
|
10
10
|
import { execFile, execFileSync } from "node:child_process";
|
|
11
|
-
import nodeHTTP from "node:http";
|
|
11
|
+
import nodeHTTP, { createServer } from "node:http";
|
|
12
12
|
import { PassThrough, Readable } from "node:stream";
|
|
13
13
|
import nodeHTTP2 from "node:http2";
|
|
14
14
|
import nodeHTTPS from "node:https";
|
|
@@ -5769,7 +5769,9 @@ const configFileSchema = object({
|
|
|
5769
5769
|
to: string()
|
|
5770
5770
|
})).optional(),
|
|
5771
5771
|
upstreamQueueConcurrency: number().int().positive().optional(),
|
|
5772
|
-
upstreamQueueMaxRetries: number().int().
|
|
5772
|
+
upstreamQueueMaxRetries: number().int().min(0).max(2).optional(),
|
|
5773
|
+
upstreamRecoveryBudgetSeconds: number().int().min(1).max(120).optional(),
|
|
5774
|
+
overloadFallbacks: record(string(), string()).optional(),
|
|
5773
5775
|
upstreamQueueBaseDelaySeconds: number().int().nonnegative().optional(),
|
|
5774
5776
|
upstreamQueueMaxDelaySeconds: number().int().positive().optional(),
|
|
5775
5777
|
gheDomain: string().optional()
|
|
@@ -5798,12 +5800,12 @@ async function readConfig() {
|
|
|
5798
5800
|
const fieldResult = schema.safeParse(rawObj[key]);
|
|
5799
5801
|
if (fieldResult.success) partial[key] = fieldResult.data;
|
|
5800
5802
|
}
|
|
5801
|
-
cachedConfig = partial;
|
|
5803
|
+
cachedConfig = sanitizeConfig(partial);
|
|
5802
5804
|
return cachedConfig;
|
|
5803
5805
|
}
|
|
5804
5806
|
const unknownKeys = Object.keys(raw).filter((key) => !KNOWN_CONFIG_KEYS.has(key));
|
|
5805
5807
|
if (unknownKeys.length > 0) consola.warn(`config.json contains unknown fields: ${unknownKeys.join(", ")}`);
|
|
5806
|
-
cachedConfig = result.data;
|
|
5808
|
+
cachedConfig = sanitizeConfig(result.data);
|
|
5807
5809
|
return cachedConfig;
|
|
5808
5810
|
} catch (error) {
|
|
5809
5811
|
if (error.code === "ENOENT") {
|
|
@@ -5815,6 +5817,33 @@ async function readConfig() {
|
|
|
5815
5817
|
return {};
|
|
5816
5818
|
}
|
|
5817
5819
|
}
|
|
5820
|
+
function sanitizeConfig(config) {
|
|
5821
|
+
if (!config.overloadFallbacks) return config;
|
|
5822
|
+
const normalizedFallbacks = {};
|
|
5823
|
+
const invalidSources = [];
|
|
5824
|
+
for (const [source, target] of Object.entries(config.overloadFallbacks)) {
|
|
5825
|
+
const normalizedSource = source.trim();
|
|
5826
|
+
const normalizedTarget = target.trim();
|
|
5827
|
+
if (!normalizedSource || !normalizedTarget || normalizedSource === normalizedTarget) {
|
|
5828
|
+
invalidSources.push(source || "<blank>");
|
|
5829
|
+
continue;
|
|
5830
|
+
}
|
|
5831
|
+
normalizedFallbacks[normalizedSource] = normalizedTarget;
|
|
5832
|
+
}
|
|
5833
|
+
const overloadFallbacks = {};
|
|
5834
|
+
for (const [source, target] of Object.entries(normalizedFallbacks)) {
|
|
5835
|
+
if (normalizedFallbacks[target] === source) {
|
|
5836
|
+
invalidSources.push(source);
|
|
5837
|
+
continue;
|
|
5838
|
+
}
|
|
5839
|
+
overloadFallbacks[source] = target;
|
|
5840
|
+
}
|
|
5841
|
+
if (invalidSources.length > 0) consola.warn(`config.json contains invalid overloadFallbacks entries: ${invalidSources.join(", ")}. Ignoring those entries.`);
|
|
5842
|
+
return {
|
|
5843
|
+
...config,
|
|
5844
|
+
overloadFallbacks
|
|
5845
|
+
};
|
|
5846
|
+
}
|
|
5818
5847
|
function getCachedConfig() {
|
|
5819
5848
|
return cachedConfig;
|
|
5820
5849
|
}
|
|
@@ -5898,6 +5927,12 @@ var ConfigStore = class {
|
|
|
5898
5927
|
getModelFallback() {
|
|
5899
5928
|
return getCachedConfig().modelFallback;
|
|
5900
5929
|
}
|
|
5930
|
+
getOverloadFallback(sourceModel) {
|
|
5931
|
+
return getCachedConfig().overloadFallbacks?.[sourceModel];
|
|
5932
|
+
}
|
|
5933
|
+
hasOverloadFallbacks() {
|
|
5934
|
+
return Object.keys(getCachedConfig().overloadFallbacks ?? {}).length > 0;
|
|
5935
|
+
}
|
|
5901
5936
|
};
|
|
5902
5937
|
const configStore = new ConfigStore();
|
|
5903
5938
|
//#endregion
|
|
@@ -5996,18 +6031,80 @@ var TranslationFailure = class extends Error {
|
|
|
5996
6031
|
}
|
|
5997
6032
|
};
|
|
5998
6033
|
//#endregion
|
|
6034
|
+
//#region src/lib/timeout-error.ts
|
|
6035
|
+
/**
|
|
6036
|
+
* Whether an error represents a request that timed out or was aborted.
|
|
6037
|
+
*
|
|
6038
|
+
* The shape differs by runtime, so the check is structural rather than a
|
|
6039
|
+
* single `name` comparison:
|
|
6040
|
+
* - Bun rejects with a flat `DOMException` named `TimeoutError` (its ~300s
|
|
6041
|
+
* `fetch` ceiling, `AbortSignal.timeout`) or `AbortError`.
|
|
6042
|
+
* - Node rejects with `TypeError('fetch failed' | 'terminated')` and puts the
|
|
6043
|
+
* real undici error on `.cause` (`HeadersTimeoutError`, `BodyTimeoutError`,
|
|
6044
|
+
* `ConnectTimeoutError`), so the top-level error carries no signal at all —
|
|
6045
|
+
* `TypeError('fetch failed')` is also what `ECONNREFUSED` and DNS failures
|
|
6046
|
+
* look like. The discriminator is the cause's `name`/`code`.
|
|
6047
|
+
*
|
|
6048
|
+
* Both runtimes enforce a ~300s upstream ceiling by default (Node's is
|
|
6049
|
+
* undici's `headersTimeout`/`bodyTimeout` default of `300e3`). It is an
|
|
6050
|
+
* **idle** timer on both, not a total-duration cap: it resets on every byte, so
|
|
6051
|
+
* it fires on a stalled stream well before the configured `--upstream-timeout`
|
|
6052
|
+
* of 1800s, and never on one that keeps streaming. See
|
|
6053
|
+
* `docs/design/streaming.md`.
|
|
6054
|
+
*
|
|
6055
|
+
* Kept in one place because the rule is checked on both sides of the stream
|
|
6056
|
+
* boundary: `src/server.ts` maps it to a 504 before the first byte, and the
|
|
6057
|
+
* Anthropic stream transducer maps it to an SSE error frame after. Two
|
|
6058
|
+
* implementations of "what counts as a timeout" is how one of them ends up
|
|
6059
|
+
* recognizing only half the errors.
|
|
6060
|
+
*/
|
|
6061
|
+
const TIMEOUT_ERROR_NAMES = new Set([
|
|
6062
|
+
"AbortError",
|
|
6063
|
+
"TimeoutError",
|
|
6064
|
+
"ConnectTimeoutError",
|
|
6065
|
+
"HeadersTimeoutError",
|
|
6066
|
+
"BodyTimeoutError"
|
|
6067
|
+
]);
|
|
6068
|
+
const TIMEOUT_ERROR_CODES = new Set([
|
|
6069
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
6070
|
+
"UND_ERR_HEADERS_TIMEOUT",
|
|
6071
|
+
"UND_ERR_BODY_TIMEOUT",
|
|
6072
|
+
"ETIMEDOUT"
|
|
6073
|
+
]);
|
|
6074
|
+
const MAX_CAUSE_DEPTH = 5;
|
|
6075
|
+
function errorCauseChainSome(value, predicate, depth = 0) {
|
|
6076
|
+
if (typeof value !== "object" || value === null) return false;
|
|
6077
|
+
const candidate = value;
|
|
6078
|
+
if (predicate(candidate)) return true;
|
|
6079
|
+
if (depth >= MAX_CAUSE_DEPTH) return false;
|
|
6080
|
+
if (errorCauseChainSome(candidate.cause, predicate, depth + 1)) return true;
|
|
6081
|
+
return Array.isArray(candidate.errors) && candidate.errors.some((inner) => errorCauseChainSome(inner, predicate, depth + 1));
|
|
6082
|
+
}
|
|
6083
|
+
function isTimeoutLikeError(error) {
|
|
6084
|
+
try {
|
|
6085
|
+
return errorCauseChainSome(error, (candidate) => typeof candidate.name === "string" && TIMEOUT_ERROR_NAMES.has(candidate.name) || typeof candidate.code === "string" && TIMEOUT_ERROR_CODES.has(candidate.code));
|
|
6086
|
+
} catch {
|
|
6087
|
+
return false;
|
|
6088
|
+
}
|
|
6089
|
+
}
|
|
6090
|
+
//#endregion
|
|
5999
6091
|
//#region src/lib/error.ts
|
|
6000
6092
|
var HTTPError = class extends Error {
|
|
6001
6093
|
status;
|
|
6002
6094
|
body;
|
|
6003
|
-
|
|
6095
|
+
headers;
|
|
6096
|
+
constructor(status, body, options = {}) {
|
|
6004
6097
|
super(body.error.message);
|
|
6005
6098
|
this.name = "HTTPError";
|
|
6006
6099
|
this.status = status;
|
|
6007
6100
|
this.body = body;
|
|
6101
|
+
this.headers = new Headers(options.headers);
|
|
6008
6102
|
}
|
|
6009
6103
|
toResponse() {
|
|
6010
|
-
return Response.json(this.body, {
|
|
6104
|
+
return Response.json(this.body, {
|
|
6105
|
+
status: this.status,
|
|
6106
|
+
headers: this.headers
|
|
6107
|
+
});
|
|
6011
6108
|
}
|
|
6012
6109
|
};
|
|
6013
6110
|
const TRANSIENT_UPSTREAM_STATUSES = new Set([
|
|
@@ -6027,13 +6124,29 @@ const TRANSIENT_UPSTREAM_STATUSES = new Set([
|
|
|
6027
6124
|
function isTransientUpstreamStatus(status) {
|
|
6028
6125
|
return TRANSIENT_UPSTREAM_STATUSES.has(status);
|
|
6029
6126
|
}
|
|
6030
|
-
|
|
6031
|
-
|
|
6032
|
-
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
|
|
6036
|
-
|
|
6127
|
+
function resolveCapacityCooldownScope(status, effectiveModel) {
|
|
6128
|
+
if (status === 429) return "account";
|
|
6129
|
+
if (status === 529) return effectiveModel ? "model" : "request";
|
|
6130
|
+
}
|
|
6131
|
+
const CONNECTION_ESTABLISHMENT_CODES = {
|
|
6132
|
+
ENOTFOUND: "dns",
|
|
6133
|
+
EAI_AGAIN: "dns",
|
|
6134
|
+
ECONNREFUSED: "connection-refused",
|
|
6135
|
+
ConnectionRefused: "connection-refused"
|
|
6136
|
+
};
|
|
6137
|
+
function isRetryableConnectionEstablishmentError(error) {
|
|
6138
|
+
if (isTimeoutLikeError(error)) return void 0;
|
|
6139
|
+
let connectionClass;
|
|
6140
|
+
try {
|
|
6141
|
+
errorCauseChainSome(error, (candidate) => {
|
|
6142
|
+
if (typeof candidate.code !== "string") return false;
|
|
6143
|
+
connectionClass = CONNECTION_ESTABLISHMENT_CODES[candidate.code];
|
|
6144
|
+
return connectionClass !== void 0;
|
|
6145
|
+
});
|
|
6146
|
+
} catch {
|
|
6147
|
+
return;
|
|
6148
|
+
}
|
|
6149
|
+
return connectionClass;
|
|
6037
6150
|
}
|
|
6038
6151
|
/**
|
|
6039
6152
|
* Reject a request locally with a 400.
|
|
@@ -6133,7 +6246,8 @@ async function throwUpstreamError(message, response) {
|
|
|
6133
6246
|
rawBody: rawText ? previewBody(rawText) : "<empty>",
|
|
6134
6247
|
headers: getDiagnosticHeaders(response)
|
|
6135
6248
|
});
|
|
6136
|
-
|
|
6249
|
+
const retryAfter = response.headers.get("retry-after");
|
|
6250
|
+
throw new HTTPError(response.status, body, retryAfter ? { headers: { "retry-after": retryAfter } } : void 0);
|
|
6137
6251
|
}
|
|
6138
6252
|
//#endregion
|
|
6139
6253
|
//#region src/util/sleep.ts
|
|
@@ -6579,17 +6693,927 @@ const GITHUB_BASE_URL = "https://github.com";
|
|
|
6579
6693
|
const GITHUB_CLIENT_ID = "Iv1.b507a08c87ecfe98";
|
|
6580
6694
|
const GITHUB_APP_SCOPES = ["read:user"].join(" ");
|
|
6581
6695
|
//#endregion
|
|
6696
|
+
//#region src/util/duration.ts
|
|
6697
|
+
/**
|
|
6698
|
+
* Formats a millisecond duration as a compact human-readable string:
|
|
6699
|
+
* `<n>ms` under one second, otherwise `<n>s` rounded to whole seconds.
|
|
6700
|
+
*/
|
|
6701
|
+
function formatDurationMs(ms) {
|
|
6702
|
+
return ms < 1e3 ? `${ms}ms` : `${Math.round(ms / 1e3)}s`;
|
|
6703
|
+
}
|
|
6704
|
+
//#endregion
|
|
6705
|
+
//#region src/lib/request-logger.ts
|
|
6706
|
+
/**
|
|
6707
|
+
* Per-request model mapping store.
|
|
6708
|
+
* Route handlers write to this; the after-response hook reads from it.
|
|
6709
|
+
* Uses WeakMap so entries are GC'd when the Request is collected.
|
|
6710
|
+
*/
|
|
6711
|
+
const requestModelMapping = /* @__PURE__ */ new WeakMap();
|
|
6712
|
+
const requestCorrelation = /* @__PURE__ */ new WeakMap();
|
|
6713
|
+
/**
|
|
6714
|
+
* Per-request start timestamp for the access log.
|
|
6715
|
+
*
|
|
6716
|
+
* Keyed on the `Request` the same way `requestCorrelation` is, so entries are
|
|
6717
|
+
* GC'd with it. This lives here rather than in `derive()` because `derive` does
|
|
6718
|
+
* not run on a route Elysia never matched — `onRequest` does, and it preserves
|
|
6719
|
+
* `Request` identity through to `onAfterResponse`. Reading a missing `derive`
|
|
6720
|
+
* value is what rendered every unmatched route's duration as the literal string
|
|
6721
|
+
* `NaNs`.
|
|
6722
|
+
*
|
|
6723
|
+
* That lifecycle behavior was measured once by hand on Bun 1.3.14 and on Node
|
|
6724
|
+
* 24.18 via `@elysiajs/node`. **The automated suite runs only under Bun**, so
|
|
6725
|
+
* the Node half is a point-in-time observation rather than standing coverage —
|
|
6726
|
+
* see `docs/solutions/testing/green-suite-is-evidence-about-one-runtime.md`. If
|
|
6727
|
+
* the Node adapter ever stops firing `onRequest` on an unmatched path, or
|
|
6728
|
+
* re-wraps the `Request` between hooks, the lookup misses and the duration
|
|
6729
|
+
* silently degrades to `-` on that runtime with a green suite.
|
|
6730
|
+
*/
|
|
6731
|
+
const requestStartTimes = /* @__PURE__ */ new WeakMap();
|
|
6732
|
+
/**
|
|
6733
|
+
* Record when a request arrived. Called from `onRequest`, which fires on every
|
|
6734
|
+
* path including ones no route matches.
|
|
6735
|
+
*
|
|
6736
|
+
* The `void` return type is load-bearing, not decoration. `WeakMap.set` returns
|
|
6737
|
+
* the WeakMap, and Elysia turns any non-undefined `onRequest` return into the
|
|
6738
|
+
* response body — a setter written as a concise arrow over `.set()` makes every
|
|
6739
|
+
* response in the proxy the string `[object WeakMap]`. `tsc` does not catch it.
|
|
6740
|
+
*/
|
|
6741
|
+
function markRequestStart(request) {
|
|
6742
|
+
requestStartTimes.set(request, Date.now());
|
|
6743
|
+
}
|
|
6744
|
+
function getRequestStart(request) {
|
|
6745
|
+
return requestStartTimes.get(request);
|
|
6746
|
+
}
|
|
6747
|
+
function getOrCreateRequestCorrelation(request) {
|
|
6748
|
+
const existing = requestCorrelation.get(request);
|
|
6749
|
+
if (existing) return existing;
|
|
6750
|
+
const requestId = crypto.randomUUID();
|
|
6751
|
+
const callerRequestId = request.headers.get("x-request-id") ?? void 0;
|
|
6752
|
+
const correlation = {
|
|
6753
|
+
requestId,
|
|
6754
|
+
...callerRequestId ? { callerRequestId } : {},
|
|
6755
|
+
responseRequestId: callerRequestId ?? requestId
|
|
6756
|
+
};
|
|
6757
|
+
requestCorrelation.set(request, correlation);
|
|
6758
|
+
return correlation;
|
|
6759
|
+
}
|
|
6760
|
+
const MAX_LOGGED_CALLER_REQUEST_ID_LENGTH = 128;
|
|
6761
|
+
const UNSAFE_CALLER_REQUEST_ID_CHARACTERS = /[^\w.:@/-]/g;
|
|
6762
|
+
function sanitizeCallerRequestId(value) {
|
|
6763
|
+
return value ? value.replace(UNSAFE_CALLER_REQUEST_ID_CHARACTERS, "_").slice(0, MAX_LOGGED_CALLER_REQUEST_ID_LENGTH) : void 0;
|
|
6764
|
+
}
|
|
6765
|
+
const RECOVERY_EVENT_OPTIONAL_FIELDS = [
|
|
6766
|
+
"retryCount",
|
|
6767
|
+
"status",
|
|
6768
|
+
"connectionClass",
|
|
6769
|
+
"effectiveModel",
|
|
6770
|
+
"scope",
|
|
6771
|
+
"activeSlots",
|
|
6772
|
+
"maxSlots",
|
|
6773
|
+
"pendingDepth",
|
|
6774
|
+
"maxPendingDepth",
|
|
6775
|
+
"queueWaitMs",
|
|
6776
|
+
"delaySource",
|
|
6777
|
+
"delayMs",
|
|
6778
|
+
"elapsedMs",
|
|
6779
|
+
"remainingBudgetMs",
|
|
6780
|
+
"nextRetryAt",
|
|
6781
|
+
"decision"
|
|
6782
|
+
];
|
|
6783
|
+
function isRoutineRecoveryEvent(fields) {
|
|
6784
|
+
return fields.event === "cooldown" || fields.event === "grant" && fields.queueWaitMs === 0 || fields.event === "retry" && fields.status !== void 0 && fields.decision === "retry";
|
|
6785
|
+
}
|
|
6786
|
+
function formatRecoveryEventLine(fields) {
|
|
6787
|
+
const kind = fields.status === 429 ? "rate limit" : fields.status === 529 ? "overload" : "recovery";
|
|
6788
|
+
const includeQueueState = fields.event === "admission" || fields.event === "grant";
|
|
6789
|
+
return [
|
|
6790
|
+
`Upstream ${kind}`,
|
|
6791
|
+
fields.decision ?? fields.event,
|
|
6792
|
+
fields.status !== void 0 ? `status=${fields.status}` : void 0,
|
|
6793
|
+
fields.effectiveModel ? `model=${JSON.stringify(fields.effectiveModel).slice(1, -1)}` : void 0,
|
|
6794
|
+
fields.scope ? `scope=${fields.scope}` : void 0,
|
|
6795
|
+
fields.retryCount !== void 0 ? `retry=${fields.retryCount}` : void 0,
|
|
6796
|
+
fields.connectionClass ? `connection=${fields.connectionClass}` : void 0,
|
|
6797
|
+
fields.delayMs !== void 0 ? `wait=${formatDurationMs(fields.delayMs)}` : void 0,
|
|
6798
|
+
fields.delaySource ? `source=${fields.delaySource}` : void 0,
|
|
6799
|
+
fields.queueWaitMs !== void 0 && fields.queueWaitMs > 0 ? `queueWait=${formatDurationMs(fields.queueWaitMs)}` : void 0,
|
|
6800
|
+
includeQueueState && fields.activeSlots !== void 0 && fields.maxSlots !== void 0 ? `slots=${fields.activeSlots}/${fields.maxSlots}` : void 0,
|
|
6801
|
+
includeQueueState && fields.pendingDepth !== void 0 && fields.maxPendingDepth !== void 0 ? `queue=${fields.pendingDepth}/${fields.maxPendingDepth}` : void 0,
|
|
6802
|
+
fields.remainingBudgetMs !== void 0 ? `budget=${formatDurationMs(fields.remainingBudgetMs)}` : void 0,
|
|
6803
|
+
`rid=${fields.requestId.slice(0, 8)}`
|
|
6804
|
+
].filter((value) => value !== void 0).join(" ");
|
|
6805
|
+
}
|
|
6806
|
+
function logRecoveryEvent(input, logger = consola) {
|
|
6807
|
+
if (logger === consola && isRoutineRecoveryEvent(input)) return;
|
|
6808
|
+
const callerRequestId = sanitizeCallerRequestId(input.callerRequestId);
|
|
6809
|
+
const fields = {
|
|
6810
|
+
requestId: input.requestId,
|
|
6811
|
+
...callerRequestId ? { callerRequestId } : {},
|
|
6812
|
+
event: input.event
|
|
6813
|
+
};
|
|
6814
|
+
for (const key of RECOVERY_EVENT_OPTIONAL_FIELDS) if (input[key] !== void 0) Object.assign(fields, { [key]: input[key] });
|
|
6815
|
+
if (logger === consola) {
|
|
6816
|
+
consola.info(formatRecoveryEventLine(fields));
|
|
6817
|
+
return;
|
|
6818
|
+
}
|
|
6819
|
+
logger.info("Upstream recovery", fields);
|
|
6820
|
+
}
|
|
6821
|
+
function setRequestModelMapping(request, info) {
|
|
6822
|
+
requestModelMapping.set(request, info);
|
|
6823
|
+
}
|
|
6824
|
+
function getRequestModelMapping(request) {
|
|
6825
|
+
return requestModelMapping.get(request);
|
|
6826
|
+
}
|
|
6827
|
+
/**
|
|
6828
|
+
* Format how long a request took, given the timestamp recorded at arrival.
|
|
6829
|
+
*
|
|
6830
|
+
* Renders `-` rather than a number when the start is missing or the arithmetic
|
|
6831
|
+
* is not finite. The caller is expected to supply a real start (see
|
|
6832
|
+
* {@link markRequestStart}); this is the shared-formatter backstop, so a future
|
|
6833
|
+
* code path that skips the `onRequest` hook degrades to an honest `-` instead
|
|
6834
|
+
* of printing `NaNs`.
|
|
6835
|
+
*/
|
|
6836
|
+
function formatElapsed(start) {
|
|
6837
|
+
if (start === void 0) return "-";
|
|
6838
|
+
const elapsed = Date.now() - start;
|
|
6839
|
+
return Number.isFinite(elapsed) ? formatDurationMs(elapsed) : "-";
|
|
6840
|
+
}
|
|
6841
|
+
function formatPath(rawUrl) {
|
|
6842
|
+
try {
|
|
6843
|
+
const url = new URL(rawUrl);
|
|
6844
|
+
return `${url.pathname}${url.search}`;
|
|
6845
|
+
} catch {
|
|
6846
|
+
return rawUrl;
|
|
6847
|
+
}
|
|
6848
|
+
}
|
|
6849
|
+
function colorizeStatus(status) {
|
|
6850
|
+
if (status >= 500) return colorize("red", status);
|
|
6851
|
+
if (status >= 400) return colorize("yellow", status);
|
|
6852
|
+
if (status >= 300) return colorize("cyan", status);
|
|
6853
|
+
return colorize("green", status);
|
|
6854
|
+
}
|
|
6855
|
+
const methodColors = {
|
|
6856
|
+
GET: "cyan",
|
|
6857
|
+
POST: "magenta",
|
|
6858
|
+
PUT: "yellow",
|
|
6859
|
+
PATCH: "yellow",
|
|
6860
|
+
DELETE: "red"
|
|
6861
|
+
};
|
|
6862
|
+
function colorizeMethod(method) {
|
|
6863
|
+
return colorize(methodColors[method] ?? "white", method);
|
|
6864
|
+
}
|
|
6865
|
+
function getEffectiveModel(info) {
|
|
6866
|
+
return info.steps.length > 0 ? info.steps.at(-1).to : info.originalModel ?? "-";
|
|
6867
|
+
}
|
|
6868
|
+
/**
|
|
6869
|
+
* Mutate `modelMapping` in place by appending a transform step.
|
|
6870
|
+
* Strategy contexts hold a reference to the same `modelMapping`,
|
|
6871
|
+
* so steps are pushed directly rather than returning a new object.
|
|
6872
|
+
*/
|
|
6873
|
+
function appendModelStepInPlace(info, tag, newModel) {
|
|
6874
|
+
const current = getEffectiveModel(info);
|
|
6875
|
+
if (newModel !== current) info.steps.push({
|
|
6876
|
+
tag,
|
|
6877
|
+
from: current,
|
|
6878
|
+
to: newModel
|
|
6879
|
+
});
|
|
6880
|
+
}
|
|
6881
|
+
function formatModelMapping(info) {
|
|
6882
|
+
if (!info) return "";
|
|
6883
|
+
const { originalModel, steps } = info;
|
|
6884
|
+
if (!originalModel && steps.length === 0) return "";
|
|
6885
|
+
const parts = [colorize("blueBright", originalModel ?? "-")];
|
|
6886
|
+
for (let i = 0; i < steps.length; i++) {
|
|
6887
|
+
const step = steps[i];
|
|
6888
|
+
const isLast = i === steps.length - 1;
|
|
6889
|
+
parts.push(colorize("dim", `-[${step.tag}]->`));
|
|
6890
|
+
parts.push(colorize(isLast ? "greenBright" : "cyanBright", step.to));
|
|
6891
|
+
}
|
|
6892
|
+
return ` ${colorize("dim", "model=")}${parts.join(" ")}`;
|
|
6893
|
+
}
|
|
6894
|
+
/**
|
|
6895
|
+
* Request logging function.
|
|
6896
|
+
* Logs a formatted request line with method, path, status, elapsed time,
|
|
6897
|
+
* and optional model mapping info.
|
|
6898
|
+
*/
|
|
6899
|
+
function logRequest(method, url, status, elapsed, modelInfo, requestId, callerRequestId) {
|
|
6900
|
+
const path = formatPath(url);
|
|
6901
|
+
const line = [
|
|
6902
|
+
colorize("dim", "<-"),
|
|
6903
|
+
colorizeMethod(method),
|
|
6904
|
+
colorize("white", path),
|
|
6905
|
+
colorizeStatus(status),
|
|
6906
|
+
colorize("dim", elapsed)
|
|
6907
|
+
].join(" ");
|
|
6908
|
+
const rid = requestId ? ` ${colorize("dim", `rid=${requestId.slice(0, 8)}`)}` : "";
|
|
6909
|
+
const safeCallerRequestId = sanitizeCallerRequestId(callerRequestId);
|
|
6910
|
+
const callerRid = safeCallerRequestId ? ` ${colorize("dim", `callerRid=${safeCallerRequestId}`)}` : "";
|
|
6911
|
+
console.log(`${line}${formatModelMapping(modelInfo)}${rid}${callerRid}`);
|
|
6912
|
+
}
|
|
6913
|
+
//#endregion
|
|
6914
|
+
//#region src/clients/upstream-queue.ts
|
|
6915
|
+
const DEFAULT_UPSTREAM_QUEUE_OPTIONS = {
|
|
6916
|
+
concurrency: 10,
|
|
6917
|
+
maxRetries: 1,
|
|
6918
|
+
baseDelayMs: 2e3,
|
|
6919
|
+
maxDelayMs: 6e4,
|
|
6920
|
+
maxQueueDepth: 1e3,
|
|
6921
|
+
recoveryBudgetMs: 60 * 1e3
|
|
6922
|
+
};
|
|
6923
|
+
const MAX_TIMER_DELAY_MS = 2147483647;
|
|
6924
|
+
const RETRY_AFTER_SECONDS_RE = /^\d+(?:\.\d+)?$/;
|
|
6925
|
+
const RETRY_AFTER_HTTP_DATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/;
|
|
6926
|
+
var TerminalUpstreamRecoveryError = class extends HTTPError {
|
|
6927
|
+
recovery;
|
|
6928
|
+
fallbackClaimed = false;
|
|
6929
|
+
constructor(source, recovery) {
|
|
6930
|
+
super(source.status, source.body, { headers: source.headers });
|
|
6931
|
+
this.name = "TerminalUpstreamRecoveryError";
|
|
6932
|
+
this.recovery = recovery;
|
|
6933
|
+
}
|
|
6934
|
+
claimFallback() {
|
|
6935
|
+
if (this.fallbackClaimed) return false;
|
|
6936
|
+
this.fallbackClaimed = true;
|
|
6937
|
+
return true;
|
|
6938
|
+
}
|
|
6939
|
+
};
|
|
6940
|
+
var LocalModelCooldownError = class extends TerminalUpstreamRecoveryError {
|
|
6941
|
+
constructor(recovery, retryAfter) {
|
|
6942
|
+
super(new HTTPError(529, { error: {
|
|
6943
|
+
message: "The selected upstream model is temporarily overloaded.",
|
|
6944
|
+
type: "overloaded_error"
|
|
6945
|
+
} }, { headers: { "retry-after": retryAfter } }), recovery);
|
|
6946
|
+
this.name = "LocalModelCooldownError";
|
|
6947
|
+
}
|
|
6948
|
+
};
|
|
6949
|
+
var FallbackCooldownError = class extends Error {
|
|
6950
|
+
scope;
|
|
6951
|
+
effectiveModel;
|
|
6952
|
+
constructor(cooldown) {
|
|
6953
|
+
super("Fallback target is locally cooled");
|
|
6954
|
+
this.name = "FallbackCooldownError";
|
|
6955
|
+
this.scope = cooldown.scope;
|
|
6956
|
+
this.effectiveModel = cooldown.effectiveModel;
|
|
6957
|
+
}
|
|
6958
|
+
};
|
|
6959
|
+
var UpstreamRequestQueue = class {
|
|
6960
|
+
sleep;
|
|
6961
|
+
now;
|
|
6962
|
+
wallNow;
|
|
6963
|
+
random;
|
|
6964
|
+
logger;
|
|
6965
|
+
setTimer;
|
|
6966
|
+
clearTimer;
|
|
6967
|
+
options;
|
|
6968
|
+
active = 0;
|
|
6969
|
+
accountNotBefore = 0;
|
|
6970
|
+
modelNotBefore = /* @__PURE__ */ new Map();
|
|
6971
|
+
drainTimer;
|
|
6972
|
+
drainTimerAt;
|
|
6973
|
+
waiters = [];
|
|
6974
|
+
terminalRecoveries = /* @__PURE__ */ new WeakSet();
|
|
6975
|
+
constructor(options = {}, deps = {}) {
|
|
6976
|
+
this.options = normalizeOptions(options);
|
|
6977
|
+
this.sleep = deps.sleep;
|
|
6978
|
+
this.now = deps.now ?? (() => performance.now());
|
|
6979
|
+
this.wallNow = deps.wallNow ?? Date.now;
|
|
6980
|
+
this.random = deps.random ?? Math.random;
|
|
6981
|
+
this.logger = deps.logger ?? consola;
|
|
6982
|
+
this.setTimer = deps.setTimeout ?? globalThis.setTimeout;
|
|
6983
|
+
this.clearTimer = deps.clearTimeout ?? globalThis.clearTimeout;
|
|
6984
|
+
}
|
|
6985
|
+
updateOptions(options) {
|
|
6986
|
+
this.options = normalizeOptions(mergeDefinedOptions(this.options, options));
|
|
6987
|
+
this.drain();
|
|
6988
|
+
}
|
|
6989
|
+
async dispatch(fetcher, inputContext, signal) {
|
|
6990
|
+
const recovery = inputContext.recovery ?? {
|
|
6991
|
+
requestId: crypto.randomUUID(),
|
|
6992
|
+
...signal ? { callerSignal: signal } : {},
|
|
6993
|
+
retryCount: 0
|
|
6994
|
+
};
|
|
6995
|
+
const context = {
|
|
6996
|
+
...inputContext,
|
|
6997
|
+
recovery
|
|
6998
|
+
};
|
|
6999
|
+
recovery.sourceModel ??= context.effectiveModel;
|
|
7000
|
+
signal?.throwIfAborted();
|
|
7001
|
+
this.throwIfFallbackCooled(context);
|
|
7002
|
+
const localCooldown = this.getActiveCooldown(context.effectiveModel);
|
|
7003
|
+
if ((typeof context.offerLocalModelCooldown === "function" ? Boolean(context.effectiveModel && context.offerLocalModelCooldown(context.effectiveModel)) : context.offerLocalModelCooldown) && localCooldown?.scope === "model" && localCooldown.notBeforeMonotonicMs > this.now()) {
|
|
7004
|
+
this.startRecovery(recovery);
|
|
7005
|
+
this.setRecoveryCooldown(recovery, localCooldown);
|
|
7006
|
+
const retryAfter = formatRetryAfter(localCooldown.notBeforeMonotonicMs - this.now());
|
|
7007
|
+
recovery.publicError = {
|
|
7008
|
+
status: 529,
|
|
7009
|
+
retryAfter
|
|
7010
|
+
};
|
|
7011
|
+
this.emitTerminal("retry", context, {
|
|
7012
|
+
retryCount: recovery.retryCount,
|
|
7013
|
+
status: 529,
|
|
7014
|
+
scope: "model",
|
|
7015
|
+
decision: "local-cooldown"
|
|
7016
|
+
});
|
|
7017
|
+
throw new LocalModelCooldownError(recovery, retryAfter);
|
|
7018
|
+
}
|
|
7019
|
+
try {
|
|
7020
|
+
return await this.runDispatch(fetcher, context, signal);
|
|
7021
|
+
} catch (error) {
|
|
7022
|
+
const connectionClass = isRetryableConnectionEstablishmentError(error);
|
|
7023
|
+
if (recovery.callerSignal?.aborted) this.emitTerminal(recovery.startedAtMonotonicMs === void 0 ? "admission" : "retry", context, {
|
|
7024
|
+
retryCount: recovery.retryCount,
|
|
7025
|
+
decision: "cancelled"
|
|
7026
|
+
});
|
|
7027
|
+
else if (signal?.aborted) this.emitTerminal("budget", context, {
|
|
7028
|
+
retryCount: recovery.retryCount,
|
|
7029
|
+
status: 504,
|
|
7030
|
+
decision: "deadline-exceeded"
|
|
7031
|
+
});
|
|
7032
|
+
else if (error instanceof HTTPError && error.status === 504 || this.remainingBudget(recovery) === 0) this.emitTerminal("budget", context, {
|
|
7033
|
+
retryCount: recovery.retryCount,
|
|
7034
|
+
status: error instanceof HTTPError ? error.status : void 0,
|
|
7035
|
+
decision: "deadline-exceeded"
|
|
7036
|
+
});
|
|
7037
|
+
else if (connectionClass && !this.canRetry(recovery)) this.emitTerminal("retry", context, {
|
|
7038
|
+
retryCount: recovery.retryCount,
|
|
7039
|
+
connectionClass,
|
|
7040
|
+
decision: "retry-exhausted"
|
|
7041
|
+
});
|
|
7042
|
+
else if (recovery.startedAtMonotonicMs !== void 0) this.emitTerminal("retry", context, {
|
|
7043
|
+
retryCount: recovery.retryCount,
|
|
7044
|
+
status: error instanceof HTTPError ? error.status : void 0,
|
|
7045
|
+
connectionClass,
|
|
7046
|
+
decision: "failed"
|
|
7047
|
+
});
|
|
7048
|
+
throw error;
|
|
7049
|
+
}
|
|
7050
|
+
}
|
|
7051
|
+
async runDispatch(fetcher, context, signal) {
|
|
7052
|
+
const { recovery } = context;
|
|
7053
|
+
let lastConnectionError;
|
|
7054
|
+
for (;;) {
|
|
7055
|
+
signal?.throwIfAborted();
|
|
7056
|
+
this.throwIfRecoveryExpired(recovery, lastConnectionError);
|
|
7057
|
+
const lease = await this.acquire(context, signal, lastConnectionError);
|
|
7058
|
+
let response;
|
|
7059
|
+
try {
|
|
7060
|
+
this.throwIfRecoveryExpired(recovery, lastConnectionError);
|
|
7061
|
+
if (context.fallbackAttempt) recovery.fallbackFetchStarted = true;
|
|
7062
|
+
response = await this.fetchBeforeDeadline(fetcher, signal, recovery, lastConnectionError);
|
|
7063
|
+
lastConnectionError = void 0;
|
|
7064
|
+
} catch (error) {
|
|
7065
|
+
lease.release();
|
|
7066
|
+
if (signal?.aborted) throw signal.reason;
|
|
7067
|
+
if (error instanceof RecoveryBudgetError) throw error.cause ?? error;
|
|
7068
|
+
const connectionClass = isRetryableConnectionEstablishmentError(error);
|
|
7069
|
+
if (!connectionClass || !context.retryable) throw error;
|
|
7070
|
+
this.startRecovery(recovery);
|
|
7071
|
+
if (!this.canRetry(recovery)) throw error;
|
|
7072
|
+
lastConnectionError = error;
|
|
7073
|
+
const delay = this.getBackoffDelay(recovery.retryCount, this.remainingBudget(recovery));
|
|
7074
|
+
recovery.retryCount++;
|
|
7075
|
+
this.emit("retry", context, {
|
|
7076
|
+
retryCount: recovery.retryCount,
|
|
7077
|
+
connectionClass,
|
|
7078
|
+
delaySource: "backoff",
|
|
7079
|
+
delayMs: delay,
|
|
7080
|
+
decision: "retry"
|
|
7081
|
+
});
|
|
7082
|
+
await this.waitForRecovery(delay, signal, recovery, error);
|
|
7083
|
+
continue;
|
|
7084
|
+
}
|
|
7085
|
+
try {
|
|
7086
|
+
const status = response.status;
|
|
7087
|
+
const scope = resolveCapacityCooldownScope(status, context.effectiveModel);
|
|
7088
|
+
const capacity = scope !== void 0;
|
|
7089
|
+
const mayReplay = context.retryable === "capacity" ? capacity : context.retryable === true && isTransientUpstreamStatus(status);
|
|
7090
|
+
let retryDelay;
|
|
7091
|
+
if (capacity) {
|
|
7092
|
+
this.startRecovery(recovery);
|
|
7093
|
+
retryDelay = this.getRetryDelay(response, recovery.retryCount, recovery);
|
|
7094
|
+
this.installCooldown(scope, context.effectiveModel, retryDelay.delayMs, context);
|
|
7095
|
+
recovery.publicError = {
|
|
7096
|
+
status,
|
|
7097
|
+
retryAfter: retryDelay.retryAfter ?? formatRetryAfter(retryDelay.delayMs)
|
|
7098
|
+
};
|
|
7099
|
+
}
|
|
7100
|
+
if (!mayReplay) return this.committed(response, lease, context, capacity ? retryDelay : void 0, capacity ? "capacity-terminal" : "upstream-terminal");
|
|
7101
|
+
this.startRecovery(recovery);
|
|
7102
|
+
if (!capacity) recovery.publicError = { status };
|
|
7103
|
+
retryDelay ??= this.getRetryDelay(response, recovery.retryCount, recovery);
|
|
7104
|
+
const remaining = this.remainingBudget(recovery);
|
|
7105
|
+
const serverMinimumDoesNotFit = retryDelay.source === "retry-after" && retryDelay.delayMs >= remaining;
|
|
7106
|
+
if (!this.canRetry(recovery) || serverMinimumDoesNotFit) {
|
|
7107
|
+
const decision = serverMinimumDoesNotFit ? "server-delay-exceeds-budget" : "retry-limit";
|
|
7108
|
+
this.emitTerminal("budget", context, {
|
|
7109
|
+
retryCount: recovery.retryCount,
|
|
7110
|
+
status,
|
|
7111
|
+
scope,
|
|
7112
|
+
delaySource: retryDelay.source,
|
|
7113
|
+
delayMs: retryDelay.delayMs,
|
|
7114
|
+
remainingBudgetMs: remaining,
|
|
7115
|
+
decision
|
|
7116
|
+
});
|
|
7117
|
+
return this.committed(response, lease, context, capacity ? retryDelay : void 0, decision);
|
|
7118
|
+
}
|
|
7119
|
+
discardResponse(response);
|
|
7120
|
+
lease.release();
|
|
7121
|
+
recovery.retryCount++;
|
|
7122
|
+
const statusLabel = status === 429 ? "rate limited (429)" : status === 529 ? "overloaded (529)" : String(status);
|
|
7123
|
+
this.logger.warn([
|
|
7124
|
+
`Upstream ${statusLabel};`,
|
|
7125
|
+
`retrying ${formatRequestContext(context)}`,
|
|
7126
|
+
`in ${formatDurationMs(retryDelay.delayMs)}`,
|
|
7127
|
+
`(attempt ${recovery.retryCount}/${recovery.retryLimit})`
|
|
7128
|
+
].join(" "));
|
|
7129
|
+
this.emit("retry", context, {
|
|
7130
|
+
retryCount: recovery.retryCount,
|
|
7131
|
+
status,
|
|
7132
|
+
scope,
|
|
7133
|
+
delaySource: retryDelay.source,
|
|
7134
|
+
delayMs: retryDelay.delayMs,
|
|
7135
|
+
decision: "retry"
|
|
7136
|
+
});
|
|
7137
|
+
await this.waitForRecovery(retryDelay.delayMs, signal, recovery);
|
|
7138
|
+
} catch (error) {
|
|
7139
|
+
discardResponse(response);
|
|
7140
|
+
lease.release();
|
|
7141
|
+
throw error;
|
|
7142
|
+
}
|
|
7143
|
+
}
|
|
7144
|
+
}
|
|
7145
|
+
async acquire(context, signal, causalError) {
|
|
7146
|
+
signal?.throwIfAborted();
|
|
7147
|
+
this.throwIfFallbackCooled(context);
|
|
7148
|
+
this.prepareCooldownWait(context);
|
|
7149
|
+
this.throwIfRecoveryExpired(context.recovery, causalError);
|
|
7150
|
+
const eligible = this.isEligible(context);
|
|
7151
|
+
if (this.active < this.options.concurrency && (eligible || this.drainTimerAt !== void 0 && this.drainTimerAt <= this.now())) {
|
|
7152
|
+
this.drain();
|
|
7153
|
+
if (eligible && this.active < this.options.concurrency) return this.grant(context, 0);
|
|
7154
|
+
}
|
|
7155
|
+
if (this.waiters.length >= this.options.maxQueueDepth) this.drain();
|
|
7156
|
+
if (this.waiters.length >= this.options.maxQueueDepth) {
|
|
7157
|
+
this.emit("admission", context, { decision: "queue-full" });
|
|
7158
|
+
throw new HTTPError(503, { error: {
|
|
7159
|
+
message: "Upstream queue full",
|
|
7160
|
+
type: "overloaded_error"
|
|
7161
|
+
} });
|
|
7162
|
+
}
|
|
7163
|
+
return new Promise((resolve, reject) => {
|
|
7164
|
+
const waiter = {
|
|
7165
|
+
context,
|
|
7166
|
+
causalError,
|
|
7167
|
+
enqueuedAt: this.now(),
|
|
7168
|
+
resolve,
|
|
7169
|
+
reject,
|
|
7170
|
+
signal
|
|
7171
|
+
};
|
|
7172
|
+
if (signal) {
|
|
7173
|
+
waiter.onAbort = () => {
|
|
7174
|
+
const index = this.waiters.indexOf(waiter);
|
|
7175
|
+
if (index === -1) return;
|
|
7176
|
+
this.waiters.splice(index, 1);
|
|
7177
|
+
reject(signal.reason);
|
|
7178
|
+
if (waiter.wakeAt === this.drainTimerAt && !this.waiters.some((candidate) => candidate.wakeAt === waiter.wakeAt)) this.scheduleNextWake();
|
|
7179
|
+
};
|
|
7180
|
+
signal.addEventListener("abort", waiter.onAbort, { once: true });
|
|
7181
|
+
}
|
|
7182
|
+
this.waiters.push(waiter);
|
|
7183
|
+
this.emit("admission", context, { decision: "queued" });
|
|
7184
|
+
this.scheduleNextWake(waiter);
|
|
7185
|
+
});
|
|
7186
|
+
}
|
|
7187
|
+
prepareCooldownWait(context) {
|
|
7188
|
+
const cooldown = this.getActiveCooldown(context.effectiveModel);
|
|
7189
|
+
if (!cooldown) return;
|
|
7190
|
+
this.startRecovery(context.recovery);
|
|
7191
|
+
this.setRecoveryCooldown(context.recovery, cooldown);
|
|
7192
|
+
const status = cooldown.scope === "account" ? 429 : 529;
|
|
7193
|
+
context.recovery.publicError = {
|
|
7194
|
+
status,
|
|
7195
|
+
retryAfter: formatRetryAfter(cooldown.notBeforeMonotonicMs - this.now())
|
|
7196
|
+
};
|
|
7197
|
+
if (cooldown.notBeforeMonotonicMs > context.recovery.deadlineMonotonicMs) throw createLocalCapacityError(context.recovery);
|
|
7198
|
+
}
|
|
7199
|
+
drain() {
|
|
7200
|
+
this.clearExpiredModels();
|
|
7201
|
+
for (let index = this.waiters.length - 1; index >= 0; index--) {
|
|
7202
|
+
const waiter = this.waiters[index];
|
|
7203
|
+
const fallbackCooldown = waiter.context.fallbackAttempt ? this.getActiveCooldown(waiter.context.effectiveModel) : void 0;
|
|
7204
|
+
if (fallbackCooldown) {
|
|
7205
|
+
this.waiters.splice(index, 1);
|
|
7206
|
+
this.cleanupWaiter(waiter);
|
|
7207
|
+
this.emit("admission", waiter.context, {
|
|
7208
|
+
scope: fallbackCooldown.scope,
|
|
7209
|
+
decision: "fallback-cooldown"
|
|
7210
|
+
});
|
|
7211
|
+
waiter.reject(new FallbackCooldownError(fallbackCooldown));
|
|
7212
|
+
continue;
|
|
7213
|
+
}
|
|
7214
|
+
const deadline = waiter.context.recovery.deadlineMonotonicMs;
|
|
7215
|
+
if (deadline !== void 0 && this.now() >= deadline) {
|
|
7216
|
+
this.waiters.splice(index, 1);
|
|
7217
|
+
this.cleanupWaiter(waiter);
|
|
7218
|
+
waiter.reject(waiter.causalError ?? createLocalCapacityError(waiter.context.recovery));
|
|
7219
|
+
}
|
|
7220
|
+
}
|
|
7221
|
+
while (this.active < this.options.concurrency) {
|
|
7222
|
+
const index = this.waiters.findIndex((waiter) => this.isEligible(waiter.context));
|
|
7223
|
+
if (index === -1) break;
|
|
7224
|
+
const waiter = this.waiters.splice(index, 1)[0];
|
|
7225
|
+
this.cleanupWaiter(waiter);
|
|
7226
|
+
waiter.resolve(this.grant(waiter.context, this.now() - waiter.enqueuedAt));
|
|
7227
|
+
}
|
|
7228
|
+
this.scheduleNextWake();
|
|
7229
|
+
}
|
|
7230
|
+
scheduleNextWake(addedWaiter) {
|
|
7231
|
+
const now = this.now();
|
|
7232
|
+
if (addedWaiter) {
|
|
7233
|
+
const wakeAt = this.getWaiterWakeAt(addedWaiter, now);
|
|
7234
|
+
addedWaiter.wakeAt = wakeAt;
|
|
7235
|
+
if (wakeAt === void 0 || this.drainTimerAt !== void 0 && wakeAt >= this.drainTimerAt) return;
|
|
7236
|
+
this.replaceDrainTimer(wakeAt, now);
|
|
7237
|
+
return;
|
|
7238
|
+
}
|
|
7239
|
+
let wakeAt;
|
|
7240
|
+
for (const waiter of this.waiters) {
|
|
7241
|
+
waiter.wakeAt = this.getWaiterWakeAt(waiter, now);
|
|
7242
|
+
if (waiter.wakeAt !== void 0) wakeAt = Math.min(wakeAt ?? Number.POSITIVE_INFINITY, waiter.wakeAt);
|
|
7243
|
+
}
|
|
7244
|
+
this.replaceDrainTimer(wakeAt, now);
|
|
7245
|
+
}
|
|
7246
|
+
getWaiterWakeAt(waiter, now) {
|
|
7247
|
+
let wakeAt;
|
|
7248
|
+
const cooldown = this.getActiveCooldown(waiter.context.effectiveModel);
|
|
7249
|
+
if (cooldown && cooldown.notBeforeMonotonicMs > now) wakeAt = cooldown.notBeforeMonotonicMs;
|
|
7250
|
+
const deadline = waiter.context.recovery.deadlineMonotonicMs;
|
|
7251
|
+
if (deadline !== void 0 && deadline > now) wakeAt = Math.min(wakeAt ?? Number.POSITIVE_INFINITY, deadline);
|
|
7252
|
+
return wakeAt;
|
|
7253
|
+
}
|
|
7254
|
+
replaceDrainTimer(wakeAt, now) {
|
|
7255
|
+
if (wakeAt === this.drainTimerAt) return;
|
|
7256
|
+
if (this.drainTimer) {
|
|
7257
|
+
this.clearTimer(this.drainTimer);
|
|
7258
|
+
this.drainTimer = void 0;
|
|
7259
|
+
this.drainTimerAt = void 0;
|
|
7260
|
+
}
|
|
7261
|
+
if (wakeAt === void 0) return;
|
|
7262
|
+
this.drainTimerAt = wakeAt;
|
|
7263
|
+
this.drainTimer = this.setTimer(() => {
|
|
7264
|
+
this.drainTimer = void 0;
|
|
7265
|
+
this.drainTimerAt = void 0;
|
|
7266
|
+
this.drain();
|
|
7267
|
+
}, Math.min(MAX_TIMER_DELAY_MS, Math.max(0, wakeAt - now)));
|
|
7268
|
+
}
|
|
7269
|
+
grant(context, queueWaitMs) {
|
|
7270
|
+
let released = false;
|
|
7271
|
+
this.active++;
|
|
7272
|
+
this.emit("grant", context, {
|
|
7273
|
+
queueWaitMs,
|
|
7274
|
+
decision: "granted"
|
|
7275
|
+
});
|
|
7276
|
+
return { release: () => {
|
|
7277
|
+
if (released) return;
|
|
7278
|
+
released = true;
|
|
7279
|
+
this.active--;
|
|
7280
|
+
this.drain();
|
|
7281
|
+
} };
|
|
7282
|
+
}
|
|
7283
|
+
isEligible(context) {
|
|
7284
|
+
return this.getActiveCooldown(context.effectiveModel) === void 0;
|
|
7285
|
+
}
|
|
7286
|
+
throwIfFallbackCooled(context) {
|
|
7287
|
+
if (!context.fallbackAttempt) return;
|
|
7288
|
+
const cooldown = this.getActiveCooldown(context.effectiveModel);
|
|
7289
|
+
if (cooldown) throw new FallbackCooldownError(cooldown);
|
|
7290
|
+
}
|
|
7291
|
+
getActiveCooldown(effectiveModel) {
|
|
7292
|
+
const now = this.now();
|
|
7293
|
+
if (this.accountNotBefore > now) return {
|
|
7294
|
+
scope: "account",
|
|
7295
|
+
notBeforeMonotonicMs: this.accountNotBefore
|
|
7296
|
+
};
|
|
7297
|
+
if (!effectiveModel) return void 0;
|
|
7298
|
+
const modelDeadline = this.modelNotBefore.get(effectiveModel);
|
|
7299
|
+
if (modelDeadline === void 0) return void 0;
|
|
7300
|
+
if (modelDeadline <= now) {
|
|
7301
|
+
this.modelNotBefore.delete(effectiveModel);
|
|
7302
|
+
return;
|
|
7303
|
+
}
|
|
7304
|
+
return {
|
|
7305
|
+
scope: "model",
|
|
7306
|
+
notBeforeMonotonicMs: modelDeadline,
|
|
7307
|
+
effectiveModel
|
|
7308
|
+
};
|
|
7309
|
+
}
|
|
7310
|
+
clearExpiredModels() {
|
|
7311
|
+
const now = this.now();
|
|
7312
|
+
for (const [model, deadline] of this.modelNotBefore) if (deadline <= now) this.modelNotBefore.delete(model);
|
|
7313
|
+
}
|
|
7314
|
+
installCooldown(scope, effectiveModel, delayMs, context) {
|
|
7315
|
+
const deadline = this.now() + delayMs;
|
|
7316
|
+
let stored = deadline;
|
|
7317
|
+
if (scope === "account") {
|
|
7318
|
+
this.accountNotBefore = Math.max(this.accountNotBefore, deadline);
|
|
7319
|
+
stored = this.accountNotBefore;
|
|
7320
|
+
} else if (scope === "model" && effectiveModel) {
|
|
7321
|
+
stored = Math.max(this.modelNotBefore.get(effectiveModel) ?? 0, deadline);
|
|
7322
|
+
this.modelNotBefore.set(effectiveModel, stored);
|
|
7323
|
+
}
|
|
7324
|
+
this.setRecoveryCooldown(context.recovery, {
|
|
7325
|
+
scope,
|
|
7326
|
+
notBeforeMonotonicMs: stored,
|
|
7327
|
+
...effectiveModel && scope === "model" ? { effectiveModel } : {}
|
|
7328
|
+
});
|
|
7329
|
+
this.emit("cooldown", context, {
|
|
7330
|
+
scope,
|
|
7331
|
+
delayMs,
|
|
7332
|
+
nextRetryAt: formatNextRetryAt(this.wallNow() + delayMs),
|
|
7333
|
+
decision: scope === "request" ? "request-local" : "installed"
|
|
7334
|
+
});
|
|
7335
|
+
this.drain();
|
|
7336
|
+
}
|
|
7337
|
+
startRecovery(recovery) {
|
|
7338
|
+
if (recovery.deadlineMonotonicMs !== void 0) return;
|
|
7339
|
+
const startedAt = this.now();
|
|
7340
|
+
recovery.startedAtMonotonicMs = startedAt;
|
|
7341
|
+
recovery.deadlineMonotonicMs = startedAt + this.options.recoveryBudgetMs;
|
|
7342
|
+
recovery.retryLimit ??= this.options.maxRetries;
|
|
7343
|
+
}
|
|
7344
|
+
setRecoveryCooldown(recovery, cooldown) {
|
|
7345
|
+
if (!recovery.cooldown || cooldown.notBeforeMonotonicMs >= recovery.cooldown.notBeforeMonotonicMs) recovery.cooldown = cooldown;
|
|
7346
|
+
}
|
|
7347
|
+
canRetry(recovery) {
|
|
7348
|
+
return recovery.retryCount < (recovery.retryLimit ?? this.options.maxRetries);
|
|
7349
|
+
}
|
|
7350
|
+
remainingBudget(recovery) {
|
|
7351
|
+
return recovery.deadlineMonotonicMs === void 0 ? this.options.recoveryBudgetMs : Math.max(0, recovery.deadlineMonotonicMs - this.now());
|
|
7352
|
+
}
|
|
7353
|
+
throwIfRecoveryExpired(recovery, lastConnectionError) {
|
|
7354
|
+
if (recovery.deadlineMonotonicMs !== void 0 && this.now() >= recovery.deadlineMonotonicMs) throw lastConnectionError ?? createLocalCapacityError(recovery);
|
|
7355
|
+
}
|
|
7356
|
+
getRetryDelay(response, attempt, recovery) {
|
|
7357
|
+
const retryAfterMs = parseRetryAfterMs(response.headers, this.wallNow());
|
|
7358
|
+
const retryAfter = response.headers.get("retry-after") ?? void 0;
|
|
7359
|
+
if (retryAfterMs !== void 0) return {
|
|
7360
|
+
delayMs: retryAfterMs,
|
|
7361
|
+
source: "retry-after",
|
|
7362
|
+
retryAfter
|
|
7363
|
+
};
|
|
7364
|
+
return {
|
|
7365
|
+
delayMs: this.getBackoffDelay(attempt, this.remainingBudget(recovery)),
|
|
7366
|
+
source: "backoff"
|
|
7367
|
+
};
|
|
7368
|
+
}
|
|
7369
|
+
getBackoffDelay(attempt, remainingBudgetMs) {
|
|
7370
|
+
const cap = Math.min(this.options.baseDelayMs * 2 ** attempt, this.options.maxDelayMs, Math.max(0, remainingBudgetMs));
|
|
7371
|
+
const random = Math.min(1, Math.max(0, this.random()));
|
|
7372
|
+
return Math.floor(cap * random);
|
|
7373
|
+
}
|
|
7374
|
+
async waitForRecovery(delayMs, signal, recovery, lastConnectionError) {
|
|
7375
|
+
const remaining = this.remainingBudget(recovery);
|
|
7376
|
+
if (delayMs > remaining) throw lastConnectionError ?? createLocalCapacityError(recovery);
|
|
7377
|
+
const deadline = createDeadlineSignal(signal, remaining, this.setTimer, this.clearTimer);
|
|
7378
|
+
try {
|
|
7379
|
+
await abortableSleep(this.sleep, delayMs, deadline.signal, this.setTimer, this.clearTimer);
|
|
7380
|
+
} catch (error) {
|
|
7381
|
+
if (signal?.aborted) throw signal.reason;
|
|
7382
|
+
if (deadline.timedOut()) throw lastConnectionError ?? createLocalCapacityError(recovery);
|
|
7383
|
+
throw error;
|
|
7384
|
+
} finally {
|
|
7385
|
+
deadline.cleanup();
|
|
7386
|
+
}
|
|
7387
|
+
this.throwIfRecoveryExpired(recovery, lastConnectionError);
|
|
7388
|
+
}
|
|
7389
|
+
async fetchBeforeDeadline(fetcher, signal, recovery, lastConnectionError) {
|
|
7390
|
+
if (recovery.deadlineMonotonicMs === void 0) return fetcher(signal);
|
|
7391
|
+
const deadline = createDeadlineSignal(signal, this.remainingBudget(recovery), this.setTimer, this.clearTimer);
|
|
7392
|
+
try {
|
|
7393
|
+
const response = await fetcher(deadline.signal);
|
|
7394
|
+
if (this.now() >= recovery.deadlineMonotonicMs) {
|
|
7395
|
+
discardResponse(response);
|
|
7396
|
+
throw new RecoveryBudgetError(recovery.fallbackFetchStarted ? createRecoveryTimeoutError() : lastConnectionError ?? createLocalCapacityError(recovery));
|
|
7397
|
+
}
|
|
7398
|
+
return response;
|
|
7399
|
+
} catch (error) {
|
|
7400
|
+
if (signal?.aborted) throw signal.reason;
|
|
7401
|
+
if (deadline.timedOut()) throw new RecoveryBudgetError(recovery.fallbackFetchStarted ? createRecoveryTimeoutError() : lastConnectionError ?? createLocalCapacityError(recovery));
|
|
7402
|
+
throw error;
|
|
7403
|
+
} finally {
|
|
7404
|
+
deadline.cleanup();
|
|
7405
|
+
}
|
|
7406
|
+
}
|
|
7407
|
+
committed(response, lease, context, retryDelay, terminalDecision = "upstream-terminal") {
|
|
7408
|
+
const { recovery } = context;
|
|
7409
|
+
if (recovery.startedAtMonotonicMs !== void 0) this.emitTerminal("retry", context, {
|
|
7410
|
+
retryCount: recovery.retryCount,
|
|
7411
|
+
status: response.status,
|
|
7412
|
+
scope: resolveCapacityCooldownScope(response.status, context.effectiveModel),
|
|
7413
|
+
decision: response.ok ? "recovered" : terminalDecision
|
|
7414
|
+
});
|
|
7415
|
+
return {
|
|
7416
|
+
response: retryDelay ? ensureRetryAfter(response, retryDelay.retryAfter ?? formatRetryAfter(retryDelay.delayMs)) : response,
|
|
7417
|
+
release: lease.release,
|
|
7418
|
+
recovery
|
|
7419
|
+
};
|
|
7420
|
+
}
|
|
7421
|
+
emit(event, context, fields) {
|
|
7422
|
+
const recovery = context.recovery;
|
|
7423
|
+
recovery.queueMetrics = {
|
|
7424
|
+
activeSlots: this.active,
|
|
7425
|
+
maxSlots: this.options.concurrency,
|
|
7426
|
+
pendingDepth: this.waiters.length,
|
|
7427
|
+
maxPendingDepth: this.options.maxQueueDepth
|
|
7428
|
+
};
|
|
7429
|
+
if (!this.logger.info) return;
|
|
7430
|
+
const eventFields = {
|
|
7431
|
+
requestId: recovery.requestId,
|
|
7432
|
+
callerRequestId: recovery.callerRequestId,
|
|
7433
|
+
event,
|
|
7434
|
+
effectiveModel: context.effectiveModel,
|
|
7435
|
+
...recovery.queueMetrics,
|
|
7436
|
+
...recovery.startedAtMonotonicMs !== void 0 ? {
|
|
7437
|
+
elapsedMs: Math.max(0, this.now() - recovery.startedAtMonotonicMs),
|
|
7438
|
+
remainingBudgetMs: this.remainingBudget(recovery)
|
|
7439
|
+
} : {},
|
|
7440
|
+
...fields
|
|
7441
|
+
};
|
|
7442
|
+
if (this.logger === consola) logRecoveryEvent(eventFields);
|
|
7443
|
+
else logRecoveryEvent(eventFields, { info: this.logger.info.bind(this.logger) });
|
|
7444
|
+
}
|
|
7445
|
+
emitTerminal(event, context, fields) {
|
|
7446
|
+
if (this.terminalRecoveries.has(context.recovery)) return;
|
|
7447
|
+
this.terminalRecoveries.add(context.recovery);
|
|
7448
|
+
this.emit(event, context, fields);
|
|
7449
|
+
}
|
|
7450
|
+
cleanupWaiter(waiter) {
|
|
7451
|
+
if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
|
|
7452
|
+
}
|
|
7453
|
+
};
|
|
7454
|
+
function createDefaultUpstreamRequestQueue() {
|
|
7455
|
+
return new UpstreamRequestQueue(DEFAULT_UPSTREAM_QUEUE_OPTIONS);
|
|
7456
|
+
}
|
|
7457
|
+
function parseRetryAfterMs(headers, now = Date.now()) {
|
|
7458
|
+
const value = headers.get("retry-after");
|
|
7459
|
+
if (!value) return void 0;
|
|
7460
|
+
if (RETRY_AFTER_SECONDS_RE.test(value)) {
|
|
7461
|
+
const milliseconds = Number(value) * 1e3;
|
|
7462
|
+
return Number.isFinite(milliseconds) ? Math.ceil(milliseconds) : void 0;
|
|
7463
|
+
}
|
|
7464
|
+
if (!RETRY_AFTER_HTTP_DATE_RE.test(value)) return void 0;
|
|
7465
|
+
const retryAt = Date.parse(value);
|
|
7466
|
+
if (Number.isNaN(retryAt)) return void 0;
|
|
7467
|
+
const serverDate = headers.get("date");
|
|
7468
|
+
const parsedServerDate = serverDate ? Date.parse(serverDate) : NaN;
|
|
7469
|
+
return Math.max(0, retryAt - (Number.isNaN(parsedServerDate) ? now : parsedServerDate));
|
|
7470
|
+
}
|
|
7471
|
+
function finiteOr(value, fallback) {
|
|
7472
|
+
return value !== void 0 && Number.isFinite(value) ? value : fallback;
|
|
7473
|
+
}
|
|
7474
|
+
function normalizeOptions(options) {
|
|
7475
|
+
return {
|
|
7476
|
+
concurrency: Math.max(1, Math.floor(finiteOr(options.concurrency, DEFAULT_UPSTREAM_QUEUE_OPTIONS.concurrency))),
|
|
7477
|
+
maxRetries: Math.min(2, Math.max(0, Math.floor(finiteOr(options.maxRetries, DEFAULT_UPSTREAM_QUEUE_OPTIONS.maxRetries)))),
|
|
7478
|
+
baseDelayMs: Math.max(0, Math.floor(finiteOr(options.baseDelayMs, DEFAULT_UPSTREAM_QUEUE_OPTIONS.baseDelayMs))),
|
|
7479
|
+
maxDelayMs: Math.max(1, Math.floor(finiteOr(options.maxDelayMs, DEFAULT_UPSTREAM_QUEUE_OPTIONS.maxDelayMs))),
|
|
7480
|
+
maxQueueDepth: Math.max(1, Math.floor(finiteOr(options.maxQueueDepth, DEFAULT_UPSTREAM_QUEUE_OPTIONS.maxQueueDepth))),
|
|
7481
|
+
recoveryBudgetMs: Math.min(120 * 1e3, Math.max(1 * 1e3, Math.floor(finiteOr(options.recoveryBudgetMs, DEFAULT_UPSTREAM_QUEUE_OPTIONS.recoveryBudgetMs))))
|
|
7482
|
+
};
|
|
7483
|
+
}
|
|
7484
|
+
function mergeDefinedOptions(current, next) {
|
|
7485
|
+
return {
|
|
7486
|
+
concurrency: next.concurrency ?? current.concurrency,
|
|
7487
|
+
maxRetries: next.maxRetries ?? current.maxRetries,
|
|
7488
|
+
baseDelayMs: next.baseDelayMs ?? current.baseDelayMs,
|
|
7489
|
+
maxDelayMs: next.maxDelayMs ?? current.maxDelayMs,
|
|
7490
|
+
maxQueueDepth: next.maxQueueDepth ?? current.maxQueueDepth,
|
|
7491
|
+
recoveryBudgetMs: next.recoveryBudgetMs ?? current.recoveryBudgetMs
|
|
7492
|
+
};
|
|
7493
|
+
}
|
|
7494
|
+
function discardResponse(response) {
|
|
7495
|
+
try {
|
|
7496
|
+
response.body?.cancel().catch(() => {});
|
|
7497
|
+
} catch {}
|
|
7498
|
+
}
|
|
7499
|
+
function ensureRetryAfter(response, retryAfter) {
|
|
7500
|
+
if (response.headers.get("retry-after") === retryAfter) return response;
|
|
7501
|
+
const headers = new Headers(response.headers);
|
|
7502
|
+
headers.set("retry-after", retryAfter);
|
|
7503
|
+
return new Response(response.body, {
|
|
7504
|
+
status: response.status,
|
|
7505
|
+
statusText: response.statusText,
|
|
7506
|
+
headers
|
|
7507
|
+
});
|
|
7508
|
+
}
|
|
7509
|
+
function formatRetryAfter(delayMs) {
|
|
7510
|
+
return String(Math.max(0, Math.ceil(delayMs / 1e3)));
|
|
7511
|
+
}
|
|
7512
|
+
function formatNextRetryAt(timestampMs) {
|
|
7513
|
+
const retryAt = new Date(timestampMs);
|
|
7514
|
+
return Number.isNaN(retryAt.getTime()) ? void 0 : retryAt.toISOString();
|
|
7515
|
+
}
|
|
7516
|
+
function formatRequestContext(context) {
|
|
7517
|
+
try {
|
|
7518
|
+
const url = new URL(context.url);
|
|
7519
|
+
return `${context.method ?? "GET"} ${url.pathname}`;
|
|
7520
|
+
} catch {
|
|
7521
|
+
return `${context.method ?? "GET"} ${context.url}`;
|
|
7522
|
+
}
|
|
7523
|
+
}
|
|
7524
|
+
function abortableSleep(sleep, ms, signal, setTimer = globalThis.setTimeout, clearTimer = globalThis.clearTimeout) {
|
|
7525
|
+
if (sleep && !signal) return sleep(ms);
|
|
7526
|
+
signal?.throwIfAborted();
|
|
7527
|
+
return new Promise((resolve, reject) => {
|
|
7528
|
+
let timer;
|
|
7529
|
+
function cleanup() {
|
|
7530
|
+
if (timer !== void 0) clearTimer(timer);
|
|
7531
|
+
signal?.removeEventListener("abort", onAbort);
|
|
7532
|
+
}
|
|
7533
|
+
function onAbort() {
|
|
7534
|
+
cleanup();
|
|
7535
|
+
reject(signal?.reason);
|
|
7536
|
+
}
|
|
7537
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
7538
|
+
if (!sleep) {
|
|
7539
|
+
timer = setTimer(() => {
|
|
7540
|
+
cleanup();
|
|
7541
|
+
resolve();
|
|
7542
|
+
}, ms);
|
|
7543
|
+
return;
|
|
7544
|
+
}
|
|
7545
|
+
sleep(ms).then(() => {
|
|
7546
|
+
cleanup();
|
|
7547
|
+
resolve();
|
|
7548
|
+
}, (error) => {
|
|
7549
|
+
cleanup();
|
|
7550
|
+
reject(error);
|
|
7551
|
+
});
|
|
7552
|
+
});
|
|
7553
|
+
}
|
|
7554
|
+
function createDeadlineSignal(parent, delayMs, setTimer, clearTimer) {
|
|
7555
|
+
const controller = new AbortController();
|
|
7556
|
+
let didTimeOut = false;
|
|
7557
|
+
const timer = setTimer(() => {
|
|
7558
|
+
didTimeOut = true;
|
|
7559
|
+
controller.abort(new DOMException("Recovery deadline exceeded", "TimeoutError"));
|
|
7560
|
+
}, Math.max(0, delayMs));
|
|
7561
|
+
return {
|
|
7562
|
+
signal: parent ? AbortSignal.any([parent, controller.signal]) : controller.signal,
|
|
7563
|
+
timedOut: () => didTimeOut,
|
|
7564
|
+
cleanup: () => clearTimer(timer)
|
|
7565
|
+
};
|
|
7566
|
+
}
|
|
7567
|
+
var RecoveryBudgetError = class extends Error {
|
|
7568
|
+
cause;
|
|
7569
|
+
constructor(cause) {
|
|
7570
|
+
super("Upstream recovery deadline exceeded");
|
|
7571
|
+
this.name = "RecoveryBudgetError";
|
|
7572
|
+
this.cause = cause;
|
|
7573
|
+
}
|
|
7574
|
+
};
|
|
7575
|
+
function createRecoveryTimeoutError() {
|
|
7576
|
+
return new HTTPError(504, { error: {
|
|
7577
|
+
message: localCapacityErrorMessage(504),
|
|
7578
|
+
type: "timeout_error"
|
|
7579
|
+
} });
|
|
7580
|
+
}
|
|
7581
|
+
function createLocalCapacityError(recovery) {
|
|
7582
|
+
const status = recovery.publicError?.status ?? 504;
|
|
7583
|
+
const retryAfter = recovery.publicError?.retryAfter ?? (recovery.cooldown ? formatRetryAfter(recovery.cooldown.notBeforeMonotonicMs - (recovery.deadlineMonotonicMs ?? 0)) : void 0);
|
|
7584
|
+
const errorType = status === 504 ? "timeout_error" : upstreamErrorType(status);
|
|
7585
|
+
const error = new HTTPError(status, { error: {
|
|
7586
|
+
message: localCapacityErrorMessage(status),
|
|
7587
|
+
type: errorType
|
|
7588
|
+
} }, retryAfter ? { headers: { "retry-after": retryAfter } } : void 0);
|
|
7589
|
+
return status === 529 ? new TerminalUpstreamRecoveryError(error, recovery) : error;
|
|
7590
|
+
}
|
|
7591
|
+
function localCapacityErrorMessage(status) {
|
|
7592
|
+
switch (status) {
|
|
7593
|
+
case 429: return "The upstream account is temporarily rate limited.";
|
|
7594
|
+
case 529: return "The selected upstream model is temporarily overloaded.";
|
|
7595
|
+
case 504: return "The upstream recovery budget was exhausted.";
|
|
7596
|
+
default: return `The last upstream attempt failed with status ${status}.`;
|
|
7597
|
+
}
|
|
7598
|
+
}
|
|
7599
|
+
//#endregion
|
|
6582
7600
|
//#region src/clients/copilot-client.ts
|
|
6583
7601
|
var CopilotClient = class {
|
|
6584
7602
|
auth;
|
|
6585
7603
|
config;
|
|
6586
7604
|
fetchImpl;
|
|
6587
7605
|
requestQueue;
|
|
7606
|
+
recovery;
|
|
7607
|
+
offerLocalModelCooldown;
|
|
7608
|
+
fallbackAttempt;
|
|
6588
7609
|
constructor(auth, config, deps) {
|
|
6589
7610
|
this.auth = auth;
|
|
6590
7611
|
this.config = config;
|
|
6591
7612
|
this.fetchImpl = deps?.fetch ?? fetch;
|
|
6592
7613
|
this.requestQueue = deps?.requestQueue;
|
|
7614
|
+
this.recovery = deps?.recovery;
|
|
7615
|
+
this.offerLocalModelCooldown = deps?.offerLocalModelCooldown ?? false;
|
|
7616
|
+
this.fallbackAttempt = deps?.fallbackAttempt ?? false;
|
|
6593
7617
|
}
|
|
6594
7618
|
requireToken() {
|
|
6595
7619
|
if (!this.auth.copilotToken) throw new Error("Copilot token not found");
|
|
@@ -6610,10 +7634,13 @@ var CopilotClient = class {
|
|
|
6610
7634
|
signal: options.signal
|
|
6611
7635
|
}
|
|
6612
7636
|
};
|
|
6613
|
-
const queuedResponse = await this.fetchWithQueue(request, options.retryable);
|
|
7637
|
+
const queuedResponse = await this.fetchWithQueue(request, options.retryable, options.effectiveModel);
|
|
6614
7638
|
const { response } = queuedResponse;
|
|
6615
7639
|
if (!response.ok) try {
|
|
6616
7640
|
await throwUpstreamError(errorMessage, response);
|
|
7641
|
+
} catch (error) {
|
|
7642
|
+
if (error instanceof HTTPError && response.status === 529 && queuedResponse.recovery) throw new TerminalUpstreamRecoveryError(error, queuedResponse.recovery);
|
|
7643
|
+
throw error;
|
|
6617
7644
|
} finally {
|
|
6618
7645
|
queuedResponse.release();
|
|
6619
7646
|
}
|
|
@@ -6634,6 +7661,7 @@ var CopilotClient = class {
|
|
|
6634
7661
|
method: "POST",
|
|
6635
7662
|
body: JSON.stringify(payload),
|
|
6636
7663
|
retryable: "capacity",
|
|
7664
|
+
effectiveModel: typeof payload.model === "string" ? payload.model : void 0,
|
|
6637
7665
|
...options
|
|
6638
7666
|
});
|
|
6639
7667
|
if (payload.stream) return withRelease(events(response), release);
|
|
@@ -6643,16 +7671,28 @@ var CopilotClient = class {
|
|
|
6643
7671
|
release();
|
|
6644
7672
|
}
|
|
6645
7673
|
}
|
|
6646
|
-
async fetchWithQueue(request, retryable) {
|
|
6647
|
-
const fetcher = () => this.fetchImpl(request.url,
|
|
7674
|
+
async fetchWithQueue(request, retryable, effectiveModel) {
|
|
7675
|
+
const fetcher = (signal) => this.fetchImpl(request.url, {
|
|
7676
|
+
...request.init,
|
|
7677
|
+
signal: signal ?? request.init.signal
|
|
7678
|
+
});
|
|
6648
7679
|
if (this.requestQueue) return this.requestQueue.dispatch(fetcher, {
|
|
6649
7680
|
method: request.init.method,
|
|
6650
7681
|
url: request.url,
|
|
6651
|
-
retryable
|
|
7682
|
+
retryable,
|
|
7683
|
+
effectiveModel,
|
|
7684
|
+
recovery: this.recovery,
|
|
7685
|
+
offerLocalModelCooldown: this.offerLocalModelCooldown,
|
|
7686
|
+
fallbackAttempt: this.fallbackAttempt
|
|
6652
7687
|
}, request.init.signal ?? void 0);
|
|
7688
|
+
const recovery = this.recovery ?? {
|
|
7689
|
+
requestId: crypto.randomUUID(),
|
|
7690
|
+
retryCount: 0
|
|
7691
|
+
};
|
|
6653
7692
|
return {
|
|
6654
|
-
response: await fetcher(),
|
|
6655
|
-
release: () => {}
|
|
7693
|
+
response: await fetcher(request.init.signal ?? void 0),
|
|
7694
|
+
release: () => {},
|
|
7695
|
+
recovery
|
|
6656
7696
|
};
|
|
6657
7697
|
}
|
|
6658
7698
|
async createChatCompletions(payload, options) {
|
|
@@ -6672,6 +7712,7 @@ var CopilotClient = class {
|
|
|
6672
7712
|
method: "POST",
|
|
6673
7713
|
body: JSON.stringify(payload),
|
|
6674
7714
|
signal: options?.signal,
|
|
7715
|
+
effectiveModel: payload.model,
|
|
6675
7716
|
retryable: true
|
|
6676
7717
|
});
|
|
6677
7718
|
}
|
|
@@ -6968,202 +8009,6 @@ function buildGitHubUrls(gheDomain) {
|
|
|
6968
8009
|
};
|
|
6969
8010
|
}
|
|
6970
8011
|
//#endregion
|
|
6971
|
-
//#region src/util/duration.ts
|
|
6972
|
-
/**
|
|
6973
|
-
* Formats a millisecond duration as a compact human-readable string:
|
|
6974
|
-
* `<n>ms` under one second, otherwise `<n>s` rounded to whole seconds.
|
|
6975
|
-
*/
|
|
6976
|
-
function formatDurationMs(ms) {
|
|
6977
|
-
return ms < 1e3 ? `${ms}ms` : `${Math.round(ms / 1e3)}s`;
|
|
6978
|
-
}
|
|
6979
|
-
//#endregion
|
|
6980
|
-
//#region src/clients/upstream-queue.ts
|
|
6981
|
-
const DEFAULT_UPSTREAM_QUEUE_OPTIONS = {
|
|
6982
|
-
concurrency: 10,
|
|
6983
|
-
maxRetries: 5,
|
|
6984
|
-
baseDelayMs: 2e3,
|
|
6985
|
-
maxDelayMs: 6e4,
|
|
6986
|
-
maxQueueDepth: 1e3
|
|
6987
|
-
};
|
|
6988
|
-
var UpstreamRequestQueue = class {
|
|
6989
|
-
sleep;
|
|
6990
|
-
now;
|
|
6991
|
-
logger;
|
|
6992
|
-
setTimer;
|
|
6993
|
-
clearTimer;
|
|
6994
|
-
options;
|
|
6995
|
-
active = 0;
|
|
6996
|
-
cooldownUntil = 0;
|
|
6997
|
-
drainTimer;
|
|
6998
|
-
waiters = [];
|
|
6999
|
-
constructor(options = {}, deps = {}) {
|
|
7000
|
-
this.options = normalizeOptions(options);
|
|
7001
|
-
this.sleep = deps.sleep ?? sleep;
|
|
7002
|
-
this.now = deps.now ?? Date.now;
|
|
7003
|
-
this.logger = deps.logger ?? consola;
|
|
7004
|
-
this.setTimer = deps.setTimeout ?? globalThis.setTimeout;
|
|
7005
|
-
this.clearTimer = deps.clearTimeout ?? globalThis.clearTimeout;
|
|
7006
|
-
}
|
|
7007
|
-
updateOptions(options) {
|
|
7008
|
-
this.options = normalizeOptions(mergeDefinedOptions(this.options, options));
|
|
7009
|
-
this.drain();
|
|
7010
|
-
}
|
|
7011
|
-
async dispatch(fetcher, context, signal) {
|
|
7012
|
-
let attempt = 0;
|
|
7013
|
-
for (;;) {
|
|
7014
|
-
signal?.throwIfAborted();
|
|
7015
|
-
const lease = await this.acquire(signal);
|
|
7016
|
-
let response;
|
|
7017
|
-
try {
|
|
7018
|
-
response = await fetcher();
|
|
7019
|
-
} catch (error) {
|
|
7020
|
-
lease.release();
|
|
7021
|
-
throw error;
|
|
7022
|
-
}
|
|
7023
|
-
const { status } = response;
|
|
7024
|
-
const isCapacityLimit = isCapacityLimitStatus(status);
|
|
7025
|
-
if (!((context.retryable === "capacity" ? isCapacityLimit : context.retryable === true && isTransientUpstreamStatus(status)) && attempt < this.options.maxRetries)) {
|
|
7026
|
-
if (isCapacityLimit) this.applyCooldown(this.getRetryDelayMs(response, 0));
|
|
7027
|
-
return {
|
|
7028
|
-
response,
|
|
7029
|
-
release: lease.release
|
|
7030
|
-
};
|
|
7031
|
-
}
|
|
7032
|
-
const delayMs = this.getRetryDelayMs(response, attempt);
|
|
7033
|
-
await discardResponse(response);
|
|
7034
|
-
if (isCapacityLimit) this.applyCooldown(delayMs);
|
|
7035
|
-
lease.release();
|
|
7036
|
-
this.logger.warn([
|
|
7037
|
-
`Upstream ${status};`,
|
|
7038
|
-
`retrying ${formatRequestContext(context)}`,
|
|
7039
|
-
`in ${formatDurationMs(delayMs)}`,
|
|
7040
|
-
`(attempt ${attempt + 1}/${this.options.maxRetries})`
|
|
7041
|
-
].join(" "));
|
|
7042
|
-
await abortableSleep(this.sleep, delayMs, signal);
|
|
7043
|
-
attempt++;
|
|
7044
|
-
}
|
|
7045
|
-
}
|
|
7046
|
-
acquire(signal) {
|
|
7047
|
-
signal?.throwIfAborted();
|
|
7048
|
-
if (this.waiters.length >= this.options.maxQueueDepth) return Promise.reject(new HTTPError(503, { error: {
|
|
7049
|
-
message: "Upstream queue full",
|
|
7050
|
-
type: "overloaded_error"
|
|
7051
|
-
} }));
|
|
7052
|
-
return new Promise((resolve, reject) => {
|
|
7053
|
-
let resolved = false;
|
|
7054
|
-
const waiter = (lease) => {
|
|
7055
|
-
resolved = true;
|
|
7056
|
-
resolve(lease);
|
|
7057
|
-
};
|
|
7058
|
-
this.waiters.push(waiter);
|
|
7059
|
-
if (signal) signal.addEventListener("abort", () => {
|
|
7060
|
-
if (resolved) return;
|
|
7061
|
-
const idx = this.waiters.indexOf(waiter);
|
|
7062
|
-
if (idx !== -1) this.waiters.splice(idx, 1);
|
|
7063
|
-
reject(signal.reason);
|
|
7064
|
-
}, { once: true });
|
|
7065
|
-
this.drain();
|
|
7066
|
-
});
|
|
7067
|
-
}
|
|
7068
|
-
drain() {
|
|
7069
|
-
if (this.drainTimer) {
|
|
7070
|
-
this.clearTimer(this.drainTimer);
|
|
7071
|
-
this.drainTimer = void 0;
|
|
7072
|
-
}
|
|
7073
|
-
const cooldownMs = this.cooldownUntil - this.now();
|
|
7074
|
-
if (cooldownMs > 0) {
|
|
7075
|
-
this.drainTimer = this.setTimer(() => this.drain(), cooldownMs);
|
|
7076
|
-
return;
|
|
7077
|
-
}
|
|
7078
|
-
while (this.active < this.options.concurrency && this.waiters.length > 0) {
|
|
7079
|
-
const resolve = this.waiters.shift();
|
|
7080
|
-
let released = false;
|
|
7081
|
-
this.active++;
|
|
7082
|
-
resolve({ release: () => {
|
|
7083
|
-
if (released) return;
|
|
7084
|
-
released = true;
|
|
7085
|
-
this.active--;
|
|
7086
|
-
this.drain();
|
|
7087
|
-
} });
|
|
7088
|
-
}
|
|
7089
|
-
}
|
|
7090
|
-
applyCooldown(delayMs) {
|
|
7091
|
-
this.cooldownUntil = Math.max(this.cooldownUntil, this.now() + delayMs);
|
|
7092
|
-
this.drain();
|
|
7093
|
-
}
|
|
7094
|
-
getRetryDelayMs(response, attempt) {
|
|
7095
|
-
const retryAfterMs = parseRetryAfterMs(response.headers, this.now());
|
|
7096
|
-
if (retryAfterMs !== void 0) return clampDelay(retryAfterMs, this.options.maxDelayMs);
|
|
7097
|
-
return clampDelay(this.options.baseDelayMs * 2 ** attempt, this.options.maxDelayMs);
|
|
7098
|
-
}
|
|
7099
|
-
};
|
|
7100
|
-
function createDefaultUpstreamRequestQueue() {
|
|
7101
|
-
return new UpstreamRequestQueue(DEFAULT_UPSTREAM_QUEUE_OPTIONS);
|
|
7102
|
-
}
|
|
7103
|
-
function parseRetryAfterMs(headers, now = Date.now()) {
|
|
7104
|
-
const retryAfter = headers.get("retry-after");
|
|
7105
|
-
if (!retryAfter) return;
|
|
7106
|
-
const retryAfterSeconds = Number.parseFloat(retryAfter);
|
|
7107
|
-
if (Number.isFinite(retryAfterSeconds)) return Math.max(0, retryAfterSeconds * 1e3);
|
|
7108
|
-
const retryAt = Date.parse(retryAfter);
|
|
7109
|
-
if (Number.isNaN(retryAt)) return;
|
|
7110
|
-
return Math.max(0, retryAt - now);
|
|
7111
|
-
}
|
|
7112
|
-
function finiteOr(value, fallback) {
|
|
7113
|
-
return value !== void 0 && Number.isFinite(value) ? value : fallback;
|
|
7114
|
-
}
|
|
7115
|
-
function normalizeOptions(options) {
|
|
7116
|
-
return {
|
|
7117
|
-
concurrency: Math.max(1, Math.floor(finiteOr(options.concurrency, DEFAULT_UPSTREAM_QUEUE_OPTIONS.concurrency))),
|
|
7118
|
-
maxRetries: Math.max(0, Math.floor(finiteOr(options.maxRetries, DEFAULT_UPSTREAM_QUEUE_OPTIONS.maxRetries))),
|
|
7119
|
-
baseDelayMs: Math.max(0, Math.floor(finiteOr(options.baseDelayMs, DEFAULT_UPSTREAM_QUEUE_OPTIONS.baseDelayMs))),
|
|
7120
|
-
maxDelayMs: Math.max(1, Math.floor(finiteOr(options.maxDelayMs, DEFAULT_UPSTREAM_QUEUE_OPTIONS.maxDelayMs))),
|
|
7121
|
-
maxQueueDepth: Math.max(1, Math.floor(finiteOr(options.maxQueueDepth, DEFAULT_UPSTREAM_QUEUE_OPTIONS.maxQueueDepth)))
|
|
7122
|
-
};
|
|
7123
|
-
}
|
|
7124
|
-
function mergeDefinedOptions(current, next) {
|
|
7125
|
-
return {
|
|
7126
|
-
concurrency: next.concurrency ?? current.concurrency,
|
|
7127
|
-
maxRetries: next.maxRetries ?? current.maxRetries,
|
|
7128
|
-
baseDelayMs: next.baseDelayMs ?? current.baseDelayMs,
|
|
7129
|
-
maxDelayMs: next.maxDelayMs ?? current.maxDelayMs,
|
|
7130
|
-
maxQueueDepth: next.maxQueueDepth ?? current.maxQueueDepth
|
|
7131
|
-
};
|
|
7132
|
-
}
|
|
7133
|
-
function clampDelay(delayMs, maxDelayMs) {
|
|
7134
|
-
return Math.min(Math.max(0, Math.ceil(delayMs)), maxDelayMs);
|
|
7135
|
-
}
|
|
7136
|
-
async function discardResponse(response) {
|
|
7137
|
-
try {
|
|
7138
|
-
await response.body?.cancel();
|
|
7139
|
-
} catch {}
|
|
7140
|
-
}
|
|
7141
|
-
function formatRequestContext(context) {
|
|
7142
|
-
try {
|
|
7143
|
-
const url = new URL(context.url);
|
|
7144
|
-
return `${context.method ?? "GET"} ${url.pathname}`;
|
|
7145
|
-
} catch {
|
|
7146
|
-
return `${context.method ?? "GET"} ${context.url}`;
|
|
7147
|
-
}
|
|
7148
|
-
}
|
|
7149
|
-
function abortableSleep(sleep, ms, signal) {
|
|
7150
|
-
if (!signal) return sleep(ms);
|
|
7151
|
-
signal.throwIfAborted();
|
|
7152
|
-
return new Promise((resolve, reject) => {
|
|
7153
|
-
let done = false;
|
|
7154
|
-
signal.addEventListener("abort", () => {
|
|
7155
|
-
if (done) return;
|
|
7156
|
-
done = true;
|
|
7157
|
-
reject(signal.reason);
|
|
7158
|
-
}, { once: true });
|
|
7159
|
-
sleep(ms).then(() => {
|
|
7160
|
-
if (done) return;
|
|
7161
|
-
done = true;
|
|
7162
|
-
resolve();
|
|
7163
|
-
});
|
|
7164
|
-
});
|
|
7165
|
-
}
|
|
7166
|
-
//#endregion
|
|
7167
8012
|
//#region src/clients/factory.ts
|
|
7168
8013
|
const upstreamRequestQueue = createDefaultUpstreamRequestQueue();
|
|
7169
8014
|
function configureUpstreamRequestQueue(options) {
|
|
@@ -7179,8 +8024,12 @@ function getClientConfig() {
|
|
|
7179
8024
|
githubApiBaseUrl: apiBaseUrl
|
|
7180
8025
|
};
|
|
7181
8026
|
}
|
|
7182
|
-
function createCopilotClient() {
|
|
7183
|
-
return new CopilotClient(authStore, getClientConfig(), {
|
|
8027
|
+
function createCopilotClient(recovery, options = {}) {
|
|
8028
|
+
return new CopilotClient(authStore, getClientConfig(), {
|
|
8029
|
+
requestQueue: upstreamRequestQueue,
|
|
8030
|
+
recovery,
|
|
8031
|
+
...options
|
|
8032
|
+
});
|
|
7184
8033
|
}
|
|
7185
8034
|
async function cacheModels(client) {
|
|
7186
8035
|
const models = await (client ?? createCopilotClient()).getModels();
|
|
@@ -7419,7 +8268,7 @@ const checkUsage = defineCommand({
|
|
|
7419
8268
|
});
|
|
7420
8269
|
//#endregion
|
|
7421
8270
|
//#region src/util/version.ts
|
|
7422
|
-
const VERSION = "0.9.
|
|
8271
|
+
const VERSION = "0.9.3";
|
|
7423
8272
|
//#endregion
|
|
7424
8273
|
//#region src/debug.ts
|
|
7425
8274
|
function getRuntimeInfo() {
|
|
@@ -7495,336 +8344,6 @@ const debug = defineCommand({
|
|
|
7495
8344
|
}
|
|
7496
8345
|
});
|
|
7497
8346
|
//#endregion
|
|
7498
|
-
//#region src/lib/tokenizer.ts
|
|
7499
|
-
const ENCODING_MAP = {
|
|
7500
|
-
o200k_base: () => import("./o200k_base-DXNwToXP.mjs"),
|
|
7501
|
-
cl100k_base: () => import("./cl100k_base-ChJqEXhP.mjs"),
|
|
7502
|
-
p50k_base: () => import("./p50k_base-Cab7w92R.mjs"),
|
|
7503
|
-
p50k_edit: () => import("./p50k_edit-DkrRw_em.mjs"),
|
|
7504
|
-
r50k_base: () => import("./r50k_base-1vVxWqTY.mjs")
|
|
7505
|
-
};
|
|
7506
|
-
const encodingCache = /* @__PURE__ */ new Map();
|
|
7507
|
-
const TOKENS_PER_MESSAGE = 3;
|
|
7508
|
-
const TOKENS_PER_NAME = 1;
|
|
7509
|
-
const REPLY_PRIMING_TOKENS = 3;
|
|
7510
|
-
const BASE_CONSTANTS = {
|
|
7511
|
-
propertyInitOverhead: 3,
|
|
7512
|
-
propertyKeyOverhead: 3,
|
|
7513
|
-
enumOverhead: -3,
|
|
7514
|
-
enumItemCost: 3,
|
|
7515
|
-
functionEndOverhead: 12
|
|
7516
|
-
};
|
|
7517
|
-
/**
|
|
7518
|
-
* Calculate tokens for tool calls
|
|
7519
|
-
*/
|
|
7520
|
-
function calculateToolCallsTokens(toolCalls, encoder, constants) {
|
|
7521
|
-
let tokens = 0;
|
|
7522
|
-
for (const toolCall of toolCalls) {
|
|
7523
|
-
tokens += constants.functionInitOverhead;
|
|
7524
|
-
tokens += encoder.encode(JSON.stringify(toolCall)).length;
|
|
7525
|
-
}
|
|
7526
|
-
tokens += constants.functionEndOverhead;
|
|
7527
|
-
return tokens;
|
|
7528
|
-
}
|
|
7529
|
-
/**
|
|
7530
|
-
* Calculate tokens for content parts
|
|
7531
|
-
*/
|
|
7532
|
-
function calculateContentPartsTokens(contentParts, encoder) {
|
|
7533
|
-
let tokens = 0;
|
|
7534
|
-
for (const part of contentParts) if (part.type === "image_url") tokens += encoder.encode(part.image_url.url).length + 85;
|
|
7535
|
-
else if (part.text) tokens += encoder.encode(part.text).length;
|
|
7536
|
-
return tokens;
|
|
7537
|
-
}
|
|
7538
|
-
/**
|
|
7539
|
-
* Calculate tokens for a single message
|
|
7540
|
-
*/
|
|
7541
|
-
function calculateMessageTokens(message, encoder, constants) {
|
|
7542
|
-
let tokens = TOKENS_PER_MESSAGE;
|
|
7543
|
-
for (const [key, value] of Object.entries(message)) {
|
|
7544
|
-
if (typeof value === "string") tokens += encoder.encode(value).length;
|
|
7545
|
-
if (key === "name") tokens += TOKENS_PER_NAME;
|
|
7546
|
-
if (key === "tool_calls") tokens += calculateToolCallsTokens(value, encoder, constants);
|
|
7547
|
-
if (key === "content" && Array.isArray(value)) tokens += calculateContentPartsTokens(value, encoder);
|
|
7548
|
-
}
|
|
7549
|
-
return tokens;
|
|
7550
|
-
}
|
|
7551
|
-
/**
|
|
7552
|
-
* Calculate tokens using custom algorithm
|
|
7553
|
-
*/
|
|
7554
|
-
function calculateTokens(messages, encoder, constants) {
|
|
7555
|
-
if (messages.length === 0) return 0;
|
|
7556
|
-
let numTokens = 0;
|
|
7557
|
-
for (const message of messages) numTokens += calculateMessageTokens(message, encoder, constants);
|
|
7558
|
-
numTokens += REPLY_PRIMING_TOKENS;
|
|
7559
|
-
return numTokens;
|
|
7560
|
-
}
|
|
7561
|
-
/**
|
|
7562
|
-
* Get the corresponding encoder module based on encoding type
|
|
7563
|
-
*/
|
|
7564
|
-
async function getEncoder(encoding) {
|
|
7565
|
-
const cached = encodingCache.get(encoding);
|
|
7566
|
-
if (cached) return cached;
|
|
7567
|
-
const supportedEncoding = encoding;
|
|
7568
|
-
if (!(supportedEncoding in ENCODING_MAP)) {
|
|
7569
|
-
const fallbackModule = await ENCODING_MAP.o200k_base();
|
|
7570
|
-
encodingCache.set(encoding, fallbackModule);
|
|
7571
|
-
return fallbackModule;
|
|
7572
|
-
}
|
|
7573
|
-
const encodingModule = await ENCODING_MAP[supportedEncoding]();
|
|
7574
|
-
encodingCache.set(encoding, encodingModule);
|
|
7575
|
-
return encodingModule;
|
|
7576
|
-
}
|
|
7577
|
-
/**
|
|
7578
|
-
* Get tokenizer type from model information
|
|
7579
|
-
*/
|
|
7580
|
-
function getTokenizerFromModel(model) {
|
|
7581
|
-
return model.capabilities.tokenizer || "o200k_base";
|
|
7582
|
-
}
|
|
7583
|
-
/**
|
|
7584
|
-
* Get model-specific constants for token calculation
|
|
7585
|
-
*/
|
|
7586
|
-
function getModelConstants(model) {
|
|
7587
|
-
const isLegacy = model.id === "gpt-3.5-turbo" || model.id === "gpt-4";
|
|
7588
|
-
return {
|
|
7589
|
-
...BASE_CONSTANTS,
|
|
7590
|
-
functionInitOverhead: isLegacy ? 10 : 7
|
|
7591
|
-
};
|
|
7592
|
-
}
|
|
7593
|
-
/**
|
|
7594
|
-
* Calculate tokens for a single parameter
|
|
7595
|
-
*/
|
|
7596
|
-
function calculateParameterTokens(key, prop, context) {
|
|
7597
|
-
const { encoder, constants } = context;
|
|
7598
|
-
let tokens = constants.propertyKeyOverhead;
|
|
7599
|
-
if (typeof prop !== "object" || prop === null) return tokens;
|
|
7600
|
-
const param = prop;
|
|
7601
|
-
const paramName = key;
|
|
7602
|
-
const paramType = param.type || "string";
|
|
7603
|
-
let paramDesc = param.description || "";
|
|
7604
|
-
if (param.enum && Array.isArray(param.enum)) {
|
|
7605
|
-
tokens += constants.enumOverhead;
|
|
7606
|
-
for (const item of param.enum) {
|
|
7607
|
-
tokens += constants.enumItemCost;
|
|
7608
|
-
tokens += encoder.encode(String(item)).length;
|
|
7609
|
-
}
|
|
7610
|
-
}
|
|
7611
|
-
if (paramDesc.endsWith(".")) paramDesc = paramDesc.slice(0, -1);
|
|
7612
|
-
const line = `${paramName}:${paramType}:${paramDesc}`;
|
|
7613
|
-
tokens += encoder.encode(line).length;
|
|
7614
|
-
const excludedKeys = new Set([
|
|
7615
|
-
"type",
|
|
7616
|
-
"description",
|
|
7617
|
-
"enum"
|
|
7618
|
-
]);
|
|
7619
|
-
for (const propertyName of Object.keys(param)) if (!excludedKeys.has(propertyName)) {
|
|
7620
|
-
const propertyValue = param[propertyName];
|
|
7621
|
-
const propertyText = typeof propertyValue === "string" ? propertyValue : JSON.stringify(propertyValue);
|
|
7622
|
-
tokens += encoder.encode(`${propertyName}:${propertyText}`).length;
|
|
7623
|
-
}
|
|
7624
|
-
return tokens;
|
|
7625
|
-
}
|
|
7626
|
-
/**
|
|
7627
|
-
* Calculate tokens for function parameters
|
|
7628
|
-
*/
|
|
7629
|
-
function calculateParametersTokens(parameters, encoder, constants) {
|
|
7630
|
-
if (!parameters || typeof parameters !== "object") return 0;
|
|
7631
|
-
const params = parameters;
|
|
7632
|
-
let tokens = 0;
|
|
7633
|
-
for (const [key, value] of Object.entries(params)) if (key === "properties") {
|
|
7634
|
-
const properties = value;
|
|
7635
|
-
if (Object.keys(properties).length > 0) {
|
|
7636
|
-
tokens += constants.propertyInitOverhead;
|
|
7637
|
-
for (const propKey of Object.keys(properties)) tokens += calculateParameterTokens(propKey, properties[propKey], {
|
|
7638
|
-
encoder,
|
|
7639
|
-
constants
|
|
7640
|
-
});
|
|
7641
|
-
}
|
|
7642
|
-
} else {
|
|
7643
|
-
const paramText = typeof value === "string" ? value : JSON.stringify(value);
|
|
7644
|
-
tokens += encoder.encode(`${key}:${paramText}`).length;
|
|
7645
|
-
}
|
|
7646
|
-
return tokens;
|
|
7647
|
-
}
|
|
7648
|
-
/**
|
|
7649
|
-
* Calculate tokens for a single tool
|
|
7650
|
-
*/
|
|
7651
|
-
function calculateToolTokens(tool, encoder, constants) {
|
|
7652
|
-
let tokens = constants.functionInitOverhead;
|
|
7653
|
-
const func = tool.function;
|
|
7654
|
-
const functionName = func.name;
|
|
7655
|
-
let functionDescription = func.description || "";
|
|
7656
|
-
if (functionDescription.endsWith(".")) functionDescription = functionDescription.slice(0, -1);
|
|
7657
|
-
const line = `${functionName}:${functionDescription}`;
|
|
7658
|
-
tokens += encoder.encode(line).length;
|
|
7659
|
-
if (typeof func.parameters === "object" && func.parameters !== null) tokens += calculateParametersTokens(func.parameters, encoder, constants);
|
|
7660
|
-
return tokens;
|
|
7661
|
-
}
|
|
7662
|
-
/**
|
|
7663
|
-
* Calculate token count for tools based on model
|
|
7664
|
-
*/
|
|
7665
|
-
function numTokensForTools(tools, encoder, constants) {
|
|
7666
|
-
let toolTokenCount = 0;
|
|
7667
|
-
for (const tool of tools) toolTokenCount += calculateToolTokens(tool, encoder, constants);
|
|
7668
|
-
toolTokenCount += constants.functionEndOverhead;
|
|
7669
|
-
return toolTokenCount;
|
|
7670
|
-
}
|
|
7671
|
-
/**
|
|
7672
|
-
* Calculate the token count of messages, supporting multiple GPT encoders
|
|
7673
|
-
*/
|
|
7674
|
-
async function getTokenCount(payload, model) {
|
|
7675
|
-
const encoder = await getEncoder(getTokenizerFromModel(model));
|
|
7676
|
-
const inputMessages = payload.messages.filter((msg) => msg.role !== "assistant");
|
|
7677
|
-
const outputMessages = payload.messages.filter((msg) => msg.role === "assistant");
|
|
7678
|
-
const constants = getModelConstants(model);
|
|
7679
|
-
let inputTokens = calculateTokens(inputMessages, encoder, constants);
|
|
7680
|
-
if (payload.tools && payload.tools.length > 0) inputTokens += numTokensForTools(payload.tools, encoder, constants);
|
|
7681
|
-
const outputTokens = calculateTokens(outputMessages, encoder, constants);
|
|
7682
|
-
return {
|
|
7683
|
-
input: inputTokens,
|
|
7684
|
-
output: outputTokens
|
|
7685
|
-
};
|
|
7686
|
-
}
|
|
7687
|
-
async function estimateResponsesInputTokens(inputItems, model) {
|
|
7688
|
-
return (await getEncoder(getTokenizerFromModel(model))).encode(JSON.stringify(inputItems)).length;
|
|
7689
|
-
}
|
|
7690
|
-
//#endregion
|
|
7691
|
-
//#region src/selfcheck.ts
|
|
7692
|
-
const PROBE_ENCODINGS = [
|
|
7693
|
-
"o200k_base",
|
|
7694
|
-
"cl100k_base",
|
|
7695
|
-
"p50k_base",
|
|
7696
|
-
"p50k_edit",
|
|
7697
|
-
"r50k_base"
|
|
7698
|
-
];
|
|
7699
|
-
const PROBE_MESSAGE = "ghc-proxy selfcheck: probe text for tokenizer chunk load";
|
|
7700
|
-
async function probeEncoding(encoding) {
|
|
7701
|
-
try {
|
|
7702
|
-
const count = await getTokenCount({ messages: [{
|
|
7703
|
-
role: "user",
|
|
7704
|
-
content: PROBE_MESSAGE
|
|
7705
|
-
}] }, {
|
|
7706
|
-
id: `selfcheck-${encoding}`,
|
|
7707
|
-
capabilities: { tokenizer: encoding }
|
|
7708
|
-
});
|
|
7709
|
-
if (count.input <= 0) throw new Error(`encoder for ${encoding} returned 0 tokens for non-empty input`);
|
|
7710
|
-
return {
|
|
7711
|
-
encoding,
|
|
7712
|
-
ok: true,
|
|
7713
|
-
tokenCount: count.input
|
|
7714
|
-
};
|
|
7715
|
-
} catch (error) {
|
|
7716
|
-
return {
|
|
7717
|
-
encoding,
|
|
7718
|
-
ok: false,
|
|
7719
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7720
|
-
};
|
|
7721
|
-
}
|
|
7722
|
-
}
|
|
7723
|
-
async function runSelfCheck(options) {
|
|
7724
|
-
const probes = await Promise.all(PROBE_ENCODINGS.map(probeEncoding));
|
|
7725
|
-
const failed = probes.filter((p) => !p.ok);
|
|
7726
|
-
const result = {
|
|
7727
|
-
ok: failed.length === 0,
|
|
7728
|
-
probes,
|
|
7729
|
-
failedCount: failed.length
|
|
7730
|
-
};
|
|
7731
|
-
if (options.json) process$1.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
7732
|
-
else {
|
|
7733
|
-
process$1.stdout.write(`ghc-proxy selfcheck — tokenizer dynamic-chunk load\n\n`);
|
|
7734
|
-
for (const probe of probes) {
|
|
7735
|
-
const mark = probe.ok ? "ok " : "FAIL";
|
|
7736
|
-
const detail = probe.ok ? `tokens=${probe.tokenCount}` : `error=${probe.error}`;
|
|
7737
|
-
process$1.stdout.write(` [${mark}] ${probe.encoding.padEnd(12)} ${detail}\n`);
|
|
7738
|
-
}
|
|
7739
|
-
process$1.stdout.write(`\n${result.ok ? "PASS" : "FAIL"} — ${probes.length - failed.length}/${probes.length} encodings loaded\n`);
|
|
7740
|
-
}
|
|
7741
|
-
if (!result.ok) process$1.exitCode = 1;
|
|
7742
|
-
}
|
|
7743
|
-
const selfcheck = defineCommand({
|
|
7744
|
-
meta: {
|
|
7745
|
-
name: "selfcheck",
|
|
7746
|
-
description: "Probe the packaged bundle for runtime regressions (loads every gpt-tokenizer dynamic chunk and encodes a probe string)."
|
|
7747
|
-
},
|
|
7748
|
-
args: { json: {
|
|
7749
|
-
type: "boolean",
|
|
7750
|
-
default: false,
|
|
7751
|
-
description: "Output probe results as JSON"
|
|
7752
|
-
} },
|
|
7753
|
-
run({ args }) {
|
|
7754
|
-
return runSelfCheck({ json: args.json });
|
|
7755
|
-
}
|
|
7756
|
-
});
|
|
7757
|
-
//#endregion
|
|
7758
|
-
//#region node_modules/proxy-from-env/index.js
|
|
7759
|
-
var DEFAULT_PORTS = {
|
|
7760
|
-
ftp: 21,
|
|
7761
|
-
gopher: 70,
|
|
7762
|
-
http: 80,
|
|
7763
|
-
https: 443,
|
|
7764
|
-
ws: 80,
|
|
7765
|
-
wss: 443
|
|
7766
|
-
};
|
|
7767
|
-
function parseUrl(urlString) {
|
|
7768
|
-
try {
|
|
7769
|
-
return new URL(urlString);
|
|
7770
|
-
} catch {
|
|
7771
|
-
return null;
|
|
7772
|
-
}
|
|
7773
|
-
}
|
|
7774
|
-
/**
|
|
7775
|
-
* @param {string|object|URL} url - The URL as a string or URL instance, or a
|
|
7776
|
-
* compatible object (such as the result from legacy url.parse).
|
|
7777
|
-
* @return {string} The URL of the proxy that should handle the request to the
|
|
7778
|
-
* given URL. If no proxy is set, this will be an empty string.
|
|
7779
|
-
*/
|
|
7780
|
-
function getProxyForUrl(url) {
|
|
7781
|
-
var parsedUrl = (typeof url === "string" ? parseUrl(url) : url) || {};
|
|
7782
|
-
var proto = parsedUrl.protocol;
|
|
7783
|
-
var hostname = parsedUrl.host;
|
|
7784
|
-
var port = parsedUrl.port;
|
|
7785
|
-
if (typeof hostname !== "string" || !hostname || typeof proto !== "string") return "";
|
|
7786
|
-
proto = proto.split(":", 1)[0];
|
|
7787
|
-
hostname = hostname.replace(/:\d*$/, "");
|
|
7788
|
-
port = parseInt(port) || DEFAULT_PORTS[proto] || 0;
|
|
7789
|
-
if (!shouldProxy(hostname, port)) return "";
|
|
7790
|
-
var proxy = getEnv(proto + "_proxy") || getEnv("all_proxy");
|
|
7791
|
-
if (proxy && proxy.indexOf("://") === -1) proxy = proto + "://" + proxy;
|
|
7792
|
-
return proxy;
|
|
7793
|
-
}
|
|
7794
|
-
/**
|
|
7795
|
-
* Determines whether a given URL should be proxied.
|
|
7796
|
-
*
|
|
7797
|
-
* @param {string} hostname - The host name of the URL.
|
|
7798
|
-
* @param {number} port - The effective port of the URL.
|
|
7799
|
-
* @returns {boolean} Whether the given URL should be proxied.
|
|
7800
|
-
* @private
|
|
7801
|
-
*/
|
|
7802
|
-
function shouldProxy(hostname, port) {
|
|
7803
|
-
var NO_PROXY = getEnv("no_proxy").toLowerCase();
|
|
7804
|
-
if (!NO_PROXY) return true;
|
|
7805
|
-
if (NO_PROXY === "*") return false;
|
|
7806
|
-
return NO_PROXY.split(/[,\s]/).every(function(proxy) {
|
|
7807
|
-
if (!proxy) return true;
|
|
7808
|
-
var parsedProxy = proxy.match(/^(.+):(\d+)$/);
|
|
7809
|
-
var parsedProxyHostname = parsedProxy ? parsedProxy[1] : proxy;
|
|
7810
|
-
var parsedProxyPort = parsedProxy ? parseInt(parsedProxy[2]) : 0;
|
|
7811
|
-
if (parsedProxyPort && parsedProxyPort !== port) return true;
|
|
7812
|
-
if (!/^[.*]/.test(parsedProxyHostname)) return hostname !== parsedProxyHostname;
|
|
7813
|
-
if (parsedProxyHostname.charAt(0) === "*") parsedProxyHostname = parsedProxyHostname.slice(1);
|
|
7814
|
-
return !hostname.endsWith(parsedProxyHostname);
|
|
7815
|
-
});
|
|
7816
|
-
}
|
|
7817
|
-
/**
|
|
7818
|
-
* Get the value for an environment variable.
|
|
7819
|
-
*
|
|
7820
|
-
* @param {string} key - The name of the environment variable.
|
|
7821
|
-
* @return {string} The value of the environment variable.
|
|
7822
|
-
* @private
|
|
7823
|
-
*/
|
|
7824
|
-
function getEnv(key) {
|
|
7825
|
-
return process.env[key.toLowerCase()] || process.env[key.toUpperCase()] || "";
|
|
7826
|
-
}
|
|
7827
|
-
//#endregion
|
|
7828
8347
|
//#region node_modules/undici/lib/core/symbols.js
|
|
7829
8348
|
var require_symbols = /* @__PURE__ */ __commonJSMin$1(((exports, module) => {
|
|
7830
8349
|
module.exports = {
|
|
@@ -29186,7 +29705,7 @@ var require_eventsource = /* @__PURE__ */ __commonJSMin$1(((exports, module) =>
|
|
|
29186
29705
|
};
|
|
29187
29706
|
}));
|
|
29188
29707
|
//#endregion
|
|
29189
|
-
//#region src/
|
|
29708
|
+
//#region src/lib/tokenizer.ts
|
|
29190
29709
|
var import_undici = (/* @__PURE__ */ __commonJSMin$1(((exports, module) => {
|
|
29191
29710
|
const Client = require_client();
|
|
29192
29711
|
const Dispatcher = require_dispatcher();
|
|
@@ -29353,6 +29872,524 @@ var import_undici = (/* @__PURE__ */ __commonJSMin$1(((exports, module) => {
|
|
|
29353
29872
|
}
|
|
29354
29873
|
module.exports.install = install;
|
|
29355
29874
|
})))();
|
|
29875
|
+
const ENCODING_MAP = {
|
|
29876
|
+
o200k_base: () => import("./o200k_base-DXNwToXP.mjs"),
|
|
29877
|
+
cl100k_base: () => import("./cl100k_base-ChJqEXhP.mjs"),
|
|
29878
|
+
p50k_base: () => import("./p50k_base-Cab7w92R.mjs"),
|
|
29879
|
+
p50k_edit: () => import("./p50k_edit-DkrRw_em.mjs"),
|
|
29880
|
+
r50k_base: () => import("./r50k_base-1vVxWqTY.mjs")
|
|
29881
|
+
};
|
|
29882
|
+
const encodingCache = /* @__PURE__ */ new Map();
|
|
29883
|
+
const TOKENS_PER_MESSAGE = 3;
|
|
29884
|
+
const TOKENS_PER_NAME = 1;
|
|
29885
|
+
const REPLY_PRIMING_TOKENS = 3;
|
|
29886
|
+
const BASE_CONSTANTS = {
|
|
29887
|
+
propertyInitOverhead: 3,
|
|
29888
|
+
propertyKeyOverhead: 3,
|
|
29889
|
+
enumOverhead: -3,
|
|
29890
|
+
enumItemCost: 3,
|
|
29891
|
+
functionEndOverhead: 12
|
|
29892
|
+
};
|
|
29893
|
+
/**
|
|
29894
|
+
* Calculate tokens for tool calls
|
|
29895
|
+
*/
|
|
29896
|
+
function calculateToolCallsTokens(toolCalls, encoder, constants) {
|
|
29897
|
+
let tokens = 0;
|
|
29898
|
+
for (const toolCall of toolCalls) {
|
|
29899
|
+
tokens += constants.functionInitOverhead;
|
|
29900
|
+
tokens += encoder.encode(JSON.stringify(toolCall)).length;
|
|
29901
|
+
}
|
|
29902
|
+
tokens += constants.functionEndOverhead;
|
|
29903
|
+
return tokens;
|
|
29904
|
+
}
|
|
29905
|
+
/**
|
|
29906
|
+
* Calculate tokens for content parts
|
|
29907
|
+
*/
|
|
29908
|
+
function calculateContentPartsTokens(contentParts, encoder) {
|
|
29909
|
+
let tokens = 0;
|
|
29910
|
+
for (const part of contentParts) if (part.type === "image_url") tokens += encoder.encode(part.image_url.url).length + 85;
|
|
29911
|
+
else if (part.text) tokens += encoder.encode(part.text).length;
|
|
29912
|
+
return tokens;
|
|
29913
|
+
}
|
|
29914
|
+
/**
|
|
29915
|
+
* Calculate tokens for a single message
|
|
29916
|
+
*/
|
|
29917
|
+
function calculateMessageTokens(message, encoder, constants) {
|
|
29918
|
+
let tokens = TOKENS_PER_MESSAGE;
|
|
29919
|
+
for (const [key, value] of Object.entries(message)) {
|
|
29920
|
+
if (typeof value === "string") tokens += encoder.encode(value).length;
|
|
29921
|
+
if (key === "name") tokens += TOKENS_PER_NAME;
|
|
29922
|
+
if (key === "tool_calls") tokens += calculateToolCallsTokens(value, encoder, constants);
|
|
29923
|
+
if (key === "content" && Array.isArray(value)) tokens += calculateContentPartsTokens(value, encoder);
|
|
29924
|
+
}
|
|
29925
|
+
return tokens;
|
|
29926
|
+
}
|
|
29927
|
+
/**
|
|
29928
|
+
* Calculate tokens using custom algorithm
|
|
29929
|
+
*/
|
|
29930
|
+
function calculateTokens(messages, encoder, constants) {
|
|
29931
|
+
if (messages.length === 0) return 0;
|
|
29932
|
+
let numTokens = 0;
|
|
29933
|
+
for (const message of messages) numTokens += calculateMessageTokens(message, encoder, constants);
|
|
29934
|
+
numTokens += REPLY_PRIMING_TOKENS;
|
|
29935
|
+
return numTokens;
|
|
29936
|
+
}
|
|
29937
|
+
/**
|
|
29938
|
+
* Get the corresponding encoder module based on encoding type
|
|
29939
|
+
*/
|
|
29940
|
+
async function getEncoder(encoding) {
|
|
29941
|
+
const cached = encodingCache.get(encoding);
|
|
29942
|
+
if (cached) return cached;
|
|
29943
|
+
const supportedEncoding = encoding;
|
|
29944
|
+
if (!(supportedEncoding in ENCODING_MAP)) {
|
|
29945
|
+
const fallbackModule = await ENCODING_MAP.o200k_base();
|
|
29946
|
+
encodingCache.set(encoding, fallbackModule);
|
|
29947
|
+
return fallbackModule;
|
|
29948
|
+
}
|
|
29949
|
+
const encodingModule = await ENCODING_MAP[supportedEncoding]();
|
|
29950
|
+
encodingCache.set(encoding, encodingModule);
|
|
29951
|
+
return encodingModule;
|
|
29952
|
+
}
|
|
29953
|
+
/**
|
|
29954
|
+
* Get tokenizer type from model information
|
|
29955
|
+
*/
|
|
29956
|
+
function getTokenizerFromModel(model) {
|
|
29957
|
+
return model.capabilities.tokenizer || "o200k_base";
|
|
29958
|
+
}
|
|
29959
|
+
/**
|
|
29960
|
+
* Get model-specific constants for token calculation
|
|
29961
|
+
*/
|
|
29962
|
+
function getModelConstants(model) {
|
|
29963
|
+
const isLegacy = model.id === "gpt-3.5-turbo" || model.id === "gpt-4";
|
|
29964
|
+
return {
|
|
29965
|
+
...BASE_CONSTANTS,
|
|
29966
|
+
functionInitOverhead: isLegacy ? 10 : 7
|
|
29967
|
+
};
|
|
29968
|
+
}
|
|
29969
|
+
/**
|
|
29970
|
+
* Calculate tokens for a single parameter
|
|
29971
|
+
*/
|
|
29972
|
+
function calculateParameterTokens(key, prop, context) {
|
|
29973
|
+
const { encoder, constants } = context;
|
|
29974
|
+
let tokens = constants.propertyKeyOverhead;
|
|
29975
|
+
if (typeof prop !== "object" || prop === null) return tokens;
|
|
29976
|
+
const param = prop;
|
|
29977
|
+
const paramName = key;
|
|
29978
|
+
const paramType = param.type || "string";
|
|
29979
|
+
let paramDesc = param.description || "";
|
|
29980
|
+
if (param.enum && Array.isArray(param.enum)) {
|
|
29981
|
+
tokens += constants.enumOverhead;
|
|
29982
|
+
for (const item of param.enum) {
|
|
29983
|
+
tokens += constants.enumItemCost;
|
|
29984
|
+
tokens += encoder.encode(String(item)).length;
|
|
29985
|
+
}
|
|
29986
|
+
}
|
|
29987
|
+
if (paramDesc.endsWith(".")) paramDesc = paramDesc.slice(0, -1);
|
|
29988
|
+
const line = `${paramName}:${paramType}:${paramDesc}`;
|
|
29989
|
+
tokens += encoder.encode(line).length;
|
|
29990
|
+
const excludedKeys = new Set([
|
|
29991
|
+
"type",
|
|
29992
|
+
"description",
|
|
29993
|
+
"enum"
|
|
29994
|
+
]);
|
|
29995
|
+
for (const propertyName of Object.keys(param)) if (!excludedKeys.has(propertyName)) {
|
|
29996
|
+
const propertyValue = param[propertyName];
|
|
29997
|
+
const propertyText = typeof propertyValue === "string" ? propertyValue : JSON.stringify(propertyValue);
|
|
29998
|
+
tokens += encoder.encode(`${propertyName}:${propertyText}`).length;
|
|
29999
|
+
}
|
|
30000
|
+
return tokens;
|
|
30001
|
+
}
|
|
30002
|
+
/**
|
|
30003
|
+
* Calculate tokens for function parameters
|
|
30004
|
+
*/
|
|
30005
|
+
function calculateParametersTokens(parameters, encoder, constants) {
|
|
30006
|
+
if (!parameters || typeof parameters !== "object") return 0;
|
|
30007
|
+
const params = parameters;
|
|
30008
|
+
let tokens = 0;
|
|
30009
|
+
for (const [key, value] of Object.entries(params)) if (key === "properties") {
|
|
30010
|
+
const properties = value;
|
|
30011
|
+
if (Object.keys(properties).length > 0) {
|
|
30012
|
+
tokens += constants.propertyInitOverhead;
|
|
30013
|
+
for (const propKey of Object.keys(properties)) tokens += calculateParameterTokens(propKey, properties[propKey], {
|
|
30014
|
+
encoder,
|
|
30015
|
+
constants
|
|
30016
|
+
});
|
|
30017
|
+
}
|
|
30018
|
+
} else {
|
|
30019
|
+
const paramText = typeof value === "string" ? value : JSON.stringify(value);
|
|
30020
|
+
tokens += encoder.encode(`${key}:${paramText}`).length;
|
|
30021
|
+
}
|
|
30022
|
+
return tokens;
|
|
30023
|
+
}
|
|
30024
|
+
/**
|
|
30025
|
+
* Calculate tokens for a single tool
|
|
30026
|
+
*/
|
|
30027
|
+
function calculateToolTokens(tool, encoder, constants) {
|
|
30028
|
+
let tokens = constants.functionInitOverhead;
|
|
30029
|
+
const func = tool.function;
|
|
30030
|
+
const functionName = func.name;
|
|
30031
|
+
let functionDescription = func.description || "";
|
|
30032
|
+
if (functionDescription.endsWith(".")) functionDescription = functionDescription.slice(0, -1);
|
|
30033
|
+
const line = `${functionName}:${functionDescription}`;
|
|
30034
|
+
tokens += encoder.encode(line).length;
|
|
30035
|
+
if (typeof func.parameters === "object" && func.parameters !== null) tokens += calculateParametersTokens(func.parameters, encoder, constants);
|
|
30036
|
+
return tokens;
|
|
30037
|
+
}
|
|
30038
|
+
/**
|
|
30039
|
+
* Calculate token count for tools based on model
|
|
30040
|
+
*/
|
|
30041
|
+
function numTokensForTools(tools, encoder, constants) {
|
|
30042
|
+
let toolTokenCount = 0;
|
|
30043
|
+
for (const tool of tools) toolTokenCount += calculateToolTokens(tool, encoder, constants);
|
|
30044
|
+
toolTokenCount += constants.functionEndOverhead;
|
|
30045
|
+
return toolTokenCount;
|
|
30046
|
+
}
|
|
30047
|
+
/**
|
|
30048
|
+
* Calculate the token count of messages, supporting multiple GPT encoders
|
|
30049
|
+
*/
|
|
30050
|
+
async function getTokenCount(payload, model) {
|
|
30051
|
+
const encoder = await getEncoder(getTokenizerFromModel(model));
|
|
30052
|
+
const inputMessages = payload.messages.filter((msg) => msg.role !== "assistant");
|
|
30053
|
+
const outputMessages = payload.messages.filter((msg) => msg.role === "assistant");
|
|
30054
|
+
const constants = getModelConstants(model);
|
|
30055
|
+
let inputTokens = calculateTokens(inputMessages, encoder, constants);
|
|
30056
|
+
if (payload.tools && payload.tools.length > 0) inputTokens += numTokensForTools(payload.tools, encoder, constants);
|
|
30057
|
+
const outputTokens = calculateTokens(outputMessages, encoder, constants);
|
|
30058
|
+
return {
|
|
30059
|
+
input: inputTokens,
|
|
30060
|
+
output: outputTokens
|
|
30061
|
+
};
|
|
30062
|
+
}
|
|
30063
|
+
async function estimateResponsesInputTokens(inputItems, model) {
|
|
30064
|
+
return (await getEncoder(getTokenizerFromModel(model))).encode(JSON.stringify(inputItems)).length;
|
|
30065
|
+
}
|
|
30066
|
+
//#endregion
|
|
30067
|
+
//#region src/selfcheck.ts
|
|
30068
|
+
const PROBE_ENCODINGS = [
|
|
30069
|
+
"o200k_base",
|
|
30070
|
+
"cl100k_base",
|
|
30071
|
+
"p50k_base",
|
|
30072
|
+
"p50k_edit",
|
|
30073
|
+
"r50k_base"
|
|
30074
|
+
];
|
|
30075
|
+
const PROBE_MESSAGE = "ghc-proxy selfcheck: probe text for tokenizer chunk load";
|
|
30076
|
+
const RUNTIME_PROBES = [
|
|
30077
|
+
["http-error-response-contract", probeHttpErrorResponseContract],
|
|
30078
|
+
["connection-error-classification", probeConnectionErrorClassification],
|
|
30079
|
+
["response-body-cancellation", probeResponseBodyCancellation],
|
|
30080
|
+
["response-commit-boundary", probeResponseCommitBoundary],
|
|
30081
|
+
["caller-cancellation", probeCallerCancellation],
|
|
30082
|
+
["protocol-payload-contract", probeProtocolPayloadContract]
|
|
30083
|
+
];
|
|
30084
|
+
async function probeEncoding(encoding) {
|
|
30085
|
+
try {
|
|
30086
|
+
const count = await getTokenCount({ messages: [{
|
|
30087
|
+
role: "user",
|
|
30088
|
+
content: PROBE_MESSAGE
|
|
30089
|
+
}] }, {
|
|
30090
|
+
id: `selfcheck-${encoding}`,
|
|
30091
|
+
capabilities: { tokenizer: encoding }
|
|
30092
|
+
});
|
|
30093
|
+
if (count.input <= 0) throw new Error(`encoder for ${encoding} returned 0 tokens for non-empty input`);
|
|
30094
|
+
return {
|
|
30095
|
+
encoding,
|
|
30096
|
+
ok: true,
|
|
30097
|
+
tokenCount: count.input
|
|
30098
|
+
};
|
|
30099
|
+
} catch (error) {
|
|
30100
|
+
return {
|
|
30101
|
+
encoding,
|
|
30102
|
+
ok: false,
|
|
30103
|
+
error: error instanceof Error ? error.message : String(error)
|
|
30104
|
+
};
|
|
30105
|
+
}
|
|
30106
|
+
}
|
|
30107
|
+
async function runRuntimeProbe(name, probe) {
|
|
30108
|
+
try {
|
|
30109
|
+
await probe();
|
|
30110
|
+
return {
|
|
30111
|
+
name,
|
|
30112
|
+
ok: true
|
|
30113
|
+
};
|
|
30114
|
+
} catch (error) {
|
|
30115
|
+
return {
|
|
30116
|
+
name,
|
|
30117
|
+
ok: false,
|
|
30118
|
+
error: error instanceof Error ? error.message : String(error)
|
|
30119
|
+
};
|
|
30120
|
+
}
|
|
30121
|
+
}
|
|
30122
|
+
async function probeHttpErrorResponseContract() {
|
|
30123
|
+
const response = new HTTPError(529, { error: {
|
|
30124
|
+
message: "upstream overloaded",
|
|
30125
|
+
type: "overloaded_error"
|
|
30126
|
+
} }, { headers: { "retry-after": "17" } }).toResponse();
|
|
30127
|
+
assertProbe(response.status === 529, `expected status 529, received ${response.status}`);
|
|
30128
|
+
assertProbe(response.headers.get("retry-after") === "17", "Retry-After was not preserved");
|
|
30129
|
+
assertProbe((await response.json()).error?.type === "overloaded_error", "error payload changed during toResponse()");
|
|
30130
|
+
}
|
|
30131
|
+
function probeConnectionErrorClassification() {
|
|
30132
|
+
assertProbe(isRetryableConnectionEstablishmentError({ code: "ConnectionRefused" }) === "connection-refused", "Bun ConnectionRefused was not classified");
|
|
30133
|
+
assertProbe(isRetryableConnectionEstablishmentError(new TypeError("fetch failed", { cause: { code: "ECONNREFUSED" } })) === "connection-refused", "Node ECONNREFUSED was not classified");
|
|
30134
|
+
assertProbe(isRetryableConnectionEstablishmentError(new TypeError("fetch failed", { cause: { code: "ENOTFOUND" } })) === "dns", "Node ENOTFOUND was not classified");
|
|
30135
|
+
assertProbe(isRetryableConnectionEstablishmentError({
|
|
30136
|
+
name: "TimeoutError",
|
|
30137
|
+
code: "ECONNREFUSED"
|
|
30138
|
+
}) === void 0, "timeout-shaped error was classified as a connection-establishment failure");
|
|
30139
|
+
}
|
|
30140
|
+
async function probeResponseBodyCancellation() {
|
|
30141
|
+
const queue = createRuntimeProbeQueue({ maxRetries: 1 });
|
|
30142
|
+
const dispatcher = process$1.versions.bun ? void 0 : new import_undici.Agent({ connections: 1 });
|
|
30143
|
+
const sockets = /* @__PURE__ */ new Set();
|
|
30144
|
+
let requests = 0;
|
|
30145
|
+
let firstResponseClosed = false;
|
|
30146
|
+
const server = createServer((_request, response) => {
|
|
30147
|
+
requests++;
|
|
30148
|
+
if (requests === 1) {
|
|
30149
|
+
response.once("close", () => {
|
|
30150
|
+
firstResponseClosed = true;
|
|
30151
|
+
});
|
|
30152
|
+
response.writeHead(529, { "retry-after": "0" });
|
|
30153
|
+
response.write("retryable response remains open");
|
|
30154
|
+
return;
|
|
30155
|
+
}
|
|
30156
|
+
response.end("ok");
|
|
30157
|
+
});
|
|
30158
|
+
server.on("connection", (socket) => {
|
|
30159
|
+
sockets.add(socket);
|
|
30160
|
+
socket.once("close", () => sockets.delete(socket));
|
|
30161
|
+
});
|
|
30162
|
+
await new Promise((resolve, reject) => {
|
|
30163
|
+
server.once("error", reject);
|
|
30164
|
+
server.listen(0, "127.0.0.1", () => {
|
|
30165
|
+
server.off("error", reject);
|
|
30166
|
+
resolve();
|
|
30167
|
+
});
|
|
30168
|
+
});
|
|
30169
|
+
try {
|
|
30170
|
+
const address = server.address();
|
|
30171
|
+
assertProbe(address !== null && typeof address === "object", "loopback server has no address");
|
|
30172
|
+
const url = `http://127.0.0.1:${address.port}/retry`;
|
|
30173
|
+
const result = await queue.dispatch((signal) => fetch(url, {
|
|
30174
|
+
signal,
|
|
30175
|
+
...dispatcher ? { dispatcher } : {}
|
|
30176
|
+
}), {
|
|
30177
|
+
url,
|
|
30178
|
+
retryable: "capacity"
|
|
30179
|
+
});
|
|
30180
|
+
try {
|
|
30181
|
+
assertProbe(result.response.status === 200, `expected retry status 200, received ${result.response.status}`);
|
|
30182
|
+
assertProbe(await result.response.text() === "ok", "retry response body changed");
|
|
30183
|
+
} finally {
|
|
30184
|
+
result.release();
|
|
30185
|
+
}
|
|
30186
|
+
assertProbe(requests === 2, `expected one retry, observed ${requests - 1}`);
|
|
30187
|
+
if (!process$1.versions.bun) assertProbe(firstResponseClosed, "retryable response did not release its transport");
|
|
30188
|
+
} finally {
|
|
30189
|
+
for (const socket of sockets) socket.destroy();
|
|
30190
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
30191
|
+
await dispatcher?.close();
|
|
30192
|
+
}
|
|
30193
|
+
}
|
|
30194
|
+
async function probeResponseCommitBoundary() {
|
|
30195
|
+
const queue = createRuntimeProbeQueue({ maxRetries: 1 });
|
|
30196
|
+
let attempts = 0;
|
|
30197
|
+
const result = await queue.dispatch(async () => {
|
|
30198
|
+
attempts++;
|
|
30199
|
+
return new Response(new ReadableStream({ start(controller) {
|
|
30200
|
+
controller.error(/* @__PURE__ */ new Error("probe stream failure"));
|
|
30201
|
+
} }));
|
|
30202
|
+
}, {
|
|
30203
|
+
url: "https://example.invalid/v1/messages",
|
|
30204
|
+
retryable: "capacity"
|
|
30205
|
+
});
|
|
30206
|
+
let bodyFailed = false;
|
|
30207
|
+
try {
|
|
30208
|
+
await result.response.text();
|
|
30209
|
+
} catch {
|
|
30210
|
+
bodyFailed = true;
|
|
30211
|
+
} finally {
|
|
30212
|
+
result.release();
|
|
30213
|
+
}
|
|
30214
|
+
assertProbe(bodyFailed, "probe stream did not fail during body consumption");
|
|
30215
|
+
assertProbe(attempts === 1, `committed response was replayed ${attempts - 1} time(s)`);
|
|
30216
|
+
}
|
|
30217
|
+
async function probeCallerCancellation() {
|
|
30218
|
+
const queue = createRuntimeProbeQueue();
|
|
30219
|
+
const controller = new AbortController();
|
|
30220
|
+
const reason = /* @__PURE__ */ new Error("selfcheck caller cancellation");
|
|
30221
|
+
let observedSignal;
|
|
30222
|
+
const pending = queue.dispatch(async (signal) => {
|
|
30223
|
+
observedSignal = signal;
|
|
30224
|
+
return new Promise((_resolve, reject) => {
|
|
30225
|
+
if (!signal) {
|
|
30226
|
+
reject(/* @__PURE__ */ new Error("queue did not pass the caller signal to fetch"));
|
|
30227
|
+
return;
|
|
30228
|
+
}
|
|
30229
|
+
signal.addEventListener("abort", () => reject(signal.reason), { once: true });
|
|
30230
|
+
});
|
|
30231
|
+
}, {
|
|
30232
|
+
url: "https://example.invalid/v1/messages",
|
|
30233
|
+
retryable: "capacity"
|
|
30234
|
+
}, controller.signal);
|
|
30235
|
+
await Promise.resolve();
|
|
30236
|
+
await Promise.resolve();
|
|
30237
|
+
controller.abort(reason);
|
|
30238
|
+
let rejection;
|
|
30239
|
+
try {
|
|
30240
|
+
await pending;
|
|
30241
|
+
} catch (error) {
|
|
30242
|
+
rejection = error;
|
|
30243
|
+
}
|
|
30244
|
+
assertProbe(observedSignal === controller.signal, "fetch did not receive the caller signal");
|
|
30245
|
+
assertProbe(rejection === reason, "caller abort reason was not preserved");
|
|
30246
|
+
}
|
|
30247
|
+
async function probeProtocolPayloadContract() {
|
|
30248
|
+
const response = new HTTPError(429, { error: {
|
|
30249
|
+
message: "rate limited",
|
|
30250
|
+
type: "rate_limit_error"
|
|
30251
|
+
} }, { headers: { "retry-after": "5" } }).toResponse();
|
|
30252
|
+
const payload = await response.json();
|
|
30253
|
+
const error = payload.error;
|
|
30254
|
+
assertProbe(Object.keys(payload).join(",") === "error", "public error payload gained a top-level extension");
|
|
30255
|
+
assertProbe(error !== void 0 && Object.keys(error).sort().join(",") === "message,type", "public error object gained a recovery extension");
|
|
30256
|
+
assertProbe([...response.headers.keys()].every((name) => !name.startsWith("x-ghc-")), "public response gained a non-standard recovery header");
|
|
30257
|
+
}
|
|
30258
|
+
function createRuntimeProbeQueue(options = {}) {
|
|
30259
|
+
return new UpstreamRequestQueue({
|
|
30260
|
+
concurrency: 1,
|
|
30261
|
+
maxRetries: options.maxRetries ?? 0,
|
|
30262
|
+
baseDelayMs: 0,
|
|
30263
|
+
maxDelayMs: 1,
|
|
30264
|
+
maxQueueDepth: 1,
|
|
30265
|
+
recoveryBudgetMs: 1e3
|
|
30266
|
+
}, {
|
|
30267
|
+
sleep: async () => {},
|
|
30268
|
+
random: () => 0,
|
|
30269
|
+
logger: {
|
|
30270
|
+
warn() {},
|
|
30271
|
+
info() {}
|
|
30272
|
+
}
|
|
30273
|
+
});
|
|
30274
|
+
}
|
|
30275
|
+
function assertProbe(condition, message) {
|
|
30276
|
+
if (!condition) throw new Error(message);
|
|
30277
|
+
}
|
|
30278
|
+
async function runSelfCheck(options) {
|
|
30279
|
+
const probes = await Promise.all(PROBE_ENCODINGS.map(probeEncoding));
|
|
30280
|
+
const runtimeProbes = await Promise.all(RUNTIME_PROBES.map(([name, probe]) => runRuntimeProbe(name, probe)));
|
|
30281
|
+
const failed = [...probes, ...runtimeProbes].filter((p) => !p.ok);
|
|
30282
|
+
const result = {
|
|
30283
|
+
ok: failed.length === 0,
|
|
30284
|
+
probes,
|
|
30285
|
+
runtimeProbes,
|
|
30286
|
+
failedCount: failed.length
|
|
30287
|
+
};
|
|
30288
|
+
if (options.json) process$1.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
30289
|
+
else {
|
|
30290
|
+
process$1.stdout.write(`ghc-proxy selfcheck — tokenizer dynamic-chunk load\n\n`);
|
|
30291
|
+
for (const probe of probes) {
|
|
30292
|
+
const mark = probe.ok ? "ok " : "FAIL";
|
|
30293
|
+
const detail = probe.ok ? `tokens=${probe.tokenCount}` : `error=${probe.error}`;
|
|
30294
|
+
process$1.stdout.write(` [${mark}] ${probe.encoding.padEnd(12)} ${detail}\n`);
|
|
30295
|
+
}
|
|
30296
|
+
process$1.stdout.write(`\nghc-proxy runtime probes\n\n`);
|
|
30297
|
+
for (const probe of runtimeProbes) {
|
|
30298
|
+
const mark = probe.ok ? "ok " : "FAIL";
|
|
30299
|
+
const detail = probe.ok ? "" : ` error=${probe.error}`;
|
|
30300
|
+
process$1.stdout.write(` [${mark}] ${probe.name}${detail}\n`);
|
|
30301
|
+
}
|
|
30302
|
+
const passed = probes.length + runtimeProbes.length - failed.length;
|
|
30303
|
+
process$1.stdout.write(`\n${result.ok ? "PASS" : "FAIL"} — ${passed}/${probes.length + runtimeProbes.length} probes passed\n`);
|
|
30304
|
+
}
|
|
30305
|
+
if (!result.ok) process$1.exitCode = 1;
|
|
30306
|
+
}
|
|
30307
|
+
const selfcheck = defineCommand({
|
|
30308
|
+
meta: {
|
|
30309
|
+
name: "selfcheck",
|
|
30310
|
+
description: "Probe the packaged bundle for tokenizer and cross-runtime regressions."
|
|
30311
|
+
},
|
|
30312
|
+
args: { json: {
|
|
30313
|
+
type: "boolean",
|
|
30314
|
+
default: false,
|
|
30315
|
+
description: "Output probe results as JSON"
|
|
30316
|
+
} },
|
|
30317
|
+
run({ args }) {
|
|
30318
|
+
return runSelfCheck({ json: args.json });
|
|
30319
|
+
}
|
|
30320
|
+
});
|
|
30321
|
+
//#endregion
|
|
30322
|
+
//#region node_modules/proxy-from-env/index.js
|
|
30323
|
+
var DEFAULT_PORTS = {
|
|
30324
|
+
ftp: 21,
|
|
30325
|
+
gopher: 70,
|
|
30326
|
+
http: 80,
|
|
30327
|
+
https: 443,
|
|
30328
|
+
ws: 80,
|
|
30329
|
+
wss: 443
|
|
30330
|
+
};
|
|
30331
|
+
function parseUrl(urlString) {
|
|
30332
|
+
try {
|
|
30333
|
+
return new URL(urlString);
|
|
30334
|
+
} catch {
|
|
30335
|
+
return null;
|
|
30336
|
+
}
|
|
30337
|
+
}
|
|
30338
|
+
/**
|
|
30339
|
+
* @param {string|object|URL} url - The URL as a string or URL instance, or a
|
|
30340
|
+
* compatible object (such as the result from legacy url.parse).
|
|
30341
|
+
* @return {string} The URL of the proxy that should handle the request to the
|
|
30342
|
+
* given URL. If no proxy is set, this will be an empty string.
|
|
30343
|
+
*/
|
|
30344
|
+
function getProxyForUrl(url) {
|
|
30345
|
+
var parsedUrl = (typeof url === "string" ? parseUrl(url) : url) || {};
|
|
30346
|
+
var proto = parsedUrl.protocol;
|
|
30347
|
+
var hostname = parsedUrl.host;
|
|
30348
|
+
var port = parsedUrl.port;
|
|
30349
|
+
if (typeof hostname !== "string" || !hostname || typeof proto !== "string") return "";
|
|
30350
|
+
proto = proto.split(":", 1)[0];
|
|
30351
|
+
hostname = hostname.replace(/:\d*$/, "");
|
|
30352
|
+
port = parseInt(port) || DEFAULT_PORTS[proto] || 0;
|
|
30353
|
+
if (!shouldProxy(hostname, port)) return "";
|
|
30354
|
+
var proxy = getEnv(proto + "_proxy") || getEnv("all_proxy");
|
|
30355
|
+
if (proxy && proxy.indexOf("://") === -1) proxy = proto + "://" + proxy;
|
|
30356
|
+
return proxy;
|
|
30357
|
+
}
|
|
30358
|
+
/**
|
|
30359
|
+
* Determines whether a given URL should be proxied.
|
|
30360
|
+
*
|
|
30361
|
+
* @param {string} hostname - The host name of the URL.
|
|
30362
|
+
* @param {number} port - The effective port of the URL.
|
|
30363
|
+
* @returns {boolean} Whether the given URL should be proxied.
|
|
30364
|
+
* @private
|
|
30365
|
+
*/
|
|
30366
|
+
function shouldProxy(hostname, port) {
|
|
30367
|
+
var NO_PROXY = getEnv("no_proxy").toLowerCase();
|
|
30368
|
+
if (!NO_PROXY) return true;
|
|
30369
|
+
if (NO_PROXY === "*") return false;
|
|
30370
|
+
return NO_PROXY.split(/[,\s]/).every(function(proxy) {
|
|
30371
|
+
if (!proxy) return true;
|
|
30372
|
+
var parsedProxy = proxy.match(/^(.+):(\d+)$/);
|
|
30373
|
+
var parsedProxyHostname = parsedProxy ? parsedProxy[1] : proxy;
|
|
30374
|
+
var parsedProxyPort = parsedProxy ? parseInt(parsedProxy[2]) : 0;
|
|
30375
|
+
if (parsedProxyPort && parsedProxyPort !== port) return true;
|
|
30376
|
+
if (!/^[.*]/.test(parsedProxyHostname)) return hostname !== parsedProxyHostname;
|
|
30377
|
+
if (parsedProxyHostname.charAt(0) === "*") parsedProxyHostname = parsedProxyHostname.slice(1);
|
|
30378
|
+
return !hostname.endsWith(parsedProxyHostname);
|
|
30379
|
+
});
|
|
30380
|
+
}
|
|
30381
|
+
/**
|
|
30382
|
+
* Get the value for an environment variable.
|
|
30383
|
+
*
|
|
30384
|
+
* @param {string} key - The name of the environment variable.
|
|
30385
|
+
* @return {string} The value of the environment variable.
|
|
30386
|
+
* @private
|
|
30387
|
+
*/
|
|
30388
|
+
function getEnv(key) {
|
|
30389
|
+
return process.env[key.toLowerCase()] || process.env[key.toUpperCase()] || "";
|
|
30390
|
+
}
|
|
30391
|
+
//#endregion
|
|
30392
|
+
//#region src/cli/proxy.ts
|
|
29356
30393
|
function initProxyFromEnv() {
|
|
29357
30394
|
if (typeof Bun !== "undefined") return;
|
|
29358
30395
|
try {
|
|
@@ -47781,148 +48818,6 @@ var node = () => {
|
|
|
47781
48818
|
};
|
|
47782
48819
|
};
|
|
47783
48820
|
//#endregion
|
|
47784
|
-
//#region src/lib/request-logger.ts
|
|
47785
|
-
/**
|
|
47786
|
-
* Per-request model mapping store.
|
|
47787
|
-
* Route handlers write to this; the after-response hook reads from it.
|
|
47788
|
-
* Uses WeakMap so entries are GC'd when the Request is collected.
|
|
47789
|
-
*/
|
|
47790
|
-
const requestModelMapping = /* @__PURE__ */ new WeakMap();
|
|
47791
|
-
function setRequestModelMapping(request, info) {
|
|
47792
|
-
requestModelMapping.set(request, info);
|
|
47793
|
-
}
|
|
47794
|
-
function getRequestModelMapping(request) {
|
|
47795
|
-
return requestModelMapping.get(request);
|
|
47796
|
-
}
|
|
47797
|
-
function formatElapsed(start) {
|
|
47798
|
-
return formatDurationMs(Date.now() - start);
|
|
47799
|
-
}
|
|
47800
|
-
function formatPath(rawUrl) {
|
|
47801
|
-
try {
|
|
47802
|
-
const url = new URL(rawUrl);
|
|
47803
|
-
return `${url.pathname}${url.search}`;
|
|
47804
|
-
} catch {
|
|
47805
|
-
return rawUrl;
|
|
47806
|
-
}
|
|
47807
|
-
}
|
|
47808
|
-
function colorizeStatus(status) {
|
|
47809
|
-
if (status >= 500) return colorize("red", status);
|
|
47810
|
-
if (status >= 400) return colorize("yellow", status);
|
|
47811
|
-
if (status >= 300) return colorize("cyan", status);
|
|
47812
|
-
return colorize("green", status);
|
|
47813
|
-
}
|
|
47814
|
-
const methodColors = {
|
|
47815
|
-
GET: "cyan",
|
|
47816
|
-
POST: "magenta",
|
|
47817
|
-
PUT: "yellow",
|
|
47818
|
-
PATCH: "yellow",
|
|
47819
|
-
DELETE: "red"
|
|
47820
|
-
};
|
|
47821
|
-
function colorizeMethod(method) {
|
|
47822
|
-
return colorize(methodColors[method] ?? "white", method);
|
|
47823
|
-
}
|
|
47824
|
-
function getEffectiveModel(info) {
|
|
47825
|
-
return info.steps.length > 0 ? info.steps.at(-1).to : info.originalModel ?? "-";
|
|
47826
|
-
}
|
|
47827
|
-
/**
|
|
47828
|
-
* Mutate `modelMapping` in place by appending a transform step.
|
|
47829
|
-
* Strategy contexts hold a reference to the same `modelMapping`,
|
|
47830
|
-
* so steps are pushed directly rather than returning a new object.
|
|
47831
|
-
*/
|
|
47832
|
-
function appendModelStepInPlace(info, tag, newModel) {
|
|
47833
|
-
const current = getEffectiveModel(info);
|
|
47834
|
-
if (newModel !== current) info.steps.push({
|
|
47835
|
-
tag,
|
|
47836
|
-
from: current,
|
|
47837
|
-
to: newModel
|
|
47838
|
-
});
|
|
47839
|
-
}
|
|
47840
|
-
function formatModelMapping(info) {
|
|
47841
|
-
if (!info) return "";
|
|
47842
|
-
const { originalModel, steps } = info;
|
|
47843
|
-
if (!originalModel && steps.length === 0) return "";
|
|
47844
|
-
const parts = [colorize("blueBright", originalModel ?? "-")];
|
|
47845
|
-
for (let i = 0; i < steps.length; i++) {
|
|
47846
|
-
const step = steps[i];
|
|
47847
|
-
const isLast = i === steps.length - 1;
|
|
47848
|
-
parts.push(colorize("dim", `-[${step.tag}]->`));
|
|
47849
|
-
parts.push(colorize(isLast ? "greenBright" : "cyanBright", step.to));
|
|
47850
|
-
}
|
|
47851
|
-
return ` ${colorize("dim", "model=")}${parts.join(" ")}`;
|
|
47852
|
-
}
|
|
47853
|
-
/**
|
|
47854
|
-
* Request logging function.
|
|
47855
|
-
* Logs a formatted request line with method, path, status, elapsed time,
|
|
47856
|
-
* and optional model mapping info.
|
|
47857
|
-
*/
|
|
47858
|
-
function logRequest(method, url, status, elapsed, modelInfo, requestId) {
|
|
47859
|
-
const path = formatPath(url);
|
|
47860
|
-
const line = [
|
|
47861
|
-
colorize("dim", "<-"),
|
|
47862
|
-
colorizeMethod(method),
|
|
47863
|
-
colorize("white", path),
|
|
47864
|
-
colorizeStatus(status),
|
|
47865
|
-
colorize("dim", elapsed)
|
|
47866
|
-
].join(" ");
|
|
47867
|
-
const rid = requestId ? ` ${colorize("dim", `rid=${requestId.slice(0, 8)}`)}` : "";
|
|
47868
|
-
console.log(`${line}${formatModelMapping(modelInfo)}${rid}`);
|
|
47869
|
-
}
|
|
47870
|
-
//#endregion
|
|
47871
|
-
//#region src/lib/timeout-error.ts
|
|
47872
|
-
/**
|
|
47873
|
-
* Whether an error represents a request that timed out or was aborted.
|
|
47874
|
-
*
|
|
47875
|
-
* The shape differs by runtime, so the check is structural rather than a
|
|
47876
|
-
* single `name` comparison:
|
|
47877
|
-
* - Bun rejects with a flat `DOMException` named `TimeoutError` (its ~300s
|
|
47878
|
-
* `fetch` ceiling, `AbortSignal.timeout`) or `AbortError`.
|
|
47879
|
-
* - Node rejects with `TypeError('fetch failed' | 'terminated')` and puts the
|
|
47880
|
-
* real undici error on `.cause` (`HeadersTimeoutError`, `BodyTimeoutError`,
|
|
47881
|
-
* `ConnectTimeoutError`), so the top-level error carries no signal at all —
|
|
47882
|
-
* `TypeError('fetch failed')` is also what `ECONNREFUSED` and DNS failures
|
|
47883
|
-
* look like. The discriminator is the cause's `name`/`code`.
|
|
47884
|
-
*
|
|
47885
|
-
* Both runtimes enforce a ~300s upstream ceiling by default (Node's is
|
|
47886
|
-
* undici's `headersTimeout`/`bodyTimeout` default of `300e3`), which fires
|
|
47887
|
-
* long before the configured `--upstream-timeout` of 1800s.
|
|
47888
|
-
*
|
|
47889
|
-
* Kept in one place because the rule is checked on both sides of the stream
|
|
47890
|
-
* boundary: `src/server.ts` maps it to a 504 before the first byte, and the
|
|
47891
|
-
* Anthropic stream transducer maps it to an SSE error frame after. Two
|
|
47892
|
-
* implementations of "what counts as a timeout" is how one of them ends up
|
|
47893
|
-
* recognizing only half the errors.
|
|
47894
|
-
*/
|
|
47895
|
-
const TIMEOUT_ERROR_NAMES = new Set([
|
|
47896
|
-
"AbortError",
|
|
47897
|
-
"TimeoutError",
|
|
47898
|
-
"ConnectTimeoutError",
|
|
47899
|
-
"HeadersTimeoutError",
|
|
47900
|
-
"BodyTimeoutError"
|
|
47901
|
-
]);
|
|
47902
|
-
const TIMEOUT_ERROR_CODES = new Set([
|
|
47903
|
-
"UND_ERR_CONNECT_TIMEOUT",
|
|
47904
|
-
"UND_ERR_HEADERS_TIMEOUT",
|
|
47905
|
-
"UND_ERR_BODY_TIMEOUT",
|
|
47906
|
-
"ETIMEDOUT"
|
|
47907
|
-
]);
|
|
47908
|
-
const MAX_CAUSE_DEPTH = 5;
|
|
47909
|
-
function matchesTimeoutShape(value, depth) {
|
|
47910
|
-
if (typeof value !== "object" || value === null) return false;
|
|
47911
|
-
const candidate = value;
|
|
47912
|
-
if (typeof candidate.name === "string" && TIMEOUT_ERROR_NAMES.has(candidate.name)) return true;
|
|
47913
|
-
if (typeof candidate.code === "string" && TIMEOUT_ERROR_CODES.has(candidate.code)) return true;
|
|
47914
|
-
if (depth >= MAX_CAUSE_DEPTH) return false;
|
|
47915
|
-
if (matchesTimeoutShape(candidate.cause, depth + 1)) return true;
|
|
47916
|
-
return Array.isArray(candidate.errors) && candidate.errors.some((inner) => matchesTimeoutShape(inner, depth + 1));
|
|
47917
|
-
}
|
|
47918
|
-
function isTimeoutLikeError(error) {
|
|
47919
|
-
try {
|
|
47920
|
-
return matchesTimeoutShape(error, 0);
|
|
47921
|
-
} catch {
|
|
47922
|
-
return false;
|
|
47923
|
-
}
|
|
47924
|
-
}
|
|
47925
|
-
//#endregion
|
|
47926
48821
|
//#region src/lib/sse-adapter.ts
|
|
47927
48822
|
/**
|
|
47928
48823
|
* Serializes Anthropic stream events into SSE output items
|
|
@@ -49198,7 +50093,7 @@ const responsesToolSchema = union([object({
|
|
|
49198
50093
|
type: literal("function"),
|
|
49199
50094
|
name: string().min(1),
|
|
49200
50095
|
parameters: jsonObjectSchema.nullable().optional(),
|
|
49201
|
-
strict: boolean().optional(),
|
|
50096
|
+
strict: boolean().nullable().optional(),
|
|
49202
50097
|
description: string().nullable().optional()
|
|
49203
50098
|
}).loose(), object({ type: string().min(1) }).catchall(unknown()).superRefine((tool, ctx) => {
|
|
49204
50099
|
if (tool.type === "function") ctx.addIssue({
|
|
@@ -49217,6 +50112,8 @@ const responsesToolChoiceSchema = union([
|
|
|
49217
50112
|
}).loose(),
|
|
49218
50113
|
object({ type: _enum([
|
|
49219
50114
|
"file_search",
|
|
50115
|
+
"web_search",
|
|
50116
|
+
"web_search_2025_08_26",
|
|
49220
50117
|
"web_search_preview",
|
|
49221
50118
|
"web_search_preview_2025_03_11",
|
|
49222
50119
|
"computer_use_preview",
|
|
@@ -49457,6 +50354,10 @@ protocolRegistry.register("embeddings", embeddingsProtocol);
|
|
|
49457
50354
|
//#endregion
|
|
49458
50355
|
//#region src/lib/upstream-signal.ts
|
|
49459
50356
|
const DEFAULT_TIMEOUT_MS = 18e5;
|
|
50357
|
+
function createUpstreamDeadlineFromConfig(now = performance.now()) {
|
|
50358
|
+
const timeoutMs = authStore.upstreamTimeoutSeconds !== void 0 ? authStore.upstreamTimeoutSeconds * 1e3 : DEFAULT_TIMEOUT_MS;
|
|
50359
|
+
return timeoutMs > 0 ? now + timeoutMs : null;
|
|
50360
|
+
}
|
|
49460
50361
|
function createUpstreamSignal(clientSignal, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
49461
50362
|
const controller = new AbortController();
|
|
49462
50363
|
const timeout = timeoutMs > 0 ? setTimeout(() => controller.abort(), timeoutMs) : void 0;
|
|
@@ -49482,8 +50383,21 @@ function createUpstreamSignal(clientSignal, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
|
49482
50383
|
* runtime instead. `isTimeoutLikeError` recognizes both runtimes' shapes so
|
|
49483
50384
|
* every path maps to a 504.
|
|
49484
50385
|
*/
|
|
49485
|
-
function createUpstreamSignalFromConfig(clientSignal) {
|
|
49486
|
-
|
|
50386
|
+
function createUpstreamSignalFromConfig(clientSignal, deadlineMonotonicMs = createUpstreamDeadlineFromConfig()) {
|
|
50387
|
+
const remainingMs = deadlineMonotonicMs === null ? void 0 : deadlineMonotonicMs - performance.now();
|
|
50388
|
+
return {
|
|
50389
|
+
...remainingMs !== void 0 && remainingMs <= 0 ? createExpiredUpstreamSignal(clientSignal) : createUpstreamSignal(clientSignal, remainingMs ?? 0),
|
|
50390
|
+
deadlineMonotonicMs
|
|
50391
|
+
};
|
|
50392
|
+
}
|
|
50393
|
+
function createExpiredUpstreamSignal(clientSignal) {
|
|
50394
|
+
const controller = new AbortController();
|
|
50395
|
+
controller.abort();
|
|
50396
|
+
return {
|
|
50397
|
+
signal: controller.signal,
|
|
50398
|
+
clientSignal,
|
|
50399
|
+
cleanup: () => {}
|
|
50400
|
+
};
|
|
49487
50401
|
}
|
|
49488
50402
|
//#endregion
|
|
49489
50403
|
//#region src/transform/constants.ts
|
|
@@ -49639,6 +50553,8 @@ function resolveRequestModel({ payload, betaHeaders, applyPolicy }) {
|
|
|
49639
50553
|
//#endregion
|
|
49640
50554
|
//#region src/pipeline/runner.ts
|
|
49641
50555
|
async function runPipeline(params, config) {
|
|
50556
|
+
const upstreamDeadlineMonotonicMs = createUpstreamDeadlineFromConfig();
|
|
50557
|
+
const recovery = createRecoveryRecord(params);
|
|
49642
50558
|
const ingested = protocolRegistry.ingest(config.protocol, params.body, params.headers);
|
|
49643
50559
|
const meta = ingested.meta;
|
|
49644
50560
|
const payload = config.afterIngest ? config.afterIngest({
|
|
@@ -49646,31 +50562,321 @@ async function runPipeline(params, config) {
|
|
|
49646
50562
|
meta,
|
|
49647
50563
|
headers: params.headers
|
|
49648
50564
|
}) : ingested.payload;
|
|
49649
|
-
const
|
|
50565
|
+
const baseSourceModel = resolveBaseModel(payload, meta, config);
|
|
50566
|
+
const fallbackPossible = shouldPreservePristinePayload(config.protocol, baseSourceModel);
|
|
50567
|
+
const pristinePayload = fallbackPossible ? structuredClone(payload) : payload;
|
|
50568
|
+
const sourceAttempt = await prepareAttempt(payload, meta, params, config, recovery, upstreamDeadlineMonotonicMs, { offerLocalModelCooldown: (sourceModel) => fallbackPossible && validateFallback(config.protocol, pristinePayload, sourceModel).ok });
|
|
50569
|
+
try {
|
|
50570
|
+
return {
|
|
50571
|
+
result: await sourceAttempt.execute(),
|
|
50572
|
+
modelMapping: sourceAttempt.modelMapping
|
|
50573
|
+
};
|
|
50574
|
+
} catch (error) {
|
|
50575
|
+
if (!(error instanceof TerminalUpstreamRecoveryError) || error.status !== 529) throw error;
|
|
50576
|
+
const sourceModel = error.recovery.sourceModel;
|
|
50577
|
+
if (!sourceModel) {
|
|
50578
|
+
emitFallbackEvent(error.recovery, void 0, "missing-source-model", 529);
|
|
50579
|
+
throw error;
|
|
50580
|
+
}
|
|
50581
|
+
const candidate = fallbackPossible ? validateFallback(config.protocol, pristinePayload, sourceModel) : {
|
|
50582
|
+
ok: false,
|
|
50583
|
+
reason: "not-configured"
|
|
50584
|
+
};
|
|
50585
|
+
if (!candidate.ok) {
|
|
50586
|
+
emitFallbackEvent(error.recovery, sourceModel, candidate.reason, 529);
|
|
50587
|
+
throw error;
|
|
50588
|
+
}
|
|
50589
|
+
if (sourceAttempt.baseModel !== resolveBaseModel(pristinePayload, meta, config) || getEffectiveModel(sourceAttempt.modelMapping) !== sourceModel) {
|
|
50590
|
+
emitFallbackEvent(error.recovery, sourceModel, "source-resolution-changed", 529);
|
|
50591
|
+
throw error;
|
|
50592
|
+
}
|
|
50593
|
+
params.signal.throwIfAborted();
|
|
50594
|
+
if (!error.claimFallback()) throw error;
|
|
50595
|
+
error.recovery.fallbackFetchStarted = false;
|
|
50596
|
+
const fallbackMapping = {
|
|
50597
|
+
originalModel: sourceAttempt.modelMapping.originalModel,
|
|
50598
|
+
steps: [...sourceAttempt.modelMapping.steps]
|
|
50599
|
+
};
|
|
50600
|
+
appendModelStepInPlace(fallbackMapping, "OVERLOAD_FALLBACK", candidate.target.id);
|
|
50601
|
+
let fallbackAttempt;
|
|
50602
|
+
try {
|
|
50603
|
+
fallbackAttempt = await prepareAttempt(structuredClone(pristinePayload), meta, params, config, error.recovery, upstreamDeadlineMonotonicMs, {
|
|
50604
|
+
target: candidate.target,
|
|
50605
|
+
modelMapping: fallbackMapping,
|
|
50606
|
+
fallbackAttempt: true
|
|
50607
|
+
});
|
|
50608
|
+
} catch {
|
|
50609
|
+
if (params.signal.aborted) throw params.signal.reason;
|
|
50610
|
+
emitFallbackEvent(error.recovery, candidate.target.id, "preflight-rejected", 529);
|
|
50611
|
+
throw error;
|
|
50612
|
+
}
|
|
50613
|
+
error.recovery.retryLimit = error.recovery.retryCount;
|
|
50614
|
+
emitFallbackEvent(error.recovery, candidate.target.id, "selected", 529);
|
|
50615
|
+
try {
|
|
50616
|
+
const result = await fallbackAttempt.execute();
|
|
50617
|
+
emitFallbackEvent(error.recovery, candidate.target.id, "succeeded");
|
|
50618
|
+
return {
|
|
50619
|
+
result: discloseActualModel(result, candidate.target.id),
|
|
50620
|
+
modelMapping: fallbackMapping
|
|
50621
|
+
};
|
|
50622
|
+
} catch (fallbackError) {
|
|
50623
|
+
if (params.signal.aborted) throw params.signal.reason;
|
|
50624
|
+
if (fallbackError instanceof FallbackCooldownError) {
|
|
50625
|
+
emitFallbackEvent(error.recovery, candidate.target.id, "target-cooldown", 529);
|
|
50626
|
+
throw error;
|
|
50627
|
+
}
|
|
50628
|
+
if (!error.recovery.fallbackFetchStarted) {
|
|
50629
|
+
emitFallbackEvent(error.recovery, candidate.target.id, "pre-fetch-failed", 529);
|
|
50630
|
+
throw error;
|
|
50631
|
+
}
|
|
50632
|
+
emitFallbackEvent(error.recovery, candidate.target.id, "target-failed", fallbackError instanceof HTTPError ? fallbackError.status : void 0, isRetryableConnectionEstablishmentError(fallbackError));
|
|
50633
|
+
throw fallbackError;
|
|
50634
|
+
}
|
|
50635
|
+
}
|
|
50636
|
+
}
|
|
50637
|
+
async function prepareAttempt(payload, meta, params, config, recovery, upstreamDeadlineMonotonicMs, options = {}) {
|
|
50638
|
+
const resolved = resolveRequestModel({
|
|
49650
50639
|
payload,
|
|
49651
50640
|
betaHeaders: meta.betaHeaders,
|
|
49652
50641
|
applyPolicy: config.applyModelPolicy
|
|
49653
50642
|
});
|
|
50643
|
+
const baseModel = resolved.model;
|
|
50644
|
+
const selectedModel = options.target ?? resolved.resolvedModel;
|
|
50645
|
+
const modelMapping = options.modelMapping ?? resolved.modelMapping;
|
|
50646
|
+
if (options.target) payload.model = options.target.id;
|
|
49654
50647
|
if (config.afterTransform) await config.afterTransform({
|
|
49655
50648
|
payload,
|
|
49656
50649
|
meta,
|
|
49657
50650
|
headers: params.headers,
|
|
49658
50651
|
selectedModel
|
|
49659
50652
|
});
|
|
49660
|
-
|
|
49661
|
-
const
|
|
49662
|
-
const
|
|
49663
|
-
|
|
49664
|
-
|
|
49665
|
-
headers: params.headers,
|
|
49666
|
-
selectedModel,
|
|
49667
|
-
copilotClient,
|
|
49668
|
-
upstreamSignal,
|
|
49669
|
-
modelMapping
|
|
50653
|
+
params.signal.throwIfAborted();
|
|
50654
|
+
const upstreamSignal = createUpstreamSignalFromConfig(params.signal, upstreamDeadlineMonotonicMs);
|
|
50655
|
+
const copilotClient = createCopilotClient(recovery, {
|
|
50656
|
+
offerLocalModelCooldown: options.offerLocalModelCooldown,
|
|
50657
|
+
fallbackAttempt: options.fallbackAttempt
|
|
49670
50658
|
});
|
|
50659
|
+
try {
|
|
50660
|
+
const ctx = config.buildStrategyContext({
|
|
50661
|
+
payload,
|
|
50662
|
+
meta,
|
|
50663
|
+
headers: params.headers,
|
|
50664
|
+
selectedModel,
|
|
50665
|
+
copilotClient,
|
|
50666
|
+
upstreamSignal,
|
|
50667
|
+
modelMapping,
|
|
50668
|
+
recovery
|
|
50669
|
+
});
|
|
50670
|
+
const entry = config.strategyRegistry.select(selectedModel, ctx);
|
|
50671
|
+
return {
|
|
50672
|
+
baseModel,
|
|
50673
|
+
modelMapping,
|
|
50674
|
+
execute: async () => {
|
|
50675
|
+
try {
|
|
50676
|
+
params.signal.throwIfAborted();
|
|
50677
|
+
return await entry.execute(ctx);
|
|
50678
|
+
} catch (error) {
|
|
50679
|
+
upstreamSignal.cleanup();
|
|
50680
|
+
throw error;
|
|
50681
|
+
}
|
|
50682
|
+
}
|
|
50683
|
+
};
|
|
50684
|
+
} catch (error) {
|
|
50685
|
+
upstreamSignal.cleanup();
|
|
50686
|
+
throw error;
|
|
50687
|
+
}
|
|
50688
|
+
}
|
|
50689
|
+
function resolveBaseModel(pristinePayload, meta, config) {
|
|
50690
|
+
return resolveRequestModel({
|
|
50691
|
+
payload: { ...pristinePayload },
|
|
50692
|
+
betaHeaders: meta.betaHeaders,
|
|
50693
|
+
applyPolicy: config.applyModelPolicy
|
|
50694
|
+
}).model;
|
|
50695
|
+
}
|
|
50696
|
+
function shouldPreservePristinePayload(protocol, baseSourceModel) {
|
|
50697
|
+
if (configStore.getOverloadFallback(baseSourceModel)?.trim()) return true;
|
|
50698
|
+
if (protocol !== "anthropic-messages" || !configStore.hasOverloadFallbacks()) return false;
|
|
50699
|
+
const model = modelCache.findById(baseSourceModel);
|
|
50700
|
+
return !model || !modelCache.supportsEndpoint(model, "/v1/messages") && !modelCache.supportsEndpoint(model, "/responses");
|
|
50701
|
+
}
|
|
50702
|
+
function validateFallback(protocol, payload, sourceModel) {
|
|
50703
|
+
const targetId = configStore.getOverloadFallback(sourceModel)?.trim();
|
|
50704
|
+
if (!targetId) return {
|
|
50705
|
+
ok: false,
|
|
50706
|
+
reason: "not-configured"
|
|
50707
|
+
};
|
|
50708
|
+
if (targetId === sourceModel) return {
|
|
50709
|
+
ok: false,
|
|
50710
|
+
reason: "same-model"
|
|
50711
|
+
};
|
|
50712
|
+
const target = modelCache.findById(targetId);
|
|
50713
|
+
if (!target) return {
|
|
50714
|
+
ok: false,
|
|
50715
|
+
reason: "unknown-target"
|
|
50716
|
+
};
|
|
50717
|
+
if (protocol === "responses" && !modelCache.supportsEndpoint(target, "/responses")) return {
|
|
50718
|
+
ok: false,
|
|
50719
|
+
reason: "unsupported-endpoint"
|
|
50720
|
+
};
|
|
50721
|
+
if (requestsTools(payload) && !modelCache.supportsToolCalls(target)) return {
|
|
50722
|
+
ok: false,
|
|
50723
|
+
reason: "unsupported-tools"
|
|
50724
|
+
};
|
|
50725
|
+
if (requestsParallelToolCalls(protocol, payload) && target.capabilities.supports.parallel_tool_calls !== true) return {
|
|
50726
|
+
ok: false,
|
|
50727
|
+
reason: "unsupported-parallel-tools"
|
|
50728
|
+
};
|
|
50729
|
+
if (requestsStreaming(payload) && target.capabilities.supports.streaming === false) return {
|
|
50730
|
+
ok: false,
|
|
50731
|
+
reason: "unsupported-streaming"
|
|
50732
|
+
};
|
|
50733
|
+
if (requestsVision(payload) && !modelCache.supportsVision(target)) return {
|
|
50734
|
+
ok: false,
|
|
50735
|
+
reason: "unsupported-vision"
|
|
50736
|
+
};
|
|
50737
|
+
if (requestsReasoningEffort(protocol, payload) && !modelCache.supportsReasoningEffort(target)) return {
|
|
50738
|
+
ok: false,
|
|
50739
|
+
reason: "unsupported-reasoning"
|
|
50740
|
+
};
|
|
50741
|
+
if (requestsThinking(protocol, payload) && !modelCache.supportsAdaptiveThinking(target) && !modelCache.supportsReasoningEffort(target)) return {
|
|
50742
|
+
ok: false,
|
|
50743
|
+
reason: "unsupported-thinking"
|
|
50744
|
+
};
|
|
50745
|
+
if (requestsStructuredOutput(protocol, payload) && !supportsStructuredOutput(protocol, target)) return {
|
|
50746
|
+
ok: false,
|
|
50747
|
+
reason: "unsupported-structured-output"
|
|
50748
|
+
};
|
|
49671
50749
|
return {
|
|
49672
|
-
|
|
49673
|
-
|
|
50750
|
+
ok: true,
|
|
50751
|
+
target
|
|
50752
|
+
};
|
|
50753
|
+
}
|
|
50754
|
+
function requestsTools(payload) {
|
|
50755
|
+
const tools = asRecord(payload)?.tools;
|
|
50756
|
+
return Array.isArray(tools) && tools.length > 0;
|
|
50757
|
+
}
|
|
50758
|
+
function requestsParallelToolCalls(protocol, payload) {
|
|
50759
|
+
if (protocol === "anthropic-messages") return requestsTools(payload);
|
|
50760
|
+
return asRecord(payload)?.parallel_tool_calls === true;
|
|
50761
|
+
}
|
|
50762
|
+
function requestsStreaming(payload) {
|
|
50763
|
+
return asRecord(payload)?.stream === true;
|
|
50764
|
+
}
|
|
50765
|
+
function requestsVision(payload) {
|
|
50766
|
+
return containsVisionPart(asRecord(payload)?.messages) || containsVisionPart(asRecord(payload)?.input);
|
|
50767
|
+
}
|
|
50768
|
+
function containsVisionPart(value) {
|
|
50769
|
+
if (Array.isArray(value)) return value.some(containsVisionPart);
|
|
50770
|
+
const record = asRecord(value);
|
|
50771
|
+
if (!record) return false;
|
|
50772
|
+
if (record.type === "image" || record.type === "image_url" || record.type === "input_image") return true;
|
|
50773
|
+
return containsVisionPart(record.content);
|
|
50774
|
+
}
|
|
50775
|
+
function requestsReasoningEffort(protocol, payload) {
|
|
50776
|
+
const record = asRecord(payload);
|
|
50777
|
+
if (!record) return false;
|
|
50778
|
+
const effort = protocol === "anthropic-messages" ? asRecord(record.output_config)?.effort : protocol === "responses" ? asRecord(record.reasoning)?.effort : record.reasoning_effort;
|
|
50779
|
+
return effort !== void 0 && effort !== "none";
|
|
50780
|
+
}
|
|
50781
|
+
function requestsThinking(protocol, payload) {
|
|
50782
|
+
const record = asRecord(payload);
|
|
50783
|
+
if (!record) return false;
|
|
50784
|
+
if (protocol === "anthropic-messages") {
|
|
50785
|
+
const type = asRecord(record.thinking)?.type;
|
|
50786
|
+
return type === "enabled" || type === "adaptive";
|
|
50787
|
+
}
|
|
50788
|
+
if (protocol === "openai-chat") return typeof record.thinking_budget === "number" && record.thinking_budget > 0;
|
|
50789
|
+
return false;
|
|
50790
|
+
}
|
|
50791
|
+
function requestsStructuredOutput(protocol, payload) {
|
|
50792
|
+
const record = asRecord(payload);
|
|
50793
|
+
if (!record) return false;
|
|
50794
|
+
if (protocol === "anthropic-messages") return asRecord(record.output_config)?.format !== void 0;
|
|
50795
|
+
if (protocol === "responses") {
|
|
50796
|
+
const type = asRecord(asRecord(record.text)?.format)?.type;
|
|
50797
|
+
return type !== void 0 && type !== "text";
|
|
50798
|
+
}
|
|
50799
|
+
const responseFormat = asRecord(record.response_format);
|
|
50800
|
+
return responseFormat?.type !== void 0 && responseFormat.type !== "text";
|
|
50801
|
+
}
|
|
50802
|
+
function supportsStructuredOutput(protocol, model) {
|
|
50803
|
+
if (protocol === "anthropic-messages") return modelCache.supportsStructuredOutputs(model) || modelCache.supportsEndpoint(model, "/responses");
|
|
50804
|
+
return model.capabilities.supports.structured_outputs ?? false;
|
|
50805
|
+
}
|
|
50806
|
+
function asRecord(value) {
|
|
50807
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : void 0;
|
|
50808
|
+
}
|
|
50809
|
+
function discloseActualModel(result, model) {
|
|
50810
|
+
if (result.kind === "json") return {
|
|
50811
|
+
kind: "json",
|
|
50812
|
+
data: replaceKnownModelIdentity(result.data, model)
|
|
50813
|
+
};
|
|
50814
|
+
return {
|
|
50815
|
+
kind: "stream",
|
|
50816
|
+
generator: discloseStreamModel(result.generator, model)
|
|
50817
|
+
};
|
|
50818
|
+
}
|
|
50819
|
+
async function* discloseStreamModel(generator, model) {
|
|
50820
|
+
for await (const chunk of generator) {
|
|
50821
|
+
if (!chunk.data || chunk.data === "[DONE]") {
|
|
50822
|
+
yield chunk;
|
|
50823
|
+
continue;
|
|
50824
|
+
}
|
|
50825
|
+
try {
|
|
50826
|
+
const parsed = JSON.parse(chunk.data);
|
|
50827
|
+
const replaced = replaceKnownModelIdentity(parsed, model);
|
|
50828
|
+
yield replaced === parsed ? chunk : {
|
|
50829
|
+
...chunk,
|
|
50830
|
+
data: JSON.stringify(replaced)
|
|
50831
|
+
};
|
|
50832
|
+
} catch {
|
|
50833
|
+
yield chunk;
|
|
50834
|
+
}
|
|
50835
|
+
}
|
|
50836
|
+
}
|
|
50837
|
+
function replaceKnownModelIdentity(value, model) {
|
|
50838
|
+
const record = asRecord(value);
|
|
50839
|
+
if (!record) return value;
|
|
50840
|
+
let changed = false;
|
|
50841
|
+
const next = { ...record };
|
|
50842
|
+
if (typeof record.model === "string") {
|
|
50843
|
+
next.model = model;
|
|
50844
|
+
changed = true;
|
|
50845
|
+
}
|
|
50846
|
+
for (const key of ["message", "response"]) {
|
|
50847
|
+
const nested = asRecord(record[key]);
|
|
50848
|
+
if (nested && typeof nested.model === "string") {
|
|
50849
|
+
next[key] = {
|
|
50850
|
+
...nested,
|
|
50851
|
+
model
|
|
50852
|
+
};
|
|
50853
|
+
changed = true;
|
|
50854
|
+
}
|
|
50855
|
+
}
|
|
50856
|
+
return changed ? next : value;
|
|
50857
|
+
}
|
|
50858
|
+
function emitFallbackEvent(recovery, effectiveModel, decision, status, connectionClass) {
|
|
50859
|
+
const now = performance.now();
|
|
50860
|
+
logRecoveryEvent({
|
|
50861
|
+
requestId: recovery.requestId,
|
|
50862
|
+
callerRequestId: recovery.callerRequestId,
|
|
50863
|
+
event: "fallback",
|
|
50864
|
+
retryCount: recovery.retryCount,
|
|
50865
|
+
effectiveModel,
|
|
50866
|
+
status,
|
|
50867
|
+
connectionClass,
|
|
50868
|
+
...recovery.queueMetrics,
|
|
50869
|
+
...recovery.startedAtMonotonicMs !== void 0 ? { elapsedMs: Math.max(0, now - recovery.startedAtMonotonicMs) } : {},
|
|
50870
|
+
...recovery.deadlineMonotonicMs !== void 0 ? { remainingBudgetMs: Math.max(0, recovery.deadlineMonotonicMs - now) } : {},
|
|
50871
|
+
decision
|
|
50872
|
+
});
|
|
50873
|
+
}
|
|
50874
|
+
function createRecoveryRecord(request) {
|
|
50875
|
+
return {
|
|
50876
|
+
requestId: request.requestId,
|
|
50877
|
+
...request.callerRequestId ? { callerRequestId: request.callerRequestId } : {},
|
|
50878
|
+
callerSignal: request.signal,
|
|
50879
|
+
retryCount: 0
|
|
49674
50880
|
};
|
|
49675
50881
|
}
|
|
49676
50882
|
//#endregion
|
|
@@ -51150,11 +52356,13 @@ const chatCompletionsStrategyRegistry = new StrategyRegistry();
|
|
|
51150
52356
|
chatCompletionsStrategyRegistry.register(chatCompletionsEntry$1);
|
|
51151
52357
|
//#endregion
|
|
51152
52358
|
//#region src/routes/chat-completions/handler.ts
|
|
51153
|
-
async function handleCompletionCore({ body, signal, headers }) {
|
|
52359
|
+
async function handleCompletionCore({ body, signal, headers, requestId, callerRequestId }) {
|
|
51154
52360
|
return runPipeline({
|
|
51155
52361
|
body,
|
|
51156
52362
|
signal,
|
|
51157
|
-
headers
|
|
52363
|
+
headers,
|
|
52364
|
+
requestId,
|
|
52365
|
+
callerRequestId
|
|
51158
52366
|
}, {
|
|
51159
52367
|
protocol: "openai-chat",
|
|
51160
52368
|
strategyRegistry: chatCompletionsStrategyRegistry,
|
|
@@ -51196,7 +52404,8 @@ function createCompletionRoutes() {
|
|
|
51196
52404
|
const { result, modelMapping } = await handleCompletionCore({
|
|
51197
52405
|
body,
|
|
51198
52406
|
signal: request.signal,
|
|
51199
|
-
headers: request.headers
|
|
52407
|
+
headers: request.headers,
|
|
52408
|
+
...getOrCreateRequestCorrelation(request)
|
|
51200
52409
|
});
|
|
51201
52410
|
const delivery = deliverResult(request, result, modelMapping);
|
|
51202
52411
|
if (!delivery.streaming) return delivery.data;
|
|
@@ -51421,12 +52630,19 @@ function normalizeSchemaNode(node) {
|
|
|
51421
52630
|
}
|
|
51422
52631
|
normalized[key] = normalizeSchemaNode(value);
|
|
51423
52632
|
}
|
|
51424
|
-
if (node.type === "object" || isRecord$2(normalized.properties)) {
|
|
51425
|
-
normalized.required = isRecord$2(normalized.properties) ? Object.keys(normalized.properties) : [];
|
|
51426
|
-
normalized.additionalProperties = false;
|
|
51427
|
-
}
|
|
51428
52633
|
return normalized;
|
|
51429
52634
|
}
|
|
52635
|
+
/**
|
|
52636
|
+
* Strip JSON Schema / OpenAPI annotations Copilot's function-schema validator
|
|
52637
|
+
* rejects, leaving the structural schema — including the caller's own
|
|
52638
|
+
* `required` array and `additionalProperties` — untouched.
|
|
52639
|
+
*
|
|
52640
|
+
* The annotation stripping is currently inert: probed 2026-08-06
|
|
52641
|
+
* (`scripts/probes/tool-strict.ts`), upstream accepts every annotation in the
|
|
52642
|
+
* list on every `/responses` model with `strict` omitted. It stays anyway — the
|
|
52643
|
+
* list was written against the upstream of 2026-04, and a probe result is a
|
|
52644
|
+
* dated snapshot rather than a permanent fact.
|
|
52645
|
+
*/
|
|
51430
52646
|
function normalizeFunctionParametersSchemaForCopilot(schema) {
|
|
51431
52647
|
if (!schema) return schema;
|
|
51432
52648
|
return normalizeSchemaNode(schema);
|
|
@@ -51742,7 +52958,6 @@ function convertAnthropicTools(tools) {
|
|
|
51742
52958
|
type: "function",
|
|
51743
52959
|
name: tool.name,
|
|
51744
52960
|
parameters: normalizeFunctionParametersSchemaForCopilot(tool.input_schema),
|
|
51745
|
-
strict: false,
|
|
51746
52961
|
...tool.description ? { description: tool.description } : {}
|
|
51747
52962
|
}));
|
|
51748
52963
|
}
|
|
@@ -52617,34 +53832,45 @@ defaultStrategyRegistry.register(responsesApiEntry);
|
|
|
52617
53832
|
defaultStrategyRegistry.register(chatCompletionsEntry);
|
|
52618
53833
|
//#endregion
|
|
52619
53834
|
//#region src/routes/messages/handler.ts
|
|
52620
|
-
async function handleMessagesCore({ body, signal, headers }) {
|
|
53835
|
+
async function handleMessagesCore({ body, signal, headers, requestId, callerRequestId }) {
|
|
52621
53836
|
let anthropicBetaHeader;
|
|
52622
|
-
|
|
52623
|
-
|
|
52624
|
-
|
|
52625
|
-
|
|
52626
|
-
|
|
52627
|
-
|
|
52628
|
-
|
|
52629
|
-
|
|
52630
|
-
|
|
52631
|
-
|
|
52632
|
-
|
|
52633
|
-
|
|
52634
|
-
|
|
52635
|
-
|
|
52636
|
-
|
|
52637
|
-
|
|
52638
|
-
|
|
52639
|
-
|
|
52640
|
-
|
|
52641
|
-
|
|
52642
|
-
|
|
52643
|
-
|
|
52644
|
-
|
|
52645
|
-
|
|
52646
|
-
|
|
52647
|
-
|
|
53837
|
+
try {
|
|
53838
|
+
return await runPipeline({
|
|
53839
|
+
body,
|
|
53840
|
+
signal,
|
|
53841
|
+
headers,
|
|
53842
|
+
requestId,
|
|
53843
|
+
callerRequestId
|
|
53844
|
+
}, {
|
|
53845
|
+
protocol: "anthropic-messages",
|
|
53846
|
+
applyModelPolicy: true,
|
|
53847
|
+
strategyRegistry: defaultStrategyRegistry,
|
|
53848
|
+
afterIngest({ payload, headers: reqHeaders }) {
|
|
53849
|
+
if (consola.level >= 4) consola.debug("Anthropic request payload:", JSON.stringify(payload));
|
|
53850
|
+
anthropicBetaHeader = processAnthropicBetaHeader(reqHeaders.get("anthropic-beta"));
|
|
53851
|
+
return payload;
|
|
53852
|
+
},
|
|
53853
|
+
buildStrategyContext({ payload, meta, headers: reqHeaders, selectedModel, copilotClient, upstreamSignal, modelMapping }) {
|
|
53854
|
+
return {
|
|
53855
|
+
copilotClient,
|
|
53856
|
+
anthropicPayload: payload,
|
|
53857
|
+
anthropicBetaHeader,
|
|
53858
|
+
selectedModel,
|
|
53859
|
+
upstreamSignal,
|
|
53860
|
+
headers: reqHeaders,
|
|
53861
|
+
requestContext: meta.requestContext ?? {},
|
|
53862
|
+
modelMapping
|
|
53863
|
+
};
|
|
53864
|
+
}
|
|
53865
|
+
});
|
|
53866
|
+
} catch (error) {
|
|
53867
|
+
if (!(error instanceof HTTPError)) throw error;
|
|
53868
|
+
const body = "type" in error.body ? error.body : {
|
|
53869
|
+
type: "error",
|
|
53870
|
+
...error.body
|
|
53871
|
+
};
|
|
53872
|
+
throw new HTTPError(error.status, body, { headers: error.headers });
|
|
53873
|
+
}
|
|
52648
53874
|
}
|
|
52649
53875
|
//#endregion
|
|
52650
53876
|
//#region src/routes/messages/route.ts
|
|
@@ -52654,7 +53880,8 @@ function createMessageRoutes() {
|
|
|
52654
53880
|
const { result, modelMapping } = await handleMessagesCore({
|
|
52655
53881
|
body,
|
|
52656
53882
|
signal: request.signal,
|
|
52657
|
-
headers: request.headers
|
|
53883
|
+
headers: request.headers,
|
|
53884
|
+
...getOrCreateRequestCorrelation(request)
|
|
52658
53885
|
});
|
|
52659
53886
|
const delivery = deliverResult(request, result, modelMapping);
|
|
52660
53887
|
if (!delivery.streaming) return delivery.data;
|
|
@@ -53066,14 +54293,16 @@ const HTTP_URL_RE = /^https?:\/\//i;
|
|
|
53066
54293
|
* emulator request prep, tool/input policies, and context management applied
|
|
53067
54294
|
* through the afterIngest / afterTransform lifecycle hooks.
|
|
53068
54295
|
*/
|
|
53069
|
-
async function handleResponsesCore({ body, signal, headers }) {
|
|
54296
|
+
async function handleResponsesCore({ body, signal, headers, requestId, callerRequestId }) {
|
|
53070
54297
|
const emulatorMode = configStore.isEmulatorEnabled();
|
|
53071
54298
|
let originalPayload;
|
|
53072
54299
|
let emulatorPrepared;
|
|
53073
54300
|
return await runPipeline({
|
|
53074
54301
|
body,
|
|
53075
54302
|
signal,
|
|
53076
|
-
headers
|
|
54303
|
+
headers,
|
|
54304
|
+
requestId,
|
|
54305
|
+
callerRequestId
|
|
53077
54306
|
}, {
|
|
53078
54307
|
protocol: "responses",
|
|
53079
54308
|
strategyRegistry: responsesStrategyRegistry,
|
|
@@ -53093,7 +54322,7 @@ async function handleResponsesCore({ body, signal, headers }) {
|
|
|
53093
54322
|
clampResponsesOutputTokens(payload);
|
|
53094
54323
|
clampResponsesReasoningEffort(payload, selectedModel);
|
|
53095
54324
|
},
|
|
53096
|
-
buildStrategyContext({ payload, meta, copilotClient, upstreamSignal }) {
|
|
54325
|
+
buildStrategyContext({ payload, meta, selectedModel, copilotClient, upstreamSignal }) {
|
|
53097
54326
|
const { vision, initiator } = getResponsesRequestOptions(payload);
|
|
53098
54327
|
const prepared = emulatorPrepared;
|
|
53099
54328
|
const requestPayload = originalPayload ?? payload;
|
|
@@ -53104,7 +54333,10 @@ async function handleResponsesCore({ body, signal, headers }) {
|
|
|
53104
54333
|
requestContext: meta.requestContext ?? {},
|
|
53105
54334
|
vision,
|
|
53106
54335
|
initiator,
|
|
53107
|
-
decorateResponse: prepared ? (response) => decorateStoredResponse(
|
|
54336
|
+
decorateResponse: prepared ? (response) => decorateStoredResponse({
|
|
54337
|
+
...response,
|
|
54338
|
+
model: selectedModel?.id ?? response.model
|
|
54339
|
+
}, requestPayload, prepared) : void 0,
|
|
53108
54340
|
onTerminalResponse: prepared ? (terminalResponse) => {
|
|
53109
54341
|
if (!prepared.shouldStore) return;
|
|
53110
54342
|
persistEmulatorResponse(terminalResponse, prepared.effectiveInputItems);
|
|
@@ -53116,16 +54348,16 @@ async function handleResponsesCore({ body, signal, headers }) {
|
|
|
53116
54348
|
function applyResponsesToolTransforms(payload) {
|
|
53117
54349
|
applyFunctionApplyPatch(payload);
|
|
53118
54350
|
applyFunctionToolCompatibilityDefaults(payload);
|
|
53119
|
-
rejectUnsupportedBuiltinTools(payload);
|
|
53120
54351
|
}
|
|
53121
54352
|
function applyFunctionToolCompatibilityDefaults(payload) {
|
|
53122
54353
|
if (!Array.isArray(payload.tools)) return;
|
|
53123
54354
|
payload.tools = payload.tools.map((tool) => {
|
|
53124
54355
|
if (!isResponseFunctionTool(tool)) return tool;
|
|
54356
|
+
const { strict, ...rest } = tool;
|
|
53125
54357
|
return {
|
|
53126
|
-
...
|
|
54358
|
+
...rest,
|
|
53127
54359
|
parameters: normalizeFunctionParametersSchemaForCopilot(tool.parameters),
|
|
53128
|
-
strict
|
|
54360
|
+
...strict != null ? { strict } : {}
|
|
53129
54361
|
};
|
|
53130
54362
|
});
|
|
53131
54363
|
}
|
|
@@ -53152,11 +54384,6 @@ function applyFunctionApplyPatch(payload) {
|
|
|
53152
54384
|
return tool;
|
|
53153
54385
|
});
|
|
53154
54386
|
}
|
|
53155
|
-
function rejectUnsupportedBuiltinTools(payload) {
|
|
53156
|
-
if (payload.tool_choice && typeof payload.tool_choice === "object" && "type" in payload.tool_choice && (payload.tool_choice.type === "web_search_preview" || payload.tool_choice.type === "web_search_preview_2025_03_11")) throwInvalidRequestError("The selected Copilot endpoint does not support the Responses web_search tool.", "tool_choice", "unsupported_tool_web_search");
|
|
53157
|
-
if (!Array.isArray(payload.tools)) return;
|
|
53158
|
-
for (const tool of payload.tools) if (tool.type === "web_search") throwInvalidRequestError("The selected Copilot endpoint does not support the Responses web_search tool.", "tools", "unsupported_tool_web_search");
|
|
53159
|
-
}
|
|
53160
54387
|
function applyResponsesInputPolicies(payload) {
|
|
53161
54388
|
payload.store = false;
|
|
53162
54389
|
stripUnresolvableInputItems(payload);
|
|
@@ -53338,7 +54565,8 @@ function createResponsesRoutes() {
|
|
|
53338
54565
|
const { result, modelMapping } = await handleResponsesCore({
|
|
53339
54566
|
body,
|
|
53340
54567
|
signal: request.signal,
|
|
53341
|
-
headers: request.headers
|
|
54568
|
+
headers: request.headers,
|
|
54569
|
+
...getOrCreateRequestCorrelation(request)
|
|
53342
54570
|
});
|
|
53343
54571
|
const delivery = deliverResult(request, result, modelMapping);
|
|
53344
54572
|
if (!delivery.streaming) return delivery.data;
|
|
@@ -53403,6 +54631,65 @@ function createUsageRoute() {
|
|
|
53403
54631
|
//#region src/server.ts
|
|
53404
54632
|
const isBun = typeof globalThis.Bun !== "undefined";
|
|
53405
54633
|
/**
|
|
54634
|
+
* Smallest and largest status an error may claim for itself.
|
|
54635
|
+
*
|
|
54636
|
+
* An error that reports a 2xx/3xx — or a nonsense number — is not describing a
|
|
54637
|
+
* failure the client can act on, so it falls through to 500 rather than turning
|
|
54638
|
+
* a thrown exception into an apparent success.
|
|
54639
|
+
*/
|
|
54640
|
+
const MIN_ERROR_STATUS = 400;
|
|
54641
|
+
const MAX_ERROR_STATUS = 599;
|
|
54642
|
+
/**
|
|
54643
|
+
* The status a thrown value claims for itself, when it claims a plausible one.
|
|
54644
|
+
*
|
|
54645
|
+
* Elysia's built-in error classes (`NotFoundError`, `ParseError`,
|
|
54646
|
+
* `ValidationError`, `InternalServerError`) each declare `status: number` as
|
|
54647
|
+
* part of their public class contract, and this repo's own `TranslationFailure`
|
|
54648
|
+
* declares `status: 400 | 502`. Reading the property instead of mapping
|
|
54649
|
+
* `code` covers all of them, and covers whatever Elysia adds next.
|
|
54650
|
+
*
|
|
54651
|
+
* The read is wrapped because this runs inside the error handler: `status` may
|
|
54652
|
+
* be a getter and `error` may be a Proxy, either of which can throw. This
|
|
54653
|
+
* hardens this function only — Elysia itself does `set.status = error.status`
|
|
54654
|
+
* after `onError` returns (`elysia/dist/compose.mjs`), so a hostile getter
|
|
54655
|
+
* still escapes `app.handle()`. Measured on the pre-fix tree as well, so that
|
|
54656
|
+
* escape is Elysia's, not something reading the property here introduced.
|
|
54657
|
+
* `isTimeoutLikeError` guards its own traversal the same way.
|
|
54658
|
+
*/
|
|
54659
|
+
function claimedErrorStatus(error) {
|
|
54660
|
+
if (typeof error !== "object" || error === null) return void 0;
|
|
54661
|
+
let status;
|
|
54662
|
+
try {
|
|
54663
|
+
if (!("status" in error)) return void 0;
|
|
54664
|
+
status = error.status;
|
|
54665
|
+
} catch {
|
|
54666
|
+
return;
|
|
54667
|
+
}
|
|
54668
|
+
return typeof status === "number" && Number.isInteger(status) && status >= MIN_ERROR_STATUS && status <= MAX_ERROR_STATUS ? status : void 0;
|
|
54669
|
+
}
|
|
54670
|
+
/**
|
|
54671
|
+
* Client-facing error `type` for a locally generated failure.
|
|
54672
|
+
*
|
|
54673
|
+
* The proxy's other error paths already classify by meaning
|
|
54674
|
+
* (`invalid_request_error`, `upstream_error`, `rate_limit_error`,
|
|
54675
|
+
* `timeout_error`), and `upstreamErrorType` in `src/lib/error.ts` states the
|
|
54676
|
+
* rule: a non-standard `type` at the proxy boundary breaks client error
|
|
54677
|
+
* handling. Honoring the thrown error's status made this branch reachable at
|
|
54678
|
+
* 404/400/422 rather than only 500, so it classifies too instead of labelling
|
|
54679
|
+
* every one of them `error`.
|
|
54680
|
+
*/
|
|
54681
|
+
function localErrorType(status) {
|
|
54682
|
+
if (status === 404) return "not_found_error";
|
|
54683
|
+
if (status >= 400 && status < 500) return "invalid_request_error";
|
|
54684
|
+
return "error";
|
|
54685
|
+
}
|
|
54686
|
+
/**
|
|
54687
|
+
* Elysia's `NotFoundError` carries the bare string `NOT_FOUND` as its message.
|
|
54688
|
+
* That is an internal token, not something a client should be shown, so an
|
|
54689
|
+
* unmatched route gets a sentence instead.
|
|
54690
|
+
*/
|
|
54691
|
+
const NOT_FOUND_MESSAGE = "Unknown endpoint. Check the request path.";
|
|
54692
|
+
/**
|
|
53406
54693
|
* Maps a thrown error to a client response.
|
|
53407
54694
|
*
|
|
53408
54695
|
* `set.status` is written on every branch because `onError` returns a fresh
|
|
@@ -53411,6 +54698,11 @@ const isBun = typeof globalThis.Bun !== "undefined";
|
|
|
53411
54698
|
* log in `onAfterResponse` reads it. Without the write-back, a 504 is logged
|
|
53412
54699
|
* as a 500.
|
|
53413
54700
|
*
|
|
54701
|
+
* Errors that carry their own plausible status keep it. Flattening every
|
|
54702
|
+
* non-HTTPError to 500 turned an unmatched route into `500 NaNs` on an
|
|
54703
|
+
* OpenAI-compatible surface where an unknown path owes the client a 404, and
|
|
54704
|
+
* hid `TranslationFailure`'s 502 behind a generic 500.
|
|
54705
|
+
*
|
|
53414
54706
|
* Exported so tests exercise this mapping rather than a copy of it.
|
|
53415
54707
|
*/
|
|
53416
54708
|
function handleRouteError({ code, error, set }) {
|
|
@@ -53422,32 +54714,33 @@ function handleRouteError({ code, error, set }) {
|
|
|
53422
54714
|
type: "timeout_error"
|
|
53423
54715
|
} }, { status: 504 });
|
|
53424
54716
|
}
|
|
53425
|
-
const
|
|
53426
|
-
|
|
54717
|
+
const status = claimedErrorStatus(error) ?? 500;
|
|
54718
|
+
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
54719
|
+
const message = code === "NOT_FOUND" ? NOT_FOUND_MESSAGE : rawMessage;
|
|
54720
|
+
set.status = status;
|
|
53427
54721
|
return Response.json({ error: {
|
|
53428
54722
|
message,
|
|
53429
|
-
type:
|
|
53430
|
-
} }, { status
|
|
54723
|
+
type: localErrorType(status)
|
|
54724
|
+
} }, { status });
|
|
53431
54725
|
}
|
|
53432
|
-
function createServer(options) {
|
|
54726
|
+
function createServer$1(options) {
|
|
53433
54727
|
return new Elysia({
|
|
53434
54728
|
adapter: isBun ? void 0 : node(),
|
|
53435
54729
|
serve: options?.idleTimeout !== void 0 ? { idleTimeout: options.idleTimeout } : void 0
|
|
53436
|
-
}).use(cors()).error({ HTTP: HTTPError }).
|
|
53437
|
-
|
|
53438
|
-
|
|
53439
|
-
|
|
54730
|
+
}).use(cors()).error({ HTTP: HTTPError }).onRequest(({ request }) => {
|
|
54731
|
+
markRequestStart(request);
|
|
54732
|
+
}).derive(({ request }) => ({ ...getOrCreateRequestCorrelation(request) })).onBeforeHandle(({ body, request, responseRequestId, set }) => {
|
|
54733
|
+
set.headers["x-request-id"] = responseRequestId;
|
|
53440
54734
|
if (request.method !== "POST") return;
|
|
53441
54735
|
const model = body && typeof body === "object" && "model" in body ? body.model : void 0;
|
|
53442
54736
|
if (typeof model === "string") setRequestModelMapping(request, {
|
|
53443
54737
|
originalModel: model,
|
|
53444
54738
|
steps: []
|
|
53445
54739
|
});
|
|
53446
|
-
}).onAfterResponse(({
|
|
53447
|
-
|
|
53448
|
-
const elapsed = formatElapsed(requestStart);
|
|
54740
|
+
}).onAfterResponse(({ callerRequestId, request, requestId, set }) => {
|
|
54741
|
+
const elapsed = formatElapsed(getRequestStart(request));
|
|
53449
54742
|
const status = typeof set.status === "number" ? set.status : 200;
|
|
53450
|
-
logRequest(request.method, request.url, status, elapsed, getRequestModelMapping(request), requestId);
|
|
54743
|
+
logRequest(request.method, request.url, status, elapsed, getRequestModelMapping(request), requestId, callerRequestId);
|
|
53451
54744
|
}).onError(({ code, error, set }) => handleRouteError({
|
|
53452
54745
|
code,
|
|
53453
54746
|
error,
|
|
@@ -53510,11 +54803,13 @@ async function runServer(options) {
|
|
|
53510
54803
|
const cachedConfig = getCachedConfig();
|
|
53511
54804
|
const upstreamQueueConcurrency = options.upstreamQueueConcurrency ?? cachedConfig.upstreamQueueConcurrency;
|
|
53512
54805
|
const upstreamQueueMaxRetries = options.upstreamQueueMaxRetries ?? cachedConfig.upstreamQueueMaxRetries;
|
|
54806
|
+
const upstreamRecoveryBudgetSeconds = options.upstreamRecoveryBudgetSeconds ?? cachedConfig.upstreamRecoveryBudgetSeconds;
|
|
53513
54807
|
const upstreamQueueBaseDelaySeconds = options.upstreamQueueBaseDelaySeconds ?? cachedConfig.upstreamQueueBaseDelaySeconds;
|
|
53514
54808
|
const upstreamQueueMaxDelaySeconds = options.upstreamQueueMaxDelaySeconds ?? cachedConfig.upstreamQueueMaxDelaySeconds;
|
|
53515
54809
|
configureUpstreamRequestQueue({
|
|
53516
54810
|
concurrency: upstreamQueueConcurrency,
|
|
53517
54811
|
maxRetries: upstreamQueueMaxRetries,
|
|
54812
|
+
recoveryBudgetMs: secondsToMs(upstreamRecoveryBudgetSeconds),
|
|
53518
54813
|
baseDelayMs: secondsToMs(upstreamQueueBaseDelaySeconds),
|
|
53519
54814
|
maxDelayMs: secondsToMs(upstreamQueueMaxDelaySeconds)
|
|
53520
54815
|
});
|
|
@@ -53529,7 +54824,7 @@ async function runServer(options) {
|
|
|
53529
54824
|
await maybeCopyClaudeCodeCommand(serverUrl);
|
|
53530
54825
|
}
|
|
53531
54826
|
printStartupBanner(serverUrl);
|
|
53532
|
-
const app = createServer({ idleTimeout: options.idleTimeoutSeconds });
|
|
54827
|
+
const app = createServer$1({ idleTimeout: options.idleTimeoutSeconds });
|
|
53533
54828
|
app.listen(options.port);
|
|
53534
54829
|
const shutdown = async () => {
|
|
53535
54830
|
consola.info("Shutting down gracefully...");
|
|
@@ -53549,6 +54844,15 @@ function parseIntArg(raw, name, fallbackMsg) {
|
|
|
53549
54844
|
}
|
|
53550
54845
|
return n;
|
|
53551
54846
|
}
|
|
54847
|
+
function parseBoundedIntArg(raw, name, fallbackMsg, min, max) {
|
|
54848
|
+
if (raw === void 0) return void 0;
|
|
54849
|
+
const value = Number(raw);
|
|
54850
|
+
if (!raw.trim() || !Number.isInteger(value) || value < min || value > max) {
|
|
54851
|
+
consola.warn(`Invalid --${name} value "${raw}". ${fallbackMsg}`);
|
|
54852
|
+
return;
|
|
54853
|
+
}
|
|
54854
|
+
return value;
|
|
54855
|
+
}
|
|
53552
54856
|
function secondsToMs(seconds) {
|
|
53553
54857
|
return seconds === void 0 ? void 0 : seconds * 1e3;
|
|
53554
54858
|
}
|
|
@@ -53633,7 +54937,11 @@ const start = defineCommand({
|
|
|
53633
54937
|
},
|
|
53634
54938
|
"upstream-queue-retries": {
|
|
53635
54939
|
type: "string",
|
|
53636
|
-
description: "Maximum retries for transient upstream responses (default:
|
|
54940
|
+
description: "Maximum retries for transient upstream responses (0-2, default: 1)"
|
|
54941
|
+
},
|
|
54942
|
+
"upstream-recovery-budget": {
|
|
54943
|
+
type: "string",
|
|
54944
|
+
description: "Recovery budget in seconds after the first retryable outcome (1-120, default: 60)"
|
|
53637
54945
|
},
|
|
53638
54946
|
"upstream-queue-base-delay": {
|
|
53639
54947
|
type: "string",
|
|
@@ -53660,7 +54968,8 @@ const start = defineCommand({
|
|
|
53660
54968
|
const idleTimeoutSeconds = parseIntArg(args["idle-timeout"], "idle-timeout", "Falling back to Bun default.");
|
|
53661
54969
|
const upstreamTimeoutSeconds = parseIntArg(args["upstream-timeout"], "upstream-timeout", "Falling back to default (300s).");
|
|
53662
54970
|
const upstreamQueueConcurrency = parseIntArg(args["upstream-queue-concurrency"], "upstream-queue-concurrency", "Using default upstream queue concurrency.");
|
|
53663
|
-
const upstreamQueueMaxRetries =
|
|
54971
|
+
const upstreamQueueMaxRetries = parseBoundedIntArg(args["upstream-queue-retries"], "upstream-queue-retries", "Using default upstream queue retry count.", 0, 2);
|
|
54972
|
+
const upstreamRecoveryBudgetSeconds = parseBoundedIntArg(args["upstream-recovery-budget"], "upstream-recovery-budget", "Using default upstream recovery budget.", 1, 120);
|
|
53664
54973
|
const upstreamQueueBaseDelaySeconds = parseIntArg(args["upstream-queue-base-delay"], "upstream-queue-base-delay", "Using default upstream queue base delay.");
|
|
53665
54974
|
const upstreamQueueMaxDelaySeconds = parseIntArg(args["upstream-queue-max-delay"], "upstream-queue-max-delay", "Using default upstream queue max delay.");
|
|
53666
54975
|
return runServer({
|
|
@@ -53678,6 +54987,7 @@ const start = defineCommand({
|
|
|
53678
54987
|
upstreamTimeoutSeconds,
|
|
53679
54988
|
upstreamQueueConcurrency,
|
|
53680
54989
|
upstreamQueueMaxRetries,
|
|
54990
|
+
upstreamRecoveryBudgetSeconds,
|
|
53681
54991
|
upstreamQueueBaseDelaySeconds,
|
|
53682
54992
|
upstreamQueueMaxDelaySeconds,
|
|
53683
54993
|
gheDomain: args["ghe-domain"],
|