fest-gateway 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +274 -0
- package/dist/cli/claude.js +326 -0
- package/dist/cli/claude.js.map +1 -0
- package/dist/cli/config.js +59 -0
- package/dist/cli/config.js.map +1 -0
- package/dist/cli/login.js +130 -0
- package/dist/cli/login.js.map +1 -0
- package/dist/cli/logout.js +20 -0
- package/dist/cli/logout.js.map +1 -0
- package/dist/cli/whoami.js +34 -0
- package/dist/cli/whoami.js.map +1 -0
- package/dist/server/adapters/anthropic.js +74 -0
- package/dist/server/adapters/anthropic.js.map +1 -0
- package/dist/server/adapters/fireworks.js +54 -0
- package/dist/server/adapters/fireworks.js.map +1 -0
- package/dist/server/adapters/index.js +16 -0
- package/dist/server/adapters/index.js.map +1 -0
- package/dist/server/adapters/registry.js +38 -0
- package/dist/server/adapters/registry.js.map +1 -0
- package/dist/server/adapters/rewrite.js +152 -0
- package/dist/server/adapters/rewrite.js.map +1 -0
- package/dist/server/adapters/url.js +24 -0
- package/dist/server/adapters/url.js.map +1 -0
- package/dist/server/api/auth.js +213 -0
- package/dist/server/api/auth.js.map +1 -0
- package/dist/server/api/live.js +108 -0
- package/dist/server/api/live.js.map +1 -0
- package/dist/server/api/models.js +129 -0
- package/dist/server/api/models.js.map +1 -0
- package/dist/server/api/oauth.js +300 -0
- package/dist/server/api/oauth.js.map +1 -0
- package/dist/server/api/routes.js +164 -0
- package/dist/server/api/routes.js.map +1 -0
- package/dist/server/api/routing-view.js +47 -0
- package/dist/server/api/routing-view.js.map +1 -0
- package/dist/server/auth/accounts.js +111 -0
- package/dist/server/auth/accounts.js.map +1 -0
- package/dist/server/auth/gateway-401.js +85 -0
- package/dist/server/auth/gateway-401.js.map +1 -0
- package/dist/server/auth/guard.js +74 -0
- package/dist/server/auth/guard.js.map +1 -0
- package/dist/server/auth/oauth.js +43 -0
- package/dist/server/auth/oauth.js.map +1 -0
- package/dist/server/auth/password.js +104 -0
- package/dist/server/auth/password.js.map +1 -0
- package/dist/server/auth/posture.js +145 -0
- package/dist/server/auth/posture.js.map +1 -0
- package/dist/server/auth/session.js +161 -0
- package/dist/server/auth/session.js.map +1 -0
- package/dist/server/bin/fest.js +589 -0
- package/dist/server/bin/fest.js.map +1 -0
- package/dist/server/config.js +109 -0
- package/dist/server/config.js.map +1 -0
- package/dist/server/credentials/provider.js +70 -0
- package/dist/server/credentials/provider.js.map +1 -0
- package/dist/server/credentials/resolve.js +89 -0
- package/dist/server/credentials/resolve.js.map +1 -0
- package/dist/server/http/body.js +66 -0
- package/dist/server/http/body.js.map +1 -0
- package/dist/server/http/errors.js +144 -0
- package/dist/server/http/errors.js.map +1 -0
- package/dist/server/http/headers.js +189 -0
- package/dist/server/http/headers.js.map +1 -0
- package/dist/server/http/pipe.js +97 -0
- package/dist/server/http/pipe.js.map +1 -0
- package/dist/server/http/server.js +226 -0
- package/dist/server/http/server.js.map +1 -0
- package/dist/server/http/sse.js +167 -0
- package/dist/server/http/sse.js.map +1 -0
- package/dist/server/http/static.js +138 -0
- package/dist/server/http/static.js.map +1 -0
- package/dist/server/ingest/live-bus.js +67 -0
- package/dist/server/ingest/live-bus.js.map +1 -0
- package/dist/server/ingest/sink.js +141 -0
- package/dist/server/ingest/sink.js.map +1 -0
- package/dist/server/log.js +31 -0
- package/dist/server/log.js.map +1 -0
- package/dist/server/pipeline/count-tokens.js +48 -0
- package/dist/server/pipeline/count-tokens.js.map +1 -0
- package/dist/server/pipeline/dispatch.js +138 -0
- package/dist/server/pipeline/dispatch.js.map +1 -0
- package/dist/server/pipeline/passthrough.js +392 -0
- package/dist/server/pipeline/passthrough.js.map +1 -0
- package/dist/server/pipeline/preflight.js +66 -0
- package/dist/server/pipeline/preflight.js.map +1 -0
- package/dist/server/pipeline/record.js +49 -0
- package/dist/server/pipeline/record.js.map +1 -0
- package/dist/server/pipeline/substitute.js +287 -0
- package/dist/server/pipeline/substitute.js.map +1 -0
- package/dist/server/routes/resolve.js +176 -0
- package/dist/server/routes/resolve.js.map +1 -0
- package/dist/server/routes/table.js +247 -0
- package/dist/server/routes/table.js.map +1 -0
- package/dist/server/routes/watch.js +101 -0
- package/dist/server/routes/watch.js.map +1 -0
- package/dist/server/secret/fingerprint.js +107 -0
- package/dist/server/secret/fingerprint.js.map +1 -0
- package/dist/server/secret/non-persistable.js +69 -0
- package/dist/server/secret/non-persistable.js.map +1 -0
- package/dist/server/store/audit.js +53 -0
- package/dist/server/store/audit.js.map +1 -0
- package/dist/server/store/bootstrap.js +53 -0
- package/dist/server/store/bootstrap.js.map +1 -0
- package/dist/server/store/db.js +157 -0
- package/dist/server/store/db.js.map +1 -0
- package/dist/server/store/ids.js +14 -0
- package/dist/server/store/ids.js.map +1 -0
- package/dist/server/store/migrations/001-init.sql +213 -0
- package/dist/server/store/migrations/002-feed-and-error-indexes.sql +26 -0
- package/dist/server/store/migrations/003-routing.sql +42 -0
- package/dist/server/store/migrations/004-admin-auth.sql +72 -0
- package/dist/server/store/migrations/005-notional-cost.sql +28 -0
- package/dist/server/store/queries.js +904 -0
- package/dist/server/store/queries.js.map +1 -0
- package/dist/server/store/retention.js +79 -0
- package/dist/server/store/retention.js.map +1 -0
- package/dist/server/store/seed.js +203 -0
- package/dist/server/store/seed.js.map +1 -0
- package/dist/server/store/tokens.js +133 -0
- package/dist/server/store/tokens.js.map +1 -0
- package/dist/server/store/write.js +330 -0
- package/dist/server/store/write.js.map +1 -0
- package/dist/server/usage/accumulator.js +209 -0
- package/dist/server/usage/accumulator.js.map +1 -0
- package/dist/server/usage/cost.js +57 -0
- package/dist/server/usage/cost.js.map +1 -0
- package/dist/server/usage/prices/catalog.js +136 -0
- package/dist/server/usage/prices/catalog.js.map +1 -0
- package/dist/server/usage/prices/refresh.js +125 -0
- package/dist/server/usage/prices/refresh.js.map +1 -0
- package/dist/server/usage/prices/snapshot.json.gz +0 -0
- package/dist/server/usage/prices/table.js +102 -0
- package/dist/server/usage/prices/table.js.map +1 -0
- package/dist/server/usage/pricing.js +103 -0
- package/dist/server/usage/pricing.js.map +1 -0
- package/dist/shared/api.js +19 -0
- package/dist/shared/api.js.map +1 -0
- package/dist/shared/base-models.js +28 -0
- package/dist/shared/base-models.js.map +1 -0
- package/dist/shared/demotion-vars.js +17 -0
- package/dist/shared/demotion-vars.js.map +1 -0
- package/dist/shared/series.js +50 -0
- package/dist/shared/series.js.map +1 -0
- package/dist/shared/types.js +40 -0
- package/dist/shared/types.js.map +1 -0
- package/dist/web/dist/assets/index-8HlKsJqt.css +2 -0
- package/dist/web/dist/assets/index-DQ7fyPVm.js +56 -0
- package/dist/web/dist/assets/index-DQ7fyPVm.js.map +1 -0
- package/dist/web/dist/assets/inter-latin-400-normal-C38fXH4l.woff2 +0 -0
- package/dist/web/dist/assets/inter-latin-400-normal-CyCys3Eg.woff +0 -0
- package/dist/web/dist/assets/inter-latin-500-normal-BL9OpVg8.woff +0 -0
- package/dist/web/dist/assets/inter-latin-500-normal-Cerq10X2.woff2 +0 -0
- package/dist/web/dist/assets/inter-latin-600-normal-CiBQ2DWP.woff +0 -0
- package/dist/web/dist/assets/inter-latin-600-normal-LgqL8muc.woff2 +0 -0
- package/dist/web/dist/assets/inter-latin-700-normal-BLAVimhd.woff +0 -0
- package/dist/web/dist/assets/inter-latin-700-normal-Yt3aPRUw.woff2 +0 -0
- package/dist/web/dist/index.html +44 -0
- package/dist/web/dist/nearform-icon.svg +4 -0
- package/package.json +82 -0
- package/routes.example.json +22 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Building a usage record.
|
|
3
|
+
*
|
|
4
|
+
* Shared by both pipelines — but only this. The two pipelines deliberately do
|
|
5
|
+
* not share a request path, because the byte-for-byte rule on the subscription
|
|
6
|
+
* side is only enforceable while there is no common code path tempting someone
|
|
7
|
+
* to add "just one" transform to it. What they SHOULD share is the shape of
|
|
8
|
+
* what they record, so that a field added for one is never silently absent for
|
|
9
|
+
* the other: a dashboard column that is populated on one path and blank on the
|
|
10
|
+
* other is worse than a column that does not exist.
|
|
11
|
+
*/
|
|
12
|
+
import { EMPTY_USAGE } from "../../shared/types.js";
|
|
13
|
+
export function buildRecord(base, partial) {
|
|
14
|
+
return {
|
|
15
|
+
id: base.id,
|
|
16
|
+
startedAt: base.startedAt,
|
|
17
|
+
endedAt: Date.now(),
|
|
18
|
+
posture: base.posture,
|
|
19
|
+
identityCarrier: base.identityCarrier,
|
|
20
|
+
callerFingerprint: base.callerFingerprint,
|
|
21
|
+
userId: base.userId,
|
|
22
|
+
tokenId: base.tokenId,
|
|
23
|
+
credentialFingerprint: base.credentialFingerprint,
|
|
24
|
+
credentialOrigin: base.credentialOrigin,
|
|
25
|
+
sessionId: base.sessionId,
|
|
26
|
+
requestedModel: null,
|
|
27
|
+
servedModel: null,
|
|
28
|
+
upstream: base.upstream,
|
|
29
|
+
stream: false,
|
|
30
|
+
httpStatus: null,
|
|
31
|
+
partial: false,
|
|
32
|
+
usage: EMPTY_USAGE,
|
|
33
|
+
costUsd: null,
|
|
34
|
+
costBasis: "none",
|
|
35
|
+
notionalCostUsd: null,
|
|
36
|
+
ttfbMs: null,
|
|
37
|
+
durationMs: Date.now() - base.startedAt,
|
|
38
|
+
bytesIn: 0,
|
|
39
|
+
bytesOut: 0,
|
|
40
|
+
upstreamRequestId: null,
|
|
41
|
+
rateLimit: null,
|
|
42
|
+
clientVersion: base.clientVersion,
|
|
43
|
+
pipeline: base.pipeline,
|
|
44
|
+
routeId: base.routeId,
|
|
45
|
+
credentialsConsidered: base.credentialsConsidered,
|
|
46
|
+
...partial,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=record.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"record.js","sourceRoot":"","sources":["../../../server/pipeline/record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAQH,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAoBpD,MAAM,UAAU,WAAW,CACzB,IAAgB,EAChB,OAAyD;IAEzD,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;QACjD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;QACvC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,cAAc,EAAE,IAAI;QACpB,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,KAAK;QACb,UAAU,EAAE,IAAI;QAChB,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,WAAW;QAClB,OAAO,EAAE,IAAI;QACb,SAAS,EAAE,MAAM;QACjB,eAAe,EAAE,IAAI;QACrB,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS;QACvC,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,CAAC;QACX,iBAAiB,EAAE,IAAI;QACvB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;QACjD,GAAG,OAAO;KACX,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The substitute pipeline: serve a request from a provider that is not the
|
|
3
|
+
* caller's own Anthropic subscription, on a credential the SERVER holds.
|
|
4
|
+
*
|
|
5
|
+
* This is the path where rewriting is allowed — and where the interesting risk
|
|
6
|
+
* lives. Three invariants hold it together, and each one is a test:
|
|
7
|
+
*
|
|
8
|
+
* 1. **The inbound credential is never forwarded.** The request is built from
|
|
9
|
+
* scratch by the adapter; inbound headers are not copied. A developer's
|
|
10
|
+
* Anthropic bearer reaching Fireworks would be disclosing their personal
|
|
11
|
+
* credential to a third party, and the only reliable way to prevent it is
|
|
12
|
+
* to never build a request from the inbound headers at all.
|
|
13
|
+
* 2. **A missing server credential refuses the request.** It does not fall
|
|
14
|
+
* back to the subscription. Succeeding on the wrong credential — or with
|
|
15
|
+
* the wrong model — is worse than a clear failure, because nobody
|
|
16
|
+
* investigates a success.
|
|
17
|
+
* 3. **Every credential considered is recorded**, including the ones skipped
|
|
18
|
+
* and why.
|
|
19
|
+
*
|
|
20
|
+
* Usage metering is unchanged from the pass-through path, because the adapter
|
|
21
|
+
* targets an Anthropic-compatible Messages endpoint: the same SSE parser and
|
|
22
|
+
* the same accumulator read the same `message_start` / `message_delta` events.
|
|
23
|
+
* A provider that needed a different stream shape would need its own reader —
|
|
24
|
+
* see `Adapter.readUsage`.
|
|
25
|
+
*/
|
|
26
|
+
import { randomUUID } from "node:crypto";
|
|
27
|
+
import { EMPTY_USAGE } from "../../shared/types.js";
|
|
28
|
+
import { anthropicError, statusForErrorType, sseErrorEvent, describeUpstreamFailure, } from "../http/errors.js";
|
|
29
|
+
import { buildDownstreamHeaders } from "../http/headers.js";
|
|
30
|
+
import { createSseParser } from "../http/sse.js";
|
|
31
|
+
import { createUsageAccumulator, usageFromJson } from "../usage/accumulator.js";
|
|
32
|
+
import { priceUsage } from "../usage/pricing.js";
|
|
33
|
+
import { ADAPTER_PRICE_PROVIDERS } from "../adapters/registry.js";
|
|
34
|
+
import { pipeWithTee, beginStream } from "../http/pipe.js";
|
|
35
|
+
import { buildRecord } from "./record.js";
|
|
36
|
+
import { log } from "../log.js";
|
|
37
|
+
/** Refuse a request that cannot be served, having recorded why. */
|
|
38
|
+
export function refuse(res, sink, base, decision, considered, message) {
|
|
39
|
+
const type = "invalid_request_error";
|
|
40
|
+
const status = statusForErrorType(type);
|
|
41
|
+
sink.record(buildRecord({
|
|
42
|
+
...base,
|
|
43
|
+
pipeline: "substitute",
|
|
44
|
+
upstream: decision.upstream?.baseUrl ?? "",
|
|
45
|
+
routeId: decision.route?.id ?? null,
|
|
46
|
+
credentialsConsidered: considered,
|
|
47
|
+
}, {
|
|
48
|
+
status: "bad_request",
|
|
49
|
+
httpStatus: status,
|
|
50
|
+
requestedModel: decision.requestedModel,
|
|
51
|
+
servedModel: null,
|
|
52
|
+
errorType: type,
|
|
53
|
+
errorMessage: message,
|
|
54
|
+
}));
|
|
55
|
+
res.writeHead(status, {
|
|
56
|
+
"content-type": "application/json",
|
|
57
|
+
// A missing server credential is not fixed by trying again. Without this
|
|
58
|
+
// the client retries ten times with backoff before showing the developer a
|
|
59
|
+
// message that was accurate on the first attempt.
|
|
60
|
+
"x-should-retry": "false",
|
|
61
|
+
});
|
|
62
|
+
// Claude Code renders this to the developer mid-task, TRUNCATED, so the
|
|
63
|
+
// first sentence has to carry the whole actionable point.
|
|
64
|
+
res.end(anthropicError(type, message));
|
|
65
|
+
}
|
|
66
|
+
export async function handleSubstitute(_req, res, ctx) {
|
|
67
|
+
const { decision, adapter, sink } = ctx;
|
|
68
|
+
const upstream = decision.upstream;
|
|
69
|
+
if (upstream === null)
|
|
70
|
+
throw new Error("substitute pipeline reached with no upstream");
|
|
71
|
+
// The served model here is a provider-native id (`accounts/fireworks/models/…`)
|
|
72
|
+
// which is ambiguous on its own, so pricing needs to know whose catalog to
|
|
73
|
+
// look in. Derived from the adapter rather than the upstream id, because an
|
|
74
|
+
// operator can name an upstream anything they like.
|
|
75
|
+
const priceProvider = ADAPTER_PRICE_PROVIDERS[upstream.adapter];
|
|
76
|
+
const recordBase = {
|
|
77
|
+
...ctx.base,
|
|
78
|
+
pipeline: "substitute",
|
|
79
|
+
upstream: upstream.baseUrl,
|
|
80
|
+
routeId: decision.route?.id ?? null,
|
|
81
|
+
credentialsConsidered: ctx.considered,
|
|
82
|
+
};
|
|
83
|
+
const finish = (partial) => {
|
|
84
|
+
sink.record(buildRecord(recordBase, partial));
|
|
85
|
+
};
|
|
86
|
+
const plan = adapter.plan({
|
|
87
|
+
path: ctx.path,
|
|
88
|
+
body: ctx.body,
|
|
89
|
+
servedModel: decision.servedModel,
|
|
90
|
+
betas: ctx.betas,
|
|
91
|
+
stream: ctx.stream,
|
|
92
|
+
secret: ctx.secret,
|
|
93
|
+
upstream,
|
|
94
|
+
});
|
|
95
|
+
const base = {
|
|
96
|
+
requestedModel: decision.requestedModel,
|
|
97
|
+
servedModel: decision.servedModel,
|
|
98
|
+
stream: ctx.stream,
|
|
99
|
+
bytesIn: plan.body.byteLength,
|
|
100
|
+
};
|
|
101
|
+
const abort = new AbortController();
|
|
102
|
+
let response;
|
|
103
|
+
try {
|
|
104
|
+
response = await fetch(plan.url, {
|
|
105
|
+
method: "POST",
|
|
106
|
+
headers: plan.headers,
|
|
107
|
+
body: plan.body,
|
|
108
|
+
signal: abort.signal,
|
|
109
|
+
redirect: "error",
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
const type = "api_error";
|
|
114
|
+
log.warn("substitute upstream failed", {
|
|
115
|
+
id: ctx.base.id,
|
|
116
|
+
upstream: upstream.id,
|
|
117
|
+
route: decision.route?.id ?? null,
|
|
118
|
+
error: String(err).slice(0, 200),
|
|
119
|
+
});
|
|
120
|
+
finish({
|
|
121
|
+
...base,
|
|
122
|
+
status: "upstream_error",
|
|
123
|
+
httpStatus: statusForErrorType(type),
|
|
124
|
+
errorType: type,
|
|
125
|
+
errorMessage: String(err).slice(0, 200),
|
|
126
|
+
});
|
|
127
|
+
res.writeHead(statusForErrorType(type), { "content-type": "application/json" });
|
|
128
|
+
res.end(anthropicError(type, `Fest: upstream ${JSON.stringify(upstream.id)} could not be reached. ` +
|
|
129
|
+
`This request was NOT retried against Anthropic — it would have used a different model.`));
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
// A 401 here means the SERVER's credential is bad, which is an operator
|
|
133
|
+
// problem, not a developer one. Recorded as `rejected` so the posture screen
|
|
134
|
+
// can say which credential failed rather than just showing a 401.
|
|
135
|
+
const considered = response.status === 401 || response.status === 403
|
|
136
|
+
? ctx.considered.map((c) => c.result === "used"
|
|
137
|
+
? { ...c, result: "rejected", reason: `upstream returned ${response.status}` }
|
|
138
|
+
: c)
|
|
139
|
+
: ctx.considered;
|
|
140
|
+
const withCreds = { ...recordBase, credentialsConsidered: considered };
|
|
141
|
+
const finishWith = (partial) => {
|
|
142
|
+
sink.record(buildRecord(withCreds, partial));
|
|
143
|
+
};
|
|
144
|
+
const downstream = buildDownstreamHeaders(response.headers, { stream: ctx.stream });
|
|
145
|
+
const upstreamRequestId = response.headers.get("request-id");
|
|
146
|
+
const common = { ...base, httpStatus: response.status, upstreamRequestId };
|
|
147
|
+
// ── upstream refused ───────────────────────────────────────────────────────
|
|
148
|
+
/**
|
|
149
|
+
* Handled before the stream/non-stream split — see the same block in
|
|
150
|
+
* `passthrough.ts` for why. It matters more here: this path can reach a
|
|
151
|
+
* provider whose validation rules are its own, so "which field did Fireworks
|
|
152
|
+
* object to" is a question only its error body can answer, and a substituted
|
|
153
|
+
* request that fails is exactly the one an operator has to explain.
|
|
154
|
+
*/
|
|
155
|
+
if (!response.ok) {
|
|
156
|
+
const failure = describeUpstreamFailure(response.status, await response.text());
|
|
157
|
+
// Logged as well as recorded — see the same block in `passthrough.ts`. The
|
|
158
|
+
// route and upstream ids are here because on this path the first question
|
|
159
|
+
// is always "which provider, chosen by which rule".
|
|
160
|
+
log.warn("upstream refused", {
|
|
161
|
+
id: ctx.base.id,
|
|
162
|
+
status: response.status,
|
|
163
|
+
type: failure.type,
|
|
164
|
+
message: failure.message,
|
|
165
|
+
requestedModel: decision.requestedModel,
|
|
166
|
+
servedModel: decision.servedModel,
|
|
167
|
+
stream: ctx.stream,
|
|
168
|
+
bytesIn: plan.body.byteLength,
|
|
169
|
+
pipeline: "substitute",
|
|
170
|
+
upstream: upstream.id,
|
|
171
|
+
route: decision.route?.id ?? null,
|
|
172
|
+
upstreamRequestId,
|
|
173
|
+
});
|
|
174
|
+
res.writeHead(response.status, buildDownstreamHeaders(response.headers, { stream: false }));
|
|
175
|
+
res.end(failure.text);
|
|
176
|
+
finishWith({
|
|
177
|
+
...common,
|
|
178
|
+
status: "upstream_error",
|
|
179
|
+
bytesOut: Buffer.byteLength(failure.text),
|
|
180
|
+
errorType: failure.type,
|
|
181
|
+
errorMessage: failure.message,
|
|
182
|
+
});
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
// ── non-streaming ──────────────────────────────────────────────────────────
|
|
186
|
+
if (!ctx.stream || response.body === null) {
|
|
187
|
+
const text = await response.text();
|
|
188
|
+
let usage = EMPTY_USAGE;
|
|
189
|
+
try {
|
|
190
|
+
usage =
|
|
191
|
+
adapter.readUsage?.(JSON.parse(text)) ?? usageFromJson(JSON.parse(text)) ?? EMPTY_USAGE;
|
|
192
|
+
}
|
|
193
|
+
catch {
|
|
194
|
+
/* relay regardless; we simply report less */
|
|
195
|
+
}
|
|
196
|
+
// Never `subscription` on this path: a server-held credential is real org
|
|
197
|
+
// spend by definition, whatever the caller presented.
|
|
198
|
+
const priced = priceUsage(decision.servedModel, usage, false, priceProvider);
|
|
199
|
+
res.writeHead(response.status, downstream);
|
|
200
|
+
res.end(text);
|
|
201
|
+
finishWith({
|
|
202
|
+
...common,
|
|
203
|
+
status: "ok",
|
|
204
|
+
usage,
|
|
205
|
+
costUsd: priced.cost,
|
|
206
|
+
costBasis: priced.basis,
|
|
207
|
+
notionalCostUsd: priced.notionalCost,
|
|
208
|
+
bytesOut: Buffer.byteLength(text),
|
|
209
|
+
});
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
// ── streaming ──────────────────────────────────────────────────────────────
|
|
213
|
+
const parser = createSseParser();
|
|
214
|
+
const acc = createUsageAccumulator();
|
|
215
|
+
beginStream(res, response.status, downstream);
|
|
216
|
+
try {
|
|
217
|
+
const result = await pipeWithTee(response.body, res, {
|
|
218
|
+
abort,
|
|
219
|
+
startedAt: ctx.base.startedAt,
|
|
220
|
+
observe: (chunk) => {
|
|
221
|
+
for (const event of parser.push(chunk))
|
|
222
|
+
acc.apply(event);
|
|
223
|
+
},
|
|
224
|
+
});
|
|
225
|
+
for (const event of parser.flush())
|
|
226
|
+
acc.apply(event);
|
|
227
|
+
const usage = acc.snapshot();
|
|
228
|
+
const priced = priceUsage(decision.servedModel, usage, false, priceProvider);
|
|
229
|
+
const streamErr = acc.streamError();
|
|
230
|
+
/**
|
|
231
|
+
* Only a 2xx reaches here — a non-2xx was recorded and relayed above, which
|
|
232
|
+
* is what keeps a provider that refuses everything from reading as a zero
|
|
233
|
+
* error rate. (It once did: status was derived purely from client aborts
|
|
234
|
+
* and in-band SSE error events, so a provider refusing before emitting any
|
|
235
|
+
* event produced a tidy, successful-looking record.)
|
|
236
|
+
*
|
|
237
|
+
* A client abort still wins over an in-band stream error: the developer
|
|
238
|
+
* walking away is the more specific fact about what happened.
|
|
239
|
+
*/
|
|
240
|
+
let status = "ok";
|
|
241
|
+
if (result.clientAborted)
|
|
242
|
+
status = "client_abort";
|
|
243
|
+
else if (streamErr)
|
|
244
|
+
status = "stream_error";
|
|
245
|
+
if (!result.clientAborted)
|
|
246
|
+
res.end();
|
|
247
|
+
finishWith({
|
|
248
|
+
...common,
|
|
249
|
+
status,
|
|
250
|
+
partial: result.clientAborted || !result.completed,
|
|
251
|
+
usage,
|
|
252
|
+
costUsd: priced.cost,
|
|
253
|
+
costBasis: priced.basis,
|
|
254
|
+
notionalCostUsd: priced.notionalCost,
|
|
255
|
+
ttfbMs: result.ttfbMs,
|
|
256
|
+
bytesOut: result.bytesOut,
|
|
257
|
+
...(streamErr ? { errorType: streamErr.type, errorMessage: streamErr.message } : {}),
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
catch (err) {
|
|
261
|
+
for (const event of parser.flush())
|
|
262
|
+
acc.apply(event);
|
|
263
|
+
const usage = acc.snapshot();
|
|
264
|
+
const priced = priceUsage(decision.servedModel, usage, false, priceProvider);
|
|
265
|
+
log.warn("substitute stream failed mid-flight", {
|
|
266
|
+
id: ctx.base.id,
|
|
267
|
+
upstream: upstream.id,
|
|
268
|
+
error: String(err).slice(0, 200),
|
|
269
|
+
});
|
|
270
|
+
if (!res.writableEnded) {
|
|
271
|
+
res.write(sseErrorEvent("api_error", "Fest: upstream connection closed mid-stream."));
|
|
272
|
+
res.end();
|
|
273
|
+
}
|
|
274
|
+
finishWith({
|
|
275
|
+
...common,
|
|
276
|
+
status: "stream_error",
|
|
277
|
+
partial: true,
|
|
278
|
+
usage,
|
|
279
|
+
costUsd: priced.cost,
|
|
280
|
+
costBasis: priced.basis,
|
|
281
|
+
notionalCostUsd: priced.notionalCost,
|
|
282
|
+
errorType: "api_error",
|
|
283
|
+
errorMessage: String(err).slice(0, 200),
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
//# sourceMappingURL=substitute.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"substitute.js","sourceRoot":"","sources":["../../../server/pipeline/substitute.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAM1C,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAgBhC,mEAAmE;AACnE,MAAM,UAAU,MAAM,CACpB,GAAmB,EACnB,IAAe,EACf,IAAqF,EACrF,QAAuB,EACvB,UAAwC,EACxC,OAAe;IAEf,MAAM,IAAI,GAAG,uBAAuB,CAAC;IACrC,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,CACT,WAAW,CACT;QACE,GAAG,IAAI;QACP,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO,IAAI,EAAE;QAC1C,OAAO,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,IAAI,IAAI;QACnC,qBAAqB,EAAE,UAAU;KAClC,EACD;QACE,MAAM,EAAE,aAAa;QACrB,UAAU,EAAE,MAAM;QAClB,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,WAAW,EAAE,IAAI;QACjB,SAAS,EAAE,IAAI;QACf,YAAY,EAAE,OAAO;KACtB,CACF,CACF,CAAC;IACF,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE;QACpB,cAAc,EAAE,kBAAkB;QAClC,yEAAyE;QACzE,2EAA2E;QAC3E,kDAAkD;QAClD,gBAAgB,EAAE,OAAO;KAC1B,CAAC,CAAC;IACH,wEAAwE;IACxE,0DAA0D;IAC1D,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAqB,EACrB,GAAmB,EACnB,GAAsB;IAEtB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC;IACxC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACnC,IAAI,QAAQ,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAEvF,gFAAgF;IAChF,2EAA2E;IAC3E,4EAA4E;IAC5E,oDAAoD;IACpD,MAAM,aAAa,GAAG,uBAAuB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEhE,MAAM,UAAU,GAAe;QAC7B,GAAG,GAAG,CAAC,IAAI;QACX,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,QAAQ,CAAC,OAAO;QAC1B,OAAO,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,IAAI,IAAI;QACnC,qBAAqB,EAAE,GAAG,CAAC,UAAU;KACtC,CAAC;IAEF,MAAM,MAAM,GAAG,CAAC,OAAyD,EAAQ,EAAE;QACjF,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACxB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,QAAQ;KACT,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG;QACX,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YAC/B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,WAAW,CAAC;QACzB,GAAG,CAAC,IAAI,CAAC,4BAA4B,EAAE;YACrC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE;YACf,QAAQ,EAAE,QAAQ,CAAC,EAAE;YACrB,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,IAAI,IAAI;YACjC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SACjC,CAAC,CAAC;QACH,MAAM,CAAC;YACL,GAAG,IAAI;YACP,MAAM,EAAE,gBAAgB;YACxB,UAAU,EAAE,kBAAkB,CAAC,IAAI,CAAC;YACpC,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SACxC,CAAC,CAAC;QACH,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAChF,GAAG,CAAC,GAAG,CACL,cAAc,CACZ,IAAI,EACJ,kBAAkB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,yBAAyB;YACpE,wFAAwF,CAC3F,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,wEAAwE;IACxE,6EAA6E;IAC7E,kEAAkE;IAClE,MAAM,UAAU,GACd,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;QAChD,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvB,CAAC,CAAC,MAAM,KAAK,MAAM;YACjB,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,UAAmB,EAAE,MAAM,EAAE,qBAAqB,QAAQ,CAAC,MAAM,EAAE,EAAE;YACvF,CAAC,CAAC,CAAC,CACN;QACH,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC;IACrB,MAAM,SAAS,GAAe,EAAE,GAAG,UAAU,EAAE,qBAAqB,EAAE,UAAU,EAAE,CAAC;IACnF,MAAM,UAAU,GAAG,CAAC,OAAyD,EAAQ,EAAE;QACrF,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,sBAAsB,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACpF,MAAM,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAE7D,MAAM,MAAM,GAAG,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IAE3E,8EAA8E;IAC9E;;;;;;OAMG;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,uBAAuB,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,2EAA2E;QAC3E,0EAA0E;QAC1E,oDAAoD;QACpD,GAAG,CAAC,IAAI,CAAC,kBAAkB,EAAE;YAC3B,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU;YAC7B,QAAQ,EAAE,YAAY;YACtB,QAAQ,EAAE,QAAQ,CAAC,EAAE;YACrB,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,IAAI,IAAI;YACjC,iBAAiB;SAClB,CAAC,CAAC;QACH,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAC5F,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,UAAU,CAAC;YACT,GAAG,MAAM;YACT,MAAM,EAAE,gBAAgB;YACxB,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC;YACzC,SAAS,EAAE,OAAO,CAAC,IAAI;YACvB,YAAY,EAAE,OAAO,CAAC,OAAO;SAC9B,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,8EAA8E;IAC9E,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,WAAW,CAAC;QACxB,IAAI,CAAC;YACH,KAAK;gBACH,OAAO,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,WAAW,CAAC;QAC5F,CAAC;QAAC,MAAM,CAAC;YACP,6CAA6C;QAC/C,CAAC;QACD,0EAA0E;QAC1E,sDAAsD;QACtD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;QAC7E,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAC3C,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACd,UAAU,CAAC;YACT,GAAG,MAAM;YACT,MAAM,EAAE,IAAI;YACZ,KAAK;YACL,OAAO,EAAE,MAAM,CAAC,IAAI;YACpB,SAAS,EAAE,MAAM,CAAC,KAAK;YACvB,eAAe,EAAE,MAAM,CAAC,YAAY;YACpC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,8EAA8E;IAC9E,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,sBAAsB,EAAE,CAAC;IAErC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAE9C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE;YACnD,KAAK;YACL,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS;YAC7B,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3D,CAAC;SACF,CAAC,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,EAAE;YAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAErD,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;QAC7E,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAEpC;;;;;;;;;WASG;QACH,IAAI,MAAM,GAAkB,IAAI,CAAC;QACjC,IAAI,MAAM,CAAC,aAAa;YAAE,MAAM,GAAG,cAAc,CAAC;aAC7C,IAAI,SAAS;YAAE,MAAM,GAAG,cAAc,CAAC;QAE5C,IAAI,CAAC,MAAM,CAAC,aAAa;YAAE,GAAG,CAAC,GAAG,EAAE,CAAC;QAErC,UAAU,CAAC;YACT,GAAG,MAAM;YACT,MAAM;YACN,OAAO,EAAE,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,SAAS;YAClD,KAAK;YACL,OAAO,EAAE,MAAM,CAAC,IAAI;YACpB,SAAS,EAAE,MAAM,CAAC,KAAK;YACvB,eAAe,EAAE,MAAM,CAAC,YAAY;YACpC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE,YAAY,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,EAAE;YAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;QAC7E,GAAG,CAAC,IAAI,CAAC,qCAAqC,EAAE;YAC9C,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE;YACf,QAAQ,EAAE,QAAQ,CAAC,EAAE;YACrB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SACjC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;YACvB,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,WAAW,EAAE,8CAA8C,CAAC,CAAC,CAAC;YACtF,GAAG,CAAC,GAAG,EAAE,CAAC;QACZ,CAAC;QACD,UAAU,CAAC;YACT,GAAG,MAAM;YACT,MAAM,EAAE,cAAc;YACtB,OAAO,EAAE,IAAI;YACb,KAAK;YACL,OAAO,EAAE,MAAM,CAAC,IAAI;YACpB,SAAS,EAAE,MAAM,CAAC,KAAK;YACvB,eAAe,EAAE,MAAM,CAAC,YAAY;YACpC,SAAS,EAAE,WAAW;YACtB,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SACxC,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pick a route for a requested model.
|
|
3
|
+
*
|
|
4
|
+
* Matching is first-match-wins in file order, with exact matches preferred over
|
|
5
|
+
* wildcards regardless of where they sit in the file.
|
|
6
|
+
*
|
|
7
|
+
* That last rule is worth stating plainly, because "first match wins" and
|
|
8
|
+
* "exact beats wildcard" are the two conventions people expect and they
|
|
9
|
+
* disagree. The reason exactness wins here: a table almost always contains a
|
|
10
|
+
* broad rule (`claude-*` -> somewhere cheap) plus a handful of deliberate
|
|
11
|
+
* exceptions. If file order alone decided, adding a new exception would require
|
|
12
|
+
* remembering to put it above the catch-all, and forgetting would silently
|
|
13
|
+
* route a model somewhere the operator did not intend — with no error, because
|
|
14
|
+
* both rules are individually valid.
|
|
15
|
+
*
|
|
16
|
+
* Within a tier, longer patterns win before file order, so `claude-opus-*`
|
|
17
|
+
* beats `claude-*` however they are written down.
|
|
18
|
+
*/
|
|
19
|
+
function matches(pattern, model) {
|
|
20
|
+
if (pattern.endsWith("*"))
|
|
21
|
+
return model.startsWith(pattern.slice(0, -1));
|
|
22
|
+
return pattern === model;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Does this route claim this model id?
|
|
26
|
+
*
|
|
27
|
+
* A route's `expose` alias is an exact match for it. The menu and the router
|
|
28
|
+
* therefore agree by construction: every id Fest publishes is an id Fest can
|
|
29
|
+
* route, and there is no way to publish one without routing it.
|
|
30
|
+
*/
|
|
31
|
+
export function claimsModel(route, model) {
|
|
32
|
+
return route.expose === model || matches(route.match, model);
|
|
33
|
+
}
|
|
34
|
+
/** Exact rules first, then longest pattern, then file order. */
|
|
35
|
+
function rank(route, index, model) {
|
|
36
|
+
// An alias hit is always exact, however wildcard the route's own match is:
|
|
37
|
+
// the developer selected this entry by name from the menu.
|
|
38
|
+
const exact = model !== undefined && route.expose === model ? true : !route.match.endsWith("*");
|
|
39
|
+
return [exact ? 0 : 1, -route.match.length, index];
|
|
40
|
+
}
|
|
41
|
+
function better(a, b) {
|
|
42
|
+
return a[0] !== b[0] ? a[0] < b[0] : a[1] !== b[1] ? a[1] < b[1] : a[2] < b[2];
|
|
43
|
+
}
|
|
44
|
+
export function resolveRoute(table, requestedModel) {
|
|
45
|
+
const passthrough = {
|
|
46
|
+
route: null,
|
|
47
|
+
upstream: null,
|
|
48
|
+
requestedModel,
|
|
49
|
+
servedModel: requestedModel,
|
|
50
|
+
pipeline: "passthrough",
|
|
51
|
+
};
|
|
52
|
+
// A request with no readable model cannot be routed. It stays on the
|
|
53
|
+
// pass-through path rather than falling into a wildcard: routing a request we
|
|
54
|
+
// could not identify is exactly how traffic ends up somewhere unintended.
|
|
55
|
+
if (requestedModel === null || requestedModel === "")
|
|
56
|
+
return passthrough;
|
|
57
|
+
let best = null;
|
|
58
|
+
let bestRank = null;
|
|
59
|
+
table.routes.forEach((route, index) => {
|
|
60
|
+
if (!claimsModel(route, requestedModel))
|
|
61
|
+
return;
|
|
62
|
+
const r = rank(route, index, requestedModel);
|
|
63
|
+
if (bestRank === null || better(r, bestRank)) {
|
|
64
|
+
best = route;
|
|
65
|
+
bestRank = r;
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
if (best === null)
|
|
69
|
+
return passthrough;
|
|
70
|
+
const route = best;
|
|
71
|
+
// A route with a null upstream is a deliberate "keep this one on the
|
|
72
|
+
// subscription", which is how an operator carves an exception out of a
|
|
73
|
+
// broad substitution rule.
|
|
74
|
+
if (route.upstream === null) {
|
|
75
|
+
return { ...passthrough, route };
|
|
76
|
+
}
|
|
77
|
+
const upstream = table.upstreams.get(route.upstream);
|
|
78
|
+
if (upstream === undefined) {
|
|
79
|
+
// Unreachable via parseRouteTable, which rejects unknown upstreams at boot.
|
|
80
|
+
// Kept as a fail-safe rather than a `!`: if it ever happens, passing the
|
|
81
|
+
// request through is the outcome that cannot leak a credential.
|
|
82
|
+
return { ...passthrough, route };
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
route,
|
|
86
|
+
upstream,
|
|
87
|
+
requestedModel,
|
|
88
|
+
servedModel: route.model ?? requestedModel,
|
|
89
|
+
pipeline: "substitute",
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
/** Client-visible Anthropic-ish ids — the same filter Claude Code applies. */
|
|
93
|
+
const ANTHROPIC_MODEL = /(claude|anthropic)/i;
|
|
94
|
+
/**
|
|
95
|
+
* The upstream that serves Anthropic models nothing else claims.
|
|
96
|
+
*
|
|
97
|
+
* Identified by ADAPTER, not by the id an operator happened to type: the
|
|
98
|
+
* `anthropic` adapter is by definition "Anthropic's own API on a key the server
|
|
99
|
+
* holds", and that is the property that makes it a safe default. Naming an
|
|
100
|
+
* upstream `anthropic` while pointing it somewhere else would otherwise decide
|
|
101
|
+
* where unrouted traffic goes.
|
|
102
|
+
*/
|
|
103
|
+
export function defaultAnthropicUpstream(table) {
|
|
104
|
+
for (const upstream of table.upstreams.values()) {
|
|
105
|
+
if (upstream.adapter === "anthropic")
|
|
106
|
+
return upstream;
|
|
107
|
+
}
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Resolve for a request that has NO caller credential to pass through.
|
|
112
|
+
*
|
|
113
|
+
* This is the key posture: the credential position holds Fest's own identity
|
|
114
|
+
* token, which is never forwarded, so pass-through cannot serve anything. An
|
|
115
|
+
* unrouted model there is not "left on the subscription" — there is no
|
|
116
|
+
* subscription — it is a guaranteed failure on the first message.
|
|
117
|
+
*
|
|
118
|
+
* So when the operator has defined an `anthropic` upstream, every Anthropic
|
|
119
|
+
* model that no route claims is served there, on the org's own key. The
|
|
120
|
+
* alternative was one exact-match route per model id, which has to be updated
|
|
121
|
+
* every time Anthropic ships a model and fails silently — as a first-message
|
|
122
|
+
* 401 — when someone forgets.
|
|
123
|
+
*
|
|
124
|
+
* Three things it deliberately does NOT do:
|
|
125
|
+
*
|
|
126
|
+
* - **It never runs in the subscription posture.** That path keeps a caller
|
|
127
|
+
* credential, and diverting it to a server-held key would bill the org for
|
|
128
|
+
* a request the developer's own plan had already covered. This function is
|
|
129
|
+
* reached only where `upstreamCredential === null`, and a plain wildcard
|
|
130
|
+
* route in `routes.json` could not express that distinction — `resolveRoute`
|
|
131
|
+
* is posture-blind, so `claude-*` -> anthropic would capture subscription
|
|
132
|
+
* traffic too. That is why this is code and not config.
|
|
133
|
+
* - **It never overrides an explicit route.** A model claimed by any route,
|
|
134
|
+
* including an `upstream: null` "keep this on pass-through" carve-out, is
|
|
135
|
+
* left exactly where the operator put it.
|
|
136
|
+
* - **It never routes a non-Anthropic id.** Sending `gpt-oss-120b` to
|
|
137
|
+
* api.anthropic.com buys a confusing 404 from a vendor who never had that
|
|
138
|
+
* model, in place of Fest's own message naming the gateway.
|
|
139
|
+
*/
|
|
140
|
+
export function resolveWithoutCallerCredential(table, requestedModel) {
|
|
141
|
+
const decision = resolveRoute(table, requestedModel);
|
|
142
|
+
// Already substituted, or claimed by an explicit route: not ours to redirect.
|
|
143
|
+
if (decision.pipeline === "substitute" || decision.route !== null)
|
|
144
|
+
return decision;
|
|
145
|
+
if (requestedModel === null || requestedModel === "")
|
|
146
|
+
return decision;
|
|
147
|
+
if (!ANTHROPIC_MODEL.test(requestedModel))
|
|
148
|
+
return decision;
|
|
149
|
+
const upstream = defaultAnthropicUpstream(table);
|
|
150
|
+
if (upstream === null)
|
|
151
|
+
return decision;
|
|
152
|
+
return {
|
|
153
|
+
route: null,
|
|
154
|
+
upstream,
|
|
155
|
+
requestedModel,
|
|
156
|
+
// Anthropic's own id, unchanged. This is a credential substitution, not a
|
|
157
|
+
// model one: the developer asked for Sonnet and gets Sonnet.
|
|
158
|
+
servedModel: requestedModel,
|
|
159
|
+
pipeline: "substitute",
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Routes in the order they are actually evaluated.
|
|
164
|
+
*
|
|
165
|
+
* Exposed for the dashboard because the file's order is NOT the evaluation
|
|
166
|
+
* order, and showing an operator the file would be showing them something
|
|
167
|
+
* subtly untrue. Seeing exceptions sorted above the catch-all they override is
|
|
168
|
+
* the fastest way to confirm a table does what its author meant.
|
|
169
|
+
*/
|
|
170
|
+
export function evaluationOrder(table) {
|
|
171
|
+
return table.routes
|
|
172
|
+
.map((route, index) => ({ route, r: rank(route, index) }))
|
|
173
|
+
.sort((a, b) => (better(a.r, b.r) ? -1 : 1))
|
|
174
|
+
.map((x) => x.route);
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=resolve.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../server/routes/resolve.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAwBH,SAAS,OAAO,CAAC,OAAe,EAAE,KAAa;IAC7C,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,OAAO,OAAO,KAAK,KAAK,CAAC;AAC3B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAY,EAAE,KAAa;IACrD,OAAO,KAAK,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED,gEAAgE;AAChE,SAAS,IAAI,CAAC,KAAY,EAAE,KAAa,EAAE,KAAc;IACvD,2EAA2E;IAC3E,2DAA2D;IAC3D,MAAM,KAAK,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAChG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,MAAM,CAAC,CAA2B,EAAE,CAA2B;IACtE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,cAA6B;IAC3E,MAAM,WAAW,GAAkB;QACjC,KAAK,EAAE,IAAI;QACX,QAAQ,EAAE,IAAI;QACd,cAAc;QACd,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,aAAa;KACxB,CAAC;IAEF,qEAAqE;IACrE,8EAA8E;IAC9E,0EAA0E;IAC1E,IAAI,cAAc,KAAK,IAAI,IAAI,cAAc,KAAK,EAAE;QAAE,OAAO,WAAW,CAAC;IAEzE,IAAI,IAAI,GAAiB,IAAI,CAAC;IAC9B,IAAI,QAAQ,GAAoC,IAAI,CAAC;IAErD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACpC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,cAAc,CAAC;YAAE,OAAO;QAChD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC7C,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,WAAW,CAAC;IACtC,MAAM,KAAK,GAAU,IAAI,CAAC;IAE1B,qEAAqE;IACrE,uEAAuE;IACvE,2BAA2B;IAC3B,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC5B,OAAO,EAAE,GAAG,WAAW,EAAE,KAAK,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,4EAA4E;QAC5E,yEAAyE;QACzE,gEAAgE;QAChE,OAAO,EAAE,GAAG,WAAW,EAAE,KAAK,EAAE,CAAC;IACnC,CAAC;IAED,OAAO;QACL,KAAK;QACL,QAAQ;QACR,cAAc;QACd,WAAW,EAAE,KAAK,CAAC,KAAK,IAAI,cAAc;QAC1C,QAAQ,EAAE,YAAY;KACvB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,eAAe,GAAG,qBAAqB,CAAC;AAE9C;;;;;;;;GAQG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAiB;IACxD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QAChD,IAAI,QAAQ,CAAC,OAAO,KAAK,WAAW;YAAE,OAAO,QAAQ,CAAC;IACxD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAiB,EACjB,cAA6B;IAE7B,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IACrD,8EAA8E;IAC9E,IAAI,QAAQ,CAAC,QAAQ,KAAK,YAAY,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,QAAQ,CAAC;IACnF,IAAI,cAAc,KAAK,IAAI,IAAI,cAAc,KAAK,EAAE;QAAE,OAAO,QAAQ,CAAC;IACtE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC;QAAE,OAAO,QAAQ,CAAC;IAE3D,MAAM,QAAQ,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;IACjD,IAAI,QAAQ,KAAK,IAAI;QAAE,OAAO,QAAQ,CAAC;IAEvC,OAAO;QACL,KAAK,EAAE,IAAI;QACX,QAAQ;QACR,cAAc;QACd,0EAA0E;QAC1E,6DAA6D;QAC7D,WAAW,EAAE,cAAc;QAC3B,QAAQ,EAAE,YAAY;KACvB,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,OAAO,KAAK,CAAC,MAAM;SAChB,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;SACzD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC"}
|