@truefoundry/ai-gateway-sdk 0.0.15 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autogen/models.d.ts +14 -1
- package/dist/autogen/schema.json +9 -1
- package/dist/errors.d.ts +2 -1
- package/dist/errors.js +3 -1
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/parity/transformCapturedProviderResponse.js +13 -5
- package/dist/parity/transformCapturedProviderResponse.js.map +1 -1
- package/dist/providers/azure-ai-inference/api.js +8 -9
- package/dist/providers/azure-ai-inference/api.js.map +1 -1
- package/dist/providers/azure-openai/api.js +3 -0
- package/dist/providers/azure-openai/api.js.map +1 -1
- package/dist/providers/azure-openai/utils.js +1 -1
- package/dist/providers/azure-openai/utils.js.map +1 -1
- package/dist/providers/bedrock/api.js +44 -9
- package/dist/providers/bedrock/api.js.map +1 -1
- package/dist/providers/bedrock/chatComplete.js +5 -0
- package/dist/providers/bedrock/chatComplete.js.map +1 -1
- package/dist/providers/bedrock/index.d.ts +50 -0
- package/dist/providers/bedrock/index.js +29 -1
- package/dist/providers/bedrock/index.js.map +1 -1
- package/dist/providers/bedrock/modelFamily.d.ts +6 -1
- package/dist/providers/bedrock/modelFamily.js +10 -1
- package/dist/providers/bedrock/modelFamily.js.map +1 -1
- package/dist/providers/bedrock/utils.d.ts +1 -1
- package/dist/providers/bedrock/utils.js +9 -1
- package/dist/providers/bedrock/utils.js.map +1 -1
- package/dist/providers/google-vertex-ai/api.d.ts +1 -0
- package/dist/providers/google-vertex-ai/api.js +6 -4
- package/dist/providers/google-vertex-ai/api.js.map +1 -1
- package/dist/providers/microsoft-foundry/api.js +36 -8
- package/dist/providers/microsoft-foundry/api.js.map +1 -1
- package/dist/services/buildFetchInit.js +6 -1
- package/dist/services/buildFetchInit.js.map +1 -1
- package/dist/services/createProviderRequest.js +126 -21
- package/dist/services/createProviderRequest.js.map +1 -1
- package/dist/services/execute.js +77 -63
- package/dist/services/execute.js.map +1 -1
- package/dist/services/handleProviderResponse.d.ts +0 -8
- package/dist/services/handleProviderResponse.js +226 -41
- package/dist/services/handleProviderResponse.js.map +1 -1
- package/dist/stream/pumpProviderAwsEventStream.js +4 -3
- package/dist/stream/pumpProviderAwsEventStream.js.map +1 -1
- package/dist/stream/pumpProviderSseStream.js +18 -16
- package/dist/stream/pumpProviderSseStream.js.map +1 -1
- package/dist/stream/shouldReadAwsEventStream.js +5 -2
- package/dist/stream/shouldReadAwsEventStream.js.map +1 -1
- package/dist/stream/types.d.ts +5 -1
- package/dist/types/contracts.d.ts +26 -0
- package/dist/types/executionReport.d.ts +3 -1
- package/dist/utils/isStreamingEnabled.d.ts +5 -3
- package/dist/utils/isStreamingEnabled.js +17 -4
- package/dist/utils/isStreamingEnabled.js.map +1 -1
- package/dist/utils/sdkRequestUrl.d.ts +3 -0
- package/dist/utils/sdkRequestUrl.js +3 -0
- package/dist/utils/sdkRequestUrl.js.map +1 -1
- package/dist/utils/sseUtils.d.ts +9 -0
- package/dist/utils/sseUtils.js +65 -0
- package/dist/utils/sseUtils.js.map +1 -1
- package/package.json +1 -1
package/dist/services/execute.js
CHANGED
|
@@ -15,7 +15,7 @@ import { deriveProviderHeaderOptions, toLowercaseHeaders } from '../utils/derive
|
|
|
15
15
|
import { logIgnoredFailure } from '../utils/ignoreWithLog.js';
|
|
16
16
|
import { isStreamingEnabledSdk } from '../utils/isStreamingEnabled.js';
|
|
17
17
|
import { logProviderCallFailure } from '../utils/providerFailureLogging.js';
|
|
18
|
-
import { EXECUTE_TRACING_BODY, EXECUTE_TRACING_URL, redactTracingRequestUrl, sdkRequestUrl, } from '../utils/sdkRequestUrl.js';
|
|
18
|
+
import { EXECUTE_TRACING_BODY, EXECUTE_TRACING_FETCH_START, EXECUTE_TRACING_HEADERS, EXECUTE_TRACING_URL, redactTracingRequestUrl, sdkRequestUrl, } from '../utils/sdkRequestUrl.js';
|
|
19
19
|
import { buildFetchInit } from './buildFetchInit.js';
|
|
20
20
|
import { createProviderRequest } from './createProviderRequest.js';
|
|
21
21
|
import { handleProviderResponse } from './handleProviderResponse.js';
|
|
@@ -103,68 +103,76 @@ async function executeRequestHandler(call, handler) {
|
|
|
103
103
|
const provider = call.providerOptions.providerAccount.provider;
|
|
104
104
|
const endpoint = call.request.endpoint;
|
|
105
105
|
const fetchStartMs = performance.now();
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
});
|
|
128
|
-
const gatewayFailure = await recordProviderFailure({ retried, url: '', call, latencyMs, streaming });
|
|
129
|
-
response = emitProviderResponseCapture(response, {
|
|
130
|
-
url: sdkRequestUrl(call.request),
|
|
131
|
-
endpoint,
|
|
132
|
-
postAdapterBody: call.request.body,
|
|
133
|
-
needsResponseTranslation: false,
|
|
134
|
-
openaiUseResponsesApi: call.execution.openaiUseResponsesApi,
|
|
135
|
-
});
|
|
136
|
-
const handled = await handleProviderResponseOrAbortCapture({
|
|
137
|
-
response,
|
|
138
|
-
endpoint,
|
|
139
|
-
provider,
|
|
140
|
-
providerOptions: call.providerOptions,
|
|
141
|
-
execution,
|
|
142
|
-
streaming,
|
|
143
|
-
yieldRawBytes: false,
|
|
144
|
-
preserveNonJsonBody: true,
|
|
145
|
-
gatewayRequestBody: call.request.body,
|
|
146
|
-
postAdapterBody: call.request.body,
|
|
147
|
-
requestUrl: sdkRequestUrl(call.request),
|
|
148
|
-
inboundPath: call.request.path, // NEW: live ctx.req.url for CLONE usage routers
|
|
149
|
-
attempts: attempt ?? 0,
|
|
150
|
-
timings: { latencyMs, fetchStartMs }, // NEW: fetchStartMs on ExecutionTimings
|
|
151
|
-
fetchStartMs,
|
|
152
|
-
modelMetadata: {
|
|
153
|
-
provider,
|
|
106
|
+
try {
|
|
107
|
+
// A ReadableStream body can be read once. Retrying the handler would send an
|
|
108
|
+
// already-consumed upload. FormData can be serialized again.
|
|
109
|
+
const execution = call.request.body instanceof ReadableStream
|
|
110
|
+
? { ...call.execution, retry: { attempts: 0, on_status_codes: [] } }
|
|
111
|
+
: call.execution;
|
|
112
|
+
const retried = await retryFromExecution('', { method: call.request.method, signal: execution.abortSignal }, execution, signal => handler({
|
|
113
|
+
request: call.request,
|
|
114
|
+
providerOptions: call.providerOptions,
|
|
115
|
+
callConfig: call.callConfig,
|
|
116
|
+
headerOptions,
|
|
117
|
+
session: call.session,
|
|
118
|
+
signal,
|
|
119
|
+
modelId,
|
|
120
|
+
}));
|
|
121
|
+
let { response } = retried;
|
|
122
|
+
const { attempt } = retried;
|
|
123
|
+
const latencyMs = performance.now() - fetchStartMs;
|
|
124
|
+
// Same as live isStreamingEnabled: ctx.get(IS_GEMINI_ROUTER).
|
|
125
|
+
const streaming = isStreamingEnabledSdk({
|
|
126
|
+
body: call.request.body,
|
|
154
127
|
endpoint,
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
128
|
+
path: call.request.path,
|
|
129
|
+
isGeminiRouter: call.callConfig.isGeminiRouter,
|
|
130
|
+
});
|
|
131
|
+
const gatewayFailure = await recordProviderFailure({ retried, url: '', call, latencyMs, streaming });
|
|
132
|
+
response = emitProviderResponseCapture(response, {
|
|
133
|
+
url: sdkRequestUrl(call.request),
|
|
134
|
+
endpoint,
|
|
135
|
+
postAdapterBody: call.request.body,
|
|
136
|
+
needsResponseTranslation: false,
|
|
137
|
+
openaiUseResponsesApi: call.execution.openaiUseResponsesApi,
|
|
138
|
+
});
|
|
139
|
+
const handled = await handleProviderResponseOrAbortCapture({
|
|
140
|
+
response,
|
|
141
|
+
endpoint,
|
|
142
|
+
provider,
|
|
143
|
+
providerOptions: call.providerOptions,
|
|
144
|
+
execution,
|
|
145
|
+
streaming,
|
|
146
|
+
yieldRawBytes: false,
|
|
147
|
+
gatewayRequestBody: call.request.body,
|
|
148
|
+
postAdapterBody: call.request.body,
|
|
149
|
+
requestUrl: sdkRequestUrl(call.request),
|
|
150
|
+
inboundPath: call.request.path, // NEW: live ctx.req.url for CLONE usage routers
|
|
151
|
+
attempts: attempt ?? 0,
|
|
152
|
+
timings: { latencyMs, fetchStartMs }, // NEW: fetchStartMs on ExecutionTimings
|
|
153
|
+
fetchStartMs,
|
|
154
|
+
modelMetadata: {
|
|
155
|
+
provider,
|
|
156
|
+
endpoint,
|
|
157
|
+
...(modelId ? { modelId } : {}),
|
|
158
|
+
},
|
|
159
|
+
strictOpenAiCompliance: call.callConfig.strictOpenAiCompliance ?? true,
|
|
160
|
+
headerOptions,
|
|
161
|
+
callConfig: call.callConfig,
|
|
162
|
+
inboundEndpoint: endpoint,
|
|
163
|
+
gatewayFailure,
|
|
164
|
+
});
|
|
165
|
+
return {
|
|
166
|
+
response: handled.response,
|
|
167
|
+
executionReport: handled.executionReport,
|
|
168
|
+
servedEndpoint: endpoint,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
catch (err) {
|
|
172
|
+
if (err && typeof err === 'object')
|
|
173
|
+
Object.assign(err, { [EXECUTE_TRACING_FETCH_START]: fetchStartMs });
|
|
174
|
+
throw err;
|
|
175
|
+
}
|
|
168
176
|
}
|
|
169
177
|
/**
|
|
170
178
|
* Run one provider call: requestHandler fetch, or translate → fetch → transform → adapt → report.
|
|
@@ -181,11 +189,14 @@ export async function executeCall(call) {
|
|
|
181
189
|
}
|
|
182
190
|
const originalBody = cloneOriginalBody(call.request.body);
|
|
183
191
|
const built = await createProviderRequest(call);
|
|
192
|
+
let fetchStartMs;
|
|
184
193
|
try {
|
|
194
|
+
// Same as live isStreamingEnabled: ctx.get(IS_GEMINI_ROUTER).
|
|
185
195
|
const streaming = isStreamingEnabledSdk({
|
|
186
196
|
body: built.postAdapterBody,
|
|
187
197
|
endpoint: built.endpoint,
|
|
188
198
|
path: call.request.path,
|
|
199
|
+
isGeminiRouter: call.callConfig.isGeminiRouter,
|
|
189
200
|
});
|
|
190
201
|
const provider = call.providerOptions.providerAccount.provider;
|
|
191
202
|
const foundationModelId = getFoundationModelId(call.providerOptions.model);
|
|
@@ -196,7 +207,7 @@ export async function executeCall(call) {
|
|
|
196
207
|
foundationModelId,
|
|
197
208
|
});
|
|
198
209
|
const fetchInit = buildFetchInit(built, call);
|
|
199
|
-
|
|
210
|
+
fetchStartMs = performance.now();
|
|
200
211
|
const retried = await retryFromExecution(built.url, fetchInit, call.execution);
|
|
201
212
|
let { response } = retried;
|
|
202
213
|
const { attempt } = retried;
|
|
@@ -259,6 +270,7 @@ export async function executeCall(call) {
|
|
|
259
270
|
servedEndpoint: built.endpoint,
|
|
260
271
|
requestUrl: redactTracingRequestUrl(built.url),
|
|
261
272
|
providerRequestBody: built.outboundBody,
|
|
273
|
+
providerRequestHeaders: built.headers,
|
|
262
274
|
};
|
|
263
275
|
}
|
|
264
276
|
catch (err) {
|
|
@@ -267,6 +279,8 @@ export async function executeCall(call) {
|
|
|
267
279
|
Object.assign(err, {
|
|
268
280
|
[EXECUTE_TRACING_URL]: redactTracingRequestUrl(built.url),
|
|
269
281
|
[EXECUTE_TRACING_BODY]: built.outboundBody,
|
|
282
|
+
[EXECUTE_TRACING_HEADERS]: built.headers,
|
|
283
|
+
...(fetchStartMs !== undefined ? { [EXECUTE_TRACING_FETCH_START]: fetchStartMs } : {}),
|
|
270
284
|
});
|
|
271
285
|
}
|
|
272
286
|
throw err;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execute.js","sourceRoot":"","sources":["../../src/services/execute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACzG,OAAO,EAAE,kBAAkB,EAAiC,MAAM,gCAAgC,CAAC;AACnG,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,MAAM,mCAAmC,CAAC;AAK9G,OAAO,EAAE,2BAA2B,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AACvG,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AACzE,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,uBAAuB,EACvB,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAoC,MAAM,0BAA0B,CAAC;AAEpG,6GAA6G;AAC7G,KAAK,UAAU,oCAAoC,CAAC,KAAkC;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4BAA4B,EAAE,CAAC;QAC/B,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,uGAAuG;AACvG,KAAK,UAAU,qBAAqB,CAAC,IAMpC;IACC,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAC1D,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,8BAA8B,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;IAC/G,IAAI,kBAAkB,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACjD,MAAM,sBAAsB,CAAC;YAC3B,QAAQ;YACR,GAAG;YACH,QAAQ,EAAE,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ;YACvD,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,IAAI;YAC1D,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,GAAG;YACtC,kBAAkB,EAAE,8BAA8B;YAClD,kBAAkB;YAClB,mBAAmB;YACnB,aAAa,EAAE,OAAO;YACtB,SAAS;YACT,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IACD,OAAO,kBAAkB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,mBAAmB,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACzG,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,IAAoB;IAC7C,IACE,IAAI,YAAY,cAAc;QAC9B,IAAI,YAAY,QAAQ;QACxB,IAAI,YAAY,WAAW;QAC3B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC;QACxB,OAAO,IAAI,KAAK,QAAQ;QACxB,IAAI,YAAY,IAAI,EACpB,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,CAAC;QACH,OAAO,eAAe,CAAC,IAAI,CAA4B,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,iBAAiB,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAA4B,CAAC;QACrE,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,iBAAiB,CAAC,iCAAiC,EAAE,SAAS,CAAC,CAAC;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAK5B;IACC,IAAI,CAAC,IAAI,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAClC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACrD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,kBAAkB;QAAE,OAAO,KAAK,CAAC;IACpE,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;AACzE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAmB;IACpD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC;IACjH,OAAO,OAAO,EAAE,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAAC,IAAmB,EAAE,OAAuB;IAC/E,MAAM,aAAa,GAAG,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;IACvC,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAEvC,6EAA6E;
|
|
1
|
+
{"version":3,"file":"execute.js","sourceRoot":"","sources":["../../src/services/execute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACzG,OAAO,EAAE,kBAAkB,EAAiC,MAAM,gCAAgC,CAAC;AACnG,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,MAAM,mCAAmC,CAAC;AAK9G,OAAO,EAAE,2BAA2B,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AACvG,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AACzE,OAAO,EACL,oBAAoB,EACpB,2BAA2B,EAC3B,uBAAuB,EACvB,mBAAmB,EACnB,uBAAuB,EACvB,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAoC,MAAM,0BAA0B,CAAC;AAEpG,6GAA6G;AAC7G,KAAK,UAAU,oCAAoC,CAAC,KAAkC;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4BAA4B,EAAE,CAAC;QAC/B,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,uGAAuG;AACvG,KAAK,UAAU,qBAAqB,CAAC,IAMpC;IACC,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAC1D,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,8BAA8B,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;IAC/G,IAAI,kBAAkB,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACjD,MAAM,sBAAsB,CAAC;YAC3B,QAAQ;YACR,GAAG;YACH,QAAQ,EAAE,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ;YACvD,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,IAAI;YAC1D,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,GAAG;YACtC,kBAAkB,EAAE,8BAA8B;YAClD,kBAAkB;YAClB,mBAAmB;YACnB,aAAa,EAAE,OAAO;YACtB,SAAS;YACT,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IACD,OAAO,kBAAkB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,mBAAmB,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACzG,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,IAAoB;IAC7C,IACE,IAAI,YAAY,cAAc;QAC9B,IAAI,YAAY,QAAQ;QACxB,IAAI,YAAY,WAAW;QAC3B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC;QACxB,OAAO,IAAI,KAAK,QAAQ;QACxB,IAAI,YAAY,IAAI,EACpB,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,CAAC;QACH,OAAO,eAAe,CAAC,IAAI,CAA4B,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,iBAAiB,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAA4B,CAAC;QACrE,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,iBAAiB,CAAC,iCAAiC,EAAE,SAAS,CAAC,CAAC;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAK5B;IACC,IAAI,CAAC,IAAI,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAClC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACrD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,kBAAkB;QAAE,OAAO,KAAK,CAAC;IACpE,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;AACzE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAmB;IACpD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC;IACjH,OAAO,OAAO,EAAE,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAAC,IAAmB,EAAE,OAAuB;IAC/E,MAAM,aAAa,GAAG,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;IACvC,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAEvC,IAAI,CAAC;QACH,6EAA6E;QAC7E,6DAA6D;QAC7D,MAAM,SAAS,GACb,IAAI,CAAC,OAAO,CAAC,IAAI,YAAY,cAAc;YACzC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,EAAE;YACpE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,kBAAkB,CACtC,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,WAAW,EAAE,EAC9D,SAAS,EACT,MAAM,CAAC,EAAE,CACP,OAAO,CAAC;YACN,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM;YACN,OAAO;SACR,CAAC,CACL,CAAC;QACF,IAAI,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAC3B,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE5B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;QACnD,8DAA8D;QAC9D,MAAM,SAAS,GAAG,qBAAqB,CAAC;YACtC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACvB,QAAQ;YACR,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACvB,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,cAAc;SAC/C,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QAErG,QAAQ,GAAG,2BAA2B,CAAC,QAAQ,EAAE;YAC/C,GAAG,EAAE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;YAChC,QAAQ;YACR,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YAClC,wBAAwB,EAAE,KAAK;YAC/B,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,qBAAqB;SAC5D,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,oCAAoC,CAAC;YACzD,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,SAAS;YACT,SAAS;YACT,aAAa,EAAE,KAAK;YACpB,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACrC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YAClC,UAAU,EAAE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;YACvC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,gDAAgD;YAChF,QAAQ,EAAE,OAAO,IAAI,CAAC;YACtB,OAAO,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,wCAAwC;YAC9E,YAAY;YACZ,aAAa,EAAE;gBACb,QAAQ;gBACR,QAAQ;gBACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChC;YACD,sBAAsB,EAAE,IAAI,CAAC,UAAU,CAAC,sBAAsB,IAAI,IAAI;YACtE,aAAa;YACb,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,eAAe,EAAE,QAAQ;YACzB,cAAc;SACf,CAAC,CAAC;QAEH,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,cAAc,EAAE,QAAQ;SACzB,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,2BAA2B,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC;QACxG,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAmB;IACnD,MAAM,yBAAyB,EAAE,CAAC;IAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,OAAO,EAAE,CAAC;QACZ,gFAAgF;QAChF,OAAO,qBAAqB,CAC1B;YACE,GAAG,IAAI;YACP,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;SAChF,EACD,OAAO,CACR,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,YAAgC,CAAC;IAErC,IAAI,CAAC;QACH,8DAA8D;QAC9D,MAAM,SAAS,GAAG,qBAAqB,CAAC;YACtC,IAAI,EAAE,KAAK,CAAC,eAAe;YAC3B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACvB,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,cAAc;SAC/C,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC;QAC/D,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC3E,MAAM,aAAa,GAAG,mBAAmB,CAAC;YACxC,SAAS;YACT,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;YACtC,QAAQ;YACR,iBAAiB;SAClB,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9C,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/E,IAAI,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAC3B,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAC5B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5G,QAAQ,GAAG,2BAA2B,CAAC,QAAQ,EAAE;YAC/C,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,wBAAwB,EAAE,KAAK,CAAC,wBAAwB;YACxD,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,qBAAqB;SAC5D,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC;QAC1E,MAAM,aAAa,GAAG;YACpB,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChC,CAAC;QAEF,MAAM,aAAa,GAAG,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,MAAM,oCAAoC,CAAC;YACzD,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ;YACR,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS;YACT,aAAa;YACb,2FAA2F;YAC3F,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACrC,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,UAAU,EAAE,KAAK,CAAC,GAAG;YACrB,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,gDAAgD;YAChF,QAAQ,EAAE,OAAO,IAAI,CAAC;YACtB,OAAO,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,wCAAwC;YAC9E,YAAY;YACZ,aAAa;YACb,sBAAsB,EAAE,IAAI,CAAC,UAAU,CAAC,sBAAsB,IAAI,IAAI;YACtE,aAAa;YACb,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;YACtC,cAAc;SACf,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC;QAErC,IAAI,KAAK,CAAC,wBAAwB,EAAE,CAAC;YACnC,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7D,IAAI,cAAc,EAAE,CAAC;gBACnB,IAAI,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACjD,aAAa,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,YAAsB,EAAE;wBAC/F,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW;qBACxC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,aAAa,GAAG,MAAM,yBAAyB,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;gBAC5G,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,aAAa;YACvB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,cAAc,EAAE,KAAK,CAAC,QAAQ;YAC9B,UAAU,EAAE,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC;YAC9C,mBAAmB,EAAE,KAAK,CAAC,YAAY;YACvC,sBAAsB,EAAE,KAAK,CAAC,OAAO;SACtC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,gGAAgG;QAChG,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,CAAC,mBAAmB,CAAC,EAAE,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC;gBACzD,CAAC,oBAAoB,CAAC,EAAE,KAAK,CAAC,YAAY;gBAC1C,CAAC,uBAAuB,CAAC,EAAE,KAAK,CAAC,OAAO;gBACxC,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,2BAA2B,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvF,CAAC,CAAC;QACL,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -28,11 +28,6 @@ export interface HandleProviderResponseInput {
|
|
|
28
28
|
callConfig?: ProviderCallConfig;
|
|
29
29
|
/** Inbound endpoint before adapter translate (live ADAPTER_ORIGINAL_ENDPOINT). */
|
|
30
30
|
inboundEndpoint?: Endpoint;
|
|
31
|
-
/**
|
|
32
|
-
* requestHandler responses (file bytes, JSONL batch output) must not be
|
|
33
|
-
* parsed and re-wrapped as application/json. JSON content types still transform.
|
|
34
|
-
*/
|
|
35
|
-
preserveNonJsonBody?: boolean;
|
|
36
31
|
/** Set when `response` was synthesized because the provider never answered. */
|
|
37
32
|
gatewayFailure?: GatewayFailure;
|
|
38
33
|
}
|
|
@@ -40,7 +35,4 @@ export interface HandleProviderResponseResult {
|
|
|
40
35
|
response: Response;
|
|
41
36
|
executionReport: Promise<ExecutionReport>;
|
|
42
37
|
}
|
|
43
|
-
/**
|
|
44
|
-
* Transform a provider Response into the client-facing Response + deferred report.
|
|
45
|
-
*/
|
|
46
38
|
export declare function handleProviderResponse(input: HandleProviderResponseInput): Promise<HandleProviderResponseResult>;
|
|
@@ -5,14 +5,18 @@
|
|
|
5
5
|
* USD cost, OTel, or logging side effects.
|
|
6
6
|
*
|
|
7
7
|
* Streaming first-chunk gate parity with live `handleStreamingMode`:
|
|
8
|
-
* -
|
|
8
|
+
* - Native `/responses` in-band frames are held until the gate resolves, while the kill switch is on
|
|
9
9
|
* - `onStreamEnd` rejects empty / detail-less streams
|
|
10
10
|
* - Gate settles before the streaming Response is returned (LB failover)
|
|
11
11
|
*/
|
|
12
|
+
import { SSE_DONE } from '../adapters/constants.js';
|
|
13
|
+
import config from '../config.js';
|
|
12
14
|
import { Provider } from '../entities/Provider.js';
|
|
13
|
-
import { TTFTTimeoutError } from '../errors.js';
|
|
15
|
+
import { ResTranslationError, TTFTTimeoutError } from '../errors.js';
|
|
16
|
+
import { PROVIDER_MALFORMED_RESPONSE_STATUS_CODE } from '../globals.js';
|
|
14
17
|
import { getActualModelId, getProviderConfig } from '../handlers/tfyUtils.js';
|
|
15
18
|
import { providerResponseCaptureAbort } from '../parity/providerResponseCapture.js';
|
|
19
|
+
import { translateDecisionResponse } from '../providers/shared/decisionDialect.js';
|
|
16
20
|
import { universalResponseFormatResponseTransform, universalResponseFormatStreamingResponseTransform, } from '../providers/shared/responseFormatUtils.js';
|
|
17
21
|
import { handleResponseTransformWithGenericError } from '../providers/utils.js';
|
|
18
22
|
import { createAnthropicFirstMeaningfulChunkValidator, createFirstChunkGateFromSignal, createResponsesInBandErrorValidator, getStreamModeSplitPattern, pumpProviderAwsEventStream, pumpProviderSseStream, resolvePumpAbortSignal, shouldReadAwsEventStream, shouldRequireAzureStreamSleep, } from '../stream/index.js';
|
|
@@ -80,14 +84,14 @@ function isJsonContentType(contentType) {
|
|
|
80
84
|
const base = contentType.split(';')[0]?.trim().toLowerCase() ?? '';
|
|
81
85
|
return base === 'application/json' || base.endsWith('+json');
|
|
82
86
|
}
|
|
83
|
-
function
|
|
87
|
+
function passthroughResponse(args, streaming) {
|
|
84
88
|
const { response, endpoint, provider, execution, gatewayRequestBody, attempts, timings, modelMetadata } = args;
|
|
85
89
|
return {
|
|
86
90
|
response,
|
|
87
91
|
executionReport: Promise.resolve({
|
|
88
92
|
status: response.status,
|
|
89
93
|
attempts,
|
|
90
|
-
streaming
|
|
94
|
+
streaming,
|
|
91
95
|
timings,
|
|
92
96
|
endReason: { kind: 'done' },
|
|
93
97
|
usage: finalizeReportUsage({
|
|
@@ -103,7 +107,8 @@ function passthroughNonStreaming(args) {
|
|
|
103
107
|
};
|
|
104
108
|
}
|
|
105
109
|
function pickFirstChunkValidators(providerOptions, endpoint) {
|
|
106
|
-
|
|
110
|
+
// Same as live handleStreamingMode. Compact has no stream param, so its body is not Responses SSE.
|
|
111
|
+
if (config.RESPONSES_IN_BAND_ERROR_FALLBACK_ENABLED && endpoint === 'createModelResponse') {
|
|
107
112
|
const responsesValidator = createResponsesInBandErrorValidator();
|
|
108
113
|
return { validateFirstChunk: responsesValidator, responsesValidator };
|
|
109
114
|
}
|
|
@@ -117,31 +122,64 @@ function pickFirstChunkValidators(providerOptions, endpoint) {
|
|
|
117
122
|
}
|
|
118
123
|
async function handleNonStreaming(args) {
|
|
119
124
|
const { response, endpoint, provider, providerOptions, execution, gatewayRequestBody, requestUrl, attempts, timings, modelMetadata, strictOpenAiCompliance, } = args;
|
|
125
|
+
const servedEndpoint = args.inboundEndpoint ?? endpoint;
|
|
126
|
+
// Live handleNonStreamingMode returns video-proxy before any body read.
|
|
127
|
+
if (servedEndpoint === 'video-proxy') {
|
|
128
|
+
return passthroughResponse(args, false);
|
|
129
|
+
}
|
|
120
130
|
const contentType = response.headers.get('content-type');
|
|
121
131
|
if (isAudioContentType(contentType)) {
|
|
122
132
|
// Minimal passthrough — full live audio tee/logging stays gateway (P7.2).
|
|
123
|
-
return
|
|
124
|
-
}
|
|
125
|
-
if (args.preserveNonJsonBody && !isJsonContentType(contentType)) {
|
|
126
|
-
return passthroughNonStreaming(args);
|
|
133
|
+
return passthroughResponse(args, false);
|
|
127
134
|
}
|
|
135
|
+
// Same as live handleNonStreamingMode: 200 file content and batch output are JSONL.
|
|
136
|
+
const jsonl = (endpoint === 'retrieveFileContent' || endpoint === 'getBatchOutput') && response.status === 200;
|
|
128
137
|
const text = await response.text();
|
|
129
|
-
let json
|
|
130
|
-
|
|
131
|
-
|
|
138
|
+
let json;
|
|
139
|
+
if (jsonl) {
|
|
140
|
+
try {
|
|
141
|
+
json = text
|
|
142
|
+
.trim()
|
|
143
|
+
.split('\n')
|
|
144
|
+
.map(line => JSON.parse(line));
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
json = { text };
|
|
148
|
+
}
|
|
132
149
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
150
|
+
else {
|
|
151
|
+
try {
|
|
152
|
+
json = JSON.parse(text);
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
if (text)
|
|
156
|
+
logIgnoredFailure('sdk.handleProviderResponse.nonStreamJsonParse', error, { provider, endpoint });
|
|
157
|
+
// Same as live handleNonStreamingMode: empty and non-JSON bodies become { text }.
|
|
158
|
+
json = { text };
|
|
159
|
+
}
|
|
136
160
|
}
|
|
137
161
|
const transformer = resolveResponseTransforms(args);
|
|
138
162
|
let transformed = json;
|
|
139
163
|
if (typeof transformer === 'function') {
|
|
140
|
-
|
|
164
|
+
try {
|
|
165
|
+
transformed = await handleResponseTransformWithGenericError(json, response.status, response.headers, strictOpenAiCompliance, requestUrl, args.postAdapterBody, transformer, provider);
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
// Live handleNonStreamingMode records provider error headers before this throw.
|
|
169
|
+
throw new ResTranslationError(error instanceof Error ? error : undefined, response.status >= 400 ? Array.from(response.headers.entries()) : undefined);
|
|
170
|
+
}
|
|
141
171
|
}
|
|
142
172
|
if (endpoint === 'chatComplete' && transformed && typeof transformed === 'object') {
|
|
143
173
|
transformed = universalResponseFormatResponseTransform(transformed, strictOpenAiCompliance);
|
|
144
174
|
}
|
|
175
|
+
const decisionResponseDialect = args.callConfig?.decisionResponseDialect;
|
|
176
|
+
if (endpoint === 'decision' &&
|
|
177
|
+
response.ok &&
|
|
178
|
+
decisionResponseDialect &&
|
|
179
|
+
transformed &&
|
|
180
|
+
typeof transformed === 'object') {
|
|
181
|
+
transformed = translateDecisionResponse(transformed, decisionResponseDialect, gatewayRequestBody);
|
|
182
|
+
}
|
|
145
183
|
// NEW wiring: settle report from CLONE liveUsageFromResponse (else PRE usageFromJson).
|
|
146
184
|
const live = liveUsageFromResponse(liveUsageScope(args), transformed, response.headers);
|
|
147
185
|
const usage = finalizeReportUsage({
|
|
@@ -154,30 +192,75 @@ async function handleNonStreaming(args) {
|
|
|
154
192
|
});
|
|
155
193
|
const finishReason = live?.finishReason ?? finishReasonFromJson(transformed);
|
|
156
194
|
const iterations = iterationsFromJson(transformed);
|
|
195
|
+
// Same as live handleNonStreamingMode: 200 + error object and no transformer becomes 500.
|
|
196
|
+
let finalStatus = response.status;
|
|
197
|
+
if (transformed && typeof transformed === 'object' && 'error' in transformed) {
|
|
198
|
+
const bodyJson = transformed;
|
|
199
|
+
if (bodyJson.error && typeof bodyJson.error === 'object') {
|
|
200
|
+
if (!transformer && response.status === 200) {
|
|
201
|
+
finalStatus = PROVIDER_MALFORMED_RESPONSE_STATUS_CODE;
|
|
202
|
+
}
|
|
203
|
+
if (finalStatus !== 200) {
|
|
204
|
+
bodyJson.error.code = String(finalStatus);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
// Same as live handleNonStreamingMode /ranking reshape, after the 200→500 check.
|
|
209
|
+
if (endpoint === 'proxy' && args.inboundPath.includes('/ranking') && transformed && typeof transformed === 'object') {
|
|
210
|
+
const bodyJson = transformed;
|
|
211
|
+
if ('detail' in bodyJson && bodyJson.detail) {
|
|
212
|
+
bodyJson.error = {
|
|
213
|
+
message: JSON.stringify(bodyJson.detail),
|
|
214
|
+
code: String(response.status),
|
|
215
|
+
type: null,
|
|
216
|
+
param: null,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
else if ('error' in bodyJson && typeof bodyJson.error !== 'object') {
|
|
220
|
+
bodyJson.error = {
|
|
221
|
+
message: String(bodyJson.error),
|
|
222
|
+
code: String(response.status),
|
|
223
|
+
type: null,
|
|
224
|
+
param: null,
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
}
|
|
157
228
|
const bodyStr = typeof transformed === 'string' ? transformed : JSON.stringify(transformed);
|
|
229
|
+
// Same as live handleNonStreamingMode: proxy keeps the provider content-type. Other bodies become JSON so text/html cannot run.
|
|
158
230
|
const headers = new Headers(response.headers);
|
|
159
|
-
headers.
|
|
231
|
+
const responseContentType = headers.get('content-type');
|
|
232
|
+
if (endpoint !== 'proxy' && (!responseContentType || !isJsonContentType(responseContentType))) {
|
|
233
|
+
headers.set('content-type', 'application/json; charset=utf-8');
|
|
234
|
+
}
|
|
160
235
|
headers.delete('content-length');
|
|
161
236
|
headers.delete('content-encoding');
|
|
162
237
|
headers.delete('transfer-encoding');
|
|
238
|
+
const executionReport = Promise.resolve({
|
|
239
|
+
status: finalStatus,
|
|
240
|
+
providerStatus: response.status,
|
|
241
|
+
attempts,
|
|
242
|
+
streaming: false,
|
|
243
|
+
timings,
|
|
244
|
+
endReason: { kind: 'done', finishReason },
|
|
245
|
+
usage,
|
|
246
|
+
iterations,
|
|
247
|
+
tokensEstimated: live?.tokensEstimated, // NEW on report (CLONE LiveUsage)
|
|
248
|
+
serviceTier: live?.serviceTier,
|
|
249
|
+
toolUsage: live?.toolUsage,
|
|
250
|
+
providerReportedCostInUsd: live?.providerReportedCostInUsd, // NEW on report
|
|
251
|
+
modelMetadata,
|
|
252
|
+
providerRequestId: providerRequestIdFromHeaders(response.headers, provider),
|
|
253
|
+
...(args.gatewayFailure ? { gatewayFailure: args.gatewayFailure } : {}),
|
|
254
|
+
});
|
|
255
|
+
// Live reads provider error headers from this status, before the 200 → 500 rewrite.
|
|
256
|
+
executionReport.providerStatus = response.status;
|
|
163
257
|
return {
|
|
164
|
-
response: new Response(bodyStr, {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
streaming: false,
|
|
169
|
-
timings,
|
|
170
|
-
endReason: { kind: 'done', finishReason },
|
|
171
|
-
usage,
|
|
172
|
-
iterations,
|
|
173
|
-
tokensEstimated: live?.tokensEstimated, // NEW on report (CLONE LiveUsage)
|
|
174
|
-
serviceTier: live?.serviceTier,
|
|
175
|
-
toolUsage: live?.toolUsage,
|
|
176
|
-
providerReportedCostInUsd: live?.providerReportedCostInUsd, // NEW on report
|
|
177
|
-
modelMetadata,
|
|
178
|
-
providerRequestId: providerRequestIdFromHeaders(response.headers, provider),
|
|
179
|
-
...(args.gatewayFailure ? { gatewayFailure: args.gatewayFailure } : {}),
|
|
258
|
+
response: new Response(bodyStr, {
|
|
259
|
+
status: finalStatus,
|
|
260
|
+
statusText: finalStatus === PROVIDER_MALFORMED_RESPONSE_STATUS_CODE ? 'Bad Gateway' : response.statusText,
|
|
261
|
+
headers,
|
|
180
262
|
}),
|
|
263
|
+
executionReport,
|
|
181
264
|
};
|
|
182
265
|
}
|
|
183
266
|
async function handleStreaming(args) {
|
|
@@ -187,7 +270,7 @@ async function handleStreaming(args) {
|
|
|
187
270
|
}
|
|
188
271
|
const transformFunction = resolveResponseTransforms(args);
|
|
189
272
|
const modelManifestType = providerOptions.model?.manifest?.type ?? '';
|
|
190
|
-
const modelId = getActualModelId(providerOptions.model) || '';
|
|
273
|
+
const modelId = args.headerOptions.aws?.awsBedrockModel || getActualModelId(providerOptions.model) || '';
|
|
191
274
|
const useAws = shouldReadAwsEventStream({
|
|
192
275
|
modelManifestType,
|
|
193
276
|
modelId,
|
|
@@ -213,7 +296,7 @@ async function handleStreaming(args) {
|
|
|
213
296
|
// The listener runs in the abort() caller, outside this capture context, so keep the abort from here.
|
|
214
297
|
const abortCapture = providerResponseCaptureAbort();
|
|
215
298
|
if (abortCapture) {
|
|
216
|
-
pumpAbort?.addEventListener('abort', abortCapture
|
|
299
|
+
pumpAbort?.addEventListener('abort', abortCapture);
|
|
217
300
|
if (pumpAbort?.aborted)
|
|
218
301
|
abortCapture();
|
|
219
302
|
}
|
|
@@ -224,13 +307,15 @@ async function handleStreaming(args) {
|
|
|
224
307
|
let ttftTimer;
|
|
225
308
|
let firstChunkGate;
|
|
226
309
|
if (gateEnabled) {
|
|
227
|
-
|
|
310
|
+
// Same as live streamHandler: use routing's timer so the first chunk clears it.
|
|
311
|
+
if (ttftEnabled && execution.ttftAbortSignal) {
|
|
312
|
+
ttftTimer = execution.ttftTimer;
|
|
313
|
+
}
|
|
314
|
+
else if (ttftEnabled) {
|
|
228
315
|
ttftController = new AbortController();
|
|
229
316
|
ttftTimer = setTimeout(() => ttftController?.abort(), execution.ttftTimeoutMs);
|
|
230
317
|
}
|
|
231
|
-
firstChunkGate = createFirstChunkGateFromSignal(ttftController?.signal, execution.ttftTimeoutMs, ttftTimer, {
|
|
232
|
-
validateFirstChunk,
|
|
233
|
-
});
|
|
318
|
+
firstChunkGate = createFirstChunkGateFromSignal(execution.ttftAbortSignal ?? ttftController?.signal, execution.ttftTimeoutMs, ttftTimer, { validateFirstChunk });
|
|
234
319
|
}
|
|
235
320
|
const deferred = createStreamingReportDeferred({
|
|
236
321
|
status: response.status,
|
|
@@ -248,6 +333,8 @@ async function handleStreaming(args) {
|
|
|
248
333
|
strictOpenAiCompliance,
|
|
249
334
|
gatewayRequest,
|
|
250
335
|
abortSignal: pumpAbort,
|
|
336
|
+
onStreamEventKey: execution.onStreamEventKey,
|
|
337
|
+
eventCounts: execution.streamEventCounts,
|
|
251
338
|
};
|
|
252
339
|
const { readable, writable } = new TransformStream();
|
|
253
340
|
const writer = writable.getWriter();
|
|
@@ -357,12 +444,14 @@ async function handleStreaming(args) {
|
|
|
357
444
|
void releaseHeldChunks();
|
|
358
445
|
};
|
|
359
446
|
const runPump = async () => {
|
|
447
|
+
execution.onStreamPumpStart?.();
|
|
360
448
|
pumpGen = useAws
|
|
361
449
|
? pumpProviderAwsEventStream({ ...pumpOptions, yieldRawBytes })
|
|
362
450
|
: pumpProviderSseStream({
|
|
363
451
|
...pumpOptions,
|
|
364
452
|
splitPattern,
|
|
365
|
-
|
|
453
|
+
// Same as live readStream: forward comments unless resolveSseCommentMode returns strip.
|
|
454
|
+
sseCommentMode: execution.sseCommentMode ?? 'forward',
|
|
366
455
|
isSleepTimeRequired: shouldRequireAzureStreamSleep(provider),
|
|
367
456
|
});
|
|
368
457
|
const gen = pumpGen;
|
|
@@ -422,6 +511,8 @@ async function handleStreaming(args) {
|
|
|
422
511
|
await closeUpstream();
|
|
423
512
|
}
|
|
424
513
|
finally {
|
|
514
|
+
if (abortCapture)
|
|
515
|
+
pumpAbort?.removeEventListener('abort', abortCapture);
|
|
425
516
|
firstChunkGate?.dispose();
|
|
426
517
|
if (ttftTimer)
|
|
427
518
|
clearTimeout(ttftTimer);
|
|
@@ -449,8 +540,12 @@ async function handleStreaming(args) {
|
|
|
449
540
|
}
|
|
450
541
|
const headers = new Headers(response.headers);
|
|
451
542
|
// Native Bedrock Converse binary path must keep provider content-type (eventstream).
|
|
543
|
+
// Live buildStreamResponse also keeps an event-stream type, charset included.
|
|
452
544
|
if (!yieldRawBytes) {
|
|
453
|
-
headers.
|
|
545
|
+
const base = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase() ?? '';
|
|
546
|
+
if (base !== 'text/event-stream') {
|
|
547
|
+
headers.set('content-type', 'text/event-stream');
|
|
548
|
+
}
|
|
454
549
|
}
|
|
455
550
|
headers.delete('content-length');
|
|
456
551
|
headers.delete('content-encoding');
|
|
@@ -463,8 +558,98 @@ async function handleStreaming(args) {
|
|
|
463
558
|
/**
|
|
464
559
|
* Transform a provider Response into the client-facing Response + deferred report.
|
|
465
560
|
*/
|
|
561
|
+
/** Proxy and TTS pass through unless a Gemini or Bedrock router must read chunks. video-proxy uses the SSE pump, same as live modelType video. */
|
|
562
|
+
function isBlindProxyStream(input) {
|
|
563
|
+
const endpoint = input.inboundEndpoint ?? input.endpoint;
|
|
564
|
+
if (endpoint !== 'proxy' && endpoint !== 'tts-proxy')
|
|
565
|
+
return false;
|
|
566
|
+
if (input.callConfig?.isGeminiRouter || input.callConfig?.isBedrockRouter)
|
|
567
|
+
return false;
|
|
568
|
+
return true;
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Live `handleProxyPassthroughWithTTFT`: forward bytes unchanged, but fail the call
|
|
572
|
+
* if no upstream frame arrives within the TTFT budget (load-balance failover).
|
|
573
|
+
*/
|
|
574
|
+
async function passthroughProxyStream(args) {
|
|
575
|
+
const ttftMs = args.execution.ttftTimeoutMs;
|
|
576
|
+
const body = args.response.body;
|
|
577
|
+
if (typeof ttftMs !== 'number' || ttftMs <= 0) {
|
|
578
|
+
return passthroughResponse(args, true);
|
|
579
|
+
}
|
|
580
|
+
// Same as live handleProxyPassthroughWithTTFT.
|
|
581
|
+
if (!body) {
|
|
582
|
+
throw new Error('Response format is invalid. Body not found');
|
|
583
|
+
}
|
|
584
|
+
const gate = createFirstChunkGateFromSignal(args.execution.ttftAbortSignal, ttftMs, args.execution.ttftTimer, undefined);
|
|
585
|
+
const { readable, writable } = new TransformStream();
|
|
586
|
+
const writer = writable.getWriter();
|
|
587
|
+
const reader = body.getReader();
|
|
588
|
+
let firstChunk = false;
|
|
589
|
+
let sawDone = false;
|
|
590
|
+
const timings = { ...args.timings };
|
|
591
|
+
args.execution.abortSignal?.addEventListener('abort', () => {
|
|
592
|
+
reader.cancel().catch(() => { });
|
|
593
|
+
});
|
|
594
|
+
void (async () => {
|
|
595
|
+
try {
|
|
596
|
+
while (true) {
|
|
597
|
+
const { done, value } = await reader.read();
|
|
598
|
+
if (done) {
|
|
599
|
+
if (!firstChunk) {
|
|
600
|
+
gate.onError(new TTFTTimeoutError(`TTFT timeout: stream ended before first chunk (timeout=${ttftMs}ms)`, ttftMs));
|
|
601
|
+
}
|
|
602
|
+
break;
|
|
603
|
+
}
|
|
604
|
+
// Same as live handleProxyPassthroughWithTTFT: the gate opens on the first frame, even if empty.
|
|
605
|
+
if (!firstChunk) {
|
|
606
|
+
firstChunk = true;
|
|
607
|
+
gate.onFirstChunk();
|
|
608
|
+
}
|
|
609
|
+
if (!value || value.byteLength === 0)
|
|
610
|
+
continue;
|
|
611
|
+
if (timings.timeToFirstTokenMs === undefined) {
|
|
612
|
+
const firstByteAt = performance.now();
|
|
613
|
+
timings.timeToFirstTokenMs = firstByteAt - args.fetchStartMs;
|
|
614
|
+
args.execution.onFirstToken?.(firstByteAt);
|
|
615
|
+
}
|
|
616
|
+
await writer.write(value);
|
|
617
|
+
// Same as live handleProxyPassthroughWithTTFT: a later abort must not look like a mid-stream cancel.
|
|
618
|
+
if (!sawDone && new TextDecoder().decode(value).includes(SSE_DONE)) {
|
|
619
|
+
sawDone = true;
|
|
620
|
+
args.execution.markStreamCompleted?.();
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
catch (error) {
|
|
625
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
626
|
+
if (!firstChunk)
|
|
627
|
+
gate.onError(err);
|
|
628
|
+
}
|
|
629
|
+
finally {
|
|
630
|
+
gate.dispose();
|
|
631
|
+
try {
|
|
632
|
+
await writer.close();
|
|
633
|
+
}
|
|
634
|
+
catch {
|
|
635
|
+
// Writer may already be closed.
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
})();
|
|
639
|
+
try {
|
|
640
|
+
await gate.promise;
|
|
641
|
+
}
|
|
642
|
+
catch (err) {
|
|
643
|
+
await reader.cancel().catch(() => { });
|
|
644
|
+
await writer.abort(err).catch(() => { });
|
|
645
|
+
throw err;
|
|
646
|
+
}
|
|
647
|
+
return passthroughResponse({ ...args, timings, response: new Response(readable, args.response) }, true);
|
|
648
|
+
}
|
|
466
649
|
export async function handleProviderResponse(input) {
|
|
467
650
|
if (input.streaming && input.response.status === 200) {
|
|
651
|
+
if (isBlindProxyStream(input))
|
|
652
|
+
return passthroughProxyStream(input);
|
|
468
653
|
return handleStreaming(input);
|
|
469
654
|
}
|
|
470
655
|
return handleNonStreaming(input);
|