@apifuse/provider-sdk 2.2.0-beta.26 → 2.2.0-beta.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/bin/apifuse-dev.ts +2 -2
- package/bin/apifuse-pack-smoke.ts +1 -1
- package/bin/apifuse-pack-types.ts +2 -1
- package/bin/apifuse-perf.ts +2 -4
- package/bin/apifuse-record.ts +1 -1
- package/dist/auth-turn/index.d.ts +1 -1
- package/dist/auth-turn/index.js +1 -1
- package/dist/ceremonies/index.js +52 -11
- package/dist/config/loader.d.ts +1 -1
- package/dist/config/loader.js +4 -2
- package/dist/contract-serialization.d.ts +1 -20
- package/dist/contract-serialization.js +8 -587
- package/dist/contract.d.ts +0 -2
- package/dist/contract.js +5 -9
- package/dist/index.d.ts +9 -8
- package/dist/index.js +6 -8
- package/dist/provider.d.ts +3 -2
- package/dist/provider.js +2 -2
- package/dist/runtime/auth-flow.js +1 -1
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +135 -12
- package/dist/runtime/instrumentation.js +1 -1
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +2 -33
- package/dist/runtime/native-network.js +2 -68
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-shared.d.ts +3 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +2 -1
- package/dist/runtime/resolver-vendors/twocaptcha.js +80 -43
- package/dist/runtime/resolver-vendors/types.d.ts +1 -1
- package/dist/runtime/resolver.d.ts +6 -10
- package/dist/runtime/resolver.js +19 -18
- package/dist/runtime/state.js +5 -115
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.js +7 -130
- package/dist/schema.d.ts +0 -63
- package/dist/schema.js +8 -808
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.js +1 -1
- package/dist/server/self-test.d.ts +13 -0
- package/dist/server/self-test.js +124 -46
- package/dist/server/serve-implementation.d.ts +199 -0
- package/dist/server/serve-implementation.js +2054 -0
- package/dist/server/serve.d.ts +1 -187
- package/dist/server/serve.js +1 -1827
- package/dist/stateful/errors.d.ts +5 -0
- package/dist/stateful/errors.js +10 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +1 -5
- package/dist/stateful/stateful-provider-session-routing.js +2 -10
- package/dist/stream.js +7 -1
- package/package.json +27 -2
- package/src/auth-turn/index.ts +1 -1
- package/src/ceremonies/index.ts +68 -18
- package/src/config/loader.ts +5 -2
- package/src/contract-serialization.ts +8 -859
- package/src/contract.ts +5 -16
- package/src/index.ts +18 -34
- package/src/provider.ts +12 -24
- package/src/runtime/auth-flow.ts +1 -1
- package/src/runtime/http.ts +155 -11
- package/src/runtime/instrumentation.ts +1 -1
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +16 -97
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-shared.ts +17 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +100 -49
- package/src/runtime/resolver-vendors/types.ts +1 -0
- package/src/runtime/resolver.ts +47 -22
- package/src/runtime/state.ts +5 -144
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +14 -157
- package/src/schema.ts +9 -1060
- package/src/serve.ts +6 -1
- package/src/server/index.ts +1 -0
- package/src/server/self-test.ts +184 -59
- package/src/server/serve-implementation.ts +3024 -0
- package/src/server/serve.ts +1 -2661
- package/src/stateful/errors.ts +12 -0
- package/src/stateful/stateful-provider-session-routing.ts +2 -11
- package/src/stream.ts +8 -1
package/src/server/serve.ts
CHANGED
|
@@ -1,2661 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
|
|
4
|
-
import { Hono } from "hono";
|
|
5
|
-
import { z } from "zod";
|
|
6
|
-
import { AuthAbortError, createAuthFlowHelpers } from "../auth.js";
|
|
7
|
-
import { validateFailClosedDeclaration } from "../declaration-validation.js";
|
|
8
|
-
import {
|
|
9
|
-
SDK_OWNED_PROVIDER_ERROR_CODES,
|
|
10
|
-
SDK_RUNTIME_OWNED_ERROR_CODES,
|
|
11
|
-
SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES,
|
|
12
|
-
} from "../error-resolution.js";
|
|
13
|
-
import {
|
|
14
|
-
AuthError,
|
|
15
|
-
isProviderError,
|
|
16
|
-
isSessionExpiredError,
|
|
17
|
-
isTransportError,
|
|
18
|
-
isValidationError,
|
|
19
|
-
ProviderError,
|
|
20
|
-
} from "../errors.js";
|
|
21
|
-
import {
|
|
22
|
-
loadProviderLocaleCatalogs,
|
|
23
|
-
localizeAuthTurn,
|
|
24
|
-
type ProviderLocaleCatalogMap,
|
|
25
|
-
} from "../i18n/catalog.js";
|
|
26
|
-
import type { ProviderLocale } from "../i18n/keys.js";
|
|
27
|
-
import {
|
|
28
|
-
categoryForStatus,
|
|
29
|
-
type ProviderErrorSource,
|
|
30
|
-
sourceForCategory,
|
|
31
|
-
isRetryableCategory,
|
|
32
|
-
PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
33
|
-
type ProviderErrorCategory,
|
|
34
|
-
} from "../observability.js";
|
|
35
|
-
import { createScratchpad } from "../runtime/auth-flow.js";
|
|
36
|
-
import { createBrowserClient } from "../runtime/browser.js";
|
|
37
|
-
import { createProviderCache } from "../runtime/cache.js";
|
|
38
|
-
import {
|
|
39
|
-
createProviderChoiceContext,
|
|
40
|
-
PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV,
|
|
41
|
-
} from "../runtime/choice.js";
|
|
42
|
-
import { createCredentialContext } from "../runtime/credential.js";
|
|
43
|
-
import { createEnvContext } from "../runtime/env.js";
|
|
44
|
-
import { executeOperation } from "../runtime/executor.js";
|
|
45
|
-
import { createHttpClient } from "../runtime/http.js";
|
|
46
|
-
import { wrapWithInstrumentation } from "../runtime/instrumentation.js";
|
|
47
|
-
import {
|
|
48
|
-
createEnvVendorCredentialResolver,
|
|
49
|
-
createNativeNetworkClient,
|
|
50
|
-
} from "../runtime/native-network.js";
|
|
51
|
-
import { getProviderBaseUrl } from "../runtime/provider.js";
|
|
52
|
-
import { createOcrClientFromEnv } from "../runtime/ocr.js";
|
|
53
|
-
import {
|
|
54
|
-
PROXY_AUTH_IP_DENIED_CODE,
|
|
55
|
-
PROXY_EDGE_AUTH_REJECTED_CODE,
|
|
56
|
-
PROXY_POOL_EXHAUSTED_CODE,
|
|
57
|
-
} from "../runtime/proxy-errors.js";
|
|
58
|
-
import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector } from "../runtime/proxy-telemetry.js";
|
|
59
|
-
import { bindResolverSignal, createResolverClientFromEnv } from "../runtime/resolver.js";
|
|
60
|
-
import {
|
|
61
|
-
assertRequiredSecretsPresent,
|
|
62
|
-
listMissingRequiredSecrets,
|
|
63
|
-
MISSING_SECRET_CODE,
|
|
64
|
-
} from "../runtime/secrets.js";
|
|
65
|
-
import {
|
|
66
|
-
createProviderRuntimeStateFromEnv,
|
|
67
|
-
createUnsupportedProviderRuntimeState,
|
|
68
|
-
} from "../runtime/state.js";
|
|
69
|
-
import { createStealthClient } from "../runtime/stealth.js";
|
|
70
|
-
import { createSttClientFromEnv } from "../runtime/stt.js";
|
|
71
|
-
import { createTraceContext } from "../runtime/trace.js";
|
|
72
|
-
import { parseSchema } from "../schema.js";
|
|
73
|
-
import {
|
|
74
|
-
STATEFUL_NONCE_HEADER as STATEFUL_FORWARDING_NONCE_HEADER,
|
|
75
|
-
STATEFUL_SIGNATURE_HEADER as STATEFUL_FORWARDING_SIGNATURE_HEADER,
|
|
76
|
-
STATEFUL_TIMESTAMP_HEADER as STATEFUL_FORWARDING_TIMESTAMP_HEADER,
|
|
77
|
-
verifyStatefulRequestSignature,
|
|
78
|
-
} from "../stateful-signing.js";
|
|
79
|
-
import { StatefulRoutingDeadlineError } from "../stateful/stateful-provider-session-routing.js";
|
|
80
|
-
import { getStealthProfile } from "../stealth/profiles.js";
|
|
81
|
-
import {
|
|
82
|
-
APIFUSE_STREAM_DONE_EVENT,
|
|
83
|
-
APIFUSE_STREAM_ERROR_EVENT,
|
|
84
|
-
encodeSseEvent,
|
|
85
|
-
error as streamError,
|
|
86
|
-
} from "../stream.js";
|
|
87
|
-
import type {
|
|
88
|
-
AuthContext,
|
|
89
|
-
AuthTurn,
|
|
90
|
-
BrowserClient,
|
|
91
|
-
FlowContext,
|
|
92
|
-
FlowContextStore,
|
|
93
|
-
HttpRetrySummary,
|
|
94
|
-
OperationDefinition,
|
|
95
|
-
OperationErrorCode,
|
|
96
|
-
OperationHttpStreamTransport,
|
|
97
|
-
OperationSseTransport,
|
|
98
|
-
OcrContext,
|
|
99
|
-
ProviderErrorStatus,
|
|
100
|
-
ProviderContext,
|
|
101
|
-
ProviderDefinition,
|
|
102
|
-
ProviderProxyPolicy,
|
|
103
|
-
ProviderRuntimeState,
|
|
104
|
-
ProviderStreamEvent,
|
|
105
|
-
ResolverContext,
|
|
106
|
-
StealthClient,
|
|
107
|
-
SttContext,
|
|
108
|
-
} from "../types.js";
|
|
109
|
-
import { VALID_OPERATION_ERROR_STATUSES } from "../types.js";
|
|
110
|
-
import {
|
|
111
|
-
createSelfTestApp,
|
|
112
|
-
createSelfTestAuthFlowInvoke,
|
|
113
|
-
createSelfTestInvoke,
|
|
114
|
-
resolveSelfTestPort,
|
|
115
|
-
} from "./self-test.js";
|
|
116
|
-
import { resolveSelfTestMasterSecrets } from "./self-test-token.js";
|
|
117
|
-
import {
|
|
118
|
-
type AuthFlowRequest,
|
|
119
|
-
AuthFlowRequestSchema,
|
|
120
|
-
type AuthFlowResponse,
|
|
121
|
-
type AuthFlowSuccessResponse,
|
|
122
|
-
OperationConnectionSchema,
|
|
123
|
-
type OperationErrorResponse,
|
|
124
|
-
type OperationRequest,
|
|
125
|
-
OperationRequestSchema,
|
|
126
|
-
type OperationResponse,
|
|
127
|
-
type OperationSuccessResponse,
|
|
128
|
-
} from "./types.js";
|
|
129
|
-
|
|
130
|
-
const DEFAULT_HOST = "0.0.0.0";
|
|
131
|
-
const DEFAULT_PORT = 3000;
|
|
132
|
-
/** Compact SDK-owned error classification emitted separately from the public response body. */
|
|
133
|
-
export const ERROR_OBSERVABILITY_HEADER = "X-ApiFuse-Error-Observability";
|
|
134
|
-
export type ErrorObservabilityDetails = {
|
|
135
|
-
category: ProviderErrorCategory;
|
|
136
|
-
taxonomyVersion: string;
|
|
137
|
-
retryable: boolean;
|
|
138
|
-
upstreamStatus?: number;
|
|
139
|
-
};
|
|
140
|
-
const AUTH_FLOW_LOCALES = ["en", "ko", "ja"] as const;
|
|
141
|
-
const retryResponseMeta = new WeakMap<ProviderContext, HttpRetrySummary>();
|
|
142
|
-
const STATEFUL_INTERNAL_OPERATIONS_ROUTE = "/__apifuse/stateful/operations";
|
|
143
|
-
const STATEFUL_FORWARDING_SOURCE_POD_HEADER = "x-apifuse-stateful-source-pod";
|
|
144
|
-
const DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS = 5 * 60_000;
|
|
145
|
-
const DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES = 10_000;
|
|
146
|
-
const STATEFUL_FORWARDING_REPLAY_BUCKET_MS = 10_000;
|
|
147
|
-
const STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS = Math.ceil(
|
|
148
|
-
STATEFUL_FORWARDING_REPLAY_BUCKET_MS / 1_000,
|
|
149
|
-
);
|
|
150
|
-
|
|
151
|
-
export const ProviderServerStatefulForwardEnvelopeSchema = z
|
|
152
|
-
.object({
|
|
153
|
-
requestId: z.string().min(1),
|
|
154
|
-
providerId: z.string().min(1),
|
|
155
|
-
operationId: z.string().min(1),
|
|
156
|
-
sessionKey: z.string().min(1),
|
|
157
|
-
connectionId: z.string().min(1),
|
|
158
|
-
serviceAccountId: z.string().min(1),
|
|
159
|
-
ownerPodId: z.string().min(1),
|
|
160
|
-
generation: z.number().int().positive(),
|
|
161
|
-
sourcePodId: z.string().min(1),
|
|
162
|
-
forwardedAt: z.string().refine((value) => Number.isFinite(Date.parse(value))),
|
|
163
|
-
deadlineAt: z
|
|
164
|
-
.string()
|
|
165
|
-
.refine((value) => Number.isFinite(Date.parse(value)))
|
|
166
|
-
.optional(),
|
|
167
|
-
idempotencyKey: z.string().min(1).optional(),
|
|
168
|
-
operationRequest: OperationRequestSchema.extend({
|
|
169
|
-
connection: OperationConnectionSchema.strict().optional(),
|
|
170
|
-
}).strict(),
|
|
171
|
-
})
|
|
172
|
-
.strict();
|
|
173
|
-
|
|
174
|
-
export type ProviderServerStatefulForwardEnvelope = Readonly<
|
|
175
|
-
z.infer<typeof ProviderServerStatefulForwardEnvelopeSchema>
|
|
176
|
-
>;
|
|
177
|
-
|
|
178
|
-
export type ProviderServerStatefulOwnerFence = Readonly<
|
|
179
|
-
Pick<
|
|
180
|
-
ProviderServerStatefulForwardEnvelope,
|
|
181
|
-
| "providerId"
|
|
182
|
-
| "sessionKey"
|
|
183
|
-
| "ownerPodId"
|
|
184
|
-
| "generation"
|
|
185
|
-
| "sourcePodId"
|
|
186
|
-
| "forwardedAt"
|
|
187
|
-
| "requestId"
|
|
188
|
-
| "idempotencyKey"
|
|
189
|
-
>
|
|
190
|
-
>;
|
|
191
|
-
|
|
192
|
-
export type ProviderServerStatefulOwnerFenceValidator = (
|
|
193
|
-
fence: ProviderServerStatefulOwnerFence,
|
|
194
|
-
signal: AbortSignal,
|
|
195
|
-
) => boolean | Promise<boolean>;
|
|
196
|
-
|
|
197
|
-
export type ProviderServerOperationExecutorInput = {
|
|
198
|
-
readonly provider: ProviderDefinition;
|
|
199
|
-
readonly operationId: string;
|
|
200
|
-
readonly ctx: ProviderContext;
|
|
201
|
-
readonly request: OperationRequest & { readonly deadlineAt?: string };
|
|
202
|
-
readonly signal?: AbortSignal;
|
|
203
|
-
readonly internalStatefulForward?: ProviderServerStatefulForwardEnvelope;
|
|
204
|
-
};
|
|
205
|
-
|
|
206
|
-
export type ProviderServerOperationExecutor = (
|
|
207
|
-
input: ProviderServerOperationExecutorInput,
|
|
208
|
-
) => Promise<unknown>;
|
|
209
|
-
|
|
210
|
-
type RequestCleanup = () => void | Promise<void>;
|
|
211
|
-
|
|
212
|
-
function createAuthStub(): AuthContext {
|
|
213
|
-
return {
|
|
214
|
-
async requestField(name) {
|
|
215
|
-
throw new ProviderError(`Auth prompt is unavailable for ${name}`, {
|
|
216
|
-
code: "AUTH_PROMPT_UNAVAILABLE",
|
|
217
|
-
});
|
|
218
|
-
},
|
|
219
|
-
};
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
function createBrowserStub(): BrowserClient {
|
|
223
|
-
return {
|
|
224
|
-
engine: "playwright-stealth",
|
|
225
|
-
async close() {},
|
|
226
|
-
async newPage() {
|
|
227
|
-
throw new ProviderError("Browser runtime is not available", {
|
|
228
|
-
code: "BROWSER_RUNTIME_UNSUPPORTED",
|
|
229
|
-
});
|
|
230
|
-
},
|
|
231
|
-
async rawPage() {
|
|
232
|
-
throw new ProviderError("Browser runtime is not available", {
|
|
233
|
-
code: "BROWSER_RUNTIME_UNSUPPORTED",
|
|
234
|
-
});
|
|
235
|
-
},
|
|
236
|
-
async withIsolatedContext() {
|
|
237
|
-
throw new ProviderError("Browser runtime is not available", {
|
|
238
|
-
code: "BROWSER_RUNTIME_UNSUPPORTED",
|
|
239
|
-
});
|
|
240
|
-
},
|
|
241
|
-
async solveChallenge() {
|
|
242
|
-
throw new ProviderError("Browser runtime is not available", {
|
|
243
|
-
code: "BROWSER_RUNTIME_UNSUPPORTED",
|
|
244
|
-
});
|
|
245
|
-
},
|
|
246
|
-
};
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
function createStealthStub(): StealthClient {
|
|
250
|
-
return {
|
|
251
|
-
async fetch() {
|
|
252
|
-
throw new ProviderError("Stealth runtime is not available", {
|
|
253
|
-
code: "STEALTH_RUNTIME_UNSUPPORTED",
|
|
254
|
-
});
|
|
255
|
-
},
|
|
256
|
-
createSession() {
|
|
257
|
-
throw new ProviderError("Stealth runtime is not available", {
|
|
258
|
-
code: "STEALTH_RUNTIME_UNSUPPORTED",
|
|
259
|
-
});
|
|
260
|
-
},
|
|
261
|
-
close() {
|
|
262
|
-
// no-op
|
|
263
|
-
},
|
|
264
|
-
};
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
function getProviderStealthBaseUrl(provider: ProviderDefinition): string | undefined {
|
|
268
|
-
const baseUrl = getProviderBaseUrl(provider);
|
|
269
|
-
if (baseUrl) {
|
|
270
|
-
return baseUrl;
|
|
271
|
-
}
|
|
272
|
-
const firstHost = provider.allowedHosts?.[0];
|
|
273
|
-
return firstHost ? `https://${firstHost}` : undefined;
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
function getProviderStealthProfile(provider: ProviderDefinition) {
|
|
277
|
-
return provider.stealth?.profile ? getStealthProfile(provider.stealth.profile) : undefined;
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
function isProductionProviderBrowserMode(provider: ProviderDefinition, env = process.env): boolean {
|
|
281
|
-
if (provider.runtime !== "browser") {
|
|
282
|
-
return false;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
if (env.APIFUSE__PROVIDER__RUNTIME === "browser") {
|
|
286
|
-
return true;
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
return env.NODE_ENV === "production" && env.APIFUSE__PROVIDER__ID === provider.id;
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
export function resolveProviderProxyAffinityKey(
|
|
293
|
-
provider: ProviderDefinition,
|
|
294
|
-
request: OperationRequest,
|
|
295
|
-
operationId: string,
|
|
296
|
-
): string {
|
|
297
|
-
const connectionKey = resolveOperationConnectionId(request) ?? request.connection?.externalRef;
|
|
298
|
-
const affinity =
|
|
299
|
-
typeof provider.proxy === "object" ? provider.proxy.session?.affinity : undefined;
|
|
300
|
-
if (affinity === "operation") {
|
|
301
|
-
return `${provider.id}/${operationId}`;
|
|
302
|
-
}
|
|
303
|
-
return connectionKey ?? provider.id;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
export function resolveProviderResolverIdentityScope(
|
|
307
|
-
provider: ProviderDefinition,
|
|
308
|
-
affinityKey: string,
|
|
309
|
-
contextId: string,
|
|
310
|
-
): string {
|
|
311
|
-
return JSON.stringify({
|
|
312
|
-
proxy: provider.proxy ?? null,
|
|
313
|
-
affinityKey,
|
|
314
|
-
contextId,
|
|
315
|
-
});
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
function resolveOperationConnectionId(request: OperationRequest): string | undefined {
|
|
319
|
-
return request.connection?.id ?? request.connectionId;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
function resolveNativeProxyPolicy(provider: ProviderDefinition): ProviderProxyPolicy | undefined {
|
|
323
|
-
if (typeof provider.proxy === "object") return provider.proxy;
|
|
324
|
-
if (provider.proxy === true) return { mode: "optional" };
|
|
325
|
-
if (provider.proxy === false) return { mode: "disabled" };
|
|
326
|
-
return undefined;
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
function createProviderContext(
|
|
330
|
-
provider: ProviderDefinition,
|
|
331
|
-
request: OperationRequest,
|
|
332
|
-
operationId: string,
|
|
333
|
-
options: ProviderServerOptions = {},
|
|
334
|
-
state: ProviderRuntimeState = createUnsupportedProviderRuntimeState(),
|
|
335
|
-
proxyTelemetry?: ProxyTelemetryCollector,
|
|
336
|
-
signal?: AbortSignal,
|
|
337
|
-
): ProviderContext {
|
|
338
|
-
const baseUrl = getProviderBaseUrl(provider);
|
|
339
|
-
const stealthBaseUrl = getProviderStealthBaseUrl(provider);
|
|
340
|
-
const stealthProfile = getProviderStealthProfile(provider);
|
|
341
|
-
const proxyClientOptions = {
|
|
342
|
-
upstream: { proxy: provider.proxy },
|
|
343
|
-
affinityKey: resolveProviderProxyAffinityKey(provider, request, operationId),
|
|
344
|
-
telemetry: proxyTelemetry,
|
|
345
|
-
};
|
|
346
|
-
const resolverIdentityScope = resolveProviderResolverIdentityScope(
|
|
347
|
-
provider,
|
|
348
|
-
proxyClientOptions.affinityKey,
|
|
349
|
-
request.requestId,
|
|
350
|
-
);
|
|
351
|
-
let wrappedContext: ProviderContext | undefined;
|
|
352
|
-
const stealthClientOptions = {
|
|
353
|
-
upstream: proxyClientOptions.upstream,
|
|
354
|
-
affinityKey: proxyClientOptions.affinityKey,
|
|
355
|
-
telemetry: proxyTelemetry,
|
|
356
|
-
};
|
|
357
|
-
|
|
358
|
-
const env = createEnvContext([
|
|
359
|
-
...(provider.secrets?.map((secret) => secret.name) ?? []),
|
|
360
|
-
PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV,
|
|
361
|
-
]);
|
|
362
|
-
const credential = createCredentialContext({
|
|
363
|
-
allowedKeys: provider.credential?.keys,
|
|
364
|
-
mode: request.connection?.mode,
|
|
365
|
-
scopes: request.connection?.scopes,
|
|
366
|
-
values: request.connection?.secrets,
|
|
367
|
-
});
|
|
368
|
-
const requestContext = {
|
|
369
|
-
connectionId: resolveOperationConnectionId(request),
|
|
370
|
-
headers: request.headers ?? {},
|
|
371
|
-
};
|
|
372
|
-
const requestState = state.forConnection(requestContext.connectionId);
|
|
373
|
-
const cache = createProviderCache({ providerId: provider.id });
|
|
374
|
-
const context = wrapWithInstrumentation({
|
|
375
|
-
env,
|
|
376
|
-
credential,
|
|
377
|
-
request: requestContext,
|
|
378
|
-
http: createHttpClient(baseUrl, {
|
|
379
|
-
...proxyClientOptions,
|
|
380
|
-
onRetrySummary: (summary) => {
|
|
381
|
-
if (summary.attempts <= 1 || !wrappedContext) return;
|
|
382
|
-
retryResponseMeta.set(wrappedContext, summary);
|
|
383
|
-
},
|
|
384
|
-
}),
|
|
385
|
-
cache,
|
|
386
|
-
state: requestState,
|
|
387
|
-
stealth: stealthBaseUrl
|
|
388
|
-
? stealthProfile
|
|
389
|
-
? createStealthClient(stealthBaseUrl, stealthProfile.name, stealthClientOptions)
|
|
390
|
-
: createStealthClient(stealthBaseUrl, stealthClientOptions)
|
|
391
|
-
: createStealthStub(),
|
|
392
|
-
browser:
|
|
393
|
-
provider.runtime === "browser"
|
|
394
|
-
? createBrowserClient({
|
|
395
|
-
allowedHosts: provider.allowedHosts,
|
|
396
|
-
cdpUrl: process.env.APIFUSE__CDP_POOL__URL,
|
|
397
|
-
headless: true,
|
|
398
|
-
requireCdpPool: isProductionProviderBrowserMode(provider),
|
|
399
|
-
stealth: true,
|
|
400
|
-
engine: provider.browser?.engine,
|
|
401
|
-
})
|
|
402
|
-
: createBrowserStub(),
|
|
403
|
-
...(provider.native
|
|
404
|
-
? {
|
|
405
|
-
native: {
|
|
406
|
-
network: createNativeNetworkClient({
|
|
407
|
-
egress: provider.native.network,
|
|
408
|
-
proxyPolicy: resolveNativeProxyPolicy(provider),
|
|
409
|
-
affinityKey: proxyClientOptions.affinityKey,
|
|
410
|
-
credentials: createEnvVendorCredentialResolver(env),
|
|
411
|
-
}),
|
|
412
|
-
},
|
|
413
|
-
}
|
|
414
|
-
: {}),
|
|
415
|
-
trace: createTraceContext(),
|
|
416
|
-
auth: createAuthStub(),
|
|
417
|
-
ocr: options.ocr ?? createOcrClientFromEnv(provider.ocr),
|
|
418
|
-
stt: options.stt ?? createSttClientFromEnv(provider.stt),
|
|
419
|
-
resolver: bindResolverSignal(
|
|
420
|
-
options.resolver ??
|
|
421
|
-
createResolverClientFromEnv(provider.resolver, undefined, {
|
|
422
|
-
allowedHosts: provider.allowedHosts,
|
|
423
|
-
cache,
|
|
424
|
-
identityScope: resolverIdentityScope,
|
|
425
|
-
}),
|
|
426
|
-
signal,
|
|
427
|
-
),
|
|
428
|
-
choice: createProviderChoiceContext({
|
|
429
|
-
providerId: provider.id,
|
|
430
|
-
env,
|
|
431
|
-
request: requestContext,
|
|
432
|
-
credential,
|
|
433
|
-
state: requestState,
|
|
434
|
-
}),
|
|
435
|
-
});
|
|
436
|
-
wrappedContext = context;
|
|
437
|
-
return context;
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
function createFlowContextStore(
|
|
441
|
-
allowedKeys: string[],
|
|
442
|
-
initialContext: Record<string, unknown> = {},
|
|
443
|
-
): {
|
|
444
|
-
context: FlowContextStore;
|
|
445
|
-
getPatch: () => Record<string, unknown | null> | undefined;
|
|
446
|
-
} {
|
|
447
|
-
const context = createScratchpad(allowedKeys, initialContext);
|
|
448
|
-
|
|
449
|
-
return {
|
|
450
|
-
context,
|
|
451
|
-
getPatch() {
|
|
452
|
-
const next = context.toJSON();
|
|
453
|
-
const patch = new Map<string, unknown | null>();
|
|
454
|
-
|
|
455
|
-
for (const [key, value] of Object.entries(next)) {
|
|
456
|
-
if (initialContext[key] !== value) {
|
|
457
|
-
patch.set(key, value);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
for (const key of Object.keys(initialContext)) {
|
|
462
|
-
if (!(key in next)) {
|
|
463
|
-
patch.set(key, null);
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
if (patch.size === 0) {
|
|
468
|
-
return undefined;
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
return Object.fromEntries(patch.entries());
|
|
472
|
-
},
|
|
473
|
-
};
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
function createAuthFlowContext(
|
|
477
|
-
provider: ProviderDefinition,
|
|
478
|
-
request: AuthFlowRequest,
|
|
479
|
-
options: ProviderServerOptions = {},
|
|
480
|
-
signal?: AbortSignal,
|
|
481
|
-
): {
|
|
482
|
-
context: FlowContext;
|
|
483
|
-
getPatch: () => Record<string, unknown | null> | undefined;
|
|
484
|
-
} {
|
|
485
|
-
const baseUrl = getProviderBaseUrl(provider);
|
|
486
|
-
const stealthBaseUrl = getProviderStealthBaseUrl(provider);
|
|
487
|
-
const stealthProfile = getProviderStealthProfile(provider);
|
|
488
|
-
const contextData = request.context ?? {};
|
|
489
|
-
const flowContextStore = createFlowContextStore(
|
|
490
|
-
provider.context?.keys ?? Object.keys(contextData),
|
|
491
|
-
contextData,
|
|
492
|
-
);
|
|
493
|
-
const proxyClientOptions = {
|
|
494
|
-
upstream: { proxy: provider.proxy },
|
|
495
|
-
affinityKey:
|
|
496
|
-
request.connectionId ??
|
|
497
|
-
request.externalRef ??
|
|
498
|
-
request.tenantId ??
|
|
499
|
-
request.providerId ??
|
|
500
|
-
provider.id,
|
|
501
|
-
};
|
|
502
|
-
const resolverIdentityScope = resolveProviderResolverIdentityScope(
|
|
503
|
-
provider,
|
|
504
|
-
proxyClientOptions.affinityKey,
|
|
505
|
-
request.requestId,
|
|
506
|
-
);
|
|
507
|
-
const stealthClientOptions = {
|
|
508
|
-
upstream: proxyClientOptions.upstream,
|
|
509
|
-
affinityKey: proxyClientOptions.affinityKey,
|
|
510
|
-
};
|
|
511
|
-
const credential = request.connection
|
|
512
|
-
? createCredentialContext({
|
|
513
|
-
allowedKeys: provider.credential?.keys,
|
|
514
|
-
mode: request.connection.mode,
|
|
515
|
-
scopes: request.connection.scopes,
|
|
516
|
-
values: request.connection.secrets,
|
|
517
|
-
})
|
|
518
|
-
: undefined;
|
|
519
|
-
const cache = createProviderCache({ providerId: provider.id });
|
|
520
|
-
|
|
521
|
-
return {
|
|
522
|
-
context: {
|
|
523
|
-
flowId: request.flowId,
|
|
524
|
-
connectionId: request.connectionId,
|
|
525
|
-
externalRef: request.externalRef,
|
|
526
|
-
tenantId: request.tenantId ?? "",
|
|
527
|
-
providerId: request.providerId ?? provider.id,
|
|
528
|
-
http: createHttpClient(baseUrl, proxyClientOptions),
|
|
529
|
-
stealth: stealthBaseUrl
|
|
530
|
-
? stealthProfile
|
|
531
|
-
? createStealthClient(stealthBaseUrl, stealthProfile.name, stealthClientOptions)
|
|
532
|
-
: createStealthClient(stealthBaseUrl, stealthClientOptions)
|
|
533
|
-
: createStealthStub(),
|
|
534
|
-
...(provider.native
|
|
535
|
-
? {
|
|
536
|
-
native: {
|
|
537
|
-
network: createNativeNetworkClient({
|
|
538
|
-
egress: provider.native.network,
|
|
539
|
-
proxyPolicy: resolveNativeProxyPolicy(provider),
|
|
540
|
-
affinityKey: proxyClientOptions.affinityKey,
|
|
541
|
-
credentials: createEnvVendorCredentialResolver(
|
|
542
|
-
createEnvContext(provider.secrets?.map((secret) => secret.name)),
|
|
543
|
-
),
|
|
544
|
-
}),
|
|
545
|
-
},
|
|
546
|
-
}
|
|
547
|
-
: {}),
|
|
548
|
-
env: createEnvContext([
|
|
549
|
-
...(provider.secrets?.map((secret) => secret.name) ?? []),
|
|
550
|
-
...(provider.auth?.mode === "oauth2_proxied"
|
|
551
|
-
? ["APIFUSE__AUTH_PROXY__URL"]
|
|
552
|
-
: []),
|
|
553
|
-
]),
|
|
554
|
-
credential,
|
|
555
|
-
context: flowContextStore.context,
|
|
556
|
-
ocr: options.ocr ?? createOcrClientFromEnv(provider.ocr),
|
|
557
|
-
stt: options.stt ?? createSttClientFromEnv(provider.stt),
|
|
558
|
-
resolver: bindResolverSignal(
|
|
559
|
-
options.resolver ??
|
|
560
|
-
createResolverClientFromEnv(provider.resolver, undefined, {
|
|
561
|
-
allowedHosts: provider.allowedHosts,
|
|
562
|
-
cache,
|
|
563
|
-
identityScope: resolverIdentityScope,
|
|
564
|
-
}),
|
|
565
|
-
signal,
|
|
566
|
-
),
|
|
567
|
-
auth: createAuthFlowHelpers({ signal }),
|
|
568
|
-
},
|
|
569
|
-
getPatch: flowContextStore.getPatch,
|
|
570
|
-
};
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
type ProviderRequestCost = {
|
|
574
|
-
durationMs: number;
|
|
575
|
-
cpuUserMicros: number;
|
|
576
|
-
cpuSystemMicros: number;
|
|
577
|
-
cpuTotalMicros: number;
|
|
578
|
-
};
|
|
579
|
-
|
|
580
|
-
type ProviderServerLogEventBase = ProviderRequestCost & {
|
|
581
|
-
providerId: string;
|
|
582
|
-
kind: "operation" | "auth";
|
|
583
|
-
route: string;
|
|
584
|
-
requestId?: string;
|
|
585
|
-
status: number;
|
|
586
|
-
};
|
|
587
|
-
|
|
588
|
-
export type ProviderServerLogEvent =
|
|
589
|
-
| (ProviderServerLogEventBase & {
|
|
590
|
-
level: "info";
|
|
591
|
-
event: "provider_request_completed";
|
|
592
|
-
})
|
|
593
|
-
| (ProviderServerLogEventBase & {
|
|
594
|
-
level: "warn" | "error";
|
|
595
|
-
event: "provider_request_failed";
|
|
596
|
-
code: string;
|
|
597
|
-
errorClass: string;
|
|
598
|
-
message: string;
|
|
599
|
-
upstreamStatus?: number;
|
|
600
|
-
errorCategory?: ProviderErrorCategory;
|
|
601
|
-
taxonomyVersion?: string;
|
|
602
|
-
retryable?: boolean;
|
|
603
|
-
signal?: "unregistered_provider_error_code";
|
|
604
|
-
signalFix?: string;
|
|
605
|
-
issues?: Array<{ path: string; code: string; message: string }>;
|
|
606
|
-
})
|
|
607
|
-
| {
|
|
608
|
-
level: "warn";
|
|
609
|
-
event: "provider_secrets_missing";
|
|
610
|
-
providerId: string;
|
|
611
|
-
missingSecrets: string[];
|
|
612
|
-
}
|
|
613
|
-
| {
|
|
614
|
-
level: "warn";
|
|
615
|
-
event: "provider_cleanup_failed";
|
|
616
|
-
providerId: string;
|
|
617
|
-
kind: "operation";
|
|
618
|
-
route: string;
|
|
619
|
-
requestId?: string;
|
|
620
|
-
resource: "browser" | "stealth";
|
|
621
|
-
errorClass: string;
|
|
622
|
-
message: string;
|
|
623
|
-
}
|
|
624
|
-
| {
|
|
625
|
-
level: "error";
|
|
626
|
-
event: "provider_shutdown_hook_failed";
|
|
627
|
-
providerId: string;
|
|
628
|
-
hookIndex: number;
|
|
629
|
-
errorClass: string;
|
|
630
|
-
message: string;
|
|
631
|
-
};
|
|
632
|
-
|
|
633
|
-
export type ProviderServerLogger = (event: ProviderServerLogEvent) => void;
|
|
634
|
-
|
|
635
|
-
export type ProviderServerOptions = {
|
|
636
|
-
logger?: ProviderServerLogger;
|
|
637
|
-
/** Optional provider-specific operation executor. Stateful providers use this to preserve provider-local runtime semantics. */
|
|
638
|
-
operationExecutor?: ProviderServerOperationExecutor;
|
|
639
|
-
/** Optional signed internal executor for stateful owner forwarding. */
|
|
640
|
-
internalOperationExecutor?: ProviderServerOperationExecutor;
|
|
641
|
-
statefulForwarding?: {
|
|
642
|
-
readonly secret: string;
|
|
643
|
-
readonly maxSkewMs?: number;
|
|
644
|
-
readonly replayCacheMaxEntries?: number;
|
|
645
|
-
/** Required fail-closed check against the SDK/runtime owner registry. */
|
|
646
|
-
readonly validateOwnerFence: ProviderServerStatefulOwnerFenceValidator;
|
|
647
|
-
};
|
|
648
|
-
/** Optional STT override for tests or custom hosts; local/prod normally resolves from env. */
|
|
649
|
-
stt?: SttContext;
|
|
650
|
-
/** Optional OCR override for tests or custom hosts; local/prod normally resolves from env. */
|
|
651
|
-
ocr?: OcrContext;
|
|
652
|
-
/** Optional resolver override for tests or custom hosts; local/prod normally resolves from env. */
|
|
653
|
-
resolver?: ResolverContext;
|
|
654
|
-
/** Optional runtime state override for tests or custom hosts. Production resolves Redis from env and fails closed when unavailable. */
|
|
655
|
-
state?: ProviderRuntimeState;
|
|
656
|
-
/** Allow process-local runtime state only for local development and tests. */
|
|
657
|
-
allowMemoryStateFallback?: boolean;
|
|
658
|
-
/**
|
|
659
|
-
* Graceful process shutdown. Hooks run in declaration order after listeners stop accepting work.
|
|
660
|
-
*
|
|
661
|
-
* @example
|
|
662
|
-
* ```ts
|
|
663
|
-
* await serve(provider, {
|
|
664
|
-
* shutdown: {
|
|
665
|
-
* hooks: [
|
|
666
|
-
* async () => { await emitter.flush(); },
|
|
667
|
-
* async () => { await sessionManager.closeAll("server-shutdown"); },
|
|
668
|
-
* async () => { await lease.release(); },
|
|
669
|
-
* async () => { await router.close(); },
|
|
670
|
-
* ],
|
|
671
|
-
* },
|
|
672
|
-
* });
|
|
673
|
-
* ```
|
|
674
|
-
*/
|
|
675
|
-
shutdown?: {
|
|
676
|
-
readonly hooks?: Array<() => Promise<void>>;
|
|
677
|
-
readonly signals?: boolean | NodeJS.Signals[];
|
|
678
|
-
readonly timeoutMs?: number;
|
|
679
|
-
};
|
|
680
|
-
};
|
|
681
|
-
|
|
682
|
-
const defaultProviderServerLogger: ProviderServerLogger = (event) => {
|
|
683
|
-
const line = JSON.stringify(event);
|
|
684
|
-
if (event.level === "info") {
|
|
685
|
-
console.log(line);
|
|
686
|
-
return;
|
|
687
|
-
}
|
|
688
|
-
console.error(line);
|
|
689
|
-
};
|
|
690
|
-
|
|
691
|
-
function startRequestCost(): {
|
|
692
|
-
startedAtMs: number;
|
|
693
|
-
cpuStart: NodeJS.CpuUsage;
|
|
694
|
-
} {
|
|
695
|
-
return {
|
|
696
|
-
startedAtMs: performance.now(),
|
|
697
|
-
cpuStart: process.cpuUsage(),
|
|
698
|
-
};
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
function finishRequestCost(input: {
|
|
702
|
-
startedAtMs: number;
|
|
703
|
-
cpuStart: NodeJS.CpuUsage;
|
|
704
|
-
}): ProviderRequestCost {
|
|
705
|
-
const cpuDelta = process.cpuUsage(input.cpuStart);
|
|
706
|
-
return {
|
|
707
|
-
durationMs: Math.max(0, Math.round(performance.now() - input.startedAtMs)),
|
|
708
|
-
cpuUserMicros: Math.max(0, cpuDelta.user),
|
|
709
|
-
cpuSystemMicros: Math.max(0, cpuDelta.system),
|
|
710
|
-
cpuTotalMicros: Math.max(0, cpuDelta.user + cpuDelta.system),
|
|
711
|
-
};
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
function zodDetails(error: z.ZodError): Array<{
|
|
715
|
-
path: string;
|
|
716
|
-
code: string;
|
|
717
|
-
message: string;
|
|
718
|
-
}> {
|
|
719
|
-
return error.issues.map((issue) => ({
|
|
720
|
-
path: issue.path.join("."),
|
|
721
|
-
code: issue.code,
|
|
722
|
-
message: issue.message,
|
|
723
|
-
}));
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
// Category-level projection with code-aware honesty overrides: a missing
|
|
727
|
-
// deployment secret is an APIFuse-side defect even though its category
|
|
728
|
-
// (credential_unavailable) usually means a caller credential problem, an
|
|
729
|
-
// internal stateful-routing deadline is APIFuse-owned despite its timeout
|
|
730
|
-
// category, and the built-in upstream failure families keep their upstream
|
|
731
|
-
// attribution even when the author left the category at the provider_error
|
|
732
|
-
// default.
|
|
733
|
-
function publicErrorSource(
|
|
734
|
-
error: unknown,
|
|
735
|
-
category: ProviderErrorCategory,
|
|
736
|
-
): ProviderErrorSource {
|
|
737
|
-
if (error instanceof StatefulRoutingDeadlineError) return "apifuse";
|
|
738
|
-
if (isProviderError(error)) {
|
|
739
|
-
if (error.code === MISSING_SECRET_CODE) return "apifuse";
|
|
740
|
-
if (error.code === "UPSTREAM_ERROR" || error.code === "BLOCKED") {
|
|
741
|
-
return "upstream_failure";
|
|
742
|
-
}
|
|
743
|
-
}
|
|
744
|
-
return sourceForCategory(category);
|
|
745
|
-
}
|
|
746
|
-
|
|
747
|
-
function toErrorResponse(
|
|
748
|
-
error: unknown,
|
|
749
|
-
requestId?: string,
|
|
750
|
-
declaredErrorCode?: OperationErrorCode,
|
|
751
|
-
): OperationErrorResponse {
|
|
752
|
-
const observability = errorObservabilityDetails(error, declaredErrorCode);
|
|
753
|
-
const source = publicErrorSource(error, observability.category);
|
|
754
|
-
if (error instanceof StatefulRoutingDeadlineError) {
|
|
755
|
-
return {
|
|
756
|
-
error: {
|
|
757
|
-
code: "STATEFUL_FORWARDING_DEADLINE_EXPIRED",
|
|
758
|
-
message: "Stateful forwarding deadline expired.",
|
|
759
|
-
...(requestId ? { requestId } : {}),
|
|
760
|
-
retryable: observability.retryable,
|
|
761
|
-
source,
|
|
762
|
-
},
|
|
763
|
-
};
|
|
764
|
-
}
|
|
765
|
-
|
|
766
|
-
if (isProviderError(error)) {
|
|
767
|
-
const details = error.details;
|
|
768
|
-
return {
|
|
769
|
-
error: {
|
|
770
|
-
code: error.code ?? "provider_error",
|
|
771
|
-
message: publicProviderErrorMessage(error),
|
|
772
|
-
...(requestId ? { requestId } : {}),
|
|
773
|
-
retryable: observability.retryable,
|
|
774
|
-
source,
|
|
775
|
-
...(error.fix ? { fix: error.fix } : {}),
|
|
776
|
-
...(details !== undefined ? { details } : {}),
|
|
777
|
-
},
|
|
778
|
-
};
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
if (error instanceof z.ZodError) {
|
|
782
|
-
return {
|
|
783
|
-
error: {
|
|
784
|
-
code: "invalid_request",
|
|
785
|
-
message: "Invalid request body",
|
|
786
|
-
...(requestId ? { requestId } : {}),
|
|
787
|
-
retryable: observability.retryable,
|
|
788
|
-
source,
|
|
789
|
-
details: zodDetails(error),
|
|
790
|
-
},
|
|
791
|
-
};
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
// A masked internal error MUST NOT be advertised as retryable: without an
|
|
795
|
-
// explicit retryable:false the hub (bori provider-backed engine) defaults 5xx
|
|
796
|
-
// to retryable:true, which turns a deterministic pre-upstream crash into an
|
|
797
|
-
// infinite START->CONTINUE->restart loop (2026-07-22 catchtable reserve RCA).
|
|
798
|
-
// We still refuse to leak message/stack — only the error class name (or the
|
|
799
|
-
// primitive type for non-Error throwables) is surfaced for ops triage.
|
|
800
|
-
return {
|
|
801
|
-
error: {
|
|
802
|
-
code: "internal_error",
|
|
803
|
-
message: "Internal error",
|
|
804
|
-
...(requestId ? { requestId } : {}),
|
|
805
|
-
retryable: observability.retryable,
|
|
806
|
-
source,
|
|
807
|
-
details: {
|
|
808
|
-
retryable: false,
|
|
809
|
-
category: "internal_error",
|
|
810
|
-
errorClass: error instanceof Error ? error.name : typeof error,
|
|
811
|
-
},
|
|
812
|
-
},
|
|
813
|
-
};
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
// Accepts `unknown` so the branded guards narrow cleanly from the top: the
|
|
817
|
-
// subtype error classes are structurally compatible with ProviderError, so
|
|
818
|
-
// narrowing from a ProviderError-typed value would collapse the negative branch
|
|
819
|
-
// to `never`. Narrowing from unknown avoids that while still recognizing errors
|
|
820
|
-
// from a duplicate SDK module instance.
|
|
821
|
-
function providerObservabilityDetails(
|
|
822
|
-
error: unknown,
|
|
823
|
-
declaredErrorCode?: OperationErrorCode,
|
|
824
|
-
): ErrorObservabilityDetails | undefined {
|
|
825
|
-
const declaredRetryable = sdkOwnsErrorResolution(error)
|
|
826
|
-
? undefined
|
|
827
|
-
: declaredErrorCode?.retryable;
|
|
828
|
-
// Session-expiry surfaces the credential_expired category + the opt-in
|
|
829
|
-
// retryable signal so Gateway/Credential Service can refresh and re-drive the
|
|
830
|
-
// operation (see design.md §4.3 D3). Without this branch the auth error would
|
|
831
|
-
// serialize as a bare 401 with no retryable/category, losing the refresh
|
|
832
|
-
// signal for exactly the retryOnAuthRefresh operations it is meant to enable.
|
|
833
|
-
if (isSessionExpiredError(error)) {
|
|
834
|
-
return {
|
|
835
|
-
category: error.options?.category ?? "credential_expired",
|
|
836
|
-
taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
837
|
-
retryable: error.options?.retryable ?? declaredRetryable ?? false,
|
|
838
|
-
};
|
|
839
|
-
}
|
|
840
|
-
// Missing-secret errors carry the canonical credential_unavailable category
|
|
841
|
-
// so Gateway/observability can attribute the failure to provisioning, not
|
|
842
|
-
// the upstream. Matched by code (not constructor) so both the SDK-owned
|
|
843
|
-
// runtime gate and any not-yet-migrated provider-thrown MISSING_SECRET
|
|
844
|
-
// serialize identically, including across duplicate SDK module instances.
|
|
845
|
-
if (isProviderError(error) && error.code === MISSING_SECRET_CODE) {
|
|
846
|
-
return {
|
|
847
|
-
category: error.options?.category ?? "credential_unavailable",
|
|
848
|
-
taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
849
|
-
retryable: error.options?.retryable ?? declaredRetryable ?? false,
|
|
850
|
-
};
|
|
851
|
-
}
|
|
852
|
-
if (!isTransportError(error)) {
|
|
853
|
-
return undefined;
|
|
854
|
-
}
|
|
855
|
-
const isProxyPoolCode =
|
|
856
|
-
error.code === PROXY_POOL_EXHAUSTED_CODE ||
|
|
857
|
-
error.code === PROXY_EDGE_AUTH_REJECTED_CODE ||
|
|
858
|
-
error.code === "PROXY_ALLOCATION_FAILED";
|
|
859
|
-
const category =
|
|
860
|
-
error.options?.category ??
|
|
861
|
-
(isProxyPoolCode
|
|
862
|
-
? "proxy_pool"
|
|
863
|
-
: error.code === PROXY_AUTH_IP_DENIED_CODE
|
|
864
|
-
? "anti_bot_blocked"
|
|
865
|
-
: error.code === "transport_timeout"
|
|
866
|
-
? "timeout"
|
|
867
|
-
: error.code === "transport_network_error"
|
|
868
|
-
? "network"
|
|
869
|
-
: error.upstreamStatus
|
|
870
|
-
? categoryForStatus(error.upstreamStatus)
|
|
871
|
-
: "upstream_http");
|
|
872
|
-
return {
|
|
873
|
-
category,
|
|
874
|
-
taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
875
|
-
retryable:
|
|
876
|
-
error.options?.retryable ??
|
|
877
|
-
(category === "upstream_http" && error.upstreamStatus
|
|
878
|
-
? error.upstreamStatus >= 500
|
|
879
|
-
: isRetryableCategory(category)),
|
|
880
|
-
...(error.upstreamStatus ? { upstreamStatus: error.upstreamStatus } : {}),
|
|
881
|
-
};
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
function errorObservabilityDetails(
|
|
885
|
-
error: unknown,
|
|
886
|
-
declaredErrorCode?: OperationErrorCode,
|
|
887
|
-
): ErrorObservabilityDetails {
|
|
888
|
-
const effectiveDeclaration = sdkOwnsErrorResolution(error) ? undefined : declaredErrorCode;
|
|
889
|
-
const providerDetails = providerObservabilityDetails(error, effectiveDeclaration);
|
|
890
|
-
if (providerDetails) return providerDetails;
|
|
891
|
-
|
|
892
|
-
if (error instanceof z.ZodError || isValidationError(error)) {
|
|
893
|
-
const declaredStatus = effectiveDeclaration?.status;
|
|
894
|
-
return {
|
|
895
|
-
category:
|
|
896
|
-
isProviderError(error) && error.options?.category
|
|
897
|
-
? error.options.category
|
|
898
|
-
: isEmittableErrorStatus(declaredStatus) &&
|
|
899
|
-
categoryForStatus(declaredStatus) === "upstream_rejected"
|
|
900
|
-
? "upstream_rejected"
|
|
901
|
-
: isEmittableErrorStatus(declaredStatus) && declaredStatus >= 500
|
|
902
|
-
? "provider_error"
|
|
903
|
-
: "input_validation",
|
|
904
|
-
taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
905
|
-
retryable: isProviderError(error)
|
|
906
|
-
? (error.options?.retryable ?? effectiveDeclaration?.retryable ?? false)
|
|
907
|
-
: false,
|
|
908
|
-
};
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
if (error instanceof StatefulRoutingDeadlineError) {
|
|
912
|
-
return {
|
|
913
|
-
category: "timeout",
|
|
914
|
-
taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
915
|
-
retryable: false,
|
|
916
|
-
};
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
if (isProviderError(error)) {
|
|
920
|
-
// Deterministic upstream refusals default to the rejection category:
|
|
921
|
-
// the UPSTREAM_REJECTED family and any operation-declared rejection
|
|
922
|
-
// status (409/410/422) classify as upstream_rejected unless the
|
|
923
|
-
// author set an explicit category.
|
|
924
|
-
const declaredStatus = effectiveDeclaration?.status;
|
|
925
|
-
const rejectionDefault =
|
|
926
|
-
error.code === "UPSTREAM_REJECTED" ||
|
|
927
|
-
(isEmittableErrorStatus(declaredStatus) &&
|
|
928
|
-
categoryForStatus(declaredStatus) === "upstream_rejected")
|
|
929
|
-
? ("upstream_rejected" as const)
|
|
930
|
-
: ("provider_error" as const);
|
|
931
|
-
return {
|
|
932
|
-
category: error.options?.category ?? rejectionDefault,
|
|
933
|
-
taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
934
|
-
retryable: error.options?.retryable ?? effectiveDeclaration?.retryable ?? false,
|
|
935
|
-
};
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
return {
|
|
939
|
-
category: "internal_error",
|
|
940
|
-
taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
|
|
941
|
-
retryable: false,
|
|
942
|
-
};
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
function responseWithErrorObservability(
|
|
946
|
-
response: Response,
|
|
947
|
-
error: unknown,
|
|
948
|
-
declaredErrorCode?: OperationErrorCode,
|
|
949
|
-
): Response {
|
|
950
|
-
const headers = new Headers(response.headers);
|
|
951
|
-
headers.set(
|
|
952
|
-
ERROR_OBSERVABILITY_HEADER,
|
|
953
|
-
JSON.stringify(errorObservabilityDetails(error, declaredErrorCode)),
|
|
954
|
-
);
|
|
955
|
-
return new Response(response.body, {
|
|
956
|
-
status: response.status,
|
|
957
|
-
statusText: response.statusText,
|
|
958
|
-
headers,
|
|
959
|
-
});
|
|
960
|
-
}
|
|
961
|
-
|
|
962
|
-
function publicProviderErrorMessage(error: ProviderError): string {
|
|
963
|
-
if (isTransportError(error)) {
|
|
964
|
-
if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
|
|
965
|
-
return error.message;
|
|
966
|
-
}
|
|
967
|
-
if (error.code === PROXY_EDGE_AUTH_REJECTED_CODE) {
|
|
968
|
-
return error.message;
|
|
969
|
-
}
|
|
970
|
-
if (error.code === PROXY_POOL_EXHAUSTED_CODE) {
|
|
971
|
-
return error.message;
|
|
972
|
-
}
|
|
973
|
-
if (error.code === "transport_timeout") return "Request timed out";
|
|
974
|
-
if (error.code === "transport_network_error") return "Network error";
|
|
975
|
-
if (error.code === "upstream_http_error" && error.status) {
|
|
976
|
-
return `Upstream request failed with status ${error.status}`;
|
|
977
|
-
}
|
|
978
|
-
if (error.status) {
|
|
979
|
-
return `Upstream request failed with status ${error.status}`;
|
|
980
|
-
}
|
|
981
|
-
return "Upstream request failed";
|
|
982
|
-
}
|
|
983
|
-
return error.message;
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
function isEmittableErrorStatus(value: unknown): value is ProviderErrorStatus {
|
|
987
|
-
return (
|
|
988
|
-
typeof value === "number" &&
|
|
989
|
-
VALID_OPERATION_ERROR_STATUSES.some((status) => status === value)
|
|
990
|
-
);
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
function toStatusCode(error: unknown, declaredErrorCode?: OperationErrorCode): ProviderErrorStatus {
|
|
994
|
-
if (error instanceof z.ZodError) {
|
|
995
|
-
return 400;
|
|
996
|
-
}
|
|
997
|
-
if (error instanceof StatefulRoutingDeadlineError) {
|
|
998
|
-
return 504;
|
|
999
|
-
}
|
|
1000
|
-
if (isProviderError(error)) {
|
|
1001
|
-
if (
|
|
1002
|
-
!sdkOwnsErrorResolution(error) &&
|
|
1003
|
-
isEmittableErrorStatus(declaredErrorCode?.status)
|
|
1004
|
-
) {
|
|
1005
|
-
return declaredErrorCode.status;
|
|
1006
|
-
}
|
|
1007
|
-
// Canonical SDK code → status mapping lives in error-resolution.ts so
|
|
1008
|
-
// the authoring lint and this runtime path share one source of truth.
|
|
1009
|
-
if (typeof error.code === "string") {
|
|
1010
|
-
const mappedStatus = SDK_STATUS_MAPPED_PROVIDER_ERROR_CODES.get(error.code);
|
|
1011
|
-
if (mappedStatus !== undefined) {
|
|
1012
|
-
return mappedStatus;
|
|
1013
|
-
}
|
|
1014
|
-
}
|
|
1015
|
-
if (isTransportError(error)) {
|
|
1016
|
-
return error.code === "transport_timeout" ? 504 : 502;
|
|
1017
|
-
}
|
|
1018
|
-
if (isValidationError(error)) {
|
|
1019
|
-
return error.options?.category === "output_validation" ? 500 : 400;
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
return 500;
|
|
1023
|
-
}
|
|
1024
|
-
|
|
1025
|
-
return 500;
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
function sdkOwnsErrorResolution(error: unknown): boolean {
|
|
1029
|
-
if (isSessionExpiredError(error)) return true;
|
|
1030
|
-
if (isTransportError(error)) return true;
|
|
1031
|
-
if (error instanceof z.ZodError) return true;
|
|
1032
|
-
if (error instanceof StatefulRoutingDeadlineError) return true;
|
|
1033
|
-
return (
|
|
1034
|
-
isProviderError(error) &&
|
|
1035
|
-
typeof error.code === "string" &&
|
|
1036
|
-
SDK_RUNTIME_OWNED_ERROR_CODES.has(error.code)
|
|
1037
|
-
);
|
|
1038
|
-
}
|
|
1039
|
-
|
|
1040
|
-
type OperationErrorCodeLookup = ReadonlyMap<string, ReadonlyMap<string, OperationErrorCode>>;
|
|
1041
|
-
|
|
1042
|
-
function buildOperationErrorCodeLookup(provider: ProviderDefinition): OperationErrorCodeLookup {
|
|
1043
|
-
return new Map(
|
|
1044
|
-
Object.entries(provider.operations).flatMap(([operationId, operation]) => {
|
|
1045
|
-
const errorCodes = operation.docs?.errorCodes;
|
|
1046
|
-
return errorCodes?.length
|
|
1047
|
-
? [[operationId, new Map(errorCodes.map((entry) => [entry.code, entry]))] as const]
|
|
1048
|
-
: [];
|
|
1049
|
-
}),
|
|
1050
|
-
);
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
function declaredErrorCodeFor(
|
|
1054
|
-
error: unknown,
|
|
1055
|
-
operationId: string | undefined,
|
|
1056
|
-
lookup: OperationErrorCodeLookup,
|
|
1057
|
-
): OperationErrorCode | undefined {
|
|
1058
|
-
if (!operationId || !isProviderError(error) || typeof error.code !== "string") return undefined;
|
|
1059
|
-
return lookup.get(operationId)?.get(error.code);
|
|
1060
|
-
}
|
|
1061
|
-
|
|
1062
|
-
function extractRequestId(raw: unknown): string | undefined {
|
|
1063
|
-
if (!raw || typeof raw !== "object") {
|
|
1064
|
-
return undefined;
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
const value = Object.getOwnPropertyDescriptor(raw, "requestId")?.value;
|
|
1068
|
-
return typeof value === "string" ? value : undefined;
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
function logProviderError(
|
|
1072
|
-
logger: ProviderServerLogger | unknown,
|
|
1073
|
-
provider: ProviderDefinition,
|
|
1074
|
-
kind: "operation" | "auth",
|
|
1075
|
-
route: string,
|
|
1076
|
-
requestId: string | undefined,
|
|
1077
|
-
error: unknown,
|
|
1078
|
-
status: number,
|
|
1079
|
-
cost: ProviderRequestCost,
|
|
1080
|
-
declaredErrorCode?: OperationErrorCode,
|
|
1081
|
-
): void {
|
|
1082
|
-
const code = isProviderError(error)
|
|
1083
|
-
? (error.code ?? "provider_error")
|
|
1084
|
-
: error instanceof z.ZodError
|
|
1085
|
-
? "invalid_request"
|
|
1086
|
-
: error instanceof StatefulRoutingDeadlineError
|
|
1087
|
-
? "STATEFUL_FORWARDING_DEADLINE_EXPIRED"
|
|
1088
|
-
: "internal_error";
|
|
1089
|
-
const errorClass = error instanceof Error ? error.name : typeof error;
|
|
1090
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1091
|
-
const details = errorObservabilityDetails(error, declaredErrorCode);
|
|
1092
|
-
const isUnregisteredProviderErrorCode =
|
|
1093
|
-
status === 500 &&
|
|
1094
|
-
isProviderError(error) &&
|
|
1095
|
-
!isValidationError(error) &&
|
|
1096
|
-
typeof error.code === "string" &&
|
|
1097
|
-
!SDK_OWNED_PROVIDER_ERROR_CODES.has(error.code) &&
|
|
1098
|
-
declaredErrorCode === undefined;
|
|
1099
|
-
const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
|
|
1100
|
-
emit({
|
|
1101
|
-
level: status >= 500 ? "error" : "warn",
|
|
1102
|
-
event: "provider_request_failed",
|
|
1103
|
-
providerId: provider.id,
|
|
1104
|
-
kind,
|
|
1105
|
-
route,
|
|
1106
|
-
...(requestId ? { requestId } : {}),
|
|
1107
|
-
status,
|
|
1108
|
-
...cost,
|
|
1109
|
-
code,
|
|
1110
|
-
errorClass,
|
|
1111
|
-
message,
|
|
1112
|
-
...(details.upstreamStatus ? { upstreamStatus: details.upstreamStatus } : {}),
|
|
1113
|
-
errorCategory: details.category,
|
|
1114
|
-
taxonomyVersion: details.taxonomyVersion,
|
|
1115
|
-
retryable: details.retryable,
|
|
1116
|
-
...(isUnregisteredProviderErrorCode
|
|
1117
|
-
? {
|
|
1118
|
-
signal: "unregistered_provider_error_code" as const,
|
|
1119
|
-
signalFix:
|
|
1120
|
-
"Declare this code (with status and retryable) in the operation's docs.errorCodes so it serves its intended status instead of 500.",
|
|
1121
|
-
}
|
|
1122
|
-
: {}),
|
|
1123
|
-
...(error instanceof z.ZodError ? { issues: zodDetails(error) } : {}),
|
|
1124
|
-
});
|
|
1125
|
-
}
|
|
1126
|
-
|
|
1127
|
-
function logProviderCleanupError(
|
|
1128
|
-
logger: ProviderServerLogger | unknown,
|
|
1129
|
-
provider: ProviderDefinition,
|
|
1130
|
-
operationId: string,
|
|
1131
|
-
requestId: string | undefined,
|
|
1132
|
-
resource: "browser" | "stealth",
|
|
1133
|
-
error: unknown,
|
|
1134
|
-
): void {
|
|
1135
|
-
const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
|
|
1136
|
-
const errorClass = error instanceof Error ? error.name : typeof error;
|
|
1137
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1138
|
-
emit({
|
|
1139
|
-
level: "warn",
|
|
1140
|
-
event: "provider_cleanup_failed",
|
|
1141
|
-
providerId: provider.id,
|
|
1142
|
-
kind: "operation",
|
|
1143
|
-
route: operationId,
|
|
1144
|
-
...(requestId ? { requestId } : {}),
|
|
1145
|
-
resource,
|
|
1146
|
-
errorClass,
|
|
1147
|
-
message,
|
|
1148
|
-
});
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
function logProviderSuccess(
|
|
1152
|
-
logger: ProviderServerLogger | unknown,
|
|
1153
|
-
provider: ProviderDefinition,
|
|
1154
|
-
kind: "operation" | "auth",
|
|
1155
|
-
route: string,
|
|
1156
|
-
requestId: string | undefined,
|
|
1157
|
-
status: number,
|
|
1158
|
-
cost: ProviderRequestCost,
|
|
1159
|
-
): void {
|
|
1160
|
-
const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
|
|
1161
|
-
emit({
|
|
1162
|
-
level: "info",
|
|
1163
|
-
event: "provider_request_completed",
|
|
1164
|
-
providerId: provider.id,
|
|
1165
|
-
kind,
|
|
1166
|
-
route,
|
|
1167
|
-
...(requestId ? { requestId } : {}),
|
|
1168
|
-
status,
|
|
1169
|
-
...cost,
|
|
1170
|
-
});
|
|
1171
|
-
}
|
|
1172
|
-
|
|
1173
|
-
function toJsonSuccessResponse(
|
|
1174
|
-
result: unknown,
|
|
1175
|
-
ctx?: ProviderContext,
|
|
1176
|
-
): Response | OperationSuccessResponse {
|
|
1177
|
-
if (result instanceof Response) {
|
|
1178
|
-
return result;
|
|
1179
|
-
}
|
|
1180
|
-
|
|
1181
|
-
if (result instanceof ReadableStream) {
|
|
1182
|
-
return new Response(result);
|
|
1183
|
-
}
|
|
1184
|
-
|
|
1185
|
-
const cacheMeta = ctx?.cache.responseMeta();
|
|
1186
|
-
const retryMeta = ctx ? retryResponseMeta.get(ctx) : undefined;
|
|
1187
|
-
const meta =
|
|
1188
|
-
cacheMeta || retryMeta
|
|
1189
|
-
? {
|
|
1190
|
-
...(cacheMeta
|
|
1191
|
-
? {
|
|
1192
|
-
cached: cacheMeta.hit,
|
|
1193
|
-
stale: cacheMeta.stale,
|
|
1194
|
-
cache: cacheMeta,
|
|
1195
|
-
}
|
|
1196
|
-
: {}),
|
|
1197
|
-
...(retryMeta ? { retry: retryMeta } : {}),
|
|
1198
|
-
}
|
|
1199
|
-
: undefined;
|
|
1200
|
-
return {
|
|
1201
|
-
data: result,
|
|
1202
|
-
...(meta ? { meta } : {}),
|
|
1203
|
-
};
|
|
1204
|
-
}
|
|
1205
|
-
|
|
1206
|
-
function isAsyncIterable<T = unknown>(value: unknown): value is AsyncIterable<T> {
|
|
1207
|
-
if (!value || typeof value !== "object") return false;
|
|
1208
|
-
const iterator = Reflect.get(value, Symbol.asyncIterator);
|
|
1209
|
-
return typeof iterator === "function";
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
function responseWithCleanup(response: Response, cleanup: RequestCleanup): Response {
|
|
1213
|
-
if (!response.body) {
|
|
1214
|
-
void cleanup();
|
|
1215
|
-
return response;
|
|
1216
|
-
}
|
|
1217
|
-
const reader = response.body.getReader();
|
|
1218
|
-
let cleaned = false;
|
|
1219
|
-
const runCleanup = async () => {
|
|
1220
|
-
if (cleaned) return;
|
|
1221
|
-
cleaned = true;
|
|
1222
|
-
await cleanup();
|
|
1223
|
-
};
|
|
1224
|
-
const body = new ReadableStream<Uint8Array>({
|
|
1225
|
-
async pull(controller) {
|
|
1226
|
-
try {
|
|
1227
|
-
const { done, value } = await reader.read();
|
|
1228
|
-
if (done) {
|
|
1229
|
-
controller.close();
|
|
1230
|
-
await runCleanup();
|
|
1231
|
-
return;
|
|
1232
|
-
}
|
|
1233
|
-
if (value) controller.enqueue(value);
|
|
1234
|
-
} catch (error) {
|
|
1235
|
-
await runCleanup();
|
|
1236
|
-
controller.error(error);
|
|
1237
|
-
}
|
|
1238
|
-
},
|
|
1239
|
-
async cancel(reason) {
|
|
1240
|
-
try {
|
|
1241
|
-
await reader.cancel(reason);
|
|
1242
|
-
} finally {
|
|
1243
|
-
await runCleanup();
|
|
1244
|
-
}
|
|
1245
|
-
},
|
|
1246
|
-
});
|
|
1247
|
-
return new Response(body, {
|
|
1248
|
-
headers: response.headers,
|
|
1249
|
-
status: response.status,
|
|
1250
|
-
statusText: response.statusText,
|
|
1251
|
-
});
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
async function validateSseEvent(
|
|
1255
|
-
operation: OperationDefinition,
|
|
1256
|
-
event: ProviderStreamEvent,
|
|
1257
|
-
): Promise<ProviderStreamEvent> {
|
|
1258
|
-
const transport = getSseTransport(operation);
|
|
1259
|
-
const schema = transport?.events?.[event.event];
|
|
1260
|
-
if (!schema) {
|
|
1261
|
-
if (event.event === APIFUSE_STREAM_ERROR_EVENT || event.event === APIFUSE_STREAM_DONE_EVENT) {
|
|
1262
|
-
return event;
|
|
1263
|
-
}
|
|
1264
|
-
throw new ProviderError(
|
|
1265
|
-
`SSE event "${event.event}" is not declared in operation transport.events.`,
|
|
1266
|
-
{
|
|
1267
|
-
code: "SSE_EVENT_UNDECLARED",
|
|
1268
|
-
category: "output_validation",
|
|
1269
|
-
retryable: false,
|
|
1270
|
-
fix: `Add "${event.event}" to transport.events or stop emitting that event.`,
|
|
1271
|
-
},
|
|
1272
|
-
);
|
|
1273
|
-
}
|
|
1274
|
-
const data = await parseSchema(schema, event.data, `transport.events.${event.event}`);
|
|
1275
|
-
return { ...event, data };
|
|
1276
|
-
}
|
|
1277
|
-
|
|
1278
|
-
function byteLength(value: Uint8Array | string): number {
|
|
1279
|
-
if (typeof value === "string") {
|
|
1280
|
-
return new TextEncoder().encode(value).byteLength;
|
|
1281
|
-
}
|
|
1282
|
-
return value.byteLength;
|
|
1283
|
-
}
|
|
1284
|
-
|
|
1285
|
-
function assertStreamPayloadWithinLimit(
|
|
1286
|
-
actualBytes: number,
|
|
1287
|
-
maxBytes: number | undefined,
|
|
1288
|
-
kind: "event" | "chunk",
|
|
1289
|
-
): void {
|
|
1290
|
-
if (maxBytes === undefined || actualBytes <= maxBytes) return;
|
|
1291
|
-
throw new ProviderError(
|
|
1292
|
-
`Stream ${kind} exceeded declared byte limit (${actualBytes} > ${maxBytes}).`,
|
|
1293
|
-
{
|
|
1294
|
-
code: kind === "event" ? "STREAM_EVENT_TOO_LARGE" : "STREAM_CHUNK_TOO_LARGE",
|
|
1295
|
-
retryable: false,
|
|
1296
|
-
category: "input_validation",
|
|
1297
|
-
fix:
|
|
1298
|
-
kind === "event"
|
|
1299
|
-
? "Emit smaller SSE events or increase transport.maxEventBytes."
|
|
1300
|
-
: "Emit smaller stream chunks or increase transport.maxChunkBytes.",
|
|
1301
|
-
},
|
|
1302
|
-
);
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1305
|
-
function toSseResponse(
|
|
1306
|
-
operation: OperationDefinition,
|
|
1307
|
-
result: AsyncIterable<ProviderStreamEvent>,
|
|
1308
|
-
cleanup: RequestCleanup,
|
|
1309
|
-
requestId?: string,
|
|
1310
|
-
): Response {
|
|
1311
|
-
const encoder = new TextEncoder();
|
|
1312
|
-
const iterator = result[Symbol.asyncIterator]();
|
|
1313
|
-
const transport = getSseTransport(operation);
|
|
1314
|
-
let done = false;
|
|
1315
|
-
let cleaned = false;
|
|
1316
|
-
const runCleanup = async () => {
|
|
1317
|
-
if (cleaned) return;
|
|
1318
|
-
cleaned = true;
|
|
1319
|
-
await cleanup();
|
|
1320
|
-
};
|
|
1321
|
-
const body = new ReadableStream<Uint8Array>({
|
|
1322
|
-
async pull(controller) {
|
|
1323
|
-
try {
|
|
1324
|
-
if (done) {
|
|
1325
|
-
controller.close();
|
|
1326
|
-
await runCleanup();
|
|
1327
|
-
return;
|
|
1328
|
-
}
|
|
1329
|
-
const next = await iterator.next();
|
|
1330
|
-
if (next.done) {
|
|
1331
|
-
done = true;
|
|
1332
|
-
controller.close();
|
|
1333
|
-
await runCleanup();
|
|
1334
|
-
return;
|
|
1335
|
-
}
|
|
1336
|
-
const validated = await validateSseEvent(operation, next.value);
|
|
1337
|
-
const encodedEvent = encodeSseEvent(validated);
|
|
1338
|
-
const encodedBytes = encoder.encode(encodedEvent);
|
|
1339
|
-
assertStreamPayloadWithinLimit(encodedBytes.byteLength, transport?.maxEventBytes, "event");
|
|
1340
|
-
controller.enqueue(encodedBytes);
|
|
1341
|
-
} catch (error) {
|
|
1342
|
-
const message = error instanceof Error ? error.message : "Stream failed";
|
|
1343
|
-
controller.enqueue(
|
|
1344
|
-
encoder.encode(
|
|
1345
|
-
encodeSseEvent(
|
|
1346
|
-
streamError("stream_error", message, {
|
|
1347
|
-
...(requestId ? { requestId } : {}),
|
|
1348
|
-
}),
|
|
1349
|
-
),
|
|
1350
|
-
),
|
|
1351
|
-
);
|
|
1352
|
-
controller.close();
|
|
1353
|
-
done = true;
|
|
1354
|
-
await runCleanup();
|
|
1355
|
-
}
|
|
1356
|
-
},
|
|
1357
|
-
async cancel(reason) {
|
|
1358
|
-
try {
|
|
1359
|
-
await iterator.return?.(reason);
|
|
1360
|
-
} finally {
|
|
1361
|
-
await runCleanup();
|
|
1362
|
-
}
|
|
1363
|
-
},
|
|
1364
|
-
});
|
|
1365
|
-
return new Response(body, {
|
|
1366
|
-
headers: {
|
|
1367
|
-
"Cache-Control": "no-cache, no-transform",
|
|
1368
|
-
Connection: "keep-alive",
|
|
1369
|
-
"Content-Type": "text/event-stream; charset=utf-8",
|
|
1370
|
-
},
|
|
1371
|
-
});
|
|
1372
|
-
}
|
|
1373
|
-
|
|
1374
|
-
function enforceStreamChunkLimit(
|
|
1375
|
-
body: ReadableStream<Uint8Array>,
|
|
1376
|
-
maxChunkBytes: number | undefined,
|
|
1377
|
-
): ReadableStream<Uint8Array> {
|
|
1378
|
-
if (maxChunkBytes === undefined) return body;
|
|
1379
|
-
const reader = body.getReader();
|
|
1380
|
-
return new ReadableStream<Uint8Array>({
|
|
1381
|
-
async pull(controller) {
|
|
1382
|
-
try {
|
|
1383
|
-
const { done, value } = await reader.read();
|
|
1384
|
-
if (done) {
|
|
1385
|
-
controller.close();
|
|
1386
|
-
return;
|
|
1387
|
-
}
|
|
1388
|
-
if (value) {
|
|
1389
|
-
assertStreamPayloadWithinLimit(byteLength(value), maxChunkBytes, "chunk");
|
|
1390
|
-
controller.enqueue(value);
|
|
1391
|
-
}
|
|
1392
|
-
} catch (error) {
|
|
1393
|
-
controller.error(error);
|
|
1394
|
-
}
|
|
1395
|
-
},
|
|
1396
|
-
cancel(reason) {
|
|
1397
|
-
return reader.cancel(reason);
|
|
1398
|
-
},
|
|
1399
|
-
});
|
|
1400
|
-
}
|
|
1401
|
-
|
|
1402
|
-
function toStreamingResponse(
|
|
1403
|
-
operation: OperationDefinition,
|
|
1404
|
-
result: unknown,
|
|
1405
|
-
cleanup: RequestCleanup,
|
|
1406
|
-
requestId?: string,
|
|
1407
|
-
): Response {
|
|
1408
|
-
const transport = operation.transport?.kind ?? "json";
|
|
1409
|
-
if (transport === "sse" && (result instanceof Response || result instanceof ReadableStream)) {
|
|
1410
|
-
void cleanup();
|
|
1411
|
-
throw new ProviderError(
|
|
1412
|
-
"SSE operations must return an AsyncIterable of typed stream.event(...) values.",
|
|
1413
|
-
{
|
|
1414
|
-
code: "SSE_RESULT_UNSUPPORTED",
|
|
1415
|
-
category: "output_validation",
|
|
1416
|
-
retryable: false,
|
|
1417
|
-
fix: "Return an async generator that yields stream.event(name, data) so APIFuse can validate event schemas and enforce event byte limits.",
|
|
1418
|
-
},
|
|
1419
|
-
);
|
|
1420
|
-
}
|
|
1421
|
-
if (result instanceof Response) {
|
|
1422
|
-
const httpTransport = getHttpStreamTransport(operation);
|
|
1423
|
-
if (httpTransport && result.body && httpTransport?.maxChunkBytes !== undefined) {
|
|
1424
|
-
return responseWithCleanup(
|
|
1425
|
-
new Response(enforceStreamChunkLimit(result.body, httpTransport.maxChunkBytes), {
|
|
1426
|
-
headers: result.headers,
|
|
1427
|
-
status: result.status,
|
|
1428
|
-
statusText: result.statusText,
|
|
1429
|
-
}),
|
|
1430
|
-
cleanup,
|
|
1431
|
-
);
|
|
1432
|
-
}
|
|
1433
|
-
return responseWithCleanup(result, cleanup);
|
|
1434
|
-
}
|
|
1435
|
-
if (result instanceof ReadableStream) {
|
|
1436
|
-
const httpTransport = getHttpStreamTransport(operation);
|
|
1437
|
-
const stream =
|
|
1438
|
-
httpTransport !== undefined
|
|
1439
|
-
? enforceStreamChunkLimit(result, httpTransport.maxChunkBytes)
|
|
1440
|
-
: result;
|
|
1441
|
-
return responseWithCleanup(
|
|
1442
|
-
new Response(stream, {
|
|
1443
|
-
headers:
|
|
1444
|
-
transport === "sse"
|
|
1445
|
-
? { "Content-Type": "text/event-stream; charset=utf-8" }
|
|
1446
|
-
: {
|
|
1447
|
-
"Content-Type":
|
|
1448
|
-
operation.transport?.kind === "http-stream"
|
|
1449
|
-
? (operation.transport.contentType ?? "application/octet-stream")
|
|
1450
|
-
: "application/octet-stream",
|
|
1451
|
-
},
|
|
1452
|
-
}),
|
|
1453
|
-
cleanup,
|
|
1454
|
-
);
|
|
1455
|
-
}
|
|
1456
|
-
if (transport === "sse" && isAsyncIterable<ProviderStreamEvent>(result)) {
|
|
1457
|
-
return toSseResponse(operation, result, cleanup, requestId);
|
|
1458
|
-
}
|
|
1459
|
-
void cleanup();
|
|
1460
|
-
throw new ProviderError(
|
|
1461
|
-
`Streaming operation returned unsupported result for transport "${transport}"`,
|
|
1462
|
-
{
|
|
1463
|
-
code: "STREAM_RESULT_UNSUPPORTED",
|
|
1464
|
-
fix: "Return an AsyncIterable of stream.event(...) values, a ReadableStream, or a Response from streaming operations.",
|
|
1465
|
-
},
|
|
1466
|
-
);
|
|
1467
|
-
}
|
|
1468
|
-
|
|
1469
|
-
function getSseTransport(operation: OperationDefinition): OperationSseTransport | undefined {
|
|
1470
|
-
return operation.transport?.kind === "sse" ? operation.transport : undefined;
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
function getHttpStreamTransport(
|
|
1474
|
-
operation: OperationDefinition,
|
|
1475
|
-
): OperationHttpStreamTransport | undefined {
|
|
1476
|
-
return operation.transport?.kind === "http-stream" ? operation.transport : undefined;
|
|
1477
|
-
}
|
|
1478
|
-
|
|
1479
|
-
function toAuthFlowResponse(
|
|
1480
|
-
result: unknown,
|
|
1481
|
-
contextPatch: Record<string, unknown | null> | undefined,
|
|
1482
|
-
): Response | AuthFlowSuccessResponse {
|
|
1483
|
-
if (result instanceof Response) {
|
|
1484
|
-
return result;
|
|
1485
|
-
}
|
|
1486
|
-
|
|
1487
|
-
if (result instanceof ReadableStream) {
|
|
1488
|
-
return new Response(result);
|
|
1489
|
-
}
|
|
1490
|
-
|
|
1491
|
-
return {
|
|
1492
|
-
data: result,
|
|
1493
|
-
...(contextPatch ? { contextPatch } : {}),
|
|
1494
|
-
};
|
|
1495
|
-
}
|
|
1496
|
-
|
|
1497
|
-
function authFlowLocaleFromHeaders(headers?: Record<string, string>): ProviderLocale {
|
|
1498
|
-
const header = Object.entries(headers ?? {}).find(
|
|
1499
|
-
([key]) => key.toLowerCase() === "accept-language",
|
|
1500
|
-
)?.[1];
|
|
1501
|
-
for (const token of (header ?? "").split(",")) {
|
|
1502
|
-
const language = token.trim().split(";")[0]?.split("-")[0]?.toLowerCase();
|
|
1503
|
-
if (isAuthFlowLocale(language)) {
|
|
1504
|
-
return language;
|
|
1505
|
-
}
|
|
1506
|
-
}
|
|
1507
|
-
return "en";
|
|
1508
|
-
}
|
|
1509
|
-
|
|
1510
|
-
function isAuthFlowLocale(value: string | undefined): value is ProviderLocale {
|
|
1511
|
-
return value === "en" || value === "ko" || value === "ja";
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
function isAuthTurn(value: unknown): value is AuthTurn {
|
|
1515
|
-
return !!value && typeof value === "object" && "kind" in value && "turnId" in value;
|
|
1516
|
-
}
|
|
1517
|
-
|
|
1518
|
-
function loadAuthFlowLocaleCatalogs(
|
|
1519
|
-
provider: ProviderDefinition,
|
|
1520
|
-
): ProviderLocaleCatalogMap | undefined {
|
|
1521
|
-
for (const providerDir of [
|
|
1522
|
-
process.cwd(),
|
|
1523
|
-
join(process.cwd(), "providers", provider.id),
|
|
1524
|
-
join(process.cwd(), "providers-staging", provider.id),
|
|
1525
|
-
]) {
|
|
1526
|
-
if (!existsSync(join(providerDir, "locales", "en.json"))) continue;
|
|
1527
|
-
try {
|
|
1528
|
-
return loadProviderLocaleCatalogs({
|
|
1529
|
-
providerDir,
|
|
1530
|
-
locales: AUTH_FLOW_LOCALES,
|
|
1531
|
-
});
|
|
1532
|
-
} catch {
|
|
1533
|
-
return undefined;
|
|
1534
|
-
}
|
|
1535
|
-
}
|
|
1536
|
-
return undefined;
|
|
1537
|
-
}
|
|
1538
|
-
|
|
1539
|
-
function materializeAuthFlowTurn(
|
|
1540
|
-
provider: ProviderDefinition,
|
|
1541
|
-
request: AuthFlowRequest,
|
|
1542
|
-
turn: AuthTurn,
|
|
1543
|
-
): AuthTurn {
|
|
1544
|
-
const catalogs = loadAuthFlowLocaleCatalogs(provider);
|
|
1545
|
-
if (!catalogs) return turn;
|
|
1546
|
-
return localizeAuthTurn(turn, {
|
|
1547
|
-
catalogs,
|
|
1548
|
-
locale: authFlowLocaleFromHeaders(request.headers),
|
|
1549
|
-
});
|
|
1550
|
-
}
|
|
1551
|
-
|
|
1552
|
-
function withAuthRequestHeaders(request: AuthFlowRequest, headers: Headers): AuthFlowRequest {
|
|
1553
|
-
return {
|
|
1554
|
-
...request,
|
|
1555
|
-
headers: {
|
|
1556
|
-
...(request.headers ?? {}),
|
|
1557
|
-
...Object.fromEntries(headers.entries()),
|
|
1558
|
-
},
|
|
1559
|
-
};
|
|
1560
|
-
}
|
|
1561
|
-
|
|
1562
|
-
async function handleOperation(
|
|
1563
|
-
provider: ProviderDefinition,
|
|
1564
|
-
request: OperationRequest,
|
|
1565
|
-
operationId: string,
|
|
1566
|
-
options: ProviderServerOptions = {},
|
|
1567
|
-
state: ProviderRuntimeState = createUnsupportedProviderRuntimeState(),
|
|
1568
|
-
proxyTelemetry?: ProxyTelemetryCollector,
|
|
1569
|
-
signal?: AbortSignal,
|
|
1570
|
-
): Promise<Response | OperationResponse> {
|
|
1571
|
-
const ctx = createProviderContext(
|
|
1572
|
-
provider,
|
|
1573
|
-
request,
|
|
1574
|
-
operationId,
|
|
1575
|
-
options,
|
|
1576
|
-
state,
|
|
1577
|
-
proxyTelemetry,
|
|
1578
|
-
signal,
|
|
1579
|
-
);
|
|
1580
|
-
const operation = provider.operations[operationId];
|
|
1581
|
-
const streaming = operation?.transport?.kind && operation.transport.kind !== "json";
|
|
1582
|
-
let cleanupCalled = false;
|
|
1583
|
-
const cleanup = async () => {
|
|
1584
|
-
if (cleanupCalled) return;
|
|
1585
|
-
cleanupCalled = true;
|
|
1586
|
-
try {
|
|
1587
|
-
ctx.stealth.close?.();
|
|
1588
|
-
} catch (error) {
|
|
1589
|
-
logProviderCleanupError(
|
|
1590
|
-
options.logger,
|
|
1591
|
-
provider,
|
|
1592
|
-
operationId,
|
|
1593
|
-
request.requestId,
|
|
1594
|
-
"stealth",
|
|
1595
|
-
error,
|
|
1596
|
-
);
|
|
1597
|
-
}
|
|
1598
|
-
try {
|
|
1599
|
-
await ctx.browser.close?.();
|
|
1600
|
-
} catch (error) {
|
|
1601
|
-
logProviderCleanupError(
|
|
1602
|
-
options.logger,
|
|
1603
|
-
provider,
|
|
1604
|
-
operationId,
|
|
1605
|
-
request.requestId,
|
|
1606
|
-
"browser",
|
|
1607
|
-
error,
|
|
1608
|
-
);
|
|
1609
|
-
}
|
|
1610
|
-
};
|
|
1611
|
-
try {
|
|
1612
|
-
const result = options.operationExecutor
|
|
1613
|
-
? await options.operationExecutor({
|
|
1614
|
-
provider,
|
|
1615
|
-
operationId,
|
|
1616
|
-
ctx,
|
|
1617
|
-
request,
|
|
1618
|
-
signal,
|
|
1619
|
-
})
|
|
1620
|
-
: await executeOperation(provider, operationId, ctx, request.input);
|
|
1621
|
-
if (streaming && operation) {
|
|
1622
|
-
return toStreamingResponse(operation, result, cleanup, request.requestId);
|
|
1623
|
-
}
|
|
1624
|
-
return toJsonSuccessResponse(result, ctx);
|
|
1625
|
-
} catch (error) {
|
|
1626
|
-
await cleanup();
|
|
1627
|
-
throw error;
|
|
1628
|
-
} finally {
|
|
1629
|
-
if (!streaming) await cleanup();
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
|
|
1633
|
-
function responseWithProviderTelemetry(
|
|
1634
|
-
response: Response,
|
|
1635
|
-
proxyTelemetry?: ProxyTelemetryCollector,
|
|
1636
|
-
): Response {
|
|
1637
|
-
const headerValue = proxyTelemetry?.toHeaderValue();
|
|
1638
|
-
const headers = new Headers(response.headers);
|
|
1639
|
-
headers.delete(PROVIDER_TELEMETRY_HEADER);
|
|
1640
|
-
if (headerValue) headers.set(PROVIDER_TELEMETRY_HEADER, headerValue);
|
|
1641
|
-
return new Response(response.body, {
|
|
1642
|
-
headers,
|
|
1643
|
-
status: response.status,
|
|
1644
|
-
statusText: response.statusText,
|
|
1645
|
-
});
|
|
1646
|
-
}
|
|
1647
|
-
|
|
1648
|
-
type AuthRoute = "start" | "continue" | "poll" | "abort" | "refresh";
|
|
1649
|
-
|
|
1650
|
-
async function handleAuthFlow(
|
|
1651
|
-
provider: ProviderDefinition,
|
|
1652
|
-
request: AuthFlowRequest,
|
|
1653
|
-
route: AuthRoute,
|
|
1654
|
-
options: ProviderServerOptions = {},
|
|
1655
|
-
signal?: AbortSignal,
|
|
1656
|
-
): Promise<Response | AuthFlowResponse> {
|
|
1657
|
-
const flow = provider.auth?.flow;
|
|
1658
|
-
if (!flow) {
|
|
1659
|
-
throw new ProviderError("Auth flow is not configured", {
|
|
1660
|
-
code: "AUTH_FLOW_NOT_CONFIGURED",
|
|
1661
|
-
});
|
|
1662
|
-
}
|
|
1663
|
-
|
|
1664
|
-
// Same SDK-owned gate as executeOperation: OAuth/credentials ceremonies
|
|
1665
|
-
// depend on declared secrets (client ids/secrets), so fail structured before
|
|
1666
|
-
// any flow code runs instead of at whatever point the ceremony first reads
|
|
1667
|
-
// the env. `abort` stays exempt: a user must always be able to cancel a
|
|
1668
|
-
// stranded flow even when provisioning is broken.
|
|
1669
|
-
const { context, getPatch } = createAuthFlowContext(provider, request, options, signal);
|
|
1670
|
-
try {
|
|
1671
|
-
if (route !== "abort") {
|
|
1672
|
-
assertRequiredSecretsPresent(provider, context.env);
|
|
1673
|
-
}
|
|
1674
|
-
const result =
|
|
1675
|
-
route === "start"
|
|
1676
|
-
? await flow.start(context)
|
|
1677
|
-
: route === "continue"
|
|
1678
|
-
? await flow.continue(context, request.input ?? {})
|
|
1679
|
-
: route === "poll"
|
|
1680
|
-
? flow.poll
|
|
1681
|
-
? await flow.poll(context)
|
|
1682
|
-
: null
|
|
1683
|
-
: route === "abort"
|
|
1684
|
-
? flow.abort
|
|
1685
|
-
? await flow.abort(context)
|
|
1686
|
-
: null
|
|
1687
|
-
: flow.refresh
|
|
1688
|
-
? await flow.refresh(context, request.input ?? {})
|
|
1689
|
-
: null;
|
|
1690
|
-
|
|
1691
|
-
if (route === "refresh" && !flow.refresh) {
|
|
1692
|
-
throw new AuthError("Provider auth flow does not support refresh.", {
|
|
1693
|
-
code: "refresh_not_supported",
|
|
1694
|
-
});
|
|
1695
|
-
}
|
|
1696
|
-
|
|
1697
|
-
const materializedResult =
|
|
1698
|
-
result &&
|
|
1699
|
-
!(result instanceof Response) &&
|
|
1700
|
-
!(result instanceof ReadableStream) &&
|
|
1701
|
-
isAuthTurn(result)
|
|
1702
|
-
? materializeAuthFlowTurn(provider, request, result)
|
|
1703
|
-
: result;
|
|
1704
|
-
return toAuthFlowResponse(materializedResult, getPatch());
|
|
1705
|
-
} catch (error) {
|
|
1706
|
-
if (error instanceof AuthAbortError) {
|
|
1707
|
-
return toAuthFlowResponse(error.turn, getPatch());
|
|
1708
|
-
}
|
|
1709
|
-
throw error;
|
|
1710
|
-
} finally {
|
|
1711
|
-
context.stealth.close?.();
|
|
1712
|
-
}
|
|
1713
|
-
}
|
|
1714
|
-
|
|
1715
|
-
class StatefulForwardingReplayCache {
|
|
1716
|
-
readonly #nonces = new Map<string, number>();
|
|
1717
|
-
readonly #expiryBuckets = new Map<number, Set<string>>();
|
|
1718
|
-
#nextExpiryBucket?: number;
|
|
1719
|
-
#latestExpiryBucket?: number;
|
|
1720
|
-
|
|
1721
|
-
constructor(private readonly maxEntries: number) {}
|
|
1722
|
-
|
|
1723
|
-
claim(nonce: string, expiresAtMs: number, nowMs: number): "accepted" | "replayed" | "full" {
|
|
1724
|
-
this.dropExpiredBuckets(nowMs);
|
|
1725
|
-
if (this.#nonces.has(nonce)) return "replayed";
|
|
1726
|
-
if (this.#nonces.size >= this.maxEntries) return "full";
|
|
1727
|
-
const expiryBucket =
|
|
1728
|
-
Math.ceil(expiresAtMs / STATEFUL_FORWARDING_REPLAY_BUCKET_MS) *
|
|
1729
|
-
STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
|
|
1730
|
-
this.#nonces.set(nonce, expiryBucket);
|
|
1731
|
-
const bucket = this.#expiryBuckets.get(expiryBucket) ?? new Set<string>();
|
|
1732
|
-
bucket.add(nonce);
|
|
1733
|
-
this.#expiryBuckets.set(expiryBucket, bucket);
|
|
1734
|
-
this.#nextExpiryBucket = Math.min(this.#nextExpiryBucket ?? expiryBucket, expiryBucket);
|
|
1735
|
-
this.#latestExpiryBucket = Math.max(this.#latestExpiryBucket ?? expiryBucket, expiryBucket);
|
|
1736
|
-
return "accepted";
|
|
1737
|
-
}
|
|
1738
|
-
|
|
1739
|
-
private dropExpiredBuckets(nowMs: number): void {
|
|
1740
|
-
if (this.#nextExpiryBucket === undefined || this.#latestExpiryBucket === undefined) return;
|
|
1741
|
-
if (nowMs >= this.#latestExpiryBucket) {
|
|
1742
|
-
this.#nonces.clear();
|
|
1743
|
-
this.#expiryBuckets.clear();
|
|
1744
|
-
this.#nextExpiryBucket = undefined;
|
|
1745
|
-
this.#latestExpiryBucket = undefined;
|
|
1746
|
-
return;
|
|
1747
|
-
}
|
|
1748
|
-
while (this.#nextExpiryBucket <= nowMs) {
|
|
1749
|
-
const bucket = this.#expiryBuckets.get(this.#nextExpiryBucket);
|
|
1750
|
-
if (bucket) {
|
|
1751
|
-
for (const cachedNonce of bucket) this.#nonces.delete(cachedNonce);
|
|
1752
|
-
this.#expiryBuckets.delete(this.#nextExpiryBucket);
|
|
1753
|
-
}
|
|
1754
|
-
this.#nextExpiryBucket += STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
|
|
1755
|
-
}
|
|
1756
|
-
}
|
|
1757
|
-
}
|
|
1758
|
-
|
|
1759
|
-
function verifyStatefulForwardingRequest(input: {
|
|
1760
|
-
readonly options: ProviderServerOptions;
|
|
1761
|
-
readonly rawBody: string;
|
|
1762
|
-
readonly headers: Headers;
|
|
1763
|
-
readonly method: string;
|
|
1764
|
-
readonly path: string;
|
|
1765
|
-
readonly replayCache: StatefulForwardingReplayCache;
|
|
1766
|
-
}): void {
|
|
1767
|
-
const config = input.options.statefulForwarding;
|
|
1768
|
-
if (!config?.secret) {
|
|
1769
|
-
throw new ProviderError("Stateful forwarding is not configured.", {
|
|
1770
|
-
code: "STATEFUL_FORWARDING_NOT_CONFIGURED",
|
|
1771
|
-
});
|
|
1772
|
-
}
|
|
1773
|
-
const timestamp = input.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "";
|
|
1774
|
-
const signature = input.headers.get(STATEFUL_FORWARDING_SIGNATURE_HEADER) ?? "";
|
|
1775
|
-
const nonce = input.headers.get(STATEFUL_FORWARDING_NONCE_HEADER) ?? "";
|
|
1776
|
-
if (!timestamp || !signature || !nonce) {
|
|
1777
|
-
throw new ProviderError("Stateful forwarding signature headers are missing.", {
|
|
1778
|
-
code: "STATEFUL_FORWARDING_SIGNATURE_MISSING",
|
|
1779
|
-
});
|
|
1780
|
-
}
|
|
1781
|
-
if (nonce.length > 256) {
|
|
1782
|
-
throw new ProviderError("Stateful forwarding nonce is invalid.", {
|
|
1783
|
-
code: "STATEFUL_FORWARDING_NONCE_INVALID",
|
|
1784
|
-
});
|
|
1785
|
-
}
|
|
1786
|
-
const timestampMs = Date.parse(timestamp);
|
|
1787
|
-
const maxSkewMs = config.maxSkewMs ?? DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS;
|
|
1788
|
-
if (!Number.isFinite(timestampMs) || Math.abs(Date.now() - timestampMs) > maxSkewMs) {
|
|
1789
|
-
throw new ProviderError(
|
|
1790
|
-
"Stateful forwarding signature timestamp is outside the allowed skew.",
|
|
1791
|
-
{ code: "STATEFUL_FORWARDING_TIMESTAMP_INVALID" },
|
|
1792
|
-
);
|
|
1793
|
-
}
|
|
1794
|
-
if (
|
|
1795
|
-
!verifyStatefulRequestSignature({
|
|
1796
|
-
secret: config.secret,
|
|
1797
|
-
timestamp,
|
|
1798
|
-
rawBody: input.rawBody,
|
|
1799
|
-
method: input.method,
|
|
1800
|
-
path: input.path,
|
|
1801
|
-
nonce,
|
|
1802
|
-
signature,
|
|
1803
|
-
})
|
|
1804
|
-
) {
|
|
1805
|
-
throw new ProviderError("Stateful forwarding signature is invalid.", {
|
|
1806
|
-
code: "STATEFUL_FORWARDING_SIGNATURE_INVALID",
|
|
1807
|
-
});
|
|
1808
|
-
}
|
|
1809
|
-
const replayResult = input.replayCache.claim(nonce, timestampMs + maxSkewMs, Date.now());
|
|
1810
|
-
if (replayResult === "replayed") {
|
|
1811
|
-
throw new ProviderError("Stateful forwarding nonce has already been used.", {
|
|
1812
|
-
code: "STATEFUL_FORWARDING_REPLAY_DETECTED",
|
|
1813
|
-
});
|
|
1814
|
-
}
|
|
1815
|
-
if (replayResult === "full") {
|
|
1816
|
-
throw new ProviderError("Stateful forwarding replay cache is at capacity.", {
|
|
1817
|
-
code: "STATEFUL_FORWARDING_REPLAY_CACHE_FULL",
|
|
1818
|
-
});
|
|
1819
|
-
}
|
|
1820
|
-
}
|
|
1821
|
-
|
|
1822
|
-
function operationRequestFromForwardingEnvelope(
|
|
1823
|
-
envelope: ProviderServerStatefulForwardEnvelope,
|
|
1824
|
-
): OperationRequest & { readonly deadlineAt?: string } {
|
|
1825
|
-
return {
|
|
1826
|
-
...envelope.operationRequest,
|
|
1827
|
-
...(envelope.deadlineAt !== undefined ? { deadlineAt: envelope.deadlineAt } : {}),
|
|
1828
|
-
};
|
|
1829
|
-
}
|
|
1830
|
-
|
|
1831
|
-
function parseStatefulForwardingEnvelope(rawBody: unknown): ProviderServerStatefulForwardEnvelope {
|
|
1832
|
-
const parsed = ProviderServerStatefulForwardEnvelopeSchema.safeParse(rawBody);
|
|
1833
|
-
if (parsed.success) return parsed.data;
|
|
1834
|
-
throw new ProviderError("Stateful forwarding envelope is invalid.", {
|
|
1835
|
-
code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
|
|
1836
|
-
details: zodDetails(parsed.error),
|
|
1837
|
-
});
|
|
1838
|
-
}
|
|
1839
|
-
|
|
1840
|
-
export function createServerApp(
|
|
1841
|
-
provider: ProviderDefinition,
|
|
1842
|
-
options: ProviderServerOptions = {},
|
|
1843
|
-
): Hono {
|
|
1844
|
-
// Fail-closed validation runs here rather than only in serve(): createServerApp
|
|
1845
|
-
// is a public export, so a cast-bypassed declaration would otherwise reach the
|
|
1846
|
-
// request path unvalidated. serve() calls into this function, so validating
|
|
1847
|
-
// here covers both entry points exactly once per app construction.
|
|
1848
|
-
validateFailClosedDeclaration(provider);
|
|
1849
|
-
validateStatefulServerConfig(options);
|
|
1850
|
-
const app = new Hono();
|
|
1851
|
-
const logger = options.logger ?? defaultProviderServerLogger;
|
|
1852
|
-
const operationErrorCodes = buildOperationErrorCodeLookup(provider);
|
|
1853
|
-
const statefulForwardingReplayCache = new StatefulForwardingReplayCache(
|
|
1854
|
-
options.statefulForwarding?.replayCacheMaxEntries ??
|
|
1855
|
-
DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES,
|
|
1856
|
-
);
|
|
1857
|
-
const state =
|
|
1858
|
-
options.state ??
|
|
1859
|
-
createProviderRuntimeStateFromEnv({
|
|
1860
|
-
providerId: provider.id,
|
|
1861
|
-
allowMemoryFallback: options.allowMemoryStateFallback === true,
|
|
1862
|
-
});
|
|
1863
|
-
|
|
1864
|
-
// Boot-time visibility for unprovisioned declared secrets: emit a structured
|
|
1865
|
-
// warn so deploy tooling/alerting sees the gap the moment the pod boots,
|
|
1866
|
-
// instead of discovering it request-by-request. Deliberately log-only — a
|
|
1867
|
-
// boot crash would trade a structured MISSING_SECRET signal for
|
|
1868
|
-
// CrashLoopBackOff. Requests still fail closed via the executeOperation gate.
|
|
1869
|
-
const missingSecretsAtBoot = listMissingRequiredSecrets(
|
|
1870
|
-
provider,
|
|
1871
|
-
createEnvContext(provider.secrets?.map((secret) => secret.name)),
|
|
1872
|
-
);
|
|
1873
|
-
if (missingSecretsAtBoot.length > 0) {
|
|
1874
|
-
logger({
|
|
1875
|
-
level: "warn",
|
|
1876
|
-
event: "provider_secrets_missing",
|
|
1877
|
-
providerId: provider.id,
|
|
1878
|
-
missingSecrets: missingSecretsAtBoot,
|
|
1879
|
-
});
|
|
1880
|
-
}
|
|
1881
|
-
|
|
1882
|
-
app.notFound((c) => {
|
|
1883
|
-
const error = new ProviderError("Not found", { code: "not_found", retryable: false });
|
|
1884
|
-
return responseWithErrorObservability(c.json(toErrorResponse(error), 404), error);
|
|
1885
|
-
});
|
|
1886
|
-
|
|
1887
|
-
app.get("/health", (c) =>
|
|
1888
|
-
c.json({
|
|
1889
|
-
status: "ok",
|
|
1890
|
-
provider: provider.id,
|
|
1891
|
-
version: provider.version,
|
|
1892
|
-
}),
|
|
1893
|
-
);
|
|
1894
|
-
|
|
1895
|
-
app.post(STATEFUL_INTERNAL_OPERATIONS_ROUTE, async (c) => {
|
|
1896
|
-
let rawBodyText = "";
|
|
1897
|
-
let rawBody: unknown;
|
|
1898
|
-
let operationId: string | undefined;
|
|
1899
|
-
const operation = "stateful-internal";
|
|
1900
|
-
const requestCost = startRequestCost();
|
|
1901
|
-
try {
|
|
1902
|
-
if (!options.internalOperationExecutor) {
|
|
1903
|
-
throw new ProviderError("Stateful internal operation executor is not configured.", {
|
|
1904
|
-
code: "STATEFUL_INTERNAL_EXECUTOR_NOT_CONFIGURED",
|
|
1905
|
-
});
|
|
1906
|
-
}
|
|
1907
|
-
rawBodyText = await c.req.raw.clone().text();
|
|
1908
|
-
verifyStatefulForwardingRequest({
|
|
1909
|
-
options,
|
|
1910
|
-
rawBody: rawBodyText,
|
|
1911
|
-
headers: c.req.raw.headers,
|
|
1912
|
-
method: c.req.raw.method,
|
|
1913
|
-
path: STATEFUL_INTERNAL_OPERATIONS_ROUTE,
|
|
1914
|
-
replayCache: statefulForwardingReplayCache,
|
|
1915
|
-
});
|
|
1916
|
-
try {
|
|
1917
|
-
rawBody = JSON.parse(rawBodyText);
|
|
1918
|
-
} catch {
|
|
1919
|
-
throw new ProviderError("Stateful forwarding envelope is not valid JSON.", {
|
|
1920
|
-
code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
|
|
1921
|
-
});
|
|
1922
|
-
}
|
|
1923
|
-
const envelope = parseStatefulForwardingEnvelope(rawBody);
|
|
1924
|
-
if (envelope.providerId !== provider.id) {
|
|
1925
|
-
throw new ProviderError(
|
|
1926
|
-
"Stateful forwarding envelope providerId does not match the served provider.",
|
|
1927
|
-
{ code: "STATEFUL_FORWARDING_PROVIDER_MISMATCH" },
|
|
1928
|
-
);
|
|
1929
|
-
}
|
|
1930
|
-
if (envelope.requestId !== envelope.operationRequest.requestId) {
|
|
1931
|
-
throw new ProviderError("Stateful forwarding requestId values do not match.", {
|
|
1932
|
-
code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
|
|
1933
|
-
});
|
|
1934
|
-
}
|
|
1935
|
-
if (
|
|
1936
|
-
envelope.sourcePodId !==
|
|
1937
|
-
(c.req.raw.headers.get(STATEFUL_FORWARDING_SOURCE_POD_HEADER) ?? "")
|
|
1938
|
-
) {
|
|
1939
|
-
throw new ProviderError("Stateful forwarding source pod does not match its header.", {
|
|
1940
|
-
code: "STATEFUL_FORWARDING_SOURCE_POD_MISMATCH",
|
|
1941
|
-
});
|
|
1942
|
-
}
|
|
1943
|
-
if (
|
|
1944
|
-
envelope.forwardedAt !== (c.req.raw.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "")
|
|
1945
|
-
) {
|
|
1946
|
-
throw new ProviderError(
|
|
1947
|
-
"Stateful forwarding forwardedAt does not match its signature timestamp.",
|
|
1948
|
-
{ code: "STATEFUL_FORWARDING_ENVELOPE_INVALID" },
|
|
1949
|
-
);
|
|
1950
|
-
}
|
|
1951
|
-
const deadlineAtMs = envelope.deadlineAt ? Date.parse(envelope.deadlineAt) : undefined;
|
|
1952
|
-
if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
|
|
1953
|
-
throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
|
|
1954
|
-
}
|
|
1955
|
-
const remainingDeadlineMs =
|
|
1956
|
-
deadlineAtMs === undefined ? undefined : deadlineAtMs - Date.now();
|
|
1957
|
-
const deadlineSignal =
|
|
1958
|
-
remainingDeadlineMs === undefined ? undefined : AbortSignal.timeout(remainingDeadlineMs);
|
|
1959
|
-
const signal = deadlineSignal
|
|
1960
|
-
? AbortSignal.any([c.req.raw.signal, deadlineSignal])
|
|
1961
|
-
: c.req.raw.signal;
|
|
1962
|
-
const ownerFenceValidation = Promise.resolve(
|
|
1963
|
-
options.statefulForwarding?.validateOwnerFence(
|
|
1964
|
-
{
|
|
1965
|
-
providerId: envelope.providerId,
|
|
1966
|
-
sessionKey: envelope.sessionKey,
|
|
1967
|
-
ownerPodId: envelope.ownerPodId,
|
|
1968
|
-
generation: envelope.generation,
|
|
1969
|
-
sourcePodId: envelope.sourcePodId,
|
|
1970
|
-
forwardedAt: envelope.forwardedAt,
|
|
1971
|
-
requestId: envelope.requestId,
|
|
1972
|
-
...(envelope.idempotencyKey ? { idempotencyKey: envelope.idempotencyKey } : {}),
|
|
1973
|
-
},
|
|
1974
|
-
signal,
|
|
1975
|
-
),
|
|
1976
|
-
);
|
|
1977
|
-
let ownerFenceValid: boolean | undefined;
|
|
1978
|
-
try {
|
|
1979
|
-
ownerFenceValid = deadlineSignal
|
|
1980
|
-
? await Promise.race([
|
|
1981
|
-
ownerFenceValidation,
|
|
1982
|
-
new Promise<never>((_resolve, reject) => {
|
|
1983
|
-
deadlineSignal.addEventListener(
|
|
1984
|
-
"abort",
|
|
1985
|
-
() =>
|
|
1986
|
-
reject(
|
|
1987
|
-
new StatefulRoutingDeadlineError(
|
|
1988
|
-
envelope.requestId,
|
|
1989
|
-
envelope.deadlineAt as string,
|
|
1990
|
-
),
|
|
1991
|
-
),
|
|
1992
|
-
{ once: true },
|
|
1993
|
-
);
|
|
1994
|
-
}),
|
|
1995
|
-
])
|
|
1996
|
-
: await ownerFenceValidation;
|
|
1997
|
-
} catch (error) {
|
|
1998
|
-
if (deadlineSignal?.aborted) {
|
|
1999
|
-
throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
|
|
2000
|
-
}
|
|
2001
|
-
throw error;
|
|
2002
|
-
}
|
|
2003
|
-
if (ownerFenceValid !== true) {
|
|
2004
|
-
throw new ProviderError("Stateful forwarding owner fence is no longer current.", {
|
|
2005
|
-
code: "STATEFUL_FORWARDING_OWNER_FENCE_INVALID",
|
|
2006
|
-
});
|
|
2007
|
-
}
|
|
2008
|
-
const request = operationRequestFromForwardingEnvelope(envelope);
|
|
2009
|
-
operationId = envelope.operationId;
|
|
2010
|
-
const ctx = createProviderContext(
|
|
2011
|
-
provider,
|
|
2012
|
-
request,
|
|
2013
|
-
operationId,
|
|
2014
|
-
options,
|
|
2015
|
-
state,
|
|
2016
|
-
undefined,
|
|
2017
|
-
signal,
|
|
2018
|
-
);
|
|
2019
|
-
if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
|
|
2020
|
-
throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
|
|
2021
|
-
}
|
|
2022
|
-
const output = await options.internalOperationExecutor({
|
|
2023
|
-
provider,
|
|
2024
|
-
operationId,
|
|
2025
|
-
ctx,
|
|
2026
|
-
request,
|
|
2027
|
-
internalStatefulForward: envelope,
|
|
2028
|
-
signal,
|
|
2029
|
-
});
|
|
2030
|
-
logProviderSuccess(
|
|
2031
|
-
logger,
|
|
2032
|
-
provider,
|
|
2033
|
-
"operation",
|
|
2034
|
-
operationId || operation,
|
|
2035
|
-
request.requestId,
|
|
2036
|
-
200,
|
|
2037
|
-
finishRequestCost(requestCost),
|
|
2038
|
-
);
|
|
2039
|
-
return c.json({ data: output });
|
|
2040
|
-
} catch (error) {
|
|
2041
|
-
const declaredErrorCode = declaredErrorCodeFor(error, operationId, operationErrorCodes);
|
|
2042
|
-
const status = toStatusCode(error, declaredErrorCode);
|
|
2043
|
-
if (isProviderError(error) && error.code === "STATEFUL_FORWARDING_REPLAY_CACHE_FULL") {
|
|
2044
|
-
c.header("Retry-After", String(STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS));
|
|
2045
|
-
}
|
|
2046
|
-
const requestId = extractRequestId(rawBody);
|
|
2047
|
-
logProviderError(
|
|
2048
|
-
logger,
|
|
2049
|
-
provider,
|
|
2050
|
-
"operation",
|
|
2051
|
-
operationId || operation,
|
|
2052
|
-
requestId,
|
|
2053
|
-
error,
|
|
2054
|
-
status,
|
|
2055
|
-
finishRequestCost(requestCost),
|
|
2056
|
-
declaredErrorCode,
|
|
2057
|
-
);
|
|
2058
|
-
return responseWithErrorObservability(
|
|
2059
|
-
c.json(toErrorResponse(error, requestId, declaredErrorCode), status),
|
|
2060
|
-
error,
|
|
2061
|
-
declaredErrorCode,
|
|
2062
|
-
);
|
|
2063
|
-
}
|
|
2064
|
-
});
|
|
2065
|
-
|
|
2066
|
-
app.post("/v1/:operation", async (c) => {
|
|
2067
|
-
let rawBody: unknown;
|
|
2068
|
-
const operation = c.req.param("operation");
|
|
2069
|
-
const proxyTelemetry = new ProxyTelemetryCollector();
|
|
2070
|
-
const requestCost = startRequestCost();
|
|
2071
|
-
try {
|
|
2072
|
-
rawBody = await c.req.raw
|
|
2073
|
-
.clone()
|
|
2074
|
-
.json()
|
|
2075
|
-
.catch(() => undefined);
|
|
2076
|
-
const body = OperationRequestSchema.parse(rawBody);
|
|
2077
|
-
const requestHeaders = Object.fromEntries(c.req.raw.headers.entries());
|
|
2078
|
-
body.headers = { ...requestHeaders, ...body.headers };
|
|
2079
|
-
const response = await handleOperation(
|
|
2080
|
-
provider,
|
|
2081
|
-
body,
|
|
2082
|
-
operation,
|
|
2083
|
-
options,
|
|
2084
|
-
state,
|
|
2085
|
-
proxyTelemetry,
|
|
2086
|
-
c.req.raw.signal,
|
|
2087
|
-
);
|
|
2088
|
-
if (response instanceof Response) {
|
|
2089
|
-
logProviderSuccess(
|
|
2090
|
-
logger,
|
|
2091
|
-
provider,
|
|
2092
|
-
"operation",
|
|
2093
|
-
operation,
|
|
2094
|
-
body.requestId,
|
|
2095
|
-
response.status,
|
|
2096
|
-
finishRequestCost(requestCost),
|
|
2097
|
-
);
|
|
2098
|
-
return responseWithProviderTelemetry(response, proxyTelemetry);
|
|
2099
|
-
}
|
|
2100
|
-
const telemetryHeader = proxyTelemetry.toHeaderValue();
|
|
2101
|
-
if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
|
|
2102
|
-
logProviderSuccess(
|
|
2103
|
-
logger,
|
|
2104
|
-
provider,
|
|
2105
|
-
"operation",
|
|
2106
|
-
operation,
|
|
2107
|
-
body.requestId,
|
|
2108
|
-
200,
|
|
2109
|
-
finishRequestCost(requestCost),
|
|
2110
|
-
);
|
|
2111
|
-
return c.json(response);
|
|
2112
|
-
} catch (error) {
|
|
2113
|
-
const declaredErrorCode = declaredErrorCodeFor(error, operation, operationErrorCodes);
|
|
2114
|
-
const status = toStatusCode(error, declaredErrorCode);
|
|
2115
|
-
const requestId = extractRequestId(rawBody);
|
|
2116
|
-
logProviderError(
|
|
2117
|
-
logger,
|
|
2118
|
-
provider,
|
|
2119
|
-
"operation",
|
|
2120
|
-
operation,
|
|
2121
|
-
requestId,
|
|
2122
|
-
error,
|
|
2123
|
-
status,
|
|
2124
|
-
finishRequestCost(requestCost),
|
|
2125
|
-
declaredErrorCode,
|
|
2126
|
-
);
|
|
2127
|
-
const telemetryHeader = proxyTelemetry.toHeaderValue();
|
|
2128
|
-
if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
|
|
2129
|
-
return responseWithErrorObservability(
|
|
2130
|
-
c.json(toErrorResponse(error, requestId, declaredErrorCode), status),
|
|
2131
|
-
error,
|
|
2132
|
-
declaredErrorCode,
|
|
2133
|
-
);
|
|
2134
|
-
}
|
|
2135
|
-
});
|
|
2136
|
-
|
|
2137
|
-
app.post("/auth/start", async (c) => {
|
|
2138
|
-
let rawBody: unknown;
|
|
2139
|
-
const requestCost = startRequestCost();
|
|
2140
|
-
try {
|
|
2141
|
-
rawBody = await c.req.raw
|
|
2142
|
-
.clone()
|
|
2143
|
-
.json()
|
|
2144
|
-
.catch(() => undefined);
|
|
2145
|
-
const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
|
|
2146
|
-
const response = await handleAuthFlow(provider, body, "start", options, c.req.raw.signal);
|
|
2147
|
-
logProviderSuccess(
|
|
2148
|
-
logger,
|
|
2149
|
-
provider,
|
|
2150
|
-
"auth",
|
|
2151
|
-
"start",
|
|
2152
|
-
body.requestId,
|
|
2153
|
-
response instanceof Response ? response.status : 200,
|
|
2154
|
-
finishRequestCost(requestCost),
|
|
2155
|
-
);
|
|
2156
|
-
return response instanceof Response ? response : c.json(response);
|
|
2157
|
-
} catch (error) {
|
|
2158
|
-
const status = toStatusCode(error);
|
|
2159
|
-
const requestId = extractRequestId(rawBody);
|
|
2160
|
-
logProviderError(
|
|
2161
|
-
logger,
|
|
2162
|
-
provider,
|
|
2163
|
-
"auth",
|
|
2164
|
-
"start",
|
|
2165
|
-
requestId,
|
|
2166
|
-
error,
|
|
2167
|
-
status,
|
|
2168
|
-
finishRequestCost(requestCost),
|
|
2169
|
-
);
|
|
2170
|
-
return responseWithErrorObservability(
|
|
2171
|
-
c.json(toErrorResponse(error, requestId), status),
|
|
2172
|
-
error,
|
|
2173
|
-
);
|
|
2174
|
-
}
|
|
2175
|
-
});
|
|
2176
|
-
|
|
2177
|
-
app.post("/auth/continue", async (c) => {
|
|
2178
|
-
let rawBody: unknown;
|
|
2179
|
-
const requestCost = startRequestCost();
|
|
2180
|
-
try {
|
|
2181
|
-
rawBody = await c.req.raw
|
|
2182
|
-
.clone()
|
|
2183
|
-
.json()
|
|
2184
|
-
.catch(() => undefined);
|
|
2185
|
-
const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
|
|
2186
|
-
const response = await handleAuthFlow(provider, body, "continue", options, c.req.raw.signal);
|
|
2187
|
-
logProviderSuccess(
|
|
2188
|
-
logger,
|
|
2189
|
-
provider,
|
|
2190
|
-
"auth",
|
|
2191
|
-
"continue",
|
|
2192
|
-
body.requestId,
|
|
2193
|
-
response instanceof Response ? response.status : 200,
|
|
2194
|
-
finishRequestCost(requestCost),
|
|
2195
|
-
);
|
|
2196
|
-
return response instanceof Response ? response : c.json(response);
|
|
2197
|
-
} catch (error) {
|
|
2198
|
-
const status = toStatusCode(error);
|
|
2199
|
-
const requestId = extractRequestId(rawBody);
|
|
2200
|
-
logProviderError(
|
|
2201
|
-
logger,
|
|
2202
|
-
provider,
|
|
2203
|
-
"auth",
|
|
2204
|
-
"continue",
|
|
2205
|
-
requestId,
|
|
2206
|
-
error,
|
|
2207
|
-
status,
|
|
2208
|
-
finishRequestCost(requestCost),
|
|
2209
|
-
);
|
|
2210
|
-
return responseWithErrorObservability(
|
|
2211
|
-
c.json(toErrorResponse(error, requestId), status),
|
|
2212
|
-
error,
|
|
2213
|
-
);
|
|
2214
|
-
}
|
|
2215
|
-
});
|
|
2216
|
-
|
|
2217
|
-
app.post("/auth/poll", async (c) => {
|
|
2218
|
-
let rawBody: unknown;
|
|
2219
|
-
const requestCost = startRequestCost();
|
|
2220
|
-
try {
|
|
2221
|
-
rawBody = await c.req.raw
|
|
2222
|
-
.clone()
|
|
2223
|
-
.json()
|
|
2224
|
-
.catch(() => undefined);
|
|
2225
|
-
const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
|
|
2226
|
-
const response = await handleAuthFlow(provider, body, "poll", options, c.req.raw.signal);
|
|
2227
|
-
logProviderSuccess(
|
|
2228
|
-
logger,
|
|
2229
|
-
provider,
|
|
2230
|
-
"auth",
|
|
2231
|
-
"poll",
|
|
2232
|
-
body.requestId,
|
|
2233
|
-
response instanceof Response ? response.status : 200,
|
|
2234
|
-
finishRequestCost(requestCost),
|
|
2235
|
-
);
|
|
2236
|
-
return response instanceof Response ? response : c.json(response);
|
|
2237
|
-
} catch (error) {
|
|
2238
|
-
const status = toStatusCode(error);
|
|
2239
|
-
const requestId = extractRequestId(rawBody);
|
|
2240
|
-
logProviderError(
|
|
2241
|
-
logger,
|
|
2242
|
-
provider,
|
|
2243
|
-
"auth",
|
|
2244
|
-
"poll",
|
|
2245
|
-
requestId,
|
|
2246
|
-
error,
|
|
2247
|
-
status,
|
|
2248
|
-
finishRequestCost(requestCost),
|
|
2249
|
-
);
|
|
2250
|
-
return responseWithErrorObservability(
|
|
2251
|
-
c.json(toErrorResponse(error, requestId), status),
|
|
2252
|
-
error,
|
|
2253
|
-
);
|
|
2254
|
-
}
|
|
2255
|
-
});
|
|
2256
|
-
|
|
2257
|
-
app.post("/auth/refresh", async (c) => {
|
|
2258
|
-
let rawBody: unknown;
|
|
2259
|
-
const requestCost = startRequestCost();
|
|
2260
|
-
try {
|
|
2261
|
-
rawBody = await c.req.raw
|
|
2262
|
-
.clone()
|
|
2263
|
-
.json()
|
|
2264
|
-
.catch(() => undefined);
|
|
2265
|
-
const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
|
|
2266
|
-
const response = await handleAuthFlow(provider, body, "refresh", options, c.req.raw.signal);
|
|
2267
|
-
logProviderSuccess(
|
|
2268
|
-
logger,
|
|
2269
|
-
provider,
|
|
2270
|
-
"auth",
|
|
2271
|
-
"refresh",
|
|
2272
|
-
body.requestId,
|
|
2273
|
-
response instanceof Response ? response.status : 200,
|
|
2274
|
-
finishRequestCost(requestCost),
|
|
2275
|
-
);
|
|
2276
|
-
return response instanceof Response ? response : c.json(response);
|
|
2277
|
-
} catch (error) {
|
|
2278
|
-
const status = toStatusCode(error);
|
|
2279
|
-
const requestId = extractRequestId(rawBody);
|
|
2280
|
-
logProviderError(
|
|
2281
|
-
logger,
|
|
2282
|
-
provider,
|
|
2283
|
-
"auth",
|
|
2284
|
-
"refresh",
|
|
2285
|
-
requestId,
|
|
2286
|
-
error,
|
|
2287
|
-
status,
|
|
2288
|
-
finishRequestCost(requestCost),
|
|
2289
|
-
);
|
|
2290
|
-
return responseWithErrorObservability(
|
|
2291
|
-
c.json(toErrorResponse(error, requestId), status),
|
|
2292
|
-
error,
|
|
2293
|
-
);
|
|
2294
|
-
}
|
|
2295
|
-
});
|
|
2296
|
-
|
|
2297
|
-
app.post("/auth/disconnect", async (c) => {
|
|
2298
|
-
let rawBody: unknown;
|
|
2299
|
-
const requestCost = startRequestCost();
|
|
2300
|
-
try {
|
|
2301
|
-
rawBody = await c.req.raw
|
|
2302
|
-
.clone()
|
|
2303
|
-
.json()
|
|
2304
|
-
.catch(() => undefined);
|
|
2305
|
-
const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
|
|
2306
|
-
const response = await handleAuthFlow(provider, body, "abort", options, c.req.raw.signal);
|
|
2307
|
-
logProviderSuccess(
|
|
2308
|
-
logger,
|
|
2309
|
-
provider,
|
|
2310
|
-
"auth",
|
|
2311
|
-
"disconnect",
|
|
2312
|
-
body.requestId,
|
|
2313
|
-
response instanceof Response ? response.status : 200,
|
|
2314
|
-
finishRequestCost(requestCost),
|
|
2315
|
-
);
|
|
2316
|
-
return response instanceof Response ? response : c.json(response);
|
|
2317
|
-
} catch (error) {
|
|
2318
|
-
const status = toStatusCode(error);
|
|
2319
|
-
const requestId = extractRequestId(rawBody);
|
|
2320
|
-
logProviderError(
|
|
2321
|
-
logger,
|
|
2322
|
-
provider,
|
|
2323
|
-
"auth",
|
|
2324
|
-
"disconnect",
|
|
2325
|
-
requestId,
|
|
2326
|
-
error,
|
|
2327
|
-
status,
|
|
2328
|
-
finishRequestCost(requestCost),
|
|
2329
|
-
);
|
|
2330
|
-
return responseWithErrorObservability(
|
|
2331
|
-
c.json(toErrorResponse(error, requestId), status),
|
|
2332
|
-
error,
|
|
2333
|
-
);
|
|
2334
|
-
}
|
|
2335
|
-
});
|
|
2336
|
-
|
|
2337
|
-
return app;
|
|
2338
|
-
}
|
|
2339
|
-
|
|
2340
|
-
function validateStatefulServerConfig(options: ProviderServerOptions): void {
|
|
2341
|
-
if (options.statefulForwarding && !options.internalOperationExecutor) {
|
|
2342
|
-
throw new Error(
|
|
2343
|
-
"Invalid provider server configuration: statefulForwarding requires internalOperationExecutor; missing option internalOperationExecutor.",
|
|
2344
|
-
);
|
|
2345
|
-
}
|
|
2346
|
-
if (options.internalOperationExecutor && !options.statefulForwarding?.secret) {
|
|
2347
|
-
throw new Error(
|
|
2348
|
-
"Invalid provider server configuration: internalOperationExecutor requires statefulForwarding.secret; missing option statefulForwarding.secret.",
|
|
2349
|
-
);
|
|
2350
|
-
}
|
|
2351
|
-
if (
|
|
2352
|
-
options.statefulForwarding &&
|
|
2353
|
-
typeof options.statefulForwarding.validateOwnerFence !== "function"
|
|
2354
|
-
) {
|
|
2355
|
-
throw new Error(
|
|
2356
|
-
"Invalid provider server configuration: statefulForwarding requires validateOwnerFence.",
|
|
2357
|
-
);
|
|
2358
|
-
}
|
|
2359
|
-
if (
|
|
2360
|
-
options.statefulForwarding?.maxSkewMs !== undefined &&
|
|
2361
|
-
(!Number.isFinite(options.statefulForwarding.maxSkewMs) ||
|
|
2362
|
-
options.statefulForwarding.maxSkewMs <= 0)
|
|
2363
|
-
) {
|
|
2364
|
-
throw new Error("Invalid provider server configuration: maxSkewMs must be positive.");
|
|
2365
|
-
}
|
|
2366
|
-
if (
|
|
2367
|
-
options.statefulForwarding?.replayCacheMaxEntries !== undefined &&
|
|
2368
|
-
(!Number.isInteger(options.statefulForwarding.replayCacheMaxEntries) ||
|
|
2369
|
-
options.statefulForwarding.replayCacheMaxEntries <= 0)
|
|
2370
|
-
) {
|
|
2371
|
-
throw new Error(
|
|
2372
|
-
"Invalid provider server configuration: replayCacheMaxEntries must be a positive integer.",
|
|
2373
|
-
);
|
|
2374
|
-
}
|
|
2375
|
-
}
|
|
2376
|
-
|
|
2377
|
-
type BunServeRuntime = {
|
|
2378
|
-
serve: (options: {
|
|
2379
|
-
port: number;
|
|
2380
|
-
hostname: string;
|
|
2381
|
-
fetch: (request: Request) => Response | Promise<Response>;
|
|
2382
|
-
}) => BunServerHandle;
|
|
2383
|
-
};
|
|
2384
|
-
|
|
2385
|
-
type BunServerHandle = {
|
|
2386
|
-
readonly port: number;
|
|
2387
|
-
stop(closeActiveConnections?: boolean): Promise<void>;
|
|
2388
|
-
};
|
|
2389
|
-
|
|
2390
|
-
function getBunServeRuntime(): BunServeRuntime | undefined {
|
|
2391
|
-
const bunValue = Object.getOwnPropertyDescriptor(globalThis, "Bun")?.value;
|
|
2392
|
-
if (!bunValue || typeof bunValue !== "object") {
|
|
2393
|
-
return undefined;
|
|
2394
|
-
}
|
|
2395
|
-
|
|
2396
|
-
const serve = Object.getOwnPropertyDescriptor(bunValue, "serve")?.value;
|
|
2397
|
-
if (typeof serve !== "function") {
|
|
2398
|
-
return undefined;
|
|
2399
|
-
}
|
|
2400
|
-
|
|
2401
|
-
return {
|
|
2402
|
-
serve(options) {
|
|
2403
|
-
return serve(options) as BunServerHandle;
|
|
2404
|
-
},
|
|
2405
|
-
};
|
|
2406
|
-
}
|
|
2407
|
-
|
|
2408
|
-
export type ProviderServerCloseOptions = {
|
|
2409
|
-
readonly timeoutMs?: number;
|
|
2410
|
-
};
|
|
2411
|
-
|
|
2412
|
-
export type ProviderServerHandle = {
|
|
2413
|
-
readonly port: number;
|
|
2414
|
-
close(options?: ProviderServerCloseOptions): Promise<void>;
|
|
2415
|
-
};
|
|
2416
|
-
|
|
2417
|
-
export interface ServeOptions extends ProviderServerOptions {
|
|
2418
|
-
host?: string;
|
|
2419
|
-
port?: number;
|
|
2420
|
-
/**
|
|
2421
|
-
* Port for the internal self-test listener (default 3001 or
|
|
2422
|
-
* APIFUSE__PROVIDER_RUNTIME__SELF_TEST_PORT). The listener only starts
|
|
2423
|
-
* when APIFUSE__PROVIDER_RUNTIME__SELF_TEST_MASTER_SECRET is present.
|
|
2424
|
-
*/
|
|
2425
|
-
selfTestPort?: number;
|
|
2426
|
-
}
|
|
2427
|
-
|
|
2428
|
-
const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
|
|
2429
|
-
const DEFAULT_SHUTDOWN_SIGNALS: NodeJS.Signals[] = ["SIGTERM", "SIGINT"];
|
|
2430
|
-
|
|
2431
|
-
type SignalServerRegistration = {
|
|
2432
|
-
readonly signals: ReadonlySet<NodeJS.Signals>;
|
|
2433
|
-
readonly close: () => Promise<void>;
|
|
2434
|
-
};
|
|
2435
|
-
|
|
2436
|
-
type ProcessSignalCoordinator = {
|
|
2437
|
-
readonly registrations: Set<SignalServerRegistration>;
|
|
2438
|
-
readonly listener: () => void;
|
|
2439
|
-
handling: boolean;
|
|
2440
|
-
};
|
|
2441
|
-
|
|
2442
|
-
const processSignalCoordinators = new Map<NodeJS.Signals, ProcessSignalCoordinator>();
|
|
2443
|
-
|
|
2444
|
-
export async function serve(
|
|
2445
|
-
provider: ProviderDefinition,
|
|
2446
|
-
options: ServeOptions = {},
|
|
2447
|
-
): Promise<ProviderServerHandle> {
|
|
2448
|
-
// Declaration validation happens inside createServerApp (the shared app
|
|
2449
|
-
// construction path), so serve() does not duplicate the call here.
|
|
2450
|
-
const bunRuntime = getBunServeRuntime();
|
|
2451
|
-
|
|
2452
|
-
if (bunRuntime === undefined) {
|
|
2453
|
-
throw new ProviderError("Bun runtime is required to start the provider server", {
|
|
2454
|
-
code: "RUNTIME_UNSUPPORTED",
|
|
2455
|
-
});
|
|
2456
|
-
}
|
|
2457
|
-
const logger = options.logger ?? defaultProviderServerLogger;
|
|
2458
|
-
const configuredTimeoutMs = shutdownTimeout(
|
|
2459
|
-
options.shutdown?.timeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS,
|
|
2460
|
-
);
|
|
2461
|
-
const configuredSignals = resolveShutdownSignals(options.shutdown?.signals ?? true);
|
|
2462
|
-
|
|
2463
|
-
const app = createServerApp(provider, {
|
|
2464
|
-
logger: options.logger,
|
|
2465
|
-
ocr: options.ocr,
|
|
2466
|
-
stt: options.stt,
|
|
2467
|
-
resolver: options.resolver,
|
|
2468
|
-
state: options.state,
|
|
2469
|
-
allowMemoryStateFallback: options.allowMemoryStateFallback,
|
|
2470
|
-
operationExecutor: options.operationExecutor,
|
|
2471
|
-
internalOperationExecutor: options.internalOperationExecutor,
|
|
2472
|
-
statefulForwarding: options.statefulForwarding,
|
|
2473
|
-
});
|
|
2474
|
-
|
|
2475
|
-
const servers: BunServerHandle[] = [];
|
|
2476
|
-
try {
|
|
2477
|
-
servers.push(
|
|
2478
|
-
bunRuntime.serve({
|
|
2479
|
-
port: options.port ?? DEFAULT_PORT,
|
|
2480
|
-
hostname: options.host ?? DEFAULT_HOST,
|
|
2481
|
-
fetch: app.fetch,
|
|
2482
|
-
}),
|
|
2483
|
-
);
|
|
2484
|
-
|
|
2485
|
-
// Internal self-test listener (health dependency inversion): a SEPARATE
|
|
2486
|
-
// socket the tenant-facing gateway never dials. Off by default — it only
|
|
2487
|
-
// starts when the shared self-test master secret env is present.
|
|
2488
|
-
const selfTestSecrets = resolveSelfTestMasterSecrets();
|
|
2489
|
-
if (selfTestSecrets) {
|
|
2490
|
-
const selfTestApp = createSelfTestApp(provider, {
|
|
2491
|
-
secrets: selfTestSecrets,
|
|
2492
|
-
invoke: createSelfTestInvoke(app),
|
|
2493
|
-
authFlow: createSelfTestAuthFlowInvoke(app),
|
|
2494
|
-
});
|
|
2495
|
-
servers.push(
|
|
2496
|
-
bunRuntime.serve({
|
|
2497
|
-
port: options.selfTestPort ?? resolveSelfTestPort(),
|
|
2498
|
-
hostname: options.host ?? DEFAULT_HOST,
|
|
2499
|
-
fetch: selfTestApp.fetch,
|
|
2500
|
-
}),
|
|
2501
|
-
);
|
|
2502
|
-
}
|
|
2503
|
-
} catch (error) {
|
|
2504
|
-
await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
|
|
2505
|
-
throw error;
|
|
2506
|
-
}
|
|
2507
|
-
|
|
2508
|
-
const server = servers[0];
|
|
2509
|
-
if (!server) throw new Error("Provider server failed to create its primary listener.");
|
|
2510
|
-
let closePromise: Promise<void> | undefined;
|
|
2511
|
-
let unregisterSignals = () => {};
|
|
2512
|
-
|
|
2513
|
-
const close = (closeOptions: ProviderServerCloseOptions = {}): Promise<void> => {
|
|
2514
|
-
if (closePromise) return closePromise;
|
|
2515
|
-
const timeoutMs = shutdownTimeout(closeOptions.timeoutMs ?? configuredTimeoutMs);
|
|
2516
|
-
closePromise = closeProviderServers({
|
|
2517
|
-
servers,
|
|
2518
|
-
hooks: options.shutdown?.hooks ?? [],
|
|
2519
|
-
timeoutMs,
|
|
2520
|
-
logger,
|
|
2521
|
-
providerId: provider.id,
|
|
2522
|
-
}).finally(() => unregisterSignals());
|
|
2523
|
-
return closePromise;
|
|
2524
|
-
};
|
|
2525
|
-
|
|
2526
|
-
try {
|
|
2527
|
-
unregisterSignals = registerForProcessSignals(configuredSignals, () =>
|
|
2528
|
-
close({ timeoutMs: configuredTimeoutMs }),
|
|
2529
|
-
);
|
|
2530
|
-
} catch (error) {
|
|
2531
|
-
unregisterSignals();
|
|
2532
|
-
await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
|
|
2533
|
-
throw error;
|
|
2534
|
-
}
|
|
2535
|
-
|
|
2536
|
-
return { port: server.port, close };
|
|
2537
|
-
}
|
|
2538
|
-
|
|
2539
|
-
function registerForProcessSignals(
|
|
2540
|
-
signals: NodeJS.Signals[],
|
|
2541
|
-
close: () => Promise<void>,
|
|
2542
|
-
): () => void {
|
|
2543
|
-
if (signals.length === 0) return () => {};
|
|
2544
|
-
const registration: SignalServerRegistration = {
|
|
2545
|
-
signals: new Set(signals),
|
|
2546
|
-
close,
|
|
2547
|
-
};
|
|
2548
|
-
let registered = true;
|
|
2549
|
-
const unregister = () => {
|
|
2550
|
-
if (!registered) return;
|
|
2551
|
-
registered = false;
|
|
2552
|
-
for (const signal of registration.signals) {
|
|
2553
|
-
const coordinator = processSignalCoordinators.get(signal);
|
|
2554
|
-
if (!coordinator) continue;
|
|
2555
|
-
coordinator.registrations.delete(registration);
|
|
2556
|
-
if (coordinator.registrations.size === 0 && !coordinator.handling) {
|
|
2557
|
-
process.removeListener(signal, coordinator.listener);
|
|
2558
|
-
processSignalCoordinators.delete(signal);
|
|
2559
|
-
}
|
|
2560
|
-
}
|
|
2561
|
-
};
|
|
2562
|
-
try {
|
|
2563
|
-
for (const signal of signals) {
|
|
2564
|
-
let coordinator = processSignalCoordinators.get(signal);
|
|
2565
|
-
if (!coordinator) {
|
|
2566
|
-
const created: ProcessSignalCoordinator = {
|
|
2567
|
-
registrations: new Set(),
|
|
2568
|
-
handling: false,
|
|
2569
|
-
listener: () => handleCoordinatedSignal(signal, created),
|
|
2570
|
-
};
|
|
2571
|
-
coordinator = created;
|
|
2572
|
-
processSignalCoordinators.set(signal, coordinator);
|
|
2573
|
-
process.on(signal, coordinator.listener);
|
|
2574
|
-
}
|
|
2575
|
-
coordinator.registrations.add(registration);
|
|
2576
|
-
}
|
|
2577
|
-
} catch (error) {
|
|
2578
|
-
unregister();
|
|
2579
|
-
throw error;
|
|
2580
|
-
}
|
|
2581
|
-
return unregister;
|
|
2582
|
-
}
|
|
2583
|
-
|
|
2584
|
-
function handleCoordinatedSignal(
|
|
2585
|
-
signal: NodeJS.Signals,
|
|
2586
|
-
coordinator: ProcessSignalCoordinator,
|
|
2587
|
-
): void {
|
|
2588
|
-
if (coordinator.handling) return;
|
|
2589
|
-
coordinator.handling = true;
|
|
2590
|
-
const registrations = [...coordinator.registrations];
|
|
2591
|
-
void Promise.allSettled(registrations.map((registration) => registration.close())).finally(() => {
|
|
2592
|
-
if (processSignalCoordinators.get(signal) === coordinator) {
|
|
2593
|
-
process.removeListener(signal, coordinator.listener);
|
|
2594
|
-
processSignalCoordinators.delete(signal);
|
|
2595
|
-
}
|
|
2596
|
-
try {
|
|
2597
|
-
process.kill(process.pid, signal);
|
|
2598
|
-
} catch {
|
|
2599
|
-
process.exitCode = 1;
|
|
2600
|
-
}
|
|
2601
|
-
});
|
|
2602
|
-
}
|
|
2603
|
-
|
|
2604
|
-
async function closeProviderServers(input: {
|
|
2605
|
-
readonly servers: BunServerHandle[];
|
|
2606
|
-
readonly hooks: Array<() => Promise<void>>;
|
|
2607
|
-
readonly timeoutMs: number;
|
|
2608
|
-
readonly logger: ProviderServerLogger;
|
|
2609
|
-
readonly providerId: string;
|
|
2610
|
-
}): Promise<void> {
|
|
2611
|
-
const deadline = Date.now() + input.timeoutMs;
|
|
2612
|
-
const gracefulStops = input.servers.map((server) => server.stop(false));
|
|
2613
|
-
for (const gracefulStop of gracefulStops) gracefulStop.catch(() => undefined);
|
|
2614
|
-
for (const [hookIndex, hook] of input.hooks.entries()) {
|
|
2615
|
-
try {
|
|
2616
|
-
await withinShutdownBudget(Promise.resolve().then(hook), deadline);
|
|
2617
|
-
} catch (error) {
|
|
2618
|
-
try {
|
|
2619
|
-
input.logger({
|
|
2620
|
-
level: "error",
|
|
2621
|
-
event: "provider_shutdown_hook_failed",
|
|
2622
|
-
providerId: input.providerId,
|
|
2623
|
-
hookIndex,
|
|
2624
|
-
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
2625
|
-
message: error instanceof Error ? error.message : "Shutdown hook failed.",
|
|
2626
|
-
});
|
|
2627
|
-
} catch {}
|
|
2628
|
-
}
|
|
2629
|
-
}
|
|
2630
|
-
const forcedStops = input.servers.map((server) => server.stop(true));
|
|
2631
|
-
await withinShutdownBudget(
|
|
2632
|
-
Promise.allSettled([...gracefulStops, ...forcedStops]).then(() => undefined),
|
|
2633
|
-
deadline,
|
|
2634
|
-
).catch(() => undefined);
|
|
2635
|
-
}
|
|
2636
|
-
|
|
2637
|
-
async function withinShutdownBudget<T>(promise: Promise<T>, deadline: number): Promise<T> {
|
|
2638
|
-
promise.catch(() => undefined);
|
|
2639
|
-
const remainingMs = Math.max(0, deadline - Date.now());
|
|
2640
|
-
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
2641
|
-
const timeout = new Promise<never>((_resolve, reject) => {
|
|
2642
|
-
timer = setTimeout(() => reject(new Error("Provider server shutdown timed out.")), remainingMs);
|
|
2643
|
-
});
|
|
2644
|
-
try {
|
|
2645
|
-
return await Promise.race([promise, timeout]);
|
|
2646
|
-
} finally {
|
|
2647
|
-
if (timer) clearTimeout(timer);
|
|
2648
|
-
}
|
|
2649
|
-
}
|
|
2650
|
-
|
|
2651
|
-
function resolveShutdownSignals(signals: boolean | NodeJS.Signals[]): NodeJS.Signals[] {
|
|
2652
|
-
if (signals === false) return [];
|
|
2653
|
-
return [...new Set(signals === true ? DEFAULT_SHUTDOWN_SIGNALS : signals)];
|
|
2654
|
-
}
|
|
2655
|
-
|
|
2656
|
-
function shutdownTimeout(value: number): number {
|
|
2657
|
-
if (!Number.isFinite(value) || value < 0) {
|
|
2658
|
-
throw new Error("Provider server shutdown timeoutMs must be a non-negative finite number.");
|
|
2659
|
-
}
|
|
2660
|
-
return value;
|
|
2661
|
-
}
|
|
1
|
+
export * from "./serve-implementation.js";
|