@apifuse/provider-sdk 2.2.0-beta.11 → 2.2.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +238 -0
- package/CHANGELOG.md +14 -0
- package/README.md +44 -2
- package/bin/apifuse-pack-smoke.ts +14 -0
- package/bin/apifuse-pack-types.ts +40 -1
- package/bin/apifuse-record.ts +622 -57
- package/bin/apifuse-submit-check.ts +43 -10
- package/dist/config/loader.d.ts +9 -1
- package/dist/config/loader.js +9 -0
- package/dist/define.d.ts +2 -1
- package/dist/define.js +61 -3
- package/dist/errors.d.ts +5 -0
- package/dist/errors.js +15 -0
- package/dist/fixture-sanitization.d.ts +26 -0
- package/dist/fixture-sanitization.js +216 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +2 -1
- package/dist/provider.d.ts +2 -1
- package/dist/provider.js +1 -0
- package/dist/runtime/http.js +86 -32
- package/dist/runtime/instrumentation.js +295 -9
- package/dist/runtime/native-network.d.ts +53 -0
- package/dist/runtime/native-network.js +477 -0
- package/dist/runtime/proxy-nodemaven.d.ts +14 -0
- package/dist/runtime/proxy-nodemaven.js +20 -2
- package/dist/runtime/request-options.d.ts +68 -1
- package/dist/runtime/request-options.js +548 -0
- package/dist/runtime/stealth.d.ts +3 -1
- package/dist/runtime/stealth.js +352 -86
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.js +1 -1
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/testing/index.d.ts +1 -1
- package/dist/testing/index.js +1 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +451 -19
- package/dist/types.d.ts +201 -7
- package/package.json +3 -1
- package/src/config/loader.ts +22 -1
- package/src/define.ts +81 -3
- package/src/errors.ts +15 -0
- package/src/fixture-sanitization.ts +247 -0
- package/src/index.ts +45 -1
- package/src/provider.ts +37 -0
- package/src/runtime/http.ts +144 -38
- package/src/runtime/instrumentation.ts +424 -8
- package/src/runtime/native-network.ts +600 -0
- package/src/runtime/proxy-nodemaven.ts +37 -2
- package/src/runtime/request-options.ts +680 -1
- package/src/runtime/stealth.ts +420 -88
- package/src/server/index.ts +4 -1
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/stream-evidence.ts +988 -0
- package/src/testing/index.ts +9 -1
- package/src/testing/run.ts +608 -12
- package/src/types.ts +235 -7
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { Impit } from "impit";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
4
|
+
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
5
|
+
import { SDKError, StealthCookieStoreVersionError, TransportError } from "../errors.js";
|
|
5
6
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
6
7
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
7
8
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
8
|
-
import {
|
|
9
|
+
import { isSensitiveKey, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, normalizeSensitiveParams, serializeRequestUrl, } from "./request-options.js";
|
|
9
10
|
const DEFAULT_PROFILE = "chrome-146";
|
|
10
11
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
11
12
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
@@ -14,6 +15,14 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN = /\bproxy\b.*\b(non[\s-]?200|connect|t
|
|
|
14
15
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
15
16
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
16
17
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE];
|
|
18
|
+
function sensitiveQueryParamNames(url) {
|
|
19
|
+
const queryStart = url.indexOf("?");
|
|
20
|
+
if (queryStart === -1)
|
|
21
|
+
return [];
|
|
22
|
+
const fragmentStart = url.indexOf("#", queryStart);
|
|
23
|
+
const query = url.slice(queryStart + 1, fragmentStart === -1 ? undefined : fragmentStart);
|
|
24
|
+
return [...new URLSearchParams(query).keys()].filter(isSensitiveKey);
|
|
25
|
+
}
|
|
17
26
|
const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
18
27
|
"chrome-120",
|
|
19
28
|
"chrome-124",
|
|
@@ -49,69 +58,128 @@ const FIREFOX_IMPIT_BY_MAJOR = {
|
|
|
49
58
|
function isRecord(value) {
|
|
50
59
|
return typeof value === "object" && value !== null;
|
|
51
60
|
}
|
|
61
|
+
const LEGACY_COOKIE_ORIGIN = "https://legacy-cookie.invalid/";
|
|
52
62
|
class CookieJarImpl {
|
|
53
63
|
cookies;
|
|
54
|
-
|
|
55
|
-
|
|
64
|
+
defaultUrl;
|
|
65
|
+
constructor(cookieStrings, defaultUrl = LEGACY_COOKIE_ORIGIN) {
|
|
66
|
+
this.cookies = new ToughCookieJar(undefined, {
|
|
67
|
+
allowSecureOnLocal: false,
|
|
68
|
+
rejectPublicSuffixes: true,
|
|
69
|
+
});
|
|
70
|
+
this.defaultUrl = this.normalizeUrl(defaultUrl) ?? LEGACY_COOKIE_ORIGIN;
|
|
56
71
|
this.setFromCookieStrings(cookieStrings);
|
|
57
72
|
}
|
|
58
|
-
|
|
73
|
+
/**
|
|
74
|
+
* URL-less legacy operations are scoped to this jar's default URL. Session
|
|
75
|
+
* jars use the client's base URL and response jars use the response URL. A
|
|
76
|
+
* flat restore has no attributes to recover, so it creates host-only Path=/
|
|
77
|
+
* cookies for that default URL instead of making them visible to every host.
|
|
78
|
+
*/
|
|
79
|
+
setFromCookieStrings(cookieStrings, url = this.defaultUrl) {
|
|
80
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
81
|
+
if (!cookieUrl)
|
|
82
|
+
return;
|
|
59
83
|
for (const cookieString of cookieStrings) {
|
|
60
|
-
|
|
61
|
-
if (!nameValue) {
|
|
62
|
-
continue;
|
|
63
|
-
}
|
|
64
|
-
const separatorIndex = nameValue.indexOf("=");
|
|
65
|
-
if (separatorIndex === -1) {
|
|
66
|
-
continue;
|
|
67
|
-
}
|
|
68
|
-
const name = nameValue.slice(0, separatorIndex).trim();
|
|
69
|
-
const value = nameValue.slice(separatorIndex + 1).trim();
|
|
70
|
-
if (name)
|
|
71
|
-
this.cookies[name] = value;
|
|
84
|
+
this.cookies.setCookieSync(cookieString, cookieUrl, { ignoreError: true });
|
|
72
85
|
}
|
|
73
86
|
}
|
|
74
|
-
get(name) {
|
|
75
|
-
return this.
|
|
87
|
+
get(name, url) {
|
|
88
|
+
return this.getAll(url)[name];
|
|
76
89
|
}
|
|
77
|
-
getAll() {
|
|
78
|
-
return
|
|
90
|
+
getAll(url) {
|
|
91
|
+
return Object.fromEntries(this.getUniqueCookies(url ?? this.defaultUrl).map((cookie) => [cookie.key, cookie.value]));
|
|
79
92
|
}
|
|
80
|
-
has(name) {
|
|
81
|
-
return Object.hasOwn(this.
|
|
93
|
+
has(name, url) {
|
|
94
|
+
return Object.hasOwn(this.getAll(url), name);
|
|
82
95
|
}
|
|
83
|
-
toString() {
|
|
84
|
-
return
|
|
85
|
-
.map((
|
|
96
|
+
toString(url) {
|
|
97
|
+
return this.getUniqueCookies(url ?? this.defaultUrl)
|
|
98
|
+
.map((cookie) => cookie.cookieString())
|
|
86
99
|
.join("; ");
|
|
87
100
|
}
|
|
88
|
-
toHeader() {
|
|
89
|
-
return this.toString();
|
|
101
|
+
toHeader(url) {
|
|
102
|
+
return this.toString(url);
|
|
90
103
|
}
|
|
91
104
|
snapshot() {
|
|
92
|
-
|
|
105
|
+
// This compatibility view deliberately enumerates the serialized store,
|
|
106
|
+
// not getAll(defaultUrl): persistence must include sibling hosts and paths.
|
|
107
|
+
// Duplicate names still collapse because a flat map cannot represent them.
|
|
108
|
+
const entries = [];
|
|
109
|
+
for (const cookie of this.serialize().jar.cookies) {
|
|
110
|
+
if (typeof cookie.key === "string" && typeof cookie.value === "string" && cookie.key) {
|
|
111
|
+
entries.push([cookie.key, cookie.value]);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return Object.fromEntries(entries);
|
|
93
115
|
}
|
|
94
116
|
restore(cookies) {
|
|
95
117
|
this.clear();
|
|
96
118
|
for (const [name, value] of Object.entries(cookies)) {
|
|
97
|
-
if (name)
|
|
98
|
-
|
|
119
|
+
if (!name)
|
|
120
|
+
continue;
|
|
121
|
+
this.cookies.setCookieSync(new Cookie({ key: name, path: "/", value }), this.defaultUrl, {
|
|
122
|
+
ignoreError: true,
|
|
123
|
+
});
|
|
99
124
|
}
|
|
100
125
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
126
|
+
serialize() {
|
|
127
|
+
const jar = this.cookies.serializeSync();
|
|
128
|
+
if (!jar) {
|
|
129
|
+
throw new SDKError("Stealth cookie store could not be serialized", {
|
|
130
|
+
code: "stealth_cookie_store_serialize_failed",
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
return { version: 1, jar };
|
|
134
|
+
}
|
|
135
|
+
deserialize(state) {
|
|
136
|
+
const version = isRecord(state) ? state.version : undefined;
|
|
137
|
+
if (version !== 1) {
|
|
138
|
+
throw new StealthCookieStoreVersionError(version);
|
|
104
139
|
}
|
|
140
|
+
// Deserialize into a new jar first so invalid state cannot partially clear
|
|
141
|
+
// or replace a live session. tough-cookie restores the cookie attributes and
|
|
142
|
+
// matching semantics represented in its own serialized format.
|
|
143
|
+
const restored = ToughCookieJar.deserializeSync(state.jar);
|
|
144
|
+
// tough-cookie 6 does not include this option in serializeSync(). Preserve
|
|
145
|
+
// the SDK's stricter setting across restoration.
|
|
146
|
+
Reflect.set(restored, "allowSecureOnLocal", false);
|
|
147
|
+
this.cookies = restored;
|
|
148
|
+
}
|
|
149
|
+
clear() {
|
|
150
|
+
this.cookies.removeAllCookiesSync();
|
|
105
151
|
}
|
|
106
|
-
find(predicate) {
|
|
107
|
-
for (const
|
|
108
|
-
const
|
|
109
|
-
if (predicate(
|
|
110
|
-
return
|
|
152
|
+
find(predicate, url) {
|
|
153
|
+
for (const cookie of this.getUniqueCookies(url ?? this.defaultUrl)) {
|
|
154
|
+
const cookieString = cookie.cookieString();
|
|
155
|
+
if (predicate(cookieString)) {
|
|
156
|
+
return cookieString;
|
|
111
157
|
}
|
|
112
158
|
}
|
|
113
159
|
return undefined;
|
|
114
160
|
}
|
|
161
|
+
normalizeUrl(url) {
|
|
162
|
+
try {
|
|
163
|
+
return new URL(url).toString();
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
getUniqueCookies(url) {
|
|
170
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
171
|
+
if (!cookieUrl)
|
|
172
|
+
return [];
|
|
173
|
+
// tough-cookie returns longer (more-specific) paths first. Keeping the
|
|
174
|
+
// first cookie for each name prevents ambiguous duplicate-name headers.
|
|
175
|
+
const names = new Set();
|
|
176
|
+
return this.cookies.getCookiesSync(cookieUrl).filter((cookie) => {
|
|
177
|
+
if (names.has(cookie.key))
|
|
178
|
+
return false;
|
|
179
|
+
names.add(cookie.key);
|
|
180
|
+
return true;
|
|
181
|
+
});
|
|
182
|
+
}
|
|
115
183
|
}
|
|
116
184
|
function closestImpitBrowser(major, candidates) {
|
|
117
185
|
let closestMajor;
|
|
@@ -175,13 +243,13 @@ function hasOwn(object, key) {
|
|
|
175
243
|
}
|
|
176
244
|
function toImpitCookieJar(cookieJar) {
|
|
177
245
|
return {
|
|
178
|
-
setCookie(cookie,
|
|
179
|
-
cookieJar.setFromCookieStrings([cookie]);
|
|
246
|
+
setCookie(cookie, url, cb) {
|
|
247
|
+
cookieJar.setFromCookieStrings([cookie], url);
|
|
180
248
|
if (typeof cb === "function")
|
|
181
249
|
cb();
|
|
182
250
|
},
|
|
183
|
-
getCookieString(
|
|
184
|
-
return cookieJar.
|
|
251
|
+
getCookieString(url) {
|
|
252
|
+
return cookieJar.toHeader(url);
|
|
185
253
|
},
|
|
186
254
|
};
|
|
187
255
|
}
|
|
@@ -233,10 +301,12 @@ function splitCombinedSetCookieHeader(headerValue) {
|
|
|
233
301
|
cookieStrings.push(finalCookie);
|
|
234
302
|
return cookieStrings;
|
|
235
303
|
}
|
|
236
|
-
export async function normalizeResponse(response, requestUrl) {
|
|
304
|
+
export async function normalizeResponse(response, requestUrl, maxBodyBytes) {
|
|
237
305
|
const headers = Object.fromEntries(response.headers.entries());
|
|
238
|
-
const cookies = new CookieJarImpl(setCookieHeadersFromResponse(response.headers));
|
|
239
|
-
const bodyBytes =
|
|
306
|
+
const cookies = new CookieJarImpl(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
307
|
+
const bodyBytes = maxBodyBytes === undefined
|
|
308
|
+
? await response.arrayBuffer()
|
|
309
|
+
: await readResponseBodyWithLimit(response, maxBodyBytes);
|
|
240
310
|
const body = new TextDecoder().decode(bodyBytes);
|
|
241
311
|
return {
|
|
242
312
|
status: response.status,
|
|
@@ -262,6 +332,75 @@ export async function normalizeResponse(response, requestUrl) {
|
|
|
262
332
|
},
|
|
263
333
|
};
|
|
264
334
|
}
|
|
335
|
+
function responseTooLargeError(maxBodyBytes, observedBytes) {
|
|
336
|
+
return new TransportError(`Response body exceeded maxBodyBytes limit of ${maxBodyBytes} bytes (observed ${observedBytes} bytes)`, {
|
|
337
|
+
code: "response_too_large",
|
|
338
|
+
category: "upstream_http",
|
|
339
|
+
retryable: false,
|
|
340
|
+
status: 0,
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
function declaredContentLength(headers) {
|
|
344
|
+
const contentLength = headers.get("content-length")?.trim();
|
|
345
|
+
if (!contentLength || !/^\d+$/.test(contentLength))
|
|
346
|
+
return undefined;
|
|
347
|
+
const parsed = Number(contentLength);
|
|
348
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
349
|
+
}
|
|
350
|
+
function abortTransportResponse(response) {
|
|
351
|
+
if (!response.abort)
|
|
352
|
+
return false;
|
|
353
|
+
try {
|
|
354
|
+
response.abort();
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
// The size error remains the primary failure if impit has already closed the response.
|
|
358
|
+
}
|
|
359
|
+
return true;
|
|
360
|
+
}
|
|
361
|
+
async function readResponseBodyWithLimit(response, maxBodyBytes) {
|
|
362
|
+
const contentLength = declaredContentLength(response.headers);
|
|
363
|
+
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
364
|
+
if (!abortTransportResponse(response)) {
|
|
365
|
+
await response.body?.cancel().catch(() => undefined);
|
|
366
|
+
}
|
|
367
|
+
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
368
|
+
}
|
|
369
|
+
if (!response.body) {
|
|
370
|
+
throw new TransportError("Response body stream is unavailable", {
|
|
371
|
+
code: "transport_stream_unavailable",
|
|
372
|
+
category: "upstream_http",
|
|
373
|
+
status: 0,
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
const reader = response.body.getReader();
|
|
377
|
+
const chunks = [];
|
|
378
|
+
let receivedBytes = 0;
|
|
379
|
+
try {
|
|
380
|
+
while (true) {
|
|
381
|
+
const { done, value } = await reader.read();
|
|
382
|
+
if (done)
|
|
383
|
+
break;
|
|
384
|
+
receivedBytes += value.byteLength;
|
|
385
|
+
if (receivedBytes > maxBodyBytes) {
|
|
386
|
+
await reader.cancel().catch(() => undefined);
|
|
387
|
+
abortTransportResponse(response);
|
|
388
|
+
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
389
|
+
}
|
|
390
|
+
chunks.push(value);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
finally {
|
|
394
|
+
reader.releaseLock();
|
|
395
|
+
}
|
|
396
|
+
const bodyBytes = new Uint8Array(receivedBytes);
|
|
397
|
+
let offset = 0;
|
|
398
|
+
for (const chunk of chunks) {
|
|
399
|
+
bodyBytes.set(chunk, offset);
|
|
400
|
+
offset += chunk.byteLength;
|
|
401
|
+
}
|
|
402
|
+
return bodyBytes.buffer;
|
|
403
|
+
}
|
|
265
404
|
function normalizeBody(body) {
|
|
266
405
|
if (body === undefined) {
|
|
267
406
|
return "";
|
|
@@ -441,7 +580,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
441
580
|
let closed = false;
|
|
442
581
|
let hasWarnedMissingProxy = false;
|
|
443
582
|
const warn = clientOptions.warn ?? console.warn;
|
|
444
|
-
const cookieJar = new CookieJarImpl([]);
|
|
583
|
+
const cookieJar = new CookieJarImpl([], baseUrl);
|
|
445
584
|
const impitCookieJar = toImpitCookieJar(cookieJar);
|
|
446
585
|
function getClient(profileName, proxyUrl, ignoreTlsErrors) {
|
|
447
586
|
if (closed) {
|
|
@@ -492,21 +631,29 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
492
631
|
}
|
|
493
632
|
const session = {
|
|
494
633
|
async fetch(url, options = {}) {
|
|
495
|
-
const method =
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
}) ??
|
|
501
|
-
(hasExplicitRetryPolicy
|
|
502
|
-
? undefined
|
|
503
|
-
: createDefaultProxyTransportRetryOptions({
|
|
634
|
+
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
635
|
+
try {
|
|
636
|
+
const method = normalizeMethod(options.method ?? "GET");
|
|
637
|
+
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
638
|
+
const stealthRetryOptions = normalizeProxyTransportRetryOptions(options.retry, {
|
|
504
639
|
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
505
640
|
label: "Stealth",
|
|
506
|
-
})
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
641
|
+
}) ??
|
|
642
|
+
(hasExplicitRetryPolicy
|
|
643
|
+
? undefined
|
|
644
|
+
: createDefaultProxyTransportRetryOptions({
|
|
645
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
646
|
+
label: "Stealth",
|
|
647
|
+
}));
|
|
648
|
+
if (stealthRetryOptions) {
|
|
649
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
650
|
+
}
|
|
651
|
+
return { hasExplicitRetryPolicy, method, stealthRetryOptions };
|
|
652
|
+
}
|
|
653
|
+
catch (error) {
|
|
654
|
+
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
655
|
+
}
|
|
656
|
+
})();
|
|
510
657
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
511
658
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
512
659
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
@@ -540,6 +687,11 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
540
687
|
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
541
688
|
let proxy;
|
|
542
689
|
let attemptProxy;
|
|
690
|
+
// Reuse the exact serialization used by this outbound attempt in its catch path.
|
|
691
|
+
let serializedUrl;
|
|
692
|
+
let fallbackSensitiveValues = [];
|
|
693
|
+
let fallbackRequestUrl;
|
|
694
|
+
let fallbackRedactedUrl;
|
|
543
695
|
const attemptStartedAt = Date.now();
|
|
544
696
|
let attemptRecorded = false;
|
|
545
697
|
const recordProxyAttempt = (outcome, errorCode, status) => {
|
|
@@ -560,6 +712,16 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
560
712
|
});
|
|
561
713
|
};
|
|
562
714
|
try {
|
|
715
|
+
const sensitiveParams = normalizeSensitiveParams(options.sensitiveParams);
|
|
716
|
+
const structural = redactUrlQueryParams(url, Object.keys(sensitiveParams ?? {}));
|
|
717
|
+
fallbackSensitiveValues = [
|
|
718
|
+
...new Set([
|
|
719
|
+
...Object.values(sensitiveParams ?? {}).map(String),
|
|
720
|
+
...structural.sensitiveValues,
|
|
721
|
+
]),
|
|
722
|
+
].filter((value) => value !== "");
|
|
723
|
+
fallbackRequestUrl = url;
|
|
724
|
+
fallbackRedactedUrl = structural.redactedUrl;
|
|
563
725
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
564
726
|
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
565
727
|
proxy = attemptProxy.url;
|
|
@@ -577,10 +739,11 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
577
739
|
const ignoreTlsErrors = Boolean(options.stealth?.insecureSkipVerify ??
|
|
578
740
|
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify));
|
|
579
741
|
const profileName = options.profile ?? defaultProfile;
|
|
580
|
-
|
|
742
|
+
serializedUrl = serializeRequestUrl(resolveUrl(baseUrl, url), options.params, sensitiveParams);
|
|
743
|
+
const { requestUrl } = serializedUrl;
|
|
581
744
|
const headers = { ...(options.headers ?? {}) };
|
|
582
745
|
if (!hasHeader(headers, "Cookie")) {
|
|
583
|
-
const cookieHeader = cookieJar.
|
|
746
|
+
const cookieHeader = cookieJar.toHeader(requestUrl);
|
|
584
747
|
if (cookieHeader)
|
|
585
748
|
headers.Cookie = cookieHeader;
|
|
586
749
|
}
|
|
@@ -594,8 +757,8 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
594
757
|
requestInit.body = normalizeBody(options.body);
|
|
595
758
|
}
|
|
596
759
|
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(requestUrl, requestInit);
|
|
597
|
-
const normalized = await normalizeResponse(response, requestUrl);
|
|
598
|
-
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
760
|
+
const normalized = await normalizeResponse(response, requestUrl, options.maxBodyBytes);
|
|
761
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
599
762
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
600
763
|
throw createProxyConnectFailureError(normalized.body);
|
|
601
764
|
}
|
|
@@ -627,12 +790,23 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
627
790
|
return normalized;
|
|
628
791
|
}
|
|
629
792
|
catch (error) {
|
|
630
|
-
const
|
|
793
|
+
const sensitiveValues = serializedUrl?.sensitiveValues ?? fallbackSensitiveValues;
|
|
794
|
+
let normalizedError;
|
|
795
|
+
try {
|
|
796
|
+
normalizedError = normalizeStealthTransportError(error);
|
|
797
|
+
}
|
|
798
|
+
catch (normalizationError) {
|
|
799
|
+
throw redactSensitiveError(normalizationError, sensitiveValues, serializedUrl?.requestUrl ?? fallbackRequestUrl, serializedUrl?.redactedUrl ?? fallbackRedactedUrl);
|
|
800
|
+
}
|
|
801
|
+
const retryErrorCode = proxyAttemptErrorCode(normalizedError);
|
|
802
|
+
const refreshableProxyError = isProxyPoolRefreshableError(normalizedError);
|
|
803
|
+
const runProxyAuthDiagnostic = shouldRunProxyAuthDiagnostic(normalizedError);
|
|
804
|
+
normalizedError = redactSensitiveError(normalizedError, sensitiveValues, serializedUrl?.requestUrl ?? fallbackRequestUrl, serializedUrl?.redactedUrl ?? fallbackRedactedUrl);
|
|
631
805
|
recordProxyAttempt("error", proxyAttemptErrorCode(normalizedError), proxyAttemptStatus(normalizedError));
|
|
632
806
|
lastError = normalizedError;
|
|
633
|
-
if (proxy && rotatesRegistryChain &&
|
|
807
|
+
if (proxy && rotatesRegistryChain && refreshableProxyError) {
|
|
634
808
|
stalePoolError = normalizedError;
|
|
635
|
-
if (
|
|
809
|
+
if (runProxyAuthDiagnostic) {
|
|
636
810
|
stalePoolDiagnosticProxy = proxy;
|
|
637
811
|
}
|
|
638
812
|
if (attempt + 1 < maxAttempts) {
|
|
@@ -662,7 +836,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
662
836
|
});
|
|
663
837
|
if (attempt + 1 < transportRetryCap &&
|
|
664
838
|
shouldRetryProxyTransportAttempt({
|
|
665
|
-
error:
|
|
839
|
+
error: { code: retryErrorCode },
|
|
666
840
|
explicitRetry: hasExplicitRetryPolicy,
|
|
667
841
|
method,
|
|
668
842
|
options: stealthRetryOptions,
|
|
@@ -712,49 +886,137 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
712
886
|
const maxHops = options.maxHops === undefined || !Number.isFinite(options.maxHops)
|
|
713
887
|
? 10
|
|
714
888
|
: Math.max(0, Math.floor(options.maxHops));
|
|
889
|
+
const { url: _url, maxHops: _maxHops, stopWhen, params, sensitiveParams, ...fetchOptions } = options;
|
|
715
890
|
const hops = [];
|
|
716
|
-
let currentUrl = resolveUrl(baseUrl, options.url);
|
|
717
891
|
let method = normalizeMethod(options.method ?? "GET");
|
|
718
892
|
let body = options.body;
|
|
719
893
|
let response;
|
|
720
894
|
const visitedRequests = new Set();
|
|
721
|
-
const
|
|
895
|
+
const initialParams = params
|
|
896
|
+
? Object.fromEntries(Object.entries(params).map(([key, value]) => [
|
|
897
|
+
key,
|
|
898
|
+
Array.isArray(value) ? [...value] : value,
|
|
899
|
+
]))
|
|
900
|
+
: undefined;
|
|
901
|
+
const normalizedSensitiveParams = normalizeSensitiveParams(sensitiveParams);
|
|
902
|
+
const initialSensitiveParams = normalizedSensitiveParams
|
|
903
|
+
? { ...normalizedSensitiveParams }
|
|
904
|
+
: undefined;
|
|
905
|
+
const sensitiveParamNames = initialSensitiveParams
|
|
906
|
+
? Object.keys(initialSensitiveParams)
|
|
907
|
+
: [];
|
|
908
|
+
const callerStructural = redactUrlQueryParams(options.url, sensitiveParamNames);
|
|
909
|
+
const sensitiveValues = new Set([
|
|
910
|
+
...Object.values(initialSensitiveParams ?? {}),
|
|
911
|
+
...callerStructural.sensitiveValues,
|
|
912
|
+
].filter((value) => value !== ""));
|
|
913
|
+
const redactRedirectUrl = (value) => {
|
|
914
|
+
const structural = redactUrlQueryParams(value, [
|
|
915
|
+
...new Set([...sensitiveParamNames, ...sensitiveQueryParamNames(value)]),
|
|
916
|
+
]);
|
|
917
|
+
for (const sensitiveValue of structural.sensitiveValues) {
|
|
918
|
+
sensitiveValues.add(sensitiveValue);
|
|
919
|
+
}
|
|
920
|
+
return redactSensitiveText(structural.redactedUrl, [...sensitiveValues]);
|
|
921
|
+
};
|
|
922
|
+
let currentUrl;
|
|
923
|
+
let initialUrl;
|
|
924
|
+
try {
|
|
925
|
+
currentUrl = resolveUrl(baseUrl, options.url);
|
|
926
|
+
redactRedirectUrl(currentUrl);
|
|
927
|
+
initialUrl = serializeRequestUrl(currentUrl, initialParams, initialSensitiveParams);
|
|
928
|
+
for (const value of initialUrl.sensitiveValues) {
|
|
929
|
+
if (value !== "")
|
|
930
|
+
sensitiveValues.add(value);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
catch (error) {
|
|
934
|
+
throw redactSensitiveError(error, [...sensitiveValues], options.url, redactRedirectUrl(options.url));
|
|
935
|
+
}
|
|
722
936
|
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
937
|
+
const outboundUrl = hopIndex === 0 ? initialUrl.requestUrl : serializeRequestUrl(currentUrl).requestUrl;
|
|
938
|
+
// Preserve params-only loop bookkeeping from before sensitiveParams:
|
|
939
|
+
// the first visited key is the caller's resolved URL, not its expanded query.
|
|
940
|
+
const visitedUrl = hopIndex === 0 && !initialSensitiveParams ? currentUrl : outboundUrl;
|
|
941
|
+
visitedRequests.add(`${method} ${visitedUrl}`);
|
|
942
|
+
try {
|
|
943
|
+
response = await session.fetch(currentUrl, {
|
|
944
|
+
...fetchOptions,
|
|
945
|
+
body,
|
|
946
|
+
method,
|
|
947
|
+
...(hopIndex === 0 && initialParams ? { params: initialParams } : {}),
|
|
948
|
+
...(hopIndex === 0 && initialSensitiveParams
|
|
949
|
+
? { sensitiveParams: initialSensitiveParams }
|
|
950
|
+
: {}),
|
|
951
|
+
redirect: "manual",
|
|
952
|
+
throwOnHttpError: false,
|
|
953
|
+
});
|
|
954
|
+
}
|
|
955
|
+
catch (error) {
|
|
956
|
+
throw redactSensitiveError(error, [...sensitiveValues], outboundUrl, redactRedirectUrl(outboundUrl));
|
|
957
|
+
}
|
|
958
|
+
// StealthResponse.url is programmatic metadata and remains raw. Only the
|
|
959
|
+
// redirect hop emitted below is a diagnostic surface.
|
|
960
|
+
const responseUrl = response.url ?? (hopIndex === 0 && initialSensitiveParams ? outboundUrl : currentUrl);
|
|
732
961
|
if (!isRedirectStatus(response.status)) {
|
|
733
962
|
return {
|
|
734
963
|
final: response,
|
|
735
964
|
hops,
|
|
736
965
|
reason: "completed",
|
|
737
966
|
cookies: cookieJar.snapshot(),
|
|
967
|
+
cookieStore: cookieJar.serialize(),
|
|
738
968
|
};
|
|
739
969
|
}
|
|
740
970
|
const location = locationHeader(response.headers);
|
|
741
|
-
const
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
971
|
+
const redactedResponseUrl = redactRedirectUrl(responseUrl);
|
|
972
|
+
const redactedLocation = location ? redactRedirectUrl(location) : undefined;
|
|
973
|
+
let nextUrl;
|
|
974
|
+
try {
|
|
975
|
+
nextUrl = location ? new URL(location, responseUrl).toString() : undefined;
|
|
976
|
+
}
|
|
977
|
+
catch (error) {
|
|
978
|
+
throw redactSensitiveError(error, [...sensitiveValues], location, redactedLocation);
|
|
979
|
+
}
|
|
980
|
+
const realHop = {
|
|
981
|
+
url: responseUrl,
|
|
746
982
|
status: response.status,
|
|
747
983
|
method,
|
|
748
984
|
...(location ? { location } : {}),
|
|
749
985
|
...(nextUrl ? { nextUrl } : {}),
|
|
750
986
|
};
|
|
987
|
+
const hop = {
|
|
988
|
+
...realHop,
|
|
989
|
+
url: redactedResponseUrl,
|
|
990
|
+
...(redactedLocation ? { location: redactedLocation } : {}),
|
|
991
|
+
...(nextUrl ? { nextUrl: redactRedirectUrl(nextUrl) } : {}),
|
|
992
|
+
};
|
|
751
993
|
hops.push(hop);
|
|
752
|
-
|
|
994
|
+
let shouldStop = false;
|
|
995
|
+
if (stopWhen) {
|
|
996
|
+
try {
|
|
997
|
+
shouldStop = await stopWhen(realHop);
|
|
998
|
+
}
|
|
999
|
+
catch (error) {
|
|
1000
|
+
let sanitizedError = error;
|
|
1001
|
+
for (const [rawUrl, safeUrl] of [
|
|
1002
|
+
[responseUrl, redactedResponseUrl],
|
|
1003
|
+
[location, redactedLocation],
|
|
1004
|
+
[nextUrl, nextUrl ? redactRedirectUrl(nextUrl) : undefined],
|
|
1005
|
+
]) {
|
|
1006
|
+
if (!rawUrl || !safeUrl)
|
|
1007
|
+
continue;
|
|
1008
|
+
sanitizedError = redactSensitiveError(sanitizedError, [...sensitiveValues], rawUrl, safeUrl);
|
|
1009
|
+
}
|
|
1010
|
+
throw sanitizedError;
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
if (shouldStop) {
|
|
753
1014
|
return {
|
|
754
1015
|
final: response,
|
|
755
1016
|
hops,
|
|
756
1017
|
reason: "stopped",
|
|
757
1018
|
cookies: cookieJar.snapshot(),
|
|
1019
|
+
cookieStore: cookieJar.serialize(),
|
|
758
1020
|
};
|
|
759
1021
|
}
|
|
760
1022
|
if (!nextUrl) {
|
|
@@ -763,6 +1025,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
763
1025
|
hops,
|
|
764
1026
|
reason: "missing_location",
|
|
765
1027
|
cookies: cookieJar.snapshot(),
|
|
1028
|
+
cookieStore: cookieJar.serialize(),
|
|
766
1029
|
};
|
|
767
1030
|
}
|
|
768
1031
|
if (hops.length > maxHops) {
|
|
@@ -771,6 +1034,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
771
1034
|
hops,
|
|
772
1035
|
reason: "max_hops",
|
|
773
1036
|
cookies: cookieJar.snapshot(),
|
|
1037
|
+
cookieStore: cookieJar.serialize(),
|
|
774
1038
|
};
|
|
775
1039
|
}
|
|
776
1040
|
const nextMethod = nextRedirectMethod(response.status, method);
|
|
@@ -783,6 +1047,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
783
1047
|
hops,
|
|
784
1048
|
reason: "loop",
|
|
785
1049
|
cookies: cookieJar.snapshot(),
|
|
1050
|
+
cookieStore: cookieJar.serialize(),
|
|
786
1051
|
};
|
|
787
1052
|
}
|
|
788
1053
|
method = nextMethod;
|
|
@@ -803,6 +1068,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
803
1068
|
hops,
|
|
804
1069
|
reason: "max_hops",
|
|
805
1070
|
cookies: cookieJar.snapshot(),
|
|
1071
|
+
cookieStore: cookieJar.serialize(),
|
|
806
1072
|
};
|
|
807
1073
|
},
|
|
808
1074
|
},
|
package/dist/server/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { createServerApp, type ProviderServerCloseOptions, type ProviderServerHandle, type ProviderServerLogEvent, type ProviderServerLogger, type ProviderServerOperationExecutor, type ProviderServerOperationExecutorInput, type ProviderServerOptions, type ProviderServerStatefulForwardEnvelope, type ServeOptions, serve, } from "./serve.js";
|
|
2
2
|
export { computeSelfTestPlanDigest, createSelfTestApp, createSelfTestAuthFlowInvoke, createSelfTestInvoke, DEFAULT_SELF_TEST_REQUEST_BUDGET_MS, isSelfTestReadOnlyOperation, PROVIDER_RUNTIME_SELF_TEST_REQUEST_BUDGET_MS_ENV, resolveSelfTestPort, SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON, SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON, SELF_TEST_HEALTHZ_PATH, SELF_TEST_PATH, SELF_TEST_SCHEMA_VERSION, type SelfTestAppOptions, type SelfTestAuthFlowInvoke, type SelfTestAuthFlowRoute, type SelfTestCaseResult, type SelfTestCaseStatus, type SelfTestOperationInvoke, type SelfTestRequest, SelfTestRequestSchema, type SelfTestResponse, } from "./self-test.js";
|
|
3
|
-
export { resolveHealthCheckInputDateTokens } from "./self-test-input-tokens.js";
|
|
3
|
+
export { type InputDateTokenCalendar, resolveHealthCheckInputDateTokens, } from "./self-test-input-tokens.js";
|
|
4
4
|
export { collectSelfTestSensitiveValues, redactSelfTestText, SELF_TEST_MAX_TEXT_LENGTH, SELF_TEST_REDACTED_PLACEHOLDER, } from "./self-test-redaction.js";
|
|
5
5
|
export { DEFAULT_SELF_TEST_PORT, deriveSelfTestToken, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV, PROVIDER_RUNTIME_SELF_TEST_PORT_ENV, resolveSelfTestMasterSecrets, type SelfTestMasterSecrets, verifySelfTestAuthorization, } from "./self-test-token.js";
|
|
6
6
|
export type { AuthFlowRequest, AuthFlowResponse, AuthFlowSuccessResponse, ConnectionMode, OperationConnection, OperationErrorResponse, OperationRequest, OperationResponse, OperationSuccessResponse, } from "./types.js";
|
package/dist/server/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { createServerApp, serve, } from "./serve.js";
|
|
2
2
|
export { computeSelfTestPlanDigest, createSelfTestApp, createSelfTestAuthFlowInvoke, createSelfTestInvoke, DEFAULT_SELF_TEST_REQUEST_BUDGET_MS, isSelfTestReadOnlyOperation, PROVIDER_RUNTIME_SELF_TEST_REQUEST_BUDGET_MS_ENV, resolveSelfTestPort, SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON, SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON, SELF_TEST_HEALTHZ_PATH, SELF_TEST_PATH, SELF_TEST_SCHEMA_VERSION, SelfTestRequestSchema, } from "./self-test.js";
|
|
3
|
-
export { resolveHealthCheckInputDateTokens } from "./self-test-input-tokens.js";
|
|
3
|
+
export { resolveHealthCheckInputDateTokens, } from "./self-test-input-tokens.js";
|
|
4
4
|
export { collectSelfTestSensitiveValues, redactSelfTestText, SELF_TEST_MAX_TEXT_LENGTH, SELF_TEST_REDACTED_PLACEHOLDER, } from "./self-test-redaction.js";
|
|
5
5
|
export { DEFAULT_SELF_TEST_PORT, deriveSelfTestToken, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV, PROVIDER_RUNTIME_SELF_TEST_PORT_ENV, resolveSelfTestMasterSecrets, verifySelfTestAuthorization, } from "./self-test-token.js";
|
|
6
6
|
export { AuthFlowRequestSchema, AuthFlowSuccessResponseSchema, ConnectionModeSchema, ErrorEnvelopeSchema, OperationConnectionSchema, OperationErrorResponseSchema, OperationRequestSchema, OperationSuccessResponseSchema, } from "./types.js";
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
export
|
|
1
|
+
export type InputDateTokenCalendar = "KST" | "UTC";
|
|
2
|
+
export declare function resolveHealthCheckInputDateTokens(value: unknown, now?: Date, calendar?: InputDateTokenCalendar): unknown;
|