@apifuse/provider-sdk 2.2.0-beta.68 → 2.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +6 -1
- package/CHANGELOG.md +4 -0
- package/README.md +10 -3
- package/dist/fixture-sanitization.js +159 -15
- package/dist/lint.js +56 -9
- package/dist/runtime/diagnostic-redactor.d.ts +6 -1
- package/dist/runtime/diagnostic-redactor.js +16 -14
- package/dist/runtime/http.js +15 -102
- package/dist/runtime/instrumentation.js +4 -1
- package/dist/runtime/redirect-hop-headers.d.ts +19 -0
- package/dist/runtime/redirect-hop-headers.js +58 -0
- package/dist/runtime/redirects.d.ts +33 -1
- package/dist/runtime/redirects.js +113 -0
- package/dist/runtime/resolver-span-attributes.d.ts +34 -0
- package/dist/runtime/resolver-span-attributes.js +93 -0
- package/dist/runtime/resolver-usage.d.ts +8 -0
- package/dist/runtime/resolver-usage.js +5 -11
- package/dist/runtime/resolver-vendors/browser.js +5 -4
- package/dist/runtime/resolver-vendors/capsolver.js +11 -10
- package/dist/runtime/resolver-vendors/twocaptcha.js +9 -8
- package/dist/runtime/resolver.js +7 -6
- package/dist/runtime/stealth-owned-headers.d.ts +21 -8
- package/dist/runtime/stealth-owned-headers.js +28 -8
- package/dist/runtime/stealth-telemetry.d.ts +1 -1
- package/dist/runtime/stealth.js +158 -46
- package/dist/runtime/trace.js +2 -2
- package/dist/server/serve-implementation.js +2 -1
- package/dist/testing/run.js +14 -7
- package/dist/trace-sanitization.js +1 -1
- package/dist/types.d.ts +30 -6
- package/package.json +2 -2
- package/src/fixture-sanitization.ts +175 -17
- package/src/lint.ts +54 -9
- package/src/runtime/diagnostic-redactor.ts +19 -13
- package/src/runtime/http.ts +22 -111
- package/src/runtime/instrumentation.ts +4 -1
- package/src/runtime/redirect-hop-headers.ts +71 -0
- package/src/runtime/redirects.ts +150 -1
- package/src/runtime/resolver-span-attributes.ts +139 -0
- package/src/runtime/resolver-usage.ts +8 -5
- package/src/runtime/resolver-vendors/browser.ts +5 -4
- package/src/runtime/resolver-vendors/capsolver.ts +11 -10
- package/src/runtime/resolver-vendors/twocaptcha.ts +9 -8
- package/src/runtime/resolver.ts +7 -6
- package/src/runtime/stealth-owned-headers.ts +34 -8
- package/src/runtime/stealth-telemetry.ts +6 -1
- package/src/runtime/stealth.ts +180 -58
- package/src/runtime/trace.ts +6 -2
- package/src/server/serve-implementation.ts +2 -1
- package/src/testing/run.ts +15 -2
- package/src/trace-sanitization.ts +1 -1
- package/src/types.ts +30 -6
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
|
|
1
3
|
import type { JsonValue } from "./contract-json.js";
|
|
2
4
|
|
|
3
5
|
export const REDACTED_FIXTURE_VALUE = "[REDACTED]";
|
|
@@ -13,6 +15,35 @@ const DIAGNOSTIC_URL_SENTINEL_RUN = new RegExp(
|
|
|
13
15
|
);
|
|
14
16
|
const PEM_PRIVATE_KEY =
|
|
15
17
|
/-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9 ]+ )?PRIVATE KEY-----/g;
|
|
18
|
+
/** RFC 9110 §11.4 `token68`, the single credential token of Bearer, Basic, Negotiate and NTLM. */
|
|
19
|
+
const TOKEN68 = "[A-Za-z0-9._~+/-]+=*";
|
|
20
|
+
/**
|
|
21
|
+
* A credential header's value is the whole RFC 9110 `credentials` production
|
|
22
|
+
* (`auth-scheme SP token68`), so the scheme and its token are one value, not two words.
|
|
23
|
+
* The key starts at a word boundary so a long word run is scanned once, not from every offset.
|
|
24
|
+
*/
|
|
25
|
+
const CREDENTIALS_HEADER_ASSIGNMENT = new RegExp(
|
|
26
|
+
String.raw`(?<![\w-])((["']?)[\w-]*authorization\2\s*[:=]\s*)[A-Za-z][\w.+-]*[ \t]+${TOKEN68}(?![^\s,;&])`,
|
|
27
|
+
"gi",
|
|
28
|
+
);
|
|
29
|
+
/**
|
|
30
|
+
* Keyless echoes of the token68 schemes whose token is always a credential. Only the scheme and
|
|
31
|
+
* its whitespace are matched, so matches never overlap a later scheme; the token is the whole
|
|
32
|
+
* opaque run that follows, so no partial redaction leaves a remainder too short for the
|
|
33
|
+
* opaque-token scan. Negotiate and NTLM match only their registered spelling so prose such as
|
|
34
|
+
* "failed to negotiate TLS" is retained.
|
|
35
|
+
*/
|
|
36
|
+
const TOKEN68_SCHEMES = [/\bBearer\s+/gi, /\b(?:Negotiate|NTLM)\s+/g] as const;
|
|
37
|
+
/** A keyless `Basic` token is a credential only when it decodes to `user-id:password`. */
|
|
38
|
+
const BASIC_SCHEME = /\bBasic\s+/gi;
|
|
39
|
+
const OPAQUE_TOKEN_CHARACTER = /[A-Za-z0-9_+/=.:~-]/;
|
|
40
|
+
const BASE64_CHARACTER = /[A-Za-z0-9+/]/;
|
|
41
|
+
const ASSIGNMENT_VALUE_DELIMITER = /[\s,;&]/;
|
|
42
|
+
/**
|
|
43
|
+
* A key and its `:`/`=`, starting at a word boundary so a word run is scanned once. Values are
|
|
44
|
+
* never consumed, so an assignment inside another value is still found.
|
|
45
|
+
*/
|
|
46
|
+
const ASSIGNMENT_KEY = /(["']?)(?<![\w-])([\w-]+)\1\s*[:=]\s*/gi;
|
|
16
47
|
|
|
17
48
|
/** Matches credential field names without treating benign prefixes such as `author` as `auth`. */
|
|
18
49
|
export function isSensitiveFixtureKey(key: string): boolean {
|
|
@@ -23,7 +54,8 @@ export function isSensitiveFixtureKey(key: string): boolean {
|
|
|
23
54
|
/^(?:authorization|authentication|auth|bearer|cookie|credential|password|passwd|privatekey|secret|session|sessionid|token)$/.test(
|
|
24
55
|
candidate,
|
|
25
56
|
) ||
|
|
26
|
-
|
|
57
|
+
// Unanchored at the start so vendor-prefixed headers (`x-api-key`) match too.
|
|
58
|
+
/(?:api|client|service|access|consumer)(?:key|secret|token)$/.test(candidate) ||
|
|
27
59
|
/(?:authorization|credential|password|passwd|privatekey|secret|sessionid|token)$/.test(
|
|
28
60
|
candidate,
|
|
29
61
|
),
|
|
@@ -71,7 +103,7 @@ export function sanitizeFixtureString(value: string): string {
|
|
|
71
103
|
retainedUrls.push(isCredentialBearingUrl(url) ? sanitizeUrlForLogs(url) : url) - 1;
|
|
72
104
|
return `APIFUSEURL${index}X`;
|
|
73
105
|
});
|
|
74
|
-
sanitized =
|
|
106
|
+
sanitized = redactCredentialSpans(sanitized, [credentialHeaderSpans, sensitiveAssignmentSpans]);
|
|
75
107
|
sanitized = sanitized.replace(OPAQUE_TOKEN_RUN, (candidate) =>
|
|
76
108
|
isSensitiveFixtureValue(candidate) ? REDACTED_FIXTURE_VALUE : candidate,
|
|
77
109
|
);
|
|
@@ -162,9 +194,12 @@ export function sanitizeDiagnosticText(value: string): string {
|
|
|
162
194
|
const index = retainedUrls.push(sanitizeUrlForLogs(url)) - 1;
|
|
163
195
|
return `${DIAGNOSTIC_URL_SENTINEL_DELIMITER}APIFUSE_URL${index}${DIAGNOSTIC_URL_SENTINEL_DELIMITER}`;
|
|
164
196
|
})
|
|
165
|
-
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, `Bearer ${REDACTED_FIXTURE_VALUE}`)
|
|
166
197
|
.replace(EMAIL_ADDRESS_RUN, REDACTED_FIXTURE_VALUE);
|
|
167
|
-
sanitized =
|
|
198
|
+
sanitized = redactCredentialSpans(sanitized, [
|
|
199
|
+
credentialHeaderSpans,
|
|
200
|
+
keylessCredentialSpans,
|
|
201
|
+
sensitiveAssignmentSpans,
|
|
202
|
+
]);
|
|
168
203
|
sanitized = sanitized.replace(OPAQUE_TOKEN_RUN, (candidate, offset: number, source: string) => {
|
|
169
204
|
if (/^(?:request|trace|correlation)[-_]?id[:=]/i.test(candidate)) return candidate;
|
|
170
205
|
const prefix = source.slice(Math.max(0, offset - 32), offset);
|
|
@@ -178,19 +213,142 @@ export function sanitizeDiagnosticText(value: string): string {
|
|
|
178
213
|
return encodeDiagnosticControls(sanitized);
|
|
179
214
|
}
|
|
180
215
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
216
|
+
type CredentialSpan = readonly [start: number, end: number];
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Every credential rule scans the same unmodified text and reports spans; the output redacts
|
|
220
|
+
* their union. No rule's rewrite can hide a key or a scheme word from another rule, so adding a
|
|
221
|
+
* rule can only redact more.
|
|
222
|
+
*/
|
|
223
|
+
function redactCredentialSpans(
|
|
224
|
+
value: string,
|
|
225
|
+
rules: readonly ((value: string) => Iterable<CredentialSpan>)[],
|
|
226
|
+
): string {
|
|
227
|
+
const spans = rules
|
|
228
|
+
.flatMap((rule) => [...rule(value)])
|
|
229
|
+
.sort((a, b) => a[0] - b[0] || b[1] - a[1]);
|
|
230
|
+
let output = "";
|
|
231
|
+
let cursor = 0;
|
|
232
|
+
let open: [number, number] | undefined;
|
|
233
|
+
const close = (span: [number, number]) => {
|
|
234
|
+
output += value.slice(cursor, span[0]) + REDACTED_FIXTURE_VALUE;
|
|
235
|
+
cursor = span[1];
|
|
236
|
+
};
|
|
237
|
+
for (const [start, end] of spans) {
|
|
238
|
+
if (open && start <= open[1]) {
|
|
239
|
+
open[1] = Math.max(open[1], end);
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
if (open) close(open);
|
|
243
|
+
open = [start, end];
|
|
244
|
+
}
|
|
245
|
+
if (open) close(open);
|
|
246
|
+
return output + value.slice(cursor);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** The scheme and token of a credential header value, as one span. */
|
|
250
|
+
function* credentialHeaderSpans(value: string): Iterable<CredentialSpan> {
|
|
251
|
+
for (const match of value.matchAll(CREDENTIALS_HEADER_ASSIGNMENT)) {
|
|
252
|
+
yield [match.index + (match[1] as string).length, match.index + match[0].length];
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Tokens after keyless `Bearer`/`Negotiate`/`NTLM`, and `Basic` tokens that decode to `user-id:password`. */
|
|
257
|
+
function* keylessCredentialSpans(value: string): Iterable<CredentialSpan> {
|
|
258
|
+
const opaqueEnds = runEnds(value, OPAQUE_TOKEN_CHARACTER);
|
|
259
|
+
for (const scheme of TOKEN68_SCHEMES) {
|
|
260
|
+
for (const match of value.matchAll(scheme)) {
|
|
261
|
+
const start = match.index + match[0].length;
|
|
262
|
+
if (opaqueEnds[start] > start) yield [start, opaqueEnds[start]];
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
const base64Ends = runEnds(value, BASE64_CHARACTER);
|
|
266
|
+
for (const match of value.matchAll(BASIC_SCHEME)) {
|
|
267
|
+
const start = match.index + match[0].length;
|
|
268
|
+
const payloadEnd = base64Ends[start];
|
|
269
|
+
let end = payloadEnd;
|
|
270
|
+
while (end < payloadEnd + 2 && value[end] === "=") end++;
|
|
271
|
+
// The base64 token must be the whole opaque run, not the prefix of a longer token.
|
|
272
|
+
if (payloadEnd === start || end !== opaqueEnds[start]) continue;
|
|
273
|
+
if (Buffer.from(value.slice(start, end), "base64").includes(0x3a)) yield [start, end];
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Values of credential-named keys; quotes around a quoted value are kept. Every key is visited,
|
|
279
|
+
* including keys inside another key's value (`auth: {"password": "x"}`), and each value's extent
|
|
280
|
+
* is read from precomputed tables, so the scan stays linear in the text length.
|
|
281
|
+
*/
|
|
282
|
+
function* sensitiveAssignmentSpans(value: string): Iterable<CredentialSpan> {
|
|
283
|
+
let extents: AssignmentValueExtents | undefined;
|
|
284
|
+
for (const key of value.matchAll(ASSIGNMENT_KEY)) {
|
|
285
|
+
const name = key[2] as string;
|
|
286
|
+
if (!isSensitiveFixtureKey(name) && name.toLowerCase() !== "key") continue;
|
|
287
|
+
extents ??= assignmentValueExtents(value);
|
|
288
|
+
const start = key.index + key[0].length;
|
|
289
|
+
const quote = value[start];
|
|
290
|
+
const closing =
|
|
291
|
+
quote === '"'
|
|
292
|
+
? extents.doubleQuoteClose[start + 1]
|
|
293
|
+
: quote === "'"
|
|
294
|
+
? extents.singleQuoteClose[start + 1]
|
|
295
|
+
: -1;
|
|
296
|
+
if (closing !== -1) {
|
|
297
|
+
yield [start + 1, closing];
|
|
298
|
+
} else if (extents.unquotedEnds[start] > start) {
|
|
299
|
+
yield [start, extents.unquotedEnds[start]];
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
type AssignmentValueExtents = {
|
|
305
|
+
/** End of the unquoted value (`[^\s,;&]+`) starting at each index. */
|
|
306
|
+
readonly unquotedEnds: Int32Array;
|
|
307
|
+
/** Index of the closing `"` when a `"(?:\\.|[^"\\])*"` body is read from each index, or -1. */
|
|
308
|
+
readonly doubleQuoteClose: Int32Array;
|
|
309
|
+
readonly singleQuoteClose: Int32Array;
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
function assignmentValueExtents(value: string): AssignmentValueExtents {
|
|
313
|
+
const unquotedEnds = new Int32Array(value.length + 1);
|
|
314
|
+
unquotedEnds[value.length] = value.length;
|
|
315
|
+
for (let index = value.length - 1; index >= 0; index--) {
|
|
316
|
+
unquotedEnds[index] = ASSIGNMENT_VALUE_DELIMITER.test(value[index] as string)
|
|
317
|
+
? index
|
|
318
|
+
: unquotedEnds[index + 1];
|
|
319
|
+
}
|
|
320
|
+
return {
|
|
321
|
+
unquotedEnds,
|
|
322
|
+
doubleQuoteClose: closingQuotes(value, '"'),
|
|
323
|
+
singleQuoteClose: closingQuotes(value, "'"),
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** Right-to-left: reading a quoted body from index i either closes at a quote or skips an escape pair. */
|
|
328
|
+
function closingQuotes(value: string, quote: string): Int32Array {
|
|
329
|
+
const closing = new Int32Array(value.length + 2).fill(-1);
|
|
330
|
+
for (let index = value.length - 1; index >= 0; index--) {
|
|
331
|
+
const character = value[index];
|
|
332
|
+
closing[index] =
|
|
333
|
+
character === quote
|
|
334
|
+
? index
|
|
335
|
+
: character === "\\"
|
|
336
|
+
? /[^\n\r\u2028\u2029]/.test(value[index + 1] ?? "\n")
|
|
337
|
+
? (closing[index + 2] as number)
|
|
338
|
+
: -1
|
|
339
|
+
: (closing[index + 1] as number);
|
|
340
|
+
}
|
|
341
|
+
return closing;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** For each index, the end of the run of `character` starting there, in one right-to-left pass. */
|
|
345
|
+
function runEnds(value: string, character: RegExp): Int32Array {
|
|
346
|
+
const ends = new Int32Array(value.length + 1);
|
|
347
|
+
ends[value.length] = value.length;
|
|
348
|
+
for (let index = value.length - 1; index >= 0; index--) {
|
|
349
|
+
ends[index] = character.test(value[index] as string) ? (ends[index + 1] as number) : index;
|
|
350
|
+
}
|
|
351
|
+
return ends;
|
|
194
352
|
}
|
|
195
353
|
|
|
196
354
|
function isCredentialBearingUrl(value: string): boolean {
|
package/src/lint.ts
CHANGED
|
@@ -1118,6 +1118,47 @@ function isStealthContextAccess(node: import("typescript").Node): boolean {
|
|
|
1118
1118
|
return false;
|
|
1119
1119
|
}
|
|
1120
1120
|
|
|
1121
|
+
function isTupleTypeNode(type: import("typescript").TypeNode | undefined): boolean {
|
|
1122
|
+
if (!type) return false;
|
|
1123
|
+
const ts = getTypeScript();
|
|
1124
|
+
if (ts.isTypeOperatorNode(type)) return isTupleTypeNode(type.type);
|
|
1125
|
+
return ts.isTupleTypeNode(type);
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
/**
|
|
1129
|
+
* True where a two-element array is consumed as a `[name, value]` header entry:
|
|
1130
|
+
* inside an entries list (`new Headers([[name, value]])`,
|
|
1131
|
+
* `Object.fromEntries([...])`), passed to a call such as `tuples.push(...)`, or
|
|
1132
|
+
* declared with a tuple type. A two-name list is bound, iterated, or handed to
|
|
1133
|
+
* a constructor instead — `const STRIP = ["sec-fetch-dest", "referer"]`,
|
|
1134
|
+
* `for (const name of [...])`, `new Set([...])` — and is not an entry. The
|
|
1135
|
+
* position decides because a Sec-Fetch-* value ("document", "navigate") is
|
|
1136
|
+
* token-shaped exactly like a header name.
|
|
1137
|
+
*/
|
|
1138
|
+
function isHeaderEntryPosition(node: import("typescript").ArrayLiteralExpression): boolean {
|
|
1139
|
+
const ts = getTypeScript();
|
|
1140
|
+
let child: import("typescript").Node = node;
|
|
1141
|
+
let parent = node.parent;
|
|
1142
|
+
while (
|
|
1143
|
+
parent &&
|
|
1144
|
+
(ts.isParenthesizedExpression(parent) ||
|
|
1145
|
+
ts.isAsExpression(parent) ||
|
|
1146
|
+
ts.isSatisfiesExpression(parent) ||
|
|
1147
|
+
ts.isTypeAssertionExpression(parent))
|
|
1148
|
+
) {
|
|
1149
|
+
if (!ts.isParenthesizedExpression(parent) && isTupleTypeNode(parent.type)) return true;
|
|
1150
|
+
child = parent;
|
|
1151
|
+
parent = parent.parent;
|
|
1152
|
+
}
|
|
1153
|
+
if (!parent) return false;
|
|
1154
|
+
if (ts.isArrayLiteralExpression(parent)) return true;
|
|
1155
|
+
if (ts.isCallExpression(parent)) return parent.arguments.some((argument) => argument === child);
|
|
1156
|
+
if (ts.isVariableDeclaration(parent)) {
|
|
1157
|
+
return parent.initializer === child && isTupleTypeNode(parent.type);
|
|
1158
|
+
}
|
|
1159
|
+
return false;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1121
1162
|
function isStaticUndefined(node: import("typescript").Node | undefined): boolean {
|
|
1122
1163
|
if (!node) return false;
|
|
1123
1164
|
const ts = getTypeScript();
|
|
@@ -1138,9 +1179,14 @@ function calleeName(node: import("typescript").CallExpression): string | undefin
|
|
|
1138
1179
|
* runtime rejection: host, connection, and accept-encoding are left alone
|
|
1139
1180
|
* because ctx.http callers set them legitimately in the same files, and
|
|
1140
1181
|
* user-agent only reaches a finding through the "user-agent" kind, which
|
|
1141
|
-
* matches a versioned literal value rather than the name
|
|
1142
|
-
*
|
|
1143
|
-
*
|
|
1182
|
+
* matches a versioned literal value rather than the name.
|
|
1183
|
+
*
|
|
1184
|
+
* This rule is the early, same-file signal only. The complete check is the
|
|
1185
|
+
* offline test harness: `runStandardTests` runs every `ctx.stealth` request
|
|
1186
|
+
* through the runtime's own `assertCallerHeadersSupported`, so a header spread
|
|
1187
|
+
* from another module, or a user-agent read from a variable or
|
|
1188
|
+
* `getStealthProfile(...)`, fails the provider's tests exactly as it would fail
|
|
1189
|
+
* in production.
|
|
1144
1190
|
*/
|
|
1145
1191
|
function isReportedOwnedHeaderName(value: string | undefined): boolean {
|
|
1146
1192
|
if (value === undefined) return false;
|
|
@@ -1247,17 +1293,16 @@ function collectBrowserVersionLiteralFindings(
|
|
|
1247
1293
|
}
|
|
1248
1294
|
|
|
1249
1295
|
if (ts.isArrayLiteralExpression(node) && node.elements[0]) {
|
|
1250
|
-
// Only a `[name, value]` tuple is a header
|
|
1251
|
-
//
|
|
1252
|
-
//
|
|
1253
|
-
// owned name.
|
|
1296
|
+
// Only a `[name, value]` tuple is a header: a two-element array in an
|
|
1297
|
+
// entry position (isHeaderEntryPosition) whose second element is not
|
|
1298
|
+
// itself an owned name.
|
|
1254
1299
|
const second = node.elements[1];
|
|
1255
1300
|
inspectHeaderEntry(
|
|
1256
1301
|
node.elements[0],
|
|
1257
1302
|
staticStringText(node.elements[0]),
|
|
1258
1303
|
second,
|
|
1259
1304
|
node.elements.length === 2 &&
|
|
1260
|
-
|
|
1305
|
+
isHeaderEntryPosition(node) &&
|
|
1261
1306
|
!isStealthOwnedHeaderName(staticStringText(second) ?? "")
|
|
1262
1307
|
? "entry"
|
|
1263
1308
|
: false,
|
|
@@ -1299,7 +1344,7 @@ function browserVersionLiteralMessage(finding: BrowserVersionLiteralFinding): st
|
|
|
1299
1344
|
case "sec-ch-ua":
|
|
1300
1345
|
return 'Hardcoded sec-ch-ua versions can disagree with the stealth TLS fingerprint. Remove the literal and let ctx.stealth generate client hints from stealth: { browser: "chrome", os: "macos" }. ctx.stealth also owns User-Agent, so omit that header there as well; for ctx.http only, derive it from getStealthProfile({ browser: "chrome", os: "macos" }).userAgent.';
|
|
1301
1346
|
case "owned-header":
|
|
1302
|
-
return `ctx.stealth owns the "${finding.literal}" header and rejects a caller value with STEALTH_HEADER_OVERRIDE_UNSUPPORTED (HTTP 500 at request time). Remove it; declare stealth: { requestClass: "navigation" | "xhr" | "post" } to drive Sec-Fetch-* and select the profile with stealth: { browser: "chrome", os: "macos" } for client hints.`;
|
|
1347
|
+
return `ctx.stealth owns the "${finding.literal}" header and rejects a caller value with STEALTH_HEADER_OVERRIDE_UNSUPPORTED (HTTP 500 at request time). Remove it; declare stealth: { requestClass: "navigation" | "form-post" | "xhr" | "post" } to drive Sec-Fetch-* and select the profile with stealth: { browser: "chrome", os: "macos" } for client hints.`;
|
|
1303
1348
|
}
|
|
1304
1349
|
}
|
|
1305
1350
|
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
REDACTED_QUERY_VALUE,
|
|
7
7
|
type SensitiveValueVariant,
|
|
8
8
|
} from "./request-options.js";
|
|
9
|
+
import { RESOLVER_CLOSED_ENUM_SPAN_ATTRIBUTES } from "./resolver-span-attributes.js";
|
|
9
10
|
|
|
10
11
|
export type DiagnosticRedactor = (text: string) => string;
|
|
11
12
|
export const REDACTION_FAILED = "[REDACTION_FAILED]";
|
|
@@ -577,11 +578,19 @@ export function redactDiagnosticAttributeKey(key: string, redact?: DiagnosticRed
|
|
|
577
578
|
export function redactCriticalDiagnosticText(text: string, redact?: DiagnosticRedactor): string {
|
|
578
579
|
return redactDiagnosticText(text, redact ? (criticalRedactors.get(redact) ?? redact) : undefined);
|
|
579
580
|
}
|
|
581
|
+
/**
|
|
582
|
+
* The redactor for one attribute value. A value its producer declares as a closed-enum literal for
|
|
583
|
+
* this key is a contract, like a typed number: exhaustion suppression skips it, while registered
|
|
584
|
+
* credentials still match. Any other value under the same key stays free text.
|
|
585
|
+
*/
|
|
580
586
|
export function diagnosticAttributeRedactor(
|
|
581
587
|
key: string,
|
|
582
588
|
redact?: DiagnosticRedactor,
|
|
589
|
+
value?: unknown,
|
|
583
590
|
): DiagnosticRedactor | undefined {
|
|
584
|
-
if (
|
|
591
|
+
if (typeof value === "string" && CLOSED_ENUM_ATTRIBUTE_VALUES.get(key)?.has(value)) {
|
|
592
|
+
return diagnosticStructuredRedactor(redact);
|
|
593
|
+
}
|
|
585
594
|
return CRITICAL_ATTRIBUTE_KEYS.has(key) && redact
|
|
586
595
|
? (criticalRedactors.get(redact) ?? redact)
|
|
587
596
|
: redact;
|
|
@@ -600,18 +609,15 @@ const CRITICAL_ATTRIBUTE_KEYS = new Set([
|
|
|
600
609
|
"http.status_code",
|
|
601
610
|
]);
|
|
602
611
|
|
|
603
|
-
// SDK-owned closed enums are contracts, like typed numbers and booleans.
|
|
604
|
-
//
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
"taxonomy",
|
|
613
|
-
"taxonomyVersion",
|
|
614
|
-
]);
|
|
612
|
+
// SDK-owned closed enums are contracts, like typed numbers and booleans. The table is the
|
|
613
|
+
// producers' own declaration of which span attributes are closed enums and of every literal each
|
|
614
|
+
// one can carry; it exempts only exhaustion suppression, and registered credentials still match.
|
|
615
|
+
const CLOSED_ENUM_ATTRIBUTE_VALUES: ReadonlyMap<string, ReadonlySet<string>> = new Map(
|
|
616
|
+
Object.entries(RESOLVER_CLOSED_ENUM_SPAN_ATTRIBUTES).map(([key, values]) => [
|
|
617
|
+
key,
|
|
618
|
+
new Set<string>(values),
|
|
619
|
+
]),
|
|
620
|
+
);
|
|
615
621
|
|
|
616
622
|
// Bigints normalize to strings for OTLP. Carry their type policy through internal
|
|
617
623
|
// detached copies without treating arbitrary numeric-looking free text as typed.
|
package/src/runtime/http.ts
CHANGED
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
resolvePolicyTransportAttemptCap,
|
|
5
5
|
resolveProxyConfigAsync,
|
|
6
6
|
} from "../config/loader.js";
|
|
7
|
-
import {
|
|
7
|
+
import { ProviderError, TransportError } from "../errors.js";
|
|
8
8
|
import { parseSseStream, readableBytes, readableLines, readableTextChunks } from "../stream.js";
|
|
9
9
|
import type {
|
|
10
10
|
HttpAttemptContext,
|
|
@@ -36,7 +36,13 @@ import {
|
|
|
36
36
|
type HttpTelemetrySink,
|
|
37
37
|
type HttpAttemptTelemetryEvent,
|
|
38
38
|
} from "./http-telemetry.js";
|
|
39
|
-
import {
|
|
39
|
+
import {
|
|
40
|
+
invalidRedirectPolicy,
|
|
41
|
+
isRedirectStatus,
|
|
42
|
+
nextSameOriginRedirectHop,
|
|
43
|
+
normalizeRedirectPolicy,
|
|
44
|
+
type RedirectPolicyTransport,
|
|
45
|
+
} from "./redirects.js";
|
|
40
46
|
import {
|
|
41
47
|
normalizeHttpRequestBody,
|
|
42
48
|
redactSensitiveError,
|
|
@@ -590,74 +596,24 @@ function resolveHttpUrl(baseUrl: string | undefined, url: string): string {
|
|
|
590
596
|
type NativeFetchInit = RequestInit & { proxy?: string };
|
|
591
597
|
|
|
592
598
|
const MAX_HTTP_REDIRECT_HOPS = 20;
|
|
593
|
-
const
|
|
599
|
+
const HTTP_REDIRECT_POLICY_TRANSPORT: RedirectPolicyTransport = {
|
|
600
|
+
label: "ctx.http",
|
|
601
|
+
maxHops: MAX_HTTP_REDIRECT_HOPS,
|
|
602
|
+
};
|
|
594
603
|
const REDIRECT_BODY_HEADERS = new Set([
|
|
595
604
|
"content-encoding",
|
|
596
605
|
"content-language",
|
|
597
606
|
"content-location",
|
|
598
607
|
"content-type",
|
|
599
608
|
]);
|
|
600
|
-
const MALFORMED_REDIRECT_TARGET = "[malformed redirect target]";
|
|
601
|
-
|
|
602
|
-
function invalidHttpRedirectPolicy(message: string, cause?: Error): TransportError {
|
|
603
|
-
return new TransportError(`Invalid ctx.http redirectPolicy: ${message}`, {
|
|
604
|
-
code: "http_redirect_policy_invalid",
|
|
605
|
-
...(cause ? { cause } : {}),
|
|
606
|
-
});
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
/** Snapshot untrusted caller input synchronously, before proxy resolution or fetch. */
|
|
610
|
-
function normalizeHttpRedirectPolicy(value: unknown): HttpRedirectPolicy | undefined {
|
|
611
|
-
if (value === undefined) return undefined;
|
|
612
|
-
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
613
|
-
throw invalidHttpRedirectPolicy("expected an object");
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
try {
|
|
617
|
-
const keys = Reflect.ownKeys(value);
|
|
618
|
-
for (const key of keys) {
|
|
619
|
-
if (typeof key !== "string" || !HTTP_REDIRECT_POLICY_FIELDS.has(key)) {
|
|
620
|
-
throw invalidHttpRedirectPolicy(`unknown field ${String(key)}`);
|
|
621
|
-
}
|
|
622
|
-
}
|
|
623
|
-
for (const field of HTTP_REDIRECT_POLICY_FIELDS) {
|
|
624
|
-
const descriptor = Object.getOwnPropertyDescriptor(value, field);
|
|
625
|
-
if (!descriptor || !("value" in descriptor)) {
|
|
626
|
-
throw invalidHttpRedirectPolicy(`${field} must be an own data property`);
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
const record = value as Record<string, unknown>;
|
|
631
|
-
if (record.mode !== "same-origin") {
|
|
632
|
-
throw invalidHttpRedirectPolicy('mode must be "same-origin"');
|
|
633
|
-
}
|
|
634
|
-
if (
|
|
635
|
-
typeof record.maxHops !== "number" ||
|
|
636
|
-
!Number.isInteger(record.maxHops) ||
|
|
637
|
-
record.maxHops < 0 ||
|
|
638
|
-
record.maxHops > MAX_HTTP_REDIRECT_HOPS
|
|
639
|
-
) {
|
|
640
|
-
throw invalidHttpRedirectPolicy(
|
|
641
|
-
`maxHops must be an integer from 0 to ${MAX_HTTP_REDIRECT_HOPS}`,
|
|
642
|
-
);
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
return { mode: "same-origin", maxHops: record.maxHops };
|
|
646
|
-
} catch (error) {
|
|
647
|
-
if (error instanceof TransportError) throw error;
|
|
648
|
-
throw invalidHttpRedirectPolicy(
|
|
649
|
-
"could not be inspected safely",
|
|
650
|
-
error instanceof Error ? error : undefined,
|
|
651
|
-
);
|
|
652
|
-
}
|
|
653
|
-
}
|
|
654
609
|
|
|
655
610
|
function snapshotHttpRedirectPolicy(options: RequestOptions): HttpRedirectPolicy | undefined {
|
|
656
611
|
try {
|
|
657
|
-
return
|
|
612
|
+
return normalizeRedirectPolicy(options.redirectPolicy, HTTP_REDIRECT_POLICY_TRANSPORT);
|
|
658
613
|
} catch (error) {
|
|
659
614
|
if (error instanceof TransportError) throw error;
|
|
660
|
-
throw
|
|
615
|
+
throw invalidRedirectPolicy(
|
|
616
|
+
HTTP_REDIRECT_POLICY_TRANSPORT,
|
|
661
617
|
"could not be read safely",
|
|
662
618
|
error instanceof Error ? error : undefined,
|
|
663
619
|
);
|
|
@@ -670,21 +626,6 @@ function withoutRedirectBodyHeaders(headers: HeadersInit | undefined): Headers {
|
|
|
670
626
|
return nextHeaders;
|
|
671
627
|
}
|
|
672
628
|
|
|
673
|
-
function redirectDiagnosticTarget(value: string): string {
|
|
674
|
-
try {
|
|
675
|
-
const parsed = new URL(value);
|
|
676
|
-
// Origin omits URL userinfo. Keeping only origin + path makes diagnostics
|
|
677
|
-
// useful while structurally excluding every query value and fragment,
|
|
678
|
-
// including attacker-chosen keys the provider did not declare sensitive.
|
|
679
|
-
const redactedQuery = parsed.search ? "?[REDACTED]" : "";
|
|
680
|
-
return parsed.origin === "null"
|
|
681
|
-
? `${parsed.protocol}<opaque-target>`
|
|
682
|
-
: `${parsed.origin}${parsed.pathname}${redactedQuery}`;
|
|
683
|
-
} catch {
|
|
684
|
-
return MALFORMED_REDIRECT_TARGET;
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
|
|
688
629
|
function discardRedirectResponseBody(response: Response): void {
|
|
689
630
|
try {
|
|
690
631
|
const cancellation = response.body?.cancel();
|
|
@@ -730,47 +671,17 @@ async function fetchWithHttpRedirectPolicy(
|
|
|
730
671
|
|
|
731
672
|
const location = response.headers.get("location");
|
|
732
673
|
discardRedirectResponseBody(response);
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
throw new HttpRedirectError(`Redirect response has malformed Location target ${target}`, {
|
|
739
|
-
reason: "missing_location",
|
|
740
|
-
target,
|
|
741
|
-
status: response.status,
|
|
742
|
-
});
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
const decision = evaluateRedirectHop({
|
|
674
|
+
const decision = nextSameOriginRedirectHop({
|
|
675
|
+
policy,
|
|
676
|
+
initialOrigin,
|
|
677
|
+
currentUrl,
|
|
678
|
+
responseUrl: currentUrl,
|
|
746
679
|
status: response.status,
|
|
747
680
|
method,
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
redirectCount: followedHops + 1,
|
|
751
|
-
maxHops: policy.maxHops,
|
|
681
|
+
location: location ?? undefined,
|
|
682
|
+
followedHops,
|
|
752
683
|
visitedRequests,
|
|
753
684
|
});
|
|
754
|
-
if (decision.kind === "stop") {
|
|
755
|
-
const target = decision.nextUrl ? redirectDiagnosticTarget(decision.nextUrl) : undefined;
|
|
756
|
-
const message = (() => {
|
|
757
|
-
switch (decision.reason) {
|
|
758
|
-
case "stopped":
|
|
759
|
-
return `Redirect policy refused cross-origin target ${target}`;
|
|
760
|
-
case "max_hops":
|
|
761
|
-
return `Redirect policy reached maxHops before target ${target}`;
|
|
762
|
-
case "loop":
|
|
763
|
-
return `Redirect loop refused target ${target}`;
|
|
764
|
-
case "missing_location":
|
|
765
|
-
return `Redirect response from ${redirectDiagnosticTarget(currentUrl)} is missing Location`;
|
|
766
|
-
}
|
|
767
|
-
})();
|
|
768
|
-
throw new HttpRedirectError(message, {
|
|
769
|
-
reason: decision.reason,
|
|
770
|
-
...(target ? { target } : {}),
|
|
771
|
-
status: response.status,
|
|
772
|
-
});
|
|
773
|
-
}
|
|
774
685
|
if (decision.nextMethod !== method) {
|
|
775
686
|
body = undefined;
|
|
776
687
|
headers = withoutRedirectBodyHeaders(headers);
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
type TraceContext,
|
|
27
27
|
} from "./trace.js";
|
|
28
28
|
import { RESOLVER_INSTRUMENTATION_METADATA } from "./resolver-shared.js";
|
|
29
|
+
import { resolverSpanAttributes } from "./resolver-span-attributes.js";
|
|
29
30
|
|
|
30
31
|
export interface InstrumentationOptions extends CreateTraceContextOptions {}
|
|
31
32
|
|
|
@@ -707,7 +708,9 @@ function wrapNamespace<T extends object>(
|
|
|
707
708
|
"resolver.solve",
|
|
708
709
|
() => Reflect.apply(value, namespaceTarget, [args[0], args[1], recorder]),
|
|
709
710
|
{
|
|
710
|
-
attributes: challengeKind
|
|
711
|
+
attributes: challengeKind
|
|
712
|
+
? resolverSpanAttributes({ challenge_kind: challengeKind })
|
|
713
|
+
: undefined,
|
|
711
714
|
},
|
|
712
715
|
);
|
|
713
716
|
};
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { getPublicSuffix } from "tough-cookie";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Request-body header names a redirect drops when it changes the method
|
|
5
|
+
* (Fetch "request-body-header name").
|
|
6
|
+
*/
|
|
7
|
+
const REDIRECT_BODY_HEADERS: ReadonlySet<string> = new Set([
|
|
8
|
+
"content-encoding",
|
|
9
|
+
"content-language",
|
|
10
|
+
"content-location",
|
|
11
|
+
"content-type",
|
|
12
|
+
]);
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Credentials a redirect never carries to another origin. Fetch removes
|
|
16
|
+
* `Authorization` on a cross-origin redirect; `Proxy-Authorization` is a
|
|
17
|
+
* credential of the same kind.
|
|
18
|
+
*/
|
|
19
|
+
const CROSS_ORIGIN_CREDENTIAL_HEADERS: ReadonlySet<string> = new Set([
|
|
20
|
+
"authorization",
|
|
21
|
+
"proxy-authorization",
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
/** One redirect hop: the URL the headers were sent to, and the next request. */
|
|
25
|
+
export interface RedirectHop {
|
|
26
|
+
readonly from: string;
|
|
27
|
+
readonly to: string;
|
|
28
|
+
readonly methodChanged: boolean;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Schemeful site (HTML "same site"): scheme plus registrable domain from the
|
|
33
|
+
* public suffix list, or the bare host when there is none (an IP address or
|
|
34
|
+
* a single-label host).
|
|
35
|
+
*/
|
|
36
|
+
function schemefulSite(url: URL): string {
|
|
37
|
+
const host = url.hostname;
|
|
38
|
+
const registrable = getPublicSuffix(host, { allowSpecialUseDomain: true, ignoreError: true });
|
|
39
|
+
return `${url.protocol}//${registrable ?? host}`;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The caller headers the next redirect hop carries, derived from the headers
|
|
44
|
+
* the previous hop sent. Both stealth redirect walkers compute every hop's
|
|
45
|
+
* headers here, so the rules hold on every path:
|
|
46
|
+
*
|
|
47
|
+
* - a method change drops the request-body headers;
|
|
48
|
+
* - a cross-origin hop drops `Authorization` and `Proxy-Authorization`;
|
|
49
|
+
* - a cross-site hop drops an explicit `Cookie`, after which the session
|
|
50
|
+
* cookie jar supplies cookies by its own domain rules, as a browser does.
|
|
51
|
+
*
|
|
52
|
+
* A header dropped on one hop stays dropped for the rest of the chain.
|
|
53
|
+
*/
|
|
54
|
+
export function redirectHopHeaders<T>(
|
|
55
|
+
hop: RedirectHop,
|
|
56
|
+
headers: Readonly<Record<string, T>>,
|
|
57
|
+
): Record<string, T> {
|
|
58
|
+
const from = new URL(hop.from);
|
|
59
|
+
const to = new URL(hop.to);
|
|
60
|
+
const crossOrigin = from.origin !== to.origin;
|
|
61
|
+
const crossSite = schemefulSite(from) !== schemefulSite(to);
|
|
62
|
+
return Object.fromEntries(
|
|
63
|
+
Object.entries(headers).filter(([name]) => {
|
|
64
|
+
const lower = name.toLowerCase();
|
|
65
|
+
if (hop.methodChanged && REDIRECT_BODY_HEADERS.has(lower)) return false;
|
|
66
|
+
if (crossOrigin && CROSS_ORIGIN_CREDENTIAL_HEADERS.has(lower)) return false;
|
|
67
|
+
if (crossSite && lower === "cookie") return false;
|
|
68
|
+
return true;
|
|
69
|
+
}),
|
|
70
|
+
);
|
|
71
|
+
}
|