@apifuse/provider-sdk 2.2.0-beta.68 → 2.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +6 -1
- package/CHANGELOG.md +4 -0
- package/README.md +10 -3
- package/dist/fixture-sanitization.js +159 -15
- package/dist/lint.js +56 -9
- package/dist/runtime/diagnostic-redactor.d.ts +6 -1
- package/dist/runtime/diagnostic-redactor.js +16 -14
- package/dist/runtime/http.js +15 -102
- package/dist/runtime/instrumentation.js +4 -1
- package/dist/runtime/redirect-hop-headers.d.ts +19 -0
- package/dist/runtime/redirect-hop-headers.js +58 -0
- package/dist/runtime/redirects.d.ts +33 -1
- package/dist/runtime/redirects.js +113 -0
- package/dist/runtime/resolver-span-attributes.d.ts +34 -0
- package/dist/runtime/resolver-span-attributes.js +93 -0
- package/dist/runtime/resolver-usage.d.ts +8 -0
- package/dist/runtime/resolver-usage.js +5 -11
- package/dist/runtime/resolver-vendors/browser.js +5 -4
- package/dist/runtime/resolver-vendors/capsolver.js +11 -10
- package/dist/runtime/resolver-vendors/twocaptcha.js +9 -8
- package/dist/runtime/resolver.js +7 -6
- package/dist/runtime/stealth-owned-headers.d.ts +21 -8
- package/dist/runtime/stealth-owned-headers.js +28 -8
- package/dist/runtime/stealth-telemetry.d.ts +1 -1
- package/dist/runtime/stealth.js +158 -46
- package/dist/runtime/trace.js +2 -2
- package/dist/server/serve-implementation.js +2 -1
- package/dist/testing/run.js +14 -7
- package/dist/trace-sanitization.js +1 -1
- package/dist/types.d.ts +30 -6
- package/package.json +2 -2
- package/src/fixture-sanitization.ts +175 -17
- package/src/lint.ts +54 -9
- package/src/runtime/diagnostic-redactor.ts +19 -13
- package/src/runtime/http.ts +22 -111
- package/src/runtime/instrumentation.ts +4 -1
- package/src/runtime/redirect-hop-headers.ts +71 -0
- package/src/runtime/redirects.ts +150 -1
- package/src/runtime/resolver-span-attributes.ts +139 -0
- package/src/runtime/resolver-usage.ts +8 -5
- package/src/runtime/resolver-vendors/browser.ts +5 -4
- package/src/runtime/resolver-vendors/capsolver.ts +11 -10
- package/src/runtime/resolver-vendors/twocaptcha.ts +9 -8
- package/src/runtime/resolver.ts +7 -6
- package/src/runtime/stealth-owned-headers.ts +34 -8
- package/src/runtime/stealth-telemetry.ts +6 -1
- package/src/runtime/stealth.ts +180 -58
- package/src/runtime/trace.ts +6 -2
- package/src/server/serve-implementation.ts +2 -1
- package/src/testing/run.ts +15 -2
- package/src/trace-sanitization.ts +1 -1
- package/src/types.ts +30 -6
package/AUTHORING.md
CHANGED
|
@@ -345,7 +345,12 @@ a Web `Response` or `{ status, headers, body }`; an unmatched call fails with
|
|
|
345
345
|
the operation, transport, and method named in the error. Browser handlers expose
|
|
346
346
|
method-level calls such as `goto`, `evaluate`, and `locator.click`, so provide a
|
|
347
347
|
canned result for each method the handler uses. Native connections similarly
|
|
348
|
-
identify `connectTcp`/`connectTls` and subsequent `write` calls.
|
|
348
|
+
identify `connectTcp`/`connectTls` and subsequent `write` calls. The offline
|
|
349
|
+
`ctx.stealth` rejects a stealth-owned request header (`User-Agent`,
|
|
350
|
+
`Sec-Fetch-*`, `sec-ch-ua*`, ...) with `STEALTH_HEADER_OVERRIDE_UNSUPPORTED`
|
|
351
|
+
before calling the stub, through the same check the production transport runs,
|
|
352
|
+
so a header spread from a shared constants module or read from a variable fails
|
|
353
|
+
the test instead of the first production request. Direct global
|
|
349
354
|
`fetch` or socket usage is outside this ProviderContext seam and should not be
|
|
350
355
|
used by provider handlers.
|
|
351
356
|
|
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -148,8 +148,12 @@ the bad request path; provider/runtime failures include `code`, `message`, and
|
|
|
148
148
|
`ctx.stealth.fetch()` when you need browser-like session or cookie control.
|
|
149
149
|
`ctx.stealth.fetch()` uses the `wreq-js`-backed browser stealth transport and
|
|
150
150
|
accepts request controls for `params`, `sensitiveParams`, `proxy`, `timeout`,
|
|
151
|
-
`maxBodyBytes`, `redirect`, `throwOnHttpError`, and
|
|
152
|
-
`stealth.insecureSkipVerify`.
|
|
151
|
+
`maxBodyBytes`, `redirect`, `redirectPolicy`, `throwOnHttpError`, and
|
|
152
|
+
`stealth.insecureSkipVerify`. A followed redirect drops `Authorization` when
|
|
153
|
+
it leaves the origin and an explicit `Cookie` when it leaves the site (the
|
|
154
|
+
session cookie jar takes over there); set
|
|
155
|
+
`redirectPolicy: { mode: "same-origin", maxHops }` to refuse a cross-origin
|
|
156
|
+
hop with `HttpRedirectError` before any request reaches it. For login
|
|
153
157
|
flows that must inspect intermediate `Location`/`Set-Cookie` headers, create
|
|
154
158
|
a session with `ctx.stealth.createSession()` and use `session.redirects.run()`;
|
|
155
159
|
inspect accumulated cookies through `session.cookies`. Select an SDK stealth
|
|
@@ -157,7 +161,10 @@ the bad request path; provider/runtime failures include `code`, `message`, and
|
|
|
157
161
|
explicitly defaults to `macos`. The transport owns `User-Agent`,
|
|
158
162
|
`Accept-Encoding`, `sec-ch-ua*`, and every `Sec-Fetch-*` header and throws
|
|
159
163
|
`STEALTH_HEADER_OVERRIDE_UNSUPPORTED` if you set one; declare
|
|
160
|
-
`stealth.requestClass: "navigation" | "xhr" | "post"` instead
|
|
164
|
+
`stealth.requestClass: "navigation" | "form-post" | "xhr" | "post"` instead
|
|
165
|
+
(`form-post` for an HTML form submission). The offline
|
|
166
|
+
`ctx.stealth` in `runStandardTests` runs the same check, so a provider test
|
|
167
|
+
fails wherever production would. Do not pin a
|
|
161
168
|
browser version
|
|
162
169
|
or tune JA3, HTTP/2 SETTINGS, or
|
|
163
170
|
pseudo-header order in provider code. Chrome, Firefox, and Safari profiles
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
1
2
|
export const REDACTED_FIXTURE_VALUE = "[REDACTED]";
|
|
2
3
|
const OPAQUE_TOKEN = /^[A-Za-z0-9_+/=.:~-]+$/;
|
|
3
4
|
const OPAQUE_TOKEN_RUN = /[A-Za-z0-9_+/=.:~-]{24,}/g;
|
|
@@ -6,12 +7,39 @@ const EMAIL_ADDRESS_RUN = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
|
|
|
6
7
|
const DIAGNOSTIC_URL_SENTINEL_DELIMITER = String.fromCodePoint(0);
|
|
7
8
|
const DIAGNOSTIC_URL_SENTINEL_RUN = new RegExp(`${DIAGNOSTIC_URL_SENTINEL_DELIMITER}APIFUSE_URL(\\d+)${DIAGNOSTIC_URL_SENTINEL_DELIMITER}`, "g");
|
|
8
9
|
const PEM_PRIVATE_KEY = /-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9 ]+ )?PRIVATE KEY-----/g;
|
|
10
|
+
/** RFC 9110 §11.4 `token68`, the single credential token of Bearer, Basic, Negotiate and NTLM. */
|
|
11
|
+
const TOKEN68 = "[A-Za-z0-9._~+/-]+=*";
|
|
12
|
+
/**
|
|
13
|
+
* A credential header's value is the whole RFC 9110 `credentials` production
|
|
14
|
+
* (`auth-scheme SP token68`), so the scheme and its token are one value, not two words.
|
|
15
|
+
* The key starts at a word boundary so a long word run is scanned once, not from every offset.
|
|
16
|
+
*/
|
|
17
|
+
const CREDENTIALS_HEADER_ASSIGNMENT = new RegExp(String.raw `(?<![\w-])((["']?)[\w-]*authorization\2\s*[:=]\s*)[A-Za-z][\w.+-]*[ \t]+${TOKEN68}(?![^\s,;&])`, "gi");
|
|
18
|
+
/**
|
|
19
|
+
* Keyless echoes of the token68 schemes whose token is always a credential. Only the scheme and
|
|
20
|
+
* its whitespace are matched, so matches never overlap a later scheme; the token is the whole
|
|
21
|
+
* opaque run that follows, so no partial redaction leaves a remainder too short for the
|
|
22
|
+
* opaque-token scan. Negotiate and NTLM match only their registered spelling so prose such as
|
|
23
|
+
* "failed to negotiate TLS" is retained.
|
|
24
|
+
*/
|
|
25
|
+
const TOKEN68_SCHEMES = [/\bBearer\s+/gi, /\b(?:Negotiate|NTLM)\s+/g];
|
|
26
|
+
/** A keyless `Basic` token is a credential only when it decodes to `user-id:password`. */
|
|
27
|
+
const BASIC_SCHEME = /\bBasic\s+/gi;
|
|
28
|
+
const OPAQUE_TOKEN_CHARACTER = /[A-Za-z0-9_+/=.:~-]/;
|
|
29
|
+
const BASE64_CHARACTER = /[A-Za-z0-9+/]/;
|
|
30
|
+
const ASSIGNMENT_VALUE_DELIMITER = /[\s,;&]/;
|
|
31
|
+
/**
|
|
32
|
+
* A key and its `:`/`=`, starting at a word boundary so a word run is scanned once. Values are
|
|
33
|
+
* never consumed, so an assignment inside another value is still found.
|
|
34
|
+
*/
|
|
35
|
+
const ASSIGNMENT_KEY = /(["']?)(?<![\w-])([\w-]+)\1\s*[:=]\s*/gi;
|
|
9
36
|
/** Matches credential field names without treating benign prefixes such as `author` as `auth`. */
|
|
10
37
|
export function isSensitiveFixtureKey(key) {
|
|
11
38
|
const normalized = key.replace(/[-_\s]/g, "").toLowerCase();
|
|
12
39
|
const candidates = [normalized, normalized.replace(/(?:value|payload|header)$/, "")];
|
|
13
40
|
return candidates.some((candidate) => /^(?:authorization|authentication|auth|bearer|cookie|credential|password|passwd|privatekey|secret|session|sessionid|token)$/.test(candidate) ||
|
|
14
|
-
|
|
41
|
+
// Unanchored at the start so vendor-prefixed headers (`x-api-key`) match too.
|
|
42
|
+
/(?:api|client|service|access|consumer)(?:key|secret|token)$/.test(candidate) ||
|
|
15
43
|
/(?:authorization|credential|password|passwd|privatekey|secret|sessionid|token)$/.test(candidate));
|
|
16
44
|
}
|
|
17
45
|
/**
|
|
@@ -50,7 +78,7 @@ export function sanitizeFixtureString(value) {
|
|
|
50
78
|
const index = retainedUrls.push(isCredentialBearingUrl(url) ? sanitizeUrlForLogs(url) : url) - 1;
|
|
51
79
|
return `APIFUSEURL${index}X`;
|
|
52
80
|
});
|
|
53
|
-
sanitized =
|
|
81
|
+
sanitized = redactCredentialSpans(sanitized, [credentialHeaderSpans, sensitiveAssignmentSpans]);
|
|
54
82
|
sanitized = sanitized.replace(OPAQUE_TOKEN_RUN, (candidate) => isSensitiveFixtureValue(candidate) ? REDACTED_FIXTURE_VALUE : candidate);
|
|
55
83
|
sanitized = sanitized.replace(/APIFUSEURL(\d+)X/g, (_match, index) => retainedUrls[Number(index)] ?? REDACTED_FIXTURE_VALUE);
|
|
56
84
|
return sanitized;
|
|
@@ -135,9 +163,12 @@ export function sanitizeDiagnosticText(value) {
|
|
|
135
163
|
const index = retainedUrls.push(sanitizeUrlForLogs(url)) - 1;
|
|
136
164
|
return `${DIAGNOSTIC_URL_SENTINEL_DELIMITER}APIFUSE_URL${index}${DIAGNOSTIC_URL_SENTINEL_DELIMITER}`;
|
|
137
165
|
})
|
|
138
|
-
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, `Bearer ${REDACTED_FIXTURE_VALUE}`)
|
|
139
166
|
.replace(EMAIL_ADDRESS_RUN, REDACTED_FIXTURE_VALUE);
|
|
140
|
-
sanitized =
|
|
167
|
+
sanitized = redactCredentialSpans(sanitized, [
|
|
168
|
+
credentialHeaderSpans,
|
|
169
|
+
keylessCredentialSpans,
|
|
170
|
+
sensitiveAssignmentSpans,
|
|
171
|
+
]);
|
|
141
172
|
sanitized = sanitized.replace(OPAQUE_TOKEN_RUN, (candidate, offset, source) => {
|
|
142
173
|
if (/^(?:request|trace|correlation)[-_]?id[:=]/i.test(candidate))
|
|
143
174
|
return candidate;
|
|
@@ -149,17 +180,130 @@ export function sanitizeDiagnosticText(value) {
|
|
|
149
180
|
sanitized = sanitized.replace(DIAGNOSTIC_URL_SENTINEL_RUN, (_match, index) => retainedUrls[Number(index)] ?? REDACTED_FIXTURE_VALUE);
|
|
150
181
|
return encodeDiagnosticControls(sanitized);
|
|
151
182
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
183
|
+
/**
|
|
184
|
+
* Every credential rule scans the same unmodified text and reports spans; the output redacts
|
|
185
|
+
* their union. No rule's rewrite can hide a key or a scheme word from another rule, so adding a
|
|
186
|
+
* rule can only redact more.
|
|
187
|
+
*/
|
|
188
|
+
function redactCredentialSpans(value, rules) {
|
|
189
|
+
const spans = rules
|
|
190
|
+
.flatMap((rule) => [...rule(value)])
|
|
191
|
+
.sort((a, b) => a[0] - b[0] || b[1] - a[1]);
|
|
192
|
+
let output = "";
|
|
193
|
+
let cursor = 0;
|
|
194
|
+
let open;
|
|
195
|
+
const close = (span) => {
|
|
196
|
+
output += value.slice(cursor, span[0]) + REDACTED_FIXTURE_VALUE;
|
|
197
|
+
cursor = span[1];
|
|
198
|
+
};
|
|
199
|
+
for (const [start, end] of spans) {
|
|
200
|
+
if (open && start <= open[1]) {
|
|
201
|
+
open[1] = Math.max(open[1], end);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
if (open)
|
|
205
|
+
close(open);
|
|
206
|
+
open = [start, end];
|
|
207
|
+
}
|
|
208
|
+
if (open)
|
|
209
|
+
close(open);
|
|
210
|
+
return output + value.slice(cursor);
|
|
211
|
+
}
|
|
212
|
+
/** The scheme and token of a credential header value, as one span. */
|
|
213
|
+
function* credentialHeaderSpans(value) {
|
|
214
|
+
for (const match of value.matchAll(CREDENTIALS_HEADER_ASSIGNMENT)) {
|
|
215
|
+
yield [match.index + match[1].length, match.index + match[0].length];
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
/** Tokens after keyless `Bearer`/`Negotiate`/`NTLM`, and `Basic` tokens that decode to `user-id:password`. */
|
|
219
|
+
function* keylessCredentialSpans(value) {
|
|
220
|
+
const opaqueEnds = runEnds(value, OPAQUE_TOKEN_CHARACTER);
|
|
221
|
+
for (const scheme of TOKEN68_SCHEMES) {
|
|
222
|
+
for (const match of value.matchAll(scheme)) {
|
|
223
|
+
const start = match.index + match[0].length;
|
|
224
|
+
if (opaqueEnds[start] > start)
|
|
225
|
+
yield [start, opaqueEnds[start]];
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
const base64Ends = runEnds(value, BASE64_CHARACTER);
|
|
229
|
+
for (const match of value.matchAll(BASIC_SCHEME)) {
|
|
230
|
+
const start = match.index + match[0].length;
|
|
231
|
+
const payloadEnd = base64Ends[start];
|
|
232
|
+
let end = payloadEnd;
|
|
233
|
+
while (end < payloadEnd + 2 && value[end] === "=")
|
|
234
|
+
end++;
|
|
235
|
+
// The base64 token must be the whole opaque run, not the prefix of a longer token.
|
|
236
|
+
if (payloadEnd === start || end !== opaqueEnds[start])
|
|
237
|
+
continue;
|
|
238
|
+
if (Buffer.from(value.slice(start, end), "base64").includes(0x3a))
|
|
239
|
+
yield [start, end];
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Values of credential-named keys; quotes around a quoted value are kept. Every key is visited,
|
|
244
|
+
* including keys inside another key's value (`auth: {"password": "x"}`), and each value's extent
|
|
245
|
+
* is read from precomputed tables, so the scan stays linear in the text length.
|
|
246
|
+
*/
|
|
247
|
+
function* sensitiveAssignmentSpans(value) {
|
|
248
|
+
let extents;
|
|
249
|
+
for (const key of value.matchAll(ASSIGNMENT_KEY)) {
|
|
250
|
+
const name = key[2];
|
|
251
|
+
if (!isSensitiveFixtureKey(name) && name.toLowerCase() !== "key")
|
|
252
|
+
continue;
|
|
253
|
+
extents ??= assignmentValueExtents(value);
|
|
254
|
+
const start = key.index + key[0].length;
|
|
255
|
+
const quote = value[start];
|
|
256
|
+
const closing = quote === '"'
|
|
257
|
+
? extents.doubleQuoteClose[start + 1]
|
|
258
|
+
: quote === "'"
|
|
259
|
+
? extents.singleQuoteClose[start + 1]
|
|
260
|
+
: -1;
|
|
261
|
+
if (closing !== -1) {
|
|
262
|
+
yield [start + 1, closing];
|
|
263
|
+
}
|
|
264
|
+
else if (extents.unquotedEnds[start] > start) {
|
|
265
|
+
yield [start, extents.unquotedEnds[start]];
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
function assignmentValueExtents(value) {
|
|
270
|
+
const unquotedEnds = new Int32Array(value.length + 1);
|
|
271
|
+
unquotedEnds[value.length] = value.length;
|
|
272
|
+
for (let index = value.length - 1; index >= 0; index--) {
|
|
273
|
+
unquotedEnds[index] = ASSIGNMENT_VALUE_DELIMITER.test(value[index])
|
|
274
|
+
? index
|
|
275
|
+
: unquotedEnds[index + 1];
|
|
276
|
+
}
|
|
277
|
+
return {
|
|
278
|
+
unquotedEnds,
|
|
279
|
+
doubleQuoteClose: closingQuotes(value, '"'),
|
|
280
|
+
singleQuoteClose: closingQuotes(value, "'"),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
/** Right-to-left: reading a quoted body from index i either closes at a quote or skips an escape pair. */
|
|
284
|
+
function closingQuotes(value, quote) {
|
|
285
|
+
const closing = new Int32Array(value.length + 2).fill(-1);
|
|
286
|
+
for (let index = value.length - 1; index >= 0; index--) {
|
|
287
|
+
const character = value[index];
|
|
288
|
+
closing[index] =
|
|
289
|
+
character === quote
|
|
290
|
+
? index
|
|
291
|
+
: character === "\\"
|
|
292
|
+
? /[^\n\r\u2028\u2029]/.test(value[index + 1] ?? "\n")
|
|
293
|
+
? closing[index + 2]
|
|
294
|
+
: -1
|
|
295
|
+
: closing[index + 1];
|
|
296
|
+
}
|
|
297
|
+
return closing;
|
|
298
|
+
}
|
|
299
|
+
/** For each index, the end of the run of `character` starting there, in one right-to-left pass. */
|
|
300
|
+
function runEnds(value, character) {
|
|
301
|
+
const ends = new Int32Array(value.length + 1);
|
|
302
|
+
ends[value.length] = value.length;
|
|
303
|
+
for (let index = value.length - 1; index >= 0; index--) {
|
|
304
|
+
ends[index] = character.test(value[index]) ? ends[index + 1] : index;
|
|
305
|
+
}
|
|
306
|
+
return ends;
|
|
163
307
|
}
|
|
164
308
|
function isCredentialBearingUrl(value) {
|
|
165
309
|
try {
|
package/dist/lint.js
CHANGED
|
@@ -821,6 +821,49 @@ function isStealthContextAccess(node) {
|
|
|
821
821
|
}
|
|
822
822
|
return false;
|
|
823
823
|
}
|
|
824
|
+
function isTupleTypeNode(type) {
|
|
825
|
+
if (!type)
|
|
826
|
+
return false;
|
|
827
|
+
const ts = getTypeScript();
|
|
828
|
+
if (ts.isTypeOperatorNode(type))
|
|
829
|
+
return isTupleTypeNode(type.type);
|
|
830
|
+
return ts.isTupleTypeNode(type);
|
|
831
|
+
}
|
|
832
|
+
/**
|
|
833
|
+
* True where a two-element array is consumed as a `[name, value]` header entry:
|
|
834
|
+
* inside an entries list (`new Headers([[name, value]])`,
|
|
835
|
+
* `Object.fromEntries([...])`), passed to a call such as `tuples.push(...)`, or
|
|
836
|
+
* declared with a tuple type. A two-name list is bound, iterated, or handed to
|
|
837
|
+
* a constructor instead — `const STRIP = ["sec-fetch-dest", "referer"]`,
|
|
838
|
+
* `for (const name of [...])`, `new Set([...])` — and is not an entry. The
|
|
839
|
+
* position decides because a Sec-Fetch-* value ("document", "navigate") is
|
|
840
|
+
* token-shaped exactly like a header name.
|
|
841
|
+
*/
|
|
842
|
+
function isHeaderEntryPosition(node) {
|
|
843
|
+
const ts = getTypeScript();
|
|
844
|
+
let child = node;
|
|
845
|
+
let parent = node.parent;
|
|
846
|
+
while (parent &&
|
|
847
|
+
(ts.isParenthesizedExpression(parent) ||
|
|
848
|
+
ts.isAsExpression(parent) ||
|
|
849
|
+
ts.isSatisfiesExpression(parent) ||
|
|
850
|
+
ts.isTypeAssertionExpression(parent))) {
|
|
851
|
+
if (!ts.isParenthesizedExpression(parent) && isTupleTypeNode(parent.type))
|
|
852
|
+
return true;
|
|
853
|
+
child = parent;
|
|
854
|
+
parent = parent.parent;
|
|
855
|
+
}
|
|
856
|
+
if (!parent)
|
|
857
|
+
return false;
|
|
858
|
+
if (ts.isArrayLiteralExpression(parent))
|
|
859
|
+
return true;
|
|
860
|
+
if (ts.isCallExpression(parent))
|
|
861
|
+
return parent.arguments.some((argument) => argument === child);
|
|
862
|
+
if (ts.isVariableDeclaration(parent)) {
|
|
863
|
+
return parent.initializer === child && isTupleTypeNode(parent.type);
|
|
864
|
+
}
|
|
865
|
+
return false;
|
|
866
|
+
}
|
|
824
867
|
function isStaticUndefined(node) {
|
|
825
868
|
if (!node)
|
|
826
869
|
return false;
|
|
@@ -842,9 +885,14 @@ function calleeName(node) {
|
|
|
842
885
|
* runtime rejection: host, connection, and accept-encoding are left alone
|
|
843
886
|
* because ctx.http callers set them legitimately in the same files, and
|
|
844
887
|
* user-agent only reaches a finding through the "user-agent" kind, which
|
|
845
|
-
* matches a versioned literal value rather than the name
|
|
846
|
-
*
|
|
847
|
-
*
|
|
888
|
+
* matches a versioned literal value rather than the name.
|
|
889
|
+
*
|
|
890
|
+
* This rule is the early, same-file signal only. The complete check is the
|
|
891
|
+
* offline test harness: `runStandardTests` runs every `ctx.stealth` request
|
|
892
|
+
* through the runtime's own `assertCallerHeadersSupported`, so a header spread
|
|
893
|
+
* from another module, or a user-agent read from a variable or
|
|
894
|
+
* `getStealthProfile(...)`, fails the provider's tests exactly as it would fail
|
|
895
|
+
* in production.
|
|
848
896
|
*/
|
|
849
897
|
function isReportedOwnedHeaderName(value) {
|
|
850
898
|
if (value === undefined)
|
|
@@ -928,13 +976,12 @@ function collectBrowserVersionLiteralFindings(fileName, source) {
|
|
|
928
976
|
: false);
|
|
929
977
|
}
|
|
930
978
|
if (ts.isArrayLiteralExpression(node) && node.elements[0]) {
|
|
931
|
-
// Only a `[name, value]` tuple is a header
|
|
932
|
-
//
|
|
933
|
-
//
|
|
934
|
-
// owned name.
|
|
979
|
+
// Only a `[name, value]` tuple is a header: a two-element array in an
|
|
980
|
+
// entry position (isHeaderEntryPosition) whose second element is not
|
|
981
|
+
// itself an owned name.
|
|
935
982
|
const second = node.elements[1];
|
|
936
983
|
inspectHeaderEntry(node.elements[0], staticStringText(node.elements[0]), second, node.elements.length === 2 &&
|
|
937
|
-
|
|
984
|
+
isHeaderEntryPosition(node) &&
|
|
938
985
|
!isStealthOwnedHeaderName(staticStringText(second) ?? "")
|
|
939
986
|
? "entry"
|
|
940
987
|
: false);
|
|
@@ -967,7 +1014,7 @@ function browserVersionLiteralMessage(finding) {
|
|
|
967
1014
|
case "sec-ch-ua":
|
|
968
1015
|
return 'Hardcoded sec-ch-ua versions can disagree with the stealth TLS fingerprint. Remove the literal and let ctx.stealth generate client hints from stealth: { browser: "chrome", os: "macos" }. ctx.stealth also owns User-Agent, so omit that header there as well; for ctx.http only, derive it from getStealthProfile({ browser: "chrome", os: "macos" }).userAgent.';
|
|
969
1016
|
case "owned-header":
|
|
970
|
-
return `ctx.stealth owns the "${finding.literal}" header and rejects a caller value with STEALTH_HEADER_OVERRIDE_UNSUPPORTED (HTTP 500 at request time). Remove it; declare stealth: { requestClass: "navigation" | "xhr" | "post" } to drive Sec-Fetch-* and select the profile with stealth: { browser: "chrome", os: "macos" } for client hints.`;
|
|
1017
|
+
return `ctx.stealth owns the "${finding.literal}" header and rejects a caller value with STEALTH_HEADER_OVERRIDE_UNSUPPORTED (HTTP 500 at request time). Remove it; declare stealth: { requestClass: "navigation" | "form-post" | "xhr" | "post" } to drive Sec-Fetch-* and select the profile with stealth: { browser: "chrome", os: "macos" } for client hints.`;
|
|
971
1018
|
}
|
|
972
1019
|
}
|
|
973
1020
|
function lintBrowserVersionLiterals(provider) {
|
|
@@ -41,7 +41,12 @@ export declare function isDiagnosticRedactionSuppressed(redact?: DiagnosticRedac
|
|
|
41
41
|
export declare function diagnosticStructuredRedactor(redact?: DiagnosticRedactor): DiagnosticRedactor | undefined;
|
|
42
42
|
export declare function redactDiagnosticAttributeKey(key: string, redact?: DiagnosticRedactor): string;
|
|
43
43
|
export declare function redactCriticalDiagnosticText(text: string, redact?: DiagnosticRedactor): string;
|
|
44
|
-
|
|
44
|
+
/**
|
|
45
|
+
* The redactor for one attribute value. A value its producer declares as a closed-enum literal for
|
|
46
|
+
* this key is a contract, like a typed number: exhaustion suppression skips it, while registered
|
|
47
|
+
* credentials still match. Any other value under the same key stays free text.
|
|
48
|
+
*/
|
|
49
|
+
export declare function diagnosticAttributeRedactor(key: string, redact?: DiagnosticRedactor, value?: unknown): DiagnosticRedactor | undefined;
|
|
45
50
|
export declare function isDiagnosticBigintAttribute(attributes: object, key: string): boolean;
|
|
46
51
|
export declare function copyDiagnosticAttributeTypes<T extends Record<string, unknown>>(source: Record<string, unknown>, target: T): T;
|
|
47
52
|
/** Reserve existing keys so independently redacted names never drop an attribute. */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
2
|
import { allSensitiveValueVariants, normalizePercentEscapes, REDACTED_QUERY_VALUE, } from "./request-options.js";
|
|
3
|
+
import { RESOLVER_CLOSED_ENUM_SPAN_ATTRIBUTES } from "./resolver-span-attributes.js";
|
|
3
4
|
export const REDACTION_FAILED = "[REDACTION_FAILED]";
|
|
4
5
|
// No supported credential source needs 1–3 character secrets. Registering these
|
|
5
6
|
// common tokens damages unrelated IDs and diagnostics even with token boundaries.
|
|
@@ -492,9 +493,15 @@ export function redactDiagnosticAttributeKey(key, redact) {
|
|
|
492
493
|
export function redactCriticalDiagnosticText(text, redact) {
|
|
493
494
|
return redactDiagnosticText(text, redact ? (criticalRedactors.get(redact) ?? redact) : undefined);
|
|
494
495
|
}
|
|
495
|
-
|
|
496
|
-
|
|
496
|
+
/**
|
|
497
|
+
* The redactor for one attribute value. A value its producer declares as a closed-enum literal for
|
|
498
|
+
* this key is a contract, like a typed number: exhaustion suppression skips it, while registered
|
|
499
|
+
* credentials still match. Any other value under the same key stays free text.
|
|
500
|
+
*/
|
|
501
|
+
export function diagnosticAttributeRedactor(key, redact, value) {
|
|
502
|
+
if (typeof value === "string" && CLOSED_ENUM_ATTRIBUTE_VALUES.get(key)?.has(value)) {
|
|
497
503
|
return diagnosticStructuredRedactor(redact);
|
|
504
|
+
}
|
|
498
505
|
return CRITICAL_ATTRIBUTE_KEYS.has(key) && redact
|
|
499
506
|
? (criticalRedactors.get(redact) ?? redact)
|
|
500
507
|
: redact;
|
|
@@ -512,18 +519,13 @@ const CRITICAL_ATTRIBUTE_KEYS = new Set([
|
|
|
512
519
|
"operation",
|
|
513
520
|
"http.status_code",
|
|
514
521
|
]);
|
|
515
|
-
// SDK-owned closed enums are contracts, like typed numbers and booleans.
|
|
516
|
-
//
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
"cacheStatus",
|
|
523
|
-
"identitySource",
|
|
524
|
-
"taxonomy",
|
|
525
|
-
"taxonomyVersion",
|
|
526
|
-
]);
|
|
522
|
+
// SDK-owned closed enums are contracts, like typed numbers and booleans. The table is the
|
|
523
|
+
// producers' own declaration of which span attributes are closed enums and of every literal each
|
|
524
|
+
// one can carry; it exempts only exhaustion suppression, and registered credentials still match.
|
|
525
|
+
const CLOSED_ENUM_ATTRIBUTE_VALUES = new Map(Object.entries(RESOLVER_CLOSED_ENUM_SPAN_ATTRIBUTES).map(([key, values]) => [
|
|
526
|
+
key,
|
|
527
|
+
new Set(values),
|
|
528
|
+
]));
|
|
527
529
|
// Bigints normalize to strings for OTLP. Carry their type policy through internal
|
|
528
530
|
// detached copies without treating arbitrary numeric-looking free text as typed.
|
|
529
531
|
const bigintAttributes = new WeakMap();
|
package/dist/runtime/http.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { policyRotatesTransportVendorChain, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, } from "../config/loader.js";
|
|
2
|
-
import {
|
|
2
|
+
import { ProviderError, TransportError } from "../errors.js";
|
|
3
3
|
import { parseSseStream, readableBytes, readableLines, readableTextChunks } from "../stream.js";
|
|
4
4
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, isProxyTransportRetryMethod, normalizeProxyTransportRetryOptions, proxyTransportRetryErrorCode, proxyTransportRetryErrorStatus, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
5
5
|
import { httpTelemetryErrorCode, } from "./http-telemetry.js";
|
|
6
|
-
import {
|
|
6
|
+
import { invalidRedirectPolicy, isRedirectStatus, nextSameOriginRedirectHop, normalizeRedirectPolicy, } from "./redirects.js";
|
|
7
7
|
import { normalizeHttpRequestBody, redactSensitiveError, redactSensitiveRequestError, serializeRequestUrl, } from "./request-options.js";
|
|
8
8
|
import { startHttpTelemetry } from "./http-telemetry-guard.js";
|
|
9
9
|
import { scopedHttpTelemetry } from "./http-telemetry-binding.js";
|
|
@@ -409,66 +409,24 @@ function resolveHttpUrl(baseUrl, url) {
|
|
|
409
409
|
return new URL(url, baseUrl ?? DEFAULT_HTTP_BASE_URL).toString();
|
|
410
410
|
}
|
|
411
411
|
const MAX_HTTP_REDIRECT_HOPS = 20;
|
|
412
|
-
const
|
|
412
|
+
const HTTP_REDIRECT_POLICY_TRANSPORT = {
|
|
413
|
+
label: "ctx.http",
|
|
414
|
+
maxHops: MAX_HTTP_REDIRECT_HOPS,
|
|
415
|
+
};
|
|
413
416
|
const REDIRECT_BODY_HEADERS = new Set([
|
|
414
417
|
"content-encoding",
|
|
415
418
|
"content-language",
|
|
416
419
|
"content-location",
|
|
417
420
|
"content-type",
|
|
418
421
|
]);
|
|
419
|
-
const MALFORMED_REDIRECT_TARGET = "[malformed redirect target]";
|
|
420
|
-
function invalidHttpRedirectPolicy(message, cause) {
|
|
421
|
-
return new TransportError(`Invalid ctx.http redirectPolicy: ${message}`, {
|
|
422
|
-
code: "http_redirect_policy_invalid",
|
|
423
|
-
...(cause ? { cause } : {}),
|
|
424
|
-
});
|
|
425
|
-
}
|
|
426
|
-
/** Snapshot untrusted caller input synchronously, before proxy resolution or fetch. */
|
|
427
|
-
function normalizeHttpRedirectPolicy(value) {
|
|
428
|
-
if (value === undefined)
|
|
429
|
-
return undefined;
|
|
430
|
-
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
431
|
-
throw invalidHttpRedirectPolicy("expected an object");
|
|
432
|
-
}
|
|
433
|
-
try {
|
|
434
|
-
const keys = Reflect.ownKeys(value);
|
|
435
|
-
for (const key of keys) {
|
|
436
|
-
if (typeof key !== "string" || !HTTP_REDIRECT_POLICY_FIELDS.has(key)) {
|
|
437
|
-
throw invalidHttpRedirectPolicy(`unknown field ${String(key)}`);
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
for (const field of HTTP_REDIRECT_POLICY_FIELDS) {
|
|
441
|
-
const descriptor = Object.getOwnPropertyDescriptor(value, field);
|
|
442
|
-
if (!descriptor || !("value" in descriptor)) {
|
|
443
|
-
throw invalidHttpRedirectPolicy(`${field} must be an own data property`);
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
const record = value;
|
|
447
|
-
if (record.mode !== "same-origin") {
|
|
448
|
-
throw invalidHttpRedirectPolicy('mode must be "same-origin"');
|
|
449
|
-
}
|
|
450
|
-
if (typeof record.maxHops !== "number" ||
|
|
451
|
-
!Number.isInteger(record.maxHops) ||
|
|
452
|
-
record.maxHops < 0 ||
|
|
453
|
-
record.maxHops > MAX_HTTP_REDIRECT_HOPS) {
|
|
454
|
-
throw invalidHttpRedirectPolicy(`maxHops must be an integer from 0 to ${MAX_HTTP_REDIRECT_HOPS}`);
|
|
455
|
-
}
|
|
456
|
-
return { mode: "same-origin", maxHops: record.maxHops };
|
|
457
|
-
}
|
|
458
|
-
catch (error) {
|
|
459
|
-
if (error instanceof TransportError)
|
|
460
|
-
throw error;
|
|
461
|
-
throw invalidHttpRedirectPolicy("could not be inspected safely", error instanceof Error ? error : undefined);
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
422
|
function snapshotHttpRedirectPolicy(options) {
|
|
465
423
|
try {
|
|
466
|
-
return
|
|
424
|
+
return normalizeRedirectPolicy(options.redirectPolicy, HTTP_REDIRECT_POLICY_TRANSPORT);
|
|
467
425
|
}
|
|
468
426
|
catch (error) {
|
|
469
427
|
if (error instanceof TransportError)
|
|
470
428
|
throw error;
|
|
471
|
-
throw
|
|
429
|
+
throw invalidRedirectPolicy(HTTP_REDIRECT_POLICY_TRANSPORT, "could not be read safely", error instanceof Error ? error : undefined);
|
|
472
430
|
}
|
|
473
431
|
}
|
|
474
432
|
function withoutRedirectBodyHeaders(headers) {
|
|
@@ -477,21 +435,6 @@ function withoutRedirectBodyHeaders(headers) {
|
|
|
477
435
|
nextHeaders.delete(name);
|
|
478
436
|
return nextHeaders;
|
|
479
437
|
}
|
|
480
|
-
function redirectDiagnosticTarget(value) {
|
|
481
|
-
try {
|
|
482
|
-
const parsed = new URL(value);
|
|
483
|
-
// Origin omits URL userinfo. Keeping only origin + path makes diagnostics
|
|
484
|
-
// useful while structurally excluding every query value and fragment,
|
|
485
|
-
// including attacker-chosen keys the provider did not declare sensitive.
|
|
486
|
-
const redactedQuery = parsed.search ? "?[REDACTED]" : "";
|
|
487
|
-
return parsed.origin === "null"
|
|
488
|
-
? `${parsed.protocol}<opaque-target>`
|
|
489
|
-
: `${parsed.origin}${parsed.pathname}${redactedQuery}`;
|
|
490
|
-
}
|
|
491
|
-
catch {
|
|
492
|
-
return MALFORMED_REDIRECT_TARGET;
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
438
|
function discardRedirectResponseBody(response) {
|
|
496
439
|
try {
|
|
497
440
|
const cancellation = response.body?.cancel();
|
|
@@ -532,47 +475,17 @@ async function fetchWithHttpRedirectPolicy(requestUrl, requestInit, policy) {
|
|
|
532
475
|
return response;
|
|
533
476
|
const location = response.headers.get("location");
|
|
534
477
|
discardRedirectResponseBody(response);
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
const target = MALFORMED_REDIRECT_TARGET;
|
|
541
|
-
throw new HttpRedirectError(`Redirect response has malformed Location target ${target}`, {
|
|
542
|
-
reason: "missing_location",
|
|
543
|
-
target,
|
|
544
|
-
status: response.status,
|
|
545
|
-
});
|
|
546
|
-
}
|
|
547
|
-
const decision = evaluateRedirectHop({
|
|
478
|
+
const decision = nextSameOriginRedirectHop({
|
|
479
|
+
policy,
|
|
480
|
+
initialOrigin,
|
|
481
|
+
currentUrl,
|
|
482
|
+
responseUrl: currentUrl,
|
|
548
483
|
status: response.status,
|
|
549
484
|
method,
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
redirectCount: followedHops + 1,
|
|
553
|
-
maxHops: policy.maxHops,
|
|
485
|
+
location: location ?? undefined,
|
|
486
|
+
followedHops,
|
|
554
487
|
visitedRequests,
|
|
555
488
|
});
|
|
556
|
-
if (decision.kind === "stop") {
|
|
557
|
-
const target = decision.nextUrl ? redirectDiagnosticTarget(decision.nextUrl) : undefined;
|
|
558
|
-
const message = (() => {
|
|
559
|
-
switch (decision.reason) {
|
|
560
|
-
case "stopped":
|
|
561
|
-
return `Redirect policy refused cross-origin target ${target}`;
|
|
562
|
-
case "max_hops":
|
|
563
|
-
return `Redirect policy reached maxHops before target ${target}`;
|
|
564
|
-
case "loop":
|
|
565
|
-
return `Redirect loop refused target ${target}`;
|
|
566
|
-
case "missing_location":
|
|
567
|
-
return `Redirect response from ${redirectDiagnosticTarget(currentUrl)} is missing Location`;
|
|
568
|
-
}
|
|
569
|
-
})();
|
|
570
|
-
throw new HttpRedirectError(message, {
|
|
571
|
-
reason: decision.reason,
|
|
572
|
-
...(target ? { target } : {}),
|
|
573
|
-
status: response.status,
|
|
574
|
-
});
|
|
575
|
-
}
|
|
576
489
|
if (decision.nextMethod !== method) {
|
|
577
490
|
body = undefined;
|
|
578
491
|
headers = withoutRedirectBodyHeaders(headers);
|
|
@@ -4,6 +4,7 @@ import { readableBytes, readableLines, readableTextChunks } from "../stream.js";
|
|
|
4
4
|
import { parseHttpRequestInvocation, isSensitiveKey, redactSensitiveError, redactSensitiveText, redactUrlQueryParams, requestOptionsFromHttpInvocation, serializeRequestUrl, } from "./request-options.js";
|
|
5
5
|
import { createTraceContext, getTraceRecorder, } from "./trace.js";
|
|
6
6
|
import { RESOLVER_INSTRUMENTATION_METADATA } from "./resolver-shared.js";
|
|
7
|
+
import { resolverSpanAttributes } from "./resolver-span-attributes.js";
|
|
7
8
|
const BROWSER_PAGE_METHODS = new Set([
|
|
8
9
|
"goto",
|
|
9
10
|
"fill",
|
|
@@ -485,7 +486,9 @@ function wrapNamespace(namespace, target, trace, shouldInstrument) {
|
|
|
485
486
|
? challenge.kind
|
|
486
487
|
: undefined;
|
|
487
488
|
return recorder.runSpan("resolver.solve", () => Reflect.apply(value, namespaceTarget, [args[0], args[1], recorder]), {
|
|
488
|
-
attributes: challengeKind
|
|
489
|
+
attributes: challengeKind
|
|
490
|
+
? resolverSpanAttributes({ challenge_kind: challengeKind })
|
|
491
|
+
: undefined,
|
|
489
492
|
});
|
|
490
493
|
};
|
|
491
494
|
wrappedMethods.set(property, wrapped);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** One redirect hop: the URL the headers were sent to, and the next request. */
|
|
2
|
+
export interface RedirectHop {
|
|
3
|
+
readonly from: string;
|
|
4
|
+
readonly to: string;
|
|
5
|
+
readonly methodChanged: boolean;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* The caller headers the next redirect hop carries, derived from the headers
|
|
9
|
+
* the previous hop sent. Both stealth redirect walkers compute every hop's
|
|
10
|
+
* headers here, so the rules hold on every path:
|
|
11
|
+
*
|
|
12
|
+
* - a method change drops the request-body headers;
|
|
13
|
+
* - a cross-origin hop drops `Authorization` and `Proxy-Authorization`;
|
|
14
|
+
* - a cross-site hop drops an explicit `Cookie`, after which the session
|
|
15
|
+
* cookie jar supplies cookies by its own domain rules, as a browser does.
|
|
16
|
+
*
|
|
17
|
+
* A header dropped on one hop stays dropped for the rest of the chain.
|
|
18
|
+
*/
|
|
19
|
+
export declare function redirectHopHeaders<T>(hop: RedirectHop, headers: Readonly<Record<string, T>>): Record<string, T>;
|