@apifuse/provider-sdk 2.2.0-beta.23 → 2.2.0-beta.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/bin/apifuse-dev.ts +2 -0
- package/bin/apifuse-pack-types.ts +234 -38
- package/bin/apifuse-perf.ts +15 -12
- package/bin/apifuse-record.ts +2 -0
- package/bin/apifuse-submit-check.ts +15 -2
- package/dist/config/loader.d.ts +8 -19
- package/dist/config/loader.js +28 -86
- package/dist/define.d.ts +4 -1
- package/dist/define.js +64 -6
- package/dist/index.d.ts +4 -3
- package/dist/index.js +2 -1
- package/dist/provider.d.ts +1 -1
- package/dist/runtime/auth-flow.js +2 -0
- package/dist/runtime/browser.js +50 -0
- package/dist/runtime/cache.d.ts +1 -0
- package/dist/runtime/cache.js +169 -15
- package/dist/runtime/http.js +0 -1
- package/dist/runtime/instrumentation.js +26 -1
- package/dist/runtime/resolver-vendors/bindings.d.ts +8 -0
- package/dist/runtime/resolver-vendors/bindings.js +15 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +24 -0
- package/dist/runtime/resolver-vendors/browser.js +287 -0
- package/dist/runtime/resolver-vendors/types.d.ts +42 -0
- package/dist/runtime/resolver-vendors/types.js +57 -0
- package/dist/runtime/resolver.d.ts +39 -0
- package/dist/runtime/resolver.js +414 -0
- package/dist/runtime/state.d.ts +3 -0
- package/dist/runtime/state.js +245 -141
- package/dist/runtime/stealth.js +3 -6
- package/dist/server/serve.d.ts +4 -1
- package/dist/server/serve.js +35 -8
- package/dist/testing/run.js +7 -0
- package/dist/types.d.ts +115 -7
- package/package.json +1 -1
- package/src/config/loader.ts +35 -111
- package/src/define.ts +105 -8
- package/src/index.ts +21 -1
- package/src/provider.ts +1 -0
- package/src/runtime/auth-flow.ts +2 -0
- package/src/runtime/browser.ts +69 -0
- package/src/runtime/cache.ts +189 -14
- package/src/runtime/http.ts +0 -1
- package/src/runtime/instrumentation.ts +36 -2
- package/src/runtime/resolver-vendors/bindings.ts +31 -0
- package/src/runtime/resolver-vendors/browser.ts +420 -0
- package/src/runtime/resolver-vendors/types.ts +113 -0
- package/src/runtime/resolver.ts +668 -0
- package/src/runtime/state.ts +323 -166
- package/src/runtime/stealth.ts +3 -6
- package/src/server/serve.ts +73 -5
- package/src/testing/run.ts +8 -0
- package/src/types.ts +133 -7
package/dist/runtime/cache.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
1
|
+
import { createHash, createHmac } from "node:crypto";
|
|
2
2
|
import { providerCacheRedisUrlFromEnv } from "../config/loader.js";
|
|
3
|
+
import { ProviderError } from "../errors.js";
|
|
3
4
|
import { createProviderRedisClient, ensureRedisReady, withRedisTimeout, } from "./redis.js";
|
|
5
|
+
export const APIFUSE__CACHE__KEY_PEPPER_ENV = "APIFUSE__CACHE__KEY_PEPPER";
|
|
4
6
|
const DEFAULT_PREFIX = "apifuse:provider-cache:v1";
|
|
5
7
|
const DEFAULT_MEMORY_MAX_ENTRIES = 1_000;
|
|
6
8
|
const DEFAULT_REDIS_TIMEOUT_MS = 150;
|
|
9
|
+
const SECRET_SCOPED_KEY_MARKER = "[secret-scoped";
|
|
7
10
|
const SECRET_FIELD_NAMES = new Set([
|
|
8
11
|
"authorization",
|
|
9
12
|
"cookie",
|
|
@@ -18,6 +21,7 @@ const SECRET_FIELD_NAMES = new Set([
|
|
|
18
21
|
"refresh_token",
|
|
19
22
|
]);
|
|
20
23
|
const sharedBackends = new Map();
|
|
24
|
+
let warnedAboutUnpepperedSecretKeys = false;
|
|
21
25
|
function backendKey(redisUrl) {
|
|
22
26
|
return redisUrl ?? "memory";
|
|
23
27
|
}
|
|
@@ -55,20 +59,157 @@ function shouldRedactField(name, extra) {
|
|
|
55
59
|
normalized.includes("password") ||
|
|
56
60
|
normalized.includes("secret"));
|
|
57
61
|
}
|
|
58
|
-
function
|
|
62
|
+
function unsupportedSecretValue(path, reason) {
|
|
63
|
+
throw new ProviderError(`Secret cache-key values must be JSON-safe; ${reason} at ${path}.`, {
|
|
64
|
+
code: "CACHE_KEY_SECRET_VALUE_UNSUPPORTED",
|
|
65
|
+
fix: "Convert the secret cache-key selector to JSON-safe primitives, arrays, or plain objects.",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
function assertJsonSafeSecretValue(value, reportedPath, ancestors = new Set()) {
|
|
69
|
+
if (value === undefined)
|
|
70
|
+
return;
|
|
71
|
+
if (value === null || typeof value === "string" || typeof value === "boolean")
|
|
72
|
+
return;
|
|
73
|
+
if (typeof value === "number") {
|
|
74
|
+
if (!Number.isFinite(value))
|
|
75
|
+
unsupportedSecretValue(reportedPath, "non-finite numbers are unsupported");
|
|
76
|
+
if (Object.is(value, -0))
|
|
77
|
+
unsupportedSecretValue(reportedPath, "negative zero is unsupported");
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
if (typeof value !== "object") {
|
|
81
|
+
unsupportedSecretValue(reportedPath, `${typeof value} values are unsupported`);
|
|
82
|
+
}
|
|
83
|
+
if (ancestors.has(value))
|
|
84
|
+
unsupportedSecretValue(reportedPath, "cyclic values are unsupported");
|
|
85
|
+
ancestors.add(value);
|
|
86
|
+
try {
|
|
87
|
+
if (Array.isArray(value)) {
|
|
88
|
+
if (Object.getOwnPropertySymbols(value).length > 0) {
|
|
89
|
+
unsupportedSecretValue(reportedPath, "symbol-keyed array properties are unsupported");
|
|
90
|
+
}
|
|
91
|
+
const expectedNames = new Set(["length"]);
|
|
92
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
93
|
+
const key = String(index);
|
|
94
|
+
expectedNames.add(key);
|
|
95
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
96
|
+
if (!descriptor)
|
|
97
|
+
unsupportedSecretValue(reportedPath, "sparse arrays are unsupported");
|
|
98
|
+
if (!descriptor.enumerable || !("value" in descriptor)) {
|
|
99
|
+
unsupportedSecretValue(reportedPath, "array accessors are unsupported");
|
|
100
|
+
}
|
|
101
|
+
assertJsonSafeSecretValue(descriptor.value, reportedPath, ancestors);
|
|
102
|
+
}
|
|
103
|
+
if (Object.getOwnPropertyNames(value).some((name) => !expectedNames.has(name))) {
|
|
104
|
+
unsupportedSecretValue(reportedPath, "custom array properties are unsupported");
|
|
105
|
+
}
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const prototype = Object.getPrototypeOf(value);
|
|
109
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
110
|
+
unsupportedSecretValue(reportedPath, "non-plain objects are unsupported");
|
|
111
|
+
}
|
|
112
|
+
if (Object.getOwnPropertySymbols(value).length > 0) {
|
|
113
|
+
unsupportedSecretValue(reportedPath, "symbol-keyed properties are unsupported");
|
|
114
|
+
}
|
|
115
|
+
for (const key of Object.getOwnPropertyNames(value)) {
|
|
116
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
117
|
+
if (!descriptor?.enumerable || !("value" in descriptor)) {
|
|
118
|
+
unsupportedSecretValue(reportedPath, "non-enumerable properties and accessors are unsupported");
|
|
119
|
+
}
|
|
120
|
+
assertJsonSafeSecretValue(descriptor.value, reportedPath, ancestors);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
ancestors.delete(value);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function containsUndefined(value) {
|
|
128
|
+
if (value === undefined)
|
|
129
|
+
return true;
|
|
130
|
+
if (Array.isArray(value))
|
|
131
|
+
return value.some(containsUndefined);
|
|
132
|
+
if (isRecord(value))
|
|
133
|
+
return Object.values(value).some(containsUndefined);
|
|
134
|
+
return false;
|
|
135
|
+
}
|
|
136
|
+
function tagSecretValue(value) {
|
|
137
|
+
if (value === undefined)
|
|
138
|
+
return ["undefined"];
|
|
139
|
+
if (value === null)
|
|
140
|
+
return ["null"];
|
|
141
|
+
if (typeof value === "string")
|
|
142
|
+
return ["string", value];
|
|
143
|
+
if (typeof value === "number")
|
|
144
|
+
return ["number", value];
|
|
145
|
+
if (typeof value === "boolean")
|
|
146
|
+
return ["boolean", value];
|
|
147
|
+
if (Array.isArray(value))
|
|
148
|
+
return ["array", value.map(tagSecretValue)];
|
|
149
|
+
return [
|
|
150
|
+
"object",
|
|
151
|
+
Object.entries(value).map(([key, entry]) => [
|
|
152
|
+
key,
|
|
153
|
+
tagSecretValue(entry),
|
|
154
|
+
]),
|
|
155
|
+
];
|
|
156
|
+
}
|
|
157
|
+
function serializeSecretValue(value) {
|
|
158
|
+
if (!containsUndefined(value))
|
|
159
|
+
return JSON.stringify([value]);
|
|
160
|
+
return `undefined-v1:${JSON.stringify(tagSecretValue(value))}`;
|
|
161
|
+
}
|
|
162
|
+
function warnAboutUnpepperedSecretKey() {
|
|
163
|
+
if (warnedAboutUnpepperedSecretKeys)
|
|
164
|
+
return;
|
|
165
|
+
warnedAboutUnpepperedSecretKeys = true;
|
|
166
|
+
console.warn(JSON.stringify({
|
|
167
|
+
level: "warn",
|
|
168
|
+
event: "provider_cache_secret_key_unpeppered",
|
|
169
|
+
message: `Secret-bearing cache keys are using unkeyed SHA-256 because ${APIFUSE__CACHE__KEY_PEPPER_ENV} is not configured.`,
|
|
170
|
+
}));
|
|
171
|
+
}
|
|
172
|
+
function normalizeKeyPart(value, extra, pepper) {
|
|
59
173
|
if (Array.isArray(value)) {
|
|
60
|
-
|
|
174
|
+
const entries = value.map((entry) => normalizeKeyPart(entry, extra, pepper));
|
|
175
|
+
return {
|
|
176
|
+
value: entries.map((entry) => entry.value),
|
|
177
|
+
secretScoped: entries.some((entry) => entry.secretScoped),
|
|
178
|
+
};
|
|
61
179
|
}
|
|
62
180
|
if (isRecord(value)) {
|
|
63
|
-
const normalized =
|
|
181
|
+
const normalized = Object.create(null);
|
|
182
|
+
let secretScoped = false;
|
|
64
183
|
for (const key of Object.keys(value).sort()) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
184
|
+
const part = shouldRedactField(key, extra)
|
|
185
|
+
? hashSecretValue(value[key], key, extra, pepper)
|
|
186
|
+
: normalizeKeyPart(value[key], extra, pepper);
|
|
187
|
+
normalized[key] = part.value;
|
|
188
|
+
secretScoped ||= part.secretScoped;
|
|
68
189
|
}
|
|
69
|
-
return normalized;
|
|
190
|
+
return { value: normalized, secretScoped };
|
|
70
191
|
}
|
|
71
|
-
return value;
|
|
192
|
+
return { value, secretScoped: false };
|
|
193
|
+
}
|
|
194
|
+
function hashSecretValue(value, fieldName, extra, pepper) {
|
|
195
|
+
assertJsonSafeSecretValue(value, `${fieldName} (inside secret value)`);
|
|
196
|
+
const canonical = serializeSecretValue(normalizeKeyPart(value, extra, pepper).value);
|
|
197
|
+
if (pepper === undefined) {
|
|
198
|
+
warnAboutUnpepperedSecretKey();
|
|
199
|
+
const digest = createHash("sha256").update(canonical).digest("hex");
|
|
200
|
+
return { value: `sha256:${digest}`, secretScoped: true };
|
|
201
|
+
}
|
|
202
|
+
const digest = createHmac("sha256", pepper).update(canonical).digest("hex");
|
|
203
|
+
return { value: `hmac-sha256:${digest}`, secretScoped: true };
|
|
204
|
+
}
|
|
205
|
+
function metadataKeys(events, secretScopedKeys) {
|
|
206
|
+
let secretScopedIndex = 0;
|
|
207
|
+
return Array.from(new Set(events.map((event) => event.key))).map((key) => {
|
|
208
|
+
if (!secretScopedKeys.has(key))
|
|
209
|
+
return key;
|
|
210
|
+
secretScopedIndex += 1;
|
|
211
|
+
return `${SECRET_SCOPED_KEY_MARKER}#${secretScopedIndex}]`;
|
|
212
|
+
});
|
|
72
213
|
}
|
|
73
214
|
function stableHash(value) {
|
|
74
215
|
return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 32);
|
|
@@ -138,10 +279,13 @@ async function withRedisFallback(operation) {
|
|
|
138
279
|
}
|
|
139
280
|
export function createProviderCache(options) {
|
|
140
281
|
const redisUrl = options.redisUrl ?? providerCacheRedisUrlFromEnv();
|
|
282
|
+
const configuredPepper = process.env[APIFUSE__CACHE__KEY_PEPPER_ENV];
|
|
283
|
+
const pepper = configuredPepper && configuredPepper.length > 0 ? configuredPepper : undefined;
|
|
141
284
|
const backend = getSharedBackend(redisUrl);
|
|
142
285
|
const memoryMaxEntries = Math.max(1, options.memoryMaxEntries ?? DEFAULT_MEMORY_MAX_ENTRIES);
|
|
143
286
|
const now = options.now ?? Date.now;
|
|
144
287
|
const events = [];
|
|
288
|
+
const secretScopedKeys = new Set();
|
|
145
289
|
function record(meta) {
|
|
146
290
|
events.push(meta);
|
|
147
291
|
}
|
|
@@ -266,8 +410,11 @@ export function createProviderCache(options) {
|
|
|
266
410
|
return {
|
|
267
411
|
key(namespace, parts, keyOptions) {
|
|
268
412
|
const extra = new Set((keyOptions?.redactFields ?? []).map((field) => field.toLowerCase()));
|
|
269
|
-
const normalized = normalizeKeyPart(parts, extra);
|
|
270
|
-
|
|
413
|
+
const normalized = normalizeKeyPart(parts, extra, pepper);
|
|
414
|
+
const key = `${DEFAULT_PREFIX}:${options.providerId}:${namespace}:${stableHash(normalized.value)}`;
|
|
415
|
+
if (normalized.secretScoped)
|
|
416
|
+
secretScopedKeys.add(key);
|
|
417
|
+
return key;
|
|
271
418
|
},
|
|
272
419
|
async get(key) {
|
|
273
420
|
const result = await read(key);
|
|
@@ -312,19 +459,25 @@ export function createProviderCache(options) {
|
|
|
312
459
|
return {
|
|
313
460
|
hit: events.some((event) => event.hit),
|
|
314
461
|
stale: events.some((event) => event.stale),
|
|
315
|
-
keys:
|
|
462
|
+
keys: metadataKeys(events, secretScopedKeys),
|
|
316
463
|
source: sourceSummary(events),
|
|
317
464
|
};
|
|
318
465
|
},
|
|
319
466
|
};
|
|
320
467
|
}
|
|
321
468
|
export function createBypassProviderCache(options) {
|
|
469
|
+
const configuredPepper = process.env[APIFUSE__CACHE__KEY_PEPPER_ENV];
|
|
470
|
+
const pepper = configuredPepper && configuredPepper.length > 0 ? configuredPepper : undefined;
|
|
322
471
|
const events = [];
|
|
472
|
+
const secretScopedKeys = new Set();
|
|
323
473
|
return {
|
|
324
474
|
key(namespace, parts, keyOptions) {
|
|
325
475
|
const extra = new Set((keyOptions?.redactFields ?? []).map((field) => field.toLowerCase()));
|
|
326
|
-
const normalized = normalizeKeyPart(parts, extra);
|
|
327
|
-
|
|
476
|
+
const normalized = normalizeKeyPart(parts, extra, pepper);
|
|
477
|
+
const key = `${DEFAULT_PREFIX}:${options.providerId}:${namespace}:${stableHash(normalized.value)}`;
|
|
478
|
+
if (normalized.secretScoped)
|
|
479
|
+
secretScopedKeys.add(key);
|
|
480
|
+
return key;
|
|
328
481
|
},
|
|
329
482
|
async get(_key) {
|
|
330
483
|
return null;
|
|
@@ -352,7 +505,7 @@ export function createBypassProviderCache(options) {
|
|
|
352
505
|
return {
|
|
353
506
|
hit: false,
|
|
354
507
|
stale: false,
|
|
355
|
-
keys:
|
|
508
|
+
keys: metadataKeys(events, secretScopedKeys),
|
|
356
509
|
source: sourceSummary(events),
|
|
357
510
|
};
|
|
358
511
|
},
|
|
@@ -365,4 +518,5 @@ export function resetProviderCacheForTests() {
|
|
|
365
518
|
backend.redis?.disconnect();
|
|
366
519
|
}
|
|
367
520
|
sharedBackends.clear();
|
|
521
|
+
warnedAboutUnpepperedSecretKeys = false;
|
|
368
522
|
}
|
package/dist/runtime/http.js
CHANGED
|
@@ -376,7 +376,6 @@ async function resolveNativeProxy(options, clientOptions, warn, proxyAttemptOffs
|
|
|
376
376
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
377
377
|
proxy: options.proxy ?? clientOptions.proxy,
|
|
378
378
|
upstream: clientOptions.upstream,
|
|
379
|
-
apifuseConfig: clientOptions.apifuseConfig,
|
|
380
379
|
proxyPolicy: clientOptions.proxyPolicy,
|
|
381
380
|
affinityKey: clientOptions.affinityKey,
|
|
382
381
|
proxyAttempt: computeProxyAttemptIndex({
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { readableBytes, readableLines, readableTextChunks } from "../stream.js";
|
|
2
2
|
import { parseHttpRequestInvocation, isSensitiveKey, redactSensitiveError, redactSensitiveText, redactUrlQueryParams, requestOptionsFromHttpInvocation, serializeRequestUrl, } from "./request-options.js";
|
|
3
3
|
import { createTraceContext, getTraceRecorder, } from "./trace.js";
|
|
4
|
+
import { RESOLVER_INSTRUMENTATION_METADATA } from "./resolver.js";
|
|
4
5
|
const BROWSER_PAGE_METHODS = new Set(["goto", "fill", "click", "type", "waitForSelector"]);
|
|
5
6
|
const DIAGNOSTIC_BASE_URL = "http://apifuse-instrumentation.invalid";
|
|
6
7
|
function isThenable(value) {
|
|
@@ -420,12 +421,35 @@ function wrapNamespace(namespace, target, trace, shouldInstrument) {
|
|
|
420
421
|
return target;
|
|
421
422
|
}
|
|
422
423
|
const wrappedMethods = new Map();
|
|
424
|
+
const resolverMetadata = namespace === "resolver" ? { target, traceRecorder: recorder } : undefined;
|
|
423
425
|
return new Proxy(target, {
|
|
424
426
|
get(namespaceTarget, property, receiver) {
|
|
427
|
+
if (property === RESOLVER_INSTRUMENTATION_METADATA && resolverMetadata) {
|
|
428
|
+
return resolverMetadata;
|
|
429
|
+
}
|
|
425
430
|
const value = Reflect.get(namespaceTarget, property, receiver);
|
|
426
431
|
if (typeof value !== "function" || property === "constructor") {
|
|
427
432
|
return value;
|
|
428
433
|
}
|
|
434
|
+
if (namespace === "resolver" && property === "solve") {
|
|
435
|
+
if (wrappedMethods.has(property)) {
|
|
436
|
+
return wrappedMethods.get(property);
|
|
437
|
+
}
|
|
438
|
+
const wrapped = (...args) => {
|
|
439
|
+
const challenge = args[0];
|
|
440
|
+
const challengeKind = typeof challenge === "object" &&
|
|
441
|
+
challenge !== null &&
|
|
442
|
+
"kind" in challenge &&
|
|
443
|
+
typeof challenge.kind === "string"
|
|
444
|
+
? challenge.kind
|
|
445
|
+
: undefined;
|
|
446
|
+
return recorder.runSpan("resolver.solve", () => Reflect.apply(value, namespaceTarget, [args[0], args[1], recorder]), {
|
|
447
|
+
attributes: challengeKind ? { challenge_kind: challengeKind } : undefined,
|
|
448
|
+
});
|
|
449
|
+
};
|
|
450
|
+
wrappedMethods.set(property, wrapped);
|
|
451
|
+
return wrapped;
|
|
452
|
+
}
|
|
429
453
|
if (namespace === "browser" && property === "newPage") {
|
|
430
454
|
if (wrappedMethods.has(property)) {
|
|
431
455
|
return wrappedMethods.get(property);
|
|
@@ -573,7 +597,8 @@ export function wrapWithInstrumentation(ctx, options = {}) {
|
|
|
573
597
|
property === "stealth" ||
|
|
574
598
|
property === "browser" ||
|
|
575
599
|
property === "session" ||
|
|
576
|
-
property === "state"
|
|
600
|
+
property === "state" ||
|
|
601
|
+
property === "resolver") {
|
|
577
602
|
const namespace = property;
|
|
578
603
|
if (wrappedTargets.has(namespace)) {
|
|
579
604
|
return wrappedTargets.get(namespace);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ProviderChallenge } from "../../types.js";
|
|
2
|
+
import type { ResolverIssuingIdentity } from "./types.js";
|
|
3
|
+
export declare const RESOLVER_CHALLENGE_BINDINGS: {
|
|
4
|
+
readonly aws_waf: "portable";
|
|
5
|
+
readonly cloudflare_interstitial: "identity_scoped";
|
|
6
|
+
};
|
|
7
|
+
export declare function resolverChallengeIsIdentityScoped(challenge: ProviderChallenge): boolean;
|
|
8
|
+
export declare function resolverChallengeIssuingIdentity(challenge: ProviderChallenge, identity: ResolverIssuingIdentity): ResolverIssuingIdentity;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export const RESOLVER_CHALLENGE_BINDINGS = {
|
|
2
|
+
aws_waf: "portable",
|
|
3
|
+
cloudflare_interstitial: "identity_scoped",
|
|
4
|
+
};
|
|
5
|
+
export function resolverChallengeIsIdentityScoped(challenge) {
|
|
6
|
+
return (RESOLVER_CHALLENGE_BINDINGS[challenge.kind] ===
|
|
7
|
+
"identity_scoped");
|
|
8
|
+
}
|
|
9
|
+
export function resolverChallengeIssuingIdentity(challenge, identity) {
|
|
10
|
+
const binding = RESOLVER_CHALLENGE_BINDINGS[challenge.kind];
|
|
11
|
+
if (binding === "portable") {
|
|
12
|
+
return { userAgent: identity.userAgent };
|
|
13
|
+
}
|
|
14
|
+
return identity;
|
|
15
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { BrowserClient, ChallengeSolution, ProviderChallenge } from "../../types.js";
|
|
2
|
+
import { type BrowserClientOptions } from "../browser.js";
|
|
3
|
+
import type { TraceRecorder } from "../trace.js";
|
|
4
|
+
import { type ResolverIdentity, type ResolverVendorAdapter } from "./types.js";
|
|
5
|
+
type BrowserClientFactory = (options: BrowserClientOptions) => BrowserClient;
|
|
6
|
+
export interface BrowserResolverVendorOptions {
|
|
7
|
+
readonly cdpUrl?: string;
|
|
8
|
+
readonly timeoutMs: number;
|
|
9
|
+
readonly pollIntervalMs?: number;
|
|
10
|
+
readonly allowedHosts: readonly string[];
|
|
11
|
+
readonly createClient?: BrowserClientFactory;
|
|
12
|
+
}
|
|
13
|
+
export type BrowserResolverSolution = Extract<ChallengeSolution, {
|
|
14
|
+
readonly form: "cookies";
|
|
15
|
+
}> & {
|
|
16
|
+
/** Unix seconds from the cookie that proved the challenge cleared. */
|
|
17
|
+
readonly expires?: number;
|
|
18
|
+
};
|
|
19
|
+
export interface BrowserResolverVendorAdapter extends ResolverVendorAdapter {
|
|
20
|
+
readonly id: "browser";
|
|
21
|
+
solve(challenge: ProviderChallenge, identity: ResolverIdentity | undefined, signal: AbortSignal, traceRecorder?: TraceRecorder): Promise<BrowserResolverSolution>;
|
|
22
|
+
}
|
|
23
|
+
export declare function createBrowserResolverVendorAdapter(options: BrowserResolverVendorOptions): BrowserResolverVendorAdapter;
|
|
24
|
+
export {};
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
import { isProviderError, ProviderError } from "../../errors.js";
|
|
2
|
+
import { createBrowserClient } from "../browser.js";
|
|
3
|
+
import { resolverChallengeIssuingIdentity } from "./bindings.js";
|
|
4
|
+
import { ResolverVendorUnavailableError, } from "./types.js";
|
|
5
|
+
const BROWSER_VENDOR_ID = "browser";
|
|
6
|
+
const DEFAULT_COOKIE_POLL_INTERVAL_MS = 100;
|
|
7
|
+
const SUCCESS_COOKIE_NAMES = {
|
|
8
|
+
aws_waf: "aws-waf-token",
|
|
9
|
+
cloudflare_interstitial: "cf_clearance",
|
|
10
|
+
};
|
|
11
|
+
class BrowserSolveTimeoutError extends Error {
|
|
12
|
+
constructor() {
|
|
13
|
+
super("Browser resolver solve budget elapsed");
|
|
14
|
+
this.name = "BrowserSolveTimeoutError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
class BrowserCleanupTimeoutError extends Error {
|
|
18
|
+
constructor(timeoutMs) {
|
|
19
|
+
super(`Browser resolver cleanup exceeded ${timeoutMs}ms`);
|
|
20
|
+
this.name = "BrowserCleanupTimeoutError";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function abortReason(signal) {
|
|
24
|
+
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
|
25
|
+
}
|
|
26
|
+
function raceWithAbort(operation, signal) {
|
|
27
|
+
if (signal.aborted) {
|
|
28
|
+
return Promise.reject(abortReason(signal));
|
|
29
|
+
}
|
|
30
|
+
return new Promise((resolve, reject) => {
|
|
31
|
+
const cleanup = () => signal.removeEventListener("abort", onAbort);
|
|
32
|
+
const onAbort = () => {
|
|
33
|
+
cleanup();
|
|
34
|
+
reject(abortReason(signal));
|
|
35
|
+
};
|
|
36
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
37
|
+
operation().then((value) => {
|
|
38
|
+
cleanup();
|
|
39
|
+
resolve(value);
|
|
40
|
+
}, (error) => {
|
|
41
|
+
cleanup();
|
|
42
|
+
reject(error);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
async function runBoundedCleanup(cleanup, timeoutMs, challengeKind, operation, traceRecorder) {
|
|
47
|
+
let timeout;
|
|
48
|
+
const boundedCleanup = async () => {
|
|
49
|
+
try {
|
|
50
|
+
await Promise.race([
|
|
51
|
+
cleanup(),
|
|
52
|
+
new Promise((_resolve, reject) => {
|
|
53
|
+
timeout = setTimeout(() => reject(new BrowserCleanupTimeoutError(timeoutMs)), timeoutMs);
|
|
54
|
+
}),
|
|
55
|
+
]);
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
if (timeout !== undefined)
|
|
59
|
+
clearTimeout(timeout);
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
if (!traceRecorder) {
|
|
63
|
+
await boundedCleanup().catch(() => undefined);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
await traceRecorder
|
|
67
|
+
.runSpan("resolver.vendor.cleanup", boundedCleanup, {
|
|
68
|
+
attributes: {
|
|
69
|
+
vendor: BROWSER_VENDOR_ID,
|
|
70
|
+
challenge_kind: challengeKind,
|
|
71
|
+
operation,
|
|
72
|
+
},
|
|
73
|
+
onError(error) {
|
|
74
|
+
return {
|
|
75
|
+
error_message: error instanceof Error ? error.message : String(error),
|
|
76
|
+
...(error instanceof Error && error.stack ? { error_stack: error.stack } : {}),
|
|
77
|
+
};
|
|
78
|
+
},
|
|
79
|
+
})
|
|
80
|
+
.catch(() => undefined);
|
|
81
|
+
}
|
|
82
|
+
async function abortableDelay(ms, signal) {
|
|
83
|
+
let timer;
|
|
84
|
+
try {
|
|
85
|
+
await raceWithAbort(() => new Promise((resolve) => {
|
|
86
|
+
timer = setTimeout(resolve, ms);
|
|
87
|
+
}), signal);
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
if (timer !== undefined)
|
|
91
|
+
clearTimeout(timer);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function isSupportedKind(kind) {
|
|
95
|
+
return Object.hasOwn(SUCCESS_COOKIE_NAMES, kind);
|
|
96
|
+
}
|
|
97
|
+
function normalizedHostname(hostname) {
|
|
98
|
+
return hostname.trim().toLowerCase().replace(/\.$/, "");
|
|
99
|
+
}
|
|
100
|
+
function assertChallengeHostAllowed(pageUrl, allowedHosts) {
|
|
101
|
+
const challengeHost = normalizedHostname(new URL(pageUrl).hostname);
|
|
102
|
+
const isAllowed = allowedHosts.some((host) => {
|
|
103
|
+
const declaredHost = normalizedHostname(host);
|
|
104
|
+
return declaredHost.length > 0 && !declaredHost.includes("*") && declaredHost === challengeHost;
|
|
105
|
+
});
|
|
106
|
+
if (isAllowed)
|
|
107
|
+
return;
|
|
108
|
+
throw new ProviderError(`Resolver challenge host "${challengeHost}" is not declared`, {
|
|
109
|
+
code: "RESOLVER_HOST_NOT_ALLOWED",
|
|
110
|
+
fix: "Add the exact challenge hostname to the provider's allowedHosts declaration.",
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
function cookieDomainSpecificity(cookie) {
|
|
114
|
+
return normalizedHostname(cookie.domain.replace(/^\./, "")).length;
|
|
115
|
+
}
|
|
116
|
+
function isHostOnlyCookieFor(cookie, hostname) {
|
|
117
|
+
return (!cookie.domain.startsWith(".") &&
|
|
118
|
+
normalizedHostname(cookie.domain) === normalizedHostname(hostname));
|
|
119
|
+
}
|
|
120
|
+
function cookieAppliesToUrl(cookie, url) {
|
|
121
|
+
const cookieDomain = normalizedHostname(cookie.domain.replace(/^\./, ""));
|
|
122
|
+
const requestHostname = normalizedHostname(url.hostname);
|
|
123
|
+
const domainMatches = cookieDomain.length > 0 &&
|
|
124
|
+
(requestHostname === cookieDomain ||
|
|
125
|
+
(cookie.domain.startsWith(".") && requestHostname.endsWith(`.${cookieDomain}`)));
|
|
126
|
+
if (!domainMatches || (cookie.secure && url.protocol !== "https:"))
|
|
127
|
+
return false;
|
|
128
|
+
const requestPath = url.pathname || "/";
|
|
129
|
+
const cookiePath = cookie.path;
|
|
130
|
+
return (cookiePath.startsWith("/") &&
|
|
131
|
+
(requestPath === cookiePath ||
|
|
132
|
+
(requestPath.startsWith(cookiePath) &&
|
|
133
|
+
(cookiePath.endsWith("/") || requestPath[cookiePath.length] === "/"))));
|
|
134
|
+
}
|
|
135
|
+
function selectSuccessCookie(cookies, successCookieName, pageUrl) {
|
|
136
|
+
const url = new URL(pageUrl);
|
|
137
|
+
return cookies
|
|
138
|
+
.filter((cookie) => cookie.name === successCookieName && cookieAppliesToUrl(cookie, url))
|
|
139
|
+
.sort((left, right) => Number(isHostOnlyCookieFor(right, url.hostname)) -
|
|
140
|
+
Number(isHostOnlyCookieFor(left, url.hostname)) ||
|
|
141
|
+
cookieDomainSpecificity(right) - cookieDomainSpecificity(left) ||
|
|
142
|
+
right.path.length - left.path.length)[0];
|
|
143
|
+
}
|
|
144
|
+
async function solveInPage(page, pageUrl, successCookieName, pollIntervalMs, signal) {
|
|
145
|
+
await raceWithAbort(() => page.goto(pageUrl), signal);
|
|
146
|
+
while (true) {
|
|
147
|
+
const cookies = await raceWithAbort(() => page.cookies(), signal);
|
|
148
|
+
const successCookie = selectSuccessCookie(cookies, successCookieName, pageUrl);
|
|
149
|
+
if (successCookie) {
|
|
150
|
+
const userAgent = await raceWithAbort(() => page.evaluate("navigator.userAgent"), signal);
|
|
151
|
+
return {
|
|
152
|
+
form: "cookies",
|
|
153
|
+
cookies: { [successCookieName]: successCookie.value },
|
|
154
|
+
userAgent,
|
|
155
|
+
...(successCookie.expires === undefined ? {} : { expires: successCookie.expires }),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
await abortableDelay(pollIntervalMs, signal);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const POOL_ALLOCATION_EXHAUSTED_CODES = new Set([
|
|
162
|
+
-32_001, // queue full
|
|
163
|
+
-32_002, // acquire timed out
|
|
164
|
+
-32_003, // shutting down
|
|
165
|
+
]);
|
|
166
|
+
function poolErrorCode(error) {
|
|
167
|
+
const code = error.code;
|
|
168
|
+
return typeof code === "number" ? code : undefined;
|
|
169
|
+
}
|
|
170
|
+
function knownUnavailableReason(error) {
|
|
171
|
+
// Source-grounded mappings:
|
|
172
|
+
// - apps/cdp-pool/src/index.ts: the JSON-RPC codes and messages below.
|
|
173
|
+
// - src/runtime/browser.ts: BROWSER_CDP_POOL_REQUIRED and the two WebSocket messages.
|
|
174
|
+
// The pool's numeric JSON-RPC code is authoritative and is preferred whenever present.
|
|
175
|
+
// The message substrings remain as a fallback for pool builds predating code
|
|
176
|
+
// propagation; they are exact strings verified against the pool source. -32004 (unknown
|
|
177
|
+
// lease) and -32006 (missing allowedHosts) are deliberately unmapped: both are caller
|
|
178
|
+
// bugs, and the next vendor would fail identically, so they propagate unchanged.
|
|
179
|
+
if (isProviderError(error)) {
|
|
180
|
+
return error.code === "BROWSER_CDP_POOL_REQUIRED" ? "missing_credentials" : undefined;
|
|
181
|
+
}
|
|
182
|
+
if (!(error instanceof Error))
|
|
183
|
+
return undefined;
|
|
184
|
+
const code = poolErrorCode(error);
|
|
185
|
+
if (code !== undefined) {
|
|
186
|
+
return POOL_ALLOCATION_EXHAUSTED_CODES.has(code) ? "allocation_exhausted" : undefined;
|
|
187
|
+
}
|
|
188
|
+
if (error.message.includes("CDP pool acquire queue is full") ||
|
|
189
|
+
error.message.includes("CDP pool acquire timed out") ||
|
|
190
|
+
error.message.includes("CDP pool is shutting down")) {
|
|
191
|
+
return "allocation_exhausted";
|
|
192
|
+
}
|
|
193
|
+
if (error.message.includes("Unable to connect to WebSocket endpoint") ||
|
|
194
|
+
error.message.includes("WebSocket closed")) {
|
|
195
|
+
return "transport_failure";
|
|
196
|
+
}
|
|
197
|
+
return undefined;
|
|
198
|
+
}
|
|
199
|
+
async function closeBrowserClient(client, timeoutMs, challengeKind, traceRecorder) {
|
|
200
|
+
const close = client?.close;
|
|
201
|
+
if (!close)
|
|
202
|
+
return;
|
|
203
|
+
await runBoundedCleanup(() => close.call(client), timeoutMs, challengeKind, "client.close", traceRecorder);
|
|
204
|
+
}
|
|
205
|
+
export function createBrowserResolverVendorAdapter(options) {
|
|
206
|
+
const createClient = options.createClient ?? createBrowserClient;
|
|
207
|
+
const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_COOKIE_POLL_INTERVAL_MS;
|
|
208
|
+
return {
|
|
209
|
+
id: BROWSER_VENDOR_ID,
|
|
210
|
+
supports(kind) {
|
|
211
|
+
return isSupportedKind(kind);
|
|
212
|
+
},
|
|
213
|
+
getIssuingIdentity(solution, requestedIdentity, challenge) {
|
|
214
|
+
if (solution.form !== "cookies" || !isSupportedKind(challenge.kind))
|
|
215
|
+
return undefined;
|
|
216
|
+
return resolverChallengeIssuingIdentity(challenge, {
|
|
217
|
+
...(requestedIdentity ? { proxyUrl: requestedIdentity.proxyUrl } : {}),
|
|
218
|
+
userAgent: solution.userAgent,
|
|
219
|
+
});
|
|
220
|
+
},
|
|
221
|
+
async solve(challenge, identity, callerSignal, traceRecorder) {
|
|
222
|
+
void identity;
|
|
223
|
+
if (!options.cdpUrl?.trim()) {
|
|
224
|
+
throw new ResolverVendorUnavailableError(BROWSER_VENDOR_ID, "missing_credentials");
|
|
225
|
+
}
|
|
226
|
+
if (!isSupportedKind(challenge.kind)) {
|
|
227
|
+
throw new TypeError(`Browser resolver does not support ${challenge.kind}`);
|
|
228
|
+
}
|
|
229
|
+
assertChallengeHostAllowed(challenge.pageUrl, options.allowedHosts);
|
|
230
|
+
const challengeKind = challenge.kind;
|
|
231
|
+
callerSignal.throwIfAborted();
|
|
232
|
+
const solveController = new AbortController();
|
|
233
|
+
const onCallerAbort = () => solveController.abort(abortReason(callerSignal));
|
|
234
|
+
callerSignal.addEventListener("abort", onCallerAbort, { once: true });
|
|
235
|
+
const timeout = setTimeout(() => solveController.abort(new BrowserSolveTimeoutError()), options.timeoutMs);
|
|
236
|
+
let client;
|
|
237
|
+
let handlerEntered = false;
|
|
238
|
+
try {
|
|
239
|
+
client = createClient({
|
|
240
|
+
allowedHosts: [...options.allowedHosts],
|
|
241
|
+
cdpUrl: options.cdpUrl.trim(),
|
|
242
|
+
requireCdpPool: true,
|
|
243
|
+
});
|
|
244
|
+
const contextOperation = client.withIsolatedContext(async (page) => {
|
|
245
|
+
handlerEntered = true;
|
|
246
|
+
return await solveInPage(page, challenge.pageUrl, SUCCESS_COOKIE_NAMES[challengeKind], pollIntervalMs, solveController.signal);
|
|
247
|
+
});
|
|
248
|
+
try {
|
|
249
|
+
return await raceWithAbort(() => contextOperation, solveController.signal);
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
if (solveController.signal.aborted) {
|
|
253
|
+
if (handlerEntered) {
|
|
254
|
+
await runBoundedCleanup(() => contextOperation, options.timeoutMs, challengeKind, "context.close", traceRecorder);
|
|
255
|
+
throw error;
|
|
256
|
+
}
|
|
257
|
+
await closeBrowserClient(client, options.timeoutMs, challengeKind, traceRecorder);
|
|
258
|
+
void contextOperation.catch(() => undefined);
|
|
259
|
+
}
|
|
260
|
+
throw error;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
catch (error) {
|
|
264
|
+
if (callerSignal.aborted)
|
|
265
|
+
throw abortReason(callerSignal);
|
|
266
|
+
if (error instanceof BrowserSolveTimeoutError) {
|
|
267
|
+
throw new ResolverVendorUnavailableError(BROWSER_VENDOR_ID, "timeout", {
|
|
268
|
+
cause: error,
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
if (error instanceof ResolverVendorUnavailableError) {
|
|
272
|
+
throw error;
|
|
273
|
+
}
|
|
274
|
+
const reason = knownUnavailableReason(error);
|
|
275
|
+
if (reason) {
|
|
276
|
+
throw new ResolverVendorUnavailableError(BROWSER_VENDOR_ID, reason, { cause: error });
|
|
277
|
+
}
|
|
278
|
+
throw error;
|
|
279
|
+
}
|
|
280
|
+
finally {
|
|
281
|
+
clearTimeout(timeout);
|
|
282
|
+
callerSignal.removeEventListener("abort", onCallerAbort);
|
|
283
|
+
await closeBrowserClient(client, options.timeoutMs, challengeKind, traceRecorder);
|
|
284
|
+
}
|
|
285
|
+
},
|
|
286
|
+
};
|
|
287
|
+
}
|