@apifuse/provider-sdk 2.2.0-beta.27 → 2.2.0-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/bin/apifuse-dev.ts +34 -5
- package/bin/apifuse-pack-smoke.ts +1 -1
- package/bin/apifuse-pack-types.ts +26 -2
- package/bin/apifuse-perf.ts +2 -4
- package/bin/apifuse-record.ts +39 -6
- package/dist/auth-turn/index.d.ts +1 -1
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +38 -0
- package/dist/ceremonies/index.js +52 -11
- package/dist/config/loader.d.ts +3 -1
- package/dist/config/loader.js +4 -2
- package/dist/index.d.ts +8 -7
- package/dist/index.js +5 -7
- package/dist/provider.d.ts +2 -1
- package/dist/provider.js +1 -1
- package/dist/runtime/auth-flow.js +1 -1
- package/dist/runtime/browser.js +45 -2
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +135 -12
- package/dist/runtime/instrumentation.js +1 -1
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +2 -33
- package/dist/runtime/native-network.js +2 -68
- package/dist/runtime/proxy-telemetry.js +3 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-public.d.ts +1 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-shared.d.ts +3 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/browser.js +14 -4
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +2 -1
- package/dist/runtime/resolver-vendors/twocaptcha.js +157 -53
- package/dist/runtime/resolver-vendors/types.d.ts +2 -2
- package/dist/runtime/resolver-vendors/types.js +3 -1
- package/dist/runtime/resolver.d.ts +15 -10
- package/dist/runtime/resolver.js +92 -23
- package/dist/runtime/state.js +5 -115
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.d.ts +1 -0
- package/dist/runtime/stealth.js +8 -131
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.js +1 -1
- package/dist/server/self-test.d.ts +13 -0
- package/dist/server/self-test.js +124 -46
- package/dist/server/serve-implementation.d.ts +199 -0
- package/dist/server/serve-implementation.js +2072 -0
- package/dist/server/serve.d.ts +1 -187
- package/dist/server/serve.js +1 -1827
- package/dist/stateful/errors.d.ts +5 -0
- package/dist/stateful/errors.js +10 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +1 -5
- package/dist/stateful/stateful-provider-session-routing.js +2 -10
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/index.js +1 -0
- package/package.json +27 -2
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +78 -0
- package/src/ceremonies/index.ts +68 -18
- package/src/config/loader.ts +8 -2
- package/src/index.ts +18 -24
- package/src/provider.ts +12 -14
- package/src/runtime/auth-flow.ts +1 -1
- package/src/runtime/browser.ts +50 -2
- package/src/runtime/http.ts +155 -11
- package/src/runtime/instrumentation.ts +1 -1
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +16 -97
- package/src/runtime/proxy-telemetry.ts +5 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-public.ts +18 -0
- package/src/runtime/resolver-shared.ts +17 -0
- package/src/runtime/resolver-vendors/browser.ts +14 -4
- package/src/runtime/resolver-vendors/twocaptcha.ts +190 -56
- package/src/runtime/resolver-vendors/types.ts +8 -2
- package/src/runtime/resolver.ts +140 -28
- package/src/runtime/state.ts +5 -144
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +15 -158
- package/src/serve.ts +6 -1
- package/src/server/index.ts +1 -0
- package/src/server/self-test.ts +184 -59
- package/src/server/serve-implementation.ts +3042 -0
- package/src/server/serve.ts +1 -2661
- package/src/stateful/errors.ts +12 -0
- package/src/stateful/stateful-provider-session-routing.ts +2 -11
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +1 -0
package/dist/runtime/resolver.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { resolveProxyConfigAsync, } from "../config/loader.js";
|
|
2
3
|
import { ProviderError } from "../errors.js";
|
|
4
|
+
import { getStealthProfile } from "../stealth/profiles.js";
|
|
3
5
|
import { resolverChallengeAllowsDirectCache, resolverChallengeIsCacheable, resolverChallengeIsIdentityScoped, resolverChallengeIssuingIdentity, } from "./resolver-vendors/bindings.js";
|
|
4
6
|
import { createBrowserResolverVendorAdapter } from "./resolver-vendors/browser.js";
|
|
5
7
|
import { assertResolverHostAllowed } from "./resolver-vendors/hosts.js";
|
|
6
8
|
import { createTwoCaptchaResolverVendorAdapter } from "./resolver-vendors/twocaptcha.js";
|
|
7
9
|
import { RESOLVER_VENDOR_CAPABILITIES, ResolverVendorUnavailableError, resolverVendorSupports, } from "./resolver-vendors/types.js";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
export
|
|
12
|
-
export
|
|
13
|
-
export const DEFAULT_RESOLVER_TIMEOUT_MS = 180_000;
|
|
10
|
+
import { createUnsupportedResolverClient, RESOLVER_INSTRUMENTATION_METADATA, } from "./resolver-shared.js";
|
|
11
|
+
import { APIFUSE__CDP_POOL__URL, APIFUSE__RESOLVER__2CAPTCHA__API_KEY, APIFUSE__RESOLVER__CAPMONSTER__API_KEY, APIFUSE__RESOLVER__CAPSOLVER__API_KEY, APIFUSE__RESOLVER__TIMEOUT_MS, DEFAULT_RESOLVER_TIMEOUT_MS, } from "./resolver-config.js";
|
|
12
|
+
import { DEFAULT_PROFILE } from "./stealth.js";
|
|
13
|
+
export { createUnsupportedResolverClient, RESOLVER_INSTRUMENTATION_METADATA, } from "./resolver-shared.js";
|
|
14
|
+
export { APIFUSE__CDP_POOL__URL, APIFUSE__RESOLVER__2CAPTCHA__API_KEY, APIFUSE__RESOLVER__CAPMONSTER__API_KEY, APIFUSE__RESOLVER__CAPSOLVER__API_KEY, APIFUSE__RESOLVER__TIMEOUT_MS, DEFAULT_RESOLVER_TIMEOUT_MS, } from "./resolver-config.js";
|
|
14
15
|
const RESOLVER_SOLUTION_CACHE_NAMESPACE = "resolver-solution";
|
|
15
16
|
const RESOLVER_SOLUTION_INDEX_CACHE_NAMESPACE = "resolver-solution-index";
|
|
16
17
|
const MIN_RESOLVER_CACHE_TTL_MS = 1_000;
|
|
@@ -52,7 +53,6 @@ const SAFE_CAUSE_MESSAGE_WORDS = new Set([
|
|
|
52
53
|
"while",
|
|
53
54
|
"writing",
|
|
54
55
|
]);
|
|
55
|
-
export const RESOLVER_INSTRUMENTATION_METADATA = Symbol.for("@apifuse/provider-sdk/runtime/resolver-instrumentation-metadata");
|
|
56
56
|
const resolverAdapterRegistry = {
|
|
57
57
|
"2captcha"(configuration, timeoutMs, allowedHosts) {
|
|
58
58
|
return createTwoCaptchaResolverVendorAdapter({
|
|
@@ -87,6 +87,20 @@ export function swapResolverAdapterFactoryForTests(vendor, factory) {
|
|
|
87
87
|
resolverAdapterRegistry[vendor] = original;
|
|
88
88
|
};
|
|
89
89
|
}
|
|
90
|
+
let resolveDefaultResolverUserAgent = () => getStealthProfile(DEFAULT_PROFILE).userAgent;
|
|
91
|
+
/** Internal test seam; deliberately not re-exported from the package root. */
|
|
92
|
+
export function swapResolverDefaultUserAgentForTests(resolver) {
|
|
93
|
+
const original = resolveDefaultResolverUserAgent;
|
|
94
|
+
resolveDefaultResolverUserAgent =
|
|
95
|
+
resolver ?? (() => getStealthProfile(DEFAULT_PROFILE).userAgent);
|
|
96
|
+
let restored = false;
|
|
97
|
+
return () => {
|
|
98
|
+
if (restored)
|
|
99
|
+
return;
|
|
100
|
+
restored = true;
|
|
101
|
+
resolveDefaultResolverUserAgent = original;
|
|
102
|
+
};
|
|
103
|
+
}
|
|
90
104
|
// This is the sole allowlist for declared vendors whose registry entry may be absent.
|
|
91
105
|
// Remove a vendor here when its adapter is registered.
|
|
92
106
|
const KNOWN_UNIMPLEMENTED_RESOLVER_VENDORS = new Set([
|
|
@@ -175,6 +189,8 @@ function adapterRequiresTransport(adapter, kind) {
|
|
|
175
189
|
function sanitizeDiagnosticUrl(rawUrl) {
|
|
176
190
|
try {
|
|
177
191
|
const parsed = new URL(rawUrl);
|
|
192
|
+
if (parsed.username || parsed.password)
|
|
193
|
+
return "[REDACTED_PROXY_URL]";
|
|
178
194
|
return `${parsed.protocol}//${parsed.host}`;
|
|
179
195
|
}
|
|
180
196
|
catch {
|
|
@@ -190,10 +206,14 @@ function sanitizeCauseMessage(message) {
|
|
|
190
206
|
.split(/\s+/)
|
|
191
207
|
.filter(Boolean)
|
|
192
208
|
.map((token) => {
|
|
193
|
-
if (token === "[REDACTED]" ||
|
|
209
|
+
if (token === "[REDACTED]" ||
|
|
210
|
+
token === "[REDACTED_PROXY_URL]" ||
|
|
211
|
+
/^[a-z][a-z\d+.-]*:\/\/[^\s]+$/i.test(token))
|
|
194
212
|
return token;
|
|
195
213
|
const word = token.replace(/^[^a-z\d]+|[^a-z\d]+$/gi, "");
|
|
196
|
-
return word.length > 0 &&
|
|
214
|
+
return word.length > 0 &&
|
|
215
|
+
word.length <= 32 &&
|
|
216
|
+
SAFE_CAUSE_MESSAGE_WORDS.has(word.toLowerCase())
|
|
197
217
|
? token
|
|
198
218
|
: "[REDACTED]";
|
|
199
219
|
});
|
|
@@ -450,6 +470,47 @@ export async function invalidateResolverSolution(resolver, challenge, solution)
|
|
|
450
470
|
onSuccess: (outcome) => ({ outcome }),
|
|
451
471
|
});
|
|
452
472
|
}
|
|
473
|
+
async function resolveResolverIdentity(proxyIntent) {
|
|
474
|
+
const userAgentSource = proxyIntent.userAgent ? "declared" : "defaulted";
|
|
475
|
+
let proxyUrl;
|
|
476
|
+
try {
|
|
477
|
+
const resolved = await resolveProxyConfigAsync({
|
|
478
|
+
upstream: proxyIntent.upstream,
|
|
479
|
+
affinityKey: proxyIntent.affinityKey,
|
|
480
|
+
telemetry: proxyIntent.telemetry
|
|
481
|
+
? {
|
|
482
|
+
...proxyIntent.telemetry,
|
|
483
|
+
recordProxyResolution(event) {
|
|
484
|
+
proxyIntent.telemetry?.recordProxyResolution({
|
|
485
|
+
...event,
|
|
486
|
+
userAgentSource,
|
|
487
|
+
});
|
|
488
|
+
},
|
|
489
|
+
}
|
|
490
|
+
: undefined,
|
|
491
|
+
});
|
|
492
|
+
proxyUrl = resolved.url;
|
|
493
|
+
if (!proxyUrl)
|
|
494
|
+
return { unavailableReason: "missing_proxy_identity", userAgentSource };
|
|
495
|
+
}
|
|
496
|
+
catch {
|
|
497
|
+
// Lease failures contain infrastructure detail that must not cross the resolver
|
|
498
|
+
// boundary. A required policy is classified by the existing fail-closed guard.
|
|
499
|
+
return { unavailableReason: "missing_proxy_identity", userAgentSource };
|
|
500
|
+
}
|
|
501
|
+
try {
|
|
502
|
+
const userAgent = proxyIntent.userAgent || resolveDefaultResolverUserAgent();
|
|
503
|
+
if (!userAgent)
|
|
504
|
+
return { unavailableReason: "missing_client_profile", userAgentSource };
|
|
505
|
+
return {
|
|
506
|
+
identity: { proxyUrl, userAgent },
|
|
507
|
+
userAgentSource,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
catch {
|
|
511
|
+
return { unavailableReason: "missing_client_profile", userAgentSource };
|
|
512
|
+
}
|
|
513
|
+
}
|
|
453
514
|
function createResolverChainClient(options) {
|
|
454
515
|
assertClientProfileTransportContract(options.clientProfile, options.transport);
|
|
455
516
|
const client = {
|
|
@@ -465,8 +526,23 @@ function createResolverChainClient(options) {
|
|
|
465
526
|
if (supportingEntries.length === 0)
|
|
466
527
|
throwUnsupportedKind(challenge.kind);
|
|
467
528
|
signal.throwIfAborted();
|
|
529
|
+
const identityResolution = options.proxyIntent
|
|
530
|
+
? await resolveResolverIdentity(options.proxyIntent)
|
|
531
|
+
: { identity: options.identity };
|
|
532
|
+
const identity = identityResolution.identity;
|
|
533
|
+
signal.throwIfAborted();
|
|
534
|
+
// Resolve a required proxy lease before consulting the cache. Solutions minted
|
|
535
|
+
// under a previous release are shared and long-lived, but a portable cached token
|
|
536
|
+
// must not bypass the upstream admission policy when no lease can be resolved.
|
|
537
|
+
const requiredProxyIdentityMissing = options.proxyIntent?.mode === "required" && identity === undefined;
|
|
538
|
+
if (requiredProxyIdentityMissing) {
|
|
539
|
+
throwExhausted(supportingEntries.map((entry) => ({
|
|
540
|
+
vendor: entry.id,
|
|
541
|
+
reason: identityResolution.unavailableReason ?? "missing_proxy_identity",
|
|
542
|
+
})));
|
|
543
|
+
}
|
|
468
544
|
if (options.cache && resolverChallengeIsCacheable(challenge)) {
|
|
469
|
-
const cached = await findCachedSolution(options.cache, challenge,
|
|
545
|
+
const cached = await findCachedSolution(options.cache, challenge, identity, options.identityScope);
|
|
470
546
|
if (cached)
|
|
471
547
|
return cached;
|
|
472
548
|
}
|
|
@@ -489,7 +565,7 @@ function createResolverChainClient(options) {
|
|
|
489
565
|
const transport = unrestrictedTransport
|
|
490
566
|
? restrictResolverTransport(unrestrictedTransport, options.allowedHosts ?? [])
|
|
491
567
|
: undefined;
|
|
492
|
-
return adapter.solve(challenge,
|
|
568
|
+
return adapter.solve(challenge, identity, signal, traceRecorder, transport);
|
|
493
569
|
};
|
|
494
570
|
const solution = traceRecorder
|
|
495
571
|
? await traceRecorder.runSpan("resolver.vendor.attempt", solveAttempt, {
|
|
@@ -497,6 +573,7 @@ function createResolverChainClient(options) {
|
|
|
497
573
|
vendor: adapter.id,
|
|
498
574
|
challenge_kind: challenge.kind,
|
|
499
575
|
client_profile: options.clientProfile,
|
|
576
|
+
resolver_identity_source: identityResolution.userAgentSource,
|
|
500
577
|
},
|
|
501
578
|
onError(error) {
|
|
502
579
|
return error instanceof ResolverVendorUnavailableError
|
|
@@ -509,9 +586,9 @@ function createResolverChainClient(options) {
|
|
|
509
586
|
resolverChallengeIsCacheable(challenge) &&
|
|
510
587
|
solution.form === "cookies" &&
|
|
511
588
|
solutionExpiryMs(solution) !== undefined) {
|
|
512
|
-
const issuingIdentity = adapter.getIssuingIdentity?.(solution,
|
|
589
|
+
const issuingIdentity = adapter.getIssuingIdentity?.(solution, identity, challenge) ??
|
|
513
590
|
resolverChallengeIssuingIdentity(challenge, {
|
|
514
|
-
...(
|
|
591
|
+
...(identity ? { proxyUrl: identity.proxyUrl } : {}),
|
|
515
592
|
userAgent: solution.userAgent,
|
|
516
593
|
});
|
|
517
594
|
if (issuingIdentity) {
|
|
@@ -544,6 +621,7 @@ export function createResolverClient(options) {
|
|
|
544
621
|
unavailableReason: options.unavailableReason,
|
|
545
622
|
cache: options.cache,
|
|
546
623
|
identity: options.identity,
|
|
624
|
+
proxyIntent: options.proxyIntent,
|
|
547
625
|
transport: options.transport,
|
|
548
626
|
createTransport: options.createTransport,
|
|
549
627
|
clientProfile: options.clientProfile,
|
|
@@ -570,16 +648,6 @@ function resolveVendorAvailability(vendor, env) {
|
|
|
570
648
|
? { vendor, available: true, configuration }
|
|
571
649
|
: { vendor, available: false, reason: "missing_credentials" };
|
|
572
650
|
}
|
|
573
|
-
export function createUnsupportedResolverClient(reason) {
|
|
574
|
-
return {
|
|
575
|
-
async solve() {
|
|
576
|
-
throw new ProviderError(reason ?? "Resolver runtime is not configured", {
|
|
577
|
-
code: "RESOLVER_UNAVAILABLE",
|
|
578
|
-
fix: "Declare resolver on the provider definition and configure vendor credentials.",
|
|
579
|
-
});
|
|
580
|
-
},
|
|
581
|
-
};
|
|
582
|
-
}
|
|
583
651
|
export function bindResolverSignal(resolver, defaultSignal) {
|
|
584
652
|
if (!defaultSignal)
|
|
585
653
|
return resolver;
|
|
@@ -620,6 +688,7 @@ function createResolverClientFromEnvInternal(config, env, options, adapterFactor
|
|
|
620
688
|
};
|
|
621
689
|
}),
|
|
622
690
|
cache: options.cache,
|
|
691
|
+
proxyIntent: options.proxyIntent,
|
|
623
692
|
identityScope: options.identityScope,
|
|
624
693
|
transport: options.transport,
|
|
625
694
|
createTransport: options.createTransport,
|
package/dist/runtime/state.js
CHANGED
|
@@ -4,7 +4,6 @@ import { ProviderError } from "../errors.js";
|
|
|
4
4
|
import { createProviderRedisClient, ensureRedisReady, withRedisTimeout, } from "./redis.js";
|
|
5
5
|
const DEFAULT_REDIS_TIMEOUT_MS = 250;
|
|
6
6
|
const REDIS_STATE_PREFIX = "apifuse:provider-state:v2";
|
|
7
|
-
const LEGACY_REDIS_STATE_PREFIX = "apifuse:provider-state:v1";
|
|
8
7
|
const PROVIDER_SCOPE_DISCRIMINATOR = "scope:provider";
|
|
9
8
|
const MISSING_CONNECTION_SCOPE_DISCRIMINATOR = "scope:connection:missing";
|
|
10
9
|
const SET_WITH_QUOTA_SCRIPT = `
|
|
@@ -15,7 +14,6 @@ local index_ttl = tonumber(ARGV[4])
|
|
|
15
14
|
local envelope = ARGV[5]
|
|
16
15
|
|
|
17
16
|
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
18
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
19
17
|
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
20
18
|
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
21
19
|
return {0, false}
|
|
@@ -24,14 +22,10 @@ end
|
|
|
24
22
|
redis.call("SET", KEYS[1], envelope, "PXAT", expires_at)
|
|
25
23
|
redis.call("ZADD", KEYS[2], expires_at, KEYS[1])
|
|
26
24
|
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
27
|
-
if ARGV[6] == "1" then
|
|
28
|
-
redis.call("SET", KEYS[3], ARGV[7], "PX", index_ttl)
|
|
29
|
-
end
|
|
30
25
|
return {1, envelope}
|
|
31
26
|
`;
|
|
32
27
|
const COMPARE_AND_SET_WITH_QUOTA_SCRIPT = `
|
|
33
28
|
local current = redis.call("GET", KEYS[1])
|
|
34
|
-
local scoped_present = current and true or false
|
|
35
29
|
if current then
|
|
36
30
|
local ok, decoded = pcall(cjson.decode, current)
|
|
37
31
|
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
@@ -44,42 +38,6 @@ local max_entries = tonumber(ARGV[3])
|
|
|
44
38
|
local expires_at = tonumber(ARGV[4])
|
|
45
39
|
local index_ttl = tonumber(ARGV[5])
|
|
46
40
|
local envelope = ARGV[6]
|
|
47
|
-
local allow_legacy_claim = ARGV[7] == "1"
|
|
48
|
-
local legacy_tombstone = ARGV[8]
|
|
49
|
-
|
|
50
|
-
if not scoped_present then
|
|
51
|
-
local legacy = redis.call("GET", KEYS[3])
|
|
52
|
-
if legacy then
|
|
53
|
-
local ok, decoded = pcall(cjson.decode, legacy)
|
|
54
|
-
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
55
|
-
legacy = false
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
if legacy and not allow_legacy_claim then
|
|
59
|
-
return {-1, false}
|
|
60
|
-
end
|
|
61
|
-
if legacy then
|
|
62
|
-
local legacy_ttl = redis.call("PTTL", KEYS[3])
|
|
63
|
-
if legacy_ttl == -1 then
|
|
64
|
-
legacy_ttl = index_ttl
|
|
65
|
-
elseif legacy_ttl < 1 then
|
|
66
|
-
legacy_ttl = 1
|
|
67
|
-
end
|
|
68
|
-
if legacy_ttl > 0 then
|
|
69
|
-
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
70
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
71
|
-
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
72
|
-
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
73
|
-
return {0, false}
|
|
74
|
-
end
|
|
75
|
-
redis.call("SET", KEYS[1], legacy, "PX", legacy_ttl)
|
|
76
|
-
redis.call("ZADD", KEYS[2], now + legacy_ttl, KEYS[1])
|
|
77
|
-
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
78
|
-
redis.call("SET", KEYS[3], legacy_tombstone, "PX", index_ttl)
|
|
79
|
-
current = legacy
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
41
|
|
|
84
42
|
local current_version = 0
|
|
85
43
|
local current_decoded = false
|
|
@@ -96,7 +54,6 @@ if current_version ~= tonumber(ARGV[1]) then
|
|
|
96
54
|
end
|
|
97
55
|
|
|
98
56
|
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
99
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
100
57
|
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
101
58
|
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
102
59
|
return {0, false}
|
|
@@ -110,60 +67,13 @@ end
|
|
|
110
67
|
redis.call("SET", KEYS[1], envelope, "PXAT", expires_at)
|
|
111
68
|
redis.call("ZADD", KEYS[2], expires_at, KEYS[1])
|
|
112
69
|
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
113
|
-
if allow_legacy_claim then
|
|
114
|
-
redis.call("SET", KEYS[3], legacy_tombstone, "PX", index_ttl)
|
|
115
|
-
end
|
|
116
70
|
return {1, envelope}
|
|
117
71
|
`;
|
|
118
72
|
const DELETE_WITH_INDEX_SCRIPT = `
|
|
119
73
|
redis.call("SET", KEYS[1], ARGV[1], "PX", ARGV[2])
|
|
120
|
-
|
|
121
|
-
redis.call("SET", KEYS[3], ARGV[1], "PX", ARGV[2])
|
|
122
|
-
end
|
|
123
|
-
redis.call("ZREM", KEYS[2], KEYS[1], KEYS[3])
|
|
74
|
+
redis.call("ZREM", KEYS[2], KEYS[1])
|
|
124
75
|
return 1
|
|
125
76
|
`;
|
|
126
|
-
// A v1 key has no connection discriminator, so it cannot safely be exposed to
|
|
127
|
-
// every v2 scope. The first concrete connection (or explicit provider scope) to
|
|
128
|
-
// request a key atomically adopts that value and tombstones v1. The missing-
|
|
129
|
-
// connection sentinel never claims legacy state.
|
|
130
|
-
const CLAIM_LEGACY_SCRIPT = `
|
|
131
|
-
local scoped = redis.call("GET", KEYS[1])
|
|
132
|
-
if scoped then
|
|
133
|
-
return {1, scoped}
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
local legacy = redis.call("GET", KEYS[3])
|
|
137
|
-
if not legacy then
|
|
138
|
-
return {1, false}
|
|
139
|
-
end
|
|
140
|
-
local ok, decoded = pcall(cjson.decode, legacy)
|
|
141
|
-
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
142
|
-
return {1, false}
|
|
143
|
-
end
|
|
144
|
-
local legacy_ttl = redis.call("PTTL", KEYS[3])
|
|
145
|
-
if legacy_ttl == -1 then
|
|
146
|
-
legacy_ttl = tonumber(ARGV[3])
|
|
147
|
-
elseif legacy_ttl < 1 then
|
|
148
|
-
legacy_ttl = 1
|
|
149
|
-
end
|
|
150
|
-
|
|
151
|
-
local now = tonumber(ARGV[1])
|
|
152
|
-
local max_entries = tonumber(ARGV[2])
|
|
153
|
-
local index_ttl = tonumber(ARGV[3])
|
|
154
|
-
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
155
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
156
|
-
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
157
|
-
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
158
|
-
return {0, false}
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
redis.call("SET", KEYS[1], legacy, "PX", legacy_ttl)
|
|
162
|
-
redis.call("ZADD", KEYS[2], now + legacy_ttl, KEYS[1])
|
|
163
|
-
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
164
|
-
redis.call("SET", KEYS[3], ARGV[4], "PX", index_ttl)
|
|
165
|
-
return {1, legacy}
|
|
166
|
-
`;
|
|
167
77
|
const redisBackends = new Map();
|
|
168
78
|
function getRedisBackend(redisUrl, injectedRedis) {
|
|
169
79
|
const existing = redisBackends.get(redisUrl);
|
|
@@ -210,12 +120,6 @@ function providerStatePrefix(providerId, namespace, scopeDiscriminator) {
|
|
|
210
120
|
function providerStateKey(providerId, namespace, scopeDiscriminator, key) {
|
|
211
121
|
return `${providerStatePrefix(providerId, namespace, scopeDiscriminator)}:${key}`;
|
|
212
122
|
}
|
|
213
|
-
function legacyProviderStatePrefix(providerId, namespace) {
|
|
214
|
-
return `${LEGACY_REDIS_STATE_PREFIX}:${providerId ?? "default"}:${namespace}`;
|
|
215
|
-
}
|
|
216
|
-
function legacyProviderStateKey(providerId, namespace, key) {
|
|
217
|
-
return `${legacyProviderStatePrefix(providerId, namespace)}:${key}`;
|
|
218
|
-
}
|
|
219
123
|
function publicStateKey(redisKey, prefixes) {
|
|
220
124
|
for (const prefix of prefixes) {
|
|
221
125
|
const prefixWithSeparator = `${prefix}:`;
|
|
@@ -311,9 +215,6 @@ class RedisProviderStateNamespace {
|
|
|
311
215
|
redisKey(key) {
|
|
312
216
|
return providerStateKey(this.providerId, this.namespaceName, this.scopeDiscriminator, key);
|
|
313
217
|
}
|
|
314
|
-
legacyRedisKey(key) {
|
|
315
|
-
return legacyProviderStateKey(this.providerId, this.namespaceName, key);
|
|
316
|
-
}
|
|
317
218
|
statePrefix() {
|
|
318
219
|
return providerStatePrefix(this.providerId, this.namespaceName, this.scopeDiscriminator);
|
|
319
220
|
}
|
|
@@ -324,9 +225,6 @@ class RedisProviderStateNamespace {
|
|
|
324
225
|
const namespaceIdentity = Buffer.from(providerStatePrefix(this.providerId, this.namespaceName, this.scopeDiscriminator), "utf8").toString("base64url");
|
|
325
226
|
return `${REDIS_STATE_PREFIX}:index:${namespaceIdentity}`;
|
|
326
227
|
}
|
|
327
|
-
canClaimLegacy() {
|
|
328
|
-
return this.scopeDiscriminator !== MISSING_CONNECTION_SCOPE_DISCRIMINATOR;
|
|
329
|
-
}
|
|
330
228
|
async indexedKeys(limit) {
|
|
331
229
|
await requireRedisReady(this.backend.redis);
|
|
332
230
|
const now = Date.now();
|
|
@@ -337,15 +235,7 @@ class RedisProviderStateNamespace {
|
|
|
337
235
|
}
|
|
338
236
|
async readRaw(key) {
|
|
339
237
|
await requireRedisReady(this.backend.redis);
|
|
340
|
-
|
|
341
|
-
return await withRequiredRedis(() => this.backend.redis.get(this.redisKey(key)));
|
|
342
|
-
}
|
|
343
|
-
const tombstone = JSON.stringify({ deleted: true });
|
|
344
|
-
const result = await withRequiredRedis(() => this.backend.redis.eval(CLAIM_LEGACY_SCRIPT, 3, this.redisKey(key), this.indexKey(), this.legacyRedisKey(key), String(Date.now()), String(this.options.maxEntries), String(parseStateDurationMs(this.options.maxTtl)), tombstone));
|
|
345
|
-
if (Array.isArray(result) && Number(result[0]) === 0) {
|
|
346
|
-
throw this.quotaExceeded();
|
|
347
|
-
}
|
|
348
|
-
return Array.isArray(result) && typeof result[1] === "string" ? result[1] : null;
|
|
238
|
+
return await withRequiredRedis(() => this.backend.redis.get(this.redisKey(key)));
|
|
349
239
|
}
|
|
350
240
|
enforceValueSize(value) {
|
|
351
241
|
const bytes = Buffer.byteLength(JSON.stringify(value), "utf8");
|
|
@@ -406,7 +296,7 @@ class RedisProviderStateNamespace {
|
|
|
406
296
|
const timing = this.writeTiming(options?.ttl);
|
|
407
297
|
const envelope = redisEnvelope(value, version, createdAt, timing.expiresAt);
|
|
408
298
|
await requireRedisReady(this.backend.redis);
|
|
409
|
-
const result = await withRequiredRedis(() => this.backend.redis.eval(SET_WITH_QUOTA_SCRIPT,
|
|
299
|
+
const result = await withRequiredRedis(() => this.backend.redis.eval(SET_WITH_QUOTA_SCRIPT, 2, this.redisKey(key), this.indexKey(), String(Date.now()), String(this.options.maxEntries), String(timing.expiresAtMs), String(timing.indexTtlMs), JSON.stringify(envelope)));
|
|
410
300
|
if (!Array.isArray(result) || Number(result[0]) !== 1) {
|
|
411
301
|
throw this.quotaExceeded();
|
|
412
302
|
}
|
|
@@ -431,7 +321,7 @@ class RedisProviderStateNamespace {
|
|
|
431
321
|
const timing = this.writeTiming(options?.ttl);
|
|
432
322
|
const envelope = redisEnvelope(value, expectedVersion + 1, createdAt, timing.expiresAt);
|
|
433
323
|
await requireRedisReady(this.backend.redis);
|
|
434
|
-
const result = await withRequiredRedis(() => this.backend.redis.eval(COMPARE_AND_SET_WITH_QUOTA_SCRIPT,
|
|
324
|
+
const result = await withRequiredRedis(() => this.backend.redis.eval(COMPARE_AND_SET_WITH_QUOTA_SCRIPT, 2, this.redisKey(key), this.indexKey(), String(expectedVersion), String(Date.now()), String(this.options.maxEntries), String(timing.expiresAtMs), String(timing.indexTtlMs), JSON.stringify(envelope)));
|
|
435
325
|
if (Array.isArray(result) && Number(result[0]) === 0) {
|
|
436
326
|
throw this.quotaExceeded();
|
|
437
327
|
}
|
|
@@ -449,7 +339,7 @@ class RedisProviderStateNamespace {
|
|
|
449
339
|
async delete(key) {
|
|
450
340
|
await requireRedisReady(this.backend.redis);
|
|
451
341
|
const tombstone = JSON.stringify({ deleted: true });
|
|
452
|
-
await withRequiredRedis(() => this.backend.redis.eval(DELETE_WITH_INDEX_SCRIPT,
|
|
342
|
+
await withRequiredRedis(() => this.backend.redis.eval(DELETE_WITH_INDEX_SCRIPT, 2, this.redisKey(key), this.indexKey(), tombstone, String(parseStateDurationMs(this.options.maxTtl))));
|
|
453
343
|
}
|
|
454
344
|
async increment(key, field, delta = 1, options) {
|
|
455
345
|
const current = (await this.get(key))?.value ?? {};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { CookieJar, StealthCookieStore, StealthCookieStoreV1, StealthSessionCookies } from "../types.js";
|
|
2
|
+
export declare class StealthCookieJar implements CookieJar, StealthSessionCookies {
|
|
3
|
+
private cookies;
|
|
4
|
+
private readonly defaultUrl;
|
|
5
|
+
constructor(cookieStrings: readonly string[], defaultUrl?: string);
|
|
6
|
+
setFromCookieStrings(cookieStrings: readonly string[], url?: string): void;
|
|
7
|
+
get(name: string, url?: string): string | undefined;
|
|
8
|
+
getAll(url?: string): Record<string, string>;
|
|
9
|
+
has(name: string, url?: string): boolean;
|
|
10
|
+
toString(url?: string): string;
|
|
11
|
+
toHeader(url?: string): string;
|
|
12
|
+
snapshot(): Record<string, string>;
|
|
13
|
+
restore(cookies: Record<string, string>): void;
|
|
14
|
+
serialize(): StealthCookieStoreV1;
|
|
15
|
+
deserialize(state: StealthCookieStore): void;
|
|
16
|
+
clear(): void;
|
|
17
|
+
find(predicate: (cookie: string) => boolean, url?: string): string | undefined;
|
|
18
|
+
private normalizeUrl;
|
|
19
|
+
private getUniqueCookies;
|
|
20
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
2
|
+
import { SDKError, StealthCookieStoreVersionError } from "../errors.js";
|
|
3
|
+
function isRecord(value) {
|
|
4
|
+
return typeof value === "object" && value !== null;
|
|
5
|
+
}
|
|
6
|
+
const LEGACY_COOKIE_ORIGIN = "https://legacy-cookie.invalid/";
|
|
7
|
+
export class StealthCookieJar {
|
|
8
|
+
cookies;
|
|
9
|
+
defaultUrl;
|
|
10
|
+
constructor(cookieStrings, defaultUrl = LEGACY_COOKIE_ORIGIN) {
|
|
11
|
+
this.cookies = new ToughCookieJar(undefined, {
|
|
12
|
+
allowSecureOnLocal: false,
|
|
13
|
+
rejectPublicSuffixes: true,
|
|
14
|
+
});
|
|
15
|
+
this.defaultUrl = this.normalizeUrl(defaultUrl) ?? LEGACY_COOKIE_ORIGIN;
|
|
16
|
+
this.setFromCookieStrings(cookieStrings);
|
|
17
|
+
}
|
|
18
|
+
setFromCookieStrings(cookieStrings, url = this.defaultUrl) {
|
|
19
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
20
|
+
if (!cookieUrl)
|
|
21
|
+
return;
|
|
22
|
+
for (const cookieString of cookieStrings) {
|
|
23
|
+
this.cookies.setCookieSync(cookieString, cookieUrl, { ignoreError: true });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
get(name, url) {
|
|
27
|
+
return this.getAll(url)[name];
|
|
28
|
+
}
|
|
29
|
+
getAll(url) {
|
|
30
|
+
return Object.fromEntries(this.getUniqueCookies(url ?? this.defaultUrl).map((cookie) => [cookie.key, cookie.value]));
|
|
31
|
+
}
|
|
32
|
+
has(name, url) {
|
|
33
|
+
return Object.hasOwn(this.getAll(url), name);
|
|
34
|
+
}
|
|
35
|
+
toString(url) {
|
|
36
|
+
return this.getUniqueCookies(url ?? this.defaultUrl)
|
|
37
|
+
.map((cookie) => cookie.cookieString())
|
|
38
|
+
.join("; ");
|
|
39
|
+
}
|
|
40
|
+
toHeader(url) {
|
|
41
|
+
return this.toString(url);
|
|
42
|
+
}
|
|
43
|
+
snapshot() {
|
|
44
|
+
const entries = [];
|
|
45
|
+
for (const cookie of this.serialize().jar.cookies) {
|
|
46
|
+
if (typeof cookie.key === "string" && typeof cookie.value === "string" && cookie.key) {
|
|
47
|
+
entries.push([cookie.key, cookie.value]);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return Object.fromEntries(entries);
|
|
51
|
+
}
|
|
52
|
+
restore(cookies) {
|
|
53
|
+
this.clear();
|
|
54
|
+
for (const [name, value] of Object.entries(cookies)) {
|
|
55
|
+
if (!name)
|
|
56
|
+
continue;
|
|
57
|
+
this.cookies.setCookieSync(new Cookie({ key: name, path: "/", value }), this.defaultUrl, {
|
|
58
|
+
ignoreError: true,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
serialize() {
|
|
63
|
+
const jar = this.cookies.serializeSync();
|
|
64
|
+
if (!jar) {
|
|
65
|
+
throw new SDKError("Stealth cookie store could not be serialized", {
|
|
66
|
+
code: "stealth_cookie_store_serialize_failed",
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
return { version: 1, jar };
|
|
70
|
+
}
|
|
71
|
+
deserialize(state) {
|
|
72
|
+
const version = isRecord(state) ? state.version : undefined;
|
|
73
|
+
if (version !== 1) {
|
|
74
|
+
throw new StealthCookieStoreVersionError(version);
|
|
75
|
+
}
|
|
76
|
+
const restored = ToughCookieJar.deserializeSync(state.jar);
|
|
77
|
+
Reflect.set(restored, "allowSecureOnLocal", false);
|
|
78
|
+
this.cookies = restored;
|
|
79
|
+
}
|
|
80
|
+
clear() {
|
|
81
|
+
this.cookies.removeAllCookiesSync();
|
|
82
|
+
}
|
|
83
|
+
find(predicate, url) {
|
|
84
|
+
for (const cookie of this.getUniqueCookies(url ?? this.defaultUrl)) {
|
|
85
|
+
const cookieString = cookie.cookieString();
|
|
86
|
+
if (predicate(cookieString))
|
|
87
|
+
return cookieString;
|
|
88
|
+
}
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
normalizeUrl(url) {
|
|
92
|
+
try {
|
|
93
|
+
return new URL(url).toString();
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
getUniqueCookies(url) {
|
|
100
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
101
|
+
if (!cookieUrl)
|
|
102
|
+
return [];
|
|
103
|
+
const names = new Set();
|
|
104
|
+
return this.cookies.getCookiesSync(cookieUrl).filter((cookie) => {
|
|
105
|
+
if (names.has(cookie.key))
|
|
106
|
+
return false;
|
|
107
|
+
names.add(cookie.key);
|
|
108
|
+
return true;
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BrowserProfile, EmulationOS } from "wreq-js";
|
|
2
2
|
import type { ProxyResolutionOptions } from "../config/loader.js";
|
|
3
3
|
import type { StealthClient, StealthResponse } from "../types.js";
|
|
4
|
+
export declare const DEFAULT_PROFILE = "chrome-146";
|
|
4
5
|
export type StealthClientOptions = ProxyResolutionOptions & {
|
|
5
6
|
warn?: (message: string) => void;
|
|
6
7
|
/**
|