@apifuse/provider-sdk 2.2.0-beta.26 → 2.2.0-beta.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/bin/apifuse-dev.ts +2 -2
- package/bin/apifuse-pack-smoke.ts +1 -1
- package/bin/apifuse-pack-types.ts +2 -1
- package/bin/apifuse-perf.ts +2 -4
- package/bin/apifuse-record.ts +1 -1
- package/dist/auth-turn/index.d.ts +1 -1
- package/dist/auth-turn/index.js +1 -1
- package/dist/ceremonies/index.js +52 -11
- package/dist/config/loader.d.ts +1 -1
- package/dist/config/loader.js +4 -2
- package/dist/contract-serialization.d.ts +1 -20
- package/dist/contract-serialization.js +8 -587
- package/dist/contract.d.ts +0 -2
- package/dist/contract.js +5 -9
- package/dist/index.d.ts +9 -8
- package/dist/index.js +6 -8
- package/dist/provider.d.ts +3 -2
- package/dist/provider.js +2 -2
- package/dist/runtime/auth-flow.js +1 -1
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +135 -12
- package/dist/runtime/instrumentation.js +1 -1
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +2 -33
- package/dist/runtime/native-network.js +2 -68
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-shared.d.ts +3 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +2 -1
- package/dist/runtime/resolver-vendors/twocaptcha.js +80 -43
- package/dist/runtime/resolver-vendors/types.d.ts +1 -1
- package/dist/runtime/resolver.d.ts +6 -10
- package/dist/runtime/resolver.js +19 -18
- package/dist/runtime/state.js +5 -115
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.js +7 -130
- package/dist/schema.d.ts +0 -63
- package/dist/schema.js +8 -808
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.js +1 -1
- package/dist/server/self-test.d.ts +13 -0
- package/dist/server/self-test.js +124 -46
- package/dist/server/serve-implementation.d.ts +199 -0
- package/dist/server/serve-implementation.js +2054 -0
- package/dist/server/serve.d.ts +1 -187
- package/dist/server/serve.js +1 -1827
- package/dist/stateful/errors.d.ts +5 -0
- package/dist/stateful/errors.js +10 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +1 -5
- package/dist/stateful/stateful-provider-session-routing.js +2 -10
- package/dist/stream.js +7 -1
- package/package.json +27 -2
- package/src/auth-turn/index.ts +1 -1
- package/src/ceremonies/index.ts +68 -18
- package/src/config/loader.ts +5 -2
- package/src/contract-serialization.ts +8 -859
- package/src/contract.ts +5 -16
- package/src/index.ts +18 -34
- package/src/provider.ts +12 -24
- package/src/runtime/auth-flow.ts +1 -1
- package/src/runtime/http.ts +155 -11
- package/src/runtime/instrumentation.ts +1 -1
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +16 -97
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-shared.ts +17 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +100 -49
- package/src/runtime/resolver-vendors/types.ts +1 -0
- package/src/runtime/resolver.ts +47 -22
- package/src/runtime/state.ts +5 -144
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +14 -157
- package/src/schema.ts +9 -1060
- package/src/serve.ts +6 -1
- package/src/server/index.ts +1 -0
- package/src/server/self-test.ts +184 -59
- package/src/server/serve-implementation.ts +3024 -0
- package/src/server/serve.ts +1 -2661
- package/src/stateful/errors.ts +12 -0
- package/src/stateful/stateful-provider-session-routing.ts +2 -11
- package/src/stream.ts +8 -1
package/src/runtime/state.ts
CHANGED
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
|
|
19
19
|
const DEFAULT_REDIS_TIMEOUT_MS = 250;
|
|
20
20
|
const REDIS_STATE_PREFIX = "apifuse:provider-state:v2";
|
|
21
|
-
const LEGACY_REDIS_STATE_PREFIX = "apifuse:provider-state:v1";
|
|
22
21
|
const PROVIDER_SCOPE_DISCRIMINATOR = "scope:provider";
|
|
23
22
|
const MISSING_CONNECTION_SCOPE_DISCRIMINATOR = "scope:connection:missing";
|
|
24
23
|
const SET_WITH_QUOTA_SCRIPT = `
|
|
@@ -29,7 +28,6 @@ local index_ttl = tonumber(ARGV[4])
|
|
|
29
28
|
local envelope = ARGV[5]
|
|
30
29
|
|
|
31
30
|
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
32
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
33
31
|
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
34
32
|
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
35
33
|
return {0, false}
|
|
@@ -38,15 +36,11 @@ end
|
|
|
38
36
|
redis.call("SET", KEYS[1], envelope, "PXAT", expires_at)
|
|
39
37
|
redis.call("ZADD", KEYS[2], expires_at, KEYS[1])
|
|
40
38
|
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
41
|
-
if ARGV[6] == "1" then
|
|
42
|
-
redis.call("SET", KEYS[3], ARGV[7], "PX", index_ttl)
|
|
43
|
-
end
|
|
44
39
|
return {1, envelope}
|
|
45
40
|
`;
|
|
46
41
|
|
|
47
42
|
const COMPARE_AND_SET_WITH_QUOTA_SCRIPT = `
|
|
48
43
|
local current = redis.call("GET", KEYS[1])
|
|
49
|
-
local scoped_present = current and true or false
|
|
50
44
|
if current then
|
|
51
45
|
local ok, decoded = pcall(cjson.decode, current)
|
|
52
46
|
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
@@ -59,42 +53,6 @@ local max_entries = tonumber(ARGV[3])
|
|
|
59
53
|
local expires_at = tonumber(ARGV[4])
|
|
60
54
|
local index_ttl = tonumber(ARGV[5])
|
|
61
55
|
local envelope = ARGV[6]
|
|
62
|
-
local allow_legacy_claim = ARGV[7] == "1"
|
|
63
|
-
local legacy_tombstone = ARGV[8]
|
|
64
|
-
|
|
65
|
-
if not scoped_present then
|
|
66
|
-
local legacy = redis.call("GET", KEYS[3])
|
|
67
|
-
if legacy then
|
|
68
|
-
local ok, decoded = pcall(cjson.decode, legacy)
|
|
69
|
-
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
70
|
-
legacy = false
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
if legacy and not allow_legacy_claim then
|
|
74
|
-
return {-1, false}
|
|
75
|
-
end
|
|
76
|
-
if legacy then
|
|
77
|
-
local legacy_ttl = redis.call("PTTL", KEYS[3])
|
|
78
|
-
if legacy_ttl == -1 then
|
|
79
|
-
legacy_ttl = index_ttl
|
|
80
|
-
elseif legacy_ttl < 1 then
|
|
81
|
-
legacy_ttl = 1
|
|
82
|
-
end
|
|
83
|
-
if legacy_ttl > 0 then
|
|
84
|
-
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
85
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
86
|
-
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
87
|
-
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
88
|
-
return {0, false}
|
|
89
|
-
end
|
|
90
|
-
redis.call("SET", KEYS[1], legacy, "PX", legacy_ttl)
|
|
91
|
-
redis.call("ZADD", KEYS[2], now + legacy_ttl, KEYS[1])
|
|
92
|
-
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
93
|
-
redis.call("SET", KEYS[3], legacy_tombstone, "PX", index_ttl)
|
|
94
|
-
current = legacy
|
|
95
|
-
end
|
|
96
|
-
end
|
|
97
|
-
end
|
|
98
56
|
|
|
99
57
|
local current_version = 0
|
|
100
58
|
local current_decoded = false
|
|
@@ -111,7 +69,6 @@ if current_version ~= tonumber(ARGV[1]) then
|
|
|
111
69
|
end
|
|
112
70
|
|
|
113
71
|
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
114
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
115
72
|
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
116
73
|
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
117
74
|
return {0, false}
|
|
@@ -125,63 +82,15 @@ end
|
|
|
125
82
|
redis.call("SET", KEYS[1], envelope, "PXAT", expires_at)
|
|
126
83
|
redis.call("ZADD", KEYS[2], expires_at, KEYS[1])
|
|
127
84
|
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
128
|
-
if allow_legacy_claim then
|
|
129
|
-
redis.call("SET", KEYS[3], legacy_tombstone, "PX", index_ttl)
|
|
130
|
-
end
|
|
131
85
|
return {1, envelope}
|
|
132
86
|
`;
|
|
133
87
|
|
|
134
88
|
const DELETE_WITH_INDEX_SCRIPT = `
|
|
135
89
|
redis.call("SET", KEYS[1], ARGV[1], "PX", ARGV[2])
|
|
136
|
-
|
|
137
|
-
redis.call("SET", KEYS[3], ARGV[1], "PX", ARGV[2])
|
|
138
|
-
end
|
|
139
|
-
redis.call("ZREM", KEYS[2], KEYS[1], KEYS[3])
|
|
90
|
+
redis.call("ZREM", KEYS[2], KEYS[1])
|
|
140
91
|
return 1
|
|
141
92
|
`;
|
|
142
93
|
|
|
143
|
-
// A v1 key has no connection discriminator, so it cannot safely be exposed to
|
|
144
|
-
// every v2 scope. The first concrete connection (or explicit provider scope) to
|
|
145
|
-
// request a key atomically adopts that value and tombstones v1. The missing-
|
|
146
|
-
// connection sentinel never claims legacy state.
|
|
147
|
-
const CLAIM_LEGACY_SCRIPT = `
|
|
148
|
-
local scoped = redis.call("GET", KEYS[1])
|
|
149
|
-
if scoped then
|
|
150
|
-
return {1, scoped}
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
local legacy = redis.call("GET", KEYS[3])
|
|
154
|
-
if not legacy then
|
|
155
|
-
return {1, false}
|
|
156
|
-
end
|
|
157
|
-
local ok, decoded = pcall(cjson.decode, legacy)
|
|
158
|
-
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
159
|
-
return {1, false}
|
|
160
|
-
end
|
|
161
|
-
local legacy_ttl = redis.call("PTTL", KEYS[3])
|
|
162
|
-
if legacy_ttl == -1 then
|
|
163
|
-
legacy_ttl = tonumber(ARGV[3])
|
|
164
|
-
elseif legacy_ttl < 1 then
|
|
165
|
-
legacy_ttl = 1
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
local now = tonumber(ARGV[1])
|
|
169
|
-
local max_entries = tonumber(ARGV[2])
|
|
170
|
-
local index_ttl = tonumber(ARGV[3])
|
|
171
|
-
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
172
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
173
|
-
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
174
|
-
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
175
|
-
return {0, false}
|
|
176
|
-
end
|
|
177
|
-
|
|
178
|
-
redis.call("SET", KEYS[1], legacy, "PX", legacy_ttl)
|
|
179
|
-
redis.call("ZADD", KEYS[2], now + legacy_ttl, KEYS[1])
|
|
180
|
-
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
181
|
-
redis.call("SET", KEYS[3], ARGV[4], "PX", index_ttl)
|
|
182
|
-
return {1, legacy}
|
|
183
|
-
`;
|
|
184
|
-
|
|
185
94
|
type RedisProviderRuntimeStateOptions = {
|
|
186
95
|
readonly redisUrl: string;
|
|
187
96
|
readonly providerId?: string;
|
|
@@ -268,18 +177,6 @@ function providerStateKey(
|
|
|
268
177
|
return `${providerStatePrefix(providerId, namespace, scopeDiscriminator)}:${key}`;
|
|
269
178
|
}
|
|
270
179
|
|
|
271
|
-
function legacyProviderStatePrefix(providerId: string | undefined, namespace: string): string {
|
|
272
|
-
return `${LEGACY_REDIS_STATE_PREFIX}:${providerId ?? "default"}:${namespace}`;
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
function legacyProviderStateKey(
|
|
276
|
-
providerId: string | undefined,
|
|
277
|
-
namespace: string,
|
|
278
|
-
key: string,
|
|
279
|
-
): string {
|
|
280
|
-
return `${legacyProviderStatePrefix(providerId, namespace)}:${key}`;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
180
|
function publicStateKey(redisKey: string, prefixes: readonly string[]): string {
|
|
284
181
|
for (const prefix of prefixes) {
|
|
285
182
|
const prefixWithSeparator = `${prefix}:`;
|
|
@@ -391,10 +288,6 @@ class RedisProviderStateNamespace implements ProviderStateNamespace {
|
|
|
391
288
|
);
|
|
392
289
|
}
|
|
393
290
|
|
|
394
|
-
private legacyRedisKey(key: string): string {
|
|
395
|
-
return legacyProviderStateKey(this.providerId, this.namespaceName, key);
|
|
396
|
-
}
|
|
397
|
-
|
|
398
291
|
private statePrefix(): string {
|
|
399
292
|
return providerStatePrefix(
|
|
400
293
|
this.providerId,
|
|
@@ -418,10 +311,6 @@ class RedisProviderStateNamespace implements ProviderStateNamespace {
|
|
|
418
311
|
return `${REDIS_STATE_PREFIX}:index:${namespaceIdentity}`;
|
|
419
312
|
}
|
|
420
313
|
|
|
421
|
-
private canClaimLegacy(): boolean {
|
|
422
|
-
return this.scopeDiscriminator !== MISSING_CONNECTION_SCOPE_DISCRIMINATOR;
|
|
423
|
-
}
|
|
424
|
-
|
|
425
314
|
private async indexedKeys(limit: number): Promise<string[]> {
|
|
426
315
|
await requireRedisReady(this.backend.redis);
|
|
427
316
|
const now = Date.now();
|
|
@@ -440,27 +329,7 @@ class RedisProviderStateNamespace implements ProviderStateNamespace {
|
|
|
440
329
|
|
|
441
330
|
private async readRaw(key: string): Promise<string | null> {
|
|
442
331
|
await requireRedisReady(this.backend.redis);
|
|
443
|
-
|
|
444
|
-
return await withRequiredRedis(() => this.backend.redis.get(this.redisKey(key)));
|
|
445
|
-
}
|
|
446
|
-
const tombstone = JSON.stringify({ deleted: true } satisfies RedisStateTombstone);
|
|
447
|
-
const result = await withRequiredRedis(() =>
|
|
448
|
-
this.backend.redis.eval(
|
|
449
|
-
CLAIM_LEGACY_SCRIPT,
|
|
450
|
-
3,
|
|
451
|
-
this.redisKey(key),
|
|
452
|
-
this.indexKey(),
|
|
453
|
-
this.legacyRedisKey(key),
|
|
454
|
-
String(Date.now()),
|
|
455
|
-
String(this.options.maxEntries),
|
|
456
|
-
String(parseStateDurationMs(this.options.maxTtl)),
|
|
457
|
-
tombstone,
|
|
458
|
-
),
|
|
459
|
-
);
|
|
460
|
-
if (Array.isArray(result) && Number(result[0]) === 0) {
|
|
461
|
-
throw this.quotaExceeded();
|
|
462
|
-
}
|
|
463
|
-
return Array.isArray(result) && typeof result[1] === "string" ? result[1] : null;
|
|
332
|
+
return await withRequiredRedis(() => this.backend.redis.get(this.redisKey(key)));
|
|
464
333
|
}
|
|
465
334
|
|
|
466
335
|
private enforceValueSize(value: unknown): void {
|
|
@@ -534,17 +403,14 @@ class RedisProviderStateNamespace implements ProviderStateNamespace {
|
|
|
534
403
|
const result = await withRequiredRedis(() =>
|
|
535
404
|
this.backend.redis.eval(
|
|
536
405
|
SET_WITH_QUOTA_SCRIPT,
|
|
537
|
-
|
|
406
|
+
2,
|
|
538
407
|
this.redisKey(key),
|
|
539
408
|
this.indexKey(),
|
|
540
|
-
this.legacyRedisKey(key),
|
|
541
409
|
String(Date.now()),
|
|
542
410
|
String(this.options.maxEntries),
|
|
543
411
|
String(timing.expiresAtMs),
|
|
544
412
|
String(timing.indexTtlMs),
|
|
545
413
|
JSON.stringify(envelope),
|
|
546
|
-
this.canClaimLegacy() ? "1" : "0",
|
|
547
|
-
JSON.stringify({ deleted: true } satisfies RedisStateTombstone),
|
|
548
414
|
),
|
|
549
415
|
);
|
|
550
416
|
if (!Array.isArray(result) || Number(result[0]) !== 1) {
|
|
@@ -590,18 +456,15 @@ class RedisProviderStateNamespace implements ProviderStateNamespace {
|
|
|
590
456
|
const result = await withRequiredRedis(() =>
|
|
591
457
|
this.backend.redis.eval(
|
|
592
458
|
COMPARE_AND_SET_WITH_QUOTA_SCRIPT,
|
|
593
|
-
|
|
459
|
+
2,
|
|
594
460
|
this.redisKey(key),
|
|
595
461
|
this.indexKey(),
|
|
596
|
-
this.legacyRedisKey(key),
|
|
597
462
|
String(expectedVersion),
|
|
598
463
|
String(Date.now()),
|
|
599
464
|
String(this.options.maxEntries),
|
|
600
465
|
String(timing.expiresAtMs),
|
|
601
466
|
String(timing.indexTtlMs),
|
|
602
467
|
JSON.stringify(envelope),
|
|
603
|
-
this.canClaimLegacy() ? "1" : "0",
|
|
604
|
-
JSON.stringify({ deleted: true } satisfies RedisStateTombstone),
|
|
605
468
|
),
|
|
606
469
|
);
|
|
607
470
|
if (Array.isArray(result) && Number(result[0]) === 0) {
|
|
@@ -627,13 +490,11 @@ class RedisProviderStateNamespace implements ProviderStateNamespace {
|
|
|
627
490
|
await withRequiredRedis(() =>
|
|
628
491
|
this.backend.redis.eval(
|
|
629
492
|
DELETE_WITH_INDEX_SCRIPT,
|
|
630
|
-
|
|
493
|
+
2,
|
|
631
494
|
this.redisKey(key),
|
|
632
495
|
this.indexKey(),
|
|
633
|
-
this.legacyRedisKey(key),
|
|
634
496
|
tombstone,
|
|
635
497
|
String(parseStateDurationMs(this.options.maxTtl)),
|
|
636
|
-
this.canClaimLegacy() ? "1" : "0",
|
|
637
498
|
),
|
|
638
499
|
);
|
|
639
500
|
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
2
|
+
|
|
3
|
+
import { SDKError, StealthCookieStoreVersionError } from "../errors.js";
|
|
4
|
+
import type {
|
|
5
|
+
CookieJar,
|
|
6
|
+
StealthCookieStore,
|
|
7
|
+
StealthCookieStoreV1,
|
|
8
|
+
StealthSessionCookies,
|
|
9
|
+
} from "../types.js";
|
|
10
|
+
|
|
11
|
+
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
12
|
+
return typeof value === "object" && value !== null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const LEGACY_COOKIE_ORIGIN = "https://legacy-cookie.invalid/";
|
|
16
|
+
|
|
17
|
+
export class StealthCookieJar implements CookieJar, StealthSessionCookies {
|
|
18
|
+
private cookies: ToughCookieJar;
|
|
19
|
+
private readonly defaultUrl: string;
|
|
20
|
+
|
|
21
|
+
constructor(cookieStrings: readonly string[], defaultUrl = LEGACY_COOKIE_ORIGIN) {
|
|
22
|
+
this.cookies = new ToughCookieJar(undefined, {
|
|
23
|
+
allowSecureOnLocal: false,
|
|
24
|
+
rejectPublicSuffixes: true,
|
|
25
|
+
});
|
|
26
|
+
this.defaultUrl = this.normalizeUrl(defaultUrl) ?? LEGACY_COOKIE_ORIGIN;
|
|
27
|
+
this.setFromCookieStrings(cookieStrings);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
setFromCookieStrings(cookieStrings: readonly string[], url = this.defaultUrl): void {
|
|
31
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
32
|
+
if (!cookieUrl) return;
|
|
33
|
+
for (const cookieString of cookieStrings) {
|
|
34
|
+
this.cookies.setCookieSync(cookieString, cookieUrl, { ignoreError: true });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
get(name: string, url?: string): string | undefined {
|
|
39
|
+
return this.getAll(url)[name];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
getAll(url?: string): Record<string, string> {
|
|
43
|
+
return Object.fromEntries(
|
|
44
|
+
this.getUniqueCookies(url ?? this.defaultUrl).map((cookie) => [cookie.key, cookie.value]),
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
has(name: string, url?: string): boolean {
|
|
49
|
+
return Object.hasOwn(this.getAll(url), name);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
toString(url?: string): string {
|
|
53
|
+
return this.getUniqueCookies(url ?? this.defaultUrl)
|
|
54
|
+
.map((cookie) => cookie.cookieString())
|
|
55
|
+
.join("; ");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
toHeader(url?: string): string {
|
|
59
|
+
return this.toString(url);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
snapshot(): Record<string, string> {
|
|
63
|
+
const entries: [string, string][] = [];
|
|
64
|
+
for (const cookie of this.serialize().jar.cookies) {
|
|
65
|
+
if (typeof cookie.key === "string" && typeof cookie.value === "string" && cookie.key) {
|
|
66
|
+
entries.push([cookie.key, cookie.value]);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return Object.fromEntries(entries);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
restore(cookies: Record<string, string>): void {
|
|
73
|
+
this.clear();
|
|
74
|
+
for (const [name, value] of Object.entries(cookies)) {
|
|
75
|
+
if (!name) continue;
|
|
76
|
+
this.cookies.setCookieSync(new Cookie({ key: name, path: "/", value }), this.defaultUrl, {
|
|
77
|
+
ignoreError: true,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
serialize(): StealthCookieStoreV1 {
|
|
83
|
+
const jar = this.cookies.serializeSync();
|
|
84
|
+
if (!jar) {
|
|
85
|
+
throw new SDKError("Stealth cookie store could not be serialized", {
|
|
86
|
+
code: "stealth_cookie_store_serialize_failed",
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
return { version: 1, jar };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
deserialize(state: StealthCookieStore): void {
|
|
93
|
+
const version = isRecord(state) ? state.version : undefined;
|
|
94
|
+
if (version !== 1) {
|
|
95
|
+
throw new StealthCookieStoreVersionError(version);
|
|
96
|
+
}
|
|
97
|
+
const restored = ToughCookieJar.deserializeSync(state.jar);
|
|
98
|
+
Reflect.set(restored, "allowSecureOnLocal", false);
|
|
99
|
+
this.cookies = restored;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
clear(): void {
|
|
103
|
+
this.cookies.removeAllCookiesSync();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
find(predicate: (cookie: string) => boolean, url?: string): string | undefined {
|
|
107
|
+
for (const cookie of this.getUniqueCookies(url ?? this.defaultUrl)) {
|
|
108
|
+
const cookieString = cookie.cookieString();
|
|
109
|
+
if (predicate(cookieString)) return cookieString;
|
|
110
|
+
}
|
|
111
|
+
return undefined;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
private normalizeUrl(url: string): string | undefined {
|
|
115
|
+
try {
|
|
116
|
+
return new URL(url).toString();
|
|
117
|
+
} catch {
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private getUniqueCookies(url: string): Cookie[] {
|
|
123
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
124
|
+
if (!cookieUrl) return [];
|
|
125
|
+
const names = new Set<string>();
|
|
126
|
+
return this.cookies.getCookiesSync(cookieUrl).filter((cookie) => {
|
|
127
|
+
if (names.has(cookie.key)) return false;
|
|
128
|
+
names.add(cookie.key);
|
|
129
|
+
return true;
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|
package/src/runtime/stealth.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
3
2
|
import type {
|
|
4
3
|
BrowserProfile,
|
|
5
4
|
EmulationOS,
|
|
@@ -18,19 +17,17 @@ import {
|
|
|
18
17
|
resolveProxyConfigAsync,
|
|
19
18
|
vendorFromResolvedSource,
|
|
20
19
|
} from "../config/loader.js";
|
|
21
|
-
import { SDKError,
|
|
20
|
+
import { SDKError, TransportError } from "../errors.js";
|
|
22
21
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
23
22
|
import type {
|
|
24
|
-
CookieJar,
|
|
25
23
|
HttpMethod,
|
|
26
24
|
StealthClient,
|
|
27
|
-
StealthCookieStore,
|
|
28
|
-
StealthCookieStoreV1,
|
|
29
25
|
StealthFetchOptions,
|
|
30
26
|
StealthRedirectHop,
|
|
31
27
|
StealthResponse,
|
|
32
28
|
StealthSession,
|
|
33
29
|
} from "../types.js";
|
|
30
|
+
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
34
31
|
import {
|
|
35
32
|
createProxyAuthIpDeniedError,
|
|
36
33
|
createProxyEdgeAuthRejectedError,
|
|
@@ -90,6 +87,10 @@ const REDIRECT_BODY_HEADERS = new Set([
|
|
|
90
87
|
"content-type",
|
|
91
88
|
]);
|
|
92
89
|
|
|
90
|
+
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
91
|
+
return typeof value === "object" && value !== null;
|
|
92
|
+
}
|
|
93
|
+
|
|
93
94
|
function sensitiveQueryParamNames(url: string): string[] {
|
|
94
95
|
const queryStart = url.indexOf("?");
|
|
95
96
|
if (queryStart === -1) return [];
|
|
@@ -156,151 +157,6 @@ type WreqSessionCacheEntry = {
|
|
|
156
157
|
tail: Promise<void>;
|
|
157
158
|
};
|
|
158
159
|
|
|
159
|
-
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
160
|
-
return typeof value === "object" && value !== null;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
const LEGACY_COOKIE_ORIGIN = "https://legacy-cookie.invalid/";
|
|
164
|
-
|
|
165
|
-
class CookieJarImpl implements CookieJar {
|
|
166
|
-
private cookies: ToughCookieJar;
|
|
167
|
-
private readonly defaultUrl: string;
|
|
168
|
-
|
|
169
|
-
constructor(cookieStrings: readonly string[], defaultUrl = LEGACY_COOKIE_ORIGIN) {
|
|
170
|
-
this.cookies = new ToughCookieJar(undefined, {
|
|
171
|
-
allowSecureOnLocal: false,
|
|
172
|
-
rejectPublicSuffixes: true,
|
|
173
|
-
});
|
|
174
|
-
this.defaultUrl = this.normalizeUrl(defaultUrl) ?? LEGACY_COOKIE_ORIGIN;
|
|
175
|
-
this.setFromCookieStrings(cookieStrings);
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/**
|
|
179
|
-
* URL-less legacy operations are scoped to this jar's default URL. Session
|
|
180
|
-
* jars use the client's base URL and response jars use the response URL. A
|
|
181
|
-
* flat restore has no attributes to recover, so it creates host-only Path=/
|
|
182
|
-
* cookies for that default URL instead of making them visible to every host.
|
|
183
|
-
*/
|
|
184
|
-
setFromCookieStrings(cookieStrings: readonly string[], url = this.defaultUrl): void {
|
|
185
|
-
const cookieUrl = this.normalizeUrl(url);
|
|
186
|
-
if (!cookieUrl) return;
|
|
187
|
-
|
|
188
|
-
for (const cookieString of cookieStrings) {
|
|
189
|
-
this.cookies.setCookieSync(cookieString, cookieUrl, { ignoreError: true });
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
get(name: string, url?: string): string | undefined {
|
|
194
|
-
return this.getAll(url)[name];
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
getAll(url?: string): Record<string, string> {
|
|
198
|
-
return Object.fromEntries(
|
|
199
|
-
this.getUniqueCookies(url ?? this.defaultUrl).map((cookie) => [cookie.key, cookie.value]),
|
|
200
|
-
);
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
has(name: string, url?: string): boolean {
|
|
204
|
-
return Object.hasOwn(this.getAll(url), name);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
toString(url?: string): string {
|
|
208
|
-
return this.getUniqueCookies(url ?? this.defaultUrl)
|
|
209
|
-
.map((cookie) => cookie.cookieString())
|
|
210
|
-
.join("; ");
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
toHeader(url?: string): string {
|
|
214
|
-
return this.toString(url);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
snapshot(): Record<string, string> {
|
|
218
|
-
// This compatibility view deliberately enumerates the serialized store,
|
|
219
|
-
// not getAll(defaultUrl): persistence must include sibling hosts and paths.
|
|
220
|
-
// Duplicate names still collapse because a flat map cannot represent them.
|
|
221
|
-
const entries: [string, string][] = [];
|
|
222
|
-
for (const cookie of this.serialize().jar.cookies) {
|
|
223
|
-
if (typeof cookie.key === "string" && typeof cookie.value === "string" && cookie.key) {
|
|
224
|
-
entries.push([cookie.key, cookie.value]);
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
return Object.fromEntries(entries);
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
restore(cookies: Record<string, string>): void {
|
|
231
|
-
this.clear();
|
|
232
|
-
for (const [name, value] of Object.entries(cookies)) {
|
|
233
|
-
if (!name) continue;
|
|
234
|
-
this.cookies.setCookieSync(new Cookie({ key: name, path: "/", value }), this.defaultUrl, {
|
|
235
|
-
ignoreError: true,
|
|
236
|
-
});
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
serialize(): StealthCookieStoreV1 {
|
|
241
|
-
const jar = this.cookies.serializeSync();
|
|
242
|
-
if (!jar) {
|
|
243
|
-
throw new SDKError("Stealth cookie store could not be serialized", {
|
|
244
|
-
code: "stealth_cookie_store_serialize_failed",
|
|
245
|
-
});
|
|
246
|
-
}
|
|
247
|
-
return { version: 1, jar };
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
deserialize(state: StealthCookieStore): void {
|
|
251
|
-
const version = isRecord(state) ? state.version : undefined;
|
|
252
|
-
if (version !== 1) {
|
|
253
|
-
throw new StealthCookieStoreVersionError(version);
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
// Deserialize into a new jar first so invalid state cannot partially clear
|
|
257
|
-
// or replace a live session. tough-cookie restores the cookie attributes and
|
|
258
|
-
// matching semantics represented in its own serialized format.
|
|
259
|
-
const restored = ToughCookieJar.deserializeSync(state.jar);
|
|
260
|
-
// tough-cookie 6 does not include this option in serializeSync(). Preserve
|
|
261
|
-
// the SDK's stricter setting across restoration.
|
|
262
|
-
Reflect.set(restored, "allowSecureOnLocal", false);
|
|
263
|
-
this.cookies = restored;
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
clear(): void {
|
|
267
|
-
this.cookies.removeAllCookiesSync();
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
find(predicate: (cookie: string) => boolean, url?: string): string | undefined {
|
|
271
|
-
for (const cookie of this.getUniqueCookies(url ?? this.defaultUrl)) {
|
|
272
|
-
const cookieString = cookie.cookieString();
|
|
273
|
-
if (predicate(cookieString)) {
|
|
274
|
-
return cookieString;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
return undefined;
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
private normalizeUrl(url: string): string | undefined {
|
|
282
|
-
try {
|
|
283
|
-
return new URL(url).toString();
|
|
284
|
-
} catch {
|
|
285
|
-
return undefined;
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
private getUniqueCookies(url: string): Cookie[] {
|
|
290
|
-
const cookieUrl = this.normalizeUrl(url);
|
|
291
|
-
if (!cookieUrl) return [];
|
|
292
|
-
|
|
293
|
-
// tough-cookie returns longer (more-specific) paths first. Keeping the
|
|
294
|
-
// first cookie for each name prevents ambiguous duplicate-name headers.
|
|
295
|
-
const names = new Set<string>();
|
|
296
|
-
return this.cookies.getCookiesSync(cookieUrl).filter((cookie) => {
|
|
297
|
-
if (names.has(cookie.key)) return false;
|
|
298
|
-
names.add(cookie.key);
|
|
299
|
-
return true;
|
|
300
|
-
});
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
|
|
304
160
|
type WreqModule = typeof import("wreq-js");
|
|
305
161
|
|
|
306
162
|
let wreqModulePromise: Promise<WreqModule> | undefined;
|
|
@@ -325,9 +181,10 @@ function parseProfileIdentifier(identifier: string): {
|
|
|
325
181
|
family: string;
|
|
326
182
|
version: number[];
|
|
327
183
|
} | null {
|
|
328
|
-
const match =
|
|
329
|
-
|
|
330
|
-
|
|
184
|
+
const match =
|
|
185
|
+
/^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(
|
|
186
|
+
identifier.toLowerCase(),
|
|
187
|
+
);
|
|
331
188
|
if (!match?.[1] || !match[2]) return null;
|
|
332
189
|
return {
|
|
333
190
|
family: match[1],
|
|
@@ -499,7 +356,7 @@ export async function normalizeResponse(
|
|
|
499
356
|
maxBodyBytes?: number,
|
|
500
357
|
): Promise<StealthResponse> {
|
|
501
358
|
const headers = Object.fromEntries(response.headers.entries());
|
|
502
|
-
const cookies = new
|
|
359
|
+
const cookies = new StealthCookieJar(
|
|
503
360
|
setCookieHeadersFromResponse(response.headers),
|
|
504
361
|
response.url ?? requestUrl,
|
|
505
362
|
);
|
|
@@ -826,7 +683,7 @@ function discardStealthRedirectBody(response: StealthTransportResponse): void {
|
|
|
826
683
|
|
|
827
684
|
async function fetchStealthRedirectChain(
|
|
828
685
|
transport: WreqSession,
|
|
829
|
-
cookieJar:
|
|
686
|
+
cookieJar: StealthCookieJar,
|
|
830
687
|
requestUrl: string,
|
|
831
688
|
method: StealthMethod,
|
|
832
689
|
options: StealthFetchOptions,
|
|
@@ -915,7 +772,7 @@ function createSessionFetcher(
|
|
|
915
772
|
let closed = false;
|
|
916
773
|
let hasWarnedMissingProxy = false;
|
|
917
774
|
const warn = clientOptions.warn ?? console.warn;
|
|
918
|
-
const cookieJar = new
|
|
775
|
+
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
919
776
|
|
|
920
777
|
async function getClientEntry(
|
|
921
778
|
profileName: string,
|
|
@@ -962,7 +819,7 @@ function createSessionFetcher(
|
|
|
962
819
|
return await operation(await entry.session);
|
|
963
820
|
} finally {
|
|
964
821
|
release();
|
|
965
|
-
|
|
822
|
+
}
|
|
966
823
|
}
|
|
967
824
|
|
|
968
825
|
async function resolveRequestProxy(
|