@apifuse/provider-sdk 2.2.0-beta.27 → 2.2.0-beta.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/bin/apifuse-dev.ts +2 -2
- package/bin/apifuse-pack-smoke.ts +1 -1
- package/bin/apifuse-pack-types.ts +2 -1
- package/bin/apifuse-perf.ts +2 -4
- package/bin/apifuse-record.ts +1 -1
- package/dist/auth-turn/index.d.ts +1 -1
- package/dist/auth-turn/index.js +1 -1
- package/dist/ceremonies/index.js +52 -11
- package/dist/config/loader.d.ts +1 -1
- package/dist/config/loader.js +4 -2
- package/dist/index.d.ts +7 -6
- package/dist/index.js +4 -6
- package/dist/provider.d.ts +2 -1
- package/dist/provider.js +1 -1
- package/dist/runtime/auth-flow.js +1 -1
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +135 -12
- package/dist/runtime/instrumentation.js +1 -1
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +2 -33
- package/dist/runtime/native-network.js +2 -68
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-shared.d.ts +3 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +2 -1
- package/dist/runtime/resolver-vendors/twocaptcha.js +80 -43
- package/dist/runtime/resolver-vendors/types.d.ts +1 -1
- package/dist/runtime/resolver.d.ts +6 -10
- package/dist/runtime/resolver.js +19 -18
- package/dist/runtime/state.js +5 -115
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.js +7 -130
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.js +1 -1
- package/dist/server/self-test.d.ts +13 -0
- package/dist/server/self-test.js +124 -46
- package/dist/server/serve-implementation.d.ts +199 -0
- package/dist/server/serve-implementation.js +2054 -0
- package/dist/server/serve.d.ts +1 -187
- package/dist/server/serve.js +1 -1827
- package/dist/stateful/errors.d.ts +5 -0
- package/dist/stateful/errors.js +10 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +1 -5
- package/dist/stateful/stateful-provider-session-routing.js +2 -10
- package/dist/stream.js +7 -1
- package/package.json +27 -2
- package/src/auth-turn/index.ts +1 -1
- package/src/ceremonies/index.ts +68 -18
- package/src/config/loader.ts +5 -2
- package/src/index.ts +18 -23
- package/src/provider.ts +12 -14
- package/src/runtime/auth-flow.ts +1 -1
- package/src/runtime/http.ts +155 -11
- package/src/runtime/instrumentation.ts +1 -1
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +16 -97
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-shared.ts +17 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +100 -49
- package/src/runtime/resolver-vendors/types.ts +1 -0
- package/src/runtime/resolver.ts +47 -22
- package/src/runtime/state.ts +5 -144
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +14 -157
- package/src/serve.ts +6 -1
- package/src/server/index.ts +1 -0
- package/src/server/self-test.ts +184 -59
- package/src/server/serve-implementation.ts +3024 -0
- package/src/server/serve.ts +1 -2661
- package/src/stateful/errors.ts +12 -0
- package/src/stateful/stateful-provider-session-routing.ts +2 -11
- package/src/stream.ts +8 -1
package/dist/runtime/state.js
CHANGED
|
@@ -4,7 +4,6 @@ import { ProviderError } from "../errors.js";
|
|
|
4
4
|
import { createProviderRedisClient, ensureRedisReady, withRedisTimeout, } from "./redis.js";
|
|
5
5
|
const DEFAULT_REDIS_TIMEOUT_MS = 250;
|
|
6
6
|
const REDIS_STATE_PREFIX = "apifuse:provider-state:v2";
|
|
7
|
-
const LEGACY_REDIS_STATE_PREFIX = "apifuse:provider-state:v1";
|
|
8
7
|
const PROVIDER_SCOPE_DISCRIMINATOR = "scope:provider";
|
|
9
8
|
const MISSING_CONNECTION_SCOPE_DISCRIMINATOR = "scope:connection:missing";
|
|
10
9
|
const SET_WITH_QUOTA_SCRIPT = `
|
|
@@ -15,7 +14,6 @@ local index_ttl = tonumber(ARGV[4])
|
|
|
15
14
|
local envelope = ARGV[5]
|
|
16
15
|
|
|
17
16
|
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
18
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
19
17
|
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
20
18
|
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
21
19
|
return {0, false}
|
|
@@ -24,14 +22,10 @@ end
|
|
|
24
22
|
redis.call("SET", KEYS[1], envelope, "PXAT", expires_at)
|
|
25
23
|
redis.call("ZADD", KEYS[2], expires_at, KEYS[1])
|
|
26
24
|
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
27
|
-
if ARGV[6] == "1" then
|
|
28
|
-
redis.call("SET", KEYS[3], ARGV[7], "PX", index_ttl)
|
|
29
|
-
end
|
|
30
25
|
return {1, envelope}
|
|
31
26
|
`;
|
|
32
27
|
const COMPARE_AND_SET_WITH_QUOTA_SCRIPT = `
|
|
33
28
|
local current = redis.call("GET", KEYS[1])
|
|
34
|
-
local scoped_present = current and true or false
|
|
35
29
|
if current then
|
|
36
30
|
local ok, decoded = pcall(cjson.decode, current)
|
|
37
31
|
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
@@ -44,42 +38,6 @@ local max_entries = tonumber(ARGV[3])
|
|
|
44
38
|
local expires_at = tonumber(ARGV[4])
|
|
45
39
|
local index_ttl = tonumber(ARGV[5])
|
|
46
40
|
local envelope = ARGV[6]
|
|
47
|
-
local allow_legacy_claim = ARGV[7] == "1"
|
|
48
|
-
local legacy_tombstone = ARGV[8]
|
|
49
|
-
|
|
50
|
-
if not scoped_present then
|
|
51
|
-
local legacy = redis.call("GET", KEYS[3])
|
|
52
|
-
if legacy then
|
|
53
|
-
local ok, decoded = pcall(cjson.decode, legacy)
|
|
54
|
-
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
55
|
-
legacy = false
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
if legacy and not allow_legacy_claim then
|
|
59
|
-
return {-1, false}
|
|
60
|
-
end
|
|
61
|
-
if legacy then
|
|
62
|
-
local legacy_ttl = redis.call("PTTL", KEYS[3])
|
|
63
|
-
if legacy_ttl == -1 then
|
|
64
|
-
legacy_ttl = index_ttl
|
|
65
|
-
elseif legacy_ttl < 1 then
|
|
66
|
-
legacy_ttl = 1
|
|
67
|
-
end
|
|
68
|
-
if legacy_ttl > 0 then
|
|
69
|
-
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
70
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
71
|
-
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
72
|
-
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
73
|
-
return {0, false}
|
|
74
|
-
end
|
|
75
|
-
redis.call("SET", KEYS[1], legacy, "PX", legacy_ttl)
|
|
76
|
-
redis.call("ZADD", KEYS[2], now + legacy_ttl, KEYS[1])
|
|
77
|
-
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
78
|
-
redis.call("SET", KEYS[3], legacy_tombstone, "PX", index_ttl)
|
|
79
|
-
current = legacy
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
41
|
|
|
84
42
|
local current_version = 0
|
|
85
43
|
local current_decoded = false
|
|
@@ -96,7 +54,6 @@ if current_version ~= tonumber(ARGV[1]) then
|
|
|
96
54
|
end
|
|
97
55
|
|
|
98
56
|
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
99
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
100
57
|
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
101
58
|
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
102
59
|
return {0, false}
|
|
@@ -110,60 +67,13 @@ end
|
|
|
110
67
|
redis.call("SET", KEYS[1], envelope, "PXAT", expires_at)
|
|
111
68
|
redis.call("ZADD", KEYS[2], expires_at, KEYS[1])
|
|
112
69
|
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
113
|
-
if allow_legacy_claim then
|
|
114
|
-
redis.call("SET", KEYS[3], legacy_tombstone, "PX", index_ttl)
|
|
115
|
-
end
|
|
116
70
|
return {1, envelope}
|
|
117
71
|
`;
|
|
118
72
|
const DELETE_WITH_INDEX_SCRIPT = `
|
|
119
73
|
redis.call("SET", KEYS[1], ARGV[1], "PX", ARGV[2])
|
|
120
|
-
|
|
121
|
-
redis.call("SET", KEYS[3], ARGV[1], "PX", ARGV[2])
|
|
122
|
-
end
|
|
123
|
-
redis.call("ZREM", KEYS[2], KEYS[1], KEYS[3])
|
|
74
|
+
redis.call("ZREM", KEYS[2], KEYS[1])
|
|
124
75
|
return 1
|
|
125
76
|
`;
|
|
126
|
-
// A v1 key has no connection discriminator, so it cannot safely be exposed to
|
|
127
|
-
// every v2 scope. The first concrete connection (or explicit provider scope) to
|
|
128
|
-
// request a key atomically adopts that value and tombstones v1. The missing-
|
|
129
|
-
// connection sentinel never claims legacy state.
|
|
130
|
-
const CLAIM_LEGACY_SCRIPT = `
|
|
131
|
-
local scoped = redis.call("GET", KEYS[1])
|
|
132
|
-
if scoped then
|
|
133
|
-
return {1, scoped}
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
local legacy = redis.call("GET", KEYS[3])
|
|
137
|
-
if not legacy then
|
|
138
|
-
return {1, false}
|
|
139
|
-
end
|
|
140
|
-
local ok, decoded = pcall(cjson.decode, legacy)
|
|
141
|
-
if ok and type(decoded) == "table" and decoded.deleted == true then
|
|
142
|
-
return {1, false}
|
|
143
|
-
end
|
|
144
|
-
local legacy_ttl = redis.call("PTTL", KEYS[3])
|
|
145
|
-
if legacy_ttl == -1 then
|
|
146
|
-
legacy_ttl = tonumber(ARGV[3])
|
|
147
|
-
elseif legacy_ttl < 1 then
|
|
148
|
-
legacy_ttl = 1
|
|
149
|
-
end
|
|
150
|
-
|
|
151
|
-
local now = tonumber(ARGV[1])
|
|
152
|
-
local max_entries = tonumber(ARGV[2])
|
|
153
|
-
local index_ttl = tonumber(ARGV[3])
|
|
154
|
-
redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", now)
|
|
155
|
-
redis.call("ZREM", KEYS[2], KEYS[3])
|
|
156
|
-
local indexed = redis.call("ZSCORE", KEYS[2], KEYS[1])
|
|
157
|
-
if not indexed and redis.call("ZCARD", KEYS[2]) >= max_entries then
|
|
158
|
-
return {0, false}
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
redis.call("SET", KEYS[1], legacy, "PX", legacy_ttl)
|
|
162
|
-
redis.call("ZADD", KEYS[2], now + legacy_ttl, KEYS[1])
|
|
163
|
-
redis.call("PEXPIRE", KEYS[2], index_ttl)
|
|
164
|
-
redis.call("SET", KEYS[3], ARGV[4], "PX", index_ttl)
|
|
165
|
-
return {1, legacy}
|
|
166
|
-
`;
|
|
167
77
|
const redisBackends = new Map();
|
|
168
78
|
function getRedisBackend(redisUrl, injectedRedis) {
|
|
169
79
|
const existing = redisBackends.get(redisUrl);
|
|
@@ -210,12 +120,6 @@ function providerStatePrefix(providerId, namespace, scopeDiscriminator) {
|
|
|
210
120
|
function providerStateKey(providerId, namespace, scopeDiscriminator, key) {
|
|
211
121
|
return `${providerStatePrefix(providerId, namespace, scopeDiscriminator)}:${key}`;
|
|
212
122
|
}
|
|
213
|
-
function legacyProviderStatePrefix(providerId, namespace) {
|
|
214
|
-
return `${LEGACY_REDIS_STATE_PREFIX}:${providerId ?? "default"}:${namespace}`;
|
|
215
|
-
}
|
|
216
|
-
function legacyProviderStateKey(providerId, namespace, key) {
|
|
217
|
-
return `${legacyProviderStatePrefix(providerId, namespace)}:${key}`;
|
|
218
|
-
}
|
|
219
123
|
function publicStateKey(redisKey, prefixes) {
|
|
220
124
|
for (const prefix of prefixes) {
|
|
221
125
|
const prefixWithSeparator = `${prefix}:`;
|
|
@@ -311,9 +215,6 @@ class RedisProviderStateNamespace {
|
|
|
311
215
|
redisKey(key) {
|
|
312
216
|
return providerStateKey(this.providerId, this.namespaceName, this.scopeDiscriminator, key);
|
|
313
217
|
}
|
|
314
|
-
legacyRedisKey(key) {
|
|
315
|
-
return legacyProviderStateKey(this.providerId, this.namespaceName, key);
|
|
316
|
-
}
|
|
317
218
|
statePrefix() {
|
|
318
219
|
return providerStatePrefix(this.providerId, this.namespaceName, this.scopeDiscriminator);
|
|
319
220
|
}
|
|
@@ -324,9 +225,6 @@ class RedisProviderStateNamespace {
|
|
|
324
225
|
const namespaceIdentity = Buffer.from(providerStatePrefix(this.providerId, this.namespaceName, this.scopeDiscriminator), "utf8").toString("base64url");
|
|
325
226
|
return `${REDIS_STATE_PREFIX}:index:${namespaceIdentity}`;
|
|
326
227
|
}
|
|
327
|
-
canClaimLegacy() {
|
|
328
|
-
return this.scopeDiscriminator !== MISSING_CONNECTION_SCOPE_DISCRIMINATOR;
|
|
329
|
-
}
|
|
330
228
|
async indexedKeys(limit) {
|
|
331
229
|
await requireRedisReady(this.backend.redis);
|
|
332
230
|
const now = Date.now();
|
|
@@ -337,15 +235,7 @@ class RedisProviderStateNamespace {
|
|
|
337
235
|
}
|
|
338
236
|
async readRaw(key) {
|
|
339
237
|
await requireRedisReady(this.backend.redis);
|
|
340
|
-
|
|
341
|
-
return await withRequiredRedis(() => this.backend.redis.get(this.redisKey(key)));
|
|
342
|
-
}
|
|
343
|
-
const tombstone = JSON.stringify({ deleted: true });
|
|
344
|
-
const result = await withRequiredRedis(() => this.backend.redis.eval(CLAIM_LEGACY_SCRIPT, 3, this.redisKey(key), this.indexKey(), this.legacyRedisKey(key), String(Date.now()), String(this.options.maxEntries), String(parseStateDurationMs(this.options.maxTtl)), tombstone));
|
|
345
|
-
if (Array.isArray(result) && Number(result[0]) === 0) {
|
|
346
|
-
throw this.quotaExceeded();
|
|
347
|
-
}
|
|
348
|
-
return Array.isArray(result) && typeof result[1] === "string" ? result[1] : null;
|
|
238
|
+
return await withRequiredRedis(() => this.backend.redis.get(this.redisKey(key)));
|
|
349
239
|
}
|
|
350
240
|
enforceValueSize(value) {
|
|
351
241
|
const bytes = Buffer.byteLength(JSON.stringify(value), "utf8");
|
|
@@ -406,7 +296,7 @@ class RedisProviderStateNamespace {
|
|
|
406
296
|
const timing = this.writeTiming(options?.ttl);
|
|
407
297
|
const envelope = redisEnvelope(value, version, createdAt, timing.expiresAt);
|
|
408
298
|
await requireRedisReady(this.backend.redis);
|
|
409
|
-
const result = await withRequiredRedis(() => this.backend.redis.eval(SET_WITH_QUOTA_SCRIPT,
|
|
299
|
+
const result = await withRequiredRedis(() => this.backend.redis.eval(SET_WITH_QUOTA_SCRIPT, 2, this.redisKey(key), this.indexKey(), String(Date.now()), String(this.options.maxEntries), String(timing.expiresAtMs), String(timing.indexTtlMs), JSON.stringify(envelope)));
|
|
410
300
|
if (!Array.isArray(result) || Number(result[0]) !== 1) {
|
|
411
301
|
throw this.quotaExceeded();
|
|
412
302
|
}
|
|
@@ -431,7 +321,7 @@ class RedisProviderStateNamespace {
|
|
|
431
321
|
const timing = this.writeTiming(options?.ttl);
|
|
432
322
|
const envelope = redisEnvelope(value, expectedVersion + 1, createdAt, timing.expiresAt);
|
|
433
323
|
await requireRedisReady(this.backend.redis);
|
|
434
|
-
const result = await withRequiredRedis(() => this.backend.redis.eval(COMPARE_AND_SET_WITH_QUOTA_SCRIPT,
|
|
324
|
+
const result = await withRequiredRedis(() => this.backend.redis.eval(COMPARE_AND_SET_WITH_QUOTA_SCRIPT, 2, this.redisKey(key), this.indexKey(), String(expectedVersion), String(Date.now()), String(this.options.maxEntries), String(timing.expiresAtMs), String(timing.indexTtlMs), JSON.stringify(envelope)));
|
|
435
325
|
if (Array.isArray(result) && Number(result[0]) === 0) {
|
|
436
326
|
throw this.quotaExceeded();
|
|
437
327
|
}
|
|
@@ -449,7 +339,7 @@ class RedisProviderStateNamespace {
|
|
|
449
339
|
async delete(key) {
|
|
450
340
|
await requireRedisReady(this.backend.redis);
|
|
451
341
|
const tombstone = JSON.stringify({ deleted: true });
|
|
452
|
-
await withRequiredRedis(() => this.backend.redis.eval(DELETE_WITH_INDEX_SCRIPT,
|
|
342
|
+
await withRequiredRedis(() => this.backend.redis.eval(DELETE_WITH_INDEX_SCRIPT, 2, this.redisKey(key), this.indexKey(), tombstone, String(parseStateDurationMs(this.options.maxTtl))));
|
|
453
343
|
}
|
|
454
344
|
async increment(key, field, delta = 1, options) {
|
|
455
345
|
const current = (await this.get(key))?.value ?? {};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { CookieJar, StealthCookieStore, StealthCookieStoreV1, StealthSessionCookies } from "../types.js";
|
|
2
|
+
export declare class StealthCookieJar implements CookieJar, StealthSessionCookies {
|
|
3
|
+
private cookies;
|
|
4
|
+
private readonly defaultUrl;
|
|
5
|
+
constructor(cookieStrings: readonly string[], defaultUrl?: string);
|
|
6
|
+
setFromCookieStrings(cookieStrings: readonly string[], url?: string): void;
|
|
7
|
+
get(name: string, url?: string): string | undefined;
|
|
8
|
+
getAll(url?: string): Record<string, string>;
|
|
9
|
+
has(name: string, url?: string): boolean;
|
|
10
|
+
toString(url?: string): string;
|
|
11
|
+
toHeader(url?: string): string;
|
|
12
|
+
snapshot(): Record<string, string>;
|
|
13
|
+
restore(cookies: Record<string, string>): void;
|
|
14
|
+
serialize(): StealthCookieStoreV1;
|
|
15
|
+
deserialize(state: StealthCookieStore): void;
|
|
16
|
+
clear(): void;
|
|
17
|
+
find(predicate: (cookie: string) => boolean, url?: string): string | undefined;
|
|
18
|
+
private normalizeUrl;
|
|
19
|
+
private getUniqueCookies;
|
|
20
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
2
|
+
import { SDKError, StealthCookieStoreVersionError } from "../errors.js";
|
|
3
|
+
function isRecord(value) {
|
|
4
|
+
return typeof value === "object" && value !== null;
|
|
5
|
+
}
|
|
6
|
+
const LEGACY_COOKIE_ORIGIN = "https://legacy-cookie.invalid/";
|
|
7
|
+
export class StealthCookieJar {
|
|
8
|
+
cookies;
|
|
9
|
+
defaultUrl;
|
|
10
|
+
constructor(cookieStrings, defaultUrl = LEGACY_COOKIE_ORIGIN) {
|
|
11
|
+
this.cookies = new ToughCookieJar(undefined, {
|
|
12
|
+
allowSecureOnLocal: false,
|
|
13
|
+
rejectPublicSuffixes: true,
|
|
14
|
+
});
|
|
15
|
+
this.defaultUrl = this.normalizeUrl(defaultUrl) ?? LEGACY_COOKIE_ORIGIN;
|
|
16
|
+
this.setFromCookieStrings(cookieStrings);
|
|
17
|
+
}
|
|
18
|
+
setFromCookieStrings(cookieStrings, url = this.defaultUrl) {
|
|
19
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
20
|
+
if (!cookieUrl)
|
|
21
|
+
return;
|
|
22
|
+
for (const cookieString of cookieStrings) {
|
|
23
|
+
this.cookies.setCookieSync(cookieString, cookieUrl, { ignoreError: true });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
get(name, url) {
|
|
27
|
+
return this.getAll(url)[name];
|
|
28
|
+
}
|
|
29
|
+
getAll(url) {
|
|
30
|
+
return Object.fromEntries(this.getUniqueCookies(url ?? this.defaultUrl).map((cookie) => [cookie.key, cookie.value]));
|
|
31
|
+
}
|
|
32
|
+
has(name, url) {
|
|
33
|
+
return Object.hasOwn(this.getAll(url), name);
|
|
34
|
+
}
|
|
35
|
+
toString(url) {
|
|
36
|
+
return this.getUniqueCookies(url ?? this.defaultUrl)
|
|
37
|
+
.map((cookie) => cookie.cookieString())
|
|
38
|
+
.join("; ");
|
|
39
|
+
}
|
|
40
|
+
toHeader(url) {
|
|
41
|
+
return this.toString(url);
|
|
42
|
+
}
|
|
43
|
+
snapshot() {
|
|
44
|
+
const entries = [];
|
|
45
|
+
for (const cookie of this.serialize().jar.cookies) {
|
|
46
|
+
if (typeof cookie.key === "string" && typeof cookie.value === "string" && cookie.key) {
|
|
47
|
+
entries.push([cookie.key, cookie.value]);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return Object.fromEntries(entries);
|
|
51
|
+
}
|
|
52
|
+
restore(cookies) {
|
|
53
|
+
this.clear();
|
|
54
|
+
for (const [name, value] of Object.entries(cookies)) {
|
|
55
|
+
if (!name)
|
|
56
|
+
continue;
|
|
57
|
+
this.cookies.setCookieSync(new Cookie({ key: name, path: "/", value }), this.defaultUrl, {
|
|
58
|
+
ignoreError: true,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
serialize() {
|
|
63
|
+
const jar = this.cookies.serializeSync();
|
|
64
|
+
if (!jar) {
|
|
65
|
+
throw new SDKError("Stealth cookie store could not be serialized", {
|
|
66
|
+
code: "stealth_cookie_store_serialize_failed",
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
return { version: 1, jar };
|
|
70
|
+
}
|
|
71
|
+
deserialize(state) {
|
|
72
|
+
const version = isRecord(state) ? state.version : undefined;
|
|
73
|
+
if (version !== 1) {
|
|
74
|
+
throw new StealthCookieStoreVersionError(version);
|
|
75
|
+
}
|
|
76
|
+
const restored = ToughCookieJar.deserializeSync(state.jar);
|
|
77
|
+
Reflect.set(restored, "allowSecureOnLocal", false);
|
|
78
|
+
this.cookies = restored;
|
|
79
|
+
}
|
|
80
|
+
clear() {
|
|
81
|
+
this.cookies.removeAllCookiesSync();
|
|
82
|
+
}
|
|
83
|
+
find(predicate, url) {
|
|
84
|
+
for (const cookie of this.getUniqueCookies(url ?? this.defaultUrl)) {
|
|
85
|
+
const cookieString = cookie.cookieString();
|
|
86
|
+
if (predicate(cookieString))
|
|
87
|
+
return cookieString;
|
|
88
|
+
}
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
normalizeUrl(url) {
|
|
92
|
+
try {
|
|
93
|
+
return new URL(url).toString();
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
getUniqueCookies(url) {
|
|
100
|
+
const cookieUrl = this.normalizeUrl(url);
|
|
101
|
+
if (!cookieUrl)
|
|
102
|
+
return [];
|
|
103
|
+
const names = new Set();
|
|
104
|
+
return this.cookies.getCookiesSync(cookieUrl).filter((cookie) => {
|
|
105
|
+
if (names.has(cookie.key))
|
|
106
|
+
return false;
|
|
107
|
+
names.add(cookie.key);
|
|
108
|
+
return true;
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
3
2
|
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
4
|
-
import { SDKError,
|
|
3
|
+
import { SDKError, TransportError } from "../errors.js";
|
|
5
4
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
5
|
+
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
6
6
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
7
7
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
8
8
|
import { evaluateRedirectHop, isRedirectStatus, nextRedirectMethod, resolveRedirectUrl, } from "./redirects.js";
|
|
@@ -22,6 +22,9 @@ const REDIRECT_BODY_HEADERS = new Set([
|
|
|
22
22
|
"content-location",
|
|
23
23
|
"content-type",
|
|
24
24
|
]);
|
|
25
|
+
function isRecord(value) {
|
|
26
|
+
return typeof value === "object" && value !== null;
|
|
27
|
+
}
|
|
25
28
|
function sensitiveQueryParamNames(url) {
|
|
26
29
|
const queryStart = url.indexOf("?");
|
|
27
30
|
if (queryStart === -1)
|
|
@@ -43,132 +46,6 @@ const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
|
43
46
|
"chrome-130-psk",
|
|
44
47
|
"edge-131",
|
|
45
48
|
]);
|
|
46
|
-
function isRecord(value) {
|
|
47
|
-
return typeof value === "object" && value !== null;
|
|
48
|
-
}
|
|
49
|
-
const LEGACY_COOKIE_ORIGIN = "https://legacy-cookie.invalid/";
|
|
50
|
-
class CookieJarImpl {
|
|
51
|
-
cookies;
|
|
52
|
-
defaultUrl;
|
|
53
|
-
constructor(cookieStrings, defaultUrl = LEGACY_COOKIE_ORIGIN) {
|
|
54
|
-
this.cookies = new ToughCookieJar(undefined, {
|
|
55
|
-
allowSecureOnLocal: false,
|
|
56
|
-
rejectPublicSuffixes: true,
|
|
57
|
-
});
|
|
58
|
-
this.defaultUrl = this.normalizeUrl(defaultUrl) ?? LEGACY_COOKIE_ORIGIN;
|
|
59
|
-
this.setFromCookieStrings(cookieStrings);
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* URL-less legacy operations are scoped to this jar's default URL. Session
|
|
63
|
-
* jars use the client's base URL and response jars use the response URL. A
|
|
64
|
-
* flat restore has no attributes to recover, so it creates host-only Path=/
|
|
65
|
-
* cookies for that default URL instead of making them visible to every host.
|
|
66
|
-
*/
|
|
67
|
-
setFromCookieStrings(cookieStrings, url = this.defaultUrl) {
|
|
68
|
-
const cookieUrl = this.normalizeUrl(url);
|
|
69
|
-
if (!cookieUrl)
|
|
70
|
-
return;
|
|
71
|
-
for (const cookieString of cookieStrings) {
|
|
72
|
-
this.cookies.setCookieSync(cookieString, cookieUrl, { ignoreError: true });
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
get(name, url) {
|
|
76
|
-
return this.getAll(url)[name];
|
|
77
|
-
}
|
|
78
|
-
getAll(url) {
|
|
79
|
-
return Object.fromEntries(this.getUniqueCookies(url ?? this.defaultUrl).map((cookie) => [cookie.key, cookie.value]));
|
|
80
|
-
}
|
|
81
|
-
has(name, url) {
|
|
82
|
-
return Object.hasOwn(this.getAll(url), name);
|
|
83
|
-
}
|
|
84
|
-
toString(url) {
|
|
85
|
-
return this.getUniqueCookies(url ?? this.defaultUrl)
|
|
86
|
-
.map((cookie) => cookie.cookieString())
|
|
87
|
-
.join("; ");
|
|
88
|
-
}
|
|
89
|
-
toHeader(url) {
|
|
90
|
-
return this.toString(url);
|
|
91
|
-
}
|
|
92
|
-
snapshot() {
|
|
93
|
-
// This compatibility view deliberately enumerates the serialized store,
|
|
94
|
-
// not getAll(defaultUrl): persistence must include sibling hosts and paths.
|
|
95
|
-
// Duplicate names still collapse because a flat map cannot represent them.
|
|
96
|
-
const entries = [];
|
|
97
|
-
for (const cookie of this.serialize().jar.cookies) {
|
|
98
|
-
if (typeof cookie.key === "string" && typeof cookie.value === "string" && cookie.key) {
|
|
99
|
-
entries.push([cookie.key, cookie.value]);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
return Object.fromEntries(entries);
|
|
103
|
-
}
|
|
104
|
-
restore(cookies) {
|
|
105
|
-
this.clear();
|
|
106
|
-
for (const [name, value] of Object.entries(cookies)) {
|
|
107
|
-
if (!name)
|
|
108
|
-
continue;
|
|
109
|
-
this.cookies.setCookieSync(new Cookie({ key: name, path: "/", value }), this.defaultUrl, {
|
|
110
|
-
ignoreError: true,
|
|
111
|
-
});
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
serialize() {
|
|
115
|
-
const jar = this.cookies.serializeSync();
|
|
116
|
-
if (!jar) {
|
|
117
|
-
throw new SDKError("Stealth cookie store could not be serialized", {
|
|
118
|
-
code: "stealth_cookie_store_serialize_failed",
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
return { version: 1, jar };
|
|
122
|
-
}
|
|
123
|
-
deserialize(state) {
|
|
124
|
-
const version = isRecord(state) ? state.version : undefined;
|
|
125
|
-
if (version !== 1) {
|
|
126
|
-
throw new StealthCookieStoreVersionError(version);
|
|
127
|
-
}
|
|
128
|
-
// Deserialize into a new jar first so invalid state cannot partially clear
|
|
129
|
-
// or replace a live session. tough-cookie restores the cookie attributes and
|
|
130
|
-
// matching semantics represented in its own serialized format.
|
|
131
|
-
const restored = ToughCookieJar.deserializeSync(state.jar);
|
|
132
|
-
// tough-cookie 6 does not include this option in serializeSync(). Preserve
|
|
133
|
-
// the SDK's stricter setting across restoration.
|
|
134
|
-
Reflect.set(restored, "allowSecureOnLocal", false);
|
|
135
|
-
this.cookies = restored;
|
|
136
|
-
}
|
|
137
|
-
clear() {
|
|
138
|
-
this.cookies.removeAllCookiesSync();
|
|
139
|
-
}
|
|
140
|
-
find(predicate, url) {
|
|
141
|
-
for (const cookie of this.getUniqueCookies(url ?? this.defaultUrl)) {
|
|
142
|
-
const cookieString = cookie.cookieString();
|
|
143
|
-
if (predicate(cookieString)) {
|
|
144
|
-
return cookieString;
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
return undefined;
|
|
148
|
-
}
|
|
149
|
-
normalizeUrl(url) {
|
|
150
|
-
try {
|
|
151
|
-
return new URL(url).toString();
|
|
152
|
-
}
|
|
153
|
-
catch {
|
|
154
|
-
return undefined;
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
getUniqueCookies(url) {
|
|
158
|
-
const cookieUrl = this.normalizeUrl(url);
|
|
159
|
-
if (!cookieUrl)
|
|
160
|
-
return [];
|
|
161
|
-
// tough-cookie returns longer (more-specific) paths first. Keeping the
|
|
162
|
-
// first cookie for each name prevents ambiguous duplicate-name headers.
|
|
163
|
-
const names = new Set();
|
|
164
|
-
return this.cookies.getCookiesSync(cookieUrl).filter((cookie) => {
|
|
165
|
-
if (names.has(cookie.key))
|
|
166
|
-
return false;
|
|
167
|
-
names.add(cookie.key);
|
|
168
|
-
return true;
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
49
|
let wreqModulePromise;
|
|
173
50
|
function getWreqModule() {
|
|
174
51
|
if (!wreqModulePromise) {
|
|
@@ -325,7 +202,7 @@ function splitCombinedSetCookieHeader(headerValue) {
|
|
|
325
202
|
}
|
|
326
203
|
export async function normalizeResponse(response, requestUrl, maxBodyBytes) {
|
|
327
204
|
const headers = Object.fromEntries(response.headers.entries());
|
|
328
|
-
const cookies = new
|
|
205
|
+
const cookies = new StealthCookieJar(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
329
206
|
const bodyBytes = maxBodyBytes === undefined
|
|
330
207
|
? await response.arrayBuffer()
|
|
331
208
|
: await readResponseBodyWithLimit(response, maxBodyBytes);
|
|
@@ -669,7 +546,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
669
546
|
let closed = false;
|
|
670
547
|
let hasWarnedMissingProxy = false;
|
|
671
548
|
const warn = clientOptions.warn ?? console.warn;
|
|
672
|
-
const cookieJar = new
|
|
549
|
+
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
673
550
|
async function getClientEntry(profileName, proxyUrl, ignoreTlsErrors) {
|
|
674
551
|
if (closed) {
|
|
675
552
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
package/dist/serve.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { createServerApp, type ServeOptions, serve } from "./server/serve.js";
|
|
1
|
+
export { createServerApp, createServerAppAsync, type ServeOptions, serve, } from "./server/serve.js";
|
package/dist/serve.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { createServerApp, serve } from "./server/serve.js";
|
|
1
|
+
export { createServerApp, createServerAppAsync, serve, } from "./server/serve.js";
|
package/dist/server/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createServerApp, ERROR_OBSERVABILITY_HEADER, type ErrorObservabilityDetails, type ProviderServerCloseOptions, type ProviderServerHandle, type ProviderServerLogEvent, type ProviderServerLogger, type ProviderServerOperationExecutor, type ProviderServerOperationExecutorInput, type ProviderServerOptions, type ProviderServerStatefulForwardEnvelope, type ServeOptions, serve, } from "./serve.js";
|
|
1
|
+
export { createServerApp, createServerAppAsync, ERROR_OBSERVABILITY_HEADER, type ErrorObservabilityDetails, type ProviderServerCloseOptions, type ProviderServerHandle, type ProviderServerLogEvent, type ProviderServerLogger, type ProviderServerOperationExecutor, type ProviderServerOperationExecutorInput, type ProviderServerOptions, type ProviderServerStatefulForwardEnvelope, type ServeOptions, serve, } from "./serve.js";
|
|
2
2
|
export { computeSelfTestPlanDigest, createSelfTestApp, createSelfTestAuthFlowInvoke, createSelfTestInvoke, DEFAULT_SELF_TEST_REQUEST_BUDGET_MS, isSelfTestReadOnlyOperation, PROVIDER_RUNTIME_SELF_TEST_REQUEST_BUDGET_MS_ENV, resolveSelfTestPort, SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON, SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON, SELF_TEST_HEALTHZ_PATH, SELF_TEST_PATH, SELF_TEST_SCHEMA_VERSION, type SelfTestAppOptions, type SelfTestAuthFlowInvoke, type SelfTestAuthFlowRoute, type SelfTestCaseResult, type SelfTestCaseStatus, type SelfTestOperationInvoke, type SelfTestRequest, SelfTestRequestSchema, type SelfTestResponse, } from "./self-test.js";
|
|
3
3
|
export { type InputDateTokenCalendar, resolveHealthCheckInputDateTokens, } from "./self-test-input-tokens.js";
|
|
4
4
|
export { collectSelfTestSensitiveValues, redactSelfTestText, SELF_TEST_MAX_TEXT_LENGTH, SELF_TEST_REDACTED_PLACEHOLDER, } from "./self-test-redaction.js";
|
package/dist/server/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createServerApp, ERROR_OBSERVABILITY_HEADER, serve, } from "./serve.js";
|
|
1
|
+
export { createServerApp, createServerAppAsync, ERROR_OBSERVABILITY_HEADER, serve, } from "./serve.js";
|
|
2
2
|
export { computeSelfTestPlanDigest, createSelfTestApp, createSelfTestAuthFlowInvoke, createSelfTestInvoke, DEFAULT_SELF_TEST_REQUEST_BUDGET_MS, isSelfTestReadOnlyOperation, PROVIDER_RUNTIME_SELF_TEST_REQUEST_BUDGET_MS_ENV, resolveSelfTestPort, SELF_TEST_AUTH_FLOW_MULTI_TURN_SKIP_REASON, SELF_TEST_AUTH_FLOW_REJECTED_SKIP_REASON, SELF_TEST_HEALTHZ_PATH, SELF_TEST_PATH, SELF_TEST_SCHEMA_VERSION, SelfTestRequestSchema, } from "./self-test.js";
|
|
3
3
|
export { resolveHealthCheckInputDateTokens, } from "./self-test-input-tokens.js";
|
|
4
4
|
export { collectSelfTestSensitiveValues, redactSelfTestText, SELF_TEST_MAX_TEXT_LENGTH, SELF_TEST_REDACTED_PLACEHOLDER, } from "./self-test-redaction.js";
|
|
@@ -51,11 +51,21 @@ export interface SelfTestResponse {
|
|
|
51
51
|
result?: SelfTestCaseResult;
|
|
52
52
|
results: SelfTestCaseResult[];
|
|
53
53
|
}
|
|
54
|
+
export interface SelfTestCancellationLogEvent {
|
|
55
|
+
level: "info";
|
|
56
|
+
event: "self_test_run_cancelled";
|
|
57
|
+
providerId: string;
|
|
58
|
+
requestId: string;
|
|
59
|
+
operationId: string;
|
|
60
|
+
caseName: string;
|
|
61
|
+
reason: string;
|
|
62
|
+
}
|
|
54
63
|
export type SelfTestOperationInvoke = (args: {
|
|
55
64
|
operationId: string;
|
|
56
65
|
input: unknown;
|
|
57
66
|
connection?: OperationConnection;
|
|
58
67
|
requestId: string;
|
|
68
|
+
signal?: AbortSignal;
|
|
59
69
|
}) => Promise<{
|
|
60
70
|
status: number;
|
|
61
71
|
data: unknown;
|
|
@@ -78,6 +88,7 @@ export type SelfTestAuthFlowInvoke = (args: {
|
|
|
78
88
|
externalRef?: string;
|
|
79
89
|
input?: Record<string, unknown>;
|
|
80
90
|
context?: Record<string, unknown>;
|
|
91
|
+
signal?: AbortSignal;
|
|
81
92
|
}) => Promise<{
|
|
82
93
|
status: number;
|
|
83
94
|
body: unknown;
|
|
@@ -112,6 +123,8 @@ export interface SelfTestAppOptions {
|
|
|
112
123
|
requestBudgetMs?: number;
|
|
113
124
|
/** Env override for secret collection + budget resolution (tests). */
|
|
114
125
|
env?: Readonly<Record<string, string | undefined>>;
|
|
126
|
+
/** Structured server logger; defaults to the same JSON console shape as the provider server. */
|
|
127
|
+
logger?: (event: SelfTestCancellationLogEvent) => void;
|
|
115
128
|
}
|
|
116
129
|
/**
|
|
117
130
|
* Stable sha256 over the provider's declared health plan (operations, case
|