@apifuse/provider-sdk 2.1.0-beta.2 → 2.1.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +172 -8
- package/CHANGELOG.md +15 -1
- package/README.md +29 -15
- package/SUBMISSION.md +86 -0
- package/bin/apifuse-dev.ts +12 -5
- package/bin/apifuse-pack-check.ts +17 -2
- package/bin/apifuse-pack-smoke.ts +133 -6
- package/bin/apifuse-perf.ts +19 -15
- package/bin/apifuse-record.ts +41 -53
- package/bin/apifuse-submit-check.ts +1052 -0
- package/bin/apifuse.ts +1 -1
- package/package.json +19 -9
- package/src/choice-token.ts +164 -0
- package/src/cli/commands.ts +24 -3
- package/src/cli/create.ts +166 -51
- package/src/cli/templates/provider/README.md.tpl +66 -7
- package/src/cli/templates/provider/dev.ts.tpl +1 -1
- package/src/cli/templates/provider/domain/README.md.tpl +3 -0
- package/src/cli/templates/provider/index.ts.tpl +5 -47
- package/src/cli/templates/provider/mappers/README.md.tpl +3 -0
- package/src/cli/templates/provider/meta.ts.tpl +7 -0
- package/src/cli/templates/provider/operations/index.ts.tpl +5 -0
- package/src/cli/templates/provider/operations/ping.ts.tpl +23 -0
- package/src/cli/templates/provider/schemas/ping.ts.tpl +16 -0
- package/src/cli/templates/provider/start.ts.tpl +1 -1
- package/src/cli/templates/provider/upstream/README.md.tpl +3 -0
- package/src/config/loader.ts +1206 -9
- package/src/define.ts +1648 -43
- package/src/errors.ts +12 -0
- package/src/i18n/catalog.ts +121 -0
- package/src/i18n/index.ts +2 -0
- package/src/i18n/keys.ts +64 -0
- package/src/index.ts +152 -8
- package/src/lint.ts +297 -42
- package/src/observability.ts +41 -0
- package/src/provider.ts +60 -3
- package/src/public-schema-field-lint.ts +237 -0
- package/src/runtime/auth-flow.ts +7 -0
- package/src/runtime/browser.ts +77 -21
- package/src/runtime/cache.ts +582 -0
- package/src/runtime/executor.ts +13 -1
- package/src/runtime/http.ts +939 -195
- package/src/runtime/insights.ts +11 -11
- package/src/runtime/instrumentation.ts +12 -4
- package/src/runtime/key-derivation.ts +1 -1
- package/src/runtime/keyring.ts +4 -3
- package/src/runtime/proxy-errors.ts +132 -0
- package/src/runtime/proxy-telemetry.ts +253 -0
- package/src/runtime/request-options.ts +66 -0
- package/src/runtime/state.ts +76 -0
- package/src/runtime/stealth.ts +1145 -0
- package/src/runtime/stt.ts +629 -0
- package/src/schema.ts +363 -1
- package/src/server/serve.ts +827 -60
- package/src/server/types.ts +35 -0
- package/src/stream.ts +210 -0
- package/src/testing/run.ts +17 -4
- package/src/types.ts +889 -50
- package/src/runtime/tls.ts +0 -434
- package/src/types/playwright-stealth.d.ts +0 -9
package/src/runtime/insights.ts
CHANGED
|
@@ -27,7 +27,7 @@ const DNS_WARN_MS = 5;
|
|
|
27
27
|
const BROWSER_IDLE_MS = 5_000;
|
|
28
28
|
const REFRESH_WARN_RATE = 0.1;
|
|
29
29
|
|
|
30
|
-
const TLS_REUSE_FIX = `const session = ctx.
|
|
30
|
+
const TLS_REUSE_FIX = `const session = ctx.stealth.createSession({ profile: 'chrome-146' });
|
|
31
31
|
const resp = await session.fetch(url, opts);`;
|
|
32
32
|
|
|
33
33
|
const TRANSFORM_FIX = `transformResponse: (raw) => {
|
|
@@ -39,11 +39,11 @@ const LARGE_RESPONSE_FIX = `const resp = await ctx.http.get('/items', {
|
|
|
39
39
|
});`;
|
|
40
40
|
|
|
41
41
|
const DNS_FIX = `// Enable DNS caching or reuse a long-lived session per host.
|
|
42
|
-
const session = ctx.
|
|
42
|
+
const session = ctx.stealth.createSession({ profile: 'chrome-146' });
|
|
43
43
|
await session.fetch(url, opts);`;
|
|
44
44
|
|
|
45
45
|
const PROXY_FIX = `// Re-check whether this operation really needs a proxy.
|
|
46
|
-
await ctx.
|
|
46
|
+
await ctx.stealth.fetch(url, { ...opts, proxy: undefined });`;
|
|
47
47
|
|
|
48
48
|
const BROWSER_FIX = `await page.waitForSelector('[data-ready="true"]', {
|
|
49
49
|
timeout: 5_000,
|
|
@@ -128,12 +128,12 @@ function makeInsight(
|
|
|
128
128
|
};
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
-
function
|
|
132
|
-
return span.name.startsWith("
|
|
131
|
+
function isStealthSpan(span: Span): boolean {
|
|
132
|
+
return span.name.startsWith("stealth.");
|
|
133
133
|
}
|
|
134
134
|
|
|
135
135
|
function isRequestSpan(span: Span): boolean {
|
|
136
|
-
return span.name === "
|
|
136
|
+
return span.name === "stealth.fetch" || span.name.startsWith("http.");
|
|
137
137
|
}
|
|
138
138
|
|
|
139
139
|
function isBrowserSpan(span: Span): boolean {
|
|
@@ -146,11 +146,11 @@ function hasProxy(span: Span): boolean {
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
function getTlsReuseInsight(spans: Span[]): InsightResult {
|
|
149
|
-
const tlsSpans = spans.filter(
|
|
149
|
+
const tlsSpans = spans.filter(isStealthSpan);
|
|
150
150
|
if (tlsSpans.length === 0) {
|
|
151
151
|
return {
|
|
152
152
|
triggered: false,
|
|
153
|
-
message: "✓
|
|
153
|
+
message: "✓ Stealth connection reuse: no stealth spans sampled yet",
|
|
154
154
|
};
|
|
155
155
|
}
|
|
156
156
|
|
|
@@ -162,14 +162,14 @@ function getTlsReuseInsight(spans: Span[]): InsightResult {
|
|
|
162
162
|
if (1 - reuseRate >= 0.8) {
|
|
163
163
|
return {
|
|
164
164
|
triggered: true,
|
|
165
|
-
message: `⚠
|
|
165
|
+
message: `⚠ Stealth connection reuse: ${formatPercent(reuseRate * 100)} reused — stealth handshakes are happening on most requests`,
|
|
166
166
|
fix: TLS_REUSE_FIX,
|
|
167
167
|
};
|
|
168
168
|
}
|
|
169
169
|
|
|
170
170
|
return {
|
|
171
171
|
triggered: false,
|
|
172
|
-
message: `✓
|
|
172
|
+
message: `✓ Stealth connection reuse: ${formatPercent(reuseRate * 100)} (good)`,
|
|
173
173
|
};
|
|
174
174
|
}
|
|
175
175
|
|
|
@@ -237,7 +237,7 @@ function getDnsRepeatedCandidate(spans: Span[]): DnsHostInsight | null {
|
|
|
237
237
|
{ dnsDurations: number[]; reuseCount: number; totalCount: number }
|
|
238
238
|
>();
|
|
239
239
|
|
|
240
|
-
for (const span of spans.filter(
|
|
240
|
+
for (const span of spans.filter(isStealthSpan)) {
|
|
241
241
|
const hostname = parseHostname(getStringAttribute(span, "url"));
|
|
242
242
|
const dnsMs = getNumberAttribute(span, "dns_ms");
|
|
243
243
|
if (!hostname || dnsMs === undefined) {
|
|
@@ -15,7 +15,12 @@ export type InstrumentedProviderContext<T extends ProviderContext> = Omit<
|
|
|
15
15
|
trace: TraceContext;
|
|
16
16
|
};
|
|
17
17
|
|
|
18
|
-
type InstrumentedNamespace =
|
|
18
|
+
type InstrumentedNamespace =
|
|
19
|
+
| "http"
|
|
20
|
+
| "stealth"
|
|
21
|
+
| "browser"
|
|
22
|
+
| "session"
|
|
23
|
+
| "state";
|
|
19
24
|
|
|
20
25
|
const BROWSER_PAGE_METHODS = new Set([
|
|
21
26
|
"goto",
|
|
@@ -105,7 +110,7 @@ function getMethod(
|
|
|
105
110
|
return methodName.toUpperCase();
|
|
106
111
|
}
|
|
107
112
|
|
|
108
|
-
if (namespace === "
|
|
113
|
+
if (namespace === "stealth") {
|
|
109
114
|
const options =
|
|
110
115
|
typeof args[1] === "object" && args[1] !== null ? args[1] : undefined;
|
|
111
116
|
if (options && "method" in options && typeof options.method === "string") {
|
|
@@ -141,7 +146,10 @@ function buildSpanAttributes(
|
|
|
141
146
|
attributes.method = method;
|
|
142
147
|
}
|
|
143
148
|
|
|
144
|
-
if (
|
|
149
|
+
if (
|
|
150
|
+
status !== undefined &&
|
|
151
|
+
(namespace === "http" || namespace === "stealth")
|
|
152
|
+
) {
|
|
145
153
|
attributes.status = status;
|
|
146
154
|
}
|
|
147
155
|
|
|
@@ -398,7 +406,7 @@ export function wrapWithInstrumentation<T extends ProviderContext>(
|
|
|
398
406
|
|
|
399
407
|
if (
|
|
400
408
|
property === "http" ||
|
|
401
|
-
property === "
|
|
409
|
+
property === "stealth" ||
|
|
402
410
|
property === "browser" ||
|
|
403
411
|
property === "session" ||
|
|
404
412
|
property === "state"
|
|
@@ -41,7 +41,7 @@ function computeInfo(providerId: string): Buffer {
|
|
|
41
41
|
export function decodeMasterKey(encoded: string): Buffer {
|
|
42
42
|
if (typeof encoded !== "string" || encoded.length === 0) {
|
|
43
43
|
throw new ConfigurationError(
|
|
44
|
-
"master key is empty; set
|
|
44
|
+
"master key is empty; set APIFUSE__KEYRING__MASTER_KEY_V{n} in the external secret manager",
|
|
45
45
|
);
|
|
46
46
|
}
|
|
47
47
|
|
package/src/runtime/keyring.ts
CHANGED
|
@@ -22,9 +22,10 @@ export interface KeyRing {
|
|
|
22
22
|
): Promise<void>;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
const DEFAULT_KEY_PREFIX = "
|
|
26
|
-
const DEFAULT_ACCEPT_LIST_VAR = "
|
|
27
|
-
const DEFAULT_WRITER_VERSION_VAR =
|
|
25
|
+
const DEFAULT_KEY_PREFIX = "APIFUSE__KEYRING__MASTER_KEY_V";
|
|
26
|
+
const DEFAULT_ACCEPT_LIST_VAR = "APIFUSE__KEYRING__MASTER_KEY_ACCEPT_LIST";
|
|
27
|
+
const DEFAULT_WRITER_VERSION_VAR =
|
|
28
|
+
"APIFUSE__KEYRING__MASTER_KEY_WRITER_VERSION";
|
|
28
29
|
|
|
29
30
|
function parseAcceptList(raw: string | undefined): number[] {
|
|
30
31
|
if (!raw || raw.trim().length === 0) {
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { TransportError } from "../errors";
|
|
2
|
+
|
|
3
|
+
export const PROXY_AUTH_IP_DENIED_CODE = "PROXY_AUTH_IP_DENIED";
|
|
4
|
+
export const PROXY_AUTH_IP_DENIED_MESSAGE =
|
|
5
|
+
"Proxy source IP is not authorized. Add the runtime egress IP to the proxy provider allowlist.";
|
|
6
|
+
export const PROXY_EDGE_AUTH_REJECTED_CODE = "PROXY_EDGE_AUTH_REJECTED";
|
|
7
|
+
export const PROXY_EDGE_AUTH_REJECTED_MESSAGE =
|
|
8
|
+
"Proxy provider rejected a candidate endpoint during authentication. The SDK will retry or refresh the proxy pool when safe.";
|
|
9
|
+
export const PROXY_POOL_STALE_CODE = "PROXY_POOL_STALE";
|
|
10
|
+
export const PROXY_EDGE_TLS_REJECTED_CODE = "PROXY_EDGE_TLS_REJECTED";
|
|
11
|
+
export const PROXY_POOL_EXHAUSTED_CODE = "PROXY_POOL_EXHAUSTED";
|
|
12
|
+
export const PROXY_POOL_EXHAUSTED_MESSAGE =
|
|
13
|
+
"Proxy provider pool was exhausted. The SDK refreshed the proxy allocation, but all candidate endpoints failed.";
|
|
14
|
+
|
|
15
|
+
const PROXY_POOL_STALE_STATUS_CODES = new Set([509, 512]);
|
|
16
|
+
const PROXY_EDGE_TLS_REJECTED_STATUS_CODES = new Set([495]);
|
|
17
|
+
|
|
18
|
+
const PROXY_AUTH_IP_DENIED_PATTERN =
|
|
19
|
+
/\b(?:source|egress|client)\s+ip\b.{0,120}\b(?:deny|denied|unauthori[sz]ed|not\s+authori[sz]ed|white\s*list|allow\s*list)\b|\b(?:white\s*list|allow\s*list)\b.{0,120}\b(?:source|egress|client)\s+ip\b/i;
|
|
20
|
+
const PROXY_EDGE_AUTH_REJECTED_PATTERN =
|
|
21
|
+
/\bauth\s+ip\s+err\b|\bproxy\b.{0,120}\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b|\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b.{0,120}\bproxy\b/i;
|
|
22
|
+
const PROXY_POOL_STALE_MESSAGE_PATTERN =
|
|
23
|
+
/\bproxy\b.{0,120}\b(?:pool|lease|expired|unavailable|exhausted|non[\s-]?200\s+code:\s*(?:509|512))\b|\bnon[\s-]?200\s+code:\s*(?:509|512)\b.{0,120}\bproxy\b|\bsmartproxy\b.{0,120}\b(?:509|512)\b/i;
|
|
24
|
+
const PROXY_EDGE_TLS_REJECTED_MESSAGE_PATTERN =
|
|
25
|
+
/\b(?:smartproxy|proxy)\b.{0,160}\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\s-]?200)\b|\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\s-]?200)\b.{0,160}\b(?:smartproxy|proxy)\b/i;
|
|
26
|
+
|
|
27
|
+
export function isProxyAuthIpDeniedMessage(message: string): boolean {
|
|
28
|
+
return PROXY_AUTH_IP_DENIED_PATTERN.test(message);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function createProxyAuthIpDeniedError(cause?: Error): TransportError {
|
|
32
|
+
return new TransportError(PROXY_AUTH_IP_DENIED_MESSAGE, {
|
|
33
|
+
code: PROXY_AUTH_IP_DENIED_CODE,
|
|
34
|
+
cause,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function isProxyEdgeAuthRejectedMessage(message: string): boolean {
|
|
39
|
+
return PROXY_EDGE_AUTH_REJECTED_PATTERN.test(message);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function createProxyEdgeAuthRejectedError(
|
|
43
|
+
cause?: Error,
|
|
44
|
+
): TransportError {
|
|
45
|
+
return new TransportError(PROXY_EDGE_AUTH_REJECTED_MESSAGE, {
|
|
46
|
+
code: PROXY_EDGE_AUTH_REJECTED_CODE,
|
|
47
|
+
cause,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function isProxyPoolStaleStatus(status: number): boolean {
|
|
52
|
+
return PROXY_POOL_STALE_STATUS_CODES.has(status);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function isProxyEdgeTlsRejectedResponse(
|
|
56
|
+
status: number,
|
|
57
|
+
evidence: string,
|
|
58
|
+
): boolean {
|
|
59
|
+
return (
|
|
60
|
+
PROXY_EDGE_TLS_REJECTED_STATUS_CODES.has(status) &&
|
|
61
|
+
PROXY_EDGE_TLS_REJECTED_MESSAGE_PATTERN.test(evidence)
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function isProxyPoolStaleMessage(message: string): boolean {
|
|
66
|
+
return PROXY_POOL_STALE_MESSAGE_PATTERN.test(message);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function isProxyPoolRefreshableError(error: unknown): boolean {
|
|
70
|
+
if (
|
|
71
|
+
error instanceof TransportError &&
|
|
72
|
+
error.code === PROXY_AUTH_IP_DENIED_CODE
|
|
73
|
+
) {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (
|
|
78
|
+
error instanceof TransportError &&
|
|
79
|
+
(error.code === PROXY_POOL_STALE_CODE ||
|
|
80
|
+
error.code === PROXY_EDGE_AUTH_REJECTED_CODE ||
|
|
81
|
+
error.code === PROXY_EDGE_TLS_REJECTED_CODE)
|
|
82
|
+
) {
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const cause = error instanceof Error ? error.cause : undefined;
|
|
87
|
+
const message = [
|
|
88
|
+
error instanceof Error ? error.message : String(error),
|
|
89
|
+
cause instanceof Error ? cause.message : "",
|
|
90
|
+
].join(" ");
|
|
91
|
+
return (
|
|
92
|
+
PROXY_POOL_STALE_MESSAGE_PATTERN.test(message) ||
|
|
93
|
+
PROXY_EDGE_AUTH_REJECTED_PATTERN.test(message)
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export const isProxyPoolStaleError = isProxyPoolRefreshableError;
|
|
98
|
+
|
|
99
|
+
export function createProxyPoolStaleError(
|
|
100
|
+
status: number,
|
|
101
|
+
cause?: Error,
|
|
102
|
+
): TransportError {
|
|
103
|
+
return new TransportError(
|
|
104
|
+
`Proxy provider pool failed with status ${status}`,
|
|
105
|
+
{
|
|
106
|
+
code: PROXY_POOL_STALE_CODE,
|
|
107
|
+
status,
|
|
108
|
+
cause,
|
|
109
|
+
},
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function createProxyEdgeTlsRejectedError(
|
|
114
|
+
status: number,
|
|
115
|
+
cause?: Error,
|
|
116
|
+
): TransportError {
|
|
117
|
+
return new TransportError(
|
|
118
|
+
`Proxy edge TLS request was rejected with status ${status}`,
|
|
119
|
+
{
|
|
120
|
+
code: PROXY_EDGE_TLS_REJECTED_CODE,
|
|
121
|
+
status,
|
|
122
|
+
cause,
|
|
123
|
+
},
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function createProxyPoolExhaustedError(cause?: Error): TransportError {
|
|
128
|
+
return new TransportError(PROXY_POOL_EXHAUSTED_MESSAGE, {
|
|
129
|
+
code: PROXY_POOL_EXHAUSTED_CODE,
|
|
130
|
+
cause,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ProxyAttemptTelemetryEvent,
|
|
3
|
+
ProxyCacheStatus,
|
|
4
|
+
ProxyResolutionTelemetryEvent,
|
|
5
|
+
ProxyTelemetrySink,
|
|
6
|
+
SmartproxyAllocatorBodyClass,
|
|
7
|
+
} from "../config/loader";
|
|
8
|
+
|
|
9
|
+
export const PROVIDER_TELEMETRY_HEADER = "X-ApiFuse-Provider-Telemetry";
|
|
10
|
+
|
|
11
|
+
type ProviderTelemetryHeader = {
|
|
12
|
+
v: 1;
|
|
13
|
+
proxy?: {
|
|
14
|
+
provider: "smartproxy";
|
|
15
|
+
cacheStatus: ProxyCacheStatus;
|
|
16
|
+
cacheHit: boolean;
|
|
17
|
+
resolutionMs: number;
|
|
18
|
+
allocatorMs?: number;
|
|
19
|
+
allocatorStatus?: number;
|
|
20
|
+
allocatorBodyClass?: SmartproxyAllocatorBodyClass;
|
|
21
|
+
allocatorAttempts?: number;
|
|
22
|
+
lockWaitMs?: number;
|
|
23
|
+
redisReadMs?: number;
|
|
24
|
+
redisWriteMs?: number;
|
|
25
|
+
poolAgeMs?: number;
|
|
26
|
+
poolExpiresInMs?: number;
|
|
27
|
+
attempts: number;
|
|
28
|
+
refreshes?: number;
|
|
29
|
+
attemptSamples?: CompactProxyAttemptSample[];
|
|
30
|
+
};
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
type CompactProxyAttemptSample = {
|
|
34
|
+
n: number;
|
|
35
|
+
a: number;
|
|
36
|
+
i?: number;
|
|
37
|
+
h?: string;
|
|
38
|
+
o: "ok" | "error";
|
|
39
|
+
c?: string;
|
|
40
|
+
s?: number;
|
|
41
|
+
d?: number;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const MAX_HEADER_BYTES = 4_096;
|
|
45
|
+
const MAX_PROXY_ATTEMPT_SAMPLES = 24;
|
|
46
|
+
|
|
47
|
+
const CACHE_STATUS_SEVERITY: Record<ProxyCacheStatus, number> = {
|
|
48
|
+
disabled: 0,
|
|
49
|
+
memory_hit: 1,
|
|
50
|
+
redis_hit: 2,
|
|
51
|
+
soft_stale_refresh: 3,
|
|
52
|
+
redis_corrupt: 4,
|
|
53
|
+
redis_error: 5,
|
|
54
|
+
lock_wait: 6,
|
|
55
|
+
allocator: 7,
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
function sumOptional(
|
|
59
|
+
left: number | undefined,
|
|
60
|
+
right: number | undefined,
|
|
61
|
+
): number | undefined {
|
|
62
|
+
const total = (left ?? 0) + (right ?? 0);
|
|
63
|
+
return total > 0 ? total : undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function maxOptional(
|
|
67
|
+
left: number | undefined,
|
|
68
|
+
right: number | undefined,
|
|
69
|
+
): number | undefined {
|
|
70
|
+
const values = [left, right].filter(
|
|
71
|
+
(value): value is number => typeof value === "number",
|
|
72
|
+
);
|
|
73
|
+
return values.length > 0 ? Math.max(...values) : undefined;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function worseStatus(
|
|
77
|
+
left: ProxyCacheStatus,
|
|
78
|
+
right: ProxyCacheStatus,
|
|
79
|
+
): ProxyCacheStatus {
|
|
80
|
+
return CACHE_STATUS_SEVERITY[right] > CACHE_STATUS_SEVERITY[left]
|
|
81
|
+
? right
|
|
82
|
+
: left;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function encodeBase64Url(value: string): string {
|
|
86
|
+
return Buffer.from(value, "utf8").toString("base64url");
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
90
|
+
#events: ProxyResolutionTelemetryEvent[] = [];
|
|
91
|
+
#attempts: ProxyAttemptTelemetryEvent[] = [];
|
|
92
|
+
|
|
93
|
+
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void {
|
|
94
|
+
this.#events.push({
|
|
95
|
+
provider: "smartproxy",
|
|
96
|
+
cacheStatus: event.cacheStatus,
|
|
97
|
+
cacheHit: event.cacheHit,
|
|
98
|
+
resolutionMs: Math.max(0, Math.floor(event.resolutionMs)),
|
|
99
|
+
allocatorMs:
|
|
100
|
+
event.allocatorMs === undefined
|
|
101
|
+
? undefined
|
|
102
|
+
: Math.max(0, Math.floor(event.allocatorMs)),
|
|
103
|
+
allocatorStatus:
|
|
104
|
+
event.allocatorStatus === undefined
|
|
105
|
+
? undefined
|
|
106
|
+
: Math.max(0, Math.floor(event.allocatorStatus)),
|
|
107
|
+
allocatorBodyClass: event.allocatorBodyClass,
|
|
108
|
+
allocatorAttempts:
|
|
109
|
+
event.allocatorAttempts === undefined
|
|
110
|
+
? undefined
|
|
111
|
+
: Math.max(1, Math.floor(event.allocatorAttempts)),
|
|
112
|
+
lockWaitMs:
|
|
113
|
+
event.lockWaitMs === undefined
|
|
114
|
+
? undefined
|
|
115
|
+
: Math.max(0, Math.floor(event.lockWaitMs)),
|
|
116
|
+
redisReadMs:
|
|
117
|
+
event.redisReadMs === undefined
|
|
118
|
+
? undefined
|
|
119
|
+
: Math.max(0, Math.floor(event.redisReadMs)),
|
|
120
|
+
redisWriteMs:
|
|
121
|
+
event.redisWriteMs === undefined
|
|
122
|
+
? undefined
|
|
123
|
+
: Math.max(0, Math.floor(event.redisWriteMs)),
|
|
124
|
+
poolAgeMs:
|
|
125
|
+
event.poolAgeMs === undefined
|
|
126
|
+
? undefined
|
|
127
|
+
: Math.max(0, Math.floor(event.poolAgeMs)),
|
|
128
|
+
poolExpiresInMs:
|
|
129
|
+
event.poolExpiresInMs === undefined
|
|
130
|
+
? undefined
|
|
131
|
+
: Math.max(0, Math.floor(event.poolExpiresInMs)),
|
|
132
|
+
attempts: Math.max(1, Math.floor(event.attempts || 1)),
|
|
133
|
+
refreshes:
|
|
134
|
+
event.refreshes === undefined
|
|
135
|
+
? undefined
|
|
136
|
+
: Math.max(0, Math.floor(event.refreshes)),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
recordProxyAttempt(event: ProxyAttemptTelemetryEvent): void {
|
|
141
|
+
if (this.#attempts.length >= MAX_PROXY_ATTEMPT_SAMPLES) return;
|
|
142
|
+
this.#attempts.push({
|
|
143
|
+
provider: "smartproxy",
|
|
144
|
+
attempt: Math.max(1, Math.floor(event.attempt || 1)),
|
|
145
|
+
...(event.poolIndex === undefined
|
|
146
|
+
? {}
|
|
147
|
+
: { poolIndex: Math.max(0, Math.floor(event.poolIndex)) }),
|
|
148
|
+
...(event.proxyHash ? { proxyHash: event.proxyHash.slice(0, 16) } : {}),
|
|
149
|
+
outcome: event.outcome === "ok" ? "ok" : "error",
|
|
150
|
+
...(event.errorCode ? { errorCode: event.errorCode.slice(0, 80) } : {}),
|
|
151
|
+
...(event.status === undefined
|
|
152
|
+
? {}
|
|
153
|
+
: { status: Math.max(0, Math.floor(event.status)) }),
|
|
154
|
+
...(event.durationMs === undefined
|
|
155
|
+
? {}
|
|
156
|
+
: { durationMs: Math.max(0, Math.floor(event.durationMs)) }),
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
toHeaderValue(): string | undefined {
|
|
161
|
+
const [first, ...rest] = this.#events;
|
|
162
|
+
if (!first) return undefined;
|
|
163
|
+
|
|
164
|
+
const aggregate = rest.reduce<ProxyResolutionTelemetryEvent>(
|
|
165
|
+
(acc, event) => ({
|
|
166
|
+
provider: "smartproxy",
|
|
167
|
+
cacheStatus: worseStatus(acc.cacheStatus, event.cacheStatus),
|
|
168
|
+
cacheHit: acc.cacheHit && event.cacheHit,
|
|
169
|
+
resolutionMs: acc.resolutionMs + event.resolutionMs,
|
|
170
|
+
allocatorMs: sumOptional(acc.allocatorMs, event.allocatorMs),
|
|
171
|
+
allocatorStatus: event.allocatorStatus ?? acc.allocatorStatus,
|
|
172
|
+
allocatorBodyClass: event.allocatorBodyClass ?? acc.allocatorBodyClass,
|
|
173
|
+
allocatorAttempts: sumOptional(
|
|
174
|
+
acc.allocatorAttempts,
|
|
175
|
+
event.allocatorAttempts,
|
|
176
|
+
),
|
|
177
|
+
lockWaitMs: sumOptional(acc.lockWaitMs, event.lockWaitMs),
|
|
178
|
+
redisReadMs: sumOptional(acc.redisReadMs, event.redisReadMs),
|
|
179
|
+
redisWriteMs: sumOptional(acc.redisWriteMs, event.redisWriteMs),
|
|
180
|
+
poolAgeMs: maxOptional(acc.poolAgeMs, event.poolAgeMs),
|
|
181
|
+
poolExpiresInMs: maxOptional(
|
|
182
|
+
acc.poolExpiresInMs,
|
|
183
|
+
event.poolExpiresInMs,
|
|
184
|
+
),
|
|
185
|
+
attempts: acc.attempts + event.attempts,
|
|
186
|
+
refreshes: sumOptional(acc.refreshes, event.refreshes),
|
|
187
|
+
}),
|
|
188
|
+
first,
|
|
189
|
+
);
|
|
190
|
+
const payload: ProviderTelemetryHeader = {
|
|
191
|
+
v: 1,
|
|
192
|
+
proxy: {
|
|
193
|
+
provider: "smartproxy",
|
|
194
|
+
cacheStatus: aggregate.cacheStatus,
|
|
195
|
+
cacheHit: aggregate.cacheHit,
|
|
196
|
+
resolutionMs: aggregate.resolutionMs,
|
|
197
|
+
...(aggregate.allocatorMs !== undefined
|
|
198
|
+
? { allocatorMs: aggregate.allocatorMs }
|
|
199
|
+
: {}),
|
|
200
|
+
...(aggregate.allocatorStatus !== undefined
|
|
201
|
+
? { allocatorStatus: aggregate.allocatorStatus }
|
|
202
|
+
: {}),
|
|
203
|
+
...(aggregate.allocatorBodyClass !== undefined
|
|
204
|
+
? { allocatorBodyClass: aggregate.allocatorBodyClass }
|
|
205
|
+
: {}),
|
|
206
|
+
...(aggregate.allocatorAttempts !== undefined
|
|
207
|
+
? { allocatorAttempts: aggregate.allocatorAttempts }
|
|
208
|
+
: {}),
|
|
209
|
+
...(aggregate.lockWaitMs !== undefined
|
|
210
|
+
? { lockWaitMs: aggregate.lockWaitMs }
|
|
211
|
+
: {}),
|
|
212
|
+
...(aggregate.redisReadMs !== undefined
|
|
213
|
+
? { redisReadMs: aggregate.redisReadMs }
|
|
214
|
+
: {}),
|
|
215
|
+
...(aggregate.redisWriteMs !== undefined
|
|
216
|
+
? { redisWriteMs: aggregate.redisWriteMs }
|
|
217
|
+
: {}),
|
|
218
|
+
...(aggregate.poolAgeMs !== undefined
|
|
219
|
+
? { poolAgeMs: aggregate.poolAgeMs }
|
|
220
|
+
: {}),
|
|
221
|
+
...(aggregate.poolExpiresInMs !== undefined
|
|
222
|
+
? { poolExpiresInMs: aggregate.poolExpiresInMs }
|
|
223
|
+
: {}),
|
|
224
|
+
attempts: aggregate.attempts,
|
|
225
|
+
...(aggregate.refreshes !== undefined
|
|
226
|
+
? { refreshes: aggregate.refreshes }
|
|
227
|
+
: {}),
|
|
228
|
+
...(this.#attempts.length > 0
|
|
229
|
+
? {
|
|
230
|
+
attemptSamples: this.#attempts.map((attempt, index) => ({
|
|
231
|
+
n: index + 1,
|
|
232
|
+
a: attempt.attempt,
|
|
233
|
+
...(attempt.poolIndex === undefined
|
|
234
|
+
? {}
|
|
235
|
+
: { i: attempt.poolIndex }),
|
|
236
|
+
...(attempt.proxyHash ? { h: attempt.proxyHash } : {}),
|
|
237
|
+
o: attempt.outcome,
|
|
238
|
+
...(attempt.errorCode ? { c: attempt.errorCode } : {}),
|
|
239
|
+
...(attempt.status === undefined ? {} : { s: attempt.status }),
|
|
240
|
+
...(attempt.durationMs === undefined
|
|
241
|
+
? {}
|
|
242
|
+
: { d: attempt.durationMs }),
|
|
243
|
+
})),
|
|
244
|
+
}
|
|
245
|
+
: {}),
|
|
246
|
+
},
|
|
247
|
+
};
|
|
248
|
+
|
|
249
|
+
const encoded = encodeBase64Url(JSON.stringify(payload));
|
|
250
|
+
if (encoded.length > MAX_HEADER_BYTES) return undefined;
|
|
251
|
+
return encoded;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
RequestParamPrimitive,
|
|
3
|
+
RequestParams,
|
|
4
|
+
RequestParamValue,
|
|
5
|
+
} from "../types";
|
|
6
|
+
|
|
7
|
+
function isParamArray(
|
|
8
|
+
value: RequestParamValue,
|
|
9
|
+
): value is readonly RequestParamPrimitive[] {
|
|
10
|
+
return Array.isArray(value);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function appendQueryValue(
|
|
14
|
+
searchParams: URLSearchParams,
|
|
15
|
+
key: string,
|
|
16
|
+
value: string | number | boolean | null | undefined,
|
|
17
|
+
): void {
|
|
18
|
+
if (value === null || value === undefined) {
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
searchParams.append(key, String(value));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function appendQueryParams(url: string, params?: RequestParams): string {
|
|
25
|
+
if (!params || Object.keys(params).length === 0) {
|
|
26
|
+
return url;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const parsed = new URL(url);
|
|
30
|
+
for (const [key, value] of Object.entries(params)) {
|
|
31
|
+
if (isParamArray(value)) {
|
|
32
|
+
for (const item of value)
|
|
33
|
+
appendQueryValue(parsed.searchParams, key, item);
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
appendQueryValue(parsed.searchParams, key, value);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return parsed.toString();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function normalizeHttpRequestBody(
|
|
43
|
+
body: unknown,
|
|
44
|
+
): string | Buffer | undefined {
|
|
45
|
+
if (body === undefined) {
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
if (typeof body === "string" || Buffer.isBuffer(body)) {
|
|
50
|
+
return body;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (body instanceof URLSearchParams) {
|
|
54
|
+
return body.toString();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (body instanceof ArrayBuffer) {
|
|
58
|
+
return Buffer.from(body);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (ArrayBuffer.isView(body)) {
|
|
62
|
+
return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return JSON.stringify(body);
|
|
66
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { ProviderError } from "../errors";
|
|
2
|
+
import type {
|
|
3
|
+
ProviderRuntimeState,
|
|
4
|
+
ProviderStateNamespace,
|
|
5
|
+
StateCasResult,
|
|
6
|
+
StateNamespaceOptions,
|
|
7
|
+
StateValue,
|
|
8
|
+
StateWriteOptions,
|
|
9
|
+
} from "../types";
|
|
10
|
+
|
|
11
|
+
export class UnsupportedProviderStateError extends ProviderError {
|
|
12
|
+
constructor(
|
|
13
|
+
message = "Provider runtime state is not available in this runtime",
|
|
14
|
+
) {
|
|
15
|
+
super(message, { code: "PROVIDER_STATE_UNSUPPORTED" });
|
|
16
|
+
this.name = "UnsupportedProviderStateError";
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
class UnsupportedProviderStateNamespace implements ProviderStateNamespace {
|
|
21
|
+
async list<T>(_options?: {
|
|
22
|
+
limit?: number;
|
|
23
|
+
prefix?: string;
|
|
24
|
+
}): Promise<StateValue<T>[]> {
|
|
25
|
+
throw new UnsupportedProviderStateError();
|
|
26
|
+
}
|
|
27
|
+
async get<T>(_key: string): Promise<StateValue<T> | null> {
|
|
28
|
+
throw new UnsupportedProviderStateError();
|
|
29
|
+
}
|
|
30
|
+
async set<T>(
|
|
31
|
+
_key: string,
|
|
32
|
+
_value: T,
|
|
33
|
+
_options?: StateWriteOptions,
|
|
34
|
+
): Promise<StateValue<T>> {
|
|
35
|
+
throw new UnsupportedProviderStateError();
|
|
36
|
+
}
|
|
37
|
+
async patch<T extends Record<string, unknown>>(
|
|
38
|
+
_key: string,
|
|
39
|
+
_partial: Partial<T>,
|
|
40
|
+
_options?: StateWriteOptions,
|
|
41
|
+
): Promise<StateValue<T>> {
|
|
42
|
+
throw new UnsupportedProviderStateError();
|
|
43
|
+
}
|
|
44
|
+
async compareAndSet<T>(
|
|
45
|
+
_key: string,
|
|
46
|
+
_expectedVersion: number,
|
|
47
|
+
_value: T,
|
|
48
|
+
_options?: StateWriteOptions,
|
|
49
|
+
): Promise<StateCasResult<T>> {
|
|
50
|
+
throw new UnsupportedProviderStateError();
|
|
51
|
+
}
|
|
52
|
+
async delete(_key: string): Promise<void> {
|
|
53
|
+
throw new UnsupportedProviderStateError();
|
|
54
|
+
}
|
|
55
|
+
async increment(
|
|
56
|
+
_key: string,
|
|
57
|
+
_field: string,
|
|
58
|
+
_delta?: number,
|
|
59
|
+
_options?: StateWriteOptions,
|
|
60
|
+
): Promise<StateValue<Record<string, unknown>>> {
|
|
61
|
+
throw new UnsupportedProviderStateError();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
class UnsupportedProviderRuntimeState implements ProviderRuntimeState {
|
|
66
|
+
namespace(
|
|
67
|
+
_name: string,
|
|
68
|
+
_options: StateNamespaceOptions,
|
|
69
|
+
): ProviderStateNamespace {
|
|
70
|
+
return new UnsupportedProviderStateNamespace();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function createUnsupportedProviderRuntimeState(): ProviderRuntimeState {
|
|
75
|
+
return new UnsupportedProviderRuntimeState();
|
|
76
|
+
}
|