@apifuse/provider-sdk 2.2.0-beta.67 → 2.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +34 -2
- package/CHANGELOG.md +8 -0
- package/README.md +18 -7
- package/bin/apifuse-record.ts +81 -21
- package/bin/apifuse-submit-check.ts +170 -28
- package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/dist/cli/templates/provider/README.md.tpl +17 -9
- package/dist/config/loader.d.ts +16 -0
- package/dist/config/loader.js +25 -8
- package/dist/error-resolution.d.ts +13 -0
- package/dist/error-resolution.js +72 -35
- package/dist/fixture-sanitization.js +159 -15
- package/dist/lint.d.ts +2 -0
- package/dist/lint.js +549 -11
- package/dist/runtime/diagnostic-redactor.d.ts +6 -1
- package/dist/runtime/diagnostic-redactor.js +16 -14
- package/dist/runtime/executor.js +3 -0
- package/dist/runtime/http.d.ts +2 -0
- package/dist/runtime/http.js +17 -103
- package/dist/runtime/instrumentation.js +4 -1
- package/dist/runtime/provider.d.ts +7 -0
- package/dist/runtime/provider.js +14 -0
- package/dist/runtime/redirect-hop-headers.d.ts +19 -0
- package/dist/runtime/redirect-hop-headers.js +58 -0
- package/dist/runtime/redirects.d.ts +33 -1
- package/dist/runtime/redirects.js +113 -0
- package/dist/runtime/resolver-span-attributes.d.ts +34 -0
- package/dist/runtime/resolver-span-attributes.js +93 -0
- package/dist/runtime/resolver-usage.d.ts +8 -0
- package/dist/runtime/resolver-usage.js +5 -11
- package/dist/runtime/resolver-vendors/browser.js +5 -4
- package/dist/runtime/resolver-vendors/capsolver.js +11 -10
- package/dist/runtime/resolver-vendors/twocaptcha.js +9 -8
- package/dist/runtime/resolver.js +7 -6
- package/dist/runtime/stealth-owned-headers.d.ts +21 -8
- package/dist/runtime/stealth-owned-headers.js +28 -8
- package/dist/runtime/stealth-telemetry.d.ts +1 -1
- package/dist/runtime/stealth.js +158 -46
- package/dist/runtime/trace.js +2 -2
- package/dist/server/index.d.ts +1 -0
- package/dist/server/serve-implementation.d.ts +2 -1
- package/dist/server/serve-implementation.js +45 -30
- package/dist/server/state-backend-report.d.ts +70 -0
- package/dist/server/state-backend-report.js +113 -0
- package/dist/testing/run.js +14 -7
- package/dist/trace-sanitization.js +1 -1
- package/dist/types.d.ts +30 -6
- package/package.json +2 -2
- package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/src/cli/templates/provider/README.md.tpl +17 -9
- package/src/config/loader.ts +49 -12
- package/src/error-resolution.ts +89 -35
- package/src/fixture-sanitization.ts +175 -17
- package/src/lint.ts +595 -10
- package/src/runtime/diagnostic-redactor.ts +19 -13
- package/src/runtime/executor.ts +3 -0
- package/src/runtime/http.ts +24 -112
- package/src/runtime/instrumentation.ts +4 -1
- package/src/runtime/provider.ts +15 -0
- package/src/runtime/redirect-hop-headers.ts +71 -0
- package/src/runtime/redirects.ts +150 -1
- package/src/runtime/resolver-span-attributes.ts +139 -0
- package/src/runtime/resolver-usage.ts +8 -5
- package/src/runtime/resolver-vendors/browser.ts +5 -4
- package/src/runtime/resolver-vendors/capsolver.ts +11 -10
- package/src/runtime/resolver-vendors/twocaptcha.ts +9 -8
- package/src/runtime/resolver.ts +7 -6
- package/src/runtime/stealth-owned-headers.ts +34 -8
- package/src/runtime/stealth-telemetry.ts +6 -1
- package/src/runtime/stealth.ts +180 -58
- package/src/runtime/trace.ts +6 -2
- package/src/server/index.ts +4 -0
- package/src/server/serve-implementation.ts +58 -31
- package/src/server/state-backend-report.ts +185 -0
- package/src/testing/run.ts +15 -2
- package/src/trace-sanitization.ts +1 -1
- package/src/types.ts +30 -6
|
@@ -4,6 +4,7 @@ import type {
|
|
|
4
4
|
ResolverVendorUnavailableError,
|
|
5
5
|
} from "./resolver-vendors/types.js";
|
|
6
6
|
import type { TraceRecorder } from "./trace.js";
|
|
7
|
+
import { resolverSpanAttributes } from "./resolver-span-attributes.js";
|
|
7
8
|
|
|
8
9
|
export const RESOLVER_PAID_USAGE_SPAN = "resolver.usage";
|
|
9
10
|
|
|
@@ -22,9 +23,11 @@ export type ResolverPaidUsageEndpoint =
|
|
|
22
23
|
* but has no published price, so it is recorded with 0 units and `billing: "unconfirmed"`
|
|
23
24
|
* rather than invented as a paid unit; the span still proves the call was made.
|
|
24
25
|
*/
|
|
26
|
+
export type ResolverPaidUsageBilling = "metered" | "unconfirmed";
|
|
27
|
+
|
|
25
28
|
const ENDPOINT_BILLING: Record<
|
|
26
29
|
ResolverPaidUsageEndpoint,
|
|
27
|
-
{ readonly billable_units: 0 | 1; readonly billing:
|
|
30
|
+
{ readonly billable_units: 0 | 1; readonly billing: ResolverPaidUsageBilling }
|
|
28
31
|
> = {
|
|
29
32
|
"capsolver:create_task": { billable_units: 1, billing: "metered" },
|
|
30
33
|
"twocaptcha:create_task": { billable_units: 1, billing: "metered" },
|
|
@@ -78,7 +81,7 @@ export async function recordPaidResolverCreate<T>(options: {
|
|
|
78
81
|
}
|
|
79
82
|
return options.create();
|
|
80
83
|
}
|
|
81
|
-
const baseAttributes = {
|
|
84
|
+
const baseAttributes = resolverSpanAttributes({
|
|
82
85
|
vendor: options.vendor,
|
|
83
86
|
challenge_kind: options.kind,
|
|
84
87
|
endpoint: options.endpoint,
|
|
@@ -87,10 +90,10 @@ export async function recordPaidResolverCreate<T>(options: {
|
|
|
87
90
|
...(options.usage ? { vendor_index: options.usage.vendorIndex } : {}),
|
|
88
91
|
...(options.round === undefined ? {} : { round: options.round }),
|
|
89
92
|
resolver_identity_scope: options.usage?.resolverIdentityScope,
|
|
90
|
-
};
|
|
93
|
+
});
|
|
91
94
|
return options.traceRecorder.runSpan(RESOLVER_PAID_USAGE_SPAN, options.create, {
|
|
92
95
|
attributes: baseAttributes,
|
|
93
|
-
onSuccess: () => ({ outcome: "success" }),
|
|
94
|
-
onError: (error) => ({ outcome: errorOutcome(error, options.signal) }),
|
|
96
|
+
onSuccess: () => resolverSpanAttributes({ outcome: "success" }),
|
|
97
|
+
onError: (error) => resolverSpanAttributes({ outcome: errorOutcome(error, options.signal) }),
|
|
95
98
|
});
|
|
96
99
|
}
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
} from "../../types.js";
|
|
9
9
|
import { type BrowserClientOptions, createBrowserClient } from "../browser.js";
|
|
10
10
|
import type { TraceRecorder } from "../trace.js";
|
|
11
|
+
import { resolverSpanAttributes } from "../resolver-span-attributes.js";
|
|
11
12
|
import type { BrowserTelemetrySink } from "../browser-telemetry.js";
|
|
12
13
|
import { resolverChallengeIssuingIdentity } from "./bindings.js";
|
|
13
14
|
import { assertResolverHostAllowed, normalizedResolverHostname } from "./hosts.js";
|
|
@@ -218,15 +219,15 @@ async function runBoundedCleanup(
|
|
|
218
219
|
|
|
219
220
|
await traceRecorder
|
|
220
221
|
.runSpan("resolver.vendor.cleanup", boundedCleanup, {
|
|
221
|
-
attributes: {
|
|
222
|
+
attributes: resolverSpanAttributes({
|
|
222
223
|
vendor: BROWSER_VENDOR_ID,
|
|
223
224
|
challenge_kind: challengeKind,
|
|
224
225
|
operation,
|
|
225
|
-
},
|
|
226
|
+
}),
|
|
226
227
|
onError(error) {
|
|
227
|
-
return {
|
|
228
|
+
return resolverSpanAttributes({
|
|
228
229
|
error_message: error instanceof Error ? error.message : String(error),
|
|
229
|
-
};
|
|
230
|
+
});
|
|
230
231
|
},
|
|
231
232
|
})
|
|
232
233
|
.catch(() => undefined);
|
|
@@ -3,6 +3,7 @@ import type { ChallengeSolution, ProviderChallenge, ProviderChallengeKind } from
|
|
|
3
3
|
import { redactSensitiveText } from "../request-options.js";
|
|
4
4
|
import { DEFAULT_STEALTH_PROFILE } from "../stealth.js";
|
|
5
5
|
import { recordPaidResolverCreate } from "../resolver-usage.js";
|
|
6
|
+
import { resolverSpanAttributes } from "../resolver-span-attributes.js";
|
|
6
7
|
import type { TraceRecorder } from "../trace.js";
|
|
7
8
|
import { assertResolverHostAllowed } from "./hosts.js";
|
|
8
9
|
import {
|
|
@@ -413,7 +414,7 @@ function spanErrorAttributes(
|
|
|
413
414
|
phase: CapsolverOperationPhase,
|
|
414
415
|
): Record<string, unknown> | undefined {
|
|
415
416
|
if (error instanceof ResolverVendorUnavailableError) {
|
|
416
|
-
return {
|
|
417
|
+
return resolverSpanAttributes({
|
|
417
418
|
unavailability_reason: error.reason,
|
|
418
419
|
transport_phase: error.phase,
|
|
419
420
|
...(error instanceof CapsolverUnavailableError
|
|
@@ -425,21 +426,21 @@ function spanErrorAttributes(
|
|
|
425
426
|
response_length: error.responseLength,
|
|
426
427
|
}
|
|
427
428
|
: {}),
|
|
428
|
-
};
|
|
429
|
+
});
|
|
429
430
|
}
|
|
430
431
|
if (error instanceof CapsolverVerdictError) {
|
|
431
|
-
return {
|
|
432
|
+
return resolverSpanAttributes({
|
|
432
433
|
verdict_reason: error.reason,
|
|
433
434
|
transport_phase: error.phase,
|
|
434
435
|
vendor_error_code: error.errorCode,
|
|
435
436
|
vendor_error_description: error.errorDescription,
|
|
436
|
-
};
|
|
437
|
+
});
|
|
437
438
|
}
|
|
438
439
|
if (error instanceof CapsolverSolveTimeoutError) {
|
|
439
|
-
return {
|
|
440
|
+
return resolverSpanAttributes({
|
|
440
441
|
unavailability_reason: "timeout",
|
|
441
442
|
transport_phase: phase,
|
|
442
|
-
};
|
|
443
|
+
});
|
|
443
444
|
}
|
|
444
445
|
return undefined;
|
|
445
446
|
}
|
|
@@ -606,10 +607,10 @@ export function createCapsolverResolverVendorAdapter(
|
|
|
606
607
|
};
|
|
607
608
|
const taskId = traceRecorder
|
|
608
609
|
? await traceRecorder.runSpan("resolver.vendor.create_task", createTask, {
|
|
609
|
-
attributes: {
|
|
610
|
+
attributes: resolverSpanAttributes({
|
|
610
611
|
vendor: CAPSOLVER_VENDOR_ID,
|
|
611
612
|
challenge_kind: challenge.kind,
|
|
612
|
-
},
|
|
613
|
+
}),
|
|
613
614
|
onError: (error) => spanErrorAttributes(error, "create_task"),
|
|
614
615
|
})
|
|
615
616
|
: await createTask();
|
|
@@ -689,10 +690,10 @@ export function createCapsolverResolverVendorAdapter(
|
|
|
689
690
|
};
|
|
690
691
|
return traceRecorder
|
|
691
692
|
? await traceRecorder.runSpan("resolver.vendor.poll_result", pollResult, {
|
|
692
|
-
attributes: {
|
|
693
|
+
attributes: resolverSpanAttributes({
|
|
693
694
|
vendor: CAPSOLVER_VENDOR_ID,
|
|
694
695
|
challenge_kind: challenge.kind,
|
|
695
|
-
},
|
|
696
|
+
}),
|
|
696
697
|
onError: (error) => spanErrorAttributes(error, "poll_result"),
|
|
697
698
|
})
|
|
698
699
|
: await pollResult();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ChallengeSolution, ProviderChallenge } from "../../types.js";
|
|
2
2
|
import type { TraceRecorder } from "../trace.js";
|
|
3
3
|
import { recordPaidResolverCreate } from "../resolver-usage.js";
|
|
4
|
+
import { resolverSpanAttributes } from "../resolver-span-attributes.js";
|
|
4
5
|
import { assertResolverHostAllowed } from "./hosts.js";
|
|
5
6
|
import {
|
|
6
7
|
type ResolverIdentity,
|
|
@@ -300,19 +301,19 @@ function spanErrorAttributes(
|
|
|
300
301
|
phase: TwoCaptchaOperationPhase,
|
|
301
302
|
): Record<string, unknown> | undefined {
|
|
302
303
|
if (error instanceof ResolverVendorUnavailableError) {
|
|
303
|
-
return {
|
|
304
|
+
return resolverSpanAttributes({
|
|
304
305
|
unavailability_reason: error.reason,
|
|
305
306
|
transport_phase: error.phase,
|
|
306
|
-
};
|
|
307
|
+
});
|
|
307
308
|
}
|
|
308
309
|
// The solve budget raises TwoCaptchaSolveTimeoutError inside the spanned closure and
|
|
309
310
|
// the outer catch converts it only after the span has been finalized, so the span has
|
|
310
311
|
// to recognize it here or slow solves lose exactly the attribution these spans add.
|
|
311
312
|
if (error instanceof TwoCaptchaSolveTimeoutError) {
|
|
312
|
-
return {
|
|
313
|
+
return resolverSpanAttributes({
|
|
313
314
|
unavailability_reason: "timeout",
|
|
314
315
|
transport_phase: phase,
|
|
315
|
-
};
|
|
316
|
+
});
|
|
316
317
|
}
|
|
317
318
|
return undefined;
|
|
318
319
|
}
|
|
@@ -468,10 +469,10 @@ export function createTwoCaptchaResolverVendorAdapter(
|
|
|
468
469
|
};
|
|
469
470
|
const taskId = traceRecorder
|
|
470
471
|
? await traceRecorder.runSpan("resolver.vendor.create_task", createTask, {
|
|
471
|
-
attributes: {
|
|
472
|
+
attributes: resolverSpanAttributes({
|
|
472
473
|
vendor: TWOCAPTCHA_VENDOR_ID,
|
|
473
474
|
challenge_kind: challenge.kind,
|
|
474
|
-
},
|
|
475
|
+
}),
|
|
475
476
|
onError: (error) => spanErrorAttributes(error, "create_task"),
|
|
476
477
|
})
|
|
477
478
|
: await createTask();
|
|
@@ -517,10 +518,10 @@ export function createTwoCaptchaResolverVendorAdapter(
|
|
|
517
518
|
};
|
|
518
519
|
return traceRecorder
|
|
519
520
|
? await traceRecorder.runSpan("resolver.vendor.poll_result", pollResult, {
|
|
520
|
-
attributes: {
|
|
521
|
+
attributes: resolverSpanAttributes({
|
|
521
522
|
vendor: TWOCAPTCHA_VENDOR_ID,
|
|
522
523
|
challenge_kind: challenge.kind,
|
|
523
|
-
},
|
|
524
|
+
}),
|
|
524
525
|
onError: (error) => spanErrorAttributes(error, "poll_result"),
|
|
525
526
|
})
|
|
526
527
|
: await pollResult();
|
package/src/runtime/resolver.ts
CHANGED
|
@@ -64,6 +64,7 @@ import {
|
|
|
64
64
|
DEFAULT_RESOLVER_TIMEOUT_MS,
|
|
65
65
|
kindRequiresClientProfile,
|
|
66
66
|
} from "./resolver-config.js";
|
|
67
|
+
import { resolverSpanAttributes } from "./resolver-span-attributes.js";
|
|
67
68
|
import { DEFAULT_STEALTH_PROFILE } from "./stealth.js";
|
|
68
69
|
import type { TraceRecorder } from "./trace.js";
|
|
69
70
|
import type { BrowserTelemetrySink } from "./browser-telemetry.js";
|
|
@@ -595,7 +596,7 @@ function unavailableAttempt(
|
|
|
595
596
|
|
|
596
597
|
function unavailableSpanAttributes(error: ResolverVendorUnavailableError): Record<string, unknown> {
|
|
597
598
|
const attempt = unavailableAttempt(error);
|
|
598
|
-
return {
|
|
599
|
+
return resolverSpanAttributes({
|
|
599
600
|
unavailability_reason: error.reason,
|
|
600
601
|
missing_fields: error.missingFields,
|
|
601
602
|
cause_name: attempt.cause?.name,
|
|
@@ -603,7 +604,7 @@ function unavailableSpanAttributes(error: ResolverVendorUnavailableError): Recor
|
|
|
603
604
|
upstream_host: attempt.upstreamHost,
|
|
604
605
|
transport_phase: attempt.phase,
|
|
605
606
|
transport_round: attempt.round,
|
|
606
|
-
};
|
|
607
|
+
});
|
|
607
608
|
}
|
|
608
609
|
|
|
609
610
|
type ResolverAttemptDiagnostics = {
|
|
@@ -994,8 +995,8 @@ async function invalidateResolverSolutionWithOutcome(
|
|
|
994
995
|
return await invalidate();
|
|
995
996
|
}
|
|
996
997
|
return await metadata.traceRecorder.runSpan("resolver.cache.invalidate", invalidate, {
|
|
997
|
-
attributes: { challenge_kind: challenge.kind },
|
|
998
|
-
onSuccess: (outcome) => ({ outcome }),
|
|
998
|
+
attributes: resolverSpanAttributes({ challenge_kind: challenge.kind }),
|
|
999
|
+
onSuccess: (outcome) => resolverSpanAttributes({ outcome }),
|
|
999
1000
|
});
|
|
1000
1001
|
}
|
|
1001
1002
|
|
|
@@ -1257,12 +1258,12 @@ function createResolverChainClient(options: {
|
|
|
1257
1258
|
};
|
|
1258
1259
|
solution = traceRecorder
|
|
1259
1260
|
? await traceRecorder.runSpan("resolver.vendor.attempt", solveAttempt, {
|
|
1260
|
-
attributes: {
|
|
1261
|
+
attributes: resolverSpanAttributes({
|
|
1261
1262
|
vendor: adapter.id,
|
|
1262
1263
|
challenge_kind: challenge.kind,
|
|
1263
1264
|
client_profile: options.clientProfile,
|
|
1264
1265
|
resolver_identity_source: identityResolution.userAgentSource,
|
|
1265
|
-
},
|
|
1266
|
+
}),
|
|
1266
1267
|
onError(error) {
|
|
1267
1268
|
return error instanceof ResolverVendorUnavailableError
|
|
1268
1269
|
? unavailableSpanAttributes(error)
|
|
@@ -1,20 +1,28 @@
|
|
|
1
|
+
import { SDKError } from "../errors.js";
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Header names the stealth transport owns.
|
|
3
5
|
*
|
|
4
6
|
* `ctx.stealth` composes these from the Chrome emulation and the request class,
|
|
5
7
|
* so a caller value is rejected at request time with `SDKError` code
|
|
6
|
-
* `STEALTH_HEADER_OVERRIDE_UNSUPPORTED` (
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* `STEALTH_HEADER_OVERRIDE_UNSUPPORTED` ({@link assertCallerHeadersSupported}).
|
|
9
|
+
* The offline test harness (`runStandardTests`) calls the same function on
|
|
10
|
+
* every `ctx.stealth` request it serves, so a provider test fails exactly where
|
|
11
|
+
* production would, whichever file or variable the header came from. That is
|
|
12
|
+
* the cross-file enforcement.
|
|
13
|
+
*
|
|
14
|
+
* The `browser-version-literal` lint reports a subset of these names — the
|
|
15
|
+
* `sec-fetch-*` and `sec-ch-ua*` families — in files that use `ctx.stealth`
|
|
16
|
+
* (`isReportedOwnedHeaderName` in lint.ts). Both read this module, so the lint
|
|
17
|
+
* cannot claim a rejection the runtime does not perform; it reports
|
|
18
|
+
* deliberately less than the runtime rejects (host, connection,
|
|
12
19
|
* accept-encoding and a non-literal user-agent stay silent — see that
|
|
13
20
|
* predicate for why), so a green `apifuse check` is not proof that no owned
|
|
14
21
|
* header is set.
|
|
15
22
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
23
|
+
* Kept off the stealth runtime: the only import is `errors.js`, whose own
|
|
24
|
+
* imports are type-only, so the lazily loaded CLI lint path does not pull the
|
|
25
|
+
* stealth runtime in.
|
|
18
26
|
*/
|
|
19
27
|
export const SDK_OWNED_EXACT_CHROME_HEADERS: ReadonlySet<string> = new Set([
|
|
20
28
|
"host",
|
|
@@ -39,3 +47,21 @@ export function isStealthOwnedHeaderName(name: string): boolean {
|
|
|
39
47
|
SDK_OWNED_EXACT_CHROME_HEADERS.has(lower) || lower.startsWith(SDK_OWNED_CHROME_HEADER_PREFIX)
|
|
40
48
|
);
|
|
41
49
|
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Throws `SDKError` `STEALTH_HEADER_OVERRIDE_UNSUPPORTED` for the first caller
|
|
53
|
+
* header `ctx.stealth` owns. An `undefined` value is skipped, as the transport
|
|
54
|
+
* drops it before sending. The stealth runtime and the offline test harness
|
|
55
|
+
* both call this, so the two cannot disagree about what is rejected.
|
|
56
|
+
*/
|
|
57
|
+
export function assertCallerHeadersSupported(
|
|
58
|
+
headers: Readonly<Record<string, string | readonly string[] | undefined>>,
|
|
59
|
+
): void {
|
|
60
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
61
|
+
if (value === undefined || !isStealthOwnedHeaderName(name)) continue;
|
|
62
|
+
throw new SDKError(
|
|
63
|
+
`Stealth transport owns the "${name.toLowerCase()}" header; remove it from headers.`,
|
|
64
|
+
{ code: "STEALTH_HEADER_OVERRIDE_UNSUPPORTED" },
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -22,7 +22,12 @@ export type StealthTelemetryErrorCode =
|
|
|
22
22
|
| "PROXY_REQUIRED"
|
|
23
23
|
| "other";
|
|
24
24
|
|
|
25
|
-
export type StealthTelemetryRequestClass =
|
|
25
|
+
export type StealthTelemetryRequestClass =
|
|
26
|
+
| "navigation"
|
|
27
|
+
| "script_navigation"
|
|
28
|
+
| "form_post"
|
|
29
|
+
| "xhr"
|
|
30
|
+
| "post";
|
|
26
31
|
|
|
27
32
|
export type StealthTelemetryAttemptKind = "request" | "resolver" | "proxy_diagnostic";
|
|
28
33
|
|