@apifuse/provider-sdk 2.2.0-beta.61 → 2.2.0-beta.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/bin/apifuse-pack-types.ts +16 -12
- package/dist/handle-meta.d.ts +18 -2
- package/dist/handle-meta.js +30 -2
- package/dist/handle.d.ts +33 -12
- package/dist/handle.js +69 -24
- package/dist/index.d.ts +3 -1
- package/dist/index.js +3 -1
- package/dist/lint.js +28 -8
- package/dist/provider.d.ts +2 -0
- package/dist/provider.js +2 -0
- package/dist/runtime/handle.js +15 -10
- package/dist/runtime/http-telemetry-binding.d.ts +10 -0
- package/dist/runtime/http-telemetry-binding.js +152 -0
- package/dist/runtime/http-telemetry-guard.d.ts +6 -0
- package/dist/runtime/http-telemetry-guard.js +150 -0
- package/dist/runtime/http-telemetry.d.ts +104 -0
- package/dist/runtime/http-telemetry.js +190 -0
- package/dist/runtime/http.d.ts +31 -0
- package/dist/runtime/http.js +222 -146
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +1 -0
- package/dist/server/serve-implementation.d.ts +2 -0
- package/dist/server/serve-implementation.js +34 -35
- package/package.json +1 -1
- package/src/handle-meta.ts +39 -4
- package/src/handle.ts +121 -38
- package/src/index.ts +19 -0
- package/src/lint.ts +30 -7
- package/src/provider.ts +10 -0
- package/src/runtime/handle.ts +20 -10
- package/src/runtime/http-telemetry-binding.ts +162 -0
- package/src/runtime/http-telemetry-guard.ts +155 -0
- package/src/runtime/http-telemetry.ts +292 -0
- package/src/runtime/http.ts +295 -173
- package/src/server/index.ts +13 -0
- package/src/server/serve-implementation.ts +41 -58
package/dist/runtime/http.js
CHANGED
|
@@ -2,9 +2,34 @@ import { policyRotatesTransportVendorChain, resolvePolicyTransportAttemptCap, re
|
|
|
2
2
|
import { HttpRedirectError, ProviderError, TransportError } from "../errors.js";
|
|
3
3
|
import { parseSseStream, readableBytes, readableLines, readableTextChunks } from "../stream.js";
|
|
4
4
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, isProxyTransportRetryMethod, normalizeProxyTransportRetryOptions, proxyTransportRetryErrorCode, proxyTransportRetryErrorStatus, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
5
|
+
import { httpTelemetryErrorCode, } from "./http-telemetry.js";
|
|
5
6
|
import { evaluateRedirectHop, isRedirectStatus, resolveRedirectUrl } from "./redirects.js";
|
|
6
7
|
import { normalizeHttpRequestBody, redactSensitiveError, redactSensitiveRequestError, serializeRequestUrl, } from "./request-options.js";
|
|
8
|
+
import { startHttpTelemetry } from "./http-telemetry-guard.js";
|
|
9
|
+
import { scopedHttpTelemetry } from "./http-telemetry-binding.js";
|
|
7
10
|
const DEFAULT_HTTP_BASE_URL = "http://localhost";
|
|
11
|
+
function httpAttemptFailure(error, status) {
|
|
12
|
+
try {
|
|
13
|
+
return {
|
|
14
|
+
e: httpTelemetryErrorCode(proxyTransportRetryErrorCode(error)),
|
|
15
|
+
status: proxyTransportRetryErrorStatus(error) ?? status,
|
|
16
|
+
...(error instanceof Error
|
|
17
|
+
? {
|
|
18
|
+
diagnostics: {
|
|
19
|
+
name: error.name,
|
|
20
|
+
message: error.message,
|
|
21
|
+
...(error.cause instanceof Error
|
|
22
|
+
? { cause: { name: error.cause.name, message: error.cause.message } }
|
|
23
|
+
: {}),
|
|
24
|
+
},
|
|
25
|
+
}
|
|
26
|
+
: {}),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return { e: "other" };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
8
33
|
function isHttpStatusOutcome(outcome) {
|
|
9
34
|
return "kind" in outcome && outcome.kind === "http-status";
|
|
10
35
|
}
|
|
@@ -527,7 +552,7 @@ function serializeHttpRequestUrl(baseUrl, url, options) {
|
|
|
527
552
|
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
528
553
|
}
|
|
529
554
|
}
|
|
530
|
-
async function fetchNativeHttp(baseUrl, url, method, options, clientOptions, warn, statusRetryCodes, proxyAttemptOffset = 0, dedupe) {
|
|
555
|
+
async function fetchNativeHttp(baseUrl, url, method, options, clientOptions, warn, statusRetryCodes, proxyAttemptOffset = 0, dedupe, observeProxy) {
|
|
531
556
|
const serializedUrl = serializeHttpRequestUrl(baseUrl, url, options);
|
|
532
557
|
const { requestUrl } = serializedUrl;
|
|
533
558
|
const controller = options.timeout ? new AbortController() : undefined;
|
|
@@ -542,6 +567,7 @@ async function fetchNativeHttp(baseUrl, url, method, options, clientOptions, war
|
|
|
542
567
|
// failures are branded as TransportErrors and count against the request
|
|
543
568
|
// deadline, exactly as an inline resolve would.
|
|
544
569
|
proxy = await resolveNativeProxy(options, clientOptions, warn, proxyAttemptOffset);
|
|
570
|
+
observeProxy?.(Boolean(proxy));
|
|
545
571
|
throwIfAmbientAborted(clientOptions.signal);
|
|
546
572
|
// For a registry allocator chain, skip an endpoint a prior attempt already
|
|
547
573
|
// tried rather than re-issuing the same request. Returning the sentinel
|
|
@@ -563,6 +589,7 @@ async function fetchNativeHttp(baseUrl, url, method, options, clientOptions, war
|
|
|
563
589
|
requestInit.body = normalizeNativeFetchBody(options.body);
|
|
564
590
|
}
|
|
565
591
|
const response = await fetchWithHttpRedirectPolicy(requestUrl, requestInit, options.redirectPolicy);
|
|
592
|
+
observeProxy?.(Boolean(proxy), response.status);
|
|
566
593
|
const headers = Object.fromEntries(response.headers.entries());
|
|
567
594
|
if (statusRetryCodes && response.status >= 400) {
|
|
568
595
|
await drainNativeResponseBody(response);
|
|
@@ -596,7 +623,7 @@ async function fetchNativeHttp(baseUrl, url, method, options, clientOptions, war
|
|
|
596
623
|
clearTimeout(timeoutHandle);
|
|
597
624
|
}
|
|
598
625
|
}
|
|
599
|
-
async function fetchNativeHttpStream(baseUrl, url, method, options, clientOptions, warn) {
|
|
626
|
+
async function fetchNativeHttpStream(baseUrl, url, method, options, clientOptions, warn, observeProxy) {
|
|
600
627
|
const serializedUrl = serializeHttpRequestUrl(baseUrl, url, options);
|
|
601
628
|
const { requestUrl } = serializedUrl;
|
|
602
629
|
const controller = options.timeout ? new AbortController() : undefined;
|
|
@@ -607,6 +634,7 @@ async function fetchNativeHttpStream(baseUrl, url, method, options, clientOption
|
|
|
607
634
|
try {
|
|
608
635
|
throwIfAmbientAborted(clientOptions.signal);
|
|
609
636
|
const proxy = await resolveNativeProxy(options, clientOptions, warn);
|
|
637
|
+
observeProxy?.(Boolean(proxy));
|
|
610
638
|
throwIfAmbientAborted(clientOptions.signal);
|
|
611
639
|
const requestInit = {
|
|
612
640
|
headers: options.headers,
|
|
@@ -676,164 +704,189 @@ export function createHttpClient(baseUrl, clientOptions = {}) {
|
|
|
676
704
|
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
677
705
|
}
|
|
678
706
|
})();
|
|
679
|
-
const
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
707
|
+
const requestStarted = performance.now();
|
|
708
|
+
const telemetry = startHttpTelemetry(scopedHttpTelemetry(clientOptions.httpTelemetry), retryOptions?.preset);
|
|
709
|
+
try {
|
|
710
|
+
return await runRequest();
|
|
711
|
+
}
|
|
712
|
+
finally {
|
|
713
|
+
telemetry.finish(performance.now() - requestStarted);
|
|
714
|
+
}
|
|
715
|
+
async function runRequest() {
|
|
716
|
+
const retryEnabled = Boolean(retryOptions &&
|
|
717
|
+
retryOptions.attempts > 1 &&
|
|
718
|
+
isProxyTransportRetryMethod(methodName, retryOptions));
|
|
719
|
+
const statusRetryEnabled = Boolean(retryEnabled &&
|
|
720
|
+
explicitRetry &&
|
|
721
|
+
retryOptions &&
|
|
722
|
+
retryOptions.statusCodes.length > 0 &&
|
|
723
|
+
headersOptions.throwOnHttpError !== false);
|
|
724
|
+
const attemptOptions = statusRetryEnabled
|
|
725
|
+
? { ...headersOptions, throwOnHttpError: false }
|
|
726
|
+
: headersOptions;
|
|
727
|
+
// Span the whole vendor chain on transport failures. Like ctx.stealth, a
|
|
728
|
+
// policy-managed proxy resolves a *different* endpoint/vendor per attempt
|
|
729
|
+
// (the flat proxyAttemptOffset rotates across the concatenated vendor pool
|
|
730
|
+
// spans), so a transport failure should advance to the next endpoint —
|
|
731
|
+
// potentially crossing into the fallback vendor — rather than stopping at
|
|
732
|
+
// the per-endpoint retry budget and stranding the request on the primary
|
|
733
|
+
// vendor. resolvePolicyTransportAttemptCap widens the cap to the chain span
|
|
734
|
+
// only for implicit, safe-method allocator requests; explicit retry
|
|
735
|
+
// policies (their documented `attempts` ceiling), unsafe methods, and
|
|
736
|
+
// static/non-registry vendors keep the retry budget. Status-code retries
|
|
737
|
+
// stay bounded by the retry budget regardless; only transport rotation gets
|
|
738
|
+
// the full span.
|
|
739
|
+
const policyProxy = (() => {
|
|
740
|
+
const policy = clientOptions.proxyPolicy ?? clientOptions.upstream?.proxy;
|
|
741
|
+
return policy && typeof policy === "object" ? policy : undefined;
|
|
742
|
+
})();
|
|
743
|
+
const usesPolicyAllocator = Boolean(policyProxy) && !options.proxy && !clientOptions.proxy;
|
|
744
|
+
const transportAttemptCap = retryOptions
|
|
745
|
+
? resolvePolicyTransportAttemptCap({
|
|
746
|
+
policy: policyProxy,
|
|
747
|
+
usesPolicyAllocator,
|
|
748
|
+
retryAttempts: retryOptions.attempts,
|
|
749
|
+
explicitRetry,
|
|
750
|
+
method: methodName,
|
|
751
|
+
})
|
|
752
|
+
: 1;
|
|
753
|
+
// Track resolved endpoints across a policy-allocator chain. Successive
|
|
754
|
+
// attempts rotate the flat offset across the concatenated vendor pool
|
|
755
|
+
// spans, but an under-filled allocation (fewer live endpoints than the
|
|
756
|
+
// configured pool size) makes the modulo mapping repeat endpoints before
|
|
757
|
+
// the offset reaches the next vendor. Rather than re-hammering an
|
|
758
|
+
// already-tried endpoint under backoff, fetchNativeHttp returns a skip
|
|
759
|
+
// sentinel for a duplicate; the loop then advances the flat offset without
|
|
760
|
+
// issuing the request, so it keeps walking toward — and into — the fallback
|
|
761
|
+
// vendor's pool span instead of stalling on the primary vendor.
|
|
762
|
+
// De-duplication is gated on the SAME predicate that widens the attempt cap
|
|
763
|
+
// (implicit, safe-method, registry-chain rotation). It must NOT engage for
|
|
764
|
+
// an explicit retry policy: there the caller's `attempts` count is the
|
|
765
|
+
// contract and each attempt must issue against whatever endpoint it resolves
|
|
766
|
+
// — even a repeat — instead of being silently skipped (which would collapse
|
|
767
|
+
// a `poolSize: 1` + `attempts: 3` request to a single fetch).
|
|
768
|
+
const dedupeAllocatorEndpoints = policyRotatesTransportVendorChain({
|
|
709
769
|
policy: policyProxy,
|
|
710
770
|
usesPolicyAllocator,
|
|
711
|
-
retryAttempts: retryOptions.attempts,
|
|
712
771
|
explicitRetry,
|
|
713
772
|
method: methodName,
|
|
714
|
-
})
|
|
715
|
-
:
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
code: "retry_exhausted",
|
|
747
|
-
});
|
|
748
|
-
}
|
|
749
|
-
if (isHttpStatusOutcome(outcome)) {
|
|
750
|
-
throw toUpstreamHttpError(outcome.status);
|
|
751
|
-
}
|
|
752
|
-
return outcome;
|
|
753
|
-
}
|
|
754
|
-
let lastError;
|
|
755
|
-
let lastErrorCode;
|
|
756
|
-
let lastStatus;
|
|
757
|
-
// `attempt` walks the flat proxy offset across the full chain span; `issued`
|
|
758
|
-
// counts requests that were actually sent (skipped duplicate offsets do not
|
|
759
|
-
// increment it). Retry summaries and the status-retry budget must reflect
|
|
760
|
-
// issued requests, not the raw offset, so they stay accurate when partial
|
|
761
|
-
// allocations skip offsets.
|
|
762
|
-
let issued = 0;
|
|
763
|
-
for (let attempt = 1; attempt <= transportAttemptCap; attempt += 1) {
|
|
764
|
-
throwIfAmbientAborted(clientOptions.signal);
|
|
765
|
-
// Whether this offset actually issued a request (vs. a skipped duplicate),
|
|
766
|
-
// so the catch counts a thrown *transport* failure once without
|
|
767
|
-
// double-counting a status outcome that already incremented before it
|
|
768
|
-
// re-threw as an upstream HTTP error.
|
|
769
|
-
let issuedThisAttempt = false;
|
|
770
|
-
try {
|
|
771
|
-
const outcome = await executeOnce(attempt - 1);
|
|
773
|
+
});
|
|
774
|
+
const dedupeContext = dedupeAllocatorEndpoints ? { attempted: new Set() } : undefined;
|
|
775
|
+
const executeOnce = async (proxyAttemptOffset = 0) => {
|
|
776
|
+
const started = performance.now();
|
|
777
|
+
let proxyUsed = false;
|
|
778
|
+
let observedStatus;
|
|
779
|
+
const record = (event) => {
|
|
780
|
+
telemetry.recordAttempt({ ...event, ms: performance.now() - started, proxyUsed });
|
|
781
|
+
};
|
|
782
|
+
try {
|
|
783
|
+
const outcome = await fetchNativeHttp(baseUrl, url, methodName, attemptOptions, clientOptions, warnOnce, statusRetryEnabled ? retryOptions?.statusCodes : undefined, proxyAttemptOffset, dedupeContext, (used, status) => {
|
|
784
|
+
proxyUsed = used;
|
|
785
|
+
observedStatus = status;
|
|
786
|
+
});
|
|
787
|
+
if (!isDedupeSkipOutcome(outcome))
|
|
788
|
+
record({
|
|
789
|
+
status: outcome.status,
|
|
790
|
+
...(isHttpStatusOutcome(outcome)
|
|
791
|
+
? { e: "upstream_http_error", statusRetry: true }
|
|
792
|
+
: {}),
|
|
793
|
+
});
|
|
794
|
+
return outcome;
|
|
795
|
+
}
|
|
796
|
+
catch (error) {
|
|
797
|
+
// Record before the retry loop can catch/rethrow a terminal status.
|
|
798
|
+
record(httpAttemptFailure(error, observedStatus));
|
|
799
|
+
throw error;
|
|
800
|
+
}
|
|
801
|
+
};
|
|
802
|
+
if (!retryEnabled || !retryOptions) {
|
|
803
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
804
|
+
const outcome = await executeOnce();
|
|
772
805
|
if (isDedupeSkipOutcome(outcome)) {
|
|
773
|
-
//
|
|
774
|
-
//
|
|
775
|
-
|
|
776
|
-
|
|
806
|
+
// Single-shot path never de-duplicates (dedupeContext is undefined),
|
|
807
|
+
// but keep the union total.
|
|
808
|
+
throw new TransportError("HTTP request produced no terminal result", {
|
|
809
|
+
code: "retry_exhausted",
|
|
810
|
+
});
|
|
777
811
|
}
|
|
778
|
-
issued += 1;
|
|
779
|
-
issuedThisAttempt = true;
|
|
780
812
|
if (isHttpStatusOutcome(outcome)) {
|
|
781
|
-
lastStatus = outcome.status;
|
|
782
|
-
if (outcome.retryable && issued < retryOptions.attempts) {
|
|
783
|
-
await sleep(computeProxyTransportRetryDelayMs(retryOptions, attempt, outcome.headers), clientOptions.signal);
|
|
784
|
-
continue;
|
|
785
|
-
}
|
|
786
813
|
throw toUpstreamHttpError(outcome.status);
|
|
787
814
|
}
|
|
788
|
-
|
|
789
|
-
if (response.status >= 400 && headersOptions.throwOnHttpError !== false) {
|
|
790
|
-
throw toUpstreamHttpError(response.status);
|
|
791
|
-
}
|
|
792
|
-
if (issued > 1) {
|
|
793
|
-
const summary = {
|
|
794
|
-
attempts: issued,
|
|
795
|
-
retries: issued - 1,
|
|
796
|
-
...(retryOptions.preset ? { preset: retryOptions.preset } : {}),
|
|
797
|
-
transport: "native",
|
|
798
|
-
...(lastErrorCode ? { lastErrorCode } : {}),
|
|
799
|
-
...(lastStatus ? { lastStatus } : {}),
|
|
800
|
-
};
|
|
801
|
-
clientOptions.onRetrySummary?.(summary);
|
|
802
|
-
}
|
|
803
|
-
return response;
|
|
815
|
+
return outcome;
|
|
804
816
|
}
|
|
805
|
-
|
|
817
|
+
let lastError;
|
|
818
|
+
// `attempt` walks the flat proxy offset across the full chain span; `issued`
|
|
819
|
+
// counts requests that were actually sent (skipped duplicate offsets do not
|
|
820
|
+
// increment it). Retry summaries and the status-retry budget must reflect
|
|
821
|
+
// issued requests, not the raw offset, so they stay accurate when partial
|
|
822
|
+
// allocations skip offsets.
|
|
823
|
+
let issued = 0;
|
|
824
|
+
for (let attempt = 1; attempt <= transportAttemptCap; attempt += 1) {
|
|
806
825
|
throwIfAmbientAborted(clientOptions.signal);
|
|
807
|
-
|
|
826
|
+
// Whether this offset actually issued a request (vs. a skipped duplicate),
|
|
827
|
+
// so the catch counts a thrown *transport* failure once without
|
|
828
|
+
// double-counting a status outcome that already incremented before it
|
|
829
|
+
// re-threw as an upstream HTTP error.
|
|
830
|
+
let issuedThisAttempt = false;
|
|
831
|
+
try {
|
|
832
|
+
const outcome = await executeOnce(attempt - 1);
|
|
833
|
+
if (isDedupeSkipOutcome(outcome)) {
|
|
834
|
+
// Duplicate endpoint from a partial allocation: advance the flat
|
|
835
|
+
// offset without issuing the request (no backoff, not a failure) so
|
|
836
|
+
// the loop keeps rotating toward the fallback vendor.
|
|
837
|
+
continue;
|
|
838
|
+
}
|
|
808
839
|
issued += 1;
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
|
|
840
|
+
issuedThisAttempt = true;
|
|
841
|
+
if (isHttpStatusOutcome(outcome)) {
|
|
842
|
+
if (outcome.retryable && issued < retryOptions.attempts) {
|
|
843
|
+
await sleep(computeProxyTransportRetryDelayMs(retryOptions, attempt, outcome.headers), clientOptions.signal);
|
|
844
|
+
continue;
|
|
845
|
+
}
|
|
846
|
+
throw toUpstreamHttpError(outcome.status);
|
|
847
|
+
}
|
|
848
|
+
const response = outcome;
|
|
849
|
+
if (response.status >= 400 && headersOptions.throwOnHttpError !== false) {
|
|
850
|
+
throw toUpstreamHttpError(response.status);
|
|
851
|
+
}
|
|
852
|
+
if (issued > 1) {
|
|
853
|
+
const summary = telemetry.toTenantRetryPayload();
|
|
854
|
+
if (summary)
|
|
855
|
+
telemetry.observe(() => clientOptions.onRetrySummary?.(summary));
|
|
856
|
+
}
|
|
857
|
+
return response;
|
|
858
|
+
}
|
|
859
|
+
catch (error) {
|
|
860
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
861
|
+
if (!issuedThisAttempt)
|
|
862
|
+
issued += 1;
|
|
863
|
+
lastError = error;
|
|
864
|
+
const proxyUsed = Boolean(error.proxyUsed);
|
|
865
|
+
if (attempt < transportAttemptCap &&
|
|
866
|
+
shouldRetryProxyTransportAttempt({
|
|
867
|
+
error,
|
|
868
|
+
explicitRetry,
|
|
869
|
+
method: methodName,
|
|
870
|
+
options: retryOptions,
|
|
871
|
+
proxyUsed,
|
|
872
|
+
})) {
|
|
873
|
+
await sleep(computeProxyTransportRetryDelayMs(retryOptions, attempt), clientOptions.signal);
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
876
|
+
throw error;
|
|
823
877
|
}
|
|
824
|
-
throw error;
|
|
825
878
|
}
|
|
879
|
+
// Reached when the attempt cap is consumed without a terminal outcome —
|
|
880
|
+
// e.g. the final offsets of a partial allocation all resolved to
|
|
881
|
+
// already-tried endpoints and were skipped. Surface the last real transport
|
|
882
|
+
// failure rather than a synthetic exhaustion error.
|
|
883
|
+
if (lastError !== undefined) {
|
|
884
|
+
throw lastError;
|
|
885
|
+
}
|
|
886
|
+
throw new TransportError("HTTP retry exhausted without a terminal result", {
|
|
887
|
+
code: "retry_exhausted",
|
|
888
|
+
});
|
|
826
889
|
}
|
|
827
|
-
// Reached when the attempt cap is consumed without a terminal outcome —
|
|
828
|
-
// e.g. the final offsets of a partial allocation all resolved to
|
|
829
|
-
// already-tried endpoints and were skipped. Surface the last real transport
|
|
830
|
-
// failure rather than a synthetic exhaustion error.
|
|
831
|
-
if (lastError !== undefined) {
|
|
832
|
-
throw lastError;
|
|
833
|
-
}
|
|
834
|
-
throw new TransportError("HTTP retry exhausted without a terminal result", {
|
|
835
|
-
code: "retry_exhausted",
|
|
836
|
-
});
|
|
837
890
|
}
|
|
838
891
|
async function streamRequest(url, method, options = {}) {
|
|
839
892
|
const { headersOptions, methodName } = (() => {
|
|
@@ -855,7 +908,30 @@ export function createHttpClient(baseUrl, clientOptions = {}) {
|
|
|
855
908
|
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
856
909
|
}
|
|
857
910
|
})();
|
|
858
|
-
|
|
911
|
+
// Streams never retry. Observe the header fetch without consuming its body.
|
|
912
|
+
const telemetry = startHttpTelemetry(scopedHttpTelemetry(clientOptions.httpTelemetry), "off");
|
|
913
|
+
const started = performance.now();
|
|
914
|
+
const alreadyAborted = clientOptions.signal?.aborted === true;
|
|
915
|
+
let proxyUsed = false;
|
|
916
|
+
const record = (event) => {
|
|
917
|
+
if (alreadyAborted)
|
|
918
|
+
return;
|
|
919
|
+
telemetry.recordAttempt({ ...event, ms: performance.now() - started, proxyUsed });
|
|
920
|
+
};
|
|
921
|
+
try {
|
|
922
|
+
const response = await fetchNativeHttpStream(baseUrl, url, methodName, headersOptions, clientOptions, warnOnce, (used) => {
|
|
923
|
+
proxyUsed = used;
|
|
924
|
+
});
|
|
925
|
+
record({ status: response.status });
|
|
926
|
+
return response;
|
|
927
|
+
}
|
|
928
|
+
catch (error) {
|
|
929
|
+
record(httpAttemptFailure(error));
|
|
930
|
+
throw error;
|
|
931
|
+
}
|
|
932
|
+
finally {
|
|
933
|
+
telemetry.finish(performance.now() - started);
|
|
934
|
+
}
|
|
859
935
|
}
|
|
860
936
|
return {
|
|
861
937
|
request: async (url, options = {}) => request(url, options.method ?? "GET", options),
|
package/dist/server/index.d.ts
CHANGED
|
@@ -12,4 +12,6 @@ export { collectSelfTestSensitiveValues, redactSelfTestText, SELF_TEST_MAX_TEXT_
|
|
|
12
12
|
export { DEFAULT_SELF_TEST_PORT, deriveSelfTestToken, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV, PROVIDER_RUNTIME_SELF_TEST_PORT_ENV, resolveSelfTestMasterSecrets, type SelfTestMasterSecrets, verifySelfTestAuthorization, } from "./self-test-token.js";
|
|
13
13
|
export type { AuthFlowRequest, AuthFlowResponse, AuthFlowSuccessResponse, ConnectionMode, OperationConnection, OperationErrorResponse, OperationRequest, OperationResponse, OperationSuccessResponse, } from "./types.js";
|
|
14
14
|
export { AuthFlowRequestSchema, AuthFlowSuccessResponseSchema, ConnectionModeSchema, ErrorEnvelopeSchema, OperationConnectionSchema, OperationErrorResponseSchema, OperationRequestSchema, OperationSuccessResponseSchema, } from "./types.js";
|
|
15
|
+
export { HttpTelemetryCollector, type HttpTelemetrySink, type HttpTelemetryRequestSink, type HttpAttemptTelemetryEvent, type HttpAttemptSample, type HttpTelemetryErrorCode, type HttpTelemetryRetryPayload, type HttpTelemetryLogPayload, type HttpTelemetryHeaderPayload, } from "../runtime/http-telemetry.js";
|
|
16
|
+
export type { HttpRetrySummary } from "../types.js";
|
|
15
17
|
export { NativeTelemetryCollector, type NativeAttemptSample, type NativeConnectTelemetryEvent, type NativeLifecycleTelemetryEvent, type NativeTelemetryDiagnostics, type NativeTelemetryErrorCode, type NativeTelemetryHeaderPayload, type NativeTelemetryKind, type NativeTelemetryLifecycleKind, type NativeTelemetryLogPayload, type NativeTelemetryOutcome, type NativeTelemetrySink, type NativeTelemetryVendorSkipReason, type NativeVendorSkipTelemetryEvent, } from "../runtime/native-telemetry.js";
|
package/dist/server/index.js
CHANGED
|
@@ -6,4 +6,5 @@ export { resolveHealthCheckInputDateTokens, } from "./self-test-input-tokens.js"
|
|
|
6
6
|
export { collectSelfTestSensitiveValues, redactSelfTestText, SELF_TEST_MAX_TEXT_LENGTH, SELF_TEST_REDACTED_PLACEHOLDER, } from "./self-test-redaction.js";
|
|
7
7
|
export { DEFAULT_SELF_TEST_PORT, deriveSelfTestToken, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_ENV, PROVIDER_RUNTIME_SELF_TEST_MASTER_SECRET_PREVIOUS_ENV, PROVIDER_RUNTIME_SELF_TEST_PORT_ENV, resolveSelfTestMasterSecrets, verifySelfTestAuthorization, } from "./self-test-token.js";
|
|
8
8
|
export { AuthFlowRequestSchema, AuthFlowSuccessResponseSchema, ConnectionModeSchema, ErrorEnvelopeSchema, OperationConnectionSchema, OperationErrorResponseSchema, OperationRequestSchema, OperationSuccessResponseSchema, } from "./types.js";
|
|
9
|
+
export { HttpTelemetryCollector, } from "../runtime/http-telemetry.js";
|
|
9
10
|
export { NativeTelemetryCollector, } from "../runtime/native-telemetry.js";
|
|
@@ -4,6 +4,7 @@ import { type ProviderEngine } from "../engine.js";
|
|
|
4
4
|
import { type ProviderErrorObservability } from "../errors.js";
|
|
5
5
|
import { type ProviderErrorCategory } from "../observability.js";
|
|
6
6
|
import type { HandleTelemetryEvent } from "../handle.js";
|
|
7
|
+
import { type HttpTelemetryLogPayload } from "../runtime/http-telemetry.js";
|
|
7
8
|
import { type ProxyTelemetryLogPayload } from "../runtime/proxy-telemetry.js";
|
|
8
9
|
import { type ResolverTelemetryLogPayload } from "../runtime/resolver-telemetry.js";
|
|
9
10
|
import type { OcrContext, ProviderContext, ProviderDefinition, ProviderFilesContext, ProviderRuntimeState, ResolverContext, SttContext } from "../types.js";
|
|
@@ -88,6 +89,7 @@ type ProviderServerLogEventBase = ProviderRequestCost & {
|
|
|
88
89
|
status: number;
|
|
89
90
|
proxy?: ProxyTelemetryLogPayload;
|
|
90
91
|
resolver?: ResolverTelemetryLogPayload;
|
|
92
|
+
http?: HttpTelemetryLogPayload;
|
|
91
93
|
};
|
|
92
94
|
export type ProviderServerLogEvent = (ProviderServerLogEventBase & {
|
|
93
95
|
level: "info";
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { NativeTelemetryCollector } from "../runtime/native-telemetry.js";
|
|
2
1
|
import { STATUS_CODES } from "node:http";
|
|
3
2
|
import { readDiagnosticEnv, withDiagnosticEnv } from "../runtime/diagnostic-env.js";
|
|
4
3
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
@@ -26,6 +25,9 @@ import { createCeremonyEgressLeaseRuntime, ENGINE_CEREMONY_EGRESS_LEASE, } from
|
|
|
26
25
|
import { createEnvContext } from "../runtime/env.js";
|
|
27
26
|
import { executeOperation } from "../runtime/executor.js";
|
|
28
27
|
import { createHttpClient } from "../runtime/http.js";
|
|
28
|
+
import { NativeTelemetryCollector } from "../runtime/native-telemetry.js";
|
|
29
|
+
import { bindHttpTelemetry, isSupportedHttpBinding, warnUnsupportedHttpBinding, } from "../runtime/http-telemetry-binding.js";
|
|
30
|
+
import { HttpTelemetryCollector } from "../runtime/http-telemetry.js";
|
|
29
31
|
import { wrapWithInstrumentation } from "../runtime/instrumentation.js";
|
|
30
32
|
import { createOcrClientFromEnv } from "../runtime/ocr.js";
|
|
31
33
|
import { getProviderBaseUrl } from "../runtime/provider.js";
|
|
@@ -83,16 +85,6 @@ function providerErrorCode(error) {
|
|
|
83
85
|
}
|
|
84
86
|
const AUTH_FLOW_LOCALES = ["en", "ko", "ja"];
|
|
85
87
|
const retryResponseMeta = new WeakMap();
|
|
86
|
-
const RETRY_LAST_ERROR_CODES = [
|
|
87
|
-
"transport_cancelled",
|
|
88
|
-
"transport_timeout",
|
|
89
|
-
"transport_network_error",
|
|
90
|
-
"upstream_http_error",
|
|
91
|
-
"other",
|
|
92
|
-
];
|
|
93
|
-
function tenantRetryLastErrorCode(value) {
|
|
94
|
-
return RETRY_LAST_ERROR_CODES.find((candidate) => candidate === value) ?? "other";
|
|
95
|
-
}
|
|
96
88
|
const STATEFUL_INTERNAL_OPERATIONS_ROUTE = "/__apifuse/stateful/operations";
|
|
97
89
|
const STATEFUL_FORWARDING_SOURCE_POD_HEADER = "x-apifuse-stateful-source-pod";
|
|
98
90
|
const DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS = 5 * 60_000;
|
|
@@ -504,7 +496,6 @@ function createProviderContext(provider, request, operationId, options, state =
|
|
|
504
496
|
};
|
|
505
497
|
const resolverIdentityScope = resolveProviderResolverIdentityScope(provider, proxyClientOptions.affinityKey, request.requestId);
|
|
506
498
|
const cache = createProviderCache({ providerId: provider.id });
|
|
507
|
-
let wrappedContext;
|
|
508
499
|
const { capabilityModules } = options;
|
|
509
500
|
const resolverOptions = createResolverRuntimeOptions(provider, cache, resolverIdentityScope, proxyPolicy, proxyClientOptions, stealthProfile, scope.resolverTelemetry);
|
|
510
501
|
const challengeRuntime = createStealthChallengeDetection(provider, capabilityModules.resolver, options.resolver, resolverOptions, scope.trace, signal);
|
|
@@ -546,11 +537,7 @@ function createProviderContext(provider, request, operationId, options, state =
|
|
|
546
537
|
http: createHttpClient(baseUrl, {
|
|
547
538
|
...proxyClientOptions,
|
|
548
539
|
...(signal ? { signal } : {}),
|
|
549
|
-
|
|
550
|
-
if (summary.attempts <= 1 || !wrappedContext)
|
|
551
|
-
return;
|
|
552
|
-
retryResponseMeta.set(wrappedContext, summary);
|
|
553
|
-
},
|
|
540
|
+
httpTelemetry: scope.httpTelemetry,
|
|
554
541
|
}),
|
|
555
542
|
cache,
|
|
556
543
|
state: requestState,
|
|
@@ -607,8 +594,30 @@ function createProviderContext(provider, request, operationId, options, state =
|
|
|
607
594
|
}),
|
|
608
595
|
}),
|
|
609
596
|
};
|
|
610
|
-
const
|
|
611
|
-
|
|
597
|
+
const candidateHttp = bindings.http;
|
|
598
|
+
const attached = options.engine.attach({ provider, bindings });
|
|
599
|
+
const hostHttp = provider.http ? attached.http : undefined;
|
|
600
|
+
const supportedHostHttp = provider.http && isSupportedHttpBinding(hostHttp);
|
|
601
|
+
if (provider.http && !supportedHostHttp)
|
|
602
|
+
warnUnsupportedHttpBinding();
|
|
603
|
+
const observedHttp = supportedHostHttp && hostHttp !== candidateHttp
|
|
604
|
+
? bindHttpTelemetry(hostHttp, scope.httpTelemetry)
|
|
605
|
+
: undefined;
|
|
606
|
+
const contextTarget = observedHttp
|
|
607
|
+
? new Proxy(attached, {
|
|
608
|
+
get(target, property, receiver) {
|
|
609
|
+
return property === "http" ? observedHttp : Reflect.get(target, property, receiver);
|
|
610
|
+
},
|
|
611
|
+
})
|
|
612
|
+
: provider.http && !supportedHostHttp
|
|
613
|
+
? new Proxy(attached, {
|
|
614
|
+
get(target, property, receiver) {
|
|
615
|
+
return property === "http" ? undefined : Reflect.get(target, property, receiver);
|
|
616
|
+
},
|
|
617
|
+
})
|
|
618
|
+
: attached;
|
|
619
|
+
const context = wrapWithInstrumentation(contextTarget);
|
|
620
|
+
retryResponseMeta.set(context, scope.httpTelemetry);
|
|
612
621
|
return context;
|
|
613
622
|
}
|
|
614
623
|
function createFlowContextStore(allowedKeys, initialContext = {}) {
|
|
@@ -719,6 +728,7 @@ function createAuthFlowContext(provider, request, route, options, state, scope,
|
|
|
719
728
|
http: createHttpClient(baseUrl, {
|
|
720
729
|
...proxyClientOptions,
|
|
721
730
|
...(signal ? { signal } : {}),
|
|
731
|
+
httpTelemetry: scope.httpTelemetry,
|
|
722
732
|
}),
|
|
723
733
|
state: state.forConnection(resolveOperationConnectionId(request)),
|
|
724
734
|
stealth: stealthBaseUrl
|
|
@@ -1481,10 +1491,12 @@ function createRequestScope(input) {
|
|
|
1481
1491
|
const proxyCollector = new ProxyTelemetryCollector();
|
|
1482
1492
|
const resolverCollector = new ResolverTelemetryCollector({ redact });
|
|
1483
1493
|
const nativeCollector = new NativeTelemetryCollector({ redact });
|
|
1494
|
+
const httpCollector = new HttpTelemetryCollector({ redact });
|
|
1484
1495
|
const telemetry = new RequestTelemetry(trace);
|
|
1485
1496
|
telemetry.register(proxyCollector);
|
|
1486
1497
|
telemetry.register(resolverCollector);
|
|
1487
1498
|
telemetry.register(nativeCollector);
|
|
1499
|
+
telemetry.register(httpCollector);
|
|
1488
1500
|
let rootRunner = (fn) => fn();
|
|
1489
1501
|
let resolveRoot;
|
|
1490
1502
|
const rootTerminal = new Promise((resolve) => {
|
|
@@ -1562,6 +1574,7 @@ function createRequestScope(input) {
|
|
|
1562
1574
|
telemetry,
|
|
1563
1575
|
resolverTelemetry: resolverCollector,
|
|
1564
1576
|
nativeTelemetry: nativeCollector,
|
|
1577
|
+
httpTelemetry: httpCollector,
|
|
1565
1578
|
seedCredentials(rawBody, kind) {
|
|
1566
1579
|
const harvested = rawRequestCredentials(rawBody, kind);
|
|
1567
1580
|
sensitiveRegistry.add(harvested.values);
|
|
@@ -1743,7 +1756,7 @@ function toJsonSuccessResponse(result, ctx) {
|
|
|
1743
1756
|
return new Response(result);
|
|
1744
1757
|
}
|
|
1745
1758
|
const cacheMeta = ctx && "cache" in ctx ? ctx.cache.responseMeta() : undefined;
|
|
1746
|
-
const
|
|
1759
|
+
const retry = ctx ? retryResponseMeta.get(ctx)?.toTenantRetryPayload() : undefined;
|
|
1747
1760
|
const cache = cacheMeta
|
|
1748
1761
|
? {
|
|
1749
1762
|
hit: cacheMeta.hit,
|
|
@@ -1753,21 +1766,7 @@ function toJsonSuccessResponse(result, ctx) {
|
|
|
1753
1766
|
...(cacheMeta.source ? { source: closedEnum(cacheMeta.source) } : {}),
|
|
1754
1767
|
}
|
|
1755
1768
|
: undefined;
|
|
1756
|
-
const
|
|
1757
|
-
? {
|
|
1758
|
-
attempts: retryMeta.attempts,
|
|
1759
|
-
retries: retryMeta.retries,
|
|
1760
|
-
...(retryMeta.preset ? { preset: closedEnum(retryMeta.preset) } : {}),
|
|
1761
|
-
transport: closedEnum(retryMeta.transport),
|
|
1762
|
-
...(retryMeta.lastErrorCode
|
|
1763
|
-
? {
|
|
1764
|
-
lastErrorCode: closedEnum(tenantRetryLastErrorCode(retryMeta.lastErrorCode)),
|
|
1765
|
-
}
|
|
1766
|
-
: {}),
|
|
1767
|
-
...(retryMeta.lastStatus ? { lastStatus: retryMeta.lastStatus } : {}),
|
|
1768
|
-
}
|
|
1769
|
-
: undefined;
|
|
1770
|
-
const meta = cacheMeta || retryMeta
|
|
1769
|
+
const meta = cacheMeta || retry
|
|
1771
1770
|
? {
|
|
1772
1771
|
...(cache
|
|
1773
1772
|
? {
|
package/package.json
CHANGED