@apifuse/provider-sdk 2.2.0-beta.52 → 2.2.0-beta.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import { createHash } from "node:crypto";
2
+ import { AsyncLocalStorage } from "node:async_hooks";
2
3
  import { existsSync } from "node:fs";
3
4
  import { createRequire } from "node:module";
4
5
  import { join } from "node:path";
@@ -88,7 +89,12 @@ import {
88
89
  import { StealthCookieJar } from "../runtime/stealth-cookies.js";
89
90
  import type * as StealthRuntimeModule from "../runtime/stealth.js";
90
91
  import { createSttClientFromEnv } from "../runtime/stt.js";
91
- import { createTraceContext, type TraceRecorder } from "../runtime/trace.js";
92
+ import {
93
+ createTraceContext,
94
+ type TraceContext as RuntimeTraceContext,
95
+ type TraceRecorder,
96
+ updateTraceContextExportMetadata,
97
+ } from "../runtime/trace.js";
92
98
  import { resolveTraceConfigFromEnv } from "../runtime/trace-config.js";
93
99
  import { parseSchema } from "../schema.js";
94
100
  import {
@@ -99,6 +105,7 @@ import {
99
105
  } from "../stateful-signing.js";
100
106
  import { StatefulRoutingDeadlineError } from "../stateful/errors.js";
101
107
  import { getStealthProfile } from "../stealth/profiles.js";
108
+ import { sanitizeTraceAttributes } from "../trace-sanitization.js";
102
109
  import {
103
110
  APIFUSE_STREAM_DONE_EVENT,
104
111
  APIFUSE_STREAM_ERROR_EVENT,
@@ -260,7 +267,17 @@ export type ProviderServerOperationExecutor<
260
267
  input: ProviderServerOperationExecutorInput<TContext>,
261
268
  ) => Promise<unknown>;
262
269
 
263
- type RequestCleanup = () => void | Promise<void>;
270
+ type RequestTerminalOutcome =
271
+ | { kind: "completed"; status: number }
272
+ | { kind: "failed"; status?: number; error: unknown }
273
+ | { kind: "cancelled"; status?: number };
274
+
275
+ type RequestStreamLifecycle = {
276
+ runStep<T>(fn: () => Promise<T>): Promise<T>;
277
+ registerCleanup(cleanup: () => void | Promise<void>): void;
278
+ terminalize(outcome: RequestTerminalOutcome): void;
279
+ cleanup(): Promise<void>;
280
+ };
264
281
 
265
282
  type ProviderCapabilityModules = {
266
283
  readonly browser?: typeof BrowserRuntimeModule;
@@ -663,16 +680,20 @@ function providerSecretNames(provider: ProviderDefinition): string[] {
663
680
  );
664
681
  }
665
682
 
683
+ type RequestScopeContext = {
684
+ trace: RuntimeTraceContext;
685
+ proxyTelemetry: ProxyTelemetryCollector;
686
+ };
687
+
666
688
  function createProviderContext(
667
689
  provider: ProviderDefinition,
668
690
  request: OperationRequest,
669
691
  operationId: string,
670
692
  options: ProviderServerRuntimeOptions,
671
693
  state: ProviderRuntimeState = createUnsupportedProviderRuntimeState(),
672
- proxyTelemetry?: ProxyTelemetryCollector,
694
+ scope: RequestScopeContext,
673
695
  signal?: AbortSignal,
674
696
  ): ProviderContext {
675
- const traceConfig = resolveTraceConfigFromEnv();
676
697
  const baseUrl = getProviderBaseUrl(provider);
677
698
  const stealthBaseUrl = getProviderStealthBaseUrl(provider);
678
699
  const stealthProfile = getProviderStealthProfile(provider);
@@ -681,7 +702,7 @@ function createProviderContext(
681
702
  const proxyClientOptions = {
682
703
  upstream: { proxy: provider.proxy },
683
704
  affinityKey: resolveProviderProxyAffinityKey(provider, request, operationId),
684
- telemetry: proxyTelemetry,
705
+ telemetry: scope.proxyTelemetry,
685
706
  engineCredentials: engineProxyCredentials,
686
707
  };
687
708
  const resolverIdentityScope = resolveProviderResolverIdentityScope(
@@ -693,7 +714,7 @@ function createProviderContext(
693
714
  const stealthClientOptions = {
694
715
  upstream: proxyClientOptions.upstream,
695
716
  affinityKey: proxyClientOptions.affinityKey,
696
- telemetry: proxyTelemetry,
717
+ telemetry: scope.proxyTelemetry,
697
718
  engineCredentials: engineProxyCredentials,
698
719
  ...(signal ? { signal } : {}),
699
720
  ...(provider.stealth ? { stealth: provider.stealth } : {}),
@@ -771,15 +792,7 @@ function createProviderContext(
771
792
  },
772
793
  }
773
794
  : {}),
774
- trace: traceConfig
775
- ? createTraceContext(
776
- resolveServerTraceContextOptions(traceConfig, {
777
- request_id: request.requestId,
778
- provider_id: provider.id,
779
- operation_id: operationId,
780
- }),
781
- )
782
- : createTraceContext(),
795
+ trace: scope.trace,
783
796
  auth: createAuthStub(),
784
797
  ocr: options.ocr ?? createOcrClientFromEnv(provider.ocr),
785
798
  stt: options.stt ?? createSttClientFromEnv(provider.stt),
@@ -885,7 +898,7 @@ function createAuthFlowContext(
885
898
  request: AuthFlowRequest,
886
899
  options: ProviderServerRuntimeOptions,
887
900
  state: ProviderRuntimeState,
888
- proxyTelemetry?: ProxyTelemetryCollector,
901
+ scope: RequestScopeContext,
889
902
  signal?: AbortSignal,
890
903
  ): {
891
904
  context: FlowContext;
@@ -904,7 +917,7 @@ function createAuthFlowContext(
904
917
  const proxyClientOptions = {
905
918
  upstream: { proxy: provider.proxy },
906
919
  affinityKey: resolveAuthFlowProxyAffinityKey(provider, request),
907
- telemetry: proxyTelemetry,
920
+ telemetry: scope.proxyTelemetry,
908
921
  engineCredentials: engineProxyCredentials,
909
922
  };
910
923
  const resolverIdentityScope = resolveProviderResolverIdentityScope(
@@ -915,7 +928,7 @@ function createAuthFlowContext(
915
928
  const stealthClientOptions = {
916
929
  upstream: proxyClientOptions.upstream,
917
930
  affinityKey: proxyClientOptions.affinityKey,
918
- telemetry: proxyTelemetry,
931
+ telemetry: scope.proxyTelemetry,
919
932
  engineCredentials: engineProxyCredentials,
920
933
  ...(signal ? { signal } : {}),
921
934
  ...(provider.stealth ? { stealth: provider.stealth } : {}),
@@ -941,71 +954,74 @@ function createAuthFlowContext(
941
954
  : undefined;
942
955
  const cache = createProviderCache({ providerId: provider.id });
943
956
 
957
+ const context: FlowContext = wrapWithInstrumentation({
958
+ flowId: request.flowId,
959
+ connectionId: resolveOperationConnectionId(request),
960
+ externalRef: request.externalRef,
961
+ tenantId: request.tenantId ?? "",
962
+ providerId: request.providerId ?? provider.id,
963
+ trace: scope.trace,
964
+ http: createHttpClient(baseUrl, {
965
+ ...proxyClientOptions,
966
+ ...(signal ? { signal } : {}),
967
+ }),
968
+ state: state.forConnection(resolveOperationConnectionId(request)),
969
+ stealth: stealthBaseUrl
970
+ ? capabilityModules.stealth
971
+ ? capabilityModules.stealth.createStealthClient(stealthBaseUrl, stealthClientOptions)
972
+ : createLazyStealthClient(logStealthCleanupError, stealthBaseUrl, stealthClientOptions)
973
+ : createStealthStub(),
974
+ ...(provider.native
975
+ ? {
976
+ native: {
977
+ network: capabilityModules.nativeNetwork!.createNativeNetworkClient({
978
+ egress: provider.native.network,
979
+ proxyPolicy: resolveNativeProxyPolicy(provider),
980
+ affinityKey: proxyClientOptions.affinityKey,
981
+ credentials: capabilityModules.nativeNetwork!.createEnvVendorCredentialResolver(
982
+ createEnvContext([...ENGINE_OWNED_PROXY_CREDENTIAL_ENV_NAMES]),
983
+ ),
984
+ }),
985
+ },
986
+ }
987
+ : {}),
988
+ env: createEnvContext([
989
+ ...providerSecretNames(provider),
990
+ ...(provider.auth?.mode === "oauth2_proxied" ? ["APIFUSE__AUTH_PROXY__URL"] : []),
991
+ ]),
992
+ credential,
993
+ context: flowContextStore.context,
994
+ ocr: options.ocr ?? createOcrClientFromEnv(provider.ocr),
995
+ stt: options.stt ?? createSttClientFromEnv(provider.stt),
996
+ resolver: capabilityModules.resolver
997
+ ? capabilityModules.resolver.bindResolverSignal(
998
+ options.resolver ??
999
+ capabilityModules.resolver.createResolverClientFromEnv(provider.resolver, undefined, {
1000
+ allowedHosts: provider.allowedHosts,
1001
+ cache,
1002
+ identityScope: resolverIdentityScope,
1003
+ ...(proxyPolicy
1004
+ ? {
1005
+ proxyIntent: {
1006
+ mode: proxyPolicy.mode,
1007
+ ...proxyClientOptions,
1008
+ ...(stealthProfile ? { userAgent: stealthProfile.userAgent } : {}),
1009
+ },
1010
+ }
1011
+ : {}),
1012
+ }),
1013
+ signal,
1014
+ )
1015
+ : bindResolverSignalWithoutRuntime(
1016
+ options.resolver ??
1017
+ createUnsupportedResolverClient("Provider does not declare resolver capability"),
1018
+ signal,
1019
+ ),
1020
+ auth: createAuthFlowHelpers({ signal }),
1021
+ });
1022
+
944
1023
  return {
945
- context: {
946
- flowId: request.flowId,
947
- connectionId: resolveOperationConnectionId(request),
948
- externalRef: request.externalRef,
949
- tenantId: request.tenantId ?? "",
950
- providerId: request.providerId ?? provider.id,
951
- http: createHttpClient(baseUrl, {
952
- ...proxyClientOptions,
953
- ...(signal ? { signal } : {}),
954
- }),
955
- state: state.forConnection(resolveOperationConnectionId(request)),
956
- stealth: stealthBaseUrl
957
- ? capabilityModules.stealth
958
- ? capabilityModules.stealth.createStealthClient(stealthBaseUrl, stealthClientOptions)
959
- : createLazyStealthClient(logStealthCleanupError, stealthBaseUrl, stealthClientOptions)
960
- : createStealthStub(),
961
- ...(provider.native
962
- ? {
963
- native: {
964
- network: capabilityModules.nativeNetwork!.createNativeNetworkClient({
965
- egress: provider.native.network,
966
- proxyPolicy: resolveNativeProxyPolicy(provider),
967
- affinityKey: proxyClientOptions.affinityKey,
968
- credentials: capabilityModules.nativeNetwork!.createEnvVendorCredentialResolver(
969
- createEnvContext([...ENGINE_OWNED_PROXY_CREDENTIAL_ENV_NAMES]),
970
- ),
971
- }),
972
- },
973
- }
974
- : {}),
975
- env: createEnvContext([
976
- ...providerSecretNames(provider),
977
- ...(provider.auth?.mode === "oauth2_proxied" ? ["APIFUSE__AUTH_PROXY__URL"] : []),
978
- ]),
979
- credential,
980
- context: flowContextStore.context,
981
- ocr: options.ocr ?? createOcrClientFromEnv(provider.ocr),
982
- stt: options.stt ?? createSttClientFromEnv(provider.stt),
983
- resolver: capabilityModules.resolver
984
- ? capabilityModules.resolver.bindResolverSignal(
985
- options.resolver ??
986
- capabilityModules.resolver.createResolverClientFromEnv(provider.resolver, undefined, {
987
- allowedHosts: provider.allowedHosts,
988
- cache,
989
- identityScope: resolverIdentityScope,
990
- ...(proxyPolicy
991
- ? {
992
- proxyIntent: {
993
- mode: proxyPolicy.mode,
994
- ...proxyClientOptions,
995
- ...(stealthProfile ? { userAgent: stealthProfile.userAgent } : {}),
996
- },
997
- }
998
- : {}),
999
- }),
1000
- signal,
1001
- )
1002
- : bindResolverSignalWithoutRuntime(
1003
- options.resolver ??
1004
- createUnsupportedResolverClient("Provider does not declare resolver capability"),
1005
- signal,
1006
- ),
1007
- auth: createAuthFlowHelpers({ signal }),
1008
- },
1024
+ context,
1009
1025
  getPatch: flowContextStore.getPatch,
1010
1026
  };
1011
1027
  }
@@ -1017,11 +1033,22 @@ type ProviderRequestCost = {
1017
1033
  cpuTotalMicros: number;
1018
1034
  };
1019
1035
 
1036
+ type RequestCorrelationIds = {
1037
+ connectionId?: string;
1038
+ flowId?: string;
1039
+ tenantId?: string;
1040
+ requestedProviderId?: string;
1041
+ };
1042
+
1020
1043
  type ProviderServerLogEventBase = ProviderRequestCost & {
1021
1044
  providerId: string;
1022
1045
  kind: "operation" | "auth";
1023
1046
  route: string;
1024
1047
  requestId?: string;
1048
+ connectionId?: string;
1049
+ flowId?: string;
1050
+ tenantId?: string;
1051
+ requestedProviderId?: string;
1025
1052
  status: number;
1026
1053
  proxy?: ProxyTelemetryLogPayload;
1027
1054
  };
@@ -1589,6 +1616,7 @@ function logProviderError(
1589
1616
  declaredErrorCode: OperationErrorCode | undefined,
1590
1617
  proxyTelemetry: ProxyTelemetryCollector | undefined,
1591
1618
  observabilityDetails: ErrorObservabilityDetails,
1619
+ correlation: RequestCorrelationIds = {},
1592
1620
  ): void {
1593
1621
  const providerCode = isProviderError(error) ? providerErrorCode(error) : undefined;
1594
1622
  const code = isProviderError(error)
@@ -1624,6 +1652,14 @@ function logProviderError(
1624
1652
  kind,
1625
1653
  route,
1626
1654
  ...(requestId ? { requestId } : {}),
1655
+ ...(correlation.connectionId !== undefined
1656
+ ? { connectionId: correlation.connectionId }
1657
+ : {}),
1658
+ ...(correlation.flowId !== undefined ? { flowId: correlation.flowId } : {}),
1659
+ ...(correlation.tenantId !== undefined ? { tenantId: correlation.tenantId } : {}),
1660
+ ...(correlation.requestedProviderId !== undefined
1661
+ ? { requestedProviderId: correlation.requestedProviderId }
1662
+ : {}),
1627
1663
  status,
1628
1664
  ...cost,
1629
1665
  ...(proxy ? { proxy } : {}),
@@ -1681,6 +1717,7 @@ function logProviderSuccess(
1681
1717
  status: number,
1682
1718
  cost: ProviderRequestCost,
1683
1719
  proxyTelemetry?: ProxyTelemetryCollector,
1720
+ correlation: RequestCorrelationIds = {},
1684
1721
  ): void {
1685
1722
  const proxy = proxyTelemetry?.toLogPayload();
1686
1723
  const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
@@ -1691,12 +1728,372 @@ function logProviderSuccess(
1691
1728
  kind,
1692
1729
  route,
1693
1730
  ...(requestId ? { requestId } : {}),
1731
+ ...(correlation.connectionId !== undefined
1732
+ ? { connectionId: correlation.connectionId }
1733
+ : {}),
1734
+ ...(correlation.flowId !== undefined ? { flowId: correlation.flowId } : {}),
1735
+ ...(correlation.tenantId !== undefined ? { tenantId: correlation.tenantId } : {}),
1736
+ ...(correlation.requestedProviderId !== undefined
1737
+ ? { requestedProviderId: correlation.requestedProviderId }
1738
+ : {}),
1694
1739
  status,
1695
1740
  ...cost,
1696
1741
  ...(proxy ? { proxy } : {}),
1697
1742
  });
1698
1743
  }
1699
1744
 
1745
+ type RequestScopeFinishResult = {
1746
+ providerTelemetryHeader?: string;
1747
+ errorObservability?: ErrorObservabilityDetails;
1748
+ };
1749
+
1750
+ type RequestScope = RequestScopeContext & {
1751
+ enrich(input: {
1752
+ route?: string;
1753
+ requestId?: string;
1754
+ operationId?: string;
1755
+ flowId?: string;
1756
+ headers?: Record<string, string>;
1757
+ correlation?: RequestCorrelationIds;
1758
+ }): void;
1759
+ run<T>(fn: () => Promise<T>): Promise<T>;
1760
+ runStreaming<T>(fn: () => Promise<T>): Promise<T>;
1761
+ runStreamStep<T>(fn: () => Promise<T>): Promise<T>;
1762
+ watchAbort(signal: AbortSignal): void;
1763
+ registerStreamCleanup(cleanup: () => void | Promise<void>): void;
1764
+ cleanupStream(): Promise<void>;
1765
+ snapshotHeaders(error?: unknown): RequestScopeFinishResult;
1766
+ terminalize(outcome: RequestTerminalOutcome): RequestScopeFinishResult;
1767
+ };
1768
+
1769
+ const STREAM_CLEANUP_TIMEOUT_MS = 100;
1770
+ const CLIENT_CANCELLED_STATUS = 400;
1771
+
1772
+ function clientCancelledError(): ProviderError {
1773
+ return new ProviderError("Request stream was cancelled by the client.", {
1774
+ code: "client_cancelled",
1775
+ category: "client_cancelled",
1776
+ retryable: false,
1777
+ });
1778
+ }
1779
+
1780
+ async function runBestEffortWithTimeout(cleanup: () => void | Promise<void>): Promise<void> {
1781
+ let timeout: ReturnType<typeof setTimeout> | undefined;
1782
+ const work = Promise.resolve()
1783
+ .then(cleanup)
1784
+ .catch(() => undefined);
1785
+ try {
1786
+ await Promise.race([
1787
+ work,
1788
+ new Promise<void>((resolve) => {
1789
+ timeout = setTimeout(resolve, STREAM_CLEANUP_TIMEOUT_MS);
1790
+ }),
1791
+ ]);
1792
+ } finally {
1793
+ if (timeout !== undefined) clearTimeout(timeout);
1794
+ }
1795
+ }
1796
+
1797
+ function traceIdFromTraceparent(headers: Record<string, string>): string | undefined {
1798
+ const traceparent = Object.entries(headers).find(
1799
+ ([name]) => name.toLowerCase() === "traceparent",
1800
+ )?.[1];
1801
+ if (!traceparent) return undefined;
1802
+ const match = /^(00)-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/.exec(traceparent);
1803
+ if (!match) return undefined;
1804
+ const traceId = match[2];
1805
+ const parentId = match[3];
1806
+ if (!traceId || !parentId || /^0+$/.test(traceId) || /^0+$/.test(parentId)) return undefined;
1807
+ return traceId;
1808
+ }
1809
+
1810
+ function requestTraceId(
1811
+ headers: Record<string, string>,
1812
+ requestId: string | undefined,
1813
+ ): string | undefined {
1814
+ return (
1815
+ traceIdFromTraceparent(headers) ??
1816
+ (requestId
1817
+ ? createHash("sha256").update(requestId, "utf8").digest("hex").slice(0, 32)
1818
+ : undefined)
1819
+ );
1820
+ }
1821
+
1822
+ function createRequestScope(input: {
1823
+ provider: ProviderDefinition;
1824
+ kind: "operation" | "auth";
1825
+ route: string;
1826
+ requestId?: string;
1827
+ operationId?: string;
1828
+ flowId?: string;
1829
+ headers: Record<string, string>;
1830
+ correlation?: RequestCorrelationIds;
1831
+ logger?: ProviderServerLogger;
1832
+ setHeader?: (name: string, value: string) => void;
1833
+ declaredErrorCode?: (error: unknown) => OperationErrorCode | undefined;
1834
+ }): RequestScope {
1835
+ const requestCost = startRequestCost();
1836
+ const traceConfig = resolveTraceConfigFromEnv();
1837
+ const proxyTelemetry = new ProxyTelemetryCollector();
1838
+ const details = {
1839
+ route: input.route,
1840
+ requestId: input.requestId,
1841
+ operationId: input.operationId,
1842
+ flowId: input.flowId,
1843
+ headers: input.headers,
1844
+ correlation: { ...(input.correlation ?? {}) },
1845
+ };
1846
+ const traceAttributes: Record<string, string> = {
1847
+ provider_id: input.provider.id,
1848
+ ...(input.operationId ? { operation_id: input.operationId } : {}),
1849
+ ...(input.flowId ? { flow_id: input.flowId } : {}),
1850
+ ...(input.requestId ? { request_id: input.requestId } : {}),
1851
+ route: input.route,
1852
+ };
1853
+ const initialTraceId = requestTraceId(input.headers, input.requestId);
1854
+ const trace: RuntimeTraceContext = traceConfig
1855
+ ? createTraceContext({
1856
+ ...resolveServerTraceContextOptions(traceConfig, traceAttributes),
1857
+ ...(initialTraceId ? { traceId: initialTraceId } : {}),
1858
+ })
1859
+ : createTraceContext();
1860
+ let rootRunner: <T>(fn: () => Promise<T>) => Promise<T> = (fn) => fn();
1861
+ let resolveRoot!: (outcome: RequestTerminalOutcome) => void;
1862
+ const rootTerminal = new Promise<RequestTerminalOutcome>((resolve) => {
1863
+ resolveRoot = resolve;
1864
+ });
1865
+ let streamingSetupPending = false;
1866
+ let rootSettled = false;
1867
+ let headersSnapshotted = false;
1868
+ let terminalOutcome: RequestTerminalOutcome | undefined;
1869
+ let finishedResult: RequestScopeFinishResult | undefined;
1870
+ let abortSignal: AbortSignal | undefined;
1871
+ let abortListener: (() => void) | undefined;
1872
+ type CleanupEntry = {
1873
+ cleanup: () => void | Promise<void>;
1874
+ promise?: Promise<void>;
1875
+ };
1876
+ const streamCleanups: CleanupEntry[] = [];
1877
+
1878
+ const terminalError = (outcome: RequestTerminalOutcome): unknown | undefined => {
1879
+ if (outcome.kind === "failed") return outcome.error;
1880
+ if (outcome.kind === "cancelled") return clientCancelledError();
1881
+ return undefined;
1882
+ };
1883
+ const root = trace.span(`request:${input.kind}:${input.route}`, async () => {
1884
+ rootRunner = AsyncLocalStorage.snapshot();
1885
+ const outcome = await rootTerminal;
1886
+ const error = terminalError(outcome);
1887
+ if (error !== undefined) throw error;
1888
+ });
1889
+ void root.catch(() => undefined);
1890
+
1891
+ const settleRoot = (): void => {
1892
+ if (!terminalOutcome || streamingSetupPending || rootSettled) return;
1893
+ rootSettled = true;
1894
+ if (abortSignal && abortListener) {
1895
+ abortSignal.removeEventListener("abort", abortListener);
1896
+ }
1897
+ resolveRoot(terminalOutcome);
1898
+ };
1899
+ const runCleanupEntry = (entry: CleanupEntry): Promise<void> => {
1900
+ entry.promise ??= runBestEffortWithTimeout(entry.cleanup);
1901
+ return entry.promise;
1902
+ };
1903
+ const cleanupStream = async (): Promise<void> => {
1904
+ await Promise.all(streamCleanups.map(runCleanupEntry));
1905
+ };
1906
+ const headerSnapshot = (error?: unknown): RequestScopeFinishResult => {
1907
+ const providerTelemetryHeader = proxyTelemetry.toHeaderValue();
1908
+ const declaredErrorCode = error === undefined ? undefined : input.declaredErrorCode?.(error);
1909
+ const errorObservability =
1910
+ error === undefined ? undefined : errorObservabilityDetails(error, declaredErrorCode);
1911
+ return {
1912
+ ...(providerTelemetryHeader ? { providerTelemetryHeader } : {}),
1913
+ ...(errorObservability ? { errorObservability } : {}),
1914
+ };
1915
+ };
1916
+ const updateTraceMetadata = (): void => {
1917
+ if (details.requestId) traceAttributes.request_id = details.requestId;
1918
+ if (details.operationId) traceAttributes.operation_id = details.operationId;
1919
+ if (details.flowId) traceAttributes.flow_id = details.flowId;
1920
+ traceAttributes.route = details.route;
1921
+ const traceId = requestTraceId(details.headers, details.requestId);
1922
+ const sanitizedAttributes = Object.fromEntries(
1923
+ Object.entries(sanitizeTraceAttributes(traceAttributes)).map(([key, value]) => [
1924
+ key,
1925
+ String(value),
1926
+ ]),
1927
+ );
1928
+ updateTraceContextExportMetadata(trace, {
1929
+ ...(traceId ? { traceId } : {}),
1930
+ resourceAttributes: sanitizedAttributes,
1931
+ });
1932
+ };
1933
+
1934
+ const scope: RequestScope = {
1935
+ trace,
1936
+ proxyTelemetry,
1937
+ enrich(enrichment): void {
1938
+ if (terminalOutcome) return;
1939
+ if (enrichment.route !== undefined) details.route = enrichment.route;
1940
+ if (enrichment.requestId !== undefined) details.requestId = enrichment.requestId;
1941
+ if (enrichment.operationId !== undefined) details.operationId = enrichment.operationId;
1942
+ if (enrichment.flowId !== undefined) details.flowId = enrichment.flowId;
1943
+ if (enrichment.headers !== undefined) details.headers = enrichment.headers;
1944
+ if (enrichment.correlation !== undefined) {
1945
+ Object.assign(details.correlation, enrichment.correlation);
1946
+ }
1947
+ updateTraceMetadata();
1948
+ },
1949
+ run<T>(fn: () => Promise<T>): Promise<T> {
1950
+ return rootRunner(fn);
1951
+ },
1952
+ async runStreaming<T>(fn: () => Promise<T>): Promise<T> {
1953
+ streamingSetupPending = true;
1954
+ try {
1955
+ return await rootRunner(fn);
1956
+ } finally {
1957
+ streamingSetupPending = false;
1958
+ settleRoot();
1959
+ }
1960
+ },
1961
+ runStreamStep<T>(fn: () => Promise<T>): Promise<T> {
1962
+ return rootRunner(fn);
1963
+ },
1964
+ watchAbort(signal): void {
1965
+ if (terminalOutcome) return;
1966
+ abortSignal = signal;
1967
+ abortListener = () => {
1968
+ scope.terminalize({ kind: "cancelled", status: CLIENT_CANCELLED_STATUS });
1969
+ void cleanupStream();
1970
+ };
1971
+ if (signal.aborted) abortListener();
1972
+ else signal.addEventListener("abort", abortListener, { once: true });
1973
+ },
1974
+ registerStreamCleanup(cleanup): void {
1975
+ const entry = { cleanup };
1976
+ streamCleanups.push(entry);
1977
+ if (terminalOutcome) void runCleanupEntry(entry);
1978
+ },
1979
+ cleanupStream,
1980
+ snapshotHeaders(error): RequestScopeFinishResult {
1981
+ headersSnapshotted = true;
1982
+ return headerSnapshot(error);
1983
+ },
1984
+ terminalize(outcome): RequestScopeFinishResult {
1985
+ if (finishedResult) return finishedResult;
1986
+ terminalOutcome = outcome;
1987
+ const error = terminalError(outcome);
1988
+ finishedResult = {};
1989
+ try {
1990
+ const declaredErrorCode =
1991
+ error === undefined ? undefined : input.declaredErrorCode?.(error);
1992
+ const status =
1993
+ outcome.status ??
1994
+ (error === undefined ? 200 : toStatusCode(error, declaredErrorCode));
1995
+ finishedResult = headerSnapshot(error);
1996
+ const cost = finishRequestCost(requestCost);
1997
+ try {
1998
+ if (error === undefined) {
1999
+ logProviderSuccess(
2000
+ input.logger,
2001
+ input.provider,
2002
+ input.kind,
2003
+ details.route,
2004
+ details.requestId,
2005
+ status,
2006
+ cost,
2007
+ proxyTelemetry,
2008
+ details.correlation,
2009
+ );
2010
+ } else {
2011
+ logProviderError(
2012
+ input.logger,
2013
+ input.provider,
2014
+ input.kind,
2015
+ details.route,
2016
+ details.requestId,
2017
+ error,
2018
+ status,
2019
+ cost,
2020
+ declaredErrorCode,
2021
+ proxyTelemetry,
2022
+ finishedResult.errorObservability as ErrorObservabilityDetails,
2023
+ details.correlation,
2024
+ );
2025
+ }
2026
+ } catch {
2027
+ // Observer callbacks cannot hold the request root open or change the response.
2028
+ }
2029
+
2030
+ if (!headersSnapshotted) {
2031
+ if (finishedResult.providerTelemetryHeader) {
2032
+ try {
2033
+ input.setHeader?.(PROVIDER_TELEMETRY_HEADER, finishedResult.providerTelemetryHeader);
2034
+ } catch {
2035
+ // Header observers are isolated independently from request settlement.
2036
+ }
2037
+ }
2038
+ if (finishedResult.errorObservability) {
2039
+ try {
2040
+ input.setHeader?.(
2041
+ ERROR_OBSERVABILITY_HEADER,
2042
+ JSON.stringify(finishedResult.errorObservability),
2043
+ );
2044
+ } catch {
2045
+ // Header observers are isolated independently from request settlement.
2046
+ }
2047
+ }
2048
+ }
2049
+ } finally {
2050
+ settleRoot();
2051
+ void cleanupStream();
2052
+ }
2053
+ return finishedResult;
2054
+ },
2055
+ };
2056
+ return scope;
2057
+ }
2058
+
2059
+ function responseWithRequestScopeHeaders(
2060
+ response: Response,
2061
+ finished: RequestScopeFinishResult,
2062
+ ): Response {
2063
+ const headers = new Headers(response.headers);
2064
+ headers.delete(PROVIDER_TELEMETRY_HEADER);
2065
+ if (finished.providerTelemetryHeader) {
2066
+ headers.set(PROVIDER_TELEMETRY_HEADER, finished.providerTelemetryHeader);
2067
+ }
2068
+ if (finished.errorObservability) {
2069
+ headers.set(ERROR_OBSERVABILITY_HEADER, JSON.stringify(finished.errorObservability));
2070
+ }
2071
+ return new Response(response.body, {
2072
+ headers,
2073
+ status: response.status,
2074
+ statusText: response.statusText,
2075
+ });
2076
+ }
2077
+
2078
+ function finalizeRequestResponse(
2079
+ scope: RequestScope,
2080
+ response: Response,
2081
+ outcome: RequestTerminalOutcome,
2082
+ ): Response {
2083
+ try {
2084
+ const error = outcome.kind === "failed" ? outcome.error : undefined;
2085
+ const finalResponse = responseWithRequestScopeHeaders(
2086
+ response,
2087
+ scope.snapshotHeaders(error),
2088
+ );
2089
+ scope.terminalize(outcome);
2090
+ return finalResponse;
2091
+ } catch (error) {
2092
+ scope.terminalize({ kind: "failed", status: 500, error });
2093
+ throw error;
2094
+ }
2095
+ }
2096
+
1700
2097
  function toJsonSuccessResponse(
1701
2098
  result: unknown,
1702
2099
  ctx?: ProviderContext,
@@ -1736,41 +2133,41 @@ function isAsyncIterable<T = unknown>(value: unknown): value is AsyncIterable<T>
1736
2133
  return typeof iterator === "function";
1737
2134
  }
1738
2135
 
1739
- function responseWithCleanup(response: Response, cleanup: RequestCleanup): Response {
2136
+ function responseWithCleanup(response: Response, lifecycle: RequestStreamLifecycle): Response {
1740
2137
  if (!response.body) {
1741
- void cleanup();
2138
+ lifecycle.terminalize({ kind: "completed", status: response.status });
2139
+ void lifecycle.cleanup();
1742
2140
  return response;
1743
2141
  }
1744
2142
  const reader = response.body.getReader();
1745
- let cleaned = false;
1746
- const runCleanup = async () => {
1747
- if (cleaned) return;
1748
- cleaned = true;
1749
- await cleanup();
1750
- };
1751
- const body = new ReadableStream<Uint8Array>({
1752
- async pull(controller) {
1753
- try {
1754
- const { done, value } = await reader.read();
1755
- if (done) {
1756
- controller.close();
1757
- await runCleanup();
1758
- return;
2143
+ let cleanupReason: unknown = "request terminalized";
2144
+ lifecycle.registerCleanup(() => lifecycle.runStep(() => reader.cancel(cleanupReason)));
2145
+ const body = new ReadableStream<Uint8Array>(
2146
+ {
2147
+ async pull(controller) {
2148
+ try {
2149
+ const { done, value } = await lifecycle.runStep(() => reader.read());
2150
+ if (done) {
2151
+ controller.close();
2152
+ lifecycle.terminalize({ kind: "completed", status: response.status });
2153
+ await lifecycle.cleanup();
2154
+ return;
2155
+ }
2156
+ if (value) controller.enqueue(value);
2157
+ } catch (error) {
2158
+ lifecycle.terminalize({ kind: "failed", error });
2159
+ await lifecycle.cleanup();
2160
+ controller.error(error);
1759
2161
  }
1760
- if (value) controller.enqueue(value);
1761
- } catch (error) {
1762
- await runCleanup();
1763
- controller.error(error);
1764
- }
1765
- },
1766
- async cancel(reason) {
1767
- try {
1768
- await reader.cancel(reason);
1769
- } finally {
1770
- await runCleanup();
1771
- }
2162
+ },
2163
+ async cancel(reason) {
2164
+ cleanupReason = reason;
2165
+ lifecycle.terminalize({ kind: "cancelled", status: CLIENT_CANCELLED_STATUS });
2166
+ await lifecycle.cleanup();
2167
+ },
1772
2168
  },
1773
- });
2169
+ { highWaterMark: 0 },
2170
+ );
1774
2171
  return new Response(body, {
1775
2172
  headers: response.headers,
1776
2173
  status: response.status,
@@ -1832,63 +2229,74 @@ function assertStreamPayloadWithinLimit(
1832
2229
  function toSseResponse(
1833
2230
  operation: OperationDefinition,
1834
2231
  result: AsyncIterable<ProviderStreamEvent>,
1835
- cleanup: RequestCleanup,
2232
+ lifecycle: RequestStreamLifecycle,
1836
2233
  requestId?: string,
1837
2234
  ): Response {
1838
2235
  const encoder = new TextEncoder();
1839
2236
  const iterator = result[Symbol.asyncIterator]();
2237
+ let cleanupReason: unknown;
2238
+ lifecycle.registerCleanup(() =>
2239
+ lifecycle.runStep(async () => {
2240
+ await iterator.return?.(cleanupReason);
2241
+ }),
2242
+ );
1840
2243
  const transport = getSseTransport(operation);
1841
2244
  let done = false;
1842
- let cleaned = false;
1843
- const runCleanup = async () => {
1844
- if (cleaned) return;
1845
- cleaned = true;
1846
- await cleanup();
1847
- };
1848
- const body = new ReadableStream<Uint8Array>({
1849
- async pull(controller) {
1850
- try {
1851
- if (done) {
2245
+ const body = new ReadableStream<Uint8Array>(
2246
+ {
2247
+ async pull(controller) {
2248
+ try {
2249
+ if (done) {
2250
+ controller.close();
2251
+ lifecycle.terminalize({ kind: "completed", status: 200 });
2252
+ await lifecycle.cleanup();
2253
+ return;
2254
+ }
2255
+ const next = await lifecycle.runStep(() => iterator.next());
2256
+ if (next.done) {
2257
+ done = true;
2258
+ controller.close();
2259
+ lifecycle.terminalize({ kind: "completed", status: 200 });
2260
+ await lifecycle.cleanup();
2261
+ return;
2262
+ }
2263
+ const encodedBytes = await lifecycle.runStep(async () => {
2264
+ const validated = await validateSseEvent(operation, next.value);
2265
+ const encodedEvent = encodeSseEvent(validated);
2266
+ const bytes = encoder.encode(encodedEvent);
2267
+ assertStreamPayloadWithinLimit(
2268
+ bytes.byteLength,
2269
+ transport?.maxEventBytes,
2270
+ "event",
2271
+ );
2272
+ return bytes;
2273
+ });
2274
+ controller.enqueue(encodedBytes);
2275
+ } catch (error) {
2276
+ const message = error instanceof Error ? error.message : "Stream failed";
2277
+ controller.enqueue(
2278
+ encoder.encode(
2279
+ encodeSseEvent(
2280
+ streamError("stream_error", message, {
2281
+ ...(requestId ? { requestId } : {}),
2282
+ }),
2283
+ ),
2284
+ ),
2285
+ );
1852
2286
  controller.close();
1853
- await runCleanup();
1854
- return;
1855
- }
1856
- const next = await iterator.next();
1857
- if (next.done) {
1858
2287
  done = true;
1859
- controller.close();
1860
- await runCleanup();
1861
- return;
2288
+ lifecycle.terminalize({ kind: "failed", error });
2289
+ await lifecycle.cleanup();
1862
2290
  }
1863
- const validated = await validateSseEvent(operation, next.value);
1864
- const encodedEvent = encodeSseEvent(validated);
1865
- const encodedBytes = encoder.encode(encodedEvent);
1866
- assertStreamPayloadWithinLimit(encodedBytes.byteLength, transport?.maxEventBytes, "event");
1867
- controller.enqueue(encodedBytes);
1868
- } catch (error) {
1869
- const message = error instanceof Error ? error.message : "Stream failed";
1870
- controller.enqueue(
1871
- encoder.encode(
1872
- encodeSseEvent(
1873
- streamError("stream_error", message, {
1874
- ...(requestId ? { requestId } : {}),
1875
- }),
1876
- ),
1877
- ),
1878
- );
1879
- controller.close();
1880
- done = true;
1881
- await runCleanup();
1882
- }
1883
- },
1884
- async cancel(reason) {
1885
- try {
1886
- await iterator.return?.(reason);
1887
- } finally {
1888
- await runCleanup();
1889
- }
2291
+ },
2292
+ async cancel(reason) {
2293
+ cleanupReason = reason;
2294
+ lifecycle.terminalize({ kind: "cancelled", status: CLIENT_CANCELLED_STATUS });
2295
+ await lifecycle.cleanup();
2296
+ },
1890
2297
  },
1891
- });
2298
+ { highWaterMark: 0 },
2299
+ );
1892
2300
  return new Response(body, {
1893
2301
  headers: {
1894
2302
  "Cache-Control": "no-cache, no-transform",
@@ -1929,12 +2337,11 @@ function enforceStreamChunkLimit(
1929
2337
  function toStreamingResponse(
1930
2338
  operation: OperationDefinition,
1931
2339
  result: unknown,
1932
- cleanup: RequestCleanup,
2340
+ lifecycle: RequestStreamLifecycle,
1933
2341
  requestId?: string,
1934
2342
  ): Response {
1935
2343
  const transport = operation.transport?.kind ?? "json";
1936
2344
  if (transport === "sse" && (result instanceof Response || result instanceof ReadableStream)) {
1937
- void cleanup();
1938
2345
  throw new ProviderError(
1939
2346
  "SSE operations must return an AsyncIterable of typed stream.event(...) values.",
1940
2347
  {
@@ -1954,10 +2361,10 @@ function toStreamingResponse(
1954
2361
  status: result.status,
1955
2362
  statusText: result.statusText,
1956
2363
  }),
1957
- cleanup,
2364
+ lifecycle,
1958
2365
  );
1959
2366
  }
1960
- return responseWithCleanup(result, cleanup);
2367
+ return responseWithCleanup(result, lifecycle);
1961
2368
  }
1962
2369
  if (result instanceof ReadableStream) {
1963
2370
  const httpTransport = getHttpStreamTransport(operation);
@@ -1977,13 +2384,12 @@ function toStreamingResponse(
1977
2384
  : "application/octet-stream",
1978
2385
  },
1979
2386
  }),
1980
- cleanup,
2387
+ lifecycle,
1981
2388
  );
1982
2389
  }
1983
2390
  if (transport === "sse" && isAsyncIterable<ProviderStreamEvent>(result)) {
1984
- return toSseResponse(operation, result, cleanup, requestId);
2391
+ return toSseResponse(operation, result, lifecycle, requestId);
1985
2392
  }
1986
- void cleanup();
1987
2393
  throw new ProviderError(
1988
2394
  `Streaming operation returned unsupported result for transport "${transport}"`,
1989
2395
  {
@@ -2092,18 +2498,10 @@ async function handleOperation(
2092
2498
  operationId: string,
2093
2499
  options: ProviderServerRuntimeOptions,
2094
2500
  state: ProviderRuntimeState = createUnsupportedProviderRuntimeState(),
2095
- proxyTelemetry?: ProxyTelemetryCollector,
2501
+ scope: RequestScope,
2096
2502
  signal?: AbortSignal,
2097
2503
  ): Promise<Response | OperationResponse> {
2098
- const ctx = createProviderContext(
2099
- provider,
2100
- request,
2101
- operationId,
2102
- options,
2103
- state,
2104
- proxyTelemetry,
2105
- signal,
2106
- );
2504
+ const ctx = createProviderContext(provider, request, operationId, options, state, scope, signal);
2107
2505
  const operation = provider.operations[operationId];
2108
2506
  const streaming = operation?.transport?.kind && operation.transport.kind !== "json";
2109
2507
  let cleanupCalled = false;
@@ -2141,6 +2539,21 @@ async function handleOperation(
2141
2539
  }
2142
2540
  }
2143
2541
  };
2542
+ const streamLifecycle: RequestStreamLifecycle = {
2543
+ runStep<T>(fn: () => Promise<T>): Promise<T> {
2544
+ return scope.runStreamStep(fn);
2545
+ },
2546
+ registerCleanup(streamCleanup): void {
2547
+ scope.registerStreamCleanup(streamCleanup);
2548
+ },
2549
+ terminalize(outcome): void {
2550
+ scope.terminalize(outcome);
2551
+ },
2552
+ cleanup(): Promise<void> {
2553
+ return scope.cleanupStream();
2554
+ },
2555
+ };
2556
+ scope.registerStreamCleanup(() => scope.runStreamStep(cleanup));
2144
2557
  try {
2145
2558
  const result = options.operationExecutor
2146
2559
  ? await options.operationExecutor({
@@ -2154,7 +2567,7 @@ async function handleOperation(
2154
2567
  env: createEnvContext(providerSecretNames(provider)),
2155
2568
  });
2156
2569
  if (streaming && operation) {
2157
- return toStreamingResponse(operation, result, cleanup, request.requestId);
2570
+ return toStreamingResponse(operation, result, streamLifecycle, request.requestId);
2158
2571
  }
2159
2572
  return toJsonSuccessResponse(result, ctx);
2160
2573
  } catch (error) {
@@ -2165,21 +2578,6 @@ async function handleOperation(
2165
2578
  }
2166
2579
  }
2167
2580
 
2168
- function responseWithProviderTelemetry(
2169
- response: Response,
2170
- proxyTelemetry?: ProxyTelemetryCollector,
2171
- ): Response {
2172
- const headerValue = proxyTelemetry?.toHeaderValue();
2173
- const headers = new Headers(response.headers);
2174
- headers.delete(PROVIDER_TELEMETRY_HEADER);
2175
- if (headerValue) headers.set(PROVIDER_TELEMETRY_HEADER, headerValue);
2176
- return new Response(response.body, {
2177
- headers,
2178
- status: response.status,
2179
- statusText: response.statusText,
2180
- });
2181
- }
2182
-
2183
2581
  type AuthRoute = "start" | "continue" | "poll" | "abort" | "refresh";
2184
2582
 
2185
2583
  async function handleAuthFlow(
@@ -2188,7 +2586,7 @@ async function handleAuthFlow(
2188
2586
  route: AuthRoute,
2189
2587
  options: ProviderServerRuntimeOptions,
2190
2588
  state: ProviderRuntimeState,
2191
- proxyTelemetry?: ProxyTelemetryCollector,
2589
+ scope: RequestScopeContext,
2192
2590
  signal?: AbortSignal,
2193
2591
  ): Promise<Response | AuthFlowResponse> {
2194
2592
  const flow = provider.auth?.flow;
@@ -2208,7 +2606,7 @@ async function handleAuthFlow(
2208
2606
  request,
2209
2607
  options,
2210
2608
  state,
2211
- proxyTelemetry,
2609
+ scope,
2212
2610
  signal,
2213
2611
  );
2214
2612
  try {
@@ -2497,7 +2895,15 @@ function createServerAppWithCapabilityModules(
2497
2895
  let rawBody: unknown;
2498
2896
  let operationId: string | undefined;
2499
2897
  const operation = "stateful-internal";
2500
- const requestCost = startRequestCost();
2898
+ const requestScope = createRequestScope({
2899
+ provider,
2900
+ kind: "operation",
2901
+ route: operation,
2902
+ headers: Object.fromEntries(c.req.raw.headers.entries()),
2903
+ logger,
2904
+ setHeader: (name, value) => c.header(name, value),
2905
+ declaredErrorCode: (error) => declaredErrorCodeFor(error, operationId, operationErrorCodes),
2906
+ });
2501
2907
  try {
2502
2908
  if (!options.internalOperationExecutor) {
2503
2909
  throw new ProviderError("Stateful internal operation executor is not configured.", {
@@ -2607,36 +3013,40 @@ function createServerAppWithCapabilityModules(
2607
3013
  }
2608
3014
  const request = operationRequestFromForwardingEnvelope(envelope);
2609
3015
  operationId = envelope.operationId;
2610
- const ctx = createProviderContext(
2611
- provider,
2612
- request,
2613
- operationId,
2614
- options,
2615
- state,
2616
- undefined,
2617
- signal,
2618
- );
2619
- if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
2620
- throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
2621
- }
2622
- const output = await options.internalOperationExecutor({
2623
- provider,
3016
+ requestScope.enrich({
3017
+ route: operationId,
3018
+ requestId: request.requestId,
2624
3019
  operationId,
2625
- ctx,
2626
- request,
2627
- internalStatefulForward: envelope,
2628
- signal,
3020
+ headers: request.headers ?? {},
3021
+ correlation: { connectionId: envelope.connectionId },
3022
+ });
3023
+ const response = await requestScope.run(async () => {
3024
+ const ctx = createProviderContext(
3025
+ provider,
3026
+ request,
3027
+ operationId as string,
3028
+ options,
3029
+ state,
3030
+ requestScope as RequestScope,
3031
+ signal,
3032
+ );
3033
+ if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
3034
+ throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
3035
+ }
3036
+ const output = await options.internalOperationExecutor!({
3037
+ provider,
3038
+ operationId: operationId as string,
3039
+ ctx,
3040
+ request,
3041
+ internalStatefulForward: envelope,
3042
+ signal,
3043
+ });
3044
+ return c.json({ data: output });
3045
+ });
3046
+ return finalizeRequestResponse(requestScope, response, {
3047
+ kind: "completed",
3048
+ status: 200,
2629
3049
  });
2630
- logProviderSuccess(
2631
- logger,
2632
- provider,
2633
- "operation",
2634
- operationId || operation,
2635
- request.requestId,
2636
- 200,
2637
- finishRequestCost(requestCost),
2638
- );
2639
- return c.json({ data: output });
2640
3050
  } catch (error) {
2641
3051
  const declaredErrorCode = declaredErrorCodeFor(error, operationId, operationErrorCodes);
2642
3052
  const status = toStatusCode(error, declaredErrorCode);
@@ -2647,32 +3057,33 @@ function createServerAppWithCapabilityModules(
2647
3057
  c.header("Retry-After", String(STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS));
2648
3058
  }
2649
3059
  const requestId = extractRequestId(rawBody);
3060
+ requestScope.enrich({
3061
+ ...(operationId ? { route: operationId, operationId } : {}),
3062
+ ...(requestId ? { requestId } : {}),
3063
+ });
2650
3064
  const observabilityDetails = errorObservabilityDetails(error, declaredErrorCode);
2651
- logProviderError(
2652
- logger,
2653
- provider,
2654
- "operation",
2655
- operationId || operation,
2656
- requestId,
2657
- error,
3065
+ const response = c.json(toErrorResponse(error, requestId, observabilityDetails), status);
3066
+ return finalizeRequestResponse(requestScope, response, {
3067
+ kind: "failed",
2658
3068
  status,
2659
- finishRequestCost(requestCost),
2660
- declaredErrorCode,
2661
- undefined,
2662
- observabilityDetails,
2663
- );
2664
- return responseWithErrorObservability(
2665
- c.json(toErrorResponse(error, requestId, observabilityDetails), status),
2666
- observabilityDetails,
2667
- );
3069
+ error,
3070
+ });
2668
3071
  }
2669
3072
  });
2670
3073
 
2671
3074
  app.post("/v1/:operation", async (c) => {
2672
3075
  let rawBody: unknown;
2673
3076
  const operation = c.req.param("operation");
2674
- const proxyTelemetry = new ProxyTelemetryCollector();
2675
- const requestCost = startRequestCost();
3077
+ const requestScope = createRequestScope({
3078
+ provider,
3079
+ kind: "operation",
3080
+ route: operation,
3081
+ operationId: operation,
3082
+ headers: Object.fromEntries(c.req.raw.headers.entries()),
3083
+ logger,
3084
+ setHeader: (name, value) => c.header(name, value),
3085
+ declaredErrorCode: (error) => declaredErrorCodeFor(error, operation, operationErrorCodes),
3086
+ });
2676
3087
  try {
2677
3088
  rawBody = await c.req.raw
2678
3089
  .clone()
@@ -2681,367 +3092,129 @@ function createServerAppWithCapabilityModules(
2681
3092
  const body = OperationRequestSchema.parse(rawBody);
2682
3093
  const requestHeaders = Object.fromEntries(c.req.raw.headers.entries());
2683
3094
  body.headers = { ...requestHeaders, ...body.headers };
2684
- const response = await handleOperation(
2685
- provider,
2686
- body,
2687
- operation,
2688
- options,
2689
- state,
2690
- proxyTelemetry,
2691
- c.req.raw.signal,
2692
- );
2693
- if (response instanceof Response) {
2694
- logProviderSuccess(
2695
- logger,
3095
+ requestScope.enrich({
3096
+ requestId: body.requestId,
3097
+ headers: body.headers,
3098
+ correlation: { connectionId: resolveOperationConnectionId(body) },
3099
+ });
3100
+ const streaming = provider.operations[operation]?.transport?.kind
3101
+ ? provider.operations[operation]?.transport?.kind !== "json"
3102
+ : false;
3103
+ if (streaming) requestScope.watchAbort(c.req.raw.signal);
3104
+ const execute = async () => {
3105
+ const handled = await handleOperation(
2696
3106
  provider,
2697
- "operation",
3107
+ body,
2698
3108
  operation,
2699
- body.requestId,
2700
- response.status,
2701
- finishRequestCost(requestCost),
2702
- proxyTelemetry,
3109
+ options,
3110
+ state,
3111
+ requestScope,
3112
+ c.req.raw.signal,
2703
3113
  );
2704
- return responseWithProviderTelemetry(response, proxyTelemetry);
2705
- }
2706
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2707
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2708
- logProviderSuccess(
2709
- logger,
2710
- provider,
2711
- "operation",
2712
- operation,
2713
- body.requestId,
2714
- 200,
2715
- finishRequestCost(requestCost),
2716
- proxyTelemetry,
2717
- );
2718
- return c.json(response);
3114
+ const response = handled instanceof Response ? handled : c.json(handled);
3115
+ return streaming
3116
+ ? responseWithRequestScopeHeaders(response, requestScope.snapshotHeaders())
3117
+ : response;
3118
+ };
3119
+ const response = streaming
3120
+ ? await requestScope.runStreaming(execute)
3121
+ : await requestScope.run(execute);
3122
+ if (streaming) return response;
3123
+ return finalizeRequestResponse(requestScope, response, {
3124
+ kind: "completed",
3125
+ status: response.status,
3126
+ });
2719
3127
  } catch (error) {
2720
3128
  const declaredErrorCode = declaredErrorCodeFor(error, operation, operationErrorCodes);
2721
3129
  const status = toStatusCode(error, declaredErrorCode);
2722
3130
  const requestId = extractRequestId(rawBody);
3131
+ requestScope.enrich({ ...(requestId ? { requestId } : {}) });
2723
3132
  const observabilityDetails = errorObservabilityDetails(error, declaredErrorCode);
2724
- logProviderError(
2725
- logger,
2726
- provider,
2727
- "operation",
2728
- operation,
2729
- requestId,
2730
- error,
2731
- status,
2732
- finishRequestCost(requestCost),
2733
- declaredErrorCode,
2734
- proxyTelemetry,
2735
- observabilityDetails,
2736
- );
2737
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2738
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2739
- return responseWithErrorObservability(
2740
- c.json(toErrorResponse(error, requestId, observabilityDetails), status),
2741
- observabilityDetails,
2742
- );
2743
- }
2744
- });
2745
-
2746
- app.post("/auth/start", async (c) => {
2747
- let rawBody: unknown;
2748
- const proxyTelemetry = new ProxyTelemetryCollector();
2749
- const requestCost = startRequestCost();
2750
- try {
2751
- rawBody = await c.req.raw
2752
- .clone()
2753
- .json()
2754
- .catch(() => undefined);
2755
- const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
2756
- const response = await handleAuthFlow(
2757
- provider,
2758
- body,
2759
- "start",
2760
- options,
2761
- state,
2762
- proxyTelemetry,
2763
- c.req.raw.signal,
2764
- );
2765
- logProviderSuccess(
2766
- logger,
2767
- provider,
2768
- "auth",
2769
- "start",
2770
- body.requestId,
2771
- response instanceof Response ? response.status : 200,
2772
- finishRequestCost(requestCost),
2773
- proxyTelemetry,
2774
- );
2775
- if (response instanceof Response)
2776
- return responseWithProviderTelemetry(response, proxyTelemetry);
2777
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2778
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2779
- return c.json(response);
2780
- } catch (error) {
2781
- const status = toStatusCode(error);
2782
- const requestId = extractRequestId(rawBody);
2783
- const observabilityDetails = errorObservabilityDetails(error);
2784
- logProviderError(
2785
- logger,
2786
- provider,
2787
- "auth",
2788
- "start",
2789
- requestId,
2790
- error,
3133
+ const response = c.json(toErrorResponse(error, requestId, observabilityDetails), status);
3134
+ return finalizeRequestResponse(requestScope, response, {
3135
+ kind: "failed",
2791
3136
  status,
2792
- finishRequestCost(requestCost),
2793
- undefined,
2794
- proxyTelemetry,
2795
- observabilityDetails,
2796
- );
2797
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2798
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2799
- return responseWithErrorObservability(
2800
- c.json(toErrorResponse(error, requestId, observabilityDetails), status),
2801
- observabilityDetails,
2802
- );
2803
- }
2804
- });
2805
-
2806
- app.post("/auth/continue", async (c) => {
2807
- let rawBody: unknown;
2808
- const proxyTelemetry = new ProxyTelemetryCollector();
2809
- const requestCost = startRequestCost();
2810
- try {
2811
- rawBody = await c.req.raw
2812
- .clone()
2813
- .json()
2814
- .catch(() => undefined);
2815
- const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
2816
- const response = await handleAuthFlow(
2817
- provider,
2818
- body,
2819
- "continue",
2820
- options,
2821
- state,
2822
- proxyTelemetry,
2823
- c.req.raw.signal,
2824
- );
2825
- logProviderSuccess(
2826
- logger,
2827
- provider,
2828
- "auth",
2829
- "continue",
2830
- body.requestId,
2831
- response instanceof Response ? response.status : 200,
2832
- finishRequestCost(requestCost),
2833
- proxyTelemetry,
2834
- );
2835
- if (response instanceof Response)
2836
- return responseWithProviderTelemetry(response, proxyTelemetry);
2837
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2838
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2839
- return c.json(response);
2840
- } catch (error) {
2841
- const status = toStatusCode(error);
2842
- const requestId = extractRequestId(rawBody);
2843
- const observabilityDetails = errorObservabilityDetails(error);
2844
- logProviderError(
2845
- logger,
2846
- provider,
2847
- "auth",
2848
- "continue",
2849
- requestId,
2850
3137
  error,
2851
- status,
2852
- finishRequestCost(requestCost),
2853
- undefined,
2854
- proxyTelemetry,
2855
- observabilityDetails,
2856
- );
2857
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2858
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2859
- return responseWithErrorObservability(
2860
- c.json(toErrorResponse(error, requestId, observabilityDetails), status),
2861
- observabilityDetails,
2862
- );
2863
- }
2864
- });
2865
-
2866
- app.post("/auth/poll", async (c) => {
2867
- let rawBody: unknown;
2868
- const proxyTelemetry = new ProxyTelemetryCollector();
2869
- const requestCost = startRequestCost();
2870
- try {
2871
- rawBody = await c.req.raw
2872
- .clone()
2873
- .json()
2874
- .catch(() => undefined);
2875
- const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
2876
- const response = await handleAuthFlow(
2877
- provider,
2878
- body,
2879
- "poll",
2880
- options,
2881
- state,
2882
- proxyTelemetry,
2883
- c.req.raw.signal,
2884
- );
2885
- logProviderSuccess(
2886
- logger,
2887
- provider,
2888
- "auth",
2889
- "poll",
2890
- body.requestId,
2891
- response instanceof Response ? response.status : 200,
2892
- finishRequestCost(requestCost),
2893
- proxyTelemetry,
2894
- );
2895
- if (response instanceof Response)
2896
- return responseWithProviderTelemetry(response, proxyTelemetry);
2897
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2898
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2899
- return c.json(response);
2900
- } catch (error) {
2901
- const status = toStatusCode(error);
2902
- const requestId = extractRequestId(rawBody);
2903
- const observabilityDetails = errorObservabilityDetails(error);
2904
- logProviderError(
2905
- logger,
2906
- provider,
2907
- "auth",
2908
- "poll",
2909
- requestId,
2910
- error,
2911
- status,
2912
- finishRequestCost(requestCost),
2913
- undefined,
2914
- proxyTelemetry,
2915
- observabilityDetails,
2916
- );
2917
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2918
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2919
- return responseWithErrorObservability(
2920
- c.json(toErrorResponse(error, requestId, observabilityDetails), status),
2921
- observabilityDetails,
2922
- );
2923
- }
2924
- });
2925
-
2926
- app.post("/auth/refresh", async (c) => {
2927
- let rawBody: unknown;
2928
- const proxyTelemetry = new ProxyTelemetryCollector();
2929
- const requestCost = startRequestCost();
2930
- try {
2931
- rawBody = await c.req.raw
2932
- .clone()
2933
- .json()
2934
- .catch(() => undefined);
2935
- const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
2936
- const response = await handleAuthFlow(
2937
- provider,
2938
- body,
2939
- "refresh",
2940
- options,
2941
- state,
2942
- proxyTelemetry,
2943
- c.req.raw.signal,
2944
- );
2945
- logProviderSuccess(
2946
- logger,
2947
- provider,
2948
- "auth",
2949
- "refresh",
2950
- body.requestId,
2951
- response instanceof Response ? response.status : 200,
2952
- finishRequestCost(requestCost),
2953
- proxyTelemetry,
2954
- );
2955
- if (response instanceof Response)
2956
- return responseWithProviderTelemetry(response, proxyTelemetry);
2957
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2958
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2959
- return c.json(response);
2960
- } catch (error) {
2961
- const status = toStatusCode(error);
2962
- const requestId = extractRequestId(rawBody);
2963
- const observabilityDetails = errorObservabilityDetails(error);
2964
- logProviderError(
2965
- logger,
2966
- provider,
2967
- "auth",
2968
- "refresh",
2969
- requestId,
2970
- error,
2971
- status,
2972
- finishRequestCost(requestCost),
2973
- undefined,
2974
- proxyTelemetry,
2975
- observabilityDetails,
2976
- );
2977
- const telemetryHeader = proxyTelemetry.toHeaderValue();
2978
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
2979
- return responseWithErrorObservability(
2980
- c.json(toErrorResponse(error, requestId, observabilityDetails), status),
2981
- observabilityDetails,
2982
- );
3138
+ });
2983
3139
  }
2984
3140
  });
2985
3141
 
2986
- app.post("/auth/disconnect", async (c) => {
2987
- let rawBody: unknown;
2988
- const proxyTelemetry = new ProxyTelemetryCollector();
2989
- const requestCost = startRequestCost();
2990
- try {
2991
- rawBody = await c.req.raw
2992
- .clone()
2993
- .json()
2994
- .catch(() => undefined);
2995
- const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
2996
- const response = await handleAuthFlow(
2997
- provider,
2998
- body,
2999
- "abort",
3000
- options,
3001
- state,
3002
- proxyTelemetry,
3003
- c.req.raw.signal,
3004
- );
3005
- logProviderSuccess(
3006
- logger,
3142
+ const authRoutes = [
3143
+ { path: "/auth/start", flowRoute: "start", logRoute: "start" },
3144
+ { path: "/auth/continue", flowRoute: "continue", logRoute: "continue" },
3145
+ { path: "/auth/poll", flowRoute: "poll", logRoute: "poll" },
3146
+ { path: "/auth/refresh", flowRoute: "refresh", logRoute: "refresh" },
3147
+ { path: "/auth/disconnect", flowRoute: "abort", logRoute: "disconnect" },
3148
+ ] as const satisfies ReadonlyArray<{
3149
+ path: string;
3150
+ flowRoute: AuthRoute;
3151
+ logRoute: string;
3152
+ }>;
3153
+
3154
+ for (const { path, flowRoute, logRoute } of authRoutes) {
3155
+ app.post(path, async (c) => {
3156
+ let rawBody: unknown;
3157
+ const requestScope = createRequestScope({
3007
3158
  provider,
3008
- "auth",
3009
- "disconnect",
3010
- body.requestId,
3011
- response instanceof Response ? response.status : 200,
3012
- finishRequestCost(requestCost),
3013
- proxyTelemetry,
3014
- );
3015
- if (response instanceof Response)
3016
- return responseWithProviderTelemetry(response, proxyTelemetry);
3017
- const telemetryHeader = proxyTelemetry.toHeaderValue();
3018
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
3019
- return c.json(response);
3020
- } catch (error) {
3021
- const status = toStatusCode(error);
3022
- const requestId = extractRequestId(rawBody);
3023
- const observabilityDetails = errorObservabilityDetails(error);
3024
- logProviderError(
3159
+ kind: "auth",
3160
+ route: logRoute,
3161
+ headers: Object.fromEntries(c.req.raw.headers.entries()),
3025
3162
  logger,
3026
- provider,
3027
- "auth",
3028
- "disconnect",
3029
- requestId,
3030
- error,
3031
- status,
3032
- finishRequestCost(requestCost),
3033
- undefined,
3034
- proxyTelemetry,
3035
- observabilityDetails,
3036
- );
3037
- const telemetryHeader = proxyTelemetry.toHeaderValue();
3038
- if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
3039
- return responseWithErrorObservability(
3040
- c.json(toErrorResponse(error, requestId, observabilityDetails), status),
3041
- observabilityDetails,
3042
- );
3043
- }
3044
- });
3163
+ setHeader: (name, value) => c.header(name, value),
3164
+ });
3165
+ try {
3166
+ rawBody = await c.req.raw
3167
+ .clone()
3168
+ .json()
3169
+ .catch(() => undefined);
3170
+ const body = withAuthRequestHeaders(
3171
+ AuthFlowRequestSchema.parse(rawBody),
3172
+ c.req.raw.headers,
3173
+ );
3174
+ requestScope.enrich({
3175
+ requestId: body.requestId,
3176
+ flowId: body.flowId,
3177
+ headers: body.headers ?? {},
3178
+ correlation: {
3179
+ connectionId: resolveOperationConnectionId(body),
3180
+ flowId: body.flowId,
3181
+ tenantId: body.tenantId,
3182
+ requestedProviderId:
3183
+ body.providerId !== undefined && body.providerId !== provider.id
3184
+ ? body.providerId
3185
+ : undefined,
3186
+ },
3187
+ });
3188
+ const response = await requestScope.run(async () => {
3189
+ const handled = await handleAuthFlow(
3190
+ provider,
3191
+ body,
3192
+ flowRoute,
3193
+ options,
3194
+ state,
3195
+ requestScope as RequestScope,
3196
+ c.req.raw.signal,
3197
+ );
3198
+ return handled instanceof Response ? handled : c.json(handled);
3199
+ });
3200
+ return finalizeRequestResponse(requestScope, response, {
3201
+ kind: "completed",
3202
+ status: response.status,
3203
+ });
3204
+ } catch (error) {
3205
+ const status = toStatusCode(error);
3206
+ const requestId = extractRequestId(rawBody);
3207
+ requestScope.enrich({ ...(requestId ? { requestId } : {}) });
3208
+ const observabilityDetails = errorObservabilityDetails(error);
3209
+ const response = c.json(toErrorResponse(error, requestId, observabilityDetails), status);
3210
+ return finalizeRequestResponse(requestScope, response, {
3211
+ kind: "failed",
3212
+ status,
3213
+ error,
3214
+ });
3215
+ }
3216
+ });
3217
+ }
3045
3218
 
3046
3219
  return app;
3047
3220
  }