@heystack/otel 0.18.0 → 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/workers.js CHANGED
@@ -7,7 +7,7 @@
7
7
  // `@opentelemetry/resources` and the platform `fetch` — all WinterCG-safe — and
8
8
  // ships its own OTLP/JSON-over-fetch span exporter so it runs on Workers/Edge
9
9
  // where the Node SDK cannot.
10
- import { context, createContextKey, INVALID_SPAN_CONTEXT, trace, SpanKind, SpanStatusCode, } from "@opentelemetry/api";
10
+ import { context, createContextKey, trace, SpanKind, SpanStatusCode, } from "@opentelemetry/api";
11
11
  import { suppressTracing, isTracingSuppressed } from "@opentelemetry/core";
12
12
  import { ROOT_CONTEXT } from "@opentelemetry/api";
13
13
  import { resourceFromAttributes } from "@opentelemetry/resources";
@@ -16,9 +16,14 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
16
16
  import { buildExporterConfig, DEFAULT_ENDPOINT, releaseResourceAttributes, } from "./core.js";
17
17
  import { isSelfSpanAttrs, safeHostname } from "./self-span.js";
18
18
  import { llmRequestAttrs, llmResponseAttrs, llmContentAttrs, llmOperationName, resolveLLMProvider, scanSseStreamForUsage, } from "./llm-enrich.js";
19
- import { instrumentEnv, makeProxy, runWithSpan } from "./workers-bindings.js";
19
+ import { instrumentEnv, keysOf, makeProxy, runWithSpan, runWithSpanKeyed, runWithSpanPreservingReturn, settlePendingKeyHashes, wrapSqlClient, } from "./workers-bindings.js";
20
+ import { NATIVELY_COVERED, NATIVE_PROPAGATES_EXTERNAL, shouldEnrichNatively } from "./workers-coverage.js";
20
21
  import { makeSampler, loadRemoteSamplingRate } from "./workers-sampler.js";
22
+ import { createNativePropagation, injectOutboundHeaders, isAiGatewayHost, NATIVE_PROPAGATION_CONTEXT_KEY, propagationTargetAllowed, propagationTraceparent, } from "./workers-propagation.js";
23
+ import { resolveRoute } from "./workers-routes.js";
21
24
  import { makeTraceparent } from "./traceparent.js";
25
+ import { NATIVE_TRACING_CONTEXT_KEY, NativeOtelSpan, getNativeTracing, nativeTracingFromContext, runNativeSpan, setNativeContextInit, } from "./workers-native.js";
26
+ import { applySpanPrivacy, makeUrlRedactor, } from "./redact.js";
22
27
  // `ExportResult` / `ExportResultCode` mirror `@opentelemetry/core`. We define
23
28
  // them inline (structurally identical) rather than import them: core is only a
24
29
  // transitive dep of sdk-trace-base and isn't reliably resolvable, and keeping it
@@ -109,8 +114,9 @@ function toStatus(status) {
109
114
  out.message = status.message;
110
115
  return out;
111
116
  }
112
- function readableSpanToOtlp(span) {
117
+ function readableSpanToOtlp(span, transform) {
113
118
  const ctx = span.spanContext();
119
+ const attrs = span.attributes;
114
120
  return {
115
121
  traceId: ctx.traceId,
116
122
  spanId: ctx.spanId,
@@ -121,7 +127,7 @@ function readableSpanToOtlp(span) {
121
127
  kind: toOtlpKind(span.kind),
122
128
  startTimeUnixNano: hrTimeToUnixNano(span.startTime),
123
129
  endTimeUnixNano: hrTimeToUnixNano(span.endTime),
124
- attributes: toKeyValues(span.attributes),
130
+ attributes: toKeyValues(transform ? transform(attrs, span) : attrs),
125
131
  events: span.events.map((ev) => ({
126
132
  timeUnixNano: hrTimeToUnixNano(ev.time),
127
133
  name: ev.name,
@@ -136,7 +142,7 @@ function readableSpanToOtlp(span) {
136
142
  * All spans in a SimpleSpanProcessor batch share the same per-request provider,
137
143
  * hence the same resource, so we emit a single resourceSpans entry.
138
144
  */
139
- export function serializeSpans(spans) {
145
+ export function serializeSpans(spans, transform) {
140
146
  const resourceAttrs = spans[0]
141
147
  ? spans[0].resource.attributes
142
148
  : {};
@@ -144,7 +150,7 @@ export function serializeSpans(spans) {
144
150
  resourceSpans: [
145
151
  {
146
152
  resource: { attributes: toKeyValues(resourceAttrs) },
147
- scopeSpans: [{ spans: spans.map(readableSpanToOtlp) }],
153
+ scopeSpans: [{ spans: spans.map((s) => readableSpanToOtlp(s, transform)) }],
148
154
  },
149
155
  ],
150
156
  };
@@ -228,6 +234,8 @@ export function isSelfSpanForTest(attrs, ingestHost) {
228
234
  // self-span guardrail satisfied and the path WinterCG-safe (pure string logic).
229
235
  // ---------------------------------------------------------------------------
230
236
  let _fetchInstrumented = false;
237
+ /** URL redactor for outbound CLIENT spans (set from config by ensureGlobalProvider). */
238
+ let _fetchRedact = makeUrlRedactor();
231
239
  let _originalFetch;
232
240
  let _fetchWrapper;
233
241
  /** Resolve the absolute URL of an outbound fetch arg (string | URL | Request). */
@@ -270,12 +278,6 @@ function injectTraceparent(input, init, traceparent) {
270
278
  }
271
279
  /** Max response body size to clone for gen_ai enrichment (512 KB). */
272
280
  const LLM_RESPONSE_SIZE_CAP = 512 * 1024;
273
- /**
274
- * Context key under which `runNativeSpan` / `withSpan` publish the request's
275
- * Cloudflare native tracer, so nested helpers (and the fetch wrapper) can join
276
- * the platform trace tree instead of creating OTel-provider spans.
277
- */
278
- const NATIVE_TRACING_CONTEXT_KEY = createContextKey("heystack.cloudflare.native-tracing");
279
281
  /**
280
282
  * Latest ingest host / ai config seen by `ensureFetchInstrumentation`. The
281
283
  * wrapper is installed ONCE per isolate but `instrument()` may reach it first
@@ -285,6 +287,37 @@ const NATIVE_TRACING_CONTEXT_KEY = createContextKey("heystack.cloudflare.native-
285
287
  */
286
288
  let _fetchIngestHost = "";
287
289
  let _fetchAiCfg = {};
290
+ let _fetchPropagateTargets;
291
+ /** True when the caller already supplied header `name` on this fetch. */
292
+ function hasHeader(input, init, name) {
293
+ try {
294
+ return new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)).has(name);
295
+ }
296
+ catch {
297
+ return false;
298
+ }
299
+ }
300
+ /** Bare ingest hostname for native mode (no exporter exists to derive it from). */
301
+ function nativeIngestHost(config) {
302
+ return safeHostname(config.endpoint ?? DEFAULT_ENDPOINT);
303
+ }
304
+ /**
305
+ * Native root helper: record the inbound parent (only when it validates) and
306
+ * the invocation's synthetic propagation pair as link attributes, then run `fn`
307
+ * with the pair published in OTel context so outbound fetches can use it.
308
+ * These are SDK-minted ids, NOT Cloudflare's native ids (which don't exist).
309
+ */
310
+ function withNativePropagation(span, inboundTraceparent, fn) {
311
+ const prop = createNativePropagation();
312
+ const parent = parseTraceparent(inboundTraceparent);
313
+ if (parent) {
314
+ span.setAttribute("heystack.parent.trace_id", parent.traceId);
315
+ span.setAttribute("heystack.parent.span_id", parent.spanId);
316
+ }
317
+ span.setAttribute("heystack.propagation.trace_id", prop.traceId);
318
+ span.setAttribute("heystack.propagation.span_id", prop.spanId);
319
+ return context.with(context.active().setValue(NATIVE_PROPAGATION_CONTEXT_KEY, prop), fn);
320
+ }
288
321
  /** A promise whose settle handles are exposed. */
289
322
  function deferred() {
290
323
  let resolve;
@@ -395,7 +428,7 @@ async function enrichLLMResponse(span, provider, response, parsedReqBody, aiCfg)
395
428
  * deferred — that is what lets usage tokens land on the span without buffering
396
429
  * or delaying the stream. No OTel provider/exporter is involved.
397
430
  */
398
- function nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch, aiCfg) {
431
+ function nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch, aiCfg, delegated = false) {
399
432
  const ready = deferred();
400
433
  const name = `${llmOperationName(url.pathname)} ${resolved.provider}`;
401
434
  let run;
@@ -404,7 +437,7 @@ function nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch
404
437
  run = nativeTracing.enterSpan(name, async (nativeSpan) => {
405
438
  invoked = true;
406
439
  // Unsampled invocation: skip all enrichment work (no tee, no clone).
407
- if (!nativeSpan.isTraced) {
440
+ if (nativeSpan.isTraced === false) {
408
441
  try {
409
442
  ready.resolve(await originalFetch(input, init));
410
443
  }
@@ -413,9 +446,11 @@ function nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch
413
446
  }
414
447
  return;
415
448
  }
416
- const adapter = new NativeSpanAdapter(nativeSpan);
449
+ const adapter = new NativeOtelSpan(nativeSpan);
417
450
  const { attrs, parsedReqBody } = llmRequestSide(resolved, url, init);
418
451
  adapter.setAttributes({ ...attrs, "server.address": url.hostname });
452
+ if (delegated)
453
+ adapter.setAttribute("heystack.ai.delegated", true);
419
454
  try {
420
455
  const response = await originalFetch(input, init);
421
456
  adapter.setAttribute("http.response.status_code", response.status);
@@ -464,10 +499,11 @@ function nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch
464
499
  * exporter's own uploads are never traced; native callers pass "" (no exporter).
465
500
  * `aiCfg` configures gen_ai enrichment (content capture is opt-in).
466
501
  */
467
- function ensureFetchInstrumentation(ingestHost, aiCfg = {}) {
502
+ function ensureFetchInstrumentation(ingestHost, aiCfg = {}, propagateTargets) {
468
503
  if (ingestHost)
469
504
  _fetchIngestHost = ingestHost;
470
505
  _fetchAiCfg = aiCfg;
506
+ _fetchPropagateTargets = propagateTargets;
471
507
  if (_fetchInstrumented)
472
508
  return;
473
509
  _fetchInstrumented = true;
@@ -487,10 +523,27 @@ function ensureFetchInstrumentation(ingestHost, aiCfg = {}) {
487
523
  const nativeTracing = active.getValue(NATIVE_TRACING_CONTEXT_KEY);
488
524
  if (nativeTracing) {
489
525
  const url = parseTarget(input);
490
- const resolved = url ? resolveLLMProvider(url, _fetchAiCfg.gateways) : null;
491
- if (!url || !resolved)
526
+ if (!url)
492
527
  return originalFetch(input, init);
493
- return nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch, _fetchAiCfg);
528
+ // Link headers: the invocation's synthetic ids, never to the Heystack
529
+ // ingest host and only to hosts allowed by `propagateTargets`.
530
+ let reqInput = input;
531
+ let reqInit = init;
532
+ const prop = active.getValue(NATIVE_PROPAGATION_CONTEXT_KEY);
533
+ const host = url.hostname.toLowerCase();
534
+ const gateway = isAiGatewayHost(url);
535
+ let injected = false;
536
+ if (prop && host !== _fetchIngestHost && propagationTargetAllowed(host, _fetchPropagateTargets)) {
537
+ injected = gateway && !hasHeader(input, init, "cf-aig-otel-trace-id");
538
+ [reqInput, reqInit] = injectOutboundHeaders(input, init, {
539
+ traceparent: propagationTraceparent(prop),
540
+ aiGateway: gateway ? { traceId: prop.traceId, parentSpanId: prop.spanId } : undefined,
541
+ });
542
+ }
543
+ const resolved = resolveLLMProvider(url, _fetchAiCfg.gateways);
544
+ if (!resolved)
545
+ return originalFetch(reqInput, reqInit);
546
+ return nativeLLMFetch(nativeTracing, resolved, url, reqInput, reqInit, originalFetch, _fetchAiCfg, injected && _fetchAiCfg.gatewayExport === true);
494
547
  }
495
548
  // Direct plane: nothing to parent to (a fetch outside a traced request).
496
549
  if (!trace.getSpan(active)) {
@@ -507,13 +560,27 @@ function ensureFetchInstrumentation(ingestHost, aiCfg = {}) {
507
560
  kind: SpanKind.CLIENT,
508
561
  attributes: {
509
562
  "http.request.method": method,
510
- "url.full": target,
563
+ // Redacted: userinfo stripped + sensitive query/fragment values
564
+ // (e.g. Gemini `?key=`, Supabase `?apikey=`) replaced with REDACTED.
565
+ "url.full": _fetchRedact(target),
511
566
  "server.address": host,
512
567
  },
513
568
  }, active);
514
569
  const sc = span.spanContext();
515
570
  const traceparent = makeTraceparent(sc);
516
- const [reqInput, reqInit] = injectTraceparent(input, init, traceparent);
571
+ let [reqInput, reqInit] = injectTraceparent(input, init, traceparent);
572
+ const directUrl = parseTarget(input);
573
+ const directGateway = directUrl !== null && isAiGatewayHost(directUrl);
574
+ if (directGateway) {
575
+ const ours = !hasHeader(input, init, "cf-aig-otel-trace-id");
576
+ // The span's REAL trace id + this CLIENT span id; existing headers win.
577
+ [reqInput, reqInit] = injectOutboundHeaders(reqInput, reqInit, {
578
+ traceparent,
579
+ aiGateway: { traceId: sc.traceId, parentSpanId: sc.spanId },
580
+ });
581
+ if (ours && _fetchAiCfg.gatewayExport === true)
582
+ span.setAttribute("heystack.ai.delegated", true);
583
+ }
517
584
  // gen_ai enrichment — request side (direct host or configured gateway).
518
585
  const url = parseTarget(input);
519
586
  const resolved = url ? resolveLLMProvider(url, _fetchAiCfg.gateways) : null;
@@ -568,6 +635,7 @@ export function __resetFetchInstrumentation() {
568
635
  _fetchWrapper = undefined;
569
636
  _fetchIngestHost = "";
570
637
  _fetchAiCfg = {};
638
+ _fetchRedact = makeUrlRedactor();
571
639
  }
572
640
  // ---------------------------------------------------------------------------
573
641
  // Exporter
@@ -630,6 +698,7 @@ function markManualFlushObserved() {
630
698
  * completes, letting the isolate be torn down and silently dropping the trace.
631
699
  */
632
700
  async function drainProvider(provider) {
701
+ await settlePendingKeyHashes();
633
702
  await provider.forceFlush().catch(() => { });
634
703
  await provider.heystackExporter.forceFlush().catch(() => { });
635
704
  }
@@ -681,8 +750,18 @@ export class HeystackSpanExporter {
681
750
  * the `pending` set + manual `flushHeystack()` when absent/unavailable.
682
751
  */
683
752
  waitUntil;
753
+ /**
754
+ * Privacy pass on every exported span — redacts URL attributes (`url.full`,
755
+ * `http.url`, `http.target`, `url.query`, …) so spans from host frameworks
756
+ * (Next/OpenNext, manual spans) are covered too, not just Heystack's own.
757
+ */
758
+ privacy;
684
759
  constructor(options) {
685
760
  const cfg = buildExporterConfig(options);
761
+ const redact = makeUrlRedactor(options);
762
+ // Client IP is gated at its source (instrument()'s SERVER span), so the
763
+ // exporter only redacts URLs and leaves deliberate manual attributes alone.
764
+ this.privacy = (attrs) => applySpanPrivacy(attrs, { redact, captureClientIp: true });
686
765
  this.url = cfg.url;
687
766
  this.ingestHost = safeHostname(cfg.url);
688
767
  this.headers = {
@@ -710,7 +789,7 @@ export class HeystackSpanExporter {
710
789
  resultCallback({ code: ExportResultCode.SUCCESS });
711
790
  return;
712
791
  }
713
- const body = JSON.stringify(serializeSpans(exportable));
792
+ const body = JSON.stringify(serializeSpans(exportable, this.privacy));
714
793
  // Build the export chain as a promise we retain, so forceFlush() can await
715
794
  // the actual network write — INCLUDING the bounded retry/backoff, so a
716
795
  // waitUntil drain keeps the isolate alive until retries settle too. It
@@ -891,87 +970,31 @@ export class HeystackSpanExporter {
891
970
  }
892
971
  }
893
972
  }
894
- function nativeTracingFromContext() {
895
- return context.active().getValue(NATIVE_TRACING_CONTEXT_KEY);
896
- }
897
973
  /**
898
- * Adapts Cloudflare's deliberately-small Span API to the OTel API shape used by
899
- * `withSpan`. Unsupported OTel operations degrade to attributes instead of
900
- * creating a second exporter/provider.
974
+ * Native mode: wrap the env's uncovered bindings (Workers AI, Vectorize,
975
+ * Workflows, Analytics Engine) so their calls become native child spans. An
976
+ * explicit `instrumentBindings` wins over the coverage gate.
901
977
  */
902
- class NativeSpanAdapter {
903
- nativeSpan;
904
- constructor(nativeSpan) {
905
- this.nativeSpan = nativeSpan;
906
- }
907
- spanContext() { return INVALID_SPAN_CONTEXT; }
908
- setAttribute(key, value) {
909
- try {
910
- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
911
- this.nativeSpan.setAttribute(key, value);
912
- }
913
- else if (Array.isArray(value)) {
914
- this.nativeSpan.setAttribute(key, JSON.stringify(value));
915
- }
916
- }
917
- catch {
918
- // Telemetry must never surface as an application error (e.g. a platform
919
- // span rejecting a value); drop the attribute instead.
920
- }
921
- return this;
922
- }
923
- setAttributes(attributes) {
924
- for (const [key, value] of Object.entries(attributes)) {
925
- if (value !== undefined)
926
- this.setAttribute(key, value);
927
- }
928
- return this;
929
- }
930
- addEvent(name, attributesOrStartTime) {
931
- const prefix = `event.${name}`;
932
- this.nativeSpan.setAttribute(prefix, true);
933
- if (attributesOrStartTime && !Array.isArray(attributesOrStartTime) && typeof attributesOrStartTime === "object") {
934
- for (const [key, value] of Object.entries(attributesOrStartTime)) {
935
- if (value !== undefined)
936
- this.setAttribute(`${prefix}.${key}`, value);
937
- }
938
- }
939
- return this;
940
- }
941
- addLink(_link) { return this; }
942
- addLinks(_links) { return this; }
943
- /** True once `recordException` stamped a specific `error.type` (the exception class). */
944
- errorTyped = false;
945
- setStatus(status) {
946
- if (status.code === SpanStatusCode.ERROR) {
947
- // Don't clobber the more specific class name a preceding recordException set.
948
- if (!this.errorTyped)
949
- this.nativeSpan.setAttribute("error.type", "error");
950
- if (status.message)
951
- this.nativeSpan.setAttribute("error.message", status.message);
952
- }
953
- return this;
954
- }
955
- updateName(_name) { return this; }
956
- end(_endTime) { }
957
- isRecording() { return this.nativeSpan.isTraced; }
958
- recordException(exception, _time) {
959
- const error = exception instanceof Error ? exception : new Error(String(exception));
960
- this.nativeSpan.setAttribute("error.type", error.name || "Error");
961
- this.errorTyped = true;
962
- this.nativeSpan.setAttribute("error.message", error.message);
963
- if (error.stack)
964
- this.nativeSpan.setAttribute("exception.stacktrace", error.stack);
965
- }
966
- }
967
- function getNativeTracing(ctx) {
968
- const candidate = ctx.tracing;
969
- return candidate && typeof candidate.enterSpan === "function" ? candidate : undefined;
978
+ function nativeInstrumentedEnv(env, tracing, config) {
979
+ if (!env || typeof env !== "object")
980
+ return env;
981
+ const explicit = config.instrumentBindings;
982
+ if (explicit === false)
983
+ return env;
984
+ return instrumentEnv(env, {
985
+ select: explicit ? explicit : true,
986
+ kindFilter: explicit ? undefined : (k) => shouldEnrichNatively(k, config.nativeEnrichment),
987
+ withSpan: (name, attrs, _kind, fn) => runNativeSpan(tracing, name, attrs, fn),
988
+ nativeNames: true,
989
+ });
970
990
  }
971
991
  function setNativeRequestAttributes(span, config, req) {
972
- const url = new URL(req.url);
973
992
  span.setAttribute("service.name", config.service);
974
- span.setAttribute("http.route", url.pathname);
993
+ // Labels the heystack.request child only; the fetch wrapper also sets the
994
+ // platform invocation span via `getActiveSpan()` when the runtime exposes it.
995
+ const route = resolveRoute(req, config);
996
+ if (route)
997
+ span.setAttribute("http.route", route);
975
998
  if (config.version)
976
999
  span.setAttribute("service.version", config.version);
977
1000
  if (config.build)
@@ -985,26 +1008,6 @@ function setNativeRequestAttributes(span, config, req) {
985
1008
  if (requestId)
986
1009
  span.setAttribute("http.request.id", requestId);
987
1010
  }
988
- async function runNativeSpan(nativeTracing, name, attrs, fn) {
989
- ensureContextManager();
990
- return nativeTracing.enterSpan(name, async (nativeSpan) => {
991
- const adapter = new NativeSpanAdapter(nativeSpan);
992
- for (const [key, value] of Object.entries(attrs)) {
993
- if (value !== undefined)
994
- adapter.setAttribute(key, value);
995
- }
996
- const nativeContext = trace.setSpan(context.active().setValue(NATIVE_TRACING_CONTEXT_KEY, nativeTracing), adapter);
997
- try {
998
- return await context.with(nativeContext, () => fn(adapter));
999
- }
1000
- catch (error) {
1001
- const err = error instanceof Error ? error : new Error(String(error));
1002
- adapter.recordException(err);
1003
- adapter.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
1004
- throw error;
1005
- }
1006
- });
1007
- }
1008
1011
  /**
1009
1012
  * Build a `BasicTracerProvider` wired to a `HeystackSpanExporter` via a
1010
1013
  * `SimpleSpanProcessor`. Exposed for advanced users who want to add manual
@@ -1022,10 +1025,18 @@ export function createTracerProvider(config) {
1022
1025
  // only when the caller supplied `version`/`build` (release attribution).
1023
1026
  resource: resourceFromAttributes({
1024
1027
  [ATTR_SERVICE_NAME]: config.service,
1028
+ // Lets ingest recognise direct-mode Workers apps as Cloudflare (route
1029
+ // templating, native-error and binding inference).
1030
+ "cloud.provider": "cloudflare",
1031
+ "cloud.platform": "cloudflare.workers",
1032
+ // Marks direct-mode SDK spans so coverage never reports a wrapped Worker as native-only.
1033
+ "heystack.sdk.mode": "direct",
1025
1034
  ...releaseResourceAttributes(config),
1026
1035
  }),
1027
1036
  spanProcessors: [new SimpleSpanProcessor(exporter)],
1028
- sampler: makeSampler(config.sampling),
1037
+ sampler: makeSampler(config.sampling, {
1038
+ trustRemoteParent: config.trustInboundSampling !== false,
1039
+ }),
1029
1040
  });
1030
1041
  // Attach the exporter so flush paths can await its in-flight fetches.
1031
1042
  return Object.assign(provider, { heystackExporter: exporter });
@@ -1125,6 +1136,7 @@ function ensureContextManager() {
1125
1136
  const mgr = getALS() ? new AlsContextManager() : new SyncStackContextManager();
1126
1137
  context.setGlobalContextManager(mgr.enable());
1127
1138
  }
1139
+ setNativeContextInit(ensureContextManager);
1128
1140
  /** Reset the context-manager registration guard. Internal/testing helper. */
1129
1141
  export function __resetContextManager() {
1130
1142
  _contextManagerRegistered = false;
@@ -1157,7 +1169,8 @@ function ensureGlobalProvider(config) {
1157
1169
  // Patch globalThis.fetch (once) so outbound subrequests get CLIENT child spans
1158
1170
  // + `traceparent` injection (distributed tracing). The exporter's own POST uses
1159
1171
  // the captured original fetch, so it is never re-entered by this wrapper.
1160
- ensureFetchInstrumentation(_provider.heystackExporter.ingestHost, config.ai ?? {});
1172
+ _fetchRedact = makeUrlRedactor(config);
1173
+ ensureFetchInstrumentation(_provider.heystackExporter.ingestHost, config.ai ?? {}, config.propagateTargets);
1161
1174
  return _provider;
1162
1175
  }
1163
1176
  /**
@@ -1190,12 +1203,32 @@ export async function flushHeystack() {
1190
1203
  markManualFlushObserved();
1191
1204
  if (!_provider)
1192
1205
  return;
1193
- if (typeof _provider.forceFlush === "function") {
1194
- await _provider.forceFlush();
1206
+ // Bounded by the same budget as the export drain so a stuck hash can't pin us.
1207
+ let timer;
1208
+ await Promise.race([
1209
+ settlePendingKeyHashes(),
1210
+ new Promise((resolve) => { timer = setTimeout(resolve, DRAIN_TIMEOUT_MS); }),
1211
+ ]).finally(() => { if (timer)
1212
+ clearTimeout(timer); });
1213
+ // Telemetry must never fail the app: OpenTelemetry's provider.forceFlush rejects (with an
1214
+ // array) when a processor fails or times out, and that would surface in an awaited
1215
+ // flushHeystack() or as an uncaught waitUntil error on every request while ingest is down.
1216
+ try {
1217
+ if (typeof _provider.forceFlush === "function") {
1218
+ await _provider.forceFlush();
1219
+ }
1220
+ }
1221
+ catch {
1222
+ // Dropped batch: the exporter already reported it; still drain what is in flight.
1195
1223
  }
1196
1224
  // The processor's forceFlush does not await our fire-and-forget fetch, so
1197
1225
  // drain the exporter's in-flight POSTs explicitly.
1198
- await _provider.heystackExporter.forceFlush();
1226
+ try {
1227
+ await _provider.heystackExporter.forceFlush();
1228
+ }
1229
+ catch {
1230
+ // Same rule: an export failure is never the app's error.
1231
+ }
1199
1232
  }
1200
1233
  /**
1201
1234
  * Wrap a Worker `ExecutionContext` so EVERY `ctx.waitUntil(promise)` ALSO
@@ -1289,6 +1322,9 @@ export function instrument(handler, config) {
1289
1322
  build: config.build,
1290
1323
  waitUntil: config.waitUntil,
1291
1324
  sampling: config.sampling,
1325
+ trustInboundSampling: config.trustInboundSampling,
1326
+ redactQueryParams: config.redactQueryParams,
1327
+ captureClientIp: config.captureClientIp,
1292
1328
  ai: config.ai,
1293
1329
  });
1294
1330
  // instrument() wires a per-request drain (see `drain` below), so exports on
@@ -1296,6 +1332,9 @@ export function instrument(handler, config) {
1296
1332
  requestDrainWired = true;
1297
1333
  return { provider, tracer: trace.getTracer("heystack") };
1298
1334
  };
1335
+ // Privacy: one redactor per instrument() call for the inbound SERVER span URL.
1336
+ const redactInboundUrl = makeUrlRedactor(config);
1337
+ const captureGeo = config.captureGeo ?? "country";
1299
1338
  // Drain BOTH the provider's span processor AND the exporter's in-flight fetch
1300
1339
  // via ctx.waitUntil, raced against an ~8s timeout (see drainWithTimeout, #25).
1301
1340
  const drain = (provider, ctx) => {
@@ -1307,7 +1346,7 @@ export function instrument(handler, config) {
1307
1346
  const originalFetch = handler.fetch?.bind(handler);
1308
1347
  if (originalFetch) {
1309
1348
  wrapped.fetch = async (req, env, ctx) => {
1310
- const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1349
+ const nativeTracing = config.nativeTracing === "direct" ? null : getNativeTracing(ctx, config.tracing);
1311
1350
  if (nativeTracing) {
1312
1351
  // `withSpan()` discovers the request's native tracer through OTel
1313
1352
  // context. Register our WinterCG-safe manager even though native mode
@@ -1316,14 +1355,25 @@ export function instrument(handler, config) {
1316
1355
  // Install the (LLM-only under native) fetch wrapper so outbound calls
1317
1356
  // to LLM providers/gateways still get gen_ai.* enrichment — native mode
1318
1357
  // never runs `setup()`, which is where the direct plane installs it.
1319
- ensureFetchInstrumentation("", config.ai ?? {});
1358
+ ensureFetchInstrumentation(nativeIngestHost(config), config.ai ?? {}, config.propagateTargets);
1359
+ // The platform's invocation (root fetch) span is what Service Health/RED
1360
+ // read. When the runtime exposes it, put the route template there too;
1361
+ // otherwise `routes` labels only the `heystack.request` child.
1362
+ try {
1363
+ const rootRoute = resolveRoute(req, config);
1364
+ if (rootRoute)
1365
+ nativeTracing.getActiveSpan?.()?.setAttribute("http.route", rootRoute);
1366
+ }
1367
+ catch { /* telemetry must never surface */ }
1320
1368
  return runNativeSpan(nativeTracing, "heystack.request", {}, async (adapter) => {
1321
1369
  setNativeRequestAttributes(adapter, config, req);
1322
- const response = await originalFetch(req, env, ctx);
1323
- adapter.setAttribute("http.response.status_code", response.status);
1324
- if (response.status >= 500)
1325
- adapter.setAttribute("error.type", "http_error");
1326
- return response;
1370
+ return withNativePropagation(adapter, req.headers.get("traceparent"), async () => {
1371
+ const response = await originalFetch(req, nativeInstrumentedEnv(env, nativeTracing, config), ctx);
1372
+ adapter.setAttribute("http.response.status_code", response.status);
1373
+ if (response.status >= 500)
1374
+ adapter.setAttribute("error.type", "http_error");
1375
+ return response;
1376
+ });
1327
1377
  });
1328
1378
  }
1329
1379
  const s = setup(env);
@@ -1362,12 +1412,16 @@ export function instrument(handler, config) {
1362
1412
  kind: SpanKind.SERVER,
1363
1413
  attributes: {
1364
1414
  "http.request.method": req.method,
1365
- "url.full": req.url,
1415
+ "url.full": redactInboundUrl(req.url),
1366
1416
  "url.path": url.pathname,
1367
1417
  "server.address": url.host,
1418
+ "cloudflare.handler_type": "fetch",
1368
1419
  },
1369
1420
  }, startCtx);
1370
- // A1 enrichment: identity, request id, client ip, geo.
1421
+ const directRoute = resolveRoute(req, config);
1422
+ if (directRoute)
1423
+ span.setAttribute("http.route", directRoute);
1424
+ // A1 enrichment: identity, request id, (opt-in) client ip, geo.
1371
1425
  // All attributes are set only when non-empty to keep spans lean.
1372
1426
  const userInfo = config.getUser?.(req);
1373
1427
  if (userInfo?.id)
@@ -1377,20 +1431,26 @@ export function instrument(handler, config) {
1377
1431
  const reqId = userInfo?.requestId ?? req.headers.get("cf-ray") ?? "";
1378
1432
  if (reqId)
1379
1433
  span.setAttribute("http.request.id", reqId);
1380
- const clientIp = req.headers.get("CF-Connecting-IP") ?? "";
1381
- if (clientIp)
1382
- span.setAttribute("client.address", clientIp);
1434
+ // End-user IP is personal data: recorded only with `captureClientIp: true`.
1435
+ if (config.captureClientIp === true) {
1436
+ const clientIp = req.headers.get("CF-Connecting-IP") ?? "";
1437
+ if (clientIp)
1438
+ span.setAttribute("client.address", clientIp);
1439
+ }
1383
1440
  // Cloudflare geo — only present on the real CF runtime; read defensively.
1441
+ // Default is country-only; region/city/ASN need `captureGeo: "full"`.
1384
1442
  const cf = req.cf;
1385
- if (cf) {
1443
+ if (cf && captureGeo) {
1386
1444
  if (typeof cf.country === "string" && cf.country)
1387
1445
  span.setAttribute("geo.country", cf.country);
1388
- if (typeof cf.region === "string" && cf.region)
1389
- span.setAttribute("geo.region", cf.region);
1390
- if (typeof cf.city === "string" && cf.city)
1391
- span.setAttribute("geo.city", cf.city);
1392
- if (cf.asn != null)
1393
- span.setAttribute("geo.asn", String(cf.asn));
1446
+ if (captureGeo === "full") {
1447
+ if (typeof cf.region === "string" && cf.region)
1448
+ span.setAttribute("geo.region", cf.region);
1449
+ if (typeof cf.city === "string" && cf.city)
1450
+ span.setAttribute("geo.city", cf.city);
1451
+ if (cf.asn != null)
1452
+ span.setAttribute("geo.asn", String(cf.asn));
1453
+ }
1394
1454
  }
1395
1455
  // FR5: response header carrying THIS span's trace + span id.
1396
1456
  const sc = span.spanContext();
@@ -1558,19 +1618,21 @@ export function instrument(handler, config) {
1558
1618
  const originalQueue = handler.queue?.bind(handler);
1559
1619
  if (originalQueue) {
1560
1620
  wrapped.queue = async (batch, env, ctx) => {
1561
- const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1621
+ const nativeTracing = config.nativeTracing === "direct" ? null : getNativeTracing(ctx, config.tracing);
1562
1622
  if (nativeTracing) {
1563
- ensureFetchInstrumentation("", config.ai ?? {});
1623
+ ensureFetchInstrumentation(nativeIngestHost(config), config.ai ?? {}, config.propagateTargets);
1564
1624
  // Native mode: Cloudflare owns queue spans — pass the ORIGINAL batch
1565
1625
  // through untouched so platform objects keep their identity.
1566
1626
  return runNativeSpan(nativeTracing, `heystack.queue ${batch.queue}`, {
1567
1627
  "service.name": config.service,
1568
1628
  "messaging.batch.message_count": batch.messages.length,
1629
+ "cloudflare.queue.batch_size": batch.messages.length,
1630
+ "cloudflare.handler_type": "queue",
1569
1631
  "messaging.destination.name": batch.queue,
1570
1632
  "messaging.operation.type": "process",
1571
1633
  "service.version": config.version,
1572
1634
  "service.build": config.build,
1573
- }, () => originalQueue(batch, env, ctx));
1635
+ }, (adapter) => withNativePropagation(adapter, null, () => originalQueue(batch, nativeInstrumentedEnv(env, nativeTracing, config), ctx)));
1574
1636
  }
1575
1637
  const s = setup(env);
1576
1638
  if (!s)
@@ -1580,7 +1642,9 @@ export function instrument(handler, config) {
1580
1642
  kind: SpanKind.CONSUMER,
1581
1643
  attributes: {
1582
1644
  "messaging.batch.message_count": batch.messages.length,
1645
+ "cloudflare.queue.batch_size": batch.messages.length,
1583
1646
  "messaging.destination.name": batch.queue,
1647
+ "cloudflare.handler_type": "queue",
1584
1648
  },
1585
1649
  });
1586
1650
  // The request's CONSUMER context, captured per-batch as an explicit parent
@@ -1620,16 +1684,18 @@ export function instrument(handler, config) {
1620
1684
  const originalScheduled = handler.scheduled?.bind(handler);
1621
1685
  if (originalScheduled) {
1622
1686
  wrapped.scheduled = async (controller, env, ctx) => {
1623
- const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1687
+ const nativeTracing = config.nativeTracing === "direct" ? null : getNativeTracing(ctx, config.tracing);
1624
1688
  if (nativeTracing) {
1625
- ensureFetchInstrumentation("", config.ai ?? {});
1689
+ ensureFetchInstrumentation(nativeIngestHost(config), config.ai ?? {}, config.propagateTargets);
1626
1690
  return runNativeSpan(nativeTracing, `heystack.scheduled ${controller.cron}`, {
1627
1691
  "service.name": config.service,
1628
1692
  "faas.trigger": "timer",
1629
1693
  "cloudflare.scheduled.cron": controller.cron,
1694
+ "faas.cron": controller.cron,
1695
+ "cloudflare.handler_type": "scheduled",
1630
1696
  "service.version": config.version,
1631
1697
  "service.build": config.build,
1632
- }, () => originalScheduled(controller, env, ctx));
1698
+ }, (adapter) => withNativePropagation(adapter, null, () => originalScheduled(controller, nativeInstrumentedEnv(env, nativeTracing, config), ctx)));
1633
1699
  }
1634
1700
  const s = setup(env);
1635
1701
  if (!s)
@@ -1639,6 +1705,8 @@ export function instrument(handler, config) {
1639
1705
  kind: SpanKind.INTERNAL,
1640
1706
  attributes: {
1641
1707
  "controller.cron": controller.cron,
1708
+ "faas.cron": controller.cron,
1709
+ "cloudflare.handler_type": "scheduled",
1642
1710
  },
1643
1711
  });
1644
1712
  try {
@@ -1662,7 +1730,7 @@ export function instrument(handler, config) {
1662
1730
  const originalEmail = handler.email?.bind(handler);
1663
1731
  if (originalEmail) {
1664
1732
  wrapped.email = async (message, env, ctx) => {
1665
- const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1733
+ const nativeTracing = config.nativeTracing === "direct" ? null : getNativeTracing(ctx, config.tracing);
1666
1734
  // PRIVACY: only from/to DOMAINS + envelope size are recorded — never the
1667
1735
  // full addresses and never any message content.
1668
1736
  const { name: emailSpanName, attrs: emailAttrs } = emailHandlerAttrs(message);
@@ -1672,7 +1740,7 @@ export function instrument(handler, config) {
1672
1740
  ...emailAttrs,
1673
1741
  "service.version": config.version,
1674
1742
  "service.build": config.build,
1675
- }, () => originalEmail(message, env, ctx));
1743
+ }, (adapter) => withNativePropagation(adapter, null, () => originalEmail(message, nativeInstrumentedEnv(env, nativeTracing, config), ctx)));
1676
1744
  }
1677
1745
  const s = setup(env);
1678
1746
  if (!s)
@@ -1680,7 +1748,7 @@ export function instrument(handler, config) {
1680
1748
  const { provider, tracer } = s;
1681
1749
  const span = tracer.startSpan(emailSpanName, {
1682
1750
  kind: SpanKind.SERVER,
1683
- attributes: emailAttrs,
1751
+ attributes: { ...emailAttrs, "cloudflare.handler_type": "email" },
1684
1752
  });
1685
1753
  try {
1686
1754
  await context.with(trace.setSpan(context.active(), span), () => originalEmail(message, env, wrapExecutionContextWithProvider(ctx, provider)));
@@ -1703,7 +1771,7 @@ export function instrument(handler, config) {
1703
1771
  const originalTail = handler.tail?.bind(handler);
1704
1772
  if (originalTail) {
1705
1773
  wrapped.tail = async (events, env, ctx) => {
1706
- const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1774
+ const nativeTracing = config.nativeTracing === "direct" ? null : getNativeTracing(ctx, config.tracing);
1707
1775
  // PRIVACY: only the event COUNT is recorded — tail payloads can contain
1708
1776
  // request URLs, headers and console output; none of it is captured.
1709
1777
  const count = Array.isArray(events) ? events.length : 0;
@@ -1722,7 +1790,7 @@ export function instrument(handler, config) {
1722
1790
  const { provider, tracer } = s;
1723
1791
  const span = tracer.startSpan("tail", {
1724
1792
  kind: SpanKind.INTERNAL,
1725
- attributes: attrs,
1793
+ attributes: { ...attrs, "cloudflare.handler_type": "tail" },
1726
1794
  });
1727
1795
  try {
1728
1796
  await context.with(trace.setSpan(context.active(), span), () => originalTail(events, env, wrapExecutionContextWithProvider(ctx, provider)));
@@ -1862,7 +1930,18 @@ function classStartSpan(opts) {
1862
1930
  * to the invocation's DurableObjectState `waitUntil`. Best-effort no-op when no
1863
1931
  * state/waitUntil is available (e.g. unit tests).
1864
1932
  */
1865
- function scheduleStateFlush(state) {
1933
+ function scheduleStateFlush(state, invocation) {
1934
+ // SQL cursors the app never finished iterating: close THIS invocation's spans
1935
+ // (without row counts); concurrent invocations on the same DO keep theirs.
1936
+ if (state && typeof state === "object") {
1937
+ const pending = pendingSqlSpans.get(state);
1938
+ if (pending) {
1939
+ for (const p of [...pending]) {
1940
+ if (invocation === undefined || p.owner === undefined || p.owner === invocation)
1941
+ p.abandon();
1942
+ }
1943
+ }
1944
+ }
1866
1945
  const waitUntil = state
1867
1946
  ?.waitUntil;
1868
1947
  if (typeof waitUntil !== "function")
@@ -1874,8 +1953,152 @@ function scheduleStateFlush(state) {
1874
1953
  // A failing waitUntil must never break the wrapped handler.
1875
1954
  }
1876
1955
  }
1877
- /** INTERNAL child spans for DurableObjectStorage get/put/delete/list ops. */
1878
- function wrapDurableObjectStorage(storage, startSpan) {
1956
+ /** Marks the DO handler invocation in context; nested spans inherit it. */
1957
+ const DO_INVOCATION_KEY = createContextKey("heystack.do.invocation");
1958
+ /** Open `sql.exec` spans per raw DurableObjectState, closed when the invocation ends. */
1959
+ const pendingSqlSpans = new WeakMap();
1960
+ /**
1961
+ * `ctx.storage.sql.exec()` → `durable_object_storage_exec`. The cursor is lazy and
1962
+ * owned by the app: we never step it. The span ends (with rows read/written) when
1963
+ * the app completes it via `toArray()`, `one()` or iterator exhaustion, and is
1964
+ * closed without counts if the invocation ends first.
1965
+ */
1966
+ function wrapDurableObjectSql(sql, startSpan, doId, pending) {
1967
+ return new Proxy(sql, {
1968
+ get(t, prop) {
1969
+ const val = Reflect.get(t, prop, t);
1970
+ if (typeof val !== "function")
1971
+ return val;
1972
+ if (prop !== "exec")
1973
+ return val.bind(t);
1974
+ return (query, ...bindings) => {
1975
+ if (isTracingSuppressed(context.active()))
1976
+ return val.call(t, query, ...bindings);
1977
+ const attrs = {};
1978
+ if (typeof query === "string")
1979
+ attrs["db.query.text"] = query;
1980
+ if (doId)
1981
+ attrs["cloudflare.durable_object.id"] = doId;
1982
+ const span = startSpan("durable_object_storage_exec", attrs, SpanKind.INTERNAL);
1983
+ let cursor;
1984
+ try {
1985
+ cursor = val.call(t, query, ...bindings);
1986
+ }
1987
+ catch (error) {
1988
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
1989
+ span.setStatus({
1990
+ code: SpanStatusCode.ERROR,
1991
+ message: error instanceof Error ? error.message : String(error),
1992
+ });
1993
+ span.end();
1994
+ throw error;
1995
+ }
1996
+ if (!cursor || typeof cursor !== "object") {
1997
+ span.end();
1998
+ return cursor;
1999
+ }
2000
+ let done = false;
2001
+ let lastActivity = Date.now();
2002
+ const owner = context.active().getValue(DO_INVOCATION_KEY);
2003
+ const finish = (counts, error) => {
2004
+ if (done)
2005
+ return;
2006
+ done = true;
2007
+ pending.delete(entry);
2008
+ try {
2009
+ if (error !== undefined) {
2010
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
2011
+ span.setStatus({
2012
+ code: SpanStatusCode.ERROR,
2013
+ message: error instanceof Error ? error.message : String(error),
2014
+ });
2015
+ }
2016
+ if (counts) {
2017
+ // Side-effect-free getters: reading them never steps the cursor.
2018
+ if (typeof cursor.rowsRead === "number") {
2019
+ span.setAttribute("cloudflare.durable_object.response.rows_read", cursor.rowsRead);
2020
+ }
2021
+ if (typeof cursor.rowsWritten === "number") {
2022
+ span.setAttribute("cloudflare.durable_object.response.rows_written", cursor.rowsWritten);
2023
+ }
2024
+ span.end();
2025
+ }
2026
+ else {
2027
+ // Abandoned: the work effectively ended at the last observed activity.
2028
+ span.end(lastActivity);
2029
+ }
2030
+ }
2031
+ catch {
2032
+ // Telemetry must never alter the cursor's behaviour.
2033
+ }
2034
+ };
2035
+ const entry = { owner, abandon: () => finish(false) };
2036
+ pending.add(entry);
2037
+ // A statement with no result columns (INSERT/UPDATE/DDL) has fully run
2038
+ // by the time exec() returns; nothing will ever be iterated.
2039
+ if (Array.isArray(cursor.columnNames) && cursor.columnNames.length === 0) {
2040
+ finish(true);
2041
+ return cursor;
2042
+ }
2043
+ const step = (fn, c, a) => {
2044
+ let r;
2045
+ try {
2046
+ r = fn.apply(c, a);
2047
+ }
2048
+ catch (error) {
2049
+ finish(false, error);
2050
+ throw error;
2051
+ }
2052
+ lastActivity = Date.now();
2053
+ if (r && r.done === true)
2054
+ finish(true);
2055
+ return r;
2056
+ };
2057
+ // One observer for the cursor and for raw() iterators: completion is
2058
+ // toArray()/one()/iterator exhaustion; nothing is ever stepped by us.
2059
+ const observe = (target) => {
2060
+ if (!target || typeof target !== "object")
2061
+ return target;
2062
+ const proxy = new Proxy(target, {
2063
+ get(c, p) {
2064
+ const v = Reflect.get(c, p, c);
2065
+ if (typeof v !== "function")
2066
+ return v;
2067
+ if (p === "toArray" || p === "one") {
2068
+ return (...a) => {
2069
+ try {
2070
+ const r = v.apply(c, a);
2071
+ finish(true);
2072
+ return r;
2073
+ }
2074
+ catch (error) {
2075
+ finish(false, error);
2076
+ throw error;
2077
+ }
2078
+ };
2079
+ }
2080
+ if (p === "next")
2081
+ return (...a) => step(v, c, a);
2082
+ if (p === Symbol.iterator) {
2083
+ return () => {
2084
+ const it = v.call(c);
2085
+ return it === c ? proxy : observe(it);
2086
+ };
2087
+ }
2088
+ if (p === "raw")
2089
+ return (...a) => observe(v.apply(c, a));
2090
+ return v.bind(c);
2091
+ },
2092
+ });
2093
+ return proxy;
2094
+ };
2095
+ return observe(cursor);
2096
+ };
2097
+ },
2098
+ });
2099
+ }
2100
+ /** INTERNAL child spans for DurableObjectStorage get/put/delete/list, alarms and SQL. */
2101
+ function wrapDurableObjectStorage(storage, startSpan, doId, pending = new Set()) {
1879
2102
  const wrapOp = (op) => async (...args) => {
1880
2103
  const underlying = storage[op];
1881
2104
  if (typeof underlying !== "function")
@@ -1884,11 +2107,10 @@ function wrapDurableObjectStorage(storage, startSpan) {
1884
2107
  return underlying.apply(storage, args);
1885
2108
  }
1886
2109
  const attrs = {};
2110
+ if (doId)
2111
+ attrs["cloudflare.durable_object.id"] = doId;
1887
2112
  const first = args[0];
1888
- if (typeof first === "string") {
1889
- attrs["cloudflare.do.storage.key"] = first;
1890
- }
1891
- else if (Array.isArray(first)) {
2113
+ if (Array.isArray(first)) {
1892
2114
  attrs["cloudflare.do.storage.key_count"] = first.length;
1893
2115
  }
1894
2116
  else if (op === "list" &&
@@ -1898,8 +2120,17 @@ function wrapDurableObjectStorage(storage, startSpan) {
1898
2120
  attrs["cloudflare.do.storage.limit"] = first.limit;
1899
2121
  }
1900
2122
  const span = startSpan(`DO storage ${op}`, attrs, SpanKind.INTERNAL);
2123
+ // Keys are user data: only the hash is ever emitted (omitted if hashing fails).
2124
+ return runWithSpanKeyed(span, "cloudflare.do.storage.key", keysOf(first), () => Promise.resolve(underlying.apply(storage, args)));
2125
+ };
2126
+ const wrapAlarm = (op) => async (...args) => {
2127
+ const underlying = storage[op];
2128
+ if (isTracingSuppressed(context.active()))
2129
+ return underlying.apply(storage, args);
2130
+ const span = startSpan(`durable_object_alarms_${op}`, doId ? { "cloudflare.durable_object.id": doId } : {}, SpanKind.INTERNAL);
1901
2131
  return runWithSpan(span, () => Promise.resolve(underlying.apply(storage, args)));
1902
2132
  };
2133
+ let tracedSql;
1903
2134
  // A dedicated proxy (not makeProxy) because `storage` is a VALUE property on
1904
2135
  // the state object, not a method — handlers there must be getters.
1905
2136
  return new Proxy(storage, {
@@ -1908,10 +2139,46 @@ function wrapDurableObjectStorage(storage, startSpan) {
1908
2139
  if ((prop === "get" || prop === "put" || prop === "delete" || prop === "list") && typeof val === "function") {
1909
2140
  return wrapOp(prop);
1910
2141
  }
2142
+ if ((prop === "getAlarm" || prop === "setAlarm" || prop === "deleteAlarm") &&
2143
+ typeof val === "function") {
2144
+ return wrapAlarm(prop);
2145
+ }
2146
+ if (prop === "sql" && val && typeof val === "object") {
2147
+ return (tracedSql ??= wrapDurableObjectSql(val, startSpan, doId, pending));
2148
+ }
1911
2149
  return typeof val === "function" ? val.bind(t) : val;
1912
2150
  },
1913
2151
  });
1914
2152
  }
2153
+ /** The DO id string from a raw DurableObjectState; undefined when unavailable. */
2154
+ function durableObjectIdOf(state) {
2155
+ try {
2156
+ const raw = state?.id?.toString?.();
2157
+ return typeof raw === "string" && raw ? raw : undefined;
2158
+ }
2159
+ catch {
2160
+ return undefined;
2161
+ }
2162
+ }
2163
+ const doIdAttrs = (state) => {
2164
+ const id = durableObjectIdOf(state);
2165
+ return id ? { "cloudflare.durable_object.id": id } : {};
2166
+ };
2167
+ /** True when a class (or an ancestor) is named `name`, e.g. cloudflare:workers' `DurableObject`. */
2168
+ function extendsNamedBase(cls, name) {
2169
+ try {
2170
+ let c = Object.getPrototypeOf(cls);
2171
+ for (let depth = 0; depth < 8 && typeof c === "function"; depth++) {
2172
+ if (c.name === name)
2173
+ return true;
2174
+ c = Object.getPrototypeOf(c);
2175
+ }
2176
+ }
2177
+ catch {
2178
+ // Exotic prototype chains are treated as plain classes.
2179
+ }
2180
+ return false;
2181
+ }
1915
2182
  /** Proxy a DurableObjectState so `state.storage` resolves to the traced proxy. */
1916
2183
  function wrapDurableObjectState(state, startSpan) {
1917
2184
  if (!state || typeof state !== "object")
@@ -1919,7 +2186,18 @@ function wrapDurableObjectState(state, startSpan) {
1919
2186
  const s = state;
1920
2187
  if (!s.storage || typeof s.storage !== "object")
1921
2188
  return state;
1922
- const tracedStorage = wrapDurableObjectStorage(s.storage, startSpan);
2189
+ let doId;
2190
+ try {
2191
+ const raw = s.id?.toString?.();
2192
+ if (typeof raw === "string" && raw)
2193
+ doId = raw;
2194
+ }
2195
+ catch {
2196
+ // An id without a usable toString() just omits the attribute.
2197
+ }
2198
+ const pending = new Set();
2199
+ pendingSqlSpans.set(s, pending);
2200
+ const tracedStorage = wrapDurableObjectStorage(s.storage, startSpan, doId, pending);
1923
2201
  return new Proxy(s, {
1924
2202
  get(t, prop, _receiver) {
1925
2203
  if (prop === "storage")
@@ -1931,6 +2209,10 @@ function wrapDurableObjectState(state, startSpan) {
1931
2209
  }
1932
2210
  /** Prototype methods that are wrapped specially (never treated as RPC). */
1933
2211
  const DO_LIFECYCLE_METHODS = new Set(["fetch", "alarm", "constructor"]);
2212
+ /** WebSocket hibernation handlers: wrapped as named lifecycle spans, never as RPC. */
2213
+ const DO_WEBSOCKET_METHODS = ["webSocketMessage", "webSocketClose", "webSocketError"];
2214
+ for (const m of DO_WEBSOCKET_METHODS)
2215
+ DO_LIFECYCLE_METHODS.add(m);
1934
2216
  const MAX_DO_RPC_METHODS = 32;
1935
2217
  /**
1936
2218
  * Decorate a Durable Object class:
@@ -1976,12 +2258,35 @@ export function wrapDurableObjectClass(cls, opts = {}) {
1976
2258
  catch {
1977
2259
  // Best-effort enumeration — a hostile getter must never break decoration.
1978
2260
  }
2261
+ // workerd's `DurableObject` base (cloudflare:workers) brand-checks its first argument, so a
2262
+ // subclass of it must receive the genuine state. Classes without that base keep getting the
2263
+ // traced view directly, which also traces storage calls made inside their constructor.
2264
+ const nativeBase = extendsNamedBase(Base, "DurableObject");
1979
2265
  class Wrapped extends Base {
1980
2266
  /** Raw (unwrapped) state captured for flush scheduling. */
1981
2267
  __heystackState;
1982
2268
  constructor(...args) {
1983
- super(...args.map((arg, i) => i === 0 ? wrapDurableObjectState(arg, startSpan) : arg));
1984
- this.__heystackState = args[0];
2269
+ const raw = args[0];
2270
+ const traced = wrapDurableObjectState(raw, startSpan);
2271
+ super(...(nativeBase ? args : [traced, ...args.slice(1)]));
2272
+ this.__heystackState = raw;
2273
+ // Swap the traced view in wherever the instance holds the state (`this.ctx` from the
2274
+ // base, or a field the user assigned), so `this.ctx.storage` calls are traced.
2275
+ if (nativeBase && traced !== raw) {
2276
+ const self = this;
2277
+ for (const key of new Set(["ctx", "state", ...Object.keys(self)])) {
2278
+ if (key === "__heystackState")
2279
+ continue;
2280
+ try {
2281
+ if (self[key] === raw) {
2282
+ Object.defineProperty(self, key, { value: traced, writable: true, configurable: true, enumerable: true });
2283
+ }
2284
+ }
2285
+ catch {
2286
+ // A non-configurable or frozen slot keeps the raw state: calls stay untraced, never broken.
2287
+ }
2288
+ }
2289
+ }
1985
2290
  }
1986
2291
  }
1987
2292
  if (typeof origFetch === "function") {
@@ -2011,9 +2316,11 @@ export function wrapDurableObjectClass(cls, opts = {}) {
2011
2316
  "http.request.method": request.method,
2012
2317
  "url.path": url.pathname,
2013
2318
  "server.address": url.host,
2319
+ "cloudflare.handler_type": "fetch",
2320
+ ...doIdAttrs(this.__heystackState),
2014
2321
  }, SpanKind.SERVER, startCtx);
2015
2322
  try {
2016
- const response = (await context.with(trace.setSpan(startCtx, span), () => origFetch.call(this, request)));
2323
+ const response = (await context.with(trace.setSpan(startCtx, span).setValue(DO_INVOCATION_KEY, span), () => origFetch.call(this, request)));
2017
2324
  span.setAttribute("http.response.status_code", response.status);
2018
2325
  span.setStatus({
2019
2326
  code: response.status >= 500 ? SpanStatusCode.ERROR : SpanStatusCode.UNSET,
@@ -2030,7 +2337,7 @@ export function wrapDurableObjectClass(cls, opts = {}) {
2030
2337
  }
2031
2338
  finally {
2032
2339
  span.end();
2033
- scheduleStateFlush(this.__heystackState);
2340
+ scheduleStateFlush(this.__heystackState, span);
2034
2341
  }
2035
2342
  };
2036
2343
  }
@@ -2039,9 +2346,9 @@ export function wrapDurableObjectClass(cls, opts = {}) {
2039
2346
  if (isTracingSuppressed(context.active())) {
2040
2347
  return origAlarm.apply(this, args);
2041
2348
  }
2042
- const span = startSpan("DO alarm", {}, SpanKind.INTERNAL);
2349
+ const span = startSpan("DO alarm", { "cloudflare.handler_type": "alarm", ...doIdAttrs(this.__heystackState) }, SpanKind.INTERNAL);
2043
2350
  try {
2044
- return await context.with(trace.setSpan(context.active(), span), () => origAlarm.apply(this, args));
2351
+ return await context.with(trace.setSpan(context.active(), span).setValue(DO_INVOCATION_KEY, span), () => origAlarm.apply(this, args));
2045
2352
  }
2046
2353
  catch (error) {
2047
2354
  span.recordException(error instanceof Error ? error : new Error(String(error)));
@@ -2053,22 +2360,20 @@ export function wrapDurableObjectClass(cls, opts = {}) {
2053
2360
  }
2054
2361
  finally {
2055
2362
  span.end();
2056
- scheduleStateFlush(this.__heystackState);
2363
+ scheduleStateFlush(this.__heystackState, span);
2057
2364
  }
2058
2365
  };
2059
2366
  }
2060
- for (const method of rpcMethods) {
2367
+ const className = Base.name || "DurableObject";
2368
+ const wrapMethod = (method, spanName, attrs) => {
2061
2369
  const orig = proto[method];
2062
2370
  Wrapped.prototype[method] = async function (...args) {
2063
2371
  if (isTracingSuppressed(context.active())) {
2064
2372
  return orig.apply(this, args);
2065
2373
  }
2066
- const span = startSpan(`DO rpc ${method}`, {
2067
- "rpc.system": "cloudflare.durable_object",
2068
- "rpc.method": method,
2069
- }, SpanKind.SERVER);
2374
+ const span = startSpan(spanName, { ...attrs, ...doIdAttrs(this.__heystackState) }, SpanKind.SERVER);
2070
2375
  try {
2071
- return await context.with(trace.setSpan(context.active(), span), () => orig.apply(this, args));
2376
+ return await context.with(trace.setSpan(context.active(), span).setValue(DO_INVOCATION_KEY, span), () => orig.apply(this, args));
2072
2377
  }
2073
2378
  catch (error) {
2074
2379
  span.recordException(error instanceof Error ? error : new Error(String(error)));
@@ -2080,9 +2385,21 @@ export function wrapDurableObjectClass(cls, opts = {}) {
2080
2385
  }
2081
2386
  finally {
2082
2387
  span.end();
2083
- scheduleStateFlush(this.__heystackState);
2388
+ scheduleStateFlush(this.__heystackState, span);
2084
2389
  }
2085
2390
  };
2391
+ };
2392
+ for (const method of DO_WEBSOCKET_METHODS) {
2393
+ if (typeof proto[method] === "function") {
2394
+ wrapMethod(method, `DO ${className} ${method}`, { "cloudflare.handler_type": method });
2395
+ }
2396
+ }
2397
+ for (const method of rpcMethods) {
2398
+ wrapMethod(method, `DO rpc ${method}`, {
2399
+ "rpc.system": "cloudflare.durable_object",
2400
+ "rpc.method": method,
2401
+ "cloudflare.handler_type": "rpc",
2402
+ });
2086
2403
  }
2087
2404
  return Wrapped;
2088
2405
  }
@@ -2225,7 +2542,7 @@ export function wrapWorkflowEntrypointClass(cls, opts = {}) {
2225
2542
  return origRun.call(this, event, step, ...rest);
2226
2543
  }
2227
2544
  const attrs = workflowRunAttrs(event);
2228
- const span = startSpan("workflow run", attrs, SpanKind.INTERNAL);
2545
+ const span = startSpan("workflow run", { ...attrs, "cloudflare.handler_type": "workflow" }, SpanKind.INTERNAL);
2229
2546
  try {
2230
2547
  return await context.with(trace.setSpan(context.active(), span), () => origRun.call(this, event, step && typeof step === "object"
2231
2548
  ? wrapWorkflowStep(step, startSpan, this.__heystackState)
@@ -2246,25 +2563,155 @@ export function wrapWorkflowEntrypointClass(cls, opts = {}) {
2246
2563
  };
2247
2564
  return Wrapped;
2248
2565
  }
2566
+ // ---------------------------------------------------------------------------
2567
+ // WorkerEntrypoint CLASS instrumentation
2568
+ // ---------------------------------------------------------------------------
2569
+ /** Handler names and RpcTarget plumbing that are never treated as RPC methods. */
2570
+ const ENTRYPOINT_NON_RPC = new Set([
2571
+ "constructor",
2572
+ "fetch",
2573
+ "tail",
2574
+ "trace",
2575
+ "scheduled",
2576
+ "queue",
2577
+ "email",
2578
+ "test",
2579
+ "then",
2580
+ "toJSON",
2581
+ "connect",
2582
+ "tailStream",
2583
+ ]);
2584
+ const ENTRYPOINT_BASES = new Set(["WorkerEntrypoint", "RpcTarget", "DurableObject"]);
2585
+ const MAX_ENTRYPOINT_RPC_METHODS = 64;
2586
+ /**
2587
+ * Decorate a `WorkerEntrypoint` class (the target of a Service binding):
2588
+ * - `fetch()` is traced exactly like an `instrument()`ed handler (SERVER span,
2589
+ * inbound `traceparent` continued, export drained via `ctx.waitUntil`).
2590
+ * - every public RPC method gets a SERVER span `rpc <method>` with
2591
+ * `jsrpc.method` / `rpc.method`. Return values (including stubs, RpcTargets
2592
+ * and custom thenables), `this` and errors pass through untouched.
2593
+ *
2594
+ * RPC spans exist in DIRECT mode only. When Cloudflare native tracing is active
2595
+ * it already emits `jsrpc` spans, so RPC calls pass straight through and `fetch`
2596
+ * takes `instrument()`'s native path. Constructor, `#private`, `_underscore` and
2597
+ * Symbol-keyed members are never wrapped.
2598
+ *
2599
+ * export default wrapWorkerEntrypointClass(class Api extends WorkerEntrypoint { ... }, { service: "api" });
2600
+ */
2601
+ export function wrapWorkerEntrypointClass(cls, cfg) {
2602
+ const Base = cls;
2603
+ const proto = Base.prototype;
2604
+ const instances = new WeakMap();
2605
+ // Enumerate public RPC methods without invoking any getter.
2606
+ const rpcMethods = [];
2607
+ try {
2608
+ let p = Base.prototype;
2609
+ for (let depth = 0; depth < 6 && p && p !== Object.prototype; depth++) {
2610
+ const ctorName = p.constructor?.name;
2611
+ if (depth > 0 && ctorName && ENTRYPOINT_BASES.has(ctorName))
2612
+ break;
2613
+ for (const key of Object.getOwnPropertyNames(p)) {
2614
+ if (ENTRYPOINT_NON_RPC.has(key) || key.startsWith("_") || key.startsWith("#"))
2615
+ continue;
2616
+ if (rpcMethods.includes(key) || rpcMethods.length >= MAX_ENTRYPOINT_RPC_METHODS)
2617
+ continue;
2618
+ if (typeof Object.getOwnPropertyDescriptor(p, key)?.value === "function")
2619
+ rpcMethods.push(key);
2620
+ }
2621
+ p = Object.getPrototypeOf(p);
2622
+ }
2623
+ }
2624
+ catch {
2625
+ // Best-effort enumeration — a hostile prototype must never break decoration.
2626
+ }
2627
+ class Wrapped extends Base {
2628
+ constructor(...args) {
2629
+ // The entrypoint's own `this.env` bindings get traced too, parented to
2630
+ // whichever RPC/fetch span is active when they are used.
2631
+ const env = args[1];
2632
+ const rawEnv = env;
2633
+ if (cfg.instrumentBindings !== false && env && typeof env === "object") {
2634
+ const native = cfg.nativeTracing === "direct" ? null : getNativeTracing(args[0], cfg.tracing);
2635
+ if (native) {
2636
+ args[1] = nativeInstrumentedEnv(env, native, cfg);
2637
+ }
2638
+ else if (cfg.instrumentBindings) {
2639
+ args[1] = instrumentEnv(env, {
2640
+ startSpan: (name, attrs, kind) => trace.getTracer("heystack").startSpan(name, { attributes: attrs, kind }, context.active()),
2641
+ select: cfg.instrumentBindings,
2642
+ });
2643
+ }
2644
+ }
2645
+ super(...args);
2646
+ instances.set(this, { ctx: args[0], env: args[1], rawEnv });
2647
+ }
2648
+ }
2649
+ try {
2650
+ Object.defineProperty(Wrapped, "name", { value: Base.name, configurable: true });
2651
+ }
2652
+ catch {
2653
+ // Non-configurable name: harmless.
2654
+ }
2655
+ const origFetch = proto.fetch;
2656
+ if (typeof origFetch === "function") {
2657
+ // Built once per instance (WorkerEntrypoint.fetch takes only the request).
2658
+ const handlers = new WeakMap();
2659
+ Wrapped.prototype.fetch = function (request) {
2660
+ const inst = instances.get(this);
2661
+ let handler = handlers.get(this);
2662
+ if (!handler) {
2663
+ handler = instrument({ fetch: (req) => origFetch.call(this, req) }, cfg);
2664
+ handlers.set(this, handler);
2665
+ }
2666
+ return handler.fetch(request, inst?.rawEnv ?? this.env, inst?.ctx ?? this.ctx);
2667
+ };
2668
+ }
2669
+ for (const method of rpcMethods) {
2670
+ const orig = proto[method];
2671
+ Wrapped.prototype[method] = function (...args) {
2672
+ if (isTracingSuppressed(context.active()))
2673
+ return orig.apply(this, args);
2674
+ const inst = instances.get(this) ?? {
2675
+ ctx: this.ctx,
2676
+ env: this.env,
2677
+ };
2678
+ // Native tracing owns jsrpc spans: pass straight through.
2679
+ if (cfg.nativeTracing !== "direct" && getNativeTracing(inst.ctx, cfg.tracing)) {
2680
+ return orig.apply(this, args);
2681
+ }
2682
+ const apiKey = cfg.apiKey ?? inst.env?.HEYSTACK_API_KEY;
2683
+ if (!apiKey) {
2684
+ warnOnceNoKey();
2685
+ return orig.apply(this, args);
2686
+ }
2687
+ const provider = ensureGlobalProvider({ ...cfg, apiKey });
2688
+ requestDrainWired = true;
2689
+ const span = trace.getTracer("heystack").startSpan(`rpc ${method}`, { kind: SpanKind.SERVER, attributes: { "jsrpc.method": method, "rpc.method": method, "cloudflare.handler_type": "rpc" } }, context.active());
2690
+ // Only the three members the shared tracer helper uses; `end` also drains.
2691
+ const tracked = {
2692
+ recordException: (e) => span.recordException(e),
2693
+ setStatus: (st) => span.setStatus(st),
2694
+ end: () => {
2695
+ span.end();
2696
+ try {
2697
+ inst.ctx?.waitUntil?.(drainWithTimeout(provider));
2698
+ }
2699
+ catch {
2700
+ // A rejected host waitUntil must not break the RPC call.
2701
+ }
2702
+ },
2703
+ };
2704
+ return runWithSpanPreservingReturn(tracked, () => context.with(trace.setSpan(context.active(), span), () => orig.apply(this, args)));
2705
+ };
2706
+ }
2707
+ return Wrapped;
2708
+ }
2249
2709
  export async function withSpan(name, attrsOrFn, maybeFn) {
2250
2710
  const attrs = typeof attrsOrFn === "function" ? undefined : attrsOrFn;
2251
2711
  const fn = (typeof attrsOrFn === "function" ? attrsOrFn : maybeFn);
2252
2712
  const nativeTracing = nativeTracingFromContext();
2253
2713
  if (nativeTracing) {
2254
- return nativeTracing.enterSpan(name, async (nativeSpan) => {
2255
- const adapter = new NativeSpanAdapter(nativeSpan);
2256
- if (attrs)
2257
- adapter.setAttributes(attrs);
2258
- const nativeContext = trace.setSpan(context.active().setValue(NATIVE_TRACING_CONTEXT_KEY, nativeTracing), adapter);
2259
- try {
2260
- return await context.with(nativeContext, () => fn(adapter));
2261
- }
2262
- catch (err) {
2263
- adapter.recordException(err instanceof Error ? err : new Error(String(err)));
2264
- adapter.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : String(err) });
2265
- throw err;
2266
- }
2267
- });
2714
+ return runNativeSpan(nativeTracing, name, attrs ?? {}, fn);
2268
2715
  }
2269
2716
  const tracer = trace.getTracer("heystack");
2270
2717
  const span = tracer.startSpan(name, attrs ? { attributes: attrs } : undefined);
@@ -2313,3 +2760,5 @@ export { llmRequestAttrs as genAiRequestAttrs } from "./llm-enrich.js";
2313
2760
  * Workers AI results (`usage.prompt_tokens`/`completion_tokens`) included.
2314
2761
  */
2315
2762
  export { llmResponseAttrs as genAiResponseAttrs } from "./llm-enrich.js";
2763
+ export { NATIVELY_COVERED, NATIVE_PROPAGATES_EXTERNAL, shouldEnrichNatively, wrapSqlClient };
2764
+ export { matchRoute } from "./workers-routes.js";