@heystack/otel 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/workers.js CHANGED
@@ -16,8 +16,9 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
16
16
  import { buildExporterConfig, DEFAULT_ENDPOINT, releaseResourceAttributes, } from "./core.js";
17
17
  import { isSelfSpanAttrs, safeHostname } from "./self-span.js";
18
18
  import { llmRequestAttrs, llmResponseAttrs, llmContentAttrs, llmOperationName, resolveLLMProvider, scanSseStreamForUsage, } from "./llm-enrich.js";
19
- import { instrumentEnv } from "./workers-bindings.js";
19
+ import { instrumentEnv, makeProxy, runWithSpan } from "./workers-bindings.js";
20
20
  import { makeSampler, loadRemoteSamplingRate } from "./workers-sampler.js";
21
+ import { makeTraceparent } from "./traceparent.js";
21
22
  // `ExportResult` / `ExportResultCode` mirror `@opentelemetry/core`. We define
22
23
  // them inline (structurally identical) rather than import them: core is only a
23
24
  // transitive dep of sdk-trace-base and isn't reliably resolvable, and keeping it
@@ -412,7 +413,7 @@ function nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch
412
413
  }
413
414
  return;
414
415
  }
415
- const adapter = new NativeSpanAdapter(nativeSpan, true);
416
+ const adapter = new NativeSpanAdapter(nativeSpan);
416
417
  const { attrs, parsedReqBody } = llmRequestSide(resolved, url, init);
417
418
  adapter.setAttributes({ ...attrs, "server.address": url.hostname });
418
419
  try {
@@ -511,7 +512,7 @@ function ensureFetchInstrumentation(ingestHost, aiCfg = {}) {
511
512
  },
512
513
  }, active);
513
514
  const sc = span.spanContext();
514
- const traceparent = `00-${sc.traceId}-${sc.spanId}-01`;
515
+ const traceparent = makeTraceparent(sc);
515
516
  const [reqInput, reqInit] = injectTraceparent(input, init, traceparent);
516
517
  // gen_ai enrichment — request side (direct host or configured gateway).
517
518
  const url = parseTarget(input);
@@ -571,6 +572,85 @@ export function __resetFetchInstrumentation() {
571
572
  // ---------------------------------------------------------------------------
572
573
  // Exporter
573
574
  // ---------------------------------------------------------------------------
575
+ /**
576
+ * Bounded in-export resilience: ONE automatic retry for network errors /
577
+ * 429 / 5xx. The backoff starts at {@link RETRY_BASE_DELAY_MS} with jitter and
578
+ * the total added wait across retries never exceeds {@link RETRY_TOTAL_CAP_MS}
579
+ * (~2s) so an ingest blip can't pin a request handler; a `Retry-After` header
580
+ * (seconds form) on 429 refines the delay within that same cap.
581
+ */
582
+ const EXPORT_MAX_ATTEMPTS = 2;
583
+ const RETRY_BASE_DELAY_MS = 250;
584
+ const RETRY_TOTAL_CAP_MS = 2_000;
585
+ /** Upper bound on concurrently-tracked in-flight exports (`pending` set). */
586
+ const MAX_PENDING_EXPORTS = 512;
587
+ function sleep(ms) {
588
+ return new Promise((resolve) => setTimeout(resolve, ms));
589
+ }
590
+ /** Parse the seconds form of `Retry-After`; HTTP-date form returns undefined. */
591
+ function parseRetryAfterSeconds(header) {
592
+ if (!header)
593
+ return undefined;
594
+ const s = Number.parseInt(header.trim(), 10);
595
+ return Number.isFinite(s) && s > 0 ? s : undefined;
596
+ }
597
+ /**
598
+ * Backoff for the single retry: honour `Retry-After` when present (429),
599
+ * otherwise base delay + jitter; always clamped into the remaining total cap.
600
+ */
601
+ function nextRetryDelayMs(remainingBudgetMs, retryAfterSeconds) {
602
+ const hinted = retryAfterSeconds !== undefined && retryAfterSeconds > 0
603
+ ? retryAfterSeconds * 1000
604
+ : RETRY_BASE_DELAY_MS + Math.random() * RETRY_BASE_DELAY_MS;
605
+ return Math.max(0, Math.min(hinted, remainingBudgetMs));
606
+ }
607
+ /**
608
+ * One-per-isolate advisory when exports complete with NEITHER a `waitUntil`
609
+ * handoff NOR an OpenNext context borrow available — i.e. delivery fell back
610
+ * to the pending set, where traces are lost unless someone drains explicitly.
611
+ */
612
+ let warnedNoHandoff = false;
613
+ let manualFlushObserved = false;
614
+ let requestDrainWired = false;
615
+ function warnIfNoHandoffOnce() {
616
+ if (manualFlushObserved || requestDrainWired || warnedNoHandoff)
617
+ return;
618
+ warnedNoHandoff = true;
619
+ console.warn("[heystack] trace exports have no waitUntil handoff (no ExecutionContext wiring detected). " +
620
+ "Wrap your Worker's ctx with wrapExecutionContext(ctx), or use instrument(), so exports are " +
621
+ "drained before isolate teardown; otherwise traces rely on flushHeystack() and may be lost.");
622
+ }
623
+ /** Record an explicit flushHeystack() so the no-handoff advisory stays quiet. */
624
+ function markManualFlushObserved() {
625
+ manualFlushObserved = true;
626
+ }
627
+ /**
628
+ * Drain BOTH the provider's span processor AND the exporter's in-flight POSTs.
629
+ * Awaiting only provider.forceFlush() would return before the export POST
630
+ * completes, letting the isolate be torn down and silently dropping the trace.
631
+ */
632
+ async function drainProvider(provider) {
633
+ await provider.forceFlush().catch(() => { });
634
+ await provider.heystackExporter.forceFlush().catch(() => { });
635
+ }
636
+ /**
637
+ * The drain raced against an ~8s timeout so a hung forceFlush (e.g. an ingest
638
+ * that never responds) can't pin the isolate until the platform CPU timeout
639
+ * kills it (#25). On timeout we stop waiting; the POST is still in-flight and
640
+ * may yet complete, we just don't block the isolate on it indefinitely.
641
+ */
642
+ function drainWithTimeout(provider) {
643
+ let timer;
644
+ const timeout = new Promise((resolve) => {
645
+ timer = setTimeout(resolve, DRAIN_TIMEOUT_MS);
646
+ });
647
+ return Promise.race([drainProvider(provider), timeout]).finally(() => {
648
+ if (timer)
649
+ clearTimeout(timer);
650
+ });
651
+ }
652
+ /** Upper bound on one drain race — shared by instrument() and wrapExecutionContext. */
653
+ const DRAIN_TIMEOUT_MS = 8_000;
574
654
  /**
575
655
  * A WinterCG-compatible OTLP/JSON span exporter. POSTs ended spans to the
576
656
  * Heystack ingest using the platform `fetch` — no Node built-ins.
@@ -581,6 +661,8 @@ export class HeystackSpanExporter {
581
661
  ingestHost;
582
662
  headers;
583
663
  shutdownState = false;
664
+ /** Exports rejected before network work because the in-flight cap was hit. */
665
+ droppedPending = 0;
584
666
  /**
585
667
  * In-flight export fetches. Each `export()` adds its settled-when-the-POST-
586
668
  * completes promise here and removes it on `finally`. `forceFlush()` awaits
@@ -629,47 +711,45 @@ export class HeystackSpanExporter {
629
711
  return;
630
712
  }
631
713
  const body = JSON.stringify(serializeSpans(exportable));
632
- // Build the fetch chain as a promise we retain, so forceFlush() can await
633
- // the actual network write. It resolves (never rejects) once the POST has
634
- // completed (success or fail) and resultCallback has been invoked.
714
+ // Build the export chain as a promise we retain, so forceFlush() can await
715
+ // the actual network write — INCLUDING the bounded retry/backoff, so a
716
+ // waitUntil drain keeps the isolate alive until retries settle too. It
717
+ // resolves (never rejects) once the POST attempts are done (success or
718
+ // final failure) and resultCallback has been invoked.
635
719
  //
636
- // The POST runs inside a tracing-suppressed context so that host fetch
637
- // auto-instrumentation (e.g. Next/OpenNext) does NOT create a CLIENT span
638
- // for it — which would otherwise be exported and re-captured, a sustained
639
- // feedback loop. As a belt-and-suspenders second layer it also uses the
640
- // CAPTURED platform fetch (when our outbound-fetch wrapper is installed), so
641
- // the export can never be re-entered by that wrapper regardless of context.
642
- const doFetch = _originalFetch ?? fetch;
643
- const p = context
644
- .with(suppressTracing(context.active()), () => doFetch(this.url, { method: "POST", headers: this.headers, body }))
645
- .then((res) => {
646
- if (res.ok) {
647
- resultCallback({ code: ExportResultCode.SUCCESS });
648
- }
649
- else {
650
- resultCallback({
651
- code: ExportResultCode.FAILED,
652
- error: new Error(`Heystack ingest responded ${res.status} ${res.statusText}`),
653
- });
654
- }
655
- })
656
- .catch((error) => {
720
+ // Each POST attempt runs inside a tracing-suppressed context so that host
721
+ // fetch auto-instrumentation (e.g. Next/OpenNext) does NOT create a CLIENT
722
+ // span for it — which would otherwise be exported and re-captured, a
723
+ // sustained feedback loop. As a belt-and-suspenders second layer it also
724
+ // uses the CAPTURED platform fetch (when our outbound-fetch wrapper is
725
+ // installed), so the export can never be re-entered by that wrapper.
726
+ const reservation = this.reservePending();
727
+ if (!reservation) {
728
+ this.droppedPending++;
657
729
  resultCallback({
658
730
  code: ExportResultCode.FAILED,
659
- error: error instanceof Error ? error : new Error(String(error)),
731
+ error: new Error(`Heystack export dropped: ${MAX_PENDING_EXPORTS} exports are already in flight`),
660
732
  });
661
- });
662
- this.pending.add(p);
663
- p.finally(() => this.pending.delete(p));
733
+ return;
734
+ }
735
+ const doFetch = _originalFetch ?? fetch;
736
+ const post = this.postWithRetry(doFetch, body, resultCallback);
737
+ // postWithRetry should not reject; handle both branches so a future change
738
+ // cannot leak the reserved slot or create an unhandled rejection.
739
+ void post.then(reservation.release, reservation.release);
740
+ const p = reservation.pending;
664
741
  // Keep the isolate alive until the POST completes. Priority:
665
742
  // 1. an explicit `waitUntil` override (set via initHeystackWorkers).
666
743
  // 2. the OpenNext Cloudflare request context's `ctx.waitUntil` — the
667
744
  // export runs during request handling, so this is available there.
668
745
  // Either path makes delivery reliable on workerd/OpenNext with no app hook.
669
- // The `pending` set + `flushHeystack()` remain as the explicit fallback.
746
+ // The `pending` set + `flushHeystack()` remain as the explicit fallback;
747
+ // when neither handoff lands we say so once per isolate (see below).
748
+ const handoff = { borrowed: false };
670
749
  if (this.waitUntil) {
671
750
  try {
672
751
  this.waitUntil(p);
752
+ handoff.borrowed = true;
673
753
  }
674
754
  catch (error) {
675
755
  this.warnWaitUntilFailed("explicit waitUntil", error);
@@ -681,21 +761,88 @@ export class HeystackSpanExporter {
681
761
  // available; otherwise await the in-flight accessor load so even those
682
762
  // early exports get handed to waitUntil once it resolves. `p` is already
683
763
  // in `pending`, so a manual `flushHeystack()` covers it regardless.
684
- this.handOffToCloudflareContext(p);
764
+ this.handOffToCloudflareContext(p, handoff);
685
765
  }
766
+ if (!handoff.borrowed) {
767
+ // Fallback pending-set mode: once THIS export settles with no handoff,
768
+ // raise the one-per-isolate advisory (suppressed for manual-flush users).
769
+ void p.then(() => warnIfNoHandoffOnce());
770
+ }
771
+ }
772
+ /**
773
+ * POST the batch with ONE bounded automatic retry for network errors / 429 /
774
+ * 5xx. Honours `Retry-After` (seconds form) on 429 within the total backoff
775
+ * cap; after retries are exhausted it preserves plain FAILED semantics via
776
+ * `resultCallback`. The returned promise never rejects.
777
+ */
778
+ postWithRetry(doFetch, body, resultCallback) {
779
+ let budgetLeft = RETRY_TOTAL_CAP_MS;
780
+ let retryAfterSeconds;
781
+ return (async () => {
782
+ let lastError;
783
+ for (let attempt = 0; attempt < EXPORT_MAX_ATTEMPTS; attempt++) {
784
+ if (attempt > 0) {
785
+ const delay = nextRetryDelayMs(budgetLeft, retryAfterSeconds);
786
+ budgetLeft -= delay;
787
+ await sleep(delay);
788
+ }
789
+ let retryable = false;
790
+ try {
791
+ const res = await context.with(suppressTracing(context.active()), () => doFetch(this.url, { method: "POST", headers: this.headers, body }));
792
+ if (res.ok) {
793
+ resultCallback({ code: ExportResultCode.SUCCESS });
794
+ return;
795
+ }
796
+ lastError = new Error(`Heystack ingest responded ${res.status} ${res.statusText}`);
797
+ retryable = res.status === 429 || res.status >= 500;
798
+ if (res.status === 429) {
799
+ retryAfterSeconds = parseRetryAfterSeconds(res.headers.get("retry-after"));
800
+ }
801
+ }
802
+ catch (error) {
803
+ lastError = error instanceof Error ? error : new Error(String(error));
804
+ retryable = true;
805
+ }
806
+ if (!retryable)
807
+ break;
808
+ }
809
+ resultCallback({
810
+ code: ExportResultCode.FAILED,
811
+ error: lastError ?? new Error("Heystack export failed"),
812
+ });
813
+ })();
814
+ }
815
+ /** Reserve a tracked slot synchronously before starting any network work. */
816
+ reservePending() {
817
+ if (this.pending.size >= MAX_PENDING_EXPORTS)
818
+ return undefined;
819
+ let release;
820
+ const pending = new Promise((resolve) => {
821
+ release = resolve;
822
+ });
823
+ this.pending.add(pending);
824
+ void pending.then(() => {
825
+ this.pending.delete(pending);
826
+ });
827
+ return { pending, release };
686
828
  }
687
829
  /**
688
830
  * Hand `p` to the OpenNext Cloudflare request context's `ctx.waitUntil`.
689
831
  * Resolves the accessor first (awaiting its in-flight load if needed) so an
690
832
  * export that fired before the dynamic import settled is still covered.
833
+ * Sets `outcome.borrowed` when (and only when) a real `waitUntil` received
834
+ * `p` — the signal the no-handoff advisory keys off.
691
835
  */
692
- handOffToCloudflareContext(p) {
836
+ handOffToCloudflareContext(p, outcome) {
693
837
  const attempt = () => {
694
838
  if (!_getCloudflareContext)
695
839
  return;
696
840
  try {
697
841
  const cf = _getCloudflareContext();
698
- cf?.ctx?.waitUntil?.(p);
842
+ if (cf?.ctx?.waitUntil) {
843
+ cf.ctx.waitUntil(p);
844
+ outcome.borrowed = true;
845
+ }
699
846
  }
700
847
  catch (error) {
701
848
  // #15: a real failure inside getCloudflareContext (e.g. called outside a
@@ -726,9 +873,9 @@ export class HeystackSpanExporter {
726
873
  "unless you call flushHeystack() or pass an explicit waitUntil.");
727
874
  }
728
875
  shutdown() {
729
- return this.forceFlush().then(() => {
730
- this.shutdownState = true;
731
- });
876
+ // Reject new work immediately, then drain everything admitted beforehand.
877
+ this.shutdownState = true;
878
+ return this.forceFlush();
732
879
  }
733
880
  /**
734
881
  * Resolve only once every in-flight export fetch has settled. This is the
@@ -737,7 +884,11 @@ export class HeystackSpanExporter {
737
884
  * `flushHeystack` and `instrument`).
738
885
  */
739
886
  async forceFlush() {
740
- await Promise.allSettled([...this.pending]);
887
+ // Exports can start from another callback while an earlier snapshot settles.
888
+ // Continue until the tracked set is actually quiescent.
889
+ while (this.pending.size > 0) {
890
+ await Promise.allSettled([...this.pending]);
891
+ }
741
892
  }
742
893
  }
743
894
  function nativeTracingFromContext() {
@@ -750,10 +901,8 @@ function nativeTracingFromContext() {
750
901
  */
751
902
  class NativeSpanAdapter {
752
903
  nativeSpan;
753
- autoEnd;
754
- constructor(nativeSpan, autoEnd) {
904
+ constructor(nativeSpan) {
755
905
  this.nativeSpan = nativeSpan;
756
- this.autoEnd = autoEnd;
757
906
  }
758
907
  spanContext() { return INVALID_SPAN_CONTEXT; }
759
908
  setAttribute(key, value) {
@@ -804,10 +953,7 @@ class NativeSpanAdapter {
804
953
  return this;
805
954
  }
806
955
  updateName(_name) { return this; }
807
- end(_endTime) {
808
- if (!this.autoEnd)
809
- this.nativeSpan.end();
810
- }
956
+ end(_endTime) { }
811
957
  isRecording() { return this.nativeSpan.isTraced; }
812
958
  recordException(exception, _time) {
813
959
  const error = exception instanceof Error ? exception : new Error(String(exception));
@@ -842,7 +988,7 @@ function setNativeRequestAttributes(span, config, req) {
842
988
  async function runNativeSpan(nativeTracing, name, attrs, fn) {
843
989
  ensureContextManager();
844
990
  return nativeTracing.enterSpan(name, async (nativeSpan) => {
845
- const adapter = new NativeSpanAdapter(nativeSpan, true);
991
+ const adapter = new NativeSpanAdapter(nativeSpan);
846
992
  for (const [key, value] of Object.entries(attrs)) {
847
993
  if (value !== undefined)
848
994
  adapter.setAttribute(key, value);
@@ -899,9 +1045,9 @@ function getALS() {
899
1045
  * context across `await` boundaries — child spans created after an `await`
900
1046
  * inside a handler are correctly parented to the request's root span.
901
1047
  *
902
- * WinterCG-safe: it does NOT import `node:async_hooks`; instead it uses the
903
- * ALS global that the runtime exposes. Falls back to `SyncStackContextManager`
904
- * when the global is absent (Deno, Bun without globals, etc.).
1048
+ * WinterCG-safe: it does NOT import the Node-only async_hooks builtin; instead
1049
+ * it uses the ALS global that the runtime exposes. Falls back to
1050
+ * `SyncStackContextManager` when the global is absent (Deno, Bun without globals, etc.).
905
1051
  */
906
1052
  export class AlsContextManager {
907
1053
  _als;
@@ -1022,10 +1168,12 @@ function ensureGlobalProvider(config) {
1022
1168
  * FLUSH: on Next-on-OpenNext (where `@opennextjs/cloudflare` is present) the
1023
1169
  * exporter automatically borrows the Cloudflare request context's
1024
1170
  * `ctx.waitUntil` so the export POST completes before the isolate is torn down
1025
- * — no app hook needed. For other workerd setups without it, call
1026
- * `flushHeystack()` (which awaits the export fetch) from a response hook, hand
1027
- * it to `ctx.waitUntil(flushHeystack())`, or pass an explicit `waitUntil` in
1028
- * the config (highest priority). Returns the provider.
1171
+ * — no app hook needed. For other workerd setups without it, wrap your
1172
+ * handler's ExecutionContext with {@link wrapExecutionContext} (every
1173
+ * `ctx.waitUntil` then also schedules the export drain), call `flushHeystack()`
1174
+ * (which awaits the export fetch) from a response hook, hand it to
1175
+ * `ctx.waitUntil(flushHeystack())`, or pass an explicit `waitUntil` in the
1176
+ * config (highest priority). Returns the provider.
1029
1177
  */
1030
1178
  export function initHeystackWorkers(config) {
1031
1179
  return ensureGlobalProvider(config);
@@ -1039,6 +1187,7 @@ export function initHeystackWorkers(config) {
1039
1187
  * isolate is torn down. Best-effort no-op if no provider is registered.
1040
1188
  */
1041
1189
  export async function flushHeystack() {
1190
+ markManualFlushObserved();
1042
1191
  if (!_provider)
1043
1192
  return;
1044
1193
  if (typeof _provider.forceFlush === "function") {
@@ -1048,10 +1197,62 @@ export async function flushHeystack() {
1048
1197
  // drain the exporter's in-flight POSTs explicitly.
1049
1198
  await _provider.heystackExporter.forceFlush();
1050
1199
  }
1200
+ /**
1201
+ * Wrap a Worker `ExecutionContext` so EVERY `ctx.waitUntil(promise)` ALSO
1202
+ * schedules the exporter drain race after that promise settles (provider +
1203
+ * in-flight POSTs, raced against an 8s timeout so a hung ingest can't pin the
1204
+ * isolate). Use on
1205
+ * `initHeystackWorkers()` / `createTracerProvider()` setups that don't go
1206
+ * through `instrument()` (which wires the same drain internally):
1207
+ *
1208
+ * export default {
1209
+ * async fetch(req, env, ctx) {
1210
+ * ctx = wrapExecutionContext(ctx);
1211
+ * return handle(req, env, ctx); // any waitUntil you do flushes exports
1212
+ * },
1213
+ * };
1214
+ *
1215
+ * Every other present or future ExecutionContext member is forwarded unchanged
1216
+ * with the real host object as its receiver. Safe to call when tracing isn't
1217
+ * initialised — the drain scheduling simply no-ops.
1218
+ */
1219
+ function wrapExecutionContextWithProvider(ctx, configuredProvider) {
1220
+ const waitUntil = (promise) => {
1221
+ ctx.waitUntil(promise);
1222
+ const provider = configuredProvider ?? _provider;
1223
+ if (!provider)
1224
+ return;
1225
+ // Background work can create spans after awaits. Start its drain only once
1226
+ // that work settles; a concurrent drain can finish before those spans exist.
1227
+ ctx.waitUntil(promise
1228
+ .catch(() => undefined)
1229
+ .then(() => drainWithTimeout(provider)));
1230
+ };
1231
+ const passThroughOnException = () => {
1232
+ ctx.passThroughOnException();
1233
+ };
1234
+ return new Proxy(ctx, {
1235
+ get(target, property) {
1236
+ if (property === "waitUntil")
1237
+ return waitUntil;
1238
+ if (property === "passThroughOnException")
1239
+ return passThroughOnException;
1240
+ const value = Reflect.get(target, property, target);
1241
+ return typeof value === "function" ? value.bind(target) : value;
1242
+ },
1243
+ set(target, property, value) {
1244
+ return Reflect.set(target, property, value, target);
1245
+ },
1246
+ });
1247
+ }
1248
+ export function wrapExecutionContext(ctx) {
1249
+ return wrapExecutionContextWithProvider(ctx);
1250
+ }
1051
1251
  /** Reset the singleton global provider. Internal/testing helper. */
1052
1252
  export function __resetProvider() {
1053
1253
  _provider = null;
1054
1254
  _remoteSamplingKicked = false;
1255
+ requestDrainWired = false;
1055
1256
  }
1056
1257
  /** Guard: the remote sampling config GET fires at most once per isolate. */
1057
1258
  let _remoteSamplingKicked = false;
@@ -1062,9 +1263,11 @@ function warnOnceNoKey() {
1062
1263
  warnedNoKey = true;
1063
1264
  console.warn("[heystack] HEYSTACK_API_KEY not set — Workers tracing disabled, handler runs untraced.");
1064
1265
  }
1065
- /** Reset the once-only no-key warning. Internal/testing helper. */
1266
+ /** Reset the once-only warnings. Internal/testing helper. */
1066
1267
  export function __resetWarnings() {
1067
1268
  warnedNoKey = false;
1269
+ warnedNoHandoff = false;
1270
+ manualFlushObserved = false;
1068
1271
  }
1069
1272
  export function instrument(handler, config) {
1070
1273
  // Resolve the API key + set up (once) the global provider. Returns null when
@@ -1088,31 +1291,15 @@ export function instrument(handler, config) {
1088
1291
  sampling: config.sampling,
1089
1292
  ai: config.ai,
1090
1293
  });
1294
+ // instrument() wires a per-request drain (see `drain` below), so exports on
1295
+ // this path are reliably flushed — the no-handoff advisory must stay quiet.
1296
+ requestDrainWired = true;
1091
1297
  return { provider, tracer: trace.getTracer("heystack") };
1092
1298
  };
1093
1299
  // Drain BOTH the provider's span processor AND the exporter's in-flight fetch
1094
- // via ctx.waitUntil. Awaiting only provider.forceFlush() would return before
1095
- // the export POST completes, letting the isolate be torn down and silently
1096
- // dropping the trace.
1097
- //
1098
- // #25: the drain is raced against an ~8s timeout so a hung forceFlush (e.g. an
1099
- // ingest that never responds) can't pin the isolate until the platform CPU
1100
- // timeout kills it. On timeout we stop waiting; the POST is still in-flight and
1101
- // may yet complete, we just don't block the isolate on it indefinitely.
1102
- const DRAIN_TIMEOUT_MS = 8_000;
1300
+ // via ctx.waitUntil, raced against an ~8s timeout (see drainWithTimeout, #25).
1103
1301
  const drain = (provider, ctx) => {
1104
- const drained = (async () => {
1105
- await provider.forceFlush().catch(() => { });
1106
- await provider.heystackExporter.forceFlush().catch(() => { });
1107
- })();
1108
- let timer;
1109
- const timeout = new Promise((resolve) => {
1110
- timer = setTimeout(resolve, DRAIN_TIMEOUT_MS);
1111
- });
1112
- ctx.waitUntil(Promise.race([drained, timeout]).finally(() => {
1113
- if (timer)
1114
- clearTimeout(timer);
1115
- }));
1302
+ ctx.waitUntil(drainWithTimeout(provider));
1116
1303
  };
1117
1304
  // Start from a shallow copy so EVERY sibling handler (tail, email, …) is
1118
1305
  // forwarded untouched; we only override fetch/queue/scheduled below.
@@ -1207,7 +1394,7 @@ export function instrument(handler, config) {
1207
1394
  }
1208
1395
  // FR5: response header carrying THIS span's trace + span id.
1209
1396
  const sc = span.spanContext();
1210
- const traceparent = `00-${sc.traceId}-${sc.spanId}-01`;
1397
+ const traceparent = makeTraceparent(sc);
1211
1398
  // The context with THIS request's SERVER span active. Used both to run the
1212
1399
  // handler and as an explicit parent fallback for binding spans.
1213
1400
  const serverCtx = trace.setSpan(startCtx, span);
@@ -1235,8 +1422,64 @@ export function instrument(handler, config) {
1235
1422
  select: config.instrumentBindings,
1236
1423
  });
1237
1424
  }
1425
+ let finished = false;
1426
+ const abortSignal = req.signal;
1427
+ let responseStatus;
1428
+ const finishSpan = (outcome, error) => {
1429
+ if (finished)
1430
+ return;
1431
+ finished = true;
1432
+ abortSignal?.removeEventListener("abort", onAbort);
1433
+ try {
1434
+ if (outcome === "aborted") {
1435
+ span.setAttribute("heystack.aborted", true);
1436
+ span.setStatus({
1437
+ code: SpanStatusCode.ERROR,
1438
+ message: "client stopped reading before the response completed",
1439
+ });
1440
+ }
1441
+ else if (outcome === "error") {
1442
+ const recorded = error instanceof Error ? error : new Error(String(error));
1443
+ span.recordException(recorded);
1444
+ span.setStatus({ code: SpanStatusCode.ERROR, message: recorded.message });
1445
+ }
1446
+ else {
1447
+ span.setStatus({
1448
+ code: responseStatus !== undefined && (responseStatus === 0 || responseStatus >= 500)
1449
+ ? SpanStatusCode.ERROR
1450
+ : SpanStatusCode.UNSET,
1451
+ });
1452
+ }
1453
+ span.end();
1454
+ }
1455
+ catch {
1456
+ // Telemetry must not change response delivery or error propagation.
1457
+ }
1458
+ try {
1459
+ drain(provider, ctx);
1460
+ }
1461
+ catch {
1462
+ // A rejected host waitUntil handoff must not break the response stream.
1463
+ }
1464
+ };
1465
+ const onAbort = () => finishSpan("aborted");
1238
1466
  try {
1239
- const response = await context.with(serverCtx, () => originalFetch(req, handlerEnv, ctx));
1467
+ const response = await context.with(serverCtx, () => originalFetch(req, handlerEnv, wrapExecutionContextWithProvider(ctx, provider)));
1468
+ responseStatus = response.status;
1469
+ if (abortSignal && !abortSignal.aborted) {
1470
+ abortSignal.addEventListener("abort", onAbort, { once: true });
1471
+ }
1472
+ else if (abortSignal?.aborted) {
1473
+ finishSpan("aborted");
1474
+ }
1475
+ // Response.error() has status 0 and immutable network-error semantics.
1476
+ // Reconstructing it is invalid (ResponseInit only accepts 200-599), and
1477
+ // there is no response header surface on which to inject traceparent.
1478
+ if (response.status === 0) {
1479
+ span.setAttribute("http.response.status_code", response.status);
1480
+ finishSpan("success");
1481
+ return response;
1482
+ }
1240
1483
  // WebSocket / connection upgrades: a 101 Switching Protocols response
1241
1484
  // carries a live server-side `webSocket` that the Workers Runtime hands
1242
1485
  // to the client ONLY if that exact Response object is returned. Rebuilding
@@ -1248,64 +1491,66 @@ export function instrument(handler, config) {
1248
1491
  // signal; `status === 101` is a belt-and-braces fallback.
1249
1492
  if (response.webSocket || response.status === 101) {
1250
1493
  span.setAttribute("http.response.status_code", response.status);
1251
- span.setStatus({ code: SpanStatusCode.UNSET });
1252
- span.end();
1253
- drain(provider, ctx);
1494
+ finishSpan("success");
1254
1495
  return response;
1255
1496
  }
1256
1497
  const headers = new Headers(response.headers);
1257
1498
  headers.set("traceparent", traceparent);
1258
1499
  appendExposeHeader(headers, "traceparent");
1259
1500
  span.setAttribute("http.response.status_code", response.status);
1260
- const finalize = () => {
1261
- span.setStatus({
1262
- code: response.status >= 500 ? SpanStatusCode.ERROR : SpanStatusCode.UNSET,
1263
- });
1264
- span.end();
1265
- drain(provider, ctx);
1266
- };
1267
1501
  // No body to stream → finalize now (redirects, 204/304, etc.).
1268
1502
  if (!response.body) {
1269
- finalize();
1503
+ finishSpan("success");
1270
1504
  return new Response(null, {
1271
1505
  status: response.status,
1272
1506
  statusText: response.statusText,
1273
1507
  headers,
1274
1508
  });
1275
1509
  }
1276
- // FR1: keep the span open until the streamed body drains; the first
1277
- // chunk records time-to-first-byte. `finished` guards double-finalize.
1510
+ // FR1: pull one chunk at a time from the original body. This preserves
1511
+ // streaming/backpressure without buffering, while observing all three
1512
+ // terminal paths: clean EOF, source error, and consumer cancellation.
1278
1513
  let firstByte = false;
1279
- let finished = false;
1280
- const monitor = new TransformStream({
1281
- transform(chunk, controller) {
1282
- if (!firstByte) {
1283
- firstByte = true;
1284
- span.addEvent("first_byte");
1514
+ let consumerCancelled = false;
1515
+ const reader = response.body.getReader();
1516
+ const monitoredBody = new ReadableStream({
1517
+ async pull(controller) {
1518
+ try {
1519
+ const result = await reader.read();
1520
+ if (consumerCancelled)
1521
+ return;
1522
+ if (result.done) {
1523
+ finishSpan("success");
1524
+ controller.close();
1525
+ return;
1526
+ }
1527
+ if (!firstByte) {
1528
+ firstByte = true;
1529
+ span.addEvent("first_byte");
1530
+ }
1531
+ controller.enqueue(result.value);
1532
+ }
1533
+ catch (error) {
1534
+ if (consumerCancelled)
1535
+ return;
1536
+ finishSpan("error", error);
1537
+ controller.error(error);
1285
1538
  }
1286
- controller.enqueue(chunk);
1287
1539
  },
1288
- flush() {
1289
- if (finished)
1290
- return;
1291
- finished = true;
1292
- finalize();
1540
+ cancel(reason) {
1541
+ consumerCancelled = true;
1542
+ finishSpan("aborted", reason);
1543
+ return reader.cancel(reason);
1293
1544
  },
1294
1545
  });
1295
- return new Response(response.body.pipeThrough(monitor), {
1546
+ return new Response(monitoredBody, {
1296
1547
  status: response.status,
1297
1548
  statusText: response.statusText,
1298
1549
  headers,
1299
1550
  });
1300
1551
  }
1301
1552
  catch (error) {
1302
- span.recordException(error instanceof Error ? error : new Error(String(error)));
1303
- span.setStatus({
1304
- code: SpanStatusCode.ERROR,
1305
- message: error instanceof Error ? error.message : String(error),
1306
- });
1307
- span.end();
1308
- drain(provider, ctx);
1553
+ finishSpan("error", error);
1309
1554
  throw error;
1310
1555
  }
1311
1556
  };
@@ -1316,6 +1561,8 @@ export function instrument(handler, config) {
1316
1561
  const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1317
1562
  if (nativeTracing) {
1318
1563
  ensureFetchInstrumentation("", config.ai ?? {});
1564
+ // Native mode: Cloudflare owns queue spans — pass the ORIGINAL batch
1565
+ // through untouched so platform objects keep their identity.
1319
1566
  return runNativeSpan(nativeTracing, `heystack.queue ${batch.queue}`, {
1320
1567
  "service.name": config.service,
1321
1568
  "messaging.batch.message_count": batch.messages.length,
@@ -1336,8 +1583,24 @@ export function instrument(handler, config) {
1336
1583
  "messaging.destination.name": batch.queue,
1337
1584
  },
1338
1585
  });
1586
+ // The request's CONSUMER context, captured per-batch as an explicit parent
1587
+ // fallback (same rationale as the binding-span serverCtx below): on
1588
+ // workerd WITHOUT nodejs_compat a message ack'd after an `await` would
1589
+ // otherwise orphan into its own single-span trace.
1590
+ const consumerCtx = trace.setSpan(context.active(), span);
1591
+ const startMessageSpan = (name, attrs) => {
1592
+ const active = context.active();
1593
+ const parentCtx = trace.getSpan(active) ? active : consumerCtx;
1594
+ return tracer.startSpan(name, { attributes: attrs }, parentCtx);
1595
+ };
1596
+ // Per-message visibility: the wrapper hands the WHOLE batch to user code
1597
+ // in one call (splitting it would change retry/ack semantics), so the true
1598
+ // per-message unit of work observable here is the explicit
1599
+ // `msg.ack()` / `msg.retry()` decision — each gets a cheap INTERNAL child
1600
+ // span when the consumer uses per-message control flow.
1601
+ const tracedBatch = wrapQueueMessageBatch(batch, startMessageSpan);
1339
1602
  try {
1340
- await context.with(trace.setSpan(context.active(), span), () => originalQueue(batch, env, ctx));
1603
+ await context.with(consumerCtx, () => originalQueue(tracedBatch, env, wrapExecutionContextWithProvider(ctx, provider)));
1341
1604
  span.setStatus({ code: SpanStatusCode.UNSET });
1342
1605
  span.end();
1343
1606
  drain(provider, ctx);
@@ -1379,7 +1642,90 @@ export function instrument(handler, config) {
1379
1642
  },
1380
1643
  });
1381
1644
  try {
1382
- await context.with(trace.setSpan(context.active(), span), () => originalScheduled(controller, env, ctx));
1645
+ await context.with(trace.setSpan(context.active(), span), () => originalScheduled(controller, env, wrapExecutionContextWithProvider(ctx, provider)));
1646
+ span.setStatus({ code: SpanStatusCode.UNSET });
1647
+ span.end();
1648
+ drain(provider, ctx);
1649
+ }
1650
+ catch (error) {
1651
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
1652
+ span.setStatus({
1653
+ code: SpanStatusCode.ERROR,
1654
+ message: error instanceof Error ? error.message : String(error),
1655
+ });
1656
+ span.end();
1657
+ drain(provider, ctx);
1658
+ throw error;
1659
+ }
1660
+ };
1661
+ }
1662
+ const originalEmail = handler.email?.bind(handler);
1663
+ if (originalEmail) {
1664
+ wrapped.email = async (message, env, ctx) => {
1665
+ const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1666
+ // PRIVACY: only from/to DOMAINS + envelope size are recorded — never the
1667
+ // full addresses and never any message content.
1668
+ const { name: emailSpanName, attrs: emailAttrs } = emailHandlerAttrs(message);
1669
+ if (nativeTracing) {
1670
+ return runNativeSpan(nativeTracing, `heystack.${emailSpanName}`, {
1671
+ "service.name": config.service,
1672
+ ...emailAttrs,
1673
+ "service.version": config.version,
1674
+ "service.build": config.build,
1675
+ }, () => originalEmail(message, env, ctx));
1676
+ }
1677
+ const s = setup(env);
1678
+ if (!s)
1679
+ return originalEmail(message, env, ctx);
1680
+ const { provider, tracer } = s;
1681
+ const span = tracer.startSpan(emailSpanName, {
1682
+ kind: SpanKind.SERVER,
1683
+ attributes: emailAttrs,
1684
+ });
1685
+ try {
1686
+ await context.with(trace.setSpan(context.active(), span), () => originalEmail(message, env, wrapExecutionContextWithProvider(ctx, provider)));
1687
+ span.setStatus({ code: SpanStatusCode.UNSET });
1688
+ span.end();
1689
+ drain(provider, ctx);
1690
+ }
1691
+ catch (error) {
1692
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
1693
+ span.setStatus({
1694
+ code: SpanStatusCode.ERROR,
1695
+ message: error instanceof Error ? error.message : String(error),
1696
+ });
1697
+ span.end();
1698
+ drain(provider, ctx);
1699
+ throw error;
1700
+ }
1701
+ };
1702
+ }
1703
+ const originalTail = handler.tail?.bind(handler);
1704
+ if (originalTail) {
1705
+ wrapped.tail = async (events, env, ctx) => {
1706
+ const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
1707
+ // PRIVACY: only the event COUNT is recorded — tail payloads can contain
1708
+ // request URLs, headers and console output; none of it is captured.
1709
+ const count = Array.isArray(events) ? events.length : 0;
1710
+ const attrs = { "cloudflare.tail.event_count": count };
1711
+ if (nativeTracing) {
1712
+ return runNativeSpan(nativeTracing, "heystack.tail", {
1713
+ "service.name": config.service,
1714
+ ...attrs,
1715
+ "service.version": config.version,
1716
+ "service.build": config.build,
1717
+ }, () => originalTail(events, env, ctx));
1718
+ }
1719
+ const s = setup(env);
1720
+ if (!s)
1721
+ return originalTail(events, env, ctx);
1722
+ const { provider, tracer } = s;
1723
+ const span = tracer.startSpan("tail", {
1724
+ kind: SpanKind.INTERNAL,
1725
+ attributes: attrs,
1726
+ });
1727
+ try {
1728
+ await context.with(trace.setSpan(context.active(), span), () => originalTail(events, env, wrapExecutionContextWithProvider(ctx, provider)));
1383
1729
  span.setStatus({ code: SpanStatusCode.UNSET });
1384
1730
  span.end();
1385
1731
  drain(provider, ctx);
@@ -1398,13 +1744,515 @@ export function instrument(handler, config) {
1398
1744
  }
1399
1745
  return wrapped;
1400
1746
  }
1747
+ // ---------------------------------------------------------------------------
1748
+ // Queue consumer: per-message ack/retry child spans
1749
+ // ---------------------------------------------------------------------------
1750
+ /**
1751
+ * Wrap a queue batch so each message's explicit `ack()` / `retry()` emits a
1752
+ * cheap INTERNAL child span. The batch itself is handed to user code in ONE
1753
+ * call — the wrapper cannot split it without changing retry/ack semantics — so
1754
+ * `queue message process` spans are deliberately NOT emitted; when a consumer
1755
+ * uses per-message control flow these ack/retry spans are the true per-message
1756
+ * unit of work. Consumers that never call ack()/retry() simply see no extra
1757
+ * spans. Non-wrapped members (ackAll/retryAll/queue/id/body/timestamp) fall
1758
+ * through to the real platform objects with correct receivers.
1759
+ */
1760
+ function wrapQueueMessageBatch(batch, startSpan) {
1761
+ const source = batch.messages;
1762
+ if (!Array.isArray(source))
1763
+ return batch;
1764
+ const messages = source.map((m) => wrapQueueMessage(m, startSpan));
1765
+ return new Proxy(batch, {
1766
+ get(t, prop, _receiver) {
1767
+ if (prop === "messages")
1768
+ return messages;
1769
+ const val = Reflect.get(t, prop, t);
1770
+ return typeof val === "function" ? val.bind(t) : val;
1771
+ },
1772
+ });
1773
+ }
1774
+ function wrapQueueMessage(message, startSpan) {
1775
+ if (!message || typeof message !== "object")
1776
+ return message;
1777
+ return new Proxy(message, {
1778
+ get(t, prop, _receiver) {
1779
+ if (prop === "ack" || prop === "retry") {
1780
+ const orig = t[prop];
1781
+ if (typeof orig !== "function")
1782
+ return orig;
1783
+ return (...args) => {
1784
+ // Bind to the REAL platform message (brand-check-safe receiver).
1785
+ if (isTracingSuppressed(context.active())) {
1786
+ orig.apply(t, args);
1787
+ return;
1788
+ }
1789
+ const attrs = { "messaging.system": "cloudflare_queues" };
1790
+ if (typeof t.attempts === "number")
1791
+ attrs["messaging.attempt"] = t.attempts;
1792
+ if (typeof t.body === "string")
1793
+ attrs["messaging.message.body.size"] = t.body.length;
1794
+ const span = startSpan(`queue message ${prop}`, attrs);
1795
+ try {
1796
+ orig.apply(t, args);
1797
+ }
1798
+ catch (error) {
1799
+ try {
1800
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
1801
+ span.setStatus({
1802
+ code: SpanStatusCode.ERROR,
1803
+ message: error instanceof Error ? error.message : String(error),
1804
+ });
1805
+ }
1806
+ catch {
1807
+ // Preserve the platform method's original exception.
1808
+ }
1809
+ throw error;
1810
+ }
1811
+ finally {
1812
+ try {
1813
+ span.end();
1814
+ }
1815
+ catch {
1816
+ // Queue decisions must remain synchronous platform operations.
1817
+ }
1818
+ }
1819
+ };
1820
+ }
1821
+ const val = Reflect.get(t, prop, t);
1822
+ return typeof val === "function" ? val.bind(t) : val;
1823
+ },
1824
+ });
1825
+ }
1826
+ // ---------------------------------------------------------------------------
1827
+ // Email handler span naming/attrs (privacy-preserving)
1828
+ // ---------------------------------------------------------------------------
1829
+ /**
1830
+ * Extract the DOMAIN of an email address, or "" for anything malformed.
1831
+ * Deliberately never returns the local part — full addresses are PII.
1832
+ */
1833
+ function emailDomain(address) {
1834
+ if (typeof address !== "string")
1835
+ return "";
1836
+ const at = address.lastIndexOf("@");
1837
+ if (at <= 0 || at === address.length - 1)
1838
+ return "";
1839
+ return address.slice(at + 1).toLowerCase();
1840
+ }
1841
+ /** SERVER-span name + attributes for an email handler invocation. */
1842
+ function emailHandlerAttrs(message) {
1843
+ const fromDomain = emailDomain(message?.from);
1844
+ const toDomain = emailDomain(message?.to);
1845
+ const attrs = {};
1846
+ if (fromDomain)
1847
+ attrs["cloudflare.email.from_domain"] = fromDomain;
1848
+ if (toDomain)
1849
+ attrs["cloudflare.email.to_domain"] = toDomain;
1850
+ const size = message?.rawSize;
1851
+ if (typeof size === "number" && Number.isFinite(size)) {
1852
+ attrs["cloudflare.email.raw_size_bytes"] = size;
1853
+ }
1854
+ return { name: fromDomain ? `email ${fromDomain}` : "email", attrs };
1855
+ }
1856
+ function classStartSpan(opts) {
1857
+ return (opts.startSpan ??
1858
+ ((name, attrs, kind, ctx) => trace.getTracer("heystack").startSpan(name, { attributes: attrs, kind }, ctx ?? context.active())));
1859
+ }
1860
+ /**
1861
+ * Keep the isolate alive until pending exports drain: hand `flushHeystack()`
1862
+ * to the invocation's DurableObjectState `waitUntil`. Best-effort no-op when no
1863
+ * state/waitUntil is available (e.g. unit tests).
1864
+ */
1865
+ function scheduleStateFlush(state) {
1866
+ const waitUntil = state
1867
+ ?.waitUntil;
1868
+ if (typeof waitUntil !== "function")
1869
+ return;
1870
+ try {
1871
+ waitUntil.call(state, flushHeystack());
1872
+ }
1873
+ catch {
1874
+ // A failing waitUntil must never break the wrapped handler.
1875
+ }
1876
+ }
1877
+ /** INTERNAL child spans for DurableObjectStorage get/put/delete/list ops. */
1878
+ function wrapDurableObjectStorage(storage, startSpan) {
1879
+ const wrapOp = (op) => async (...args) => {
1880
+ const underlying = storage[op];
1881
+ if (typeof underlying !== "function")
1882
+ return undefined;
1883
+ if (isTracingSuppressed(context.active())) {
1884
+ return underlying.apply(storage, args);
1885
+ }
1886
+ const attrs = {};
1887
+ const first = args[0];
1888
+ if (typeof first === "string") {
1889
+ attrs["cloudflare.do.storage.key"] = first;
1890
+ }
1891
+ else if (Array.isArray(first)) {
1892
+ attrs["cloudflare.do.storage.key_count"] = first.length;
1893
+ }
1894
+ else if (op === "list" &&
1895
+ first &&
1896
+ typeof first === "object" &&
1897
+ typeof first.limit === "number") {
1898
+ attrs["cloudflare.do.storage.limit"] = first.limit;
1899
+ }
1900
+ const span = startSpan(`DO storage ${op}`, attrs, SpanKind.INTERNAL);
1901
+ return runWithSpan(span, () => Promise.resolve(underlying.apply(storage, args)));
1902
+ };
1903
+ // A dedicated proxy (not makeProxy) because `storage` is a VALUE property on
1904
+ // the state object, not a method — handlers there must be getters.
1905
+ return new Proxy(storage, {
1906
+ get(t, prop, _receiver) {
1907
+ const val = Reflect.get(t, prop, t);
1908
+ if ((prop === "get" || prop === "put" || prop === "delete" || prop === "list") && typeof val === "function") {
1909
+ return wrapOp(prop);
1910
+ }
1911
+ return typeof val === "function" ? val.bind(t) : val;
1912
+ },
1913
+ });
1914
+ }
1915
+ /** Proxy a DurableObjectState so `state.storage` resolves to the traced proxy. */
1916
+ function wrapDurableObjectState(state, startSpan) {
1917
+ if (!state || typeof state !== "object")
1918
+ return state;
1919
+ const s = state;
1920
+ if (!s.storage || typeof s.storage !== "object")
1921
+ return state;
1922
+ const tracedStorage = wrapDurableObjectStorage(s.storage, startSpan);
1923
+ return new Proxy(s, {
1924
+ get(t, prop, _receiver) {
1925
+ if (prop === "storage")
1926
+ return tracedStorage;
1927
+ const val = Reflect.get(t, prop, t);
1928
+ return typeof val === "function" ? val.bind(t) : val;
1929
+ },
1930
+ });
1931
+ }
1932
+ /** Prototype methods that are wrapped specially (never treated as RPC). */
1933
+ const DO_LIFECYCLE_METHODS = new Set(["fetch", "alarm", "constructor"]);
1934
+ const MAX_DO_RPC_METHODS = 32;
1935
+ /**
1936
+ * Decorate a Durable Object class:
1937
+ * - constructor: wraps the incoming `state` so `state.storage` operations emit
1938
+ * INTERNAL child spans (`DO storage get|put|delete|list`) with key/count attrs.
1939
+ * - `fetch()`: SERVER span named `<METHOD> <path>` (like instrument()'s fetch
1940
+ * wrapper), continuing an inbound W3C traceparent, recording status/duration.
1941
+ * - `alarm()`: INTERNAL span.
1942
+ * - RPC prototype methods: SERVER span `DO rpc <method>` with rpc.method attrs.
1943
+ *
1944
+ * Every root span schedules flushHeystack() via `state.waitUntil` so exports
1945
+ * complete before isolate teardown. Instance arrow-function methods assigned in
1946
+ * the constructor are not enumerable on the prototype and stay unwrapped.
1947
+ *
1948
+ * export class Counter extends DurableObject {
1949
+ * // ...
1950
+ * }
1951
+ * export const Counter = wrapDurableObjectClass(BaseCounter); // or decorate in place
1952
+ */
1953
+ export function wrapDurableObjectClass(cls, opts = {}) {
1954
+ const Base = cls;
1955
+ const startSpan = classStartSpan(opts);
1956
+ const proto = Base.prototype;
1957
+ const origFetch = proto.fetch;
1958
+ const origAlarm = proto.alarm;
1959
+ // Enumerate the RPC surface (prototype chain, lifecycle handlers excluded).
1960
+ const rpcMethods = [];
1961
+ try {
1962
+ let p = Base.prototype;
1963
+ for (let depth = 0; depth < 5 && p && p !== Object.prototype && rpcMethods.length < MAX_DO_RPC_METHODS; depth++) {
1964
+ for (const key of Object.getOwnPropertyNames(p)) {
1965
+ if (DO_LIFECYCLE_METHODS.has(key))
1966
+ continue;
1967
+ if (!rpcMethods.includes(key) && typeof proto[key] === "function") {
1968
+ rpcMethods.push(key);
1969
+ if (rpcMethods.length >= MAX_DO_RPC_METHODS)
1970
+ break;
1971
+ }
1972
+ }
1973
+ p = Object.getPrototypeOf(p);
1974
+ }
1975
+ }
1976
+ catch {
1977
+ // Best-effort enumeration — a hostile getter must never break decoration.
1978
+ }
1979
+ class Wrapped extends Base {
1980
+ /** Raw (unwrapped) state captured for flush scheduling. */
1981
+ __heystackState;
1982
+ constructor(...args) {
1983
+ super(...args.map((arg, i) => i === 0 ? wrapDurableObjectState(arg, startSpan) : arg));
1984
+ this.__heystackState = args[0];
1985
+ }
1986
+ }
1987
+ if (typeof origFetch === "function") {
1988
+ Wrapped.prototype.fetch = async function (request) {
1989
+ if (isTracingSuppressed(context.active())) {
1990
+ return origFetch.call(this, request);
1991
+ }
1992
+ // Continue an inbound W3C traceparent so caller→DO is one distributed trace.
1993
+ let startCtx = context.active();
1994
+ const parent = parseTraceparent(request.headers.get("traceparent"));
1995
+ if (parent) {
1996
+ startCtx = trace.setSpanContext(startCtx, {
1997
+ traceId: parent.traceId,
1998
+ spanId: parent.spanId,
1999
+ traceFlags: parent.traceFlags,
2000
+ isRemote: true,
2001
+ });
2002
+ }
2003
+ let url;
2004
+ try {
2005
+ url = new URL(request.url);
2006
+ }
2007
+ catch {
2008
+ url = new URL("https://durable-object.invalid/");
2009
+ }
2010
+ const span = startSpan(`${request.method} ${url.pathname}`, {
2011
+ "http.request.method": request.method,
2012
+ "url.path": url.pathname,
2013
+ "server.address": url.host,
2014
+ }, SpanKind.SERVER, startCtx);
2015
+ try {
2016
+ const response = (await context.with(trace.setSpan(startCtx, span), () => origFetch.call(this, request)));
2017
+ span.setAttribute("http.response.status_code", response.status);
2018
+ span.setStatus({
2019
+ code: response.status >= 500 ? SpanStatusCode.ERROR : SpanStatusCode.UNSET,
2020
+ });
2021
+ return response;
2022
+ }
2023
+ catch (error) {
2024
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
2025
+ span.setStatus({
2026
+ code: SpanStatusCode.ERROR,
2027
+ message: error instanceof Error ? error.message : String(error),
2028
+ });
2029
+ throw error;
2030
+ }
2031
+ finally {
2032
+ span.end();
2033
+ scheduleStateFlush(this.__heystackState);
2034
+ }
2035
+ };
2036
+ }
2037
+ if (typeof origAlarm === "function") {
2038
+ Wrapped.prototype.alarm = async function (...args) {
2039
+ if (isTracingSuppressed(context.active())) {
2040
+ return origAlarm.apply(this, args);
2041
+ }
2042
+ const span = startSpan("DO alarm", {}, SpanKind.INTERNAL);
2043
+ try {
2044
+ return await context.with(trace.setSpan(context.active(), span), () => origAlarm.apply(this, args));
2045
+ }
2046
+ catch (error) {
2047
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
2048
+ span.setStatus({
2049
+ code: SpanStatusCode.ERROR,
2050
+ message: error instanceof Error ? error.message : String(error),
2051
+ });
2052
+ throw error;
2053
+ }
2054
+ finally {
2055
+ span.end();
2056
+ scheduleStateFlush(this.__heystackState);
2057
+ }
2058
+ };
2059
+ }
2060
+ for (const method of rpcMethods) {
2061
+ const orig = proto[method];
2062
+ Wrapped.prototype[method] = async function (...args) {
2063
+ if (isTracingSuppressed(context.active())) {
2064
+ return orig.apply(this, args);
2065
+ }
2066
+ const span = startSpan(`DO rpc ${method}`, {
2067
+ "rpc.system": "cloudflare.durable_object",
2068
+ "rpc.method": method,
2069
+ }, SpanKind.SERVER);
2070
+ try {
2071
+ return await context.with(trace.setSpan(context.active(), span), () => orig.apply(this, args));
2072
+ }
2073
+ catch (error) {
2074
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
2075
+ span.setStatus({
2076
+ code: SpanStatusCode.ERROR,
2077
+ message: error instanceof Error ? error.message : String(error),
2078
+ });
2079
+ throw error;
2080
+ }
2081
+ finally {
2082
+ span.end();
2083
+ scheduleStateFlush(this.__heystackState);
2084
+ }
2085
+ };
2086
+ }
2087
+ return Wrapped;
2088
+ }
2089
+ // ---------------------------------------------------------------------------
2090
+ // Workflows ENTRYPOINT class instrumentation
2091
+ // ---------------------------------------------------------------------------
2092
+ /** Step methods that are async WAITS — traced as short marker spans, never held open. */
2093
+ const WORKFLOW_WAIT_METHODS = ["sleep", "sleepUntil", "waitForEvent", "waitForName"];
2094
+ /** Marker-span attributes for a wait call (duration/timestamp only — no payload). */
2095
+ function workflowWaitAttrs(method, args) {
2096
+ if (method === "sleep" && args.length > 1 && args[1] != null) {
2097
+ return { "cloudflare.workflow.sleep_duration": String(args[1]) };
2098
+ }
2099
+ if (method === "sleepUntil" && args.length > 1 && args[1] != null) {
2100
+ const ts = args[1] instanceof Date ? args[1].toISOString() : String(args[1]);
2101
+ return { "cloudflare.workflow.sleep_until": ts };
2102
+ }
2103
+ return undefined;
2104
+ }
2105
+ /** Privacy-safe attributes from Cloudflare's real WorkflowEvent contract. */
2106
+ function workflowRunAttrs(event) {
2107
+ if (!event || typeof event !== "object")
2108
+ return {};
2109
+ const value = event;
2110
+ const attrs = {};
2111
+ if (typeof value.workflowName === "string") {
2112
+ attrs["cloudflare.workflow.name"] = value.workflowName;
2113
+ }
2114
+ if (typeof value.instanceId === "string") {
2115
+ attrs["cloudflare.workflow.instance_id"] = value.instanceId;
2116
+ }
2117
+ if (value.timestamp instanceof Date && Number.isFinite(value.timestamp.getTime())) {
2118
+ attrs["cloudflare.workflow.timestamp"] = value.timestamp.toISOString();
2119
+ }
2120
+ if (value.schedule && typeof value.schedule === "object") {
2121
+ if (typeof value.schedule.cron === "string") {
2122
+ attrs["cloudflare.workflow.schedule.cron"] = value.schedule.cron;
2123
+ }
2124
+ if (typeof value.schedule.scheduledTime === "number" &&
2125
+ Number.isFinite(value.schedule.scheduledTime)) {
2126
+ attrs["cloudflare.workflow.schedule.scheduled_time"] = value.schedule.scheduledTime;
2127
+ }
2128
+ }
2129
+ return attrs;
2130
+ }
2131
+ /**
2132
+ * Wrap a WorkflowStep so `do(name, fn)` executes fn inside a step span while
2133
+ * preserving engine semantics (we wrap FN and delegate to the real step.do —
2134
+ * the workflow engine owns execution/caching across replays), and wait-style
2135
+ * methods become short MARKER spans that end immediately after the wait is
2136
+ * initiated (a workflow can suspend for minutes; holding a span open across a
2137
+ * sleep would poison duration metrics).
2138
+ */
2139
+ function wrapWorkflowStep(step, startSpan, state) {
2140
+ const handlers = {};
2141
+ if (typeof step.do === "function") {
2142
+ handlers.do = (name, fn, ...rest) => {
2143
+ if (isTracingSuppressed(context.active()) || typeof fn !== "function") {
2144
+ return step.do(name, fn, ...rest);
2145
+ }
2146
+ return step.do(name, async (...fnArgs) => {
2147
+ const span = startSpan(`step ${String(name)}`, {}, SpanKind.INTERNAL);
2148
+ try {
2149
+ return await context.with(trace.setSpan(context.active(), span), () => Promise.resolve(fn(...fnArgs)));
2150
+ }
2151
+ catch (error) {
2152
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
2153
+ span.setStatus({
2154
+ code: SpanStatusCode.ERROR,
2155
+ message: error instanceof Error ? error.message : String(error),
2156
+ });
2157
+ throw error;
2158
+ }
2159
+ finally {
2160
+ span.end();
2161
+ }
2162
+ }, ...rest);
2163
+ };
2164
+ }
2165
+ for (const method of WORKFLOW_WAIT_METHODS) {
2166
+ if (typeof step[method] !== "function")
2167
+ continue;
2168
+ handlers[method] = (...args) => {
2169
+ if (isTracingSuppressed(context.active())) {
2170
+ return step[method](...args);
2171
+ }
2172
+ const nameArg = typeof args[0] === "string" ? ` ${args[0]}` : "";
2173
+ const span = startSpan(`workflow ${method}${nameArg}`, workflowWaitAttrs(method, args) ?? {}, SpanKind.INTERNAL);
2174
+ try {
2175
+ // Marker semantics: end the span synchronously after initiating the
2176
+ // wait — the returned promise is awaited by the caller OUTSIDE the span.
2177
+ return step[method](...args);
2178
+ }
2179
+ catch (error) {
2180
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
2181
+ span.setStatus({
2182
+ code: SpanStatusCode.ERROR,
2183
+ message: error instanceof Error ? error.message : String(error),
2184
+ });
2185
+ throw error;
2186
+ }
2187
+ finally {
2188
+ span.end();
2189
+ // The caller suspends only after this wrapper returns the wait promise,
2190
+ // so hand the ended marker span to waitUntil before that can happen.
2191
+ scheduleStateFlush(state);
2192
+ }
2193
+ };
2194
+ }
2195
+ return makeProxy(step, handlers);
2196
+ }
2197
+ /**
2198
+ * Decorate a Cloudflare Workflows entrypoint class so its `run(event, step)`
2199
+ * emits a `workflow run` INTERNAL span with workflow identity/schedule metadata
2200
+ * (never payload content), and the handed-out
2201
+ * step object's methods become traced versions (see {@link wrapWorkflowStep}):
2202
+ * `step.do(name, fn)` → INTERNAL span named `step <name>` with success/error
2203
+ * status (step retries manifest as whole-run replays by the engine and are not
2204
+ * observable at this layer, so no retry attribute is emitted), and
2205
+ * sleep/sleepUntil/waitForEvent/waitForName → short marker spans.
2206
+ *
2207
+ * Every completed run schedules flushHeystack() via the entrypoint state's
2208
+ * `waitUntil` so exports survive isolate teardown.
2209
+ */
2210
+ export function wrapWorkflowEntrypointClass(cls, opts = {}) {
2211
+ const Base = cls;
2212
+ const startSpan = classStartSpan(opts);
2213
+ const origRun = Base.prototype.run;
2214
+ if (typeof origRun !== "function")
2215
+ return cls;
2216
+ class Wrapped extends Base {
2217
+ __heystackState;
2218
+ constructor(...args) {
2219
+ super(...args);
2220
+ this.__heystackState = args[0];
2221
+ }
2222
+ }
2223
+ Wrapped.prototype.run = async function (event, step, ...rest) {
2224
+ if (isTracingSuppressed(context.active())) {
2225
+ return origRun.call(this, event, step, ...rest);
2226
+ }
2227
+ const attrs = workflowRunAttrs(event);
2228
+ const span = startSpan("workflow run", attrs, SpanKind.INTERNAL);
2229
+ try {
2230
+ return await context.with(trace.setSpan(context.active(), span), () => origRun.call(this, event, step && typeof step === "object"
2231
+ ? wrapWorkflowStep(step, startSpan, this.__heystackState)
2232
+ : step, ...rest));
2233
+ }
2234
+ catch (error) {
2235
+ span.recordException(error instanceof Error ? error : new Error(String(error)));
2236
+ span.setStatus({
2237
+ code: SpanStatusCode.ERROR,
2238
+ message: error instanceof Error ? error.message : String(error),
2239
+ });
2240
+ throw error;
2241
+ }
2242
+ finally {
2243
+ span.end();
2244
+ scheduleStateFlush(this.__heystackState);
2245
+ }
2246
+ };
2247
+ return Wrapped;
2248
+ }
1401
2249
  export async function withSpan(name, attrsOrFn, maybeFn) {
1402
2250
  const attrs = typeof attrsOrFn === "function" ? undefined : attrsOrFn;
1403
2251
  const fn = (typeof attrsOrFn === "function" ? attrsOrFn : maybeFn);
1404
2252
  const nativeTracing = nativeTracingFromContext();
1405
2253
  if (nativeTracing) {
1406
2254
  return nativeTracing.enterSpan(name, async (nativeSpan) => {
1407
- const adapter = new NativeSpanAdapter(nativeSpan, true);
2255
+ const adapter = new NativeSpanAdapter(nativeSpan);
1408
2256
  if (attrs)
1409
2257
  adapter.setAttributes(attrs);
1410
2258
  const nativeContext = trace.setSpan(context.active().setValue(NATIVE_TRACING_CONTEXT_KEY, nativeTracing), adapter);