@heystack/otel 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/traceparent.d.ts +8 -0
- package/dist/traceparent.js +5 -0
- package/dist/workers-agents.d.ts +1 -1
- package/dist/workers-agents.js +1 -1
- package/dist/workers-bindings.d.ts +11 -1
- package/dist/workers-bindings.js +286 -6
- package/dist/workers.d.ts +91 -20
- package/dist/workers.js +962 -114
- package/package.json +1 -1
package/dist/workers.js
CHANGED
|
@@ -16,8 +16,9 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
|
16
16
|
import { buildExporterConfig, DEFAULT_ENDPOINT, releaseResourceAttributes, } from "./core.js";
|
|
17
17
|
import { isSelfSpanAttrs, safeHostname } from "./self-span.js";
|
|
18
18
|
import { llmRequestAttrs, llmResponseAttrs, llmContentAttrs, llmOperationName, resolveLLMProvider, scanSseStreamForUsage, } from "./llm-enrich.js";
|
|
19
|
-
import { instrumentEnv } from "./workers-bindings.js";
|
|
19
|
+
import { instrumentEnv, makeProxy, runWithSpan } from "./workers-bindings.js";
|
|
20
20
|
import { makeSampler, loadRemoteSamplingRate } from "./workers-sampler.js";
|
|
21
|
+
import { makeTraceparent } from "./traceparent.js";
|
|
21
22
|
// `ExportResult` / `ExportResultCode` mirror `@opentelemetry/core`. We define
|
|
22
23
|
// them inline (structurally identical) rather than import them: core is only a
|
|
23
24
|
// transitive dep of sdk-trace-base and isn't reliably resolvable, and keeping it
|
|
@@ -412,7 +413,7 @@ function nativeLLMFetch(nativeTracing, resolved, url, input, init, originalFetch
|
|
|
412
413
|
}
|
|
413
414
|
return;
|
|
414
415
|
}
|
|
415
|
-
const adapter = new NativeSpanAdapter(nativeSpan
|
|
416
|
+
const adapter = new NativeSpanAdapter(nativeSpan);
|
|
416
417
|
const { attrs, parsedReqBody } = llmRequestSide(resolved, url, init);
|
|
417
418
|
adapter.setAttributes({ ...attrs, "server.address": url.hostname });
|
|
418
419
|
try {
|
|
@@ -511,7 +512,7 @@ function ensureFetchInstrumentation(ingestHost, aiCfg = {}) {
|
|
|
511
512
|
},
|
|
512
513
|
}, active);
|
|
513
514
|
const sc = span.spanContext();
|
|
514
|
-
const traceparent =
|
|
515
|
+
const traceparent = makeTraceparent(sc);
|
|
515
516
|
const [reqInput, reqInit] = injectTraceparent(input, init, traceparent);
|
|
516
517
|
// gen_ai enrichment — request side (direct host or configured gateway).
|
|
517
518
|
const url = parseTarget(input);
|
|
@@ -571,6 +572,85 @@ export function __resetFetchInstrumentation() {
|
|
|
571
572
|
// ---------------------------------------------------------------------------
|
|
572
573
|
// Exporter
|
|
573
574
|
// ---------------------------------------------------------------------------
|
|
575
|
+
/**
|
|
576
|
+
* Bounded in-export resilience: ONE automatic retry for network errors /
|
|
577
|
+
* 429 / 5xx. The backoff starts at {@link RETRY_BASE_DELAY_MS} with jitter and
|
|
578
|
+
* the total added wait across retries never exceeds {@link RETRY_TOTAL_CAP_MS}
|
|
579
|
+
* (~2s) so an ingest blip can't pin a request handler; a `Retry-After` header
|
|
580
|
+
* (seconds form) on 429 refines the delay within that same cap.
|
|
581
|
+
*/
|
|
582
|
+
const EXPORT_MAX_ATTEMPTS = 2;
|
|
583
|
+
const RETRY_BASE_DELAY_MS = 250;
|
|
584
|
+
const RETRY_TOTAL_CAP_MS = 2_000;
|
|
585
|
+
/** Upper bound on concurrently-tracked in-flight exports (`pending` set). */
|
|
586
|
+
const MAX_PENDING_EXPORTS = 512;
|
|
587
|
+
function sleep(ms) {
|
|
588
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
589
|
+
}
|
|
590
|
+
/** Parse the seconds form of `Retry-After`; HTTP-date form returns undefined. */
|
|
591
|
+
function parseRetryAfterSeconds(header) {
|
|
592
|
+
if (!header)
|
|
593
|
+
return undefined;
|
|
594
|
+
const s = Number.parseInt(header.trim(), 10);
|
|
595
|
+
return Number.isFinite(s) && s > 0 ? s : undefined;
|
|
596
|
+
}
|
|
597
|
+
/**
|
|
598
|
+
* Backoff for the single retry: honour `Retry-After` when present (429),
|
|
599
|
+
* otherwise base delay + jitter; always clamped into the remaining total cap.
|
|
600
|
+
*/
|
|
601
|
+
function nextRetryDelayMs(remainingBudgetMs, retryAfterSeconds) {
|
|
602
|
+
const hinted = retryAfterSeconds !== undefined && retryAfterSeconds > 0
|
|
603
|
+
? retryAfterSeconds * 1000
|
|
604
|
+
: RETRY_BASE_DELAY_MS + Math.random() * RETRY_BASE_DELAY_MS;
|
|
605
|
+
return Math.max(0, Math.min(hinted, remainingBudgetMs));
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* One-per-isolate advisory when exports complete with NEITHER a `waitUntil`
|
|
609
|
+
* handoff NOR an OpenNext context borrow available — i.e. delivery fell back
|
|
610
|
+
* to the pending set, where traces are lost unless someone drains explicitly.
|
|
611
|
+
*/
|
|
612
|
+
let warnedNoHandoff = false;
|
|
613
|
+
let manualFlushObserved = false;
|
|
614
|
+
let requestDrainWired = false;
|
|
615
|
+
function warnIfNoHandoffOnce() {
|
|
616
|
+
if (manualFlushObserved || requestDrainWired || warnedNoHandoff)
|
|
617
|
+
return;
|
|
618
|
+
warnedNoHandoff = true;
|
|
619
|
+
console.warn("[heystack] trace exports have no waitUntil handoff (no ExecutionContext wiring detected). " +
|
|
620
|
+
"Wrap your Worker's ctx with wrapExecutionContext(ctx), or use instrument(), so exports are " +
|
|
621
|
+
"drained before isolate teardown; otherwise traces rely on flushHeystack() and may be lost.");
|
|
622
|
+
}
|
|
623
|
+
/** Record an explicit flushHeystack() so the no-handoff advisory stays quiet. */
|
|
624
|
+
function markManualFlushObserved() {
|
|
625
|
+
manualFlushObserved = true;
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Drain BOTH the provider's span processor AND the exporter's in-flight POSTs.
|
|
629
|
+
* Awaiting only provider.forceFlush() would return before the export POST
|
|
630
|
+
* completes, letting the isolate be torn down and silently dropping the trace.
|
|
631
|
+
*/
|
|
632
|
+
async function drainProvider(provider) {
|
|
633
|
+
await provider.forceFlush().catch(() => { });
|
|
634
|
+
await provider.heystackExporter.forceFlush().catch(() => { });
|
|
635
|
+
}
|
|
636
|
+
/**
|
|
637
|
+
* The drain raced against an ~8s timeout so a hung forceFlush (e.g. an ingest
|
|
638
|
+
* that never responds) can't pin the isolate until the platform CPU timeout
|
|
639
|
+
* kills it (#25). On timeout we stop waiting; the POST is still in-flight and
|
|
640
|
+
* may yet complete, we just don't block the isolate on it indefinitely.
|
|
641
|
+
*/
|
|
642
|
+
function drainWithTimeout(provider) {
|
|
643
|
+
let timer;
|
|
644
|
+
const timeout = new Promise((resolve) => {
|
|
645
|
+
timer = setTimeout(resolve, DRAIN_TIMEOUT_MS);
|
|
646
|
+
});
|
|
647
|
+
return Promise.race([drainProvider(provider), timeout]).finally(() => {
|
|
648
|
+
if (timer)
|
|
649
|
+
clearTimeout(timer);
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
/** Upper bound on one drain race — shared by instrument() and wrapExecutionContext. */
|
|
653
|
+
const DRAIN_TIMEOUT_MS = 8_000;
|
|
574
654
|
/**
|
|
575
655
|
* A WinterCG-compatible OTLP/JSON span exporter. POSTs ended spans to the
|
|
576
656
|
* Heystack ingest using the platform `fetch` — no Node built-ins.
|
|
@@ -581,6 +661,8 @@ export class HeystackSpanExporter {
|
|
|
581
661
|
ingestHost;
|
|
582
662
|
headers;
|
|
583
663
|
shutdownState = false;
|
|
664
|
+
/** Exports rejected before network work because the in-flight cap was hit. */
|
|
665
|
+
droppedPending = 0;
|
|
584
666
|
/**
|
|
585
667
|
* In-flight export fetches. Each `export()` adds its settled-when-the-POST-
|
|
586
668
|
* completes promise here and removes it on `finally`. `forceFlush()` awaits
|
|
@@ -629,47 +711,45 @@ export class HeystackSpanExporter {
|
|
|
629
711
|
return;
|
|
630
712
|
}
|
|
631
713
|
const body = JSON.stringify(serializeSpans(exportable));
|
|
632
|
-
// Build the
|
|
633
|
-
// the actual network write
|
|
634
|
-
//
|
|
714
|
+
// Build the export chain as a promise we retain, so forceFlush() can await
|
|
715
|
+
// the actual network write — INCLUDING the bounded retry/backoff, so a
|
|
716
|
+
// waitUntil drain keeps the isolate alive until retries settle too. It
|
|
717
|
+
// resolves (never rejects) once the POST attempts are done (success or
|
|
718
|
+
// final failure) and resultCallback has been invoked.
|
|
635
719
|
//
|
|
636
|
-
//
|
|
637
|
-
// auto-instrumentation (e.g. Next/OpenNext) does NOT create a CLIENT
|
|
638
|
-
// for it — which would otherwise be exported and re-captured, a
|
|
639
|
-
// feedback loop. As a belt-and-suspenders second layer it also
|
|
640
|
-
// CAPTURED platform fetch (when our outbound-fetch wrapper is
|
|
641
|
-
// the export can never be re-entered by that wrapper
|
|
642
|
-
const
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
.then((res) => {
|
|
646
|
-
if (res.ok) {
|
|
647
|
-
resultCallback({ code: ExportResultCode.SUCCESS });
|
|
648
|
-
}
|
|
649
|
-
else {
|
|
650
|
-
resultCallback({
|
|
651
|
-
code: ExportResultCode.FAILED,
|
|
652
|
-
error: new Error(`Heystack ingest responded ${res.status} ${res.statusText}`),
|
|
653
|
-
});
|
|
654
|
-
}
|
|
655
|
-
})
|
|
656
|
-
.catch((error) => {
|
|
720
|
+
// Each POST attempt runs inside a tracing-suppressed context so that host
|
|
721
|
+
// fetch auto-instrumentation (e.g. Next/OpenNext) does NOT create a CLIENT
|
|
722
|
+
// span for it — which would otherwise be exported and re-captured, a
|
|
723
|
+
// sustained feedback loop. As a belt-and-suspenders second layer it also
|
|
724
|
+
// uses the CAPTURED platform fetch (when our outbound-fetch wrapper is
|
|
725
|
+
// installed), so the export can never be re-entered by that wrapper.
|
|
726
|
+
const reservation = this.reservePending();
|
|
727
|
+
if (!reservation) {
|
|
728
|
+
this.droppedPending++;
|
|
657
729
|
resultCallback({
|
|
658
730
|
code: ExportResultCode.FAILED,
|
|
659
|
-
error:
|
|
731
|
+
error: new Error(`Heystack export dropped: ${MAX_PENDING_EXPORTS} exports are already in flight`),
|
|
660
732
|
});
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
const doFetch = _originalFetch ?? fetch;
|
|
736
|
+
const post = this.postWithRetry(doFetch, body, resultCallback);
|
|
737
|
+
// postWithRetry should not reject; handle both branches so a future change
|
|
738
|
+
// cannot leak the reserved slot or create an unhandled rejection.
|
|
739
|
+
void post.then(reservation.release, reservation.release);
|
|
740
|
+
const p = reservation.pending;
|
|
664
741
|
// Keep the isolate alive until the POST completes. Priority:
|
|
665
742
|
// 1. an explicit `waitUntil` override (set via initHeystackWorkers).
|
|
666
743
|
// 2. the OpenNext Cloudflare request context's `ctx.waitUntil` — the
|
|
667
744
|
// export runs during request handling, so this is available there.
|
|
668
745
|
// Either path makes delivery reliable on workerd/OpenNext with no app hook.
|
|
669
|
-
// The `pending` set + `flushHeystack()` remain as the explicit fallback
|
|
746
|
+
// The `pending` set + `flushHeystack()` remain as the explicit fallback;
|
|
747
|
+
// when neither handoff lands we say so once per isolate (see below).
|
|
748
|
+
const handoff = { borrowed: false };
|
|
670
749
|
if (this.waitUntil) {
|
|
671
750
|
try {
|
|
672
751
|
this.waitUntil(p);
|
|
752
|
+
handoff.borrowed = true;
|
|
673
753
|
}
|
|
674
754
|
catch (error) {
|
|
675
755
|
this.warnWaitUntilFailed("explicit waitUntil", error);
|
|
@@ -681,21 +761,88 @@ export class HeystackSpanExporter {
|
|
|
681
761
|
// available; otherwise await the in-flight accessor load so even those
|
|
682
762
|
// early exports get handed to waitUntil once it resolves. `p` is already
|
|
683
763
|
// in `pending`, so a manual `flushHeystack()` covers it regardless.
|
|
684
|
-
this.handOffToCloudflareContext(p);
|
|
764
|
+
this.handOffToCloudflareContext(p, handoff);
|
|
685
765
|
}
|
|
766
|
+
if (!handoff.borrowed) {
|
|
767
|
+
// Fallback pending-set mode: once THIS export settles with no handoff,
|
|
768
|
+
// raise the one-per-isolate advisory (suppressed for manual-flush users).
|
|
769
|
+
void p.then(() => warnIfNoHandoffOnce());
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* POST the batch with ONE bounded automatic retry for network errors / 429 /
|
|
774
|
+
* 5xx. Honours `Retry-After` (seconds form) on 429 within the total backoff
|
|
775
|
+
* cap; after retries are exhausted it preserves plain FAILED semantics via
|
|
776
|
+
* `resultCallback`. The returned promise never rejects.
|
|
777
|
+
*/
|
|
778
|
+
postWithRetry(doFetch, body, resultCallback) {
|
|
779
|
+
let budgetLeft = RETRY_TOTAL_CAP_MS;
|
|
780
|
+
let retryAfterSeconds;
|
|
781
|
+
return (async () => {
|
|
782
|
+
let lastError;
|
|
783
|
+
for (let attempt = 0; attempt < EXPORT_MAX_ATTEMPTS; attempt++) {
|
|
784
|
+
if (attempt > 0) {
|
|
785
|
+
const delay = nextRetryDelayMs(budgetLeft, retryAfterSeconds);
|
|
786
|
+
budgetLeft -= delay;
|
|
787
|
+
await sleep(delay);
|
|
788
|
+
}
|
|
789
|
+
let retryable = false;
|
|
790
|
+
try {
|
|
791
|
+
const res = await context.with(suppressTracing(context.active()), () => doFetch(this.url, { method: "POST", headers: this.headers, body }));
|
|
792
|
+
if (res.ok) {
|
|
793
|
+
resultCallback({ code: ExportResultCode.SUCCESS });
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
lastError = new Error(`Heystack ingest responded ${res.status} ${res.statusText}`);
|
|
797
|
+
retryable = res.status === 429 || res.status >= 500;
|
|
798
|
+
if (res.status === 429) {
|
|
799
|
+
retryAfterSeconds = parseRetryAfterSeconds(res.headers.get("retry-after"));
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
catch (error) {
|
|
803
|
+
lastError = error instanceof Error ? error : new Error(String(error));
|
|
804
|
+
retryable = true;
|
|
805
|
+
}
|
|
806
|
+
if (!retryable)
|
|
807
|
+
break;
|
|
808
|
+
}
|
|
809
|
+
resultCallback({
|
|
810
|
+
code: ExportResultCode.FAILED,
|
|
811
|
+
error: lastError ?? new Error("Heystack export failed"),
|
|
812
|
+
});
|
|
813
|
+
})();
|
|
814
|
+
}
|
|
815
|
+
/** Reserve a tracked slot synchronously before starting any network work. */
|
|
816
|
+
reservePending() {
|
|
817
|
+
if (this.pending.size >= MAX_PENDING_EXPORTS)
|
|
818
|
+
return undefined;
|
|
819
|
+
let release;
|
|
820
|
+
const pending = new Promise((resolve) => {
|
|
821
|
+
release = resolve;
|
|
822
|
+
});
|
|
823
|
+
this.pending.add(pending);
|
|
824
|
+
void pending.then(() => {
|
|
825
|
+
this.pending.delete(pending);
|
|
826
|
+
});
|
|
827
|
+
return { pending, release };
|
|
686
828
|
}
|
|
687
829
|
/**
|
|
688
830
|
* Hand `p` to the OpenNext Cloudflare request context's `ctx.waitUntil`.
|
|
689
831
|
* Resolves the accessor first (awaiting its in-flight load if needed) so an
|
|
690
832
|
* export that fired before the dynamic import settled is still covered.
|
|
833
|
+
* Sets `outcome.borrowed` when (and only when) a real `waitUntil` received
|
|
834
|
+
* `p` — the signal the no-handoff advisory keys off.
|
|
691
835
|
*/
|
|
692
|
-
handOffToCloudflareContext(p) {
|
|
836
|
+
handOffToCloudflareContext(p, outcome) {
|
|
693
837
|
const attempt = () => {
|
|
694
838
|
if (!_getCloudflareContext)
|
|
695
839
|
return;
|
|
696
840
|
try {
|
|
697
841
|
const cf = _getCloudflareContext();
|
|
698
|
-
cf?.ctx?.waitUntil
|
|
842
|
+
if (cf?.ctx?.waitUntil) {
|
|
843
|
+
cf.ctx.waitUntil(p);
|
|
844
|
+
outcome.borrowed = true;
|
|
845
|
+
}
|
|
699
846
|
}
|
|
700
847
|
catch (error) {
|
|
701
848
|
// #15: a real failure inside getCloudflareContext (e.g. called outside a
|
|
@@ -726,9 +873,9 @@ export class HeystackSpanExporter {
|
|
|
726
873
|
"unless you call flushHeystack() or pass an explicit waitUntil.");
|
|
727
874
|
}
|
|
728
875
|
shutdown() {
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
876
|
+
// Reject new work immediately, then drain everything admitted beforehand.
|
|
877
|
+
this.shutdownState = true;
|
|
878
|
+
return this.forceFlush();
|
|
732
879
|
}
|
|
733
880
|
/**
|
|
734
881
|
* Resolve only once every in-flight export fetch has settled. This is the
|
|
@@ -737,7 +884,11 @@ export class HeystackSpanExporter {
|
|
|
737
884
|
* `flushHeystack` and `instrument`).
|
|
738
885
|
*/
|
|
739
886
|
async forceFlush() {
|
|
740
|
-
|
|
887
|
+
// Exports can start from another callback while an earlier snapshot settles.
|
|
888
|
+
// Continue until the tracked set is actually quiescent.
|
|
889
|
+
while (this.pending.size > 0) {
|
|
890
|
+
await Promise.allSettled([...this.pending]);
|
|
891
|
+
}
|
|
741
892
|
}
|
|
742
893
|
}
|
|
743
894
|
function nativeTracingFromContext() {
|
|
@@ -750,10 +901,8 @@ function nativeTracingFromContext() {
|
|
|
750
901
|
*/
|
|
751
902
|
class NativeSpanAdapter {
|
|
752
903
|
nativeSpan;
|
|
753
|
-
|
|
754
|
-
constructor(nativeSpan, autoEnd) {
|
|
904
|
+
constructor(nativeSpan) {
|
|
755
905
|
this.nativeSpan = nativeSpan;
|
|
756
|
-
this.autoEnd = autoEnd;
|
|
757
906
|
}
|
|
758
907
|
spanContext() { return INVALID_SPAN_CONTEXT; }
|
|
759
908
|
setAttribute(key, value) {
|
|
@@ -804,10 +953,7 @@ class NativeSpanAdapter {
|
|
|
804
953
|
return this;
|
|
805
954
|
}
|
|
806
955
|
updateName(_name) { return this; }
|
|
807
|
-
end(_endTime) {
|
|
808
|
-
if (!this.autoEnd)
|
|
809
|
-
this.nativeSpan.end();
|
|
810
|
-
}
|
|
956
|
+
end(_endTime) { }
|
|
811
957
|
isRecording() { return this.nativeSpan.isTraced; }
|
|
812
958
|
recordException(exception, _time) {
|
|
813
959
|
const error = exception instanceof Error ? exception : new Error(String(exception));
|
|
@@ -842,7 +988,7 @@ function setNativeRequestAttributes(span, config, req) {
|
|
|
842
988
|
async function runNativeSpan(nativeTracing, name, attrs, fn) {
|
|
843
989
|
ensureContextManager();
|
|
844
990
|
return nativeTracing.enterSpan(name, async (nativeSpan) => {
|
|
845
|
-
const adapter = new NativeSpanAdapter(nativeSpan
|
|
991
|
+
const adapter = new NativeSpanAdapter(nativeSpan);
|
|
846
992
|
for (const [key, value] of Object.entries(attrs)) {
|
|
847
993
|
if (value !== undefined)
|
|
848
994
|
adapter.setAttribute(key, value);
|
|
@@ -899,9 +1045,9 @@ function getALS() {
|
|
|
899
1045
|
* context across `await` boundaries — child spans created after an `await`
|
|
900
1046
|
* inside a handler are correctly parented to the request's root span.
|
|
901
1047
|
*
|
|
902
|
-
* WinterCG-safe: it does NOT import
|
|
903
|
-
* ALS global that the runtime exposes. Falls back to
|
|
904
|
-
* when the global is absent (Deno, Bun without globals, etc.).
|
|
1048
|
+
* WinterCG-safe: it does NOT import the Node-only async_hooks builtin; instead
|
|
1049
|
+
* it uses the ALS global that the runtime exposes. Falls back to
|
|
1050
|
+
* `SyncStackContextManager` when the global is absent (Deno, Bun without globals, etc.).
|
|
905
1051
|
*/
|
|
906
1052
|
export class AlsContextManager {
|
|
907
1053
|
_als;
|
|
@@ -1022,10 +1168,12 @@ function ensureGlobalProvider(config) {
|
|
|
1022
1168
|
* FLUSH: on Next-on-OpenNext (where `@opennextjs/cloudflare` is present) the
|
|
1023
1169
|
* exporter automatically borrows the Cloudflare request context's
|
|
1024
1170
|
* `ctx.waitUntil` so the export POST completes before the isolate is torn down
|
|
1025
|
-
* — no app hook needed. For other workerd setups without it,
|
|
1026
|
-
*
|
|
1027
|
-
*
|
|
1028
|
-
* the
|
|
1171
|
+
* — no app hook needed. For other workerd setups without it, wrap your
|
|
1172
|
+
* handler's ExecutionContext with {@link wrapExecutionContext} (every
|
|
1173
|
+
* `ctx.waitUntil` then also schedules the export drain), call `flushHeystack()`
|
|
1174
|
+
* (which awaits the export fetch) from a response hook, hand it to
|
|
1175
|
+
* `ctx.waitUntil(flushHeystack())`, or pass an explicit `waitUntil` in the
|
|
1176
|
+
* config (highest priority). Returns the provider.
|
|
1029
1177
|
*/
|
|
1030
1178
|
export function initHeystackWorkers(config) {
|
|
1031
1179
|
return ensureGlobalProvider(config);
|
|
@@ -1039,6 +1187,7 @@ export function initHeystackWorkers(config) {
|
|
|
1039
1187
|
* isolate is torn down. Best-effort no-op if no provider is registered.
|
|
1040
1188
|
*/
|
|
1041
1189
|
export async function flushHeystack() {
|
|
1190
|
+
markManualFlushObserved();
|
|
1042
1191
|
if (!_provider)
|
|
1043
1192
|
return;
|
|
1044
1193
|
if (typeof _provider.forceFlush === "function") {
|
|
@@ -1048,10 +1197,62 @@ export async function flushHeystack() {
|
|
|
1048
1197
|
// drain the exporter's in-flight POSTs explicitly.
|
|
1049
1198
|
await _provider.heystackExporter.forceFlush();
|
|
1050
1199
|
}
|
|
1200
|
+
/**
|
|
1201
|
+
* Wrap a Worker `ExecutionContext` so EVERY `ctx.waitUntil(promise)` ALSO
|
|
1202
|
+
* schedules the exporter drain race after that promise settles (provider +
|
|
1203
|
+
* in-flight POSTs, raced against an 8s timeout so a hung ingest can't pin the
|
|
1204
|
+
* isolate). Use on
|
|
1205
|
+
* `initHeystackWorkers()` / `createTracerProvider()` setups that don't go
|
|
1206
|
+
* through `instrument()` (which wires the same drain internally):
|
|
1207
|
+
*
|
|
1208
|
+
* export default {
|
|
1209
|
+
* async fetch(req, env, ctx) {
|
|
1210
|
+
* ctx = wrapExecutionContext(ctx);
|
|
1211
|
+
* return handle(req, env, ctx); // any waitUntil you do flushes exports
|
|
1212
|
+
* },
|
|
1213
|
+
* };
|
|
1214
|
+
*
|
|
1215
|
+
* Every other present or future ExecutionContext member is forwarded unchanged
|
|
1216
|
+
* with the real host object as its receiver. Safe to call when tracing isn't
|
|
1217
|
+
* initialised — the drain scheduling simply no-ops.
|
|
1218
|
+
*/
|
|
1219
|
+
function wrapExecutionContextWithProvider(ctx, configuredProvider) {
|
|
1220
|
+
const waitUntil = (promise) => {
|
|
1221
|
+
ctx.waitUntil(promise);
|
|
1222
|
+
const provider = configuredProvider ?? _provider;
|
|
1223
|
+
if (!provider)
|
|
1224
|
+
return;
|
|
1225
|
+
// Background work can create spans after awaits. Start its drain only once
|
|
1226
|
+
// that work settles; a concurrent drain can finish before those spans exist.
|
|
1227
|
+
ctx.waitUntil(promise
|
|
1228
|
+
.catch(() => undefined)
|
|
1229
|
+
.then(() => drainWithTimeout(provider)));
|
|
1230
|
+
};
|
|
1231
|
+
const passThroughOnException = () => {
|
|
1232
|
+
ctx.passThroughOnException();
|
|
1233
|
+
};
|
|
1234
|
+
return new Proxy(ctx, {
|
|
1235
|
+
get(target, property) {
|
|
1236
|
+
if (property === "waitUntil")
|
|
1237
|
+
return waitUntil;
|
|
1238
|
+
if (property === "passThroughOnException")
|
|
1239
|
+
return passThroughOnException;
|
|
1240
|
+
const value = Reflect.get(target, property, target);
|
|
1241
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
1242
|
+
},
|
|
1243
|
+
set(target, property, value) {
|
|
1244
|
+
return Reflect.set(target, property, value, target);
|
|
1245
|
+
},
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
export function wrapExecutionContext(ctx) {
|
|
1249
|
+
return wrapExecutionContextWithProvider(ctx);
|
|
1250
|
+
}
|
|
1051
1251
|
/** Reset the singleton global provider. Internal/testing helper. */
|
|
1052
1252
|
export function __resetProvider() {
|
|
1053
1253
|
_provider = null;
|
|
1054
1254
|
_remoteSamplingKicked = false;
|
|
1255
|
+
requestDrainWired = false;
|
|
1055
1256
|
}
|
|
1056
1257
|
/** Guard: the remote sampling config GET fires at most once per isolate. */
|
|
1057
1258
|
let _remoteSamplingKicked = false;
|
|
@@ -1062,9 +1263,11 @@ function warnOnceNoKey() {
|
|
|
1062
1263
|
warnedNoKey = true;
|
|
1063
1264
|
console.warn("[heystack] HEYSTACK_API_KEY not set — Workers tracing disabled, handler runs untraced.");
|
|
1064
1265
|
}
|
|
1065
|
-
/** Reset the once-only
|
|
1266
|
+
/** Reset the once-only warnings. Internal/testing helper. */
|
|
1066
1267
|
export function __resetWarnings() {
|
|
1067
1268
|
warnedNoKey = false;
|
|
1269
|
+
warnedNoHandoff = false;
|
|
1270
|
+
manualFlushObserved = false;
|
|
1068
1271
|
}
|
|
1069
1272
|
export function instrument(handler, config) {
|
|
1070
1273
|
// Resolve the API key + set up (once) the global provider. Returns null when
|
|
@@ -1088,31 +1291,15 @@ export function instrument(handler, config) {
|
|
|
1088
1291
|
sampling: config.sampling,
|
|
1089
1292
|
ai: config.ai,
|
|
1090
1293
|
});
|
|
1294
|
+
// instrument() wires a per-request drain (see `drain` below), so exports on
|
|
1295
|
+
// this path are reliably flushed — the no-handoff advisory must stay quiet.
|
|
1296
|
+
requestDrainWired = true;
|
|
1091
1297
|
return { provider, tracer: trace.getTracer("heystack") };
|
|
1092
1298
|
};
|
|
1093
1299
|
// Drain BOTH the provider's span processor AND the exporter's in-flight fetch
|
|
1094
|
-
// via ctx.waitUntil
|
|
1095
|
-
// the export POST completes, letting the isolate be torn down and silently
|
|
1096
|
-
// dropping the trace.
|
|
1097
|
-
//
|
|
1098
|
-
// #25: the drain is raced against an ~8s timeout so a hung forceFlush (e.g. an
|
|
1099
|
-
// ingest that never responds) can't pin the isolate until the platform CPU
|
|
1100
|
-
// timeout kills it. On timeout we stop waiting; the POST is still in-flight and
|
|
1101
|
-
// may yet complete, we just don't block the isolate on it indefinitely.
|
|
1102
|
-
const DRAIN_TIMEOUT_MS = 8_000;
|
|
1300
|
+
// via ctx.waitUntil, raced against an ~8s timeout (see drainWithTimeout, #25).
|
|
1103
1301
|
const drain = (provider, ctx) => {
|
|
1104
|
-
|
|
1105
|
-
await provider.forceFlush().catch(() => { });
|
|
1106
|
-
await provider.heystackExporter.forceFlush().catch(() => { });
|
|
1107
|
-
})();
|
|
1108
|
-
let timer;
|
|
1109
|
-
const timeout = new Promise((resolve) => {
|
|
1110
|
-
timer = setTimeout(resolve, DRAIN_TIMEOUT_MS);
|
|
1111
|
-
});
|
|
1112
|
-
ctx.waitUntil(Promise.race([drained, timeout]).finally(() => {
|
|
1113
|
-
if (timer)
|
|
1114
|
-
clearTimeout(timer);
|
|
1115
|
-
}));
|
|
1302
|
+
ctx.waitUntil(drainWithTimeout(provider));
|
|
1116
1303
|
};
|
|
1117
1304
|
// Start from a shallow copy so EVERY sibling handler (tail, email, …) is
|
|
1118
1305
|
// forwarded untouched; we only override fetch/queue/scheduled below.
|
|
@@ -1207,7 +1394,7 @@ export function instrument(handler, config) {
|
|
|
1207
1394
|
}
|
|
1208
1395
|
// FR5: response header carrying THIS span's trace + span id.
|
|
1209
1396
|
const sc = span.spanContext();
|
|
1210
|
-
const traceparent =
|
|
1397
|
+
const traceparent = makeTraceparent(sc);
|
|
1211
1398
|
// The context with THIS request's SERVER span active. Used both to run the
|
|
1212
1399
|
// handler and as an explicit parent fallback for binding spans.
|
|
1213
1400
|
const serverCtx = trace.setSpan(startCtx, span);
|
|
@@ -1235,8 +1422,64 @@ export function instrument(handler, config) {
|
|
|
1235
1422
|
select: config.instrumentBindings,
|
|
1236
1423
|
});
|
|
1237
1424
|
}
|
|
1425
|
+
let finished = false;
|
|
1426
|
+
const abortSignal = req.signal;
|
|
1427
|
+
let responseStatus;
|
|
1428
|
+
const finishSpan = (outcome, error) => {
|
|
1429
|
+
if (finished)
|
|
1430
|
+
return;
|
|
1431
|
+
finished = true;
|
|
1432
|
+
abortSignal?.removeEventListener("abort", onAbort);
|
|
1433
|
+
try {
|
|
1434
|
+
if (outcome === "aborted") {
|
|
1435
|
+
span.setAttribute("heystack.aborted", true);
|
|
1436
|
+
span.setStatus({
|
|
1437
|
+
code: SpanStatusCode.ERROR,
|
|
1438
|
+
message: "client stopped reading before the response completed",
|
|
1439
|
+
});
|
|
1440
|
+
}
|
|
1441
|
+
else if (outcome === "error") {
|
|
1442
|
+
const recorded = error instanceof Error ? error : new Error(String(error));
|
|
1443
|
+
span.recordException(recorded);
|
|
1444
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: recorded.message });
|
|
1445
|
+
}
|
|
1446
|
+
else {
|
|
1447
|
+
span.setStatus({
|
|
1448
|
+
code: responseStatus !== undefined && (responseStatus === 0 || responseStatus >= 500)
|
|
1449
|
+
? SpanStatusCode.ERROR
|
|
1450
|
+
: SpanStatusCode.UNSET,
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
span.end();
|
|
1454
|
+
}
|
|
1455
|
+
catch {
|
|
1456
|
+
// Telemetry must not change response delivery or error propagation.
|
|
1457
|
+
}
|
|
1458
|
+
try {
|
|
1459
|
+
drain(provider, ctx);
|
|
1460
|
+
}
|
|
1461
|
+
catch {
|
|
1462
|
+
// A rejected host waitUntil handoff must not break the response stream.
|
|
1463
|
+
}
|
|
1464
|
+
};
|
|
1465
|
+
const onAbort = () => finishSpan("aborted");
|
|
1238
1466
|
try {
|
|
1239
|
-
const response = await context.with(serverCtx, () => originalFetch(req, handlerEnv, ctx));
|
|
1467
|
+
const response = await context.with(serverCtx, () => originalFetch(req, handlerEnv, wrapExecutionContextWithProvider(ctx, provider)));
|
|
1468
|
+
responseStatus = response.status;
|
|
1469
|
+
if (abortSignal && !abortSignal.aborted) {
|
|
1470
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
1471
|
+
}
|
|
1472
|
+
else if (abortSignal?.aborted) {
|
|
1473
|
+
finishSpan("aborted");
|
|
1474
|
+
}
|
|
1475
|
+
// Response.error() has status 0 and immutable network-error semantics.
|
|
1476
|
+
// Reconstructing it is invalid (ResponseInit only accepts 200-599), and
|
|
1477
|
+
// there is no response header surface on which to inject traceparent.
|
|
1478
|
+
if (response.status === 0) {
|
|
1479
|
+
span.setAttribute("http.response.status_code", response.status);
|
|
1480
|
+
finishSpan("success");
|
|
1481
|
+
return response;
|
|
1482
|
+
}
|
|
1240
1483
|
// WebSocket / connection upgrades: a 101 Switching Protocols response
|
|
1241
1484
|
// carries a live server-side `webSocket` that the Workers Runtime hands
|
|
1242
1485
|
// to the client ONLY if that exact Response object is returned. Rebuilding
|
|
@@ -1248,64 +1491,66 @@ export function instrument(handler, config) {
|
|
|
1248
1491
|
// signal; `status === 101` is a belt-and-braces fallback.
|
|
1249
1492
|
if (response.webSocket || response.status === 101) {
|
|
1250
1493
|
span.setAttribute("http.response.status_code", response.status);
|
|
1251
|
-
|
|
1252
|
-
span.end();
|
|
1253
|
-
drain(provider, ctx);
|
|
1494
|
+
finishSpan("success");
|
|
1254
1495
|
return response;
|
|
1255
1496
|
}
|
|
1256
1497
|
const headers = new Headers(response.headers);
|
|
1257
1498
|
headers.set("traceparent", traceparent);
|
|
1258
1499
|
appendExposeHeader(headers, "traceparent");
|
|
1259
1500
|
span.setAttribute("http.response.status_code", response.status);
|
|
1260
|
-
const finalize = () => {
|
|
1261
|
-
span.setStatus({
|
|
1262
|
-
code: response.status >= 500 ? SpanStatusCode.ERROR : SpanStatusCode.UNSET,
|
|
1263
|
-
});
|
|
1264
|
-
span.end();
|
|
1265
|
-
drain(provider, ctx);
|
|
1266
|
-
};
|
|
1267
1501
|
// No body to stream → finalize now (redirects, 204/304, etc.).
|
|
1268
1502
|
if (!response.body) {
|
|
1269
|
-
|
|
1503
|
+
finishSpan("success");
|
|
1270
1504
|
return new Response(null, {
|
|
1271
1505
|
status: response.status,
|
|
1272
1506
|
statusText: response.statusText,
|
|
1273
1507
|
headers,
|
|
1274
1508
|
});
|
|
1275
1509
|
}
|
|
1276
|
-
// FR1:
|
|
1277
|
-
//
|
|
1510
|
+
// FR1: pull one chunk at a time from the original body. This preserves
|
|
1511
|
+
// streaming/backpressure without buffering, while observing all three
|
|
1512
|
+
// terminal paths: clean EOF, source error, and consumer cancellation.
|
|
1278
1513
|
let firstByte = false;
|
|
1279
|
-
let
|
|
1280
|
-
const
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1514
|
+
let consumerCancelled = false;
|
|
1515
|
+
const reader = response.body.getReader();
|
|
1516
|
+
const monitoredBody = new ReadableStream({
|
|
1517
|
+
async pull(controller) {
|
|
1518
|
+
try {
|
|
1519
|
+
const result = await reader.read();
|
|
1520
|
+
if (consumerCancelled)
|
|
1521
|
+
return;
|
|
1522
|
+
if (result.done) {
|
|
1523
|
+
finishSpan("success");
|
|
1524
|
+
controller.close();
|
|
1525
|
+
return;
|
|
1526
|
+
}
|
|
1527
|
+
if (!firstByte) {
|
|
1528
|
+
firstByte = true;
|
|
1529
|
+
span.addEvent("first_byte");
|
|
1530
|
+
}
|
|
1531
|
+
controller.enqueue(result.value);
|
|
1532
|
+
}
|
|
1533
|
+
catch (error) {
|
|
1534
|
+
if (consumerCancelled)
|
|
1535
|
+
return;
|
|
1536
|
+
finishSpan("error", error);
|
|
1537
|
+
controller.error(error);
|
|
1285
1538
|
}
|
|
1286
|
-
controller.enqueue(chunk);
|
|
1287
1539
|
},
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
finalize();
|
|
1540
|
+
cancel(reason) {
|
|
1541
|
+
consumerCancelled = true;
|
|
1542
|
+
finishSpan("aborted", reason);
|
|
1543
|
+
return reader.cancel(reason);
|
|
1293
1544
|
},
|
|
1294
1545
|
});
|
|
1295
|
-
return new Response(
|
|
1546
|
+
return new Response(monitoredBody, {
|
|
1296
1547
|
status: response.status,
|
|
1297
1548
|
statusText: response.statusText,
|
|
1298
1549
|
headers,
|
|
1299
1550
|
});
|
|
1300
1551
|
}
|
|
1301
1552
|
catch (error) {
|
|
1302
|
-
|
|
1303
|
-
span.setStatus({
|
|
1304
|
-
code: SpanStatusCode.ERROR,
|
|
1305
|
-
message: error instanceof Error ? error.message : String(error),
|
|
1306
|
-
});
|
|
1307
|
-
span.end();
|
|
1308
|
-
drain(provider, ctx);
|
|
1553
|
+
finishSpan("error", error);
|
|
1309
1554
|
throw error;
|
|
1310
1555
|
}
|
|
1311
1556
|
};
|
|
@@ -1316,6 +1561,8 @@ export function instrument(handler, config) {
|
|
|
1316
1561
|
const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
|
|
1317
1562
|
if (nativeTracing) {
|
|
1318
1563
|
ensureFetchInstrumentation("", config.ai ?? {});
|
|
1564
|
+
// Native mode: Cloudflare owns queue spans — pass the ORIGINAL batch
|
|
1565
|
+
// through untouched so platform objects keep their identity.
|
|
1319
1566
|
return runNativeSpan(nativeTracing, `heystack.queue ${batch.queue}`, {
|
|
1320
1567
|
"service.name": config.service,
|
|
1321
1568
|
"messaging.batch.message_count": batch.messages.length,
|
|
@@ -1336,8 +1583,24 @@ export function instrument(handler, config) {
|
|
|
1336
1583
|
"messaging.destination.name": batch.queue,
|
|
1337
1584
|
},
|
|
1338
1585
|
});
|
|
1586
|
+
// The request's CONSUMER context, captured per-batch as an explicit parent
|
|
1587
|
+
// fallback (same rationale as the binding-span serverCtx below): on
|
|
1588
|
+
// workerd WITHOUT nodejs_compat a message ack'd after an `await` would
|
|
1589
|
+
// otherwise orphan into its own single-span trace.
|
|
1590
|
+
const consumerCtx = trace.setSpan(context.active(), span);
|
|
1591
|
+
const startMessageSpan = (name, attrs) => {
|
|
1592
|
+
const active = context.active();
|
|
1593
|
+
const parentCtx = trace.getSpan(active) ? active : consumerCtx;
|
|
1594
|
+
return tracer.startSpan(name, { attributes: attrs }, parentCtx);
|
|
1595
|
+
};
|
|
1596
|
+
// Per-message visibility: the wrapper hands the WHOLE batch to user code
|
|
1597
|
+
// in one call (splitting it would change retry/ack semantics), so the true
|
|
1598
|
+
// per-message unit of work observable here is the explicit
|
|
1599
|
+
// `msg.ack()` / `msg.retry()` decision — each gets a cheap INTERNAL child
|
|
1600
|
+
// span when the consumer uses per-message control flow.
|
|
1601
|
+
const tracedBatch = wrapQueueMessageBatch(batch, startMessageSpan);
|
|
1339
1602
|
try {
|
|
1340
|
-
await context.with(
|
|
1603
|
+
await context.with(consumerCtx, () => originalQueue(tracedBatch, env, wrapExecutionContextWithProvider(ctx, provider)));
|
|
1341
1604
|
span.setStatus({ code: SpanStatusCode.UNSET });
|
|
1342
1605
|
span.end();
|
|
1343
1606
|
drain(provider, ctx);
|
|
@@ -1379,7 +1642,90 @@ export function instrument(handler, config) {
|
|
|
1379
1642
|
},
|
|
1380
1643
|
});
|
|
1381
1644
|
try {
|
|
1382
|
-
await context.with(trace.setSpan(context.active(), span), () => originalScheduled(controller, env, ctx));
|
|
1645
|
+
await context.with(trace.setSpan(context.active(), span), () => originalScheduled(controller, env, wrapExecutionContextWithProvider(ctx, provider)));
|
|
1646
|
+
span.setStatus({ code: SpanStatusCode.UNSET });
|
|
1647
|
+
span.end();
|
|
1648
|
+
drain(provider, ctx);
|
|
1649
|
+
}
|
|
1650
|
+
catch (error) {
|
|
1651
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
1652
|
+
span.setStatus({
|
|
1653
|
+
code: SpanStatusCode.ERROR,
|
|
1654
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1655
|
+
});
|
|
1656
|
+
span.end();
|
|
1657
|
+
drain(provider, ctx);
|
|
1658
|
+
throw error;
|
|
1659
|
+
}
|
|
1660
|
+
};
|
|
1661
|
+
}
|
|
1662
|
+
const originalEmail = handler.email?.bind(handler);
|
|
1663
|
+
if (originalEmail) {
|
|
1664
|
+
wrapped.email = async (message, env, ctx) => {
|
|
1665
|
+
const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
|
|
1666
|
+
// PRIVACY: only from/to DOMAINS + envelope size are recorded — never the
|
|
1667
|
+
// full addresses and never any message content.
|
|
1668
|
+
const { name: emailSpanName, attrs: emailAttrs } = emailHandlerAttrs(message);
|
|
1669
|
+
if (nativeTracing) {
|
|
1670
|
+
return runNativeSpan(nativeTracing, `heystack.${emailSpanName}`, {
|
|
1671
|
+
"service.name": config.service,
|
|
1672
|
+
...emailAttrs,
|
|
1673
|
+
"service.version": config.version,
|
|
1674
|
+
"service.build": config.build,
|
|
1675
|
+
}, () => originalEmail(message, env, ctx));
|
|
1676
|
+
}
|
|
1677
|
+
const s = setup(env);
|
|
1678
|
+
if (!s)
|
|
1679
|
+
return originalEmail(message, env, ctx);
|
|
1680
|
+
const { provider, tracer } = s;
|
|
1681
|
+
const span = tracer.startSpan(emailSpanName, {
|
|
1682
|
+
kind: SpanKind.SERVER,
|
|
1683
|
+
attributes: emailAttrs,
|
|
1684
|
+
});
|
|
1685
|
+
try {
|
|
1686
|
+
await context.with(trace.setSpan(context.active(), span), () => originalEmail(message, env, wrapExecutionContextWithProvider(ctx, provider)));
|
|
1687
|
+
span.setStatus({ code: SpanStatusCode.UNSET });
|
|
1688
|
+
span.end();
|
|
1689
|
+
drain(provider, ctx);
|
|
1690
|
+
}
|
|
1691
|
+
catch (error) {
|
|
1692
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
1693
|
+
span.setStatus({
|
|
1694
|
+
code: SpanStatusCode.ERROR,
|
|
1695
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1696
|
+
});
|
|
1697
|
+
span.end();
|
|
1698
|
+
drain(provider, ctx);
|
|
1699
|
+
throw error;
|
|
1700
|
+
}
|
|
1701
|
+
};
|
|
1702
|
+
}
|
|
1703
|
+
const originalTail = handler.tail?.bind(handler);
|
|
1704
|
+
if (originalTail) {
|
|
1705
|
+
wrapped.tail = async (events, env, ctx) => {
|
|
1706
|
+
const nativeTracing = config.nativeTracing === "direct" ? undefined : getNativeTracing(ctx);
|
|
1707
|
+
// PRIVACY: only the event COUNT is recorded — tail payloads can contain
|
|
1708
|
+
// request URLs, headers and console output; none of it is captured.
|
|
1709
|
+
const count = Array.isArray(events) ? events.length : 0;
|
|
1710
|
+
const attrs = { "cloudflare.tail.event_count": count };
|
|
1711
|
+
if (nativeTracing) {
|
|
1712
|
+
return runNativeSpan(nativeTracing, "heystack.tail", {
|
|
1713
|
+
"service.name": config.service,
|
|
1714
|
+
...attrs,
|
|
1715
|
+
"service.version": config.version,
|
|
1716
|
+
"service.build": config.build,
|
|
1717
|
+
}, () => originalTail(events, env, ctx));
|
|
1718
|
+
}
|
|
1719
|
+
const s = setup(env);
|
|
1720
|
+
if (!s)
|
|
1721
|
+
return originalTail(events, env, ctx);
|
|
1722
|
+
const { provider, tracer } = s;
|
|
1723
|
+
const span = tracer.startSpan("tail", {
|
|
1724
|
+
kind: SpanKind.INTERNAL,
|
|
1725
|
+
attributes: attrs,
|
|
1726
|
+
});
|
|
1727
|
+
try {
|
|
1728
|
+
await context.with(trace.setSpan(context.active(), span), () => originalTail(events, env, wrapExecutionContextWithProvider(ctx, provider)));
|
|
1383
1729
|
span.setStatus({ code: SpanStatusCode.UNSET });
|
|
1384
1730
|
span.end();
|
|
1385
1731
|
drain(provider, ctx);
|
|
@@ -1398,13 +1744,515 @@ export function instrument(handler, config) {
|
|
|
1398
1744
|
}
|
|
1399
1745
|
return wrapped;
|
|
1400
1746
|
}
|
|
1747
|
+
// ---------------------------------------------------------------------------
|
|
1748
|
+
// Queue consumer: per-message ack/retry child spans
|
|
1749
|
+
// ---------------------------------------------------------------------------
|
|
1750
|
+
/**
|
|
1751
|
+
* Wrap a queue batch so each message's explicit `ack()` / `retry()` emits a
|
|
1752
|
+
* cheap INTERNAL child span. The batch itself is handed to user code in ONE
|
|
1753
|
+
* call — the wrapper cannot split it without changing retry/ack semantics — so
|
|
1754
|
+
* `queue message process` spans are deliberately NOT emitted; when a consumer
|
|
1755
|
+
* uses per-message control flow these ack/retry spans are the true per-message
|
|
1756
|
+
* unit of work. Consumers that never call ack()/retry() simply see no extra
|
|
1757
|
+
* spans. Non-wrapped members (ackAll/retryAll/queue/id/body/timestamp) fall
|
|
1758
|
+
* through to the real platform objects with correct receivers.
|
|
1759
|
+
*/
|
|
1760
|
+
function wrapQueueMessageBatch(batch, startSpan) {
|
|
1761
|
+
const source = batch.messages;
|
|
1762
|
+
if (!Array.isArray(source))
|
|
1763
|
+
return batch;
|
|
1764
|
+
const messages = source.map((m) => wrapQueueMessage(m, startSpan));
|
|
1765
|
+
return new Proxy(batch, {
|
|
1766
|
+
get(t, prop, _receiver) {
|
|
1767
|
+
if (prop === "messages")
|
|
1768
|
+
return messages;
|
|
1769
|
+
const val = Reflect.get(t, prop, t);
|
|
1770
|
+
return typeof val === "function" ? val.bind(t) : val;
|
|
1771
|
+
},
|
|
1772
|
+
});
|
|
1773
|
+
}
|
|
1774
|
+
function wrapQueueMessage(message, startSpan) {
|
|
1775
|
+
if (!message || typeof message !== "object")
|
|
1776
|
+
return message;
|
|
1777
|
+
return new Proxy(message, {
|
|
1778
|
+
get(t, prop, _receiver) {
|
|
1779
|
+
if (prop === "ack" || prop === "retry") {
|
|
1780
|
+
const orig = t[prop];
|
|
1781
|
+
if (typeof orig !== "function")
|
|
1782
|
+
return orig;
|
|
1783
|
+
return (...args) => {
|
|
1784
|
+
// Bind to the REAL platform message (brand-check-safe receiver).
|
|
1785
|
+
if (isTracingSuppressed(context.active())) {
|
|
1786
|
+
orig.apply(t, args);
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
const attrs = { "messaging.system": "cloudflare_queues" };
|
|
1790
|
+
if (typeof t.attempts === "number")
|
|
1791
|
+
attrs["messaging.attempt"] = t.attempts;
|
|
1792
|
+
if (typeof t.body === "string")
|
|
1793
|
+
attrs["messaging.message.body.size"] = t.body.length;
|
|
1794
|
+
const span = startSpan(`queue message ${prop}`, attrs);
|
|
1795
|
+
try {
|
|
1796
|
+
orig.apply(t, args);
|
|
1797
|
+
}
|
|
1798
|
+
catch (error) {
|
|
1799
|
+
try {
|
|
1800
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
1801
|
+
span.setStatus({
|
|
1802
|
+
code: SpanStatusCode.ERROR,
|
|
1803
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
catch {
|
|
1807
|
+
// Preserve the platform method's original exception.
|
|
1808
|
+
}
|
|
1809
|
+
throw error;
|
|
1810
|
+
}
|
|
1811
|
+
finally {
|
|
1812
|
+
try {
|
|
1813
|
+
span.end();
|
|
1814
|
+
}
|
|
1815
|
+
catch {
|
|
1816
|
+
// Queue decisions must remain synchronous platform operations.
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
};
|
|
1820
|
+
}
|
|
1821
|
+
const val = Reflect.get(t, prop, t);
|
|
1822
|
+
return typeof val === "function" ? val.bind(t) : val;
|
|
1823
|
+
},
|
|
1824
|
+
});
|
|
1825
|
+
}
|
|
1826
|
+
// ---------------------------------------------------------------------------
|
|
1827
|
+
// Email handler span naming/attrs (privacy-preserving)
|
|
1828
|
+
// ---------------------------------------------------------------------------
|
|
1829
|
+
/**
|
|
1830
|
+
* Extract the DOMAIN of an email address, or "" for anything malformed.
|
|
1831
|
+
* Deliberately never returns the local part — full addresses are PII.
|
|
1832
|
+
*/
|
|
1833
|
+
function emailDomain(address) {
|
|
1834
|
+
if (typeof address !== "string")
|
|
1835
|
+
return "";
|
|
1836
|
+
const at = address.lastIndexOf("@");
|
|
1837
|
+
if (at <= 0 || at === address.length - 1)
|
|
1838
|
+
return "";
|
|
1839
|
+
return address.slice(at + 1).toLowerCase();
|
|
1840
|
+
}
|
|
1841
|
+
/** SERVER-span name + attributes for an email handler invocation. */
|
|
1842
|
+
function emailHandlerAttrs(message) {
|
|
1843
|
+
const fromDomain = emailDomain(message?.from);
|
|
1844
|
+
const toDomain = emailDomain(message?.to);
|
|
1845
|
+
const attrs = {};
|
|
1846
|
+
if (fromDomain)
|
|
1847
|
+
attrs["cloudflare.email.from_domain"] = fromDomain;
|
|
1848
|
+
if (toDomain)
|
|
1849
|
+
attrs["cloudflare.email.to_domain"] = toDomain;
|
|
1850
|
+
const size = message?.rawSize;
|
|
1851
|
+
if (typeof size === "number" && Number.isFinite(size)) {
|
|
1852
|
+
attrs["cloudflare.email.raw_size_bytes"] = size;
|
|
1853
|
+
}
|
|
1854
|
+
return { name: fromDomain ? `email ${fromDomain}` : "email", attrs };
|
|
1855
|
+
}
|
|
1856
|
+
function classStartSpan(opts) {
|
|
1857
|
+
return (opts.startSpan ??
|
|
1858
|
+
((name, attrs, kind, ctx) => trace.getTracer("heystack").startSpan(name, { attributes: attrs, kind }, ctx ?? context.active())));
|
|
1859
|
+
}
|
|
1860
|
+
/**
|
|
1861
|
+
* Keep the isolate alive until pending exports drain: hand `flushHeystack()`
|
|
1862
|
+
* to the invocation's DurableObjectState `waitUntil`. Best-effort no-op when no
|
|
1863
|
+
* state/waitUntil is available (e.g. unit tests).
|
|
1864
|
+
*/
|
|
1865
|
+
function scheduleStateFlush(state) {
|
|
1866
|
+
const waitUntil = state
|
|
1867
|
+
?.waitUntil;
|
|
1868
|
+
if (typeof waitUntil !== "function")
|
|
1869
|
+
return;
|
|
1870
|
+
try {
|
|
1871
|
+
waitUntil.call(state, flushHeystack());
|
|
1872
|
+
}
|
|
1873
|
+
catch {
|
|
1874
|
+
// A failing waitUntil must never break the wrapped handler.
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
/** INTERNAL child spans for DurableObjectStorage get/put/delete/list ops. */
|
|
1878
|
+
function wrapDurableObjectStorage(storage, startSpan) {
|
|
1879
|
+
const wrapOp = (op) => async (...args) => {
|
|
1880
|
+
const underlying = storage[op];
|
|
1881
|
+
if (typeof underlying !== "function")
|
|
1882
|
+
return undefined;
|
|
1883
|
+
if (isTracingSuppressed(context.active())) {
|
|
1884
|
+
return underlying.apply(storage, args);
|
|
1885
|
+
}
|
|
1886
|
+
const attrs = {};
|
|
1887
|
+
const first = args[0];
|
|
1888
|
+
if (typeof first === "string") {
|
|
1889
|
+
attrs["cloudflare.do.storage.key"] = first;
|
|
1890
|
+
}
|
|
1891
|
+
else if (Array.isArray(first)) {
|
|
1892
|
+
attrs["cloudflare.do.storage.key_count"] = first.length;
|
|
1893
|
+
}
|
|
1894
|
+
else if (op === "list" &&
|
|
1895
|
+
first &&
|
|
1896
|
+
typeof first === "object" &&
|
|
1897
|
+
typeof first.limit === "number") {
|
|
1898
|
+
attrs["cloudflare.do.storage.limit"] = first.limit;
|
|
1899
|
+
}
|
|
1900
|
+
const span = startSpan(`DO storage ${op}`, attrs, SpanKind.INTERNAL);
|
|
1901
|
+
return runWithSpan(span, () => Promise.resolve(underlying.apply(storage, args)));
|
|
1902
|
+
};
|
|
1903
|
+
// A dedicated proxy (not makeProxy) because `storage` is a VALUE property on
|
|
1904
|
+
// the state object, not a method — handlers there must be getters.
|
|
1905
|
+
return new Proxy(storage, {
|
|
1906
|
+
get(t, prop, _receiver) {
|
|
1907
|
+
const val = Reflect.get(t, prop, t);
|
|
1908
|
+
if ((prop === "get" || prop === "put" || prop === "delete" || prop === "list") && typeof val === "function") {
|
|
1909
|
+
return wrapOp(prop);
|
|
1910
|
+
}
|
|
1911
|
+
return typeof val === "function" ? val.bind(t) : val;
|
|
1912
|
+
},
|
|
1913
|
+
});
|
|
1914
|
+
}
|
|
1915
|
+
/** Proxy a DurableObjectState so `state.storage` resolves to the traced proxy. */
|
|
1916
|
+
function wrapDurableObjectState(state, startSpan) {
|
|
1917
|
+
if (!state || typeof state !== "object")
|
|
1918
|
+
return state;
|
|
1919
|
+
const s = state;
|
|
1920
|
+
if (!s.storage || typeof s.storage !== "object")
|
|
1921
|
+
return state;
|
|
1922
|
+
const tracedStorage = wrapDurableObjectStorage(s.storage, startSpan);
|
|
1923
|
+
return new Proxy(s, {
|
|
1924
|
+
get(t, prop, _receiver) {
|
|
1925
|
+
if (prop === "storage")
|
|
1926
|
+
return tracedStorage;
|
|
1927
|
+
const val = Reflect.get(t, prop, t);
|
|
1928
|
+
return typeof val === "function" ? val.bind(t) : val;
|
|
1929
|
+
},
|
|
1930
|
+
});
|
|
1931
|
+
}
|
|
1932
|
+
/** Prototype methods that are wrapped specially (never treated as RPC). */
|
|
1933
|
+
const DO_LIFECYCLE_METHODS = new Set(["fetch", "alarm", "constructor"]);
|
|
1934
|
+
const MAX_DO_RPC_METHODS = 32;
|
|
1935
|
+
/**
|
|
1936
|
+
* Decorate a Durable Object class:
|
|
1937
|
+
* - constructor: wraps the incoming `state` so `state.storage` operations emit
|
|
1938
|
+
* INTERNAL child spans (`DO storage get|put|delete|list`) with key/count attrs.
|
|
1939
|
+
* - `fetch()`: SERVER span named `<METHOD> <path>` (like instrument()'s fetch
|
|
1940
|
+
* wrapper), continuing an inbound W3C traceparent, recording status/duration.
|
|
1941
|
+
* - `alarm()`: INTERNAL span.
|
|
1942
|
+
* - RPC prototype methods: SERVER span `DO rpc <method>` with rpc.method attrs.
|
|
1943
|
+
*
|
|
1944
|
+
* Every root span schedules flushHeystack() via `state.waitUntil` so exports
|
|
1945
|
+
* complete before isolate teardown. Instance arrow-function methods assigned in
|
|
1946
|
+
* the constructor are not enumerable on the prototype and stay unwrapped.
|
|
1947
|
+
*
|
|
1948
|
+
* export class Counter extends DurableObject {
|
|
1949
|
+
* // ...
|
|
1950
|
+
* }
|
|
1951
|
+
* export const Counter = wrapDurableObjectClass(BaseCounter); // or decorate in place
|
|
1952
|
+
*/
|
|
1953
|
+
export function wrapDurableObjectClass(cls, opts = {}) {
|
|
1954
|
+
const Base = cls;
|
|
1955
|
+
const startSpan = classStartSpan(opts);
|
|
1956
|
+
const proto = Base.prototype;
|
|
1957
|
+
const origFetch = proto.fetch;
|
|
1958
|
+
const origAlarm = proto.alarm;
|
|
1959
|
+
// Enumerate the RPC surface (prototype chain, lifecycle handlers excluded).
|
|
1960
|
+
const rpcMethods = [];
|
|
1961
|
+
try {
|
|
1962
|
+
let p = Base.prototype;
|
|
1963
|
+
for (let depth = 0; depth < 5 && p && p !== Object.prototype && rpcMethods.length < MAX_DO_RPC_METHODS; depth++) {
|
|
1964
|
+
for (const key of Object.getOwnPropertyNames(p)) {
|
|
1965
|
+
if (DO_LIFECYCLE_METHODS.has(key))
|
|
1966
|
+
continue;
|
|
1967
|
+
if (!rpcMethods.includes(key) && typeof proto[key] === "function") {
|
|
1968
|
+
rpcMethods.push(key);
|
|
1969
|
+
if (rpcMethods.length >= MAX_DO_RPC_METHODS)
|
|
1970
|
+
break;
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
p = Object.getPrototypeOf(p);
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
catch {
|
|
1977
|
+
// Best-effort enumeration — a hostile getter must never break decoration.
|
|
1978
|
+
}
|
|
1979
|
+
class Wrapped extends Base {
|
|
1980
|
+
/** Raw (unwrapped) state captured for flush scheduling. */
|
|
1981
|
+
__heystackState;
|
|
1982
|
+
constructor(...args) {
|
|
1983
|
+
super(...args.map((arg, i) => i === 0 ? wrapDurableObjectState(arg, startSpan) : arg));
|
|
1984
|
+
this.__heystackState = args[0];
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
if (typeof origFetch === "function") {
|
|
1988
|
+
Wrapped.prototype.fetch = async function (request) {
|
|
1989
|
+
if (isTracingSuppressed(context.active())) {
|
|
1990
|
+
return origFetch.call(this, request);
|
|
1991
|
+
}
|
|
1992
|
+
// Continue an inbound W3C traceparent so caller→DO is one distributed trace.
|
|
1993
|
+
let startCtx = context.active();
|
|
1994
|
+
const parent = parseTraceparent(request.headers.get("traceparent"));
|
|
1995
|
+
if (parent) {
|
|
1996
|
+
startCtx = trace.setSpanContext(startCtx, {
|
|
1997
|
+
traceId: parent.traceId,
|
|
1998
|
+
spanId: parent.spanId,
|
|
1999
|
+
traceFlags: parent.traceFlags,
|
|
2000
|
+
isRemote: true,
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
let url;
|
|
2004
|
+
try {
|
|
2005
|
+
url = new URL(request.url);
|
|
2006
|
+
}
|
|
2007
|
+
catch {
|
|
2008
|
+
url = new URL("https://durable-object.invalid/");
|
|
2009
|
+
}
|
|
2010
|
+
const span = startSpan(`${request.method} ${url.pathname}`, {
|
|
2011
|
+
"http.request.method": request.method,
|
|
2012
|
+
"url.path": url.pathname,
|
|
2013
|
+
"server.address": url.host,
|
|
2014
|
+
}, SpanKind.SERVER, startCtx);
|
|
2015
|
+
try {
|
|
2016
|
+
const response = (await context.with(trace.setSpan(startCtx, span), () => origFetch.call(this, request)));
|
|
2017
|
+
span.setAttribute("http.response.status_code", response.status);
|
|
2018
|
+
span.setStatus({
|
|
2019
|
+
code: response.status >= 500 ? SpanStatusCode.ERROR : SpanStatusCode.UNSET,
|
|
2020
|
+
});
|
|
2021
|
+
return response;
|
|
2022
|
+
}
|
|
2023
|
+
catch (error) {
|
|
2024
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
2025
|
+
span.setStatus({
|
|
2026
|
+
code: SpanStatusCode.ERROR,
|
|
2027
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2028
|
+
});
|
|
2029
|
+
throw error;
|
|
2030
|
+
}
|
|
2031
|
+
finally {
|
|
2032
|
+
span.end();
|
|
2033
|
+
scheduleStateFlush(this.__heystackState);
|
|
2034
|
+
}
|
|
2035
|
+
};
|
|
2036
|
+
}
|
|
2037
|
+
if (typeof origAlarm === "function") {
|
|
2038
|
+
Wrapped.prototype.alarm = async function (...args) {
|
|
2039
|
+
if (isTracingSuppressed(context.active())) {
|
|
2040
|
+
return origAlarm.apply(this, args);
|
|
2041
|
+
}
|
|
2042
|
+
const span = startSpan("DO alarm", {}, SpanKind.INTERNAL);
|
|
2043
|
+
try {
|
|
2044
|
+
return await context.with(trace.setSpan(context.active(), span), () => origAlarm.apply(this, args));
|
|
2045
|
+
}
|
|
2046
|
+
catch (error) {
|
|
2047
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
2048
|
+
span.setStatus({
|
|
2049
|
+
code: SpanStatusCode.ERROR,
|
|
2050
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2051
|
+
});
|
|
2052
|
+
throw error;
|
|
2053
|
+
}
|
|
2054
|
+
finally {
|
|
2055
|
+
span.end();
|
|
2056
|
+
scheduleStateFlush(this.__heystackState);
|
|
2057
|
+
}
|
|
2058
|
+
};
|
|
2059
|
+
}
|
|
2060
|
+
for (const method of rpcMethods) {
|
|
2061
|
+
const orig = proto[method];
|
|
2062
|
+
Wrapped.prototype[method] = async function (...args) {
|
|
2063
|
+
if (isTracingSuppressed(context.active())) {
|
|
2064
|
+
return orig.apply(this, args);
|
|
2065
|
+
}
|
|
2066
|
+
const span = startSpan(`DO rpc ${method}`, {
|
|
2067
|
+
"rpc.system": "cloudflare.durable_object",
|
|
2068
|
+
"rpc.method": method,
|
|
2069
|
+
}, SpanKind.SERVER);
|
|
2070
|
+
try {
|
|
2071
|
+
return await context.with(trace.setSpan(context.active(), span), () => orig.apply(this, args));
|
|
2072
|
+
}
|
|
2073
|
+
catch (error) {
|
|
2074
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
2075
|
+
span.setStatus({
|
|
2076
|
+
code: SpanStatusCode.ERROR,
|
|
2077
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2078
|
+
});
|
|
2079
|
+
throw error;
|
|
2080
|
+
}
|
|
2081
|
+
finally {
|
|
2082
|
+
span.end();
|
|
2083
|
+
scheduleStateFlush(this.__heystackState);
|
|
2084
|
+
}
|
|
2085
|
+
};
|
|
2086
|
+
}
|
|
2087
|
+
return Wrapped;
|
|
2088
|
+
}
|
|
2089
|
+
// ---------------------------------------------------------------------------
|
|
2090
|
+
// Workflows ENTRYPOINT class instrumentation
|
|
2091
|
+
// ---------------------------------------------------------------------------
|
|
2092
|
+
/** Step methods that are async WAITS — traced as short marker spans, never held open. */
|
|
2093
|
+
const WORKFLOW_WAIT_METHODS = ["sleep", "sleepUntil", "waitForEvent", "waitForName"];
|
|
2094
|
+
/** Marker-span attributes for a wait call (duration/timestamp only — no payload). */
|
|
2095
|
+
function workflowWaitAttrs(method, args) {
|
|
2096
|
+
if (method === "sleep" && args.length > 1 && args[1] != null) {
|
|
2097
|
+
return { "cloudflare.workflow.sleep_duration": String(args[1]) };
|
|
2098
|
+
}
|
|
2099
|
+
if (method === "sleepUntil" && args.length > 1 && args[1] != null) {
|
|
2100
|
+
const ts = args[1] instanceof Date ? args[1].toISOString() : String(args[1]);
|
|
2101
|
+
return { "cloudflare.workflow.sleep_until": ts };
|
|
2102
|
+
}
|
|
2103
|
+
return undefined;
|
|
2104
|
+
}
|
|
2105
|
+
/** Privacy-safe attributes from Cloudflare's real WorkflowEvent contract. */
|
|
2106
|
+
function workflowRunAttrs(event) {
|
|
2107
|
+
if (!event || typeof event !== "object")
|
|
2108
|
+
return {};
|
|
2109
|
+
const value = event;
|
|
2110
|
+
const attrs = {};
|
|
2111
|
+
if (typeof value.workflowName === "string") {
|
|
2112
|
+
attrs["cloudflare.workflow.name"] = value.workflowName;
|
|
2113
|
+
}
|
|
2114
|
+
if (typeof value.instanceId === "string") {
|
|
2115
|
+
attrs["cloudflare.workflow.instance_id"] = value.instanceId;
|
|
2116
|
+
}
|
|
2117
|
+
if (value.timestamp instanceof Date && Number.isFinite(value.timestamp.getTime())) {
|
|
2118
|
+
attrs["cloudflare.workflow.timestamp"] = value.timestamp.toISOString();
|
|
2119
|
+
}
|
|
2120
|
+
if (value.schedule && typeof value.schedule === "object") {
|
|
2121
|
+
if (typeof value.schedule.cron === "string") {
|
|
2122
|
+
attrs["cloudflare.workflow.schedule.cron"] = value.schedule.cron;
|
|
2123
|
+
}
|
|
2124
|
+
if (typeof value.schedule.scheduledTime === "number" &&
|
|
2125
|
+
Number.isFinite(value.schedule.scheduledTime)) {
|
|
2126
|
+
attrs["cloudflare.workflow.schedule.scheduled_time"] = value.schedule.scheduledTime;
|
|
2127
|
+
}
|
|
2128
|
+
}
|
|
2129
|
+
return attrs;
|
|
2130
|
+
}
|
|
2131
|
+
/**
|
|
2132
|
+
* Wrap a WorkflowStep so `do(name, fn)` executes fn inside a step span while
|
|
2133
|
+
* preserving engine semantics (we wrap FN and delegate to the real step.do —
|
|
2134
|
+
* the workflow engine owns execution/caching across replays), and wait-style
|
|
2135
|
+
* methods become short MARKER spans that end immediately after the wait is
|
|
2136
|
+
* initiated (a workflow can suspend for minutes; holding a span open across a
|
|
2137
|
+
* sleep would poison duration metrics).
|
|
2138
|
+
*/
|
|
2139
|
+
function wrapWorkflowStep(step, startSpan, state) {
|
|
2140
|
+
const handlers = {};
|
|
2141
|
+
if (typeof step.do === "function") {
|
|
2142
|
+
handlers.do = (name, fn, ...rest) => {
|
|
2143
|
+
if (isTracingSuppressed(context.active()) || typeof fn !== "function") {
|
|
2144
|
+
return step.do(name, fn, ...rest);
|
|
2145
|
+
}
|
|
2146
|
+
return step.do(name, async (...fnArgs) => {
|
|
2147
|
+
const span = startSpan(`step ${String(name)}`, {}, SpanKind.INTERNAL);
|
|
2148
|
+
try {
|
|
2149
|
+
return await context.with(trace.setSpan(context.active(), span), () => Promise.resolve(fn(...fnArgs)));
|
|
2150
|
+
}
|
|
2151
|
+
catch (error) {
|
|
2152
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
2153
|
+
span.setStatus({
|
|
2154
|
+
code: SpanStatusCode.ERROR,
|
|
2155
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2156
|
+
});
|
|
2157
|
+
throw error;
|
|
2158
|
+
}
|
|
2159
|
+
finally {
|
|
2160
|
+
span.end();
|
|
2161
|
+
}
|
|
2162
|
+
}, ...rest);
|
|
2163
|
+
};
|
|
2164
|
+
}
|
|
2165
|
+
for (const method of WORKFLOW_WAIT_METHODS) {
|
|
2166
|
+
if (typeof step[method] !== "function")
|
|
2167
|
+
continue;
|
|
2168
|
+
handlers[method] = (...args) => {
|
|
2169
|
+
if (isTracingSuppressed(context.active())) {
|
|
2170
|
+
return step[method](...args);
|
|
2171
|
+
}
|
|
2172
|
+
const nameArg = typeof args[0] === "string" ? ` ${args[0]}` : "";
|
|
2173
|
+
const span = startSpan(`workflow ${method}${nameArg}`, workflowWaitAttrs(method, args) ?? {}, SpanKind.INTERNAL);
|
|
2174
|
+
try {
|
|
2175
|
+
// Marker semantics: end the span synchronously after initiating the
|
|
2176
|
+
// wait — the returned promise is awaited by the caller OUTSIDE the span.
|
|
2177
|
+
return step[method](...args);
|
|
2178
|
+
}
|
|
2179
|
+
catch (error) {
|
|
2180
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
2181
|
+
span.setStatus({
|
|
2182
|
+
code: SpanStatusCode.ERROR,
|
|
2183
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2184
|
+
});
|
|
2185
|
+
throw error;
|
|
2186
|
+
}
|
|
2187
|
+
finally {
|
|
2188
|
+
span.end();
|
|
2189
|
+
// The caller suspends only after this wrapper returns the wait promise,
|
|
2190
|
+
// so hand the ended marker span to waitUntil before that can happen.
|
|
2191
|
+
scheduleStateFlush(state);
|
|
2192
|
+
}
|
|
2193
|
+
};
|
|
2194
|
+
}
|
|
2195
|
+
return makeProxy(step, handlers);
|
|
2196
|
+
}
|
|
2197
|
+
/**
|
|
2198
|
+
* Decorate a Cloudflare Workflows entrypoint class so its `run(event, step)`
|
|
2199
|
+
* emits a `workflow run` INTERNAL span with workflow identity/schedule metadata
|
|
2200
|
+
* (never payload content), and the handed-out
|
|
2201
|
+
* step object's methods become traced versions (see {@link wrapWorkflowStep}):
|
|
2202
|
+
* `step.do(name, fn)` → INTERNAL span named `step <name>` with success/error
|
|
2203
|
+
* status (step retries manifest as whole-run replays by the engine and are not
|
|
2204
|
+
* observable at this layer, so no retry attribute is emitted), and
|
|
2205
|
+
* sleep/sleepUntil/waitForEvent/waitForName → short marker spans.
|
|
2206
|
+
*
|
|
2207
|
+
* Every completed run schedules flushHeystack() via the entrypoint state's
|
|
2208
|
+
* `waitUntil` so exports survive isolate teardown.
|
|
2209
|
+
*/
|
|
2210
|
+
export function wrapWorkflowEntrypointClass(cls, opts = {}) {
|
|
2211
|
+
const Base = cls;
|
|
2212
|
+
const startSpan = classStartSpan(opts);
|
|
2213
|
+
const origRun = Base.prototype.run;
|
|
2214
|
+
if (typeof origRun !== "function")
|
|
2215
|
+
return cls;
|
|
2216
|
+
class Wrapped extends Base {
|
|
2217
|
+
__heystackState;
|
|
2218
|
+
constructor(...args) {
|
|
2219
|
+
super(...args);
|
|
2220
|
+
this.__heystackState = args[0];
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
Wrapped.prototype.run = async function (event, step, ...rest) {
|
|
2224
|
+
if (isTracingSuppressed(context.active())) {
|
|
2225
|
+
return origRun.call(this, event, step, ...rest);
|
|
2226
|
+
}
|
|
2227
|
+
const attrs = workflowRunAttrs(event);
|
|
2228
|
+
const span = startSpan("workflow run", attrs, SpanKind.INTERNAL);
|
|
2229
|
+
try {
|
|
2230
|
+
return await context.with(trace.setSpan(context.active(), span), () => origRun.call(this, event, step && typeof step === "object"
|
|
2231
|
+
? wrapWorkflowStep(step, startSpan, this.__heystackState)
|
|
2232
|
+
: step, ...rest));
|
|
2233
|
+
}
|
|
2234
|
+
catch (error) {
|
|
2235
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
2236
|
+
span.setStatus({
|
|
2237
|
+
code: SpanStatusCode.ERROR,
|
|
2238
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2239
|
+
});
|
|
2240
|
+
throw error;
|
|
2241
|
+
}
|
|
2242
|
+
finally {
|
|
2243
|
+
span.end();
|
|
2244
|
+
scheduleStateFlush(this.__heystackState);
|
|
2245
|
+
}
|
|
2246
|
+
};
|
|
2247
|
+
return Wrapped;
|
|
2248
|
+
}
|
|
1401
2249
|
export async function withSpan(name, attrsOrFn, maybeFn) {
|
|
1402
2250
|
const attrs = typeof attrsOrFn === "function" ? undefined : attrsOrFn;
|
|
1403
2251
|
const fn = (typeof attrsOrFn === "function" ? attrsOrFn : maybeFn);
|
|
1404
2252
|
const nativeTracing = nativeTracingFromContext();
|
|
1405
2253
|
if (nativeTracing) {
|
|
1406
2254
|
return nativeTracing.enterSpan(name, async (nativeSpan) => {
|
|
1407
|
-
const adapter = new NativeSpanAdapter(nativeSpan
|
|
2255
|
+
const adapter = new NativeSpanAdapter(nativeSpan);
|
|
1408
2256
|
if (attrs)
|
|
1409
2257
|
adapter.setAttributes(attrs);
|
|
1410
2258
|
const nativeContext = trace.setSpan(context.active().setValue(NATIVE_TRACING_CONTEXT_KEY, nativeTracing), adapter);
|