@heystack/otel 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -4
- package/dist/workers-bindings.d.ts +26 -8
- package/dist/workers-bindings.js +167 -7
- package/dist/workers-sampler.d.ts +23 -1
- package/dist/workers-sampler.js +49 -2
- package/dist/workers.d.ts +6 -0
- package/dist/workers.js +22 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -90,7 +90,7 @@ export default instrument(
|
|
|
90
90
|
getUser: (req) => ({
|
|
91
91
|
id: req.headers.get("x-user-id") ?? undefined,
|
|
92
92
|
}),
|
|
93
|
-
instrumentBindings: true, // auto-trace D1/KV/R2/Vectorize
|
|
93
|
+
instrumentBindings: true, // auto-trace D1/KV/R2/Vectorize/AI/Queues/Service bindings
|
|
94
94
|
},
|
|
95
95
|
);
|
|
96
96
|
```
|
|
@@ -116,8 +116,8 @@ Set the key as a secret: `wrangler secret put HEYSTACK_API_KEY`.
|
|
|
116
116
|
| `service` | `string` | **Required.** Service name that appears in the Heystack console. |
|
|
117
117
|
| `apiKey` | `string?` | Defaults to `env.HEYSTACK_API_KEY`. |
|
|
118
118
|
| `getUser` | `(req: Request) => { id?, session?, requestId? } \| undefined` | Called per request. `id` → `enduser.id`, `session` → `session.id`, `requestId` → `http.request.id` (falls back to the `cf-ray` header). |
|
|
119
|
-
| `instrumentBindings` | `boolean \| string[]` | `true` = auto child spans for all detected D1/KV/R2/Vectorize bindings; `string[]` = only the named bindings. Default `false`. |
|
|
120
|
-
| `sampling` | `{ rate?: number }` | Head-sampling rate 0–1
|
|
119
|
+
| `instrumentBindings` | `boolean \| string[]` | `true` = auto child spans for all detected D1 / KV / R2 / Vectorize / Workers AI / Queue producer / Service binding bindings; `string[]` = only the named bindings. Default `false`. |
|
|
120
|
+
| `sampling` | `{ rate?: number } \| { remote: true }` | Head-sampling configuration. `{ rate }`: keep a deterministic fraction of fresh root traces (0–1; default `1` = keep all). `{ remote: true }`: fetch the rate from the Heystack config endpoint instead — lets you change it centrally without redeploying. Cold isolates keep all traffic until the first config fetch resolves; fails open if the config can't be reached. Parent-respecting in both modes: a request arriving with a sampled `traceparent` is always recorded. See [Head sampling](#head-sampling) below. |
|
|
121
121
|
| `waitUntil` | `(p: Promise<unknown>) => void` | Override the isolate keep-alive hook; defaults to the auto-detected `ctx.waitUntil`. |
|
|
122
122
|
| `endpoint` | `string?` | Override the ingest endpoint (advanced). |
|
|
123
123
|
|
|
@@ -138,6 +138,19 @@ The sampler is **deterministic by trace ID**: the same trace always makes the sa
|
|
|
138
138
|
|
|
139
139
|
The `rate` governs **fresh root traces only** (no inbound `traceparent`, or `traceparent` with `sampled=0`). A request arriving with a sampled inbound `traceparent` (`sampled=1`) is always recorded — the parent's decision is respected, so a distributed trace is never split mid-flight by the child's sample rate.
|
|
140
140
|
|
|
141
|
+
### Remote sampling
|
|
142
|
+
|
|
143
|
+
`sampling: { remote: true }` lets Heystack control the rate centrally — change it from the dashboard without redeploying:
|
|
144
|
+
|
|
145
|
+
```ts
|
|
146
|
+
export default instrument(worker, {
|
|
147
|
+
service: "my-api",
|
|
148
|
+
sampling: { remote: true },
|
|
149
|
+
});
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
On startup the worker fetches its configured rate from the Heystack config endpoint. **Cold isolates keep all traffic until that first fetch resolves** (fails open — nothing is dropped while loading). If the config endpoint can't be reached, the worker keeps everything. The same parent-respecting rule applies: a request with an inbound sampled `traceparent` is always recorded regardless of the fetched rate.
|
|
153
|
+
|
|
141
154
|
### Automatic tracing
|
|
142
155
|
|
|
143
156
|
`instrument()` traces the following automatically, with no additional config:
|
|
@@ -146,7 +159,7 @@ The `rate` governs **fresh root traces only** (no inbound `traceparent`, or `tra
|
|
|
146
159
|
- **Outbound `fetch`** — each outbound subrequest while a request span is active gets a CLIENT child span (`http.request.method`, `url.full`, `server.address`, `http.response.status_code`). A W3C `traceparent` header is injected into the subrequest so a downstream Heystack-instrumented service continues the same trace (distributed tracing across services). The exporter's own ingest POST is never traced.
|
|
147
160
|
- **Queue consumers (`queue`)** — a CONSUMER span per batch, with `messaging.destination.name` (queue name) and `messaging.batch.message_count`.
|
|
148
161
|
- **Scheduled handlers (`scheduled`)** — an INTERNAL span per invocation, with `controller.cron`.
|
|
149
|
-
- **Binding calls** (when `instrumentBindings` is set) — a child span for every D1 query (`db.statement`), KV read/write, R2 operation, and
|
|
162
|
+
- **Binding calls** (when `instrumentBindings` is set) — a child span for every D1 query (`db.statement`), KV read/write, R2 operation, Vectorize query, Workers AI inference (`gen_ai.*` attributes including model name and token usage), Queue `.send`/`.sendBatch` (PRODUCER spans with `messaging.*` attributes), and Service binding `.fetch` calls (CLIENT spans with `traceparent` injected so calls to other Workers stitch into the same distributed trace).
|
|
150
163
|
|
|
151
164
|
### Client enrichment
|
|
152
165
|
|
|
@@ -292,6 +305,8 @@ As belt-and-suspenders the exporter also drops any span whose HTTP target points
|
|
|
292
305
|
|
|
293
306
|
## Migration / versioning
|
|
294
307
|
|
|
308
|
+
- **`0.8.0`** — **`/workers`: Workers AI, Queue producer, and Service binding instrumentation.** `instrumentBindings: true` now auto-wraps three additional binding types: `env.AI.run()` emits CLIENT spans with `gen_ai.system`, `gen_ai.request.model`, and `gen_ai.usage.input_tokens`/`output_tokens` (streaming results are never consumed); Queue `.send`/`.sendBatch` emit PRODUCER spans with `messaging.*` attributes including batch size; Service binding `.fetch` emits a CLIENT span and injects a W3C `traceparent` header into the outgoing request so calls to other Workers appear in the same distributed trace. `startSpan` factory now accepts an optional `SpanKind` for correct CLIENT/PRODUCER categorisation. No breaking changes.
|
|
309
|
+
- **`0.7.0`** — **`/workers`: remote sampling (`sampling: { remote: true }`).** New `sampling` variant that fetches the head-sampling rate from the Heystack config endpoint at runtime, so you can change it from the console without redeploying. Cold isolates keep all traffic until the first config fetch resolves (fails open). If the config endpoint is unreachable, the worker keeps everything. Same parent-respecting rule as `sampling: { rate }`. No breaking changes; existing `sampling: { rate }` configs are unchanged.
|
|
295
310
|
- **`0.6.0`** — **`/workers`: head sampling (`sampling: { rate }`).** New optional `WorkersConfig` field: `sampling.rate` (0–1, default `1`). Keeps a deterministic fraction of fresh root traces — the drop decision is made in the worker before export (no egress, no ingest cost). Parent-respecting: requests arriving with a sampled `traceparent` are always recorded. Consistent with server-side sampling (same trace-ID hash). No breaking changes; all new options are optional. See [Head sampling](#head-sampling).
|
|
296
311
|
- **`0.5.0`** — **`/workers`: identity enrichment, binding tracing, outbound-fetch tracing, manual span helpers.** New `WorkersConfig` options: `getUser` (attach `enduser.id`/`session.id`/`http.request.id` per request from a synchronous callback), `instrumentBindings` (auto child spans for D1/KV/R2/Vectorize — `true` = all detected, or a `string[]` to select). Outbound `fetch` calls made inside a traced handler automatically get CLIENT child spans with `traceparent` injection (distributed tracing across services). New ergonomic exports from `/workers`: `withSpan(name, attrs?, fn)` runs a function inside a named child span (auto-parented, exceptions recorded, `span.end()` in `finally`); `addEvent(name, attrs?)` adds an event to the active span. No breaking changes; all new options are optional.
|
|
297
312
|
- **`0.4.3`** — **feedback-loop guard extended to the Node path (cost fix).** The self-instrumentation loop the Workers path fixed in 0.3.1/0.3.2 was still live on plain Node / Next-on-Node: the OTLP-over-`http` exporter's `POST /v1/traces` was auto-instrumented and re-exported, so ~77% of ingested spans in production were the exporter tracing itself — needless ingest + ClickHouse compute. 0.4.3 ignores ingest-host requests in the HTTP/undici auto-instrumentations **and** filters self-spans at the exporter boundary (covers caller-supplied `instrumentations` too). The hostname matcher is now a shared module used by both `/node` and `/workers`. No API change. **Action: upgrade and redeploy any Node/Next-on-Node app** — it cuts ingested span volume sharply.
|
|
@@ -1,33 +1,51 @@
|
|
|
1
|
-
import { type Span } from "@opentelemetry/api";
|
|
1
|
+
import { SpanKind, type Span } from "@opentelemetry/api";
|
|
2
2
|
export interface InstrumentBindingsOpts {
|
|
3
3
|
/**
|
|
4
4
|
* Factory that creates and starts a new child span. Called at binding method
|
|
5
5
|
* invocation time (inside the traced handler scope), so `context.active()`
|
|
6
6
|
* at that moment correctly parents to the root span.
|
|
7
7
|
*
|
|
8
|
-
* For integration: pass `(name, attrs) => tracer.startSpan(name, { attributes: attrs }, context.active())`.
|
|
8
|
+
* For integration: pass `(name, attrs, kind) => tracer.startSpan(name, { attributes: attrs, kind }, context.active())`.
|
|
9
9
|
* For unit tests: inject a fake so no global provider is required.
|
|
10
10
|
*/
|
|
11
|
-
startSpan: (name: string, attrs: Record<string, unknown
|
|
11
|
+
startSpan: (name: string, attrs: Record<string, unknown>, kind?: SpanKind) => Span;
|
|
12
12
|
/**
|
|
13
|
-
* `true` → auto-detect and wrap all D1/KV/R2/Vectorize bindings.
|
|
13
|
+
* `true` → auto-detect and wrap all D1/KV/R2/Vectorize/AI/Queue/Service bindings.
|
|
14
14
|
* `string[]` → only wrap bindings whose env key is listed.
|
|
15
15
|
*/
|
|
16
16
|
select: boolean | string[];
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* Build a new args array for `fetch(input, init?)` with a `traceparent` header
|
|
20
|
+
* injected WITHOUT mutating the caller's original Request or init object.
|
|
21
|
+
*
|
|
22
|
+
* - `Request` first arg → rebuilt as `new Request(original, { headers })`.
|
|
23
|
+
* - String/URL first arg → spread a new init object with an augmented Headers.
|
|
24
|
+
*/
|
|
25
|
+
export declare function injectTraceparentArgs(args: any[], traceparent: string): any[];
|
|
18
26
|
/**
|
|
19
27
|
* Wrap an env object's Cloudflare bindings so that each binding operation
|
|
20
28
|
* emits a child span under the currently-active OTel context.
|
|
21
29
|
*
|
|
22
|
-
* Detects binding type by duck-typing
|
|
23
|
-
*
|
|
24
|
-
*
|
|
30
|
+
* Detects binding type by duck-typing:
|
|
31
|
+
* - D1: `prepare`
|
|
32
|
+
* - R2: `get`+`put`+`head`
|
|
33
|
+
* - KV: `get`+`put`+`list`
|
|
34
|
+
* - Vectorize: `query`+`upsert`
|
|
35
|
+
* - Workers AI: `run` (without `prepare`)
|
|
36
|
+
* - Queue producer: `send`+`sendBatch`
|
|
37
|
+
* - Service binding / Fetcher: `fetch` (last branch; most generic)
|
|
38
|
+
*
|
|
39
|
+
* Unrecognised bindings are passed through unchanged.
|
|
25
40
|
*
|
|
26
41
|
* Each wrapped binding is a `Proxy` over the original — non-wrapped prototype
|
|
27
42
|
* methods fall through to the real binding so no functionality is lost.
|
|
28
43
|
*
|
|
44
|
+
* Service-binding spans inject a W3C `traceparent` header into outgoing
|
|
45
|
+
* requests so calls to other Workers stitch into one distributed trace.
|
|
46
|
+
*
|
|
29
47
|
* @param env - The Worker env / binding bag.
|
|
30
|
-
* @param opts - `startSpan` factory + `select` filter.
|
|
48
|
+
* @param opts - `startSpan` factory (now accepts optional `SpanKind`) + `select` filter.
|
|
31
49
|
* @returns A shallow copy of `env` with selected bindings replaced by proxies.
|
|
32
50
|
*/
|
|
33
51
|
export declare function instrumentEnv<E extends Record<string, unknown>>(env: E, opts: InstrumentBindingsOpts): E;
|
package/dist/workers-bindings.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
// ---------------------------------------------------------------------------
|
|
2
2
|
// Cloudflare binding instrumentation for @heystack/otel/workers.
|
|
3
3
|
//
|
|
4
|
-
// Wraps D1, KV, R2,
|
|
5
|
-
//
|
|
4
|
+
// Wraps D1, KV, R2, Vectorize, Workers AI, Queue producers, and Service
|
|
5
|
+
// bindings with OTel child spans so that every binding operation is visible
|
|
6
|
+
// as a child of the active request span.
|
|
6
7
|
//
|
|
7
8
|
// WinterCG-safe: no `node:*` imports. Span factory is injected so the logic
|
|
8
9
|
// is pure and unit-testable without a global provider.
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
|
-
import { context, SpanStatusCode } from "@opentelemetry/api";
|
|
11
|
+
import { context, SpanKind, SpanStatusCode } from "@opentelemetry/api";
|
|
11
12
|
import { isTracingSuppressed } from "@opentelemetry/core";
|
|
12
13
|
// ---------------------------------------------------------------------------
|
|
13
14
|
// Duck-type detectors — conservative; require the distinctive method set
|
|
@@ -31,6 +32,24 @@ function isVectorizeLike(b) {
|
|
|
31
32
|
return (typeof b?.query === "function" &&
|
|
32
33
|
typeof b?.upsert === "function");
|
|
33
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* Workers AI: has `run` but NOT `prepare` (which would match D1 first anyway).
|
|
37
|
+
* The `prepare` exclusion is defensive — D1 is already matched before this branch.
|
|
38
|
+
*/
|
|
39
|
+
function isWorkersAILike(b) {
|
|
40
|
+
return (typeof b?.run === "function" &&
|
|
41
|
+
typeof b?.prepare !== "function");
|
|
42
|
+
}
|
|
43
|
+
/** Queue producer: exposes both `send` and `sendBatch`. */
|
|
44
|
+
function isQueueLike(b) {
|
|
45
|
+
return (typeof b?.send === "function" &&
|
|
46
|
+
typeof b?.sendBatch === "function");
|
|
47
|
+
}
|
|
48
|
+
/** Service binding / Fetcher: last branch — anything with `.fetch`. Prior branches
|
|
49
|
+
* exclude D1/R2/KV/Vectorize/AI/Queue, so order alone is sufficient. */
|
|
50
|
+
function isServiceLike(b) {
|
|
51
|
+
return typeof b?.fetch === "function";
|
|
52
|
+
}
|
|
34
53
|
// ---------------------------------------------------------------------------
|
|
35
54
|
// Span lifecycle helper
|
|
36
55
|
// ---------------------------------------------------------------------------
|
|
@@ -161,21 +180,153 @@ function wrapVectorize(binding, opts, indexName) {
|
|
|
161
180
|
});
|
|
162
181
|
}
|
|
163
182
|
// ---------------------------------------------------------------------------
|
|
183
|
+
// Workers AI wrapper
|
|
184
|
+
// ---------------------------------------------------------------------------
|
|
185
|
+
function wrapWorkersAI(binding, opts, _name) {
|
|
186
|
+
return makeProxy(binding, {
|
|
187
|
+
run: async (...args) => {
|
|
188
|
+
if (isTracingSuppressed(context.active())) {
|
|
189
|
+
return binding.run(...args);
|
|
190
|
+
}
|
|
191
|
+
const attrs = {
|
|
192
|
+
"gen_ai.system": "cloudflare.workers_ai",
|
|
193
|
+
};
|
|
194
|
+
if (typeof args[0] === "string") {
|
|
195
|
+
attrs["gen_ai.request.model"] = args[0];
|
|
196
|
+
}
|
|
197
|
+
const span = opts.startSpan("AI run", attrs, SpanKind.CLIENT);
|
|
198
|
+
try {
|
|
199
|
+
const result = await binding.run(...args);
|
|
200
|
+
// Best-effort usage extraction — never read/await/tee the stream.
|
|
201
|
+
try {
|
|
202
|
+
if (result &&
|
|
203
|
+
typeof result === "object" &&
|
|
204
|
+
!(result instanceof ReadableStream) &&
|
|
205
|
+
typeof result.usage === "object") {
|
|
206
|
+
const usage = result.usage;
|
|
207
|
+
if (typeof usage.prompt_tokens === "number") {
|
|
208
|
+
span.setAttribute("gen_ai.usage.input_tokens", usage.prompt_tokens);
|
|
209
|
+
}
|
|
210
|
+
if (typeof usage.completion_tokens === "number") {
|
|
211
|
+
span.setAttribute("gen_ai.usage.output_tokens", usage.completion_tokens);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
// Swallow — a weird shape must never throw.
|
|
217
|
+
}
|
|
218
|
+
return result;
|
|
219
|
+
}
|
|
220
|
+
catch (err) {
|
|
221
|
+
span.recordException(err instanceof Error ? err : new Error(String(err)));
|
|
222
|
+
span.setStatus({
|
|
223
|
+
code: SpanStatusCode.ERROR,
|
|
224
|
+
message: err instanceof Error ? err.message : String(err),
|
|
225
|
+
});
|
|
226
|
+
throw err;
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
229
|
+
span.end();
|
|
230
|
+
}
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
// ---------------------------------------------------------------------------
|
|
235
|
+
// Queue producer wrapper
|
|
236
|
+
// ---------------------------------------------------------------------------
|
|
237
|
+
function wrapQueue(binding, opts, name) {
|
|
238
|
+
const baseAttrs = {
|
|
239
|
+
"messaging.system": "cloudflare_queues",
|
|
240
|
+
"messaging.destination.name": name,
|
|
241
|
+
};
|
|
242
|
+
return makeProxy(binding, {
|
|
243
|
+
send: async (...args) => {
|
|
244
|
+
if (isTracingSuppressed(context.active())) {
|
|
245
|
+
return binding.send(...args);
|
|
246
|
+
}
|
|
247
|
+
const span = opts.startSpan("Queue send", { ...baseAttrs, "messaging.operation": "send" }, SpanKind.PRODUCER);
|
|
248
|
+
return runWithSpan(span, () => binding.send(...args));
|
|
249
|
+
},
|
|
250
|
+
sendBatch: async (...args) => {
|
|
251
|
+
if (isTracingSuppressed(context.active())) {
|
|
252
|
+
return binding.sendBatch(...args);
|
|
253
|
+
}
|
|
254
|
+
const batchAttrs = {
|
|
255
|
+
...baseAttrs,
|
|
256
|
+
"messaging.operation": "sendBatch",
|
|
257
|
+
};
|
|
258
|
+
if (Array.isArray(args[0])) {
|
|
259
|
+
batchAttrs["messaging.batch.message_count"] = args[0].length;
|
|
260
|
+
}
|
|
261
|
+
const span = opts.startSpan("Queue sendBatch", batchAttrs, SpanKind.PRODUCER);
|
|
262
|
+
return runWithSpan(span, () => binding.sendBatch(...args));
|
|
263
|
+
},
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
// ---------------------------------------------------------------------------
|
|
267
|
+
// Service binding / Fetcher wrapper + traceparent injection
|
|
268
|
+
// ---------------------------------------------------------------------------
|
|
269
|
+
/**
|
|
270
|
+
* Build a new args array for `fetch(input, init?)` with a `traceparent` header
|
|
271
|
+
* injected WITHOUT mutating the caller's original Request or init object.
|
|
272
|
+
*
|
|
273
|
+
* - `Request` first arg → rebuilt as `new Request(original, { headers })`.
|
|
274
|
+
* - String/URL first arg → spread a new init object with an augmented Headers.
|
|
275
|
+
*/
|
|
276
|
+
export function injectTraceparentArgs(args, traceparent) {
|
|
277
|
+
const [input, ...rest] = args;
|
|
278
|
+
if (input instanceof Request) {
|
|
279
|
+
const headers = new Headers(input.headers);
|
|
280
|
+
headers.set("traceparent", traceparent);
|
|
281
|
+
return [new Request(input, { headers }), ...rest];
|
|
282
|
+
}
|
|
283
|
+
// String or URL
|
|
284
|
+
const init = { ...(rest[0] ?? {}) };
|
|
285
|
+
const headers = new Headers(init.headers);
|
|
286
|
+
headers.set("traceparent", traceparent);
|
|
287
|
+
init.headers = headers;
|
|
288
|
+
return [input, init];
|
|
289
|
+
}
|
|
290
|
+
function wrapService(binding, opts, name) {
|
|
291
|
+
return makeProxy(binding, {
|
|
292
|
+
fetch: async (...args) => {
|
|
293
|
+
if (isTracingSuppressed(context.active())) {
|
|
294
|
+
return binding.fetch(...args);
|
|
295
|
+
}
|
|
296
|
+
const span = opts.startSpan(`Service ${name} fetch`, { "peer.service": name }, SpanKind.CLIENT);
|
|
297
|
+
const sc = span.spanContext();
|
|
298
|
+
const traceparent = `00-${sc.traceId}-${sc.spanId}-01`;
|
|
299
|
+
const injectedArgs = injectTraceparentArgs(args, traceparent);
|
|
300
|
+
return runWithSpan(span, () => binding.fetch(...injectedArgs));
|
|
301
|
+
},
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
// ---------------------------------------------------------------------------
|
|
164
305
|
// Main export
|
|
165
306
|
// ---------------------------------------------------------------------------
|
|
166
307
|
/**
|
|
167
308
|
* Wrap an env object's Cloudflare bindings so that each binding operation
|
|
168
309
|
* emits a child span under the currently-active OTel context.
|
|
169
310
|
*
|
|
170
|
-
* Detects binding type by duck-typing
|
|
171
|
-
*
|
|
172
|
-
*
|
|
311
|
+
* Detects binding type by duck-typing:
|
|
312
|
+
* - D1: `prepare`
|
|
313
|
+
* - R2: `get`+`put`+`head`
|
|
314
|
+
* - KV: `get`+`put`+`list`
|
|
315
|
+
* - Vectorize: `query`+`upsert`
|
|
316
|
+
* - Workers AI: `run` (without `prepare`)
|
|
317
|
+
* - Queue producer: `send`+`sendBatch`
|
|
318
|
+
* - Service binding / Fetcher: `fetch` (last branch; most generic)
|
|
319
|
+
*
|
|
320
|
+
* Unrecognised bindings are passed through unchanged.
|
|
173
321
|
*
|
|
174
322
|
* Each wrapped binding is a `Proxy` over the original — non-wrapped prototype
|
|
175
323
|
* methods fall through to the real binding so no functionality is lost.
|
|
176
324
|
*
|
|
325
|
+
* Service-binding spans inject a W3C `traceparent` header into outgoing
|
|
326
|
+
* requests so calls to other Workers stitch into one distributed trace.
|
|
327
|
+
*
|
|
177
328
|
* @param env - The Worker env / binding bag.
|
|
178
|
-
* @param opts - `startSpan` factory + `select` filter.
|
|
329
|
+
* @param opts - `startSpan` factory (now accepts optional `SpanKind`) + `select` filter.
|
|
179
330
|
* @returns A shallow copy of `env` with selected bindings replaced by proxies.
|
|
180
331
|
*/
|
|
181
332
|
export function instrumentEnv(env, opts) {
|
|
@@ -199,6 +350,15 @@ export function instrumentEnv(env, opts) {
|
|
|
199
350
|
else if (isVectorizeLike(binding)) {
|
|
200
351
|
result[key] = wrapVectorize(binding, opts, key);
|
|
201
352
|
}
|
|
353
|
+
else if (isWorkersAILike(binding)) {
|
|
354
|
+
result[key] = wrapWorkersAI(binding, opts, key);
|
|
355
|
+
}
|
|
356
|
+
else if (isQueueLike(binding)) {
|
|
357
|
+
result[key] = wrapQueue(binding, opts, key);
|
|
358
|
+
}
|
|
359
|
+
else if (isServiceLike(binding)) {
|
|
360
|
+
result[key] = wrapService(binding, opts, key);
|
|
361
|
+
}
|
|
202
362
|
// Unrecognised bindings are left as-is.
|
|
203
363
|
}
|
|
204
364
|
return result;
|
|
@@ -2,12 +2,34 @@ import { type Sampler, type SamplingResult } from "@opentelemetry/sdk-trace-base
|
|
|
2
2
|
import type { Context, Attributes, Link, SpanKind } from "@opentelemetry/api";
|
|
3
3
|
export declare function fnv01(s: string): number;
|
|
4
4
|
export declare function traceKept(traceId: string, rate: number): boolean;
|
|
5
|
+
/** Test-only: set the remote rate directly. */
|
|
6
|
+
export declare function __setRemoteRate(n: number): void;
|
|
7
|
+
/** Test-only: read the current remote rate. */
|
|
8
|
+
export declare function __getRemoteRate(): number;
|
|
5
9
|
export declare class HeystackRatioSampler implements Sampler {
|
|
6
10
|
private readonly rate;
|
|
7
|
-
constructor(rate: number);
|
|
11
|
+
constructor(rate: number | (() => number));
|
|
8
12
|
shouldSample(_ctx: Context, traceId: string, _name: string, _kind: SpanKind, _attrs: Attributes, _links: Link[]): SamplingResult;
|
|
9
13
|
toString(): string;
|
|
10
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* Fetch the sampling rate from the Heystack ingest config endpoint and update
|
|
17
|
+
* the module-level `_remoteRate` ref. Call once per isolate (guarded in
|
|
18
|
+
* `workers.ts` via `_remoteSamplingKicked`). Uses the provided `fetchImpl`
|
|
19
|
+
* (the captured pre-patch fetch) so the GET is never re-entered by outbound
|
|
20
|
+
* fetch instrumentation and is wrapped in `suppressTracing` at the call site
|
|
21
|
+
* in `workers.ts` (belt-and-suspenders against self-tracing).
|
|
22
|
+
*
|
|
23
|
+
* Fail-open: any network failure, non-200, or parse error leaves `_remoteRate`
|
|
24
|
+
* at its current value (initially 1 = keep-all). Remote sampling must never
|
|
25
|
+
* drop telemetry due to a config-service outage.
|
|
26
|
+
*/
|
|
27
|
+
export declare function loadRemoteSamplingRate(opts: {
|
|
28
|
+
endpoint: string;
|
|
29
|
+
apiKey: string;
|
|
30
|
+
fetchImpl: typeof fetch;
|
|
31
|
+
}): Promise<void>;
|
|
11
32
|
export declare function makeSampler(sampling?: {
|
|
12
33
|
rate?: number;
|
|
34
|
+
remote?: boolean;
|
|
13
35
|
}): Sampler;
|
package/dist/workers-sampler.js
CHANGED
|
@@ -16,23 +16,70 @@ export function traceKept(traceId, rate) {
|
|
|
16
16
|
return false;
|
|
17
17
|
return fnv01(traceId) < rate;
|
|
18
18
|
}
|
|
19
|
+
// Module-level mutable ref for the remote-loaded rate. Starts at 1 (keep-all)
|
|
20
|
+
// so cold-isolate requests are fully preserved until the config fetch resolves.
|
|
21
|
+
const _remoteRate = { value: 1 };
|
|
22
|
+
/** Test-only: set the remote rate directly. */
|
|
23
|
+
export function __setRemoteRate(n) {
|
|
24
|
+
_remoteRate.value = n;
|
|
25
|
+
}
|
|
26
|
+
/** Test-only: read the current remote rate. */
|
|
27
|
+
export function __getRemoteRate() {
|
|
28
|
+
return _remoteRate.value;
|
|
29
|
+
}
|
|
19
30
|
export class HeystackRatioSampler {
|
|
20
31
|
rate;
|
|
21
32
|
constructor(rate) {
|
|
22
33
|
this.rate = rate;
|
|
23
34
|
}
|
|
24
35
|
shouldSample(_ctx, traceId, _name, _kind, _attrs, _links) {
|
|
36
|
+
const rate = typeof this.rate === "function" ? this.rate() : this.rate;
|
|
25
37
|
return {
|
|
26
|
-
decision: traceKept(traceId,
|
|
38
|
+
decision: traceKept(traceId, rate)
|
|
27
39
|
? SamplingDecision.RECORD_AND_SAMPLED
|
|
28
40
|
: SamplingDecision.NOT_RECORD,
|
|
29
41
|
};
|
|
30
42
|
}
|
|
31
43
|
toString() {
|
|
32
|
-
|
|
44
|
+
const r = typeof this.rate === "function" ? this.rate() : this.rate;
|
|
45
|
+
return `HeystackRatioSampler{${r}}`;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Fetch the sampling rate from the Heystack ingest config endpoint and update
|
|
50
|
+
* the module-level `_remoteRate` ref. Call once per isolate (guarded in
|
|
51
|
+
* `workers.ts` via `_remoteSamplingKicked`). Uses the provided `fetchImpl`
|
|
52
|
+
* (the captured pre-patch fetch) so the GET is never re-entered by outbound
|
|
53
|
+
* fetch instrumentation and is wrapped in `suppressTracing` at the call site
|
|
54
|
+
* in `workers.ts` (belt-and-suspenders against self-tracing).
|
|
55
|
+
*
|
|
56
|
+
* Fail-open: any network failure, non-200, or parse error leaves `_remoteRate`
|
|
57
|
+
* at its current value (initially 1 = keep-all). Remote sampling must never
|
|
58
|
+
* drop telemetry due to a config-service outage.
|
|
59
|
+
*/
|
|
60
|
+
export async function loadRemoteSamplingRate(opts) {
|
|
61
|
+
try {
|
|
62
|
+
const url = `${opts.endpoint.replace(/\/+$/, "")}/v1/sampling/config`;
|
|
63
|
+
const res = await opts.fetchImpl(url, {
|
|
64
|
+
headers: { Authorization: `Bearer ${opts.apiKey}` },
|
|
65
|
+
});
|
|
66
|
+
if (!res.ok)
|
|
67
|
+
return; // fail open — keep current rate
|
|
68
|
+
const cfg = (await res.json());
|
|
69
|
+
const r = Number(cfg?.trace_sample_rate);
|
|
70
|
+
if (Number.isFinite(r) && r >= 0 && r <= 1)
|
|
71
|
+
_remoteRate.value = r;
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
/* fail open: leave rate at keep-all */
|
|
33
75
|
}
|
|
34
76
|
}
|
|
35
77
|
export function makeSampler(sampling) {
|
|
78
|
+
if (sampling?.remote) {
|
|
79
|
+
// Dynamic rate: reads from the per-isolate ref that loadRemoteSamplingRate sets.
|
|
80
|
+
// Starts at 1 (keep-all) until the config fetch resolves on the first request.
|
|
81
|
+
return new ParentBasedSampler({ root: new HeystackRatioSampler(() => _remoteRate.value) });
|
|
82
|
+
}
|
|
36
83
|
const rate = sampling?.rate;
|
|
37
84
|
if (rate === undefined || rate >= 1)
|
|
38
85
|
return new AlwaysOnSampler();
|
package/dist/workers.d.ts
CHANGED
|
@@ -168,9 +168,15 @@ export interface WorkersConfig {
|
|
|
168
168
|
* Note: head sampling is parent-respecting, so an incoming request carrying a
|
|
169
169
|
* sampled `traceparent` is still recorded even at `rate: 0` (it is not an
|
|
170
170
|
* absolute kill-switch; it governs only fresh/root traces).
|
|
171
|
+
*
|
|
172
|
+
* When `remote: true`, the sampling rate is fetched once per isolate from
|
|
173
|
+
* `{endpoint}/v1/sampling/config` on the first request. Cold-isolate spans
|
|
174
|
+
* are kept (rate=1) until the fetch resolves. On any failure the rate stays
|
|
175
|
+
* at keep-all (fail-open). Incompatible with inline `rate` (remote wins).
|
|
171
176
|
*/
|
|
172
177
|
sampling?: {
|
|
173
178
|
rate?: number;
|
|
179
|
+
remote?: boolean;
|
|
174
180
|
};
|
|
175
181
|
}
|
|
176
182
|
/**
|
package/dist/workers.js
CHANGED
|
@@ -13,10 +13,10 @@ import { ROOT_CONTEXT } from "@opentelemetry/api";
|
|
|
13
13
|
import { Resource } from "@opentelemetry/resources";
|
|
14
14
|
import { BasicTracerProvider, SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-base";
|
|
15
15
|
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
16
|
-
import { buildExporterConfig } from "./core.js";
|
|
16
|
+
import { buildExporterConfig, DEFAULT_ENDPOINT } from "./core.js";
|
|
17
17
|
import { isSelfSpanAttrs, safeHostname } from "./self-span.js";
|
|
18
18
|
import { instrumentEnv } from "./workers-bindings.js";
|
|
19
|
-
import { makeSampler } from "./workers-sampler.js";
|
|
19
|
+
import { makeSampler, loadRemoteSamplingRate } from "./workers-sampler.js";
|
|
20
20
|
// `ExportResult` / `ExportResultCode` mirror `@opentelemetry/core`. We define
|
|
21
21
|
// them inline (structurally identical) rather than import them: core is only a
|
|
22
22
|
// transitive dep of sdk-trace-base and isn't reliably resolvable, and keeping it
|
|
@@ -696,7 +696,10 @@ export async function flushHeystack() {
|
|
|
696
696
|
/** Reset the singleton global provider. Internal/testing helper. */
|
|
697
697
|
export function __resetProvider() {
|
|
698
698
|
_provider = null;
|
|
699
|
+
_remoteSamplingKicked = false;
|
|
699
700
|
}
|
|
701
|
+
/** Guard: the remote sampling config GET fires at most once per isolate. */
|
|
702
|
+
let _remoteSamplingKicked = false;
|
|
700
703
|
let warnedNoKey = false;
|
|
701
704
|
function warnOnceNoKey() {
|
|
702
705
|
if (warnedNoKey)
|
|
@@ -763,6 +766,22 @@ export function instrument(handler, config) {
|
|
|
763
766
|
if (!s)
|
|
764
767
|
return originalFetch(req, env, ctx);
|
|
765
768
|
const { provider, tracer } = s;
|
|
769
|
+
// Once per isolate: kick off the remote sampling config GET so the rate
|
|
770
|
+
// is available for subsequent requests without a redeploy. Uses the
|
|
771
|
+
// captured pre-patch fetch under suppressTracing — never self-traced,
|
|
772
|
+
// never looped. Fail-open: any error leaves the rate at 1 (keep-all).
|
|
773
|
+
if (config.sampling?.remote && !_remoteSamplingKicked) {
|
|
774
|
+
_remoteSamplingKicked = true;
|
|
775
|
+
const resolvedKey = config.apiKey ?? env?.HEYSTACK_API_KEY;
|
|
776
|
+
if (resolvedKey) {
|
|
777
|
+
const ep = config.endpoint ?? DEFAULT_ENDPOINT;
|
|
778
|
+
ctx.waitUntil(context.with(suppressTracing(context.active()), () => loadRemoteSamplingRate({
|
|
779
|
+
endpoint: ep,
|
|
780
|
+
apiKey: resolvedKey,
|
|
781
|
+
fetchImpl: _originalFetch ?? fetch,
|
|
782
|
+
})));
|
|
783
|
+
}
|
|
784
|
+
}
|
|
766
785
|
const url = new URL(req.url);
|
|
767
786
|
// FR5: continue an inbound W3C traceparent so tap→server is one trace.
|
|
768
787
|
const parent = parseTraceparent(req.headers.get("traceparent"));
|
|
@@ -819,7 +838,7 @@ export function instrument(handler, config) {
|
|
|
819
838
|
if (config.instrumentBindings) {
|
|
820
839
|
const binTracer = trace.getTracer("heystack");
|
|
821
840
|
handlerEnv = instrumentEnv(env, {
|
|
822
|
-
startSpan: (name, attrs) => binTracer.startSpan(name, { attributes: attrs }, context.active()),
|
|
841
|
+
startSpan: (name, attrs, kind) => binTracer.startSpan(name, { attributes: attrs, kind }, context.active()),
|
|
823
842
|
select: config.instrumentBindings,
|
|
824
843
|
});
|
|
825
844
|
}
|