@watchtowerobservability/node 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +39 -8
  2. package/dist/cjs/env.d.ts +12 -0
  3. package/dist/cjs/env.js +37 -0
  4. package/dist/cjs/errors.d.ts +8 -0
  5. package/dist/cjs/errors.js +23 -0
  6. package/dist/cjs/index.d.ts +29 -9
  7. package/dist/cjs/index.js +87 -49
  8. package/dist/cjs/instrument.d.ts +7 -0
  9. package/dist/cjs/instrument.js +51 -0
  10. package/dist/cjs/integrations/genai-agents.d.ts +4 -1
  11. package/dist/cjs/integrations/genai-agents.js +27 -24
  12. package/dist/cjs/integrations/genai-frameworks.d.ts +25 -0
  13. package/dist/cjs/integrations/genai-frameworks.js +211 -0
  14. package/dist/cjs/integrations/genai.d.ts +1 -0
  15. package/dist/cjs/integrations/genai.js +7 -2
  16. package/dist/cjs/integrations/next.d.ts +4 -1
  17. package/dist/cjs/integrations/next.js +13 -7
  18. package/dist/cjs/load.js +36 -5
  19. package/dist/cjs/origin.d.ts +9 -0
  20. package/dist/cjs/origin.js +43 -0
  21. package/dist/cjs/redact.d.ts +8 -0
  22. package/dist/cjs/redact.js +43 -0
  23. package/dist/cjs/register.d.ts +1 -0
  24. package/dist/cjs/register.js +29 -0
  25. package/dist/cjs/sampling.d.ts +53 -0
  26. package/dist/cjs/sampling.js +98 -0
  27. package/dist/esm/env.d.ts +12 -0
  28. package/dist/esm/env.js +33 -0
  29. package/dist/esm/errors.d.ts +8 -0
  30. package/dist/esm/errors.js +22 -0
  31. package/dist/esm/index.d.ts +29 -9
  32. package/dist/esm/index.js +87 -50
  33. package/dist/esm/instrument.d.ts +7 -0
  34. package/dist/esm/instrument.js +48 -0
  35. package/dist/esm/integrations/genai-agents.d.ts +4 -1
  36. package/dist/esm/integrations/genai-agents.js +4 -4
  37. package/dist/esm/integrations/genai-frameworks.d.ts +25 -0
  38. package/dist/esm/integrations/genai-frameworks.js +206 -0
  39. package/dist/esm/integrations/genai.d.ts +1 -0
  40. package/dist/esm/integrations/genai.js +3 -1
  41. package/dist/esm/integrations/next.d.ts +4 -1
  42. package/dist/esm/integrations/next.js +11 -6
  43. package/dist/esm/load.js +78 -2
  44. package/dist/esm/origin.d.ts +9 -0
  45. package/dist/esm/origin.js +39 -0
  46. package/dist/esm/redact.d.ts +8 -0
  47. package/dist/esm/redact.js +39 -0
  48. package/dist/esm/register.d.ts +1 -0
  49. package/dist/esm/register.js +27 -0
  50. package/dist/esm/sampling.d.ts +53 -0
  51. package/dist/esm/sampling.js +91 -0
  52. package/package.json +33 -2
@@ -0,0 +1,53 @@
1
+ import type { Attributes, Context, Link, SpanKind } from '@opentelemetry/api';
2
+ import type { Sampler, SamplingResult } from '@opentelemetry/sdk-trace';
3
+ export interface SamplingRule {
4
+ name?: string;
5
+ enabled?: boolean | null;
6
+ match?: {
7
+ service?: string;
8
+ env?: string;
9
+ span_name_prefix?: string;
10
+ span_name_glob?: string;
11
+ span_name_regex?: string;
12
+ attrs?: Record<string, string>;
13
+ min_duration_ms?: number | null;
14
+ status?: string;
15
+ };
16
+ action?: {
17
+ kind?: string;
18
+ ratio?: number | null;
19
+ };
20
+ }
21
+ export interface RemoteConfig {
22
+ strategyType?: string;
23
+ probabilisticSampling?: {
24
+ samplingRate: number;
25
+ };
26
+ rules?: SamplingRule[];
27
+ }
28
+ /** fnv64a of the trace id's bytes, as a fraction of 2^64; matches the gateway's `ratioKeeps`. */
29
+ export declare function traceFraction(traceIdHex: string): number;
30
+ export declare function globMatches(glob: string, text: string): boolean;
31
+ /** The keep ratio for a span under `cfg` (1 = keep). First matching ratio/drop rule wins. */
32
+ export declare function ratioFor(cfg: RemoteConfig, service: string, env: string, name: string, attrs: Attributes): number;
33
+ export interface RemoteSamplerOptions {
34
+ endpoint: string;
35
+ headers: Record<string, string>;
36
+ service: string;
37
+ env?: string;
38
+ intervalMs?: number;
39
+ fetch?: typeof fetch;
40
+ }
41
+ export declare class RemoteSampler implements Sampler {
42
+ private readonly o;
43
+ cfg: RemoteConfig;
44
+ private etag;
45
+ private timer?;
46
+ private readonly fetchFn;
47
+ constructor(o: RemoteSamplerOptions);
48
+ /** One poll; a failure or 304 keeps the current config. */
49
+ poll(): Promise<void>;
50
+ stop(): void;
51
+ shouldSample(_c: Context, traceId: string, name: string, _k: SpanKind, attributes: Attributes, _l: Link[]): SamplingResult;
52
+ toString(): string;
53
+ }
@@ -0,0 +1,98 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.RemoteSampler = void 0;
4
+ exports.traceFraction = traceFraction;
5
+ exports.globMatches = globMatches;
6
+ exports.ratioFor = ratioFor;
7
+ const NOT_RECORD = 0;
8
+ const RECORD_AND_SAMPLED = 2;
9
+ const FNV_OFFSET = 0xcbf29ce484222325n;
10
+ const FNV_PRIME = 0x100000001b3n;
11
+ const MASK = 0xffffffffffffffffn;
12
+ const TWO_64 = 18446744073709551616;
13
+ /** fnv64a of the trace id's bytes, as a fraction of 2^64; matches the gateway's `ratioKeeps`. */
14
+ function traceFraction(traceIdHex) {
15
+ let h = FNV_OFFSET;
16
+ for (let i = 0; i + 1 < traceIdHex.length; i += 2)
17
+ h = ((h ^ BigInt(parseInt(traceIdHex.slice(i, i + 2), 16))) * FNV_PRIME) & MASK;
18
+ return Number(h) / TWO_64;
19
+ }
20
+ function globMatches(glob, text) {
21
+ const re = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.');
22
+ return new RegExp(`^${re}$`, 's').test(text);
23
+ }
24
+ function ruleMatches(r, service, env, name, attrs) {
25
+ const m = r.match ?? {};
26
+ // duration and status are only known at the end of the span: the gateway decides those
27
+ if (m.min_duration_ms != null || m.status)
28
+ return false;
29
+ if ((m.service && m.service !== service) || (m.env && m.env !== env))
30
+ return false;
31
+ if (m.span_name_prefix && !name.startsWith(m.span_name_prefix))
32
+ return false;
33
+ if (m.span_name_glob && !globMatches(m.span_name_glob, name))
34
+ return false;
35
+ if (m.span_name_regex) {
36
+ try {
37
+ if (!new RegExp(m.span_name_regex).test(name))
38
+ return false;
39
+ }
40
+ catch {
41
+ return false;
42
+ }
43
+ }
44
+ return Object.entries(m.attrs ?? {}).every(([k, v]) => String(attrs[k]) === v);
45
+ }
46
+ /** The keep ratio for a span under `cfg` (1 = keep). First matching ratio/drop rule wins. */
47
+ function ratioFor(cfg, service, env, name, attrs) {
48
+ if (cfg.strategyType === 'PROBABILISTIC')
49
+ return clamp(cfg.probabilisticSampling?.samplingRate);
50
+ for (const r of cfg.rules ?? []) {
51
+ if (r.enabled === false || !ruleMatches(r, service, env, name, attrs))
52
+ continue;
53
+ if (r.action?.kind === 'drop')
54
+ return 0;
55
+ if (r.action?.kind === 'ratio' && r.action.ratio != null)
56
+ return clamp(r.action.ratio);
57
+ return 1; // keep, tail, rate: decided later
58
+ }
59
+ return 1;
60
+ }
61
+ const clamp = (r) => (typeof r === 'number' && Number.isFinite(r) ? Math.min(1, Math.max(0, r)) : 1);
62
+ class RemoteSampler {
63
+ o;
64
+ cfg = {};
65
+ etag = '';
66
+ timer;
67
+ fetchFn;
68
+ constructor(o) {
69
+ this.o = o;
70
+ this.fetchFn = o.fetch ?? globalThis.fetch;
71
+ void this.poll();
72
+ this.timer = setInterval(() => void this.poll(), Math.max(1000, o.intervalMs ?? 30_000));
73
+ this.timer.unref?.();
74
+ }
75
+ /** One poll; a failure or 304 keeps the current config. */
76
+ async poll() {
77
+ try {
78
+ const q = new URLSearchParams({ service: this.o.service, ...(this.o.env ? { env: this.o.env } : {}) });
79
+ const res = await this.fetchFn(`${this.o.endpoint}/v1/sampling?${q}`, {
80
+ headers: { ...this.o.headers, ...(this.etag ? { 'If-None-Match': this.etag } : {}) },
81
+ });
82
+ if (res.status === 304 || !res.ok)
83
+ return;
84
+ this.cfg = (await res.json());
85
+ this.etag = res.headers.get('ETag') ?? '';
86
+ }
87
+ catch { /* fallback: keep the last good config */ }
88
+ }
89
+ stop() { if (this.timer)
90
+ clearInterval(this.timer); }
91
+ shouldSample(_c, traceId, name, _k, attributes, _l) {
92
+ const r = ratioFor(this.cfg, this.o.service, this.o.env ?? '', name, attributes);
93
+ const keep = r >= 1 || (r > 0 && traceFraction(traceId) < r);
94
+ return { decision: keep ? RECORD_AND_SAMPLED : NOT_RECORD };
95
+ }
96
+ toString() { return 'WatchtowerRemoteSampler'; }
97
+ }
98
+ exports.RemoteSampler = RemoteSampler;
@@ -0,0 +1,12 @@
1
+ /** Parse `k=v,k2=v2` (OTEL_RESOURCE_ATTRIBUTES / OTEL_EXPORTER_OTLP_HEADERS); values are percent-decoded. */
2
+ export declare function parseKv(s: string | undefined): Record<string, string>;
3
+ export interface OtelEnv {
4
+ endpoint?: string;
5
+ /** Header names are lower-cased by nobody: they are kept as written, except `Authorization` is matched case-insensitively. */
6
+ headers: Record<string, string>;
7
+ /** `Authorization: Bearer <key>` from OTEL_EXPORTER_OTLP_HEADERS, the key only. */
8
+ bearer?: string;
9
+ serviceName?: string;
10
+ resource: Record<string, string>;
11
+ }
12
+ export declare function otelEnv(env?: NodeJS.ProcessEnv): OtelEnv;
@@ -0,0 +1,33 @@
1
+ // Standard OTEL_* environment variables, used as fallbacks after the WT_* ones.
2
+ /** Parse `k=v,k2=v2` (OTEL_RESOURCE_ATTRIBUTES / OTEL_EXPORTER_OTLP_HEADERS); values are percent-decoded. */
3
+ export function parseKv(s) {
4
+ const out = {};
5
+ for (const part of (s || '').split(',')) {
6
+ const i = part.indexOf('=');
7
+ if (i <= 0)
8
+ continue;
9
+ const dec = (x) => { try {
10
+ return decodeURIComponent(x.trim());
11
+ }
12
+ catch {
13
+ return x.trim();
14
+ } };
15
+ out[dec(part.slice(0, i))] = dec(part.slice(i + 1));
16
+ }
17
+ return out;
18
+ }
19
+ export function otelEnv(env = process.env) {
20
+ const headers = parseKv(env.OTEL_EXPORTER_OTLP_HEADERS);
21
+ const authKey = Object.keys(headers).find((k) => k.toLowerCase() === 'authorization');
22
+ const auth = authKey ? headers[authKey] : undefined;
23
+ if (authKey)
24
+ delete headers[authKey];
25
+ const resource = parseKv(env.OTEL_RESOURCE_ATTRIBUTES);
26
+ return {
27
+ endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT || undefined,
28
+ headers,
29
+ bearer: auth && /^bearer\s+/i.test(auth) ? auth.replace(/^bearer\s+/i, '') : undefined,
30
+ serviceName: env.OTEL_SERVICE_NAME || resource['service.name'] || undefined,
31
+ resource,
32
+ };
33
+ }
@@ -2,7 +2,15 @@ export interface CaptureContext {
2
2
  /** false for crashes (uncaught); default true. Keys starting with `wt.` are dropped. */
3
3
  handled?: boolean;
4
4
  attributes?: Record<string, string | number | boolean | undefined>;
5
+ /**
6
+ * Exception replay: variables to show on the throwing frame. Node cannot read another frame's locals, so
7
+ * the caller names them. Secret-looking names are masked, values are shortened; the project must also turn
8
+ * on "exception replay" in its issue settings or the server drops them.
9
+ */
10
+ locals?: Record<string, unknown>;
5
11
  }
12
+ /** `exception.locals` for the innermost stack frame of `e`: the function and line come from its stack, the values from `locals`. */
13
+ export declare function localsAttribute(e: Error, locals: Record<string, unknown>): string | undefined;
6
14
  /** The slice of `process` the hooks need; tests pass an EventEmitter. */
7
15
  export interface ProcessLike {
8
16
  on(event: string, listener: (arg: unknown) => void): unknown;
@@ -1,6 +1,25 @@
1
1
  // Error capture: an exception becomes an OTel span event (exception.*) with trace and span ids.
2
2
  import { trace, SpanStatusCode } from '@opentelemetry/api';
3
3
  const captured = new WeakMap(); // error object -> traceId, so framework + process hooks never double-report
4
+ const SENSITIVE = /pass|secret|token|api[-_]?key|auth|cookie|session|credit|card|ssn|private|bearer/i;
5
+ /** `exception.locals` for the innermost stack frame of `e`: the function and line come from its stack, the values from `locals`. */
6
+ export function localsAttribute(e, locals) {
7
+ const m = /^\s+at (?:(\S+) \()?(.*?):(\d+):\d+\)?$/m.exec(e.stack || '');
8
+ const vars = {};
9
+ for (const [k, v] of Object.entries(locals).slice(0, 20)) {
10
+ let text;
11
+ try {
12
+ text = typeof v === 'string' ? v : JSON.stringify(v) ?? String(v);
13
+ }
14
+ catch {
15
+ text = '<unrepresentable>';
16
+ }
17
+ vars[k] = SENSITIVE.test(k) ? '[Filtered]' : text.slice(0, 120);
18
+ }
19
+ if (!Object.keys(vars).length)
20
+ return undefined;
21
+ return JSON.stringify([{ function: m?.[1] ?? '', filename: m?.[2] ?? '', lineno: m ? Number(m[3]) : undefined, vars }]);
22
+ }
4
23
  export function dropReserved(attrs) {
5
24
  return Object.fromEntries(Object.entries(attrs || {}).filter(([k]) => !k.startsWith('wt.')));
6
25
  }
@@ -15,7 +34,10 @@ export function captureException(err, ctx) {
15
34
  const active = trace.getActiveSpan();
16
35
  const span = active && active.isRecording() ? active : trace.getTracer('watchtower').startSpan('exception');
17
36
  span.setAttributes(attrs);
37
+ const replay = o.locals ? localsAttribute(e, o.locals) : undefined;
18
38
  span.recordException(e);
39
+ if (replay)
40
+ span.setAttribute('exception.locals', replay); // the OTel JS exception event takes no attributes
19
41
  span.setStatus({ code: SpanStatusCode.ERROR, message: e.message });
20
42
  const traceId = span.spanContext().traceId;
21
43
  if (span !== active)
@@ -7,6 +7,7 @@ import type { LoggerProvider } from '@opentelemetry/sdk-logs';
7
7
  import type { MeterProvider } from '@opentelemetry/sdk-metrics';
8
8
  import { captureException } from './errors.js';
9
9
  import type { CaptureContext } from './errors.js';
10
+ import { RemoteSampler } from './sampling.js';
10
11
  import type { Profiler } from './profiling.js';
11
12
  export { captureException, metrics, trace };
12
13
  export { setCaptureRules, capturePayload } from './payload.js';
@@ -14,40 +15,56 @@ export type { CaptureRule } from './payload.js';
14
15
  export type { CaptureContext };
15
16
  export declare const DEFAULT_ENDPOINT = "http://localhost:4318";
16
17
  export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
18
+ /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
19
+ export declare function callerOf(stack: string | undefined): Record<string, string | number>;
17
20
  export interface InitOptions {
18
- /** Ingest key; defaults to WT_INGEST_KEY. */
21
+ /** Ingest key; defaults to WT_INGEST_KEY, then a `Bearer` Authorization in OTEL_EXPORTER_OTLP_HEADERS. */
19
22
  apiKey?: string;
20
23
  /** Defaults to WT_ENVIRONMENT. */
21
24
  environment?: string;
22
- /** Defaults to WT_RELEASE. */
25
+ /** Sent as `service.version`; the server links an issue's first release by matching it to an existing release's version. Defaults to WT_RELEASE, then `service.version` in OTEL_RESOURCE_ATTRIBUTES. */
23
26
  release?: string;
24
27
  /** Hook uncaughtException / unhandledRejection (default true). */
25
28
  captureProcessErrors?: boolean;
26
- /** Replace the log exporter (tests). */
29
+ /** Replace the log exporter (tests); setting it turns logs on unless `logs` is false. Exported through a batch processor. */
27
30
  logExporter?: unknown;
28
31
  /** Export OTLP logs; defaults to WATCHTOWER_LOGS (on unless `off`; OTEL_LOGS_EXPORTER=none also turns it off). */
29
32
  logs?: boolean;
33
+ /** Stamp code.filepath/code.lineno of the `log()` caller (reads the stack); defaults to WATCHTOWER_LOG_CALLER=1. */
34
+ logCaller?: boolean;
30
35
  /** Export OTLP metrics; defaults to WATCHTOWER_METRICS (on unless `off`; OTEL_METRICS_EXPORTER=none also turns it off). */
31
36
  metrics?: boolean;
32
- /** Replace the metric exporter (tests). */
37
+ /** Replace the metric exporter (tests); setting it turns metrics on unless `metrics` is false. */
33
38
  metricExporter?: unknown;
34
- /** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT or http://localhost:4318. */
39
+ /** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT, then OTEL_EXPORTER_OTLP_ENDPOINT, then http://localhost:4318. */
35
40
  endpoint?: string;
36
41
  /** REST API base URL for getPrompt/score; defaults to WT_API_URL or endpoint. */
37
42
  apiUrl?: string;
38
43
  project?: string;
39
- /** Defaults to WT_SERVICE. */
44
+ /** Defaults to WT_SERVICE, then OTEL_SERVICE_NAME, then `service.name` in OTEL_RESOURCE_ATTRIBUTES, then `app`. */
40
45
  serviceName?: string;
41
- /** Resource attributes; keys starting with `wt.` are dropped. */
46
+ /** Resource attributes (on top of OTEL_RESOURCE_ATTRIBUTES); keys starting with `wt.` are dropped. */
42
47
  resource?: Record<string, string | number | boolean>;
43
- /** Called for every span attribute before export; return the value to keep. */
48
+ /**
49
+ * Called before export for every span attribute, every span event's and link's attributes (so `exception.message`
50
+ * and `exception.stacktrace`), the span name (key `span.name`), the status message (`status.message`) and each log
51
+ * body (`log.body`) and log attribute; return the value to keep.
52
+ */
44
53
  redact?: (key: string, value: unknown) => unknown;
45
54
  /** Directory for the offline buffer. */
46
55
  bufferDir?: string;
56
+ /** Apply the project's head sampling rules in the SDK, polled from /v1/sampling; defaults to WT_REMOTE_SAMPLING=1. Spans dropped here never reach the gateway, so its exact request counts see only what is kept. */
57
+ remoteSampling?: boolean;
58
+ /** Seconds between sampling polls; defaults to WT_SAMPLING_INTERVAL_S or 30. */
59
+ samplingIntervalS?: number;
60
+ /** Stamp code.filepath / code.lineno / code.function on spans (the code that started the span); defaults to WT_CODE_ORIGIN=1. */
61
+ codeOrigin?: boolean;
47
62
  /** Fetch the service's payload capture rules (APM-016) and record bodies/GraphQL where a rule matches; defaults to WT_CAPTURE_RULES=1. */
48
63
  captureRules?: boolean;
49
64
  /** Replace the span exporter (tests). */
50
65
  exporter?: unknown;
66
+ /** Patch node:http, fetch (undici), pg and ioredis for those whose `@opentelemetry/instrumentation-*` peer is installed; defaults to on unless WT_INSTRUMENT=0. `init` only: use the `/register` preload so it runs before the app imports them. */
67
+ instrument?: boolean;
51
68
  fetch?: typeof fetch;
52
69
  }
53
70
  export interface EvalRow {
@@ -77,9 +94,10 @@ export declare class Client {
77
94
  readonly project: string | undefined;
78
95
  readonly headers: {
79
96
  Authorization: string;
80
- };
97
+ } & Record<string, string>;
81
98
  readonly fetch: typeof fetch;
82
99
  readonly redact: InitOptions['redact'];
100
+ private readonly logCaller;
83
101
  readonly provider: TracerProvider;
84
102
  readonly resource: Resource;
85
103
  readonly tracer: Tracer;
@@ -89,6 +107,8 @@ export declare class Client {
89
107
  readonly meterProvider?: MeterProvider;
90
108
  private readonly stopRuntime;
91
109
  uninstall?: () => void;
110
+ /** Set when remote sampling is on. */
111
+ remoteSampler?: RemoteSampler;
92
112
  /** Set when WT_PROFILING=1 started the continuous profiler. */
93
113
  profiler?: Promise<Profiler | undefined>;
94
114
  constructor(opts: InitOptions);
package/dist/esm/index.js CHANGED
@@ -1,17 +1,27 @@
1
1
  // Thin Watchtower layer over the OpenTelemetry Node SDK: key header, endpoint default,
2
2
  // offline buffer, wt.*-safe resource attributes, redaction hook, getPrompt/score/evaluate/observe.
3
3
  import { inspect } from 'node:util';
4
- import { context, metrics, propagation, trace } from '@opentelemetry/api';
4
+ import { context, diag, metrics, propagation, trace } from '@opentelemetry/api';
5
5
  import { need as needPeer } from './load.js';
6
+ import { otelEnv } from './env.js';
7
+ import { instrumentAll } from './instrument.js';
8
+ import { LOG_BODY_KEY, redactor } from './redact.js';
6
9
  import { SpoolExporter, spoolLogs } from './buffer.js';
7
10
  import { startMetrics, switchOn } from './metrics.js';
8
11
  import * as nodePath from 'node:path';
9
12
  import { captureException, installProcessHandlers, dropReserved } from './errors.js';
10
13
  import { loadCaptureRules } from './payload.js';
11
14
  import { startProfiling } from './profiling.js';
15
+ import { RemoteSampler } from './sampling.js';
16
+ import { codeOrigin } from './origin.js';
12
17
  export { captureException, metrics, trace };
13
18
  export { setCaptureRules, capturePayload } from './payload.js';
14
19
  export const DEFAULT_ENDPOINT = 'http://localhost:4318';
20
+ /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
21
+ export function callerOf(stack) {
22
+ const m = /\(?((?:file:\/\/)?[^()\s]+):(\d+):\d+\)?$/.exec((stack ?? '').split('\n')[2] ?? '');
23
+ return m ? { 'code.filepath': m[1].replace(/^file:\/\//, ''), 'code.lineno': Number(m[2]) } : {};
24
+ }
15
25
  const SEVERITY = { trace: 1, debug: 5, info: 9, warn: 13, error: 17, fatal: 21 };
16
26
  /** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
17
27
  export function safeResource(attrs) {
@@ -25,6 +35,7 @@ export class Client {
25
35
  headers;
26
36
  fetch;
27
37
  redact;
38
+ logCaller;
28
39
  provider;
29
40
  resource;
30
41
  tracer;
@@ -34,31 +45,36 @@ export class Client {
34
45
  meterProvider;
35
46
  stopRuntime = () => { };
36
47
  uninstall;
48
+ /** Set when remote sampling is on. */
49
+ remoteSampler;
37
50
  /** Set when WT_PROFILING=1 started the continuous profiler. */
38
51
  profiler;
39
52
  constructor(opts) {
40
53
  const o = opts || {};
41
54
  const env = process.env;
42
- const apiKey = o.apiKey || env.WT_INGEST_KEY;
55
+ const oe = otelEnv(env);
56
+ const apiKey = o.apiKey || env.WT_INGEST_KEY || oe.bearer;
43
57
  if (!apiKey)
44
58
  throw new Error('apiKey is required (or set WT_INGEST_KEY)');
45
- this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || DEFAULT_ENDPOINT).replace(/\/+$/, '');
59
+ this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || oe.endpoint || DEFAULT_ENDPOINT).replace(/\/+$/, '');
46
60
  this.apiUrl = (o.apiUrl || env.WT_API_URL || this.endpoint).replace(/\/+$/, '');
47
61
  this.project = o.project || env.WT_PROJECT_ID;
48
- this.headers = { Authorization: `Bearer ${apiKey}` };
62
+ this.headers = { ...oe.headers, Authorization: `Bearer ${apiKey}` };
49
63
  this.fetch = o.fetch || globalThis.fetch;
50
- const environment = o.environment || env.WT_ENVIRONMENT;
51
- const release = o.release || env.WT_RELEASE;
64
+ const environment = o.environment || env.WT_ENVIRONMENT || oe.resource['deployment.environment.name'];
65
+ const release = o.release || env.WT_RELEASE || oe.resource['service.version'];
52
66
  // the OTel SDK packages are optional peers, loaded here (not at import time) with a clear error when missing
53
67
  const { resourceFromAttributes } = needPeer('@opentelemetry/resources');
54
68
  const resource = resourceFromAttributes({
55
- 'service.name': o.serviceName || env.WT_SERVICE || 'app',
69
+ ...safeResource(oe.resource),
70
+ 'service.name': o.serviceName || env.WT_SERVICE || oe.serviceName || 'app',
56
71
  ...(environment ? { 'deployment.environment.name': environment } : {}),
57
72
  ...(release ? { 'service.version': release } : {}),
58
73
  ...safeResource(o.resource),
59
74
  });
60
75
  this.resource = resource;
61
76
  this.redact = o.redact;
77
+ this.logCaller = o.logCaller ?? env.WATCHTOWER_LOG_CALLER === '1';
62
78
  let exporter = o.exporter;
63
79
  if (!exporter) {
64
80
  const url = `${this.endpoint}/v1/traces`;
@@ -70,30 +86,42 @@ export class Client {
70
86
  }
71
87
  // the redactor runs first, so the batch processor exports redacted attributes
72
88
  const { BatchSpanProcessor, TracerProvider: TP } = needPeer('@opentelemetry/sdk-trace');
89
+ let sampler;
90
+ if (o.remoteSampling ?? env.WT_REMOTE_SAMPLING === '1') {
91
+ const { ParentBasedSampler } = needPeer('@opentelemetry/sdk-trace');
92
+ this.remoteSampler = new RemoteSampler({
93
+ endpoint: this.endpoint, headers: this.headers, service: String(resource.attributes['service.name']), env: environment,
94
+ intervalMs: (o.samplingIntervalS ?? (Number(env.WT_SAMPLING_INTERVAL_S) || 30)) * 1000, fetch: this.fetch,
95
+ });
96
+ sampler = new ParentBasedSampler({ root: this.remoteSampler });
97
+ }
73
98
  this.provider = new TP({
74
99
  resource,
75
- spanProcessors: [...(o.redact ? [redactor(o.redact)] : []), new BatchSpanProcessor({ exporter })],
100
+ ...(sampler ? { sampler: sampler } : {}),
101
+ spanProcessors: [
102
+ ...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [codeOrigin()] : []),
103
+ ...(o.redact ? [redactor(o.redact)] : []),
104
+ new BatchSpanProcessor({ exporter }),
105
+ ],
76
106
  });
77
107
  this.tracer = this.provider.getTracer('watchtower');
78
- const logsOn = switchOn('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs);
108
+ // an explicit exporter never switches the other signals off: only `logs` / `metrics` / the env switches do
109
+ const logsOn = switchOn('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs ?? (o.logExporter ? true : undefined));
79
110
  let logExporter = logsOn ? o.logExporter : undefined;
80
- if (logsOn && !logExporter && !o.exporter) {
111
+ if (logsOn && !logExporter) {
81
112
  const url = `${this.endpoint}/v1/logs`;
82
113
  const { OTLPLogExporter } = needPeer('@opentelemetry/exporter-logs-otlp-http');
83
114
  logExporter = new OTLPLogExporter({ url, headers: this.headers });
84
115
  if (o.bufferDir)
85
116
  logExporter = spoolLogs(logExporter, nodePath.join(o.bufferDir, 'logs'), url, this.headers, this.fetch);
86
117
  }
87
- const { LoggerProvider: LP, SimpleLogRecordProcessor, BatchLogRecordProcessor } = needPeer('@opentelemetry/sdk-logs');
118
+ const { LoggerProvider: LP, BatchLogRecordProcessor } = needPeer('@opentelemetry/sdk-logs');
88
119
  this.loggerProvider = new LP({
89
120
  resource,
90
- processors: !logExporter ? [] : [o.logExporter
91
- ? new SimpleLogRecordProcessor({ exporter: logExporter })
92
- : new BatchLogRecordProcessor({ exporter: logExporter })],
121
+ processors: !logExporter ? [] : [new BatchLogRecordProcessor({ exporter: logExporter })],
93
122
  });
94
123
  this.logger = this.loggerProvider.getLogger('watchtower');
95
- // with a test span exporter only an explicit metric exporter is used, as for logs
96
- if (switchOn('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics) && (o.metricExporter || !o.exporter)) {
124
+ if (switchOn('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics ?? (o.metricExporter ? true : undefined))) {
97
125
  const m = startMetrics(this.endpoint, this.headers, resource, o.bufferDir, this.fetch, o.metricExporter);
98
126
  this.meterProvider = m.provider;
99
127
  this.stopRuntime = m.stop;
@@ -106,8 +134,12 @@ export class Client {
106
134
  log(level, body, attributes) {
107
135
  let attrs = dropReserved(attributes);
108
136
  const redact = this.redact;
109
- if (redact)
137
+ if (redact) {
110
138
  attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => [k, redact(k, v)]));
139
+ body = String(redact(LOG_BODY_KEY, body) ?? '');
140
+ }
141
+ if (this.logCaller)
142
+ attrs = { ...attrs, ...callerOf(new Error().stack) };
111
143
  this.logger.emit({
112
144
  severityNumber: SEVERITY[level] || SEVERITY.info,
113
145
  severityText: String(level).toUpperCase(),
@@ -215,41 +247,46 @@ export class Client {
215
247
  }
216
248
  shutdown() {
217
249
  this.stopRuntime();
250
+ this.remoteSampler?.stop();
218
251
  return Promise.all([this.profiler?.then((p) => p?.stop()), this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
219
252
  }
220
253
  }
221
- function redactor(hook) {
222
- return {
223
- onStart() { },
224
- onEnd(span) {
225
- for (const k of Object.keys(span.attributes))
226
- span.attributes[k] = hook(k, span.attributes[k]);
227
- },
228
- shutdown: () => Promise.resolve(),
229
- forceFlush: () => Promise.resolve(),
230
- };
254
+ // One state per process, not per module copy: a CommonJS and an ESM copy of this package (a `-r` preload plus an `import`
255
+ // app, say) must share the client or flush()/shutdown() in one copy would miss the init() of the other.
256
+ const state = (globalThis[Symbol.for('watchtower.node.state')] ??= {});
257
+ /**
258
+ * Register the global tracer provider, context manager, propagator and meter provider. OTel allows each to be set once
259
+ * (a second set returns false and changes nothing), so a provider the app or an earlier instrumentation registered is
260
+ * left alone and a warning says so; only globals a previous `init` installed are torn down on a re-init.
261
+ */
262
+ function registerGlobals(c) {
263
+ if (state.ownsGlobals) { // a re-init replaces what the previous init installed
264
+ trace.disable();
265
+ context.disable();
266
+ propagation.disable();
267
+ metrics.disable();
268
+ }
269
+ const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
270
+ const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
271
+ const ok = [
272
+ trace.setGlobalTracerProvider(c.provider),
273
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()),
274
+ propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })),
275
+ ...(c.meterProvider ? [metrics.setGlobalMeterProvider(c.meterProvider)] : []),
276
+ ];
277
+ state.ownsGlobals = ok[0] === true;
278
+ if (ok.includes(false))
279
+ diag.warn('watchtower: an OpenTelemetry global (tracer provider, context manager, propagator or meter provider) was already registered and was kept; call init() before other OTel setup to own them');
231
280
  }
232
- let current;
233
281
  export function init(opts) {
234
- if (current && current.uninstall)
235
- current.uninstall();
236
- void current?.profiler?.then((p) => p?.stop());
282
+ if (state.current && state.current.uninstall)
283
+ state.current.uninstall();
284
+ void state.current?.profiler?.then((p) => p?.stop());
237
285
  const c = new Client(opts);
238
- current = c;
239
- // Global tracer + async context so trace.getActiveSpan() works; a re-init replaces the previous client's.
240
- trace.disable();
241
- context.disable();
242
- propagation.disable();
243
- const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
244
- const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
245
- context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
246
- propagation.setGlobalPropagator(new CompositePropagator({
247
- propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
248
- }));
249
- trace.setGlobalTracerProvider(c.provider);
250
- if (c.meterProvider) {
251
- metrics.disable(); // a re-init replaces the previous client's provider
252
- metrics.setGlobalMeterProvider(c.meterProvider);
286
+ state.current = c;
287
+ registerGlobals(c);
288
+ if (opts?.instrument ?? !['0', 'off', 'false', 'no'].includes((process.env.WT_INSTRUMENT || '').trim().toLowerCase())) {
289
+ instrumentAll({ tracerProvider: c.provider, meterProvider: c.meterProvider }, [c.endpoint, c.apiUrl]);
253
290
  }
254
291
  if (process.env.WT_PROFILING === '1') {
255
292
  const attrs = c.resource.attributes;
@@ -266,15 +303,15 @@ export function init(opts) {
266
303
  return c;
267
304
  }
268
305
  function need() {
269
- if (!current)
306
+ if (!state.current)
270
307
  throw new Error('watchtower.init() has not been called');
271
- return current;
308
+ return state.current;
272
309
  }
273
- export const flush = () => (current ? current.forceFlush() : Promise.resolve());
310
+ export const flush = () => (state.current ? state.current.forceFlush() : Promise.resolve());
274
311
  export const log = (l, b, a) => need().log(l, b, a);
275
312
  export const getPrompt = (id) => need().getPrompt(id);
276
313
  export const score = (t, v, o) => need().score(t, v, o);
277
314
  export const evaluate = (d, t, s, o) => need().evaluate(d, t, s, o);
278
315
  export const evaluateResume = (d, t, s, prev, o) => need().evaluateResume(d, t, s, prev, o);
279
316
  export const observe = (n, f) => need().observe(n, f);
280
- export const shutdown = () => (current ? current.shutdown() : Promise.resolve());
317
+ export const shutdown = () => (state.current ? state.current.shutdown() : Promise.resolve());
@@ -0,0 +1,7 @@
1
+ interface Providers {
2
+ tracerProvider: unknown;
3
+ meterProvider?: unknown;
4
+ }
5
+ /** Register whichever of the four instrumentations is installed. Returns the package names that were enabled. */
6
+ export declare function instrumentAll(p: Providers, selfUrls: string[]): string[];
7
+ export {};
@@ -0,0 +1,48 @@
1
+ // Auto-instrumentation: node:http, fetch (undici), pg and ioredis, each only when its optional peer package is installed.
2
+ import { load } from './load.js';
3
+ const ENTRIES = [
4
+ {
5
+ pkg: '@opentelemetry/instrumentation-http', ctor: 'HttpInstrumentation',
6
+ // our own exports and API calls must not become spans
7
+ opts: (self) => ({ ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
8
+ },
9
+ {
10
+ pkg: '@opentelemetry/instrumentation-undici', ctor: 'UndiciInstrumentation',
11
+ opts: (self) => ({ ignoreRequestHook: (r) => self.some((u) => r.origin === u.origin) }),
12
+ },
13
+ { pkg: '@opentelemetry/instrumentation-pg', ctor: 'PgInstrumentation', opts: () => ({}) },
14
+ { pkg: '@opentelemetry/instrumentation-ioredis', ctor: 'IORedisInstrumentation', opts: () => ({}) },
15
+ ];
16
+ /** Register whichever of the four instrumentations is installed. Returns the package names that were enabled. */
17
+ export function instrumentAll(p, selfUrls) {
18
+ let registerInstrumentations;
19
+ try {
20
+ ({ registerInstrumentations } = load('@opentelemetry/instrumentation'));
21
+ }
22
+ catch {
23
+ return []; // optional peer not installed
24
+ }
25
+ const self = selfUrls.flatMap((u) => { try {
26
+ return [new URL(u)];
27
+ }
28
+ catch {
29
+ return [];
30
+ } });
31
+ const found = [];
32
+ const names = [];
33
+ for (const e of ENTRIES) {
34
+ try {
35
+ const Ctor = load(e.pkg)[e.ctor];
36
+ if (Ctor) {
37
+ found.push(new Ctor(e.opts(self)));
38
+ names.push(e.pkg);
39
+ }
40
+ }
41
+ catch {
42
+ // package not installed (or failed to construct): skip it, never break init
43
+ }
44
+ }
45
+ if (found.length)
46
+ registerInstrumentations({ instrumentations: found, tracerProvider: p.tracerProvider, meterProvider: p.meterProvider });
47
+ return names;
48
+ }
@@ -1,5 +1,8 @@
1
- import type { Span } from '@opentelemetry/api';
1
+ import type { Attributes, Span } from '@opentelemetry/api';
2
2
  type Obj = Record<string, unknown>;
3
+ export declare const obj: (v: unknown) => Obj;
4
+ export declare const s: (v: unknown) => string;
5
+ export declare function start(name: string, attrs: Attributes, parent?: Span, startTime?: number): Span;
3
6
  /** The part of LlamaIndex.TS `Settings.callbackManager` used here. */
4
7
  export interface CallbackManagerLike {
5
8
  on(event: string, handler: (e: {