@watchtowerobservability/node 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +88 -9
  2. package/dist/cjs/checkin.d.ts +7 -0
  3. package/dist/cjs/checkin.js +18 -0
  4. package/dist/cjs/env.d.ts +12 -0
  5. package/dist/cjs/env.js +37 -0
  6. package/dist/cjs/errors.d.ts +8 -0
  7. package/dist/cjs/errors.js +23 -0
  8. package/dist/cjs/filter.d.ts +2 -0
  9. package/dist/cjs/filter.js +16 -0
  10. package/dist/cjs/index.d.ts +48 -10
  11. package/dist/cjs/index.js +119 -53
  12. package/dist/cjs/instrument.d.ts +10 -0
  13. package/dist/cjs/instrument.js +74 -0
  14. package/dist/cjs/integrations/db.d.ts +3 -0
  15. package/dist/cjs/integrations/db.js +4 -1
  16. package/dist/cjs/integrations/genai-agents.d.ts +4 -1
  17. package/dist/cjs/integrations/genai-agents.js +27 -24
  18. package/dist/cjs/integrations/genai-frameworks.d.ts +25 -0
  19. package/dist/cjs/integrations/genai-frameworks.js +211 -0
  20. package/dist/cjs/integrations/genai.d.ts +1 -0
  21. package/dist/cjs/integrations/genai.js +7 -2
  22. package/dist/cjs/integrations/next-build.d.ts +31 -0
  23. package/dist/cjs/integrations/next-build.js +95 -0
  24. package/dist/cjs/integrations/next-tunnel.d.ts +14 -0
  25. package/dist/cjs/integrations/next-tunnel.js +52 -0
  26. package/dist/cjs/integrations/next.d.ts +24 -1
  27. package/dist/cjs/integrations/next.js +39 -7
  28. package/dist/cjs/load.js +37 -5
  29. package/dist/cjs/next-sourcemaps-cli.d.ts +2 -0
  30. package/dist/cjs/next-sourcemaps-cli.js +9 -0
  31. package/dist/cjs/origin.d.ts +9 -0
  32. package/dist/cjs/origin.js +43 -0
  33. package/dist/cjs/redact.d.ts +8 -0
  34. package/dist/cjs/redact.js +43 -0
  35. package/dist/cjs/register.d.ts +1 -0
  36. package/dist/cjs/register.js +29 -0
  37. package/dist/cjs/sampling.d.ts +53 -0
  38. package/dist/cjs/sampling.js +98 -0
  39. package/dist/cjs/scrub.d.ts +7 -0
  40. package/dist/cjs/scrub.js +39 -0
  41. package/dist/cjs/shutdown.d.ts +14 -0
  42. package/dist/cjs/shutdown.js +26 -0
  43. package/dist/cjs/sql.js +11 -1
  44. package/dist/cjs/version.d.ts +2 -0
  45. package/dist/cjs/version.js +5 -0
  46. package/dist/esm/checkin.d.ts +7 -0
  47. package/dist/esm/checkin.js +15 -0
  48. package/dist/esm/env.d.ts +12 -0
  49. package/dist/esm/env.js +33 -0
  50. package/dist/esm/errors.d.ts +8 -0
  51. package/dist/esm/errors.js +22 -0
  52. package/dist/esm/filter.d.ts +2 -0
  53. package/dist/esm/filter.js +13 -0
  54. package/dist/esm/index.d.ts +48 -10
  55. package/dist/esm/index.js +117 -53
  56. package/dist/esm/instrument.d.ts +10 -0
  57. package/dist/esm/instrument.js +71 -0
  58. package/dist/esm/integrations/db.d.ts +3 -0
  59. package/dist/esm/integrations/db.js +5 -2
  60. package/dist/esm/integrations/genai-agents.d.ts +4 -1
  61. package/dist/esm/integrations/genai-agents.js +4 -4
  62. package/dist/esm/integrations/genai-frameworks.d.ts +25 -0
  63. package/dist/esm/integrations/genai-frameworks.js +206 -0
  64. package/dist/esm/integrations/genai.d.ts +1 -0
  65. package/dist/esm/integrations/genai.js +3 -1
  66. package/dist/esm/integrations/next-build.d.ts +31 -0
  67. package/dist/esm/integrations/next-build.js +58 -0
  68. package/dist/esm/integrations/next-tunnel.d.ts +14 -0
  69. package/dist/esm/integrations/next-tunnel.js +49 -0
  70. package/dist/esm/integrations/next.d.ts +24 -1
  71. package/dist/esm/integrations/next.js +32 -6
  72. package/dist/esm/load.js +80 -2
  73. package/dist/esm/next-sourcemaps-cli.d.ts +2 -0
  74. package/dist/esm/next-sourcemaps-cli.js +7 -0
  75. package/dist/esm/origin.d.ts +9 -0
  76. package/dist/esm/origin.js +39 -0
  77. package/dist/esm/redact.d.ts +8 -0
  78. package/dist/esm/redact.js +39 -0
  79. package/dist/esm/register.d.ts +1 -0
  80. package/dist/esm/register.js +27 -0
  81. package/dist/esm/sampling.d.ts +53 -0
  82. package/dist/esm/sampling.js +91 -0
  83. package/dist/esm/scrub.d.ts +7 -0
  84. package/dist/esm/scrub.js +34 -0
  85. package/dist/esm/shutdown.d.ts +14 -0
  86. package/dist/esm/shutdown.js +23 -0
  87. package/dist/esm/sql.js +11 -1
  88. package/dist/esm/version.d.ts +2 -0
  89. package/dist/esm/version.js +2 -0
  90. package/package.json +40 -2
@@ -2,11 +2,12 @@ import { inspect } from 'node:util';
2
2
  import { metrics, trace } from '@opentelemetry/api';
3
3
  import type { Tracer } from '@opentelemetry/api';
4
4
  import type { Resource } from '@opentelemetry/resources';
5
- import type { TracerProvider } from '@opentelemetry/sdk-trace';
5
+ import type { ReadableSpan, TracerProvider } from '@opentelemetry/sdk-trace';
6
6
  import type { LoggerProvider } from '@opentelemetry/sdk-logs';
7
7
  import type { MeterProvider } from '@opentelemetry/sdk-metrics';
8
8
  import { captureException } from './errors.js';
9
9
  import type { CaptureContext } from './errors.js';
10
+ import { RemoteSampler } from './sampling.js';
10
11
  import type { Profiler } from './profiling.js';
11
12
  export { captureException, metrics, trace };
12
13
  export { setCaptureRules, capturePayload } from './payload.js';
@@ -14,40 +15,71 @@ export type { CaptureRule } from './payload.js';
14
15
  export type { CaptureContext };
15
16
  export declare const DEFAULT_ENDPOINT = "http://localhost:4318";
16
17
  export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
18
+ /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
19
+ export declare function callerOf(stack: string | undefined): Record<string, string | number>;
17
20
  export interface InitOptions {
18
- /** Ingest key; defaults to WT_INGEST_KEY. */
21
+ /** Ingest key; defaults to WT_INGEST_KEY, then a `Bearer` Authorization in OTEL_EXPORTER_OTLP_HEADERS. */
19
22
  apiKey?: string;
20
23
  /** Defaults to WT_ENVIRONMENT. */
21
24
  environment?: string;
22
- /** Defaults to WT_RELEASE. */
25
+ /** Sent as `service.version`; the server links an issue's first release by matching it to an existing release's version. Defaults to WT_RELEASE, then `service.version` in OTEL_RESOURCE_ATTRIBUTES. */
23
26
  release?: string;
24
27
  /** Hook uncaughtException / unhandledRejection (default true). */
25
28
  captureProcessErrors?: boolean;
26
- /** Replace the log exporter (tests). */
29
+ /** Replace the log exporter (tests); setting it turns logs on unless `logs` is false. Exported through a batch processor. */
27
30
  logExporter?: unknown;
28
31
  /** Export OTLP logs; defaults to WATCHTOWER_LOGS (on unless `off`; OTEL_LOGS_EXPORTER=none also turns it off). */
29
32
  logs?: boolean;
33
+ /** Stamp code.filepath/code.lineno of the `log()` caller (reads the stack); defaults to WATCHTOWER_LOG_CALLER=1. */
34
+ logCaller?: boolean;
30
35
  /** Export OTLP metrics; defaults to WATCHTOWER_METRICS (on unless `off`; OTEL_METRICS_EXPORTER=none also turns it off). */
31
36
  metrics?: boolean;
32
- /** Replace the metric exporter (tests). */
37
+ /** Replace the metric exporter (tests); setting it turns metrics on unless `metrics` is false. */
33
38
  metricExporter?: unknown;
34
- /** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT or http://localhost:4318. */
39
+ /** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT, then OTEL_EXPORTER_OTLP_ENDPOINT, then http://localhost:4318. */
35
40
  endpoint?: string;
36
41
  /** REST API base URL for getPrompt/score; defaults to WT_API_URL or endpoint. */
37
42
  apiUrl?: string;
38
43
  project?: string;
39
- /** Defaults to WT_SERVICE. */
44
+ /** Defaults to WT_SERVICE, then OTEL_SERVICE_NAME, then `service.name` in OTEL_RESOURCE_ATTRIBUTES, then `app`. */
40
45
  serviceName?: string;
41
- /** Resource attributes; keys starting with `wt.` are dropped. */
46
+ /** Resource attributes (on top of OTEL_RESOURCE_ATTRIBUTES); keys starting with `wt.` are dropped. */
42
47
  resource?: Record<string, string | number | boolean>;
43
- /** Called for every span attribute before export; return the value to keep. */
48
+ /**
49
+ * Called before export for every span attribute, every span event's and link's attributes (so `exception.message`
50
+ * and `exception.stacktrace`), the span name (key `span.name`), the status message (`status.message`) and each log
51
+ * body (`log.body`) and log attribute; return the value to keep.
52
+ */
44
53
  redact?: (key: string, value: unknown) => unknown;
45
54
  /** Directory for the offline buffer. */
46
55
  bufferDir?: string;
56
+ /** Apply the project's head sampling rules in the SDK, polled from /v1/sampling; defaults to WT_REMOTE_SAMPLING=1. Spans dropped here never reach the gateway, so its exact request counts see only what is kept. */
57
+ remoteSampling?: boolean;
58
+ /** Seconds between sampling polls; defaults to WT_SAMPLING_INTERVAL_S or 30. */
59
+ samplingIntervalS?: number;
60
+ /** Stamp code.filepath / code.lineno / code.function on spans (the code that started the span); defaults to WT_CODE_ORIGIN=1. */
61
+ codeOrigin?: boolean;
47
62
  /** Fetch the service's payload capture rules (APM-016) and record bodies/GraphQL where a rule matches; defaults to WT_CAPTURE_RULES=1. */
48
63
  captureRules?: boolean;
49
64
  /** Replace the span exporter (tests). */
50
65
  exporter?: unknown;
66
+ /**
67
+ * Scrub URLs on every span, including spans other libraries create (Next.js, node:http, fetch): the query string is dropped
68
+ * (`url.query` removed) and path segments that look like secrets or ids (>= 24 chars, or >= 10 chars mixing letters and digits,
69
+ * UUIDs included) become `:redacted`, in `http.target`, `url.full`, `url.path`, `http.url` and URLs in span names.
70
+ * `http.route` is kept. Default true; `false` records URLs as the instrumentation produced them.
71
+ */
72
+ urlScrubbing?: boolean;
73
+ /** Return false to drop a finished span before export (the `/next` entry uses it for `nextInternals`). */
74
+ spanFilter?: (span: ReadableSpan) => boolean;
75
+ /**
76
+ * Patch node:http, fetch (undici), pg and ioredis for those whose `@opentelemetry/instrumentation-*` peer is installed; defaults to on unless WT_INSTRUMENT=0.
77
+ * `'auto'` (or WT_INSTRUMENT=auto) also instruments whichever of postgres.js, better-sqlite3 and mysql2 is installed and logs one line naming what it patched
78
+ * (`instrumentLog: false` or WT_INSTRUMENT_LOG=0 silences it). `init` only: use the `/register` preload so it runs before the app imports them.
79
+ */
80
+ instrument?: boolean | 'auto';
81
+ /** Print the one-line list of what `instrument: 'auto'` patched (default true). */
82
+ instrumentLog?: boolean;
51
83
  fetch?: typeof fetch;
52
84
  }
53
85
  export interface EvalRow {
@@ -77,9 +109,10 @@ export declare class Client {
77
109
  readonly project: string | undefined;
78
110
  readonly headers: {
79
111
  Authorization: string;
80
- };
112
+ } & Record<string, string>;
81
113
  readonly fetch: typeof fetch;
82
114
  readonly redact: InitOptions['redact'];
115
+ private readonly logCaller;
83
116
  readonly provider: TracerProvider;
84
117
  readonly resource: Resource;
85
118
  readonly tracer: Tracer;
@@ -89,11 +122,15 @@ export declare class Client {
89
122
  readonly meterProvider?: MeterProvider;
90
123
  private readonly stopRuntime;
91
124
  uninstall?: () => void;
125
+ /** Set when remote sampling is on. */
126
+ remoteSampler?: RemoteSampler;
92
127
  /** Set when WT_PROFILING=1 started the continuous profiler. */
93
128
  profiler?: Promise<Profiler | undefined>;
94
129
  constructor(opts: InitOptions);
95
130
  /** Emit an OTLP log record; trace and span ids come from the active context. */
96
131
  log(level: LogLevel, body: string, attributes?: Record<string, unknown>): void;
132
+ /** Run `fn` as a cron / scheduler job with check-ins to the monitor `monitorSlug` (needs the `ingest:checkins` key scope): `in_progress`, then `ok` or `error` with the duration. */
133
+ checkin<T>(monitorSlug: string, fn: () => T | Promise<T>): Promise<T>;
97
134
  forceFlush(): Promise<unknown>;
98
135
  toJSON(): {
99
136
  endpoint: string;
@@ -115,6 +152,7 @@ export declare class Client {
115
152
  shutdown(): Promise<void>;
116
153
  }
117
154
  export declare function init(opts: InitOptions): Client;
155
+ export declare const checkin: <T>(slug: string, fn: () => T | Promise<T>) => Promise<T>;
118
156
  export declare const flush: () => Promise<unknown>;
119
157
  export declare const log: (l: LogLevel, b: string, a?: Record<string, unknown>) => void;
120
158
  export declare const getPrompt: (id: string) => Promise<unknown>;
package/dist/esm/index.js CHANGED
@@ -1,17 +1,32 @@
1
1
  // Thin Watchtower layer over the OpenTelemetry Node SDK: key header, endpoint default,
2
2
  // offline buffer, wt.*-safe resource attributes, redaction hook, getPrompt/score/evaluate/observe.
3
3
  import { inspect } from 'node:util';
4
- import { context, metrics, propagation, trace } from '@opentelemetry/api';
4
+ import { context, diag, metrics, propagation, trace } from '@opentelemetry/api';
5
5
  import { need as needPeer } from './load.js';
6
+ import { otelEnv } from './env.js';
7
+ import { instrumentAll } from './instrument.js';
8
+ import { LOG_BODY_KEY, redactor } from './redact.js';
9
+ import { urlScrubber } from './scrub.js';
10
+ import { filterExporter } from './filter.js';
11
+ import { checkin as runCheckin } from './checkin.js';
12
+ import { installSignalFlush } from './shutdown.js';
13
+ import { VERSION } from './version.js';
6
14
  import { SpoolExporter, spoolLogs } from './buffer.js';
7
15
  import { startMetrics, switchOn } from './metrics.js';
8
16
  import * as nodePath from 'node:path';
9
17
  import { captureException, installProcessHandlers, dropReserved } from './errors.js';
10
18
  import { loadCaptureRules } from './payload.js';
11
19
  import { startProfiling } from './profiling.js';
20
+ import { RemoteSampler } from './sampling.js';
21
+ import { codeOrigin } from './origin.js';
12
22
  export { captureException, metrics, trace };
13
23
  export { setCaptureRules, capturePayload } from './payload.js';
14
24
  export const DEFAULT_ENDPOINT = 'http://localhost:4318';
25
+ /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
26
+ export function callerOf(stack) {
27
+ const m = /\(?((?:file:\/\/)?[^()\s]+):(\d+):\d+\)?$/.exec((stack ?? '').split('\n')[2] ?? '');
28
+ return m ? { 'code.filepath': m[1].replace(/^file:\/\//, ''), 'code.lineno': Number(m[2]) } : {};
29
+ }
15
30
  const SEVERITY = { trace: 1, debug: 5, info: 9, warn: 13, error: 17, fatal: 21 };
16
31
  /** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
17
32
  export function safeResource(attrs) {
@@ -25,6 +40,7 @@ export class Client {
25
40
  headers;
26
41
  fetch;
27
42
  redact;
43
+ logCaller;
28
44
  provider;
29
45
  resource;
30
46
  tracer;
@@ -34,31 +50,43 @@ export class Client {
34
50
  meterProvider;
35
51
  stopRuntime = () => { };
36
52
  uninstall;
53
+ /** Set when remote sampling is on. */
54
+ remoteSampler;
37
55
  /** Set when WT_PROFILING=1 started the continuous profiler. */
38
56
  profiler;
39
57
  constructor(opts) {
40
58
  const o = opts || {};
41
59
  const env = process.env;
42
- const apiKey = o.apiKey || env.WT_INGEST_KEY;
60
+ const oe = otelEnv(env);
61
+ const apiKey = o.apiKey || env.WT_INGEST_KEY || oe.bearer;
43
62
  if (!apiKey)
44
63
  throw new Error('apiKey is required (or set WT_INGEST_KEY)');
45
- this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || DEFAULT_ENDPOINT).replace(/\/+$/, '');
64
+ this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || oe.endpoint || DEFAULT_ENDPOINT).replace(/\/+$/, '');
46
65
  this.apiUrl = (o.apiUrl || env.WT_API_URL || this.endpoint).replace(/\/+$/, '');
47
66
  this.project = o.project || env.WT_PROJECT_ID;
48
- this.headers = { Authorization: `Bearer ${apiKey}` };
67
+ this.headers = { ...oe.headers, Authorization: `Bearer ${apiKey}` };
49
68
  this.fetch = o.fetch || globalThis.fetch;
50
- const environment = o.environment || env.WT_ENVIRONMENT;
51
- const release = o.release || env.WT_RELEASE;
69
+ const environment = o.environment || env.WT_ENVIRONMENT || oe.resource['deployment.environment.name'];
70
+ const release = o.release || env.WT_RELEASE || oe.resource['service.version'];
52
71
  // the OTel SDK packages are optional peers, loaded here (not at import time) with a clear error when missing
53
- const { resourceFromAttributes } = needPeer('@opentelemetry/resources');
72
+ const { resourceFromAttributes, defaultResource } = needPeer('@opentelemetry/resources');
73
+ // the SDK's own telemetry.sdk.* (name, language, version): a bare resourceFromAttributes does not add them
74
+ const sdk = (defaultResource().attributes ?? {});
54
75
  const resource = resourceFromAttributes({
55
- 'service.name': o.serviceName || env.WT_SERVICE || 'app',
76
+ 'telemetry.sdk.name': String(sdk['telemetry.sdk.name'] ?? 'opentelemetry'),
77
+ 'telemetry.sdk.language': String(sdk['telemetry.sdk.language'] ?? 'nodejs'),
78
+ 'telemetry.sdk.version': String(sdk['telemetry.sdk.version'] ?? 'unknown'),
79
+ 'telemetry.distro.name': 'watchtower',
80
+ 'telemetry.distro.version': VERSION,
81
+ ...safeResource(oe.resource),
82
+ 'service.name': o.serviceName || env.WT_SERVICE || oe.serviceName || 'app',
56
83
  ...(environment ? { 'deployment.environment.name': environment } : {}),
57
84
  ...(release ? { 'service.version': release } : {}),
58
85
  ...safeResource(o.resource),
59
86
  });
60
87
  this.resource = resource;
61
88
  this.redact = o.redact;
89
+ this.logCaller = o.logCaller ?? env.WATCHTOWER_LOG_CALLER === '1';
62
90
  let exporter = o.exporter;
63
91
  if (!exporter) {
64
92
  const url = `${this.endpoint}/v1/traces`;
@@ -68,32 +96,47 @@ export class Client {
68
96
  if (o.bufferDir)
69
97
  exporter = new SpoolExporter(exporter, nodePath.join(o.bufferDir, 'traces'), url, this.headers, this.fetch);
70
98
  }
71
- // the redactor runs first, so the batch processor exports redacted attributes
99
+ if (o.spanFilter)
100
+ exporter = filterExporter(exporter, o.spanFilter);
101
+ // the scrubber and redactor run first, so the batch processor exports scrubbed attributes (the user's `redact` sees scrubbed URLs)
72
102
  const { BatchSpanProcessor, TracerProvider: TP } = needPeer('@opentelemetry/sdk-trace');
103
+ let sampler;
104
+ if (o.remoteSampling ?? env.WT_REMOTE_SAMPLING === '1') {
105
+ const { ParentBasedSampler } = needPeer('@opentelemetry/sdk-trace');
106
+ this.remoteSampler = new RemoteSampler({
107
+ endpoint: this.endpoint, headers: this.headers, service: String(resource.attributes['service.name']), env: environment,
108
+ intervalMs: (o.samplingIntervalS ?? (Number(env.WT_SAMPLING_INTERVAL_S) || 30)) * 1000, fetch: this.fetch,
109
+ });
110
+ sampler = new ParentBasedSampler({ root: this.remoteSampler });
111
+ }
73
112
  this.provider = new TP({
74
113
  resource,
75
- spanProcessors: [...(o.redact ? [redactor(o.redact)] : []), new BatchSpanProcessor({ exporter })],
114
+ ...(sampler ? { sampler: sampler } : {}),
115
+ spanProcessors: [
116
+ ...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [codeOrigin()] : []),
117
+ ...(o.urlScrubbing ?? true ? [urlScrubber()] : []),
118
+ ...(o.redact ? [redactor(o.redact)] : []),
119
+ new BatchSpanProcessor({ exporter }),
120
+ ],
76
121
  });
77
122
  this.tracer = this.provider.getTracer('watchtower');
78
- const logsOn = switchOn('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs);
123
+ // an explicit exporter never switches the other signals off: only `logs` / `metrics` / the env switches do
124
+ const logsOn = switchOn('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs ?? (o.logExporter ? true : undefined));
79
125
  let logExporter = logsOn ? o.logExporter : undefined;
80
- if (logsOn && !logExporter && !o.exporter) {
126
+ if (logsOn && !logExporter) {
81
127
  const url = `${this.endpoint}/v1/logs`;
82
128
  const { OTLPLogExporter } = needPeer('@opentelemetry/exporter-logs-otlp-http');
83
129
  logExporter = new OTLPLogExporter({ url, headers: this.headers });
84
130
  if (o.bufferDir)
85
131
  logExporter = spoolLogs(logExporter, nodePath.join(o.bufferDir, 'logs'), url, this.headers, this.fetch);
86
132
  }
87
- const { LoggerProvider: LP, SimpleLogRecordProcessor, BatchLogRecordProcessor } = needPeer('@opentelemetry/sdk-logs');
133
+ const { LoggerProvider: LP, BatchLogRecordProcessor } = needPeer('@opentelemetry/sdk-logs');
88
134
  this.loggerProvider = new LP({
89
135
  resource,
90
- processors: !logExporter ? [] : [o.logExporter
91
- ? new SimpleLogRecordProcessor({ exporter: logExporter })
92
- : new BatchLogRecordProcessor({ exporter: logExporter })],
136
+ processors: !logExporter ? [] : [new BatchLogRecordProcessor({ exporter: logExporter })],
93
137
  });
94
138
  this.logger = this.loggerProvider.getLogger('watchtower');
95
- // with a test span exporter only an explicit metric exporter is used, as for logs
96
- if (switchOn('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics) && (o.metricExporter || !o.exporter)) {
139
+ if (switchOn('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics ?? (o.metricExporter ? true : undefined))) {
97
140
  const m = startMetrics(this.endpoint, this.headers, resource, o.bufferDir, this.fetch, o.metricExporter);
98
141
  this.meterProvider = m.provider;
99
142
  this.stopRuntime = m.stop;
@@ -106,8 +149,12 @@ export class Client {
106
149
  log(level, body, attributes) {
107
150
  let attrs = dropReserved(attributes);
108
151
  const redact = this.redact;
109
- if (redact)
152
+ if (redact) {
110
153
  attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => [k, redact(k, v)]));
154
+ body = String(redact(LOG_BODY_KEY, body) ?? '');
155
+ }
156
+ if (this.logCaller)
157
+ attrs = { ...attrs, ...callerOf(new Error().stack) };
111
158
  this.logger.emit({
112
159
  severityNumber: SEVERITY[level] || SEVERITY.info,
113
160
  severityText: String(level).toUpperCase(),
@@ -115,6 +162,10 @@ export class Client {
115
162
  attributes: attrs,
116
163
  });
117
164
  }
165
+ /** Run `fn` as a cron / scheduler job with check-ins to the monitor `monitorSlug` (needs the `ingest:checkins` key scope): `in_progress`, then `ok` or `error` with the duration. */
166
+ checkin(monitorSlug, fn) {
167
+ return runCheckin(this, monitorSlug, fn);
168
+ }
118
169
  forceFlush() {
119
170
  return Promise.all([this.provider.forceFlush(), this.loggerProvider.forceFlush(), this.meterProvider?.forceFlush()]);
120
171
  }
@@ -215,41 +266,51 @@ export class Client {
215
266
  }
216
267
  shutdown() {
217
268
  this.stopRuntime();
269
+ this.remoteSampler?.stop();
218
270
  return Promise.all([this.profiler?.then((p) => p?.stop()), this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
219
271
  }
220
272
  }
221
- function redactor(hook) {
222
- return {
223
- onStart() { },
224
- onEnd(span) {
225
- for (const k of Object.keys(span.attributes))
226
- span.attributes[k] = hook(k, span.attributes[k]);
227
- },
228
- shutdown: () => Promise.resolve(),
229
- forceFlush: () => Promise.resolve(),
230
- };
273
+ // One state per process, not per module copy: a CommonJS and an ESM copy of this package (a `-r` preload plus an `import`
274
+ // app, say) must share the client or flush()/shutdown() in one copy would miss the init() of the other.
275
+ const state = (globalThis[Symbol.for('watchtower.node.state')] ??= {});
276
+ /**
277
+ * Register the global tracer provider, context manager, propagator and meter provider. OTel allows each to be set once
278
+ * (a second set returns false and changes nothing), so a provider the app or an earlier instrumentation registered is
279
+ * left alone and a warning says so; only globals a previous `init` installed are torn down on a re-init.
280
+ */
281
+ function registerGlobals(c) {
282
+ if (state.ownsGlobals) { // a re-init replaces what the previous init installed
283
+ trace.disable();
284
+ context.disable();
285
+ propagation.disable();
286
+ metrics.disable();
287
+ }
288
+ const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
289
+ const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
290
+ const ok = [
291
+ trace.setGlobalTracerProvider(c.provider),
292
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()),
293
+ propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })),
294
+ ...(c.meterProvider ? [metrics.setGlobalMeterProvider(c.meterProvider)] : []),
295
+ ];
296
+ state.ownsGlobals = ok[0] === true;
297
+ if (ok.includes(false))
298
+ diag.warn('watchtower: an OpenTelemetry global (tracer provider, context manager, propagator or meter provider) was already registered and was kept; call init() before other OTel setup to own them');
231
299
  }
232
- let current;
233
300
  export function init(opts) {
234
- if (current && current.uninstall)
235
- current.uninstall();
236
- void current?.profiler?.then((p) => p?.stop());
301
+ if (state.current && state.current.uninstall)
302
+ state.current.uninstall();
303
+ void state.current?.profiler?.then((p) => p?.stop());
237
304
  const c = new Client(opts);
238
- current = c;
239
- // Global tracer + async context so trace.getActiveSpan() works; a re-init replaces the previous client's.
240
- trace.disable();
241
- context.disable();
242
- propagation.disable();
243
- const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
244
- const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
245
- context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
246
- propagation.setGlobalPropagator(new CompositePropagator({
247
- propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
248
- }));
249
- trace.setGlobalTracerProvider(c.provider);
250
- if (c.meterProvider) {
251
- metrics.disable(); // a re-init replaces the previous client's provider
252
- metrics.setGlobalMeterProvider(c.meterProvider);
305
+ state.current = c;
306
+ registerGlobals(c);
307
+ const env = (process.env.WT_INSTRUMENT || '').trim().toLowerCase();
308
+ const mode = opts?.instrument ?? (env === 'auto' ? 'auto' : !['0', 'off', 'false', 'no'].includes(env));
309
+ if (mode) {
310
+ const names = instrumentAll({ tracerProvider: c.provider, meterProvider: c.meterProvider }, [c.endpoint, c.apiUrl], mode === 'auto');
311
+ if (mode === 'auto' && (opts?.instrumentLog ?? !['0', 'off', 'false', 'no'].includes((process.env.WT_INSTRUMENT_LOG || '').trim().toLowerCase()))) {
312
+ console.info(`[watchtower] instrumented: ${names.map((n) => n.replace('@opentelemetry/instrumentation-', '')).join(', ') || 'nothing (no supported package found)'}`);
313
+ }
253
314
  }
254
315
  if (process.env.WT_PROFILING === '1') {
255
316
  const attrs = c.resource.attributes;
@@ -261,20 +322,23 @@ export function init(opts) {
261
322
  }).catch(() => undefined); // best effort: never break init
262
323
  }
263
324
  if ((opts || {}).captureProcessErrors !== false) {
264
- c.uninstall = installProcessHandlers(() => c.forceFlush());
325
+ const offErrors = installProcessHandlers(() => c.forceFlush());
326
+ const offSignals = installSignalFlush(() => c.forceFlush());
327
+ c.uninstall = () => { offErrors(); offSignals(); };
265
328
  }
266
329
  return c;
267
330
  }
268
331
  function need() {
269
- if (!current)
332
+ if (!state.current)
270
333
  throw new Error('watchtower.init() has not been called');
271
- return current;
334
+ return state.current;
272
335
  }
273
- export const flush = () => (current ? current.forceFlush() : Promise.resolve());
336
+ export const checkin = (slug, fn) => need().checkin(slug, fn);
337
+ export const flush = () => (state.current ? state.current.forceFlush() : Promise.resolve());
274
338
  export const log = (l, b, a) => need().log(l, b, a);
275
339
  export const getPrompt = (id) => need().getPrompt(id);
276
340
  export const score = (t, v, o) => need().score(t, v, o);
277
341
  export const evaluate = (d, t, s, o) => need().evaluate(d, t, s, o);
278
342
  export const evaluateResume = (d, t, s, prev, o) => need().evaluateResume(d, t, s, prev, o);
279
343
  export const observe = (n, f) => need().observe(n, f);
280
- export const shutdown = () => (current ? current.shutdown() : Promise.resolve());
344
+ export const shutdown = () => (state.current ? state.current.shutdown() : Promise.resolve());
@@ -0,0 +1,10 @@
1
+ interface Providers {
2
+ tracerProvider: unknown;
3
+ meterProvider?: unknown;
4
+ }
5
+ /**
6
+ * Register whichever of the OTel instrumentations is installed (http, undici, pg, ioredis; with `auto` also mysql2) and,
7
+ * with `auto`, patch postgres.js and better-sqlite3 when installed. Returns the names that were enabled.
8
+ */
9
+ export declare function instrumentAll(p: Providers, selfUrls: string[], auto?: boolean): string[];
10
+ export {};
@@ -0,0 +1,71 @@
1
+ // Auto-instrumentation: node:http, fetch (undici), pg and ioredis, each only when its optional peer package is installed.
2
+ import { load } from './load.js';
3
+ import { instrumentBetterSqlite3 } from './integrations/sqlite.js';
4
+ import { instrumentPostgres } from './integrations/postgres.js';
5
+ const ENTRIES = [
6
+ {
7
+ pkg: '@opentelemetry/instrumentation-http', ctor: 'HttpInstrumentation',
8
+ // our own exports and API calls must not become spans
9
+ opts: (self) => ({ ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
10
+ },
11
+ {
12
+ pkg: '@opentelemetry/instrumentation-undici', ctor: 'UndiciInstrumentation',
13
+ opts: (self) => ({ ignoreRequestHook: (r) => self.some((u) => r.origin === u.origin) }),
14
+ },
15
+ { pkg: '@opentelemetry/instrumentation-pg', ctor: 'PgInstrumentation', opts: () => ({}) },
16
+ { pkg: '@opentelemetry/instrumentation-ioredis', ctor: 'IORedisInstrumentation', opts: () => ({}) },
17
+ { pkg: '@opentelemetry/instrumentation-mysql2', ctor: 'MySQL2Instrumentation', opts: () => ({}), auto: true },
18
+ ];
19
+ /** Drivers patched in place (no OTel package): `instrument: 'auto'` only. Each loads the app's installed copy or is skipped. */
20
+ const DRIVERS = [
21
+ { name: 'postgres.js', run: () => { load('postgres'); instrumentPostgres(); } }, // only when `postgres` is installed
22
+ { name: 'better-sqlite3', run: () => instrumentBetterSqlite3(load('better-sqlite3')) },
23
+ ];
24
+ /**
25
+ * Register whichever of the OTel instrumentations is installed (http, undici, pg, ioredis; with `auto` also mysql2) and,
26
+ * with `auto`, patch postgres.js and better-sqlite3 when installed. Returns the names that were enabled.
27
+ */
28
+ export function instrumentAll(p, selfUrls, auto = false) {
29
+ let registerInstrumentations;
30
+ try {
31
+ ({ registerInstrumentations } = load('@opentelemetry/instrumentation'));
32
+ }
33
+ catch {
34
+ return auto ? patchDrivers([]) : []; // optional peer not installed
35
+ }
36
+ const self = selfUrls.flatMap((u) => { try {
37
+ return [new URL(u)];
38
+ }
39
+ catch {
40
+ return [];
41
+ } });
42
+ const found = [];
43
+ const names = [];
44
+ for (const e of ENTRIES) {
45
+ if (e.auto && !auto)
46
+ continue;
47
+ try {
48
+ const Ctor = load(e.pkg)[e.ctor];
49
+ if (Ctor) {
50
+ found.push(new Ctor(e.opts(self)));
51
+ names.push(e.pkg);
52
+ }
53
+ }
54
+ catch {
55
+ // package not installed (or failed to construct): skip it, never break init
56
+ }
57
+ }
58
+ if (found.length)
59
+ registerInstrumentations({ instrumentations: found, tracerProvider: p.tracerProvider, meterProvider: p.meterProvider });
60
+ return auto ? patchDrivers(names) : names;
61
+ }
62
+ function patchDrivers(names) {
63
+ for (const d of DRIVERS) {
64
+ try {
65
+ d.run();
66
+ names.push(d.name);
67
+ }
68
+ catch { /* driver not installed */ }
69
+ }
70
+ return names;
71
+ }
@@ -5,6 +5,9 @@ export interface DbOptions {
5
5
  /** Flag a statement run more than this many times under one parent span. Default:
6
6
  * `WATCHTOWER_DB_N_PLUS_ONE` or 5; 0 turns it off. */
7
7
  nPlusOne?: number;
8
+ /** Record a query only when a span is active (a request, job or `observe` span). Default true (`WATCHTOWER_DB_REQUIRE_PARENT=0`
9
+ * turns it off): startup DDL, migrations and polling loops run outside any span and would otherwise each become a root trace. */
10
+ requireParent?: boolean;
8
11
  }
9
12
  export interface DbOp {
10
13
  /** `db.system.name`, e.g. `sqlite`. */
@@ -2,7 +2,7 @@
2
2
  // statement hash and N+1 hint, connection-pool metrics and embedded-store gauges.
3
3
  import * as fs from 'node:fs';
4
4
  import * as nodePath from 'node:path';
5
- import { context, createContextKey, metrics, trace, SpanKind, SpanStatusCode } from '@opentelemetry/api';
5
+ import { context, createContextKey, INVALID_SPAN_CONTEXT, metrics, trace, SpanKind, SpanStatusCode } from '@opentelemetry/api';
6
6
  import { isPgDialect, normalizeAndHash, querySummary } from '../sql.js';
7
7
  /** Set inside an instrumented call so a wrapper the library calls internally adds no second span. */
8
8
  const INSIDE = createContextKey('watchtower.db.inside');
@@ -27,6 +27,10 @@ function repeats(parent, key) {
27
27
  }
28
28
  /** A CLIENT span for `op` (not made active): summary, hash, N+1 hint. Prefer `traced`. */
29
29
  export function startDbSpan(op, opts, parent, startTime) {
30
+ const p = trace.getSpan(parent)?.spanContext();
31
+ const requireParent = opts?.requireParent ?? !['0', 'false', 'off', 'no'].includes((process.env.WATCHTOWER_DB_REQUIRE_PARENT || '').trim().toLowerCase());
32
+ if (requireParent && !(p && trace.isSpanContextValid(p)))
33
+ return trace.wrapSpanContext(INVALID_SPAN_CONTEXT); // non-recording: nothing is exported
30
34
  const attrs = { 'db.system.name': op.system, ...op.attributes };
31
35
  let { operation, collection } = op;
32
36
  let name = [operation, collection].filter(Boolean).join(' ') || op.system;
@@ -49,7 +53,6 @@ export function startDbSpan(op, opts, parent, startTime) {
49
53
  attrs['db.collection.name'] = collection;
50
54
  if (op.namespace)
51
55
  attrs['db.namespace'] = op.namespace;
52
- const p = trace.getSpan(parent)?.spanContext();
53
56
  const max = threshold(opts);
54
57
  if (p && max > 0) {
55
58
  const n = repeats(p.spanId, key);
@@ -1,5 +1,8 @@
1
- import type { Span } from '@opentelemetry/api';
1
+ import type { Attributes, Span } from '@opentelemetry/api';
2
2
  type Obj = Record<string, unknown>;
3
+ export declare const obj: (v: unknown) => Obj;
4
+ export declare const s: (v: unknown) => string;
5
+ export declare function start(name: string, attrs: Attributes, parent?: Span, startTime?: number): Span;
3
6
  /** The part of LlamaIndex.TS `Settings.callbackManager` used here. */
4
7
  export interface CallbackManagerLike {
5
8
  on(event: string, handler: (e: {
@@ -4,12 +4,12 @@
4
4
  // argument text is recorded. Both take the framework's own object, so neither package is a dependency.
5
5
  import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
6
6
  import { applyExtract, extract } from './genai-stream.js';
7
- const obj = (v) => (v && typeof v === 'object' ? v : {});
8
- const s = (v) => (typeof v === 'string' ? v : '');
9
- function start(name, attrs, parent) {
7
+ export const obj = (v) => (v && typeof v === 'object' ? v : {});
8
+ export const s = (v) => (typeof v === 'string' ? v : '');
9
+ export function start(name, attrs, parent, startTime) {
10
10
  const ctx = parent ? trace.setSpan(context.active(), parent) : context.active();
11
11
  const clean = Object.fromEntries(Object.entries(attrs).filter(([, v]) => v !== undefined && v !== ''));
12
- return trace.getTracer('watchtower').startSpan(name, { kind: SpanKind.INTERNAL, attributes: clean }, ctx);
12
+ return trace.getTracer('watchtower').startSpan(name, { kind: SpanKind.INTERNAL, attributes: clean, ...(startTime !== undefined ? { startTime } : {}) }, ctx);
13
13
  }
14
14
  /** instrumentLlamaIndex(Settings) (from `llamaindex`): llm-start/stream/end and tool call events as spans. */
15
15
  export function instrumentLlamaIndex(settings) {
@@ -0,0 +1,25 @@
1
+ import type { Span } from '@opentelemetry/api';
2
+ type Fn = (...a: unknown[]) => unknown;
3
+ /**
4
+ * langchainCallbackHandler(): pass it as `callbacks: [handler]` to a LangChain.js model, chain or LangGraph.js graph
5
+ * (`graph.invoke(input, { callbacks: [handler] })`). LangGraph nodes (chains whose metadata names `langgraph_node`)
6
+ * and the graph itself become `invoke_agent` spans, tools `execute_tool`, model calls `chat`, nested as they run;
7
+ * plain LCEL chains get no span. Returns a plain handler object (LangChain wraps it) with the open spans for tests.
8
+ */
9
+ export declare function langchainCallbackHandler(): Record<string, unknown> & {
10
+ open: Map<string, Span>;
11
+ };
12
+ /**
13
+ * instrumentVercelAI({ generateText, streamText }) with the functions from `ai`: returns wrapped copies (ES module
14
+ * exports cannot be patched). Pass `agentName` to name the run (default: the `experimental_telemetry.functionId`, else `agent`).
15
+ * const { generateText } = instrumentVercelAI({ generateText: ai.generateText });
16
+ * A multi-step loop (`maxSteps` / `stopWhen`) gives one `invoke_agent` span with a `chat` span per step and an
17
+ * `execute_tool` span per tool call.
18
+ */
19
+ export declare function instrumentVercelAI<T extends {
20
+ generateText?: Fn;
21
+ streamText?: Fn;
22
+ }>(ai: T, agentName?: string): T;
23
+ /** instrumentMastra(agent): wraps `agent.generate` and `agent.stream` of a Mastra `Agent`; the run is named after the agent. Returns the agent. */
24
+ export declare function instrumentMastra<T>(agent: T): T;
25
+ export {};