@watchtowerobservability/node 0.7.4 → 0.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +36 -1
  2. package/README.md +96 -12
  3. package/dist/cjs/attach.d.ts +7 -0
  4. package/dist/cjs/attach.js +29 -0
  5. package/dist/cjs/checkin.js +4 -13
  6. package/dist/cjs/cron/monitor.d.ts +47 -0
  7. package/dist/cjs/cron/monitor.js +95 -0
  8. package/dist/cjs/cron/schedulers.d.ts +36 -0
  9. package/dist/cjs/cron/schedulers.js +65 -0
  10. package/dist/cjs/env.d.ts +10 -0
  11. package/dist/cjs/env.js +98 -1
  12. package/dist/cjs/index.d.ts +50 -6
  13. package/dist/cjs/index.js +114 -37
  14. package/dist/cjs/instrument-for.d.ts +17 -0
  15. package/dist/cjs/instrument-for.js +44 -0
  16. package/dist/cjs/instrument.d.ts +4 -0
  17. package/dist/cjs/instrument.js +32 -1
  18. package/dist/cjs/integrations/bullmq.d.ts +8 -2
  19. package/dist/cjs/integrations/bullmq.js +32 -5
  20. package/dist/cjs/integrations/db.d.ts +7 -2
  21. package/dist/cjs/integrations/db.js +32 -4
  22. package/dist/cjs/integrations/fastify.d.ts +1 -1
  23. package/dist/cjs/integrations/fastify.js +3 -2
  24. package/dist/cjs/integrations/logs.js +16 -5
  25. package/dist/cjs/integrations/next-build.d.ts +25 -8
  26. package/dist/cjs/integrations/next-build.js +159 -34
  27. package/dist/cjs/integrations/next-tunnel.d.ts +6 -1
  28. package/dist/cjs/integrations/next-tunnel.js +13 -2
  29. package/dist/cjs/integrations/postgres.d.ts +2 -0
  30. package/dist/cjs/integrations/postgres.js +43 -1
  31. package/dist/cjs/integrations/temporal.d.ts +97 -0
  32. package/dist/cjs/integrations/temporal.js +144 -0
  33. package/dist/cjs/integrations/util.d.ts +17 -0
  34. package/dist/cjs/integrations/util.js +38 -0
  35. package/dist/cjs/load.d.ts +6 -0
  36. package/dist/cjs/load.js +28 -4
  37. package/dist/cjs/next-doctor-cli.d.ts +2 -0
  38. package/dist/cjs/next-doctor-cli.js +15 -0
  39. package/dist/cjs/next-doctor.d.ts +2 -0
  40. package/dist/cjs/next-doctor.js +50 -0
  41. package/dist/cjs/redact-presets.d.ts +16 -0
  42. package/dist/cjs/redact-presets.js +66 -0
  43. package/dist/cjs/register.js +19 -6
  44. package/dist/cjs/version.d.ts +1 -1
  45. package/dist/cjs/version.js +1 -1
  46. package/dist/esm/attach.d.ts +7 -0
  47. package/dist/esm/attach.js +25 -0
  48. package/dist/esm/checkin.js +4 -13
  49. package/dist/esm/cron/monitor.d.ts +47 -0
  50. package/dist/esm/cron/monitor.js +85 -0
  51. package/dist/esm/cron/schedulers.d.ts +36 -0
  52. package/dist/esm/cron/schedulers.js +61 -0
  53. package/dist/esm/env.d.ts +10 -0
  54. package/dist/esm/env.js +62 -0
  55. package/dist/esm/index.d.ts +50 -6
  56. package/dist/esm/index.js +109 -39
  57. package/dist/esm/instrument-for.d.ts +17 -0
  58. package/dist/esm/instrument-for.js +41 -0
  59. package/dist/esm/instrument.d.ts +4 -0
  60. package/dist/esm/instrument.js +30 -1
  61. package/dist/esm/integrations/bullmq.d.ts +8 -2
  62. package/dist/esm/integrations/bullmq.js +32 -5
  63. package/dist/esm/integrations/db.d.ts +7 -2
  64. package/dist/esm/integrations/db.js +29 -4
  65. package/dist/esm/integrations/fastify.d.ts +1 -1
  66. package/dist/esm/integrations/fastify.js +4 -3
  67. package/dist/esm/integrations/logs.js +16 -5
  68. package/dist/esm/integrations/next-build.d.ts +25 -8
  69. package/dist/esm/integrations/next-build.js +158 -34
  70. package/dist/esm/integrations/next-tunnel.d.ts +6 -1
  71. package/dist/esm/integrations/next-tunnel.js +13 -2
  72. package/dist/esm/integrations/postgres.d.ts +2 -0
  73. package/dist/esm/integrations/postgres.js +43 -2
  74. package/dist/esm/integrations/temporal.d.ts +97 -0
  75. package/dist/esm/integrations/temporal.js +139 -0
  76. package/dist/esm/integrations/util.d.ts +17 -0
  77. package/dist/esm/integrations/util.js +35 -0
  78. package/dist/esm/load.d.ts +6 -0
  79. package/dist/esm/load.js +21 -3
  80. package/dist/esm/next-doctor-cli.d.ts +2 -0
  81. package/dist/esm/next-doctor-cli.js +13 -0
  82. package/dist/esm/next-doctor.d.ts +2 -0
  83. package/dist/esm/next-doctor.js +14 -0
  84. package/dist/esm/redact-presets.d.ts +16 -0
  85. package/dist/esm/redact-presets.js +58 -0
  86. package/dist/esm/register.js +20 -7
  87. package/dist/esm/version.d.ts +1 -1
  88. package/dist/esm/version.js +1 -1
  89. package/package.json +42 -1
package/dist/esm/index.js CHANGED
@@ -2,14 +2,18 @@
2
2
  // offline buffer, wt.*-safe resource attributes, redaction hook, getPrompt/score/evaluate/observe.
3
3
  import { inspect } from 'node:util';
4
4
  import { context, diag, metrics, propagation, trace } from '@opentelemetry/api';
5
- import { need as needPeer } from './load.js';
6
- import { otelEnv } from './env.js';
5
+ import { missingPeers, need as needPeer, STANDALONE_HINT } from './load.js';
6
+ import { otelEnv, resolveRelease } from './env.js';
7
+ import { dbRequireParent, setHttpServerActive } from './integrations/db.js';
7
8
  import { instrumentAll } from './instrument.js';
8
9
  import { LOG_BODY_KEY, redactor } from './redact.js';
9
- import { urlScrubber } from './scrub.js';
10
+ import { resolveRedact } from './redact-presets.js';
11
+ import { foreignProvider, attachProcessors } from './attach.js';
12
+ import { scrubUrl, urlScrubber } from './scrub.js';
10
13
  import { filterExporter } from './filter.js';
11
14
  import { nextInternalsFilter } from './next-filter.js';
12
15
  import { checkin as runCheckin } from './checkin.js';
16
+ import { checkIn as checkInFor, withMonitor as withMonitorFor } from './cron/monitor.js';
13
17
  import { installSignalFlush } from './shutdown.js';
14
18
  import { VERSION } from './version.js';
15
19
  import { SpoolExporter, spoolLogs } from './buffer.js';
@@ -21,7 +25,10 @@ import { startProfiling } from './profiling.js';
21
25
  import { RemoteSampler } from './sampling.js';
22
26
  import { codeOrigin } from './origin.js';
23
27
  export { captureException, metrics, trace };
28
+ export { instrument } from './instrument-for.js';
29
+ export { droppedDbSpans } from './integrations/db.js';
24
30
  export { setCaptureRules, capturePayload } from './payload.js';
31
+ export { instrumentNestSchedule, instrumentNodeCron } from './cron/schedulers.js';
25
32
  export const DEFAULT_ENDPOINT = 'http://localhost:4318';
26
33
  /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
27
34
  export function callerOf(stack) {
@@ -41,6 +48,16 @@ export class Client {
41
48
  headers;
42
49
  fetch;
43
50
  redact;
51
+ scrubUrls;
52
+ /** What `init` found: `blocked` lists the OTel globals that were already registered and kept. */
53
+ traces = 'ok';
54
+ metrics = 'ok';
55
+ logs = 'ok';
56
+ blocked = [];
57
+ warnings = [];
58
+ /** True when the exporter was attached to another provider's pipeline (`otel`). */
59
+ attached = false;
60
+ spanProcessors = [];
44
61
  logCaller;
45
62
  provider;
46
63
  resource;
@@ -68,7 +85,7 @@ export class Client {
68
85
  this.headers = { ...oe.headers, Authorization: `Bearer ${apiKey}` };
69
86
  this.fetch = o.fetch || globalThis.fetch;
70
87
  const environment = o.environment || env.WT_ENVIRONMENT || oe.resource['deployment.environment.name'];
71
- const release = o.release || env.WT_RELEASE || oe.resource['service.version'];
88
+ const release = resolveRelease(o.release, env, oe.resource, process.cwd());
72
89
  // the OTel SDK packages are optional peers, loaded here (not at import time) with a clear error when missing
73
90
  const { resourceFromAttributes, defaultResource } = needPeer('@opentelemetry/resources');
74
91
  // the SDK's own telemetry.sdk.* (name, language, version): a bare resourceFromAttributes does not add them
@@ -86,7 +103,9 @@ export class Client {
86
103
  ...safeResource(o.resource),
87
104
  });
88
105
  this.resource = resource;
89
- this.redact = o.redact;
106
+ const red = resolveRedact(o.redact);
107
+ this.redact = red.hook;
108
+ this.scrubUrls = o.urlScrubbing ?? red.preset !== 'off';
90
109
  this.logCaller = o.logCaller ?? env.WATCHTOWER_LOG_CALLER === '1';
91
110
  let exporter = o.exporter;
92
111
  if (!exporter) {
@@ -110,19 +129,30 @@ export class Client {
110
129
  });
111
130
  sampler = new ParentBasedSampler({ root: this.remoteSampler });
112
131
  }
113
- this.provider = new TP({
114
- resource,
115
- ...(sampler ? { sampler: sampler } : {}),
116
- spanProcessors: [
117
- ...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [codeOrigin()] : []),
118
- ...(o.urlScrubbing ?? true ? [urlScrubber()] : []),
119
- ...(o.redact ? [redactor(o.redact)] : []),
120
- new BatchSpanProcessor({ exporter }),
121
- ],
122
- });
132
+ const procs = [
133
+ ...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [codeOrigin()] : []),
134
+ ...(o.urlScrubbing ?? red.preset !== 'off' ? [urlScrubber()] : []),
135
+ ...(red.hook ? [redactor(red.hook)] : []),
136
+ new BatchSpanProcessor({ exporter }),
137
+ ];
138
+ const mode = o.otel ?? 'auto';
139
+ const foreign = mode !== 'own' && !state.ownsGlobals ? foreignProvider() : undefined;
140
+ if (foreign && attachProcessors(foreign, procs)) {
141
+ this.attached = true;
142
+ this.spanProcessors = procs;
143
+ this.provider = foreign;
144
+ if (sampler)
145
+ this.warnings.push('remoteSampling is not applied when attached to another tracer provider');
146
+ }
147
+ else {
148
+ if (foreign || mode === 'attach')
149
+ this.warnings.push(foreign ? 'otel: the existing tracer provider has no span-processor hook; using our own' : "otel: 'attach' but no tracer provider is registered yet; using our own");
150
+ this.provider = new TP({ resource, ...(sampler ? { sampler: sampler } : {}), spanProcessors: procs });
151
+ }
123
152
  this.tracer = this.provider.getTracer('watchtower');
124
153
  // an explicit exporter never switches the other signals off: only `logs` / `metrics` / the env switches do
125
154
  const logsOn = switchOn('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs ?? (o.logExporter ? true : undefined));
155
+ this.logs = logsOn ? 'ok' : 'disabled';
126
156
  let logExporter = logsOn ? o.logExporter : undefined;
127
157
  if (logsOn && !logExporter) {
128
158
  const url = `${this.endpoint}/v1/logs`;
@@ -142,6 +172,9 @@ export class Client {
142
172
  this.meterProvider = m.provider;
143
173
  this.stopRuntime = m.stop;
144
174
  }
175
+ else {
176
+ this.metrics = 'disabled';
177
+ }
145
178
  if (o.captureRules ?? env.WT_CAPTURE_RULES === '1') {
146
179
  void loadCaptureRules({ apiUrl: this.apiUrl, key: apiKey, service: String(resource.attributes['service.name']), fetch: this.fetch });
147
180
  }
@@ -151,9 +184,18 @@ export class Client {
151
184
  let attrs = dropReserved(attributes);
152
185
  const redact = this.redact;
153
186
  if (redact) {
154
- attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => [k, redact(k, v)]));
187
+ // dotted keys (flattened nested objects) also go through the hook under their leaf name, so a hook keyed on `url` sees `req.url`
188
+ attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => {
189
+ v = redact(k, v);
190
+ const leaf = k.slice(k.lastIndexOf('.') + 1);
191
+ return [k, leaf !== k ? redact(leaf, v) : v];
192
+ }));
155
193
  body = String(redact(LOG_BODY_KEY, body) ?? '');
156
194
  }
195
+ if (this.scrubUrls)
196
+ for (const k of Object.keys(attrs))
197
+ if (typeof attrs[k] === 'string' && /(^|\.)(url|uri|target)$/.test(k))
198
+ attrs[k] = scrubUrl(attrs[k]);
157
199
  if (this.logCaller)
158
200
  attrs = { ...attrs, ...callerOf(new Error().stack) };
159
201
  this.logger.emit({
@@ -168,7 +210,8 @@ export class Client {
168
210
  return runCheckin(this, monitorSlug, fn);
169
211
  }
170
212
  forceFlush() {
171
- return Promise.all([this.provider.forceFlush(), this.loggerProvider.forceFlush(), this.meterProvider?.forceFlush()]);
213
+ const spans = this.attached ? Promise.all(this.spanProcessors.map((p) => p.forceFlush())) : this.provider.forceFlush();
214
+ return Promise.all([spans, this.loggerProvider.forceFlush(), this.meterProvider?.forceFlush()]);
172
215
  }
173
216
  // keep the key out of console.log / JSON.stringify
174
217
  toJSON() {
@@ -268,7 +311,7 @@ export class Client {
268
311
  shutdown() {
269
312
  this.stopRuntime();
270
313
  this.remoteSampler?.stop();
271
- return Promise.all([this.profiler?.then((p) => p?.stop()), this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
314
+ return Promise.all([this.profiler?.then((p) => p?.stop()), this.attached ? Promise.all(this.spanProcessors.map((p) => p.shutdown())) : this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
272
315
  }
273
316
  }
274
317
  // One state per process, not per module copy: a CommonJS and an ESM copy of this package (a `-r` preload plus an `import`
@@ -280,25 +323,42 @@ const state = (globalThis[Symbol.for('watchtower.node.state')] ??= {});
280
323
  * left alone and a warning says so; only globals a previous `init` installed are torn down on a re-init.
281
324
  */
282
325
  function registerGlobals(c) {
326
+ const blocked = [];
283
327
  if (state.ownsGlobals) { // a re-init replaces what the previous init installed
284
328
  trace.disable();
285
329
  context.disable();
286
330
  propagation.disable();
287
331
  metrics.disable();
288
332
  }
289
- const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
290
- const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
291
- const ok = [
292
- trace.setGlobalTracerProvider(c.provider),
293
- context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()),
294
- propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })),
295
- ...(c.meterProvider ? [metrics.setGlobalMeterProvider(c.meterProvider)] : []),
296
- ];
297
- state.ownsGlobals = ok[0] === true;
298
- if (ok.includes(false))
299
- diag.warn('watchtower: an OpenTelemetry global (tracer provider, context manager, propagator or meter provider) was already registered and was kept; call init() before other OTel setup to own them');
333
+ if (!c.attached) { // attached: the other setup owns the provider, context manager and propagator
334
+ const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
335
+ const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
336
+ const ownTracer = trace.setGlobalTracerProvider(c.provider);
337
+ state.ownsGlobals = ownTracer === true;
338
+ if (!ownTracer)
339
+ blocked.push('tracer-provider');
340
+ if (!context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()))
341
+ blocked.push('context-manager');
342
+ if (!propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })))
343
+ blocked.push('propagator');
344
+ }
345
+ if (c.meterProvider && !metrics.setGlobalMeterProvider(c.meterProvider))
346
+ blocked.push('meter-provider');
347
+ c.blocked = blocked;
348
+ if (blocked.includes('tracer-provider'))
349
+ c.traces = 'blocked';
350
+ if (blocked.includes('meter-provider'))
351
+ c.metrics = 'blocked';
352
+ if (blocked.length) {
353
+ const w = `watchtower: OpenTelemetry global(s) already registered and kept: ${blocked.join(', ')}; call init() before other OTel setup to own them, or leave \`otel\` on 'auto' to attach to the existing provider`;
354
+ c.warnings.push(w);
355
+ diag.warn(w);
356
+ }
300
357
  }
301
358
  export function init(opts) {
359
+ const missing = missingPeers();
360
+ if (missing.length)
361
+ console.warn(`[watchtower] missing OpenTelemetry package(s): ${missing.join(', ')}. Next.js standalone: ${STANDALONE_HINT}. Otherwise: npm install ${missing.join(' ')}`);
302
362
  const nextMode = opts?.nextInternals ?? (process.env.NEXT_RUNTIME ? 'sample' : undefined);
303
363
  if (nextMode) {
304
364
  const own = nextInternalsFilter(nextMode, opts?.nextInternalsRate);
@@ -315,11 +375,12 @@ export function init(opts) {
315
375
  const mode = opts?.instrument ?? (env === 'auto' ? 'auto' : !['0', 'off', 'false', 'no'].includes(env));
316
376
  if (mode) {
317
377
  const names = instrumentAll({ tracerProvider: c.provider, meterProvider: c.meterProvider }, [c.endpoint, c.apiUrl], mode === 'auto');
378
+ setHttpServerActive(names.includes('@opentelemetry/instrumentation-http'));
318
379
  const logOn = opts?.instrumentLog ?? !['0', 'off', 'false', 'no'].includes((process.env.WT_INSTRUMENT_LOG || '').trim().toLowerCase());
319
380
  if (mode === 'auto' && logOn) {
320
381
  console.info(`[watchtower] instrumented: ${names.map((n) => n.replace('@opentelemetry/instrumentation-', '')).join(', ') || 'nothing (no supported package found)'}`);
321
382
  }
322
- if (logOn && !['0', 'false', 'off', 'no'].includes((process.env.WATCHTOWER_DB_REQUIRE_PARENT || '').trim().toLowerCase())) {
383
+ if (logOn && dbRequireParent(undefined)) {
323
384
  console.info('watchtower: requireParent on; queries outside a span are not recorded (WATCHTOWER_DB_REQUIRE_PARENT=0 to keep them)');
324
385
  }
325
386
  }
@@ -339,17 +400,26 @@ export function init(opts) {
339
400
  }
340
401
  return c;
341
402
  }
403
+ /** Every public helper is a no-op before `init()` (like `captureException`), so shared code can call them in any order. */
342
404
  function need() {
343
- if (!state.current)
344
- throw new Error('watchtower.init() has not been called');
405
+ if (!state.current && !state.offWarned && process.env.NODE_ENV !== 'test') {
406
+ state.offWarned = true; // once per process: a dropped call must not go unnoticed, nor flood the log
407
+ console.warn(`[watchtower] telemetry is OFF: ${state.offReason ?? 'init() was not called'}. Calls to log, observe, checkin and the other helpers are dropped.`);
408
+ }
345
409
  return state.current;
346
410
  }
347
- export const checkin = (slug, fn) => need().checkin(slug, fn);
411
+ /** Why telemetry is off (set by the preload when it cannot start); shown on the first dropped helper call. */
412
+ export const setOffReason = (r) => { state.offReason = r; };
413
+ /** One check-in for the monitor `slug`. Fire and forget by default (5 s cap, never throws); `{ wait: true }` awaits it. */
414
+ export const checkIn = (slug, status, o) => { const c = need(); return c ? checkInFor(c, slug, status, o) : Promise.resolve(); };
415
+ /** `withMonitor(slug, fn, { schedule, timezone, checkinMargin, maxRuntime, failureThreshold, wait })`: `fn` as a monitored job; the monitor is created or updated from `schedule`. */
416
+ export const withMonitor = (slug, fn, o) => { const c = need(); return c ? withMonitorFor(c, slug, fn, o) : Promise.resolve().then(fn); };
417
+ export const checkin = (slug, fn) => need()?.checkin(slug, fn) ?? Promise.resolve().then(fn);
348
418
  export const flush = () => (state.current ? state.current.forceFlush() : Promise.resolve());
349
- export const log = (l, b, a) => need().log(l, b, a);
350
- export const getPrompt = (id) => need().getPrompt(id);
351
- export const score = (t, v, o) => need().score(t, v, o);
352
- export const evaluate = (d, t, s, o) => need().evaluate(d, t, s, o);
353
- export const evaluateResume = (d, t, s, prev, o) => need().evaluateResume(d, t, s, prev, o);
354
- export const observe = (n, f) => need().observe(n, f);
419
+ export const log = (l, b, a) => need()?.log(l, b, a);
420
+ export const getPrompt = (id) => need()?.getPrompt(id) ?? Promise.resolve(undefined);
421
+ export const score = (t, v, o) => need()?.score(t, v, o) ?? Promise.resolve(undefined);
422
+ export const evaluate = (d, t, s, o) => need()?.evaluate(d, t, s, o) ?? Promise.resolve([]);
423
+ export const evaluateResume = (d, t, s, prev, o) => need()?.evaluateResume(d, t, s, prev, o) ?? Promise.resolve([]);
424
+ export const observe = (n, f) => need()?.observe(n, f) ?? f;
355
425
  export const shutdown = () => (state.current ? state.current.shutdown() : Promise.resolve());
@@ -0,0 +1,17 @@
1
+ export type Framework = 'nest' | 'express' | 'next' | 'bullmq' | 'pg' | 'postgres' | 'ioredis';
2
+ export interface InstrumentOptions {
3
+ /** URLs never traced as outgoing calls (our own endpoint); `init` passes its own. */
4
+ selfUrls?: string[];
5
+ }
6
+ /**
7
+ * `instrument('nest' | 'express' | 'next' | 'bullmq' | 'pg' | 'postgres' | 'ioredis')`. Call it after `init()`.
8
+ *
9
+ * - `nest`, `express`: node:http server instrumentation; server spans are renamed `METHOD /route/:param` from the matched route (Nest on Express).
10
+ * - `next`: http and undici (fetch); Next's own span filtering is `init`'s `nextInternals` (applied when `NEXT_RUNTIME` is set).
11
+ * - `bullmq`: producer and worker spans with trace context in the job (`instrumentBullMQ`); order does not matter, it patches the classes.
12
+ * - `postgres`: postgres.js through its diagnostics channel; order does not matter.
13
+ * - `pg`, `ioredis`: the OTel instrumentation, which hooks `require`: call before the first `require('pg')` / `import 'pg'` (best: the `/register` preload).
14
+ *
15
+ * Returns the packages or integrations enabled by this call (empty when already on, or its peer is not installed).
16
+ */
17
+ export declare function instrument(name: Framework, opts?: InstrumentOptions): string[];
@@ -0,0 +1,41 @@
1
+ // One call per framework: `instrument('nest')`. Applies the right instrumentation and returns what it enabled.
2
+ import { load } from './load.js';
3
+ import { instrumentOne } from './instrument.js';
4
+ import { setHttpServerActive } from './integrations/db.js';
5
+ import { instrumentPostgres } from './integrations/postgres.js';
6
+ import { instrumentBullMQ } from './integrations/bullmq.js';
7
+ const HTTP = '@opentelemetry/instrumentation-http';
8
+ const UNDICI = '@opentelemetry/instrumentation-undici';
9
+ /**
10
+ * `instrument('nest' | 'express' | 'next' | 'bullmq' | 'pg' | 'postgres' | 'ioredis')`. Call it after `init()`.
11
+ *
12
+ * - `nest`, `express`: node:http server instrumentation; server spans are renamed `METHOD /route/:param` from the matched route (Nest on Express).
13
+ * - `next`: http and undici (fetch); Next's own span filtering is `init`'s `nextInternals` (applied when `NEXT_RUNTIME` is set).
14
+ * - `bullmq`: producer and worker spans with trace context in the job (`instrumentBullMQ`); order does not matter, it patches the classes.
15
+ * - `postgres`: postgres.js through its diagnostics channel; order does not matter.
16
+ * - `pg`, `ioredis`: the OTel instrumentation, which hooks `require`: call before the first `require('pg')` / `import 'pg'` (best: the `/register` preload).
17
+ *
18
+ * Returns the packages or integrations enabled by this call (empty when already on, or its peer is not installed).
19
+ */
20
+ export function instrument(name, opts = {}) {
21
+ const self = opts.selfUrls ?? [];
22
+ const on = (pkgs) => {
23
+ const done = pkgs.filter((p) => instrumentOne(p, self));
24
+ if (pkgs.includes(HTTP) && done.includes(HTTP))
25
+ setHttpServerActive(true);
26
+ return done;
27
+ };
28
+ switch (name) {
29
+ case 'nest':
30
+ case 'express': return on([HTTP]);
31
+ case 'next': return on([HTTP, UNDICI]);
32
+ case 'pg': return on(['@opentelemetry/instrumentation-pg']);
33
+ case 'ioredis': return on(['@opentelemetry/instrumentation-ioredis']);
34
+ case 'postgres':
35
+ load('postgres');
36
+ instrumentPostgres();
37
+ return ['postgres.js'];
38
+ case 'bullmq': return instrumentBullMQ(load('bullmq')) ? ['bullmq'] : [];
39
+ default: throw new Error(`watchtower: instrument(${JSON.stringify(name)}): expected nest, express, next, bullmq, pg, postgres or ioredis`);
40
+ }
41
+ }
@@ -1,7 +1,11 @@
1
+ /** Instrumentation packages already registered in this process (by `init` or `instrument()`), so none is patched twice. */
2
+ export declare const registered: Set<string>;
1
3
  interface Providers {
2
4
  tracerProvider: unknown;
3
5
  meterProvider?: unknown;
4
6
  }
7
+ /** Register one entry by package name (for `instrument(name)`); false when already registered or not installed. Uses the global providers. */
8
+ export declare function instrumentOne(pkg: string, selfUrls?: string[]): boolean;
5
9
  /**
6
10
  * Register whichever of the OTel instrumentations is installed (http, undici, pg, ioredis; with `auto` also mysql2) and,
7
11
  * with `auto`, patch postgres.js and better-sqlite3 when installed. Returns the names that were enabled.
@@ -2,11 +2,15 @@
2
2
  import { load } from './load.js';
3
3
  import { instrumentBetterSqlite3 } from './integrations/sqlite.js';
4
4
  import { instrumentPostgres } from './integrations/postgres.js';
5
+ import { routeNamer } from './integrations/util.js';
6
+ /** Instrumentation packages already registered in this process (by `init` or `instrument()`), so none is patched twice. */
7
+ export const registered = new Set();
5
8
  const ENTRIES = [
6
9
  {
7
10
  pkg: '@opentelemetry/instrumentation-http', ctor: 'HttpInstrumentation',
8
11
  // our own exports and API calls must not become spans
9
- opts: (self) => ({ ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
12
+ // routeNamer: server spans become `GET /users/:id` from the matched Express / Nest route (http alone names them just `GET`)
13
+ opts: (self) => ({ applyCustomAttributesOnSpan: routeNamer, ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
10
14
  },
11
15
  {
12
16
  pkg: '@opentelemetry/instrumentation-undici', ctor: 'UndiciInstrumentation',
@@ -16,6 +20,30 @@ const ENTRIES = [
16
20
  { pkg: '@opentelemetry/instrumentation-ioredis', ctor: 'IORedisInstrumentation', opts: () => ({}) },
17
21
  { pkg: '@opentelemetry/instrumentation-mysql2', ctor: 'MySQL2Instrumentation', opts: () => ({}), auto: true },
18
22
  ];
23
+ /** Register one entry by package name (for `instrument(name)`); false when already registered or not installed. Uses the global providers. */
24
+ export function instrumentOne(pkg, selfUrls = []) {
25
+ const e = ENTRIES.find((x) => x.pkg === pkg);
26
+ if (!e || registered.has(pkg))
27
+ return false;
28
+ const self = selfUrls.flatMap((u) => { try {
29
+ return [new URL(u)];
30
+ }
31
+ catch {
32
+ return [];
33
+ } });
34
+ try {
35
+ const { registerInstrumentations } = load('@opentelemetry/instrumentation');
36
+ const Ctor = load(pkg)[e.ctor];
37
+ if (!Ctor)
38
+ return false;
39
+ registerInstrumentations({ instrumentations: [new Ctor(e.opts(self))] });
40
+ registered.add(pkg);
41
+ return true;
42
+ }
43
+ catch {
44
+ return false;
45
+ } // peer not installed
46
+ }
19
47
  /** Drivers patched in place (no OTel package): `instrument: 'auto'` only. Each loads the app's installed copy or is skipped. */
20
48
  const DRIVERS = [
21
49
  { name: 'postgres.js', run: () => { load('postgres'); instrumentPostgres(); } }, // only when `postgres` is installed
@@ -49,6 +77,7 @@ export function instrumentAll(p, selfUrls, auto = false) {
49
77
  if (Ctor) {
50
78
  found.push(new Ctor(e.opts(self)));
51
79
  names.push(e.pkg);
80
+ registered.add(e.pkg);
52
81
  }
53
82
  }
54
83
  catch {
@@ -8,12 +8,18 @@ interface BullMQLike {
8
8
  prototype: Record<string | symbol, unknown>;
9
9
  };
10
10
  }
11
+ export interface BullMQOptions {
12
+ /** Report a job's failure as an error when it is the last attempt (no retry left): queue, job name and id, attempt, attemptsMade and attempts as attributes. */
13
+ captureFailures?: boolean;
14
+ /** Run repeatable jobs (`repeat.pattern`/`cron`) as cron monitors named `<queue>-<job name>`; needs the `ingest:checkins` key scope. */
15
+ cronMonitors?: boolean;
16
+ }
11
17
  /**
12
18
  * `import * as bullmq from 'bullmq'; instrumentBullMQ(bullmq)` (ESM or CJS): PRODUCER spans on
13
19
  * `Queue.add`/`addBulk`, CONSUMER spans around each job a `Worker` processes, parented by the
14
20
  * producer's context carried in job data under `_wt` (stripped before your processor runs; object
15
21
  * data only). Call it once, before creating queues and workers. Returns false when this bullmq
16
- * has no `Worker.callProcessJob` (consumer spans off; producer spans still on).
22
+ * has no `Worker.callProcessJob` (consumer spans off; producer spans still on). Options: see `BullMQOptions`.
17
23
  * shortcut: sandboxed (file path) processors run in a child process and get no consumer span.
18
24
  */
19
- export declare function instrumentBullMQ(bullmq: BullMQLike): boolean;
25
+ export declare function instrumentBullMQ(bullmq: BullMQLike, options?: BullMQOptions): boolean;
@@ -1,5 +1,7 @@
1
1
  import { context, propagation, trace, SpanKind, SpanStatusCode } from '@opentelemetry/api';
2
2
  import { CONTEXT_KEY, injectData as inject, isPlainObject } from './messaging.js';
3
+ import { captureException } from '../errors.js';
4
+ import { monitorJob, slugify } from '../cron/monitor.js';
3
5
  export { CONTEXT_KEY };
4
6
  const PATCHED = Symbol.for('watchtower.bullmq');
5
7
  const tracer = () => trace.getTracer('watchtower');
@@ -9,12 +11,13 @@ const base = (queue, op, name) => ({
9
11
  'messaging.operation.type': op,
10
12
  'messaging.operation.name': name,
11
13
  });
12
- async function run(span, fn) {
14
+ async function run(span, fn, recorded) {
13
15
  try {
14
16
  return await fn();
15
17
  }
16
18
  catch (e) {
17
- span.recordException(e);
19
+ if (!recorded?.())
20
+ span.recordException(e);
18
21
  span.setStatus({ code: SpanStatusCode.ERROR, message: e?.message });
19
22
  throw e;
20
23
  }
@@ -27,10 +30,10 @@ async function run(span, fn) {
27
30
  * `Queue.add`/`addBulk`, CONSUMER spans around each job a `Worker` processes, parented by the
28
31
  * producer's context carried in job data under `_wt` (stripped before your processor runs; object
29
32
  * data only). Call it once, before creating queues and workers. Returns false when this bullmq
30
- * has no `Worker.callProcessJob` (consumer spans off; producer spans still on).
33
+ * has no `Worker.callProcessJob` (consumer spans off; producer spans still on). Options: see `BullMQOptions`.
31
34
  * shortcut: sandboxed (file path) processors run in a child process and get no consumer span.
32
35
  */
33
- export function instrumentBullMQ(bullmq) {
36
+ export function instrumentBullMQ(bullmq, options = {}) {
34
37
  const q = bullmq.Queue.prototype;
35
38
  const w = bullmq.Worker.prototype;
36
39
  if (q[PATCHED])
@@ -73,7 +76,31 @@ export function instrumentBullMQ(bullmq) {
73
76
  attrs['messaging.message.id'] = job.id;
74
77
  if (job.opts?.delay)
75
78
  attrs['messaging.bullmq.job.delay_ms'] = job.opts.delay;
76
- return tracer().startActiveSpan(`process ${this.name}`, { kind: SpanKind.CONSUMER, attributes: attrs }, parent, (span) => run(span, () => call.call(this, job, ...rest)));
79
+ const queue = this.name;
80
+ const rep = job.opts?.repeat;
81
+ const cron = options.cronMonitors ? rep?.pattern ?? rep?.cron : undefined;
82
+ return tracer().startActiveSpan(`process ${queue}`, { kind: SpanKind.CONSUMER, attributes: attrs }, parent, (span) => {
83
+ let recorded = false;
84
+ const exec = async () => {
85
+ try {
86
+ return await call.call(this, job, ...rest);
87
+ }
88
+ catch (e) {
89
+ const attempts = job.opts?.attempts ?? 1;
90
+ const made = (job.attemptsMade ?? 0) + 1;
91
+ if (options.captureFailures && (made >= attempts || e?.name === 'UnrecoverableError')) {
92
+ captureException(e, { attributes: {
93
+ 'bullmq.queue': queue, 'bullmq.job.name': job.name, 'bullmq.job.id': job.id, 'bullmq.job.attempt': made,
94
+ 'bullmq.job.attempts_made': made, 'bullmq.job.attempts': attempts,
95
+ } });
96
+ recorded = true;
97
+ }
98
+ throw e;
99
+ }
100
+ };
101
+ const slug = cron ? slugify(`${queue}-${job.name}`) : '';
102
+ return run(span, () => (slug ? monitorJob(slug, exec, { schedule: cron, timezone: rep?.tz }) : exec()), () => recorded);
103
+ });
77
104
  };
78
105
  return true;
79
106
  }
@@ -5,8 +5,7 @@ export interface DbOptions {
5
5
  /** Flag a statement run more than this many times under one parent span. Default:
6
6
  * `WATCHTOWER_DB_N_PLUS_ONE` or 5; 0 turns it off. */
7
7
  nPlusOne?: number;
8
- /** Record a query only when a span is active (a request, job or `observe` span). Default true (`WATCHTOWER_DB_REQUIRE_PARENT=0`
9
- * turns it off): startup DDL, migrations and polling loops run outside any span and would otherwise each become a root trace. */
8
+ /** Record a query only when a span is active (a request, job or `observe` span). Default: true when HTTP server instrumentation is active, else false (`WATCHTOWER_DB_REQUIRE_PARENT=0/1` overrides): startup DDL, migrations and polling loops run outside any span and would otherwise each become a root trace. */
10
9
  requireParent?: boolean;
11
10
  }
12
11
  export interface DbOp {
@@ -22,6 +21,12 @@ export interface DbOp {
22
21
  export declare const inside: (ctx?: Context) => boolean;
23
22
  /** `parent` with `span` active; `nested` marks it as inside an instrumented call (no inner spans). */
24
23
  export declare const enter: (parent: Context, span: Span, nested?: boolean) => Context;
24
+ /** Set by `init()`: whether node:http server instrumentation is on (requests then give queries a parent span). */
25
+ export declare const setHttpServerActive: (v: boolean) => void;
26
+ /** Explicit option, then `WATCHTOWER_DB_REQUIRE_PARENT`, else only when HTTP server instrumentation is active (a queue worker has no request spans, so its queries become root spans). */
27
+ export declare function dbRequireParent(opt: boolean | undefined): boolean;
28
+ /** Datastore spans dropped so far because `requireParent` found no active parent span. */
29
+ export declare const droppedDbSpans: () => number;
25
30
  /** A CLIENT span for `op` (not made active): summary, hash, N+1 hint. Prefer `traced`. */
26
31
  export declare function startDbSpan(op: DbOp, opts: DbOptions | undefined, parent: Context, startTime?: TimeInput): Span;
27
32
  /** A finished CLIENT span for a query a library reports after the fact (ORM loggers): it ends now. */
@@ -2,7 +2,7 @@
2
2
  // statement hash and N+1 hint, connection-pool metrics and embedded-store gauges.
3
3
  import * as fs from 'node:fs';
4
4
  import * as nodePath from 'node:path';
5
- import { context, createContextKey, INVALID_SPAN_CONTEXT, metrics, trace, SpanKind, SpanStatusCode } from '@opentelemetry/api';
5
+ import { context, createContextKey, diag, INVALID_SPAN_CONTEXT, metrics, trace, SpanKind, SpanStatusCode } from '@opentelemetry/api';
6
6
  import { isPgDialect, normalizeAndHash, querySummary } from '../sql.js';
7
7
  /** Set inside an instrumented call so a wrapper the library calls internally adds no second span. */
8
8
  const INSIDE = createContextKey('watchtower.db.inside');
@@ -25,12 +25,37 @@ function repeats(parent, key) {
25
25
  m.set(key, n);
26
26
  return n;
27
27
  }
28
+ let httpServerActive = false;
29
+ /** Set by `init()`: whether node:http server instrumentation is on (requests then give queries a parent span). */
30
+ export const setHttpServerActive = (v) => { httpServerActive = v; };
31
+ /** Explicit option, then `WATCHTOWER_DB_REQUIRE_PARENT`, else only when HTTP server instrumentation is active (a queue worker has no request spans, so its queries become root spans). */
32
+ export function dbRequireParent(opt) {
33
+ if (opt !== undefined)
34
+ return opt;
35
+ const e = (process.env.WATCHTOWER_DB_REQUIRE_PARENT || '').trim().toLowerCase();
36
+ if (['0', 'false', 'off', 'no'].includes(e))
37
+ return false;
38
+ if (['1', 'true', 'on', 'yes'].includes(e))
39
+ return true;
40
+ return httpServerActive;
41
+ }
42
+ let droppedSpans = 0;
43
+ let droppedCounter;
44
+ /** Datastore spans dropped so far because `requireParent` found no active parent span. */
45
+ export const droppedDbSpans = () => droppedSpans;
46
+ /** Count a dropped span: the `wt.sdk.db.dropped_spans` counter (exported with the SDK's metrics) and one diag warning. */
47
+ function dropped() {
48
+ if (droppedSpans++ === 0)
49
+ diag.warn('watchtower: a datastore span was dropped because it has no parent span (requireParent); counted in wt.sdk.db.dropped_spans. Set WATCHTOWER_DB_REQUIRE_PARENT=0 to keep such queries as root spans');
50
+ droppedCounter ??= metrics.getMeter('watchtower').createCounter('wt.sdk.db.dropped_spans', { description: 'Datastore spans dropped for lack of a parent span (requireParent)' });
51
+ droppedCounter.add(1);
52
+ return trace.wrapSpanContext(INVALID_SPAN_CONTEXT);
53
+ }
28
54
  /** A CLIENT span for `op` (not made active): summary, hash, N+1 hint. Prefer `traced`. */
29
55
  export function startDbSpan(op, opts, parent, startTime) {
30
56
  const p = trace.getSpan(parent)?.spanContext();
31
- const requireParent = opts?.requireParent ?? !['0', 'false', 'off', 'no'].includes((process.env.WATCHTOWER_DB_REQUIRE_PARENT || '').trim().toLowerCase());
32
- if (requireParent && !(p && trace.isSpanContextValid(p)))
33
- return trace.wrapSpanContext(INVALID_SPAN_CONTEXT); // non-recording: nothing is exported
57
+ if (dbRequireParent(opts?.requireParent) && !(p && trace.isSpanContextValid(p)))
58
+ return dropped(); // non-recording: nothing is exported
34
59
  const attrs = { 'db.system.name': op.system, ...op.attributes };
35
60
  let { operation, collection } = op;
36
61
  let name = [operation, collection].filter(Boolean).join(' ') || op.system;
@@ -2,5 +2,5 @@
2
2
  export interface FastifyLike {
3
3
  addHook(name: string, fn: (...args: any[]) => void): unknown;
4
4
  }
5
- /** await app.register(plugin): server span per request plus 5xx error capture. */
5
+ /** await app.register(plugin): server span per request (or the http instrumentation's own server span, enriched) plus 5xx error capture. */
6
6
  export declare function plugin(fastify: FastifyLike, _opts: unknown, done: (err?: Error) => void): void;
@@ -1,13 +1,14 @@
1
1
  import { context } from '@opentelemetry/api';
2
2
  import { captureException } from '../errors.js';
3
3
  import { capturePayload, captureActive } from '../payload.js';
4
- import { serverSpan, statusOf } from './util.js';
4
+ import { adoptServerSpan, serverSpan, statusOf } from './util.js';
5
5
  const KEY = Symbol('wt.span');
6
6
  const SENT = Symbol('wt.sent');
7
- /** await app.register(plugin): server span per request plus 5xx error capture. */
7
+ /** await app.register(plugin): server span per request (or the http instrumentation's own server span, enriched) plus 5xx error capture. */
8
8
  export function plugin(fastify, _opts, done) {
9
9
  fastify.addHook('onRequest', (req, _reply, next) => {
10
- const s = serverSpan(req.method, req.url);
10
+ // with the http instrumentation active its server span is enriched (route, status) instead of starting a second one
11
+ const s = adoptServerSpan(req.method) ?? serverSpan(req.method, req.url);
11
12
  req[KEY] = s;
12
13
  context.with(s.ctx, next);
13
14
  });
@@ -4,11 +4,22 @@ import { load } from '../load.js';
4
4
  const defaultSink = { log: (level, body, attrs) => wt.log(level, body, attrs) };
5
5
  const PINO = { 10: 'trace', 20: 'debug', 30: 'info', 40: 'warn', 50: 'error', 60: 'fatal' };
6
6
  const CORE = new Set(['level', 'time', 'msg', 'pid', 'hostname', 'v']);
7
- const flat = (o, skip) => {
8
- const attrs = {};
9
- for (const [k, v] of Object.entries(o))
10
- if (!skip || !skip.has(k))
11
- attrs[k] = typeof v === 'object' ? JSON.stringify(v) : v;
7
+ const MAX_DEPTH = 4;
8
+ const plain = (v) => !!v && typeof v === 'object' && Object.getPrototypeOf(v) === Object.prototype;
9
+ const prim = (v) => v == null || ['string', 'number', 'boolean'].includes(typeof v);
10
+ /** Nested plain objects become dotted attributes (`req.url`) up to depth 4 so `redact` and the presets see each leaf; arrays of primitives stay arrays, anything else is a JSON string. */
11
+ const flat = (o, skip, prefix = '', depth = 1, attrs = {}) => {
12
+ for (const [k, v] of Object.entries(o)) {
13
+ if (skip && skip.has(k))
14
+ continue;
15
+ const key = prefix + k;
16
+ if (plain(v) && depth < MAX_DEPTH)
17
+ flat(v, undefined, `${key}.`, depth + 1, attrs);
18
+ else if (Array.isArray(v) && v.every(prim))
19
+ attrs[key] = v;
20
+ else
21
+ attrs[key] = typeof v === 'object' && v !== null ? JSON.stringify(v) : v;
22
+ }
12
23
  return attrs;
13
24
  };
14
25
  /** pino(opts, pinoStream()) or pino.multistream([{ stream: pinoStream() }, { stream: process.stdout }]). */