@watchtowerobservability/node 0.7.3 → 0.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +40 -1
  2. package/README.md +96 -12
  3. package/dist/cjs/attach.d.ts +7 -0
  4. package/dist/cjs/attach.js +29 -0
  5. package/dist/cjs/checkin.js +4 -13
  6. package/dist/cjs/cron/monitor.d.ts +47 -0
  7. package/dist/cjs/cron/monitor.js +95 -0
  8. package/dist/cjs/cron/schedulers.d.ts +36 -0
  9. package/dist/cjs/cron/schedulers.js +65 -0
  10. package/dist/cjs/env.d.ts +10 -0
  11. package/dist/cjs/env.js +98 -1
  12. package/dist/cjs/index.d.ts +50 -6
  13. package/dist/cjs/index.js +114 -37
  14. package/dist/cjs/instrument-for.d.ts +17 -0
  15. package/dist/cjs/instrument-for.js +44 -0
  16. package/dist/cjs/instrument.d.ts +4 -0
  17. package/dist/cjs/instrument.js +32 -1
  18. package/dist/cjs/integrations/bullmq.d.ts +8 -2
  19. package/dist/cjs/integrations/bullmq.js +32 -5
  20. package/dist/cjs/integrations/db.d.ts +7 -2
  21. package/dist/cjs/integrations/db.js +32 -4
  22. package/dist/cjs/integrations/fastify.d.ts +1 -1
  23. package/dist/cjs/integrations/fastify.js +3 -2
  24. package/dist/cjs/integrations/logs.js +16 -5
  25. package/dist/cjs/integrations/next-build.d.ts +25 -8
  26. package/dist/cjs/integrations/next-build.js +159 -34
  27. package/dist/cjs/integrations/next-tunnel.d.ts +6 -1
  28. package/dist/cjs/integrations/next-tunnel.js +13 -2
  29. package/dist/cjs/integrations/postgres.d.ts +2 -0
  30. package/dist/cjs/integrations/postgres.js +43 -1
  31. package/dist/cjs/integrations/temporal.d.ts +97 -0
  32. package/dist/cjs/integrations/temporal.js +144 -0
  33. package/dist/cjs/integrations/util.d.ts +17 -0
  34. package/dist/cjs/integrations/util.js +38 -0
  35. package/dist/cjs/load.d.ts +6 -0
  36. package/dist/cjs/load.js +28 -4
  37. package/dist/cjs/next-doctor-cli.d.ts +2 -0
  38. package/dist/cjs/next-doctor-cli.js +15 -0
  39. package/dist/cjs/next-doctor.d.ts +2 -0
  40. package/dist/cjs/next-doctor.js +50 -0
  41. package/dist/cjs/pprof.d.ts +2 -0
  42. package/dist/cjs/pprof.js +18 -5
  43. package/dist/cjs/profiling.d.ts +31 -0
  44. package/dist/cjs/profiling.js +67 -11
  45. package/dist/cjs/redact-presets.d.ts +16 -0
  46. package/dist/cjs/redact-presets.js +66 -0
  47. package/dist/cjs/register.js +19 -6
  48. package/dist/cjs/version.d.ts +1 -1
  49. package/dist/cjs/version.js +1 -1
  50. package/dist/esm/attach.d.ts +7 -0
  51. package/dist/esm/attach.js +25 -0
  52. package/dist/esm/checkin.js +4 -13
  53. package/dist/esm/cron/monitor.d.ts +47 -0
  54. package/dist/esm/cron/monitor.js +85 -0
  55. package/dist/esm/cron/schedulers.d.ts +36 -0
  56. package/dist/esm/cron/schedulers.js +61 -0
  57. package/dist/esm/env.d.ts +10 -0
  58. package/dist/esm/env.js +62 -0
  59. package/dist/esm/index.d.ts +50 -6
  60. package/dist/esm/index.js +109 -39
  61. package/dist/esm/instrument-for.d.ts +17 -0
  62. package/dist/esm/instrument-for.js +41 -0
  63. package/dist/esm/instrument.d.ts +4 -0
  64. package/dist/esm/instrument.js +30 -1
  65. package/dist/esm/integrations/bullmq.d.ts +8 -2
  66. package/dist/esm/integrations/bullmq.js +32 -5
  67. package/dist/esm/integrations/db.d.ts +7 -2
  68. package/dist/esm/integrations/db.js +29 -4
  69. package/dist/esm/integrations/fastify.d.ts +1 -1
  70. package/dist/esm/integrations/fastify.js +4 -3
  71. package/dist/esm/integrations/logs.js +16 -5
  72. package/dist/esm/integrations/next-build.d.ts +25 -8
  73. package/dist/esm/integrations/next-build.js +158 -34
  74. package/dist/esm/integrations/next-tunnel.d.ts +6 -1
  75. package/dist/esm/integrations/next-tunnel.js +13 -2
  76. package/dist/esm/integrations/postgres.d.ts +2 -0
  77. package/dist/esm/integrations/postgres.js +43 -2
  78. package/dist/esm/integrations/temporal.d.ts +97 -0
  79. package/dist/esm/integrations/temporal.js +139 -0
  80. package/dist/esm/integrations/util.d.ts +17 -0
  81. package/dist/esm/integrations/util.js +35 -0
  82. package/dist/esm/load.d.ts +6 -0
  83. package/dist/esm/load.js +21 -3
  84. package/dist/esm/next-doctor-cli.d.ts +2 -0
  85. package/dist/esm/next-doctor-cli.js +13 -0
  86. package/dist/esm/next-doctor.d.ts +2 -0
  87. package/dist/esm/next-doctor.js +14 -0
  88. package/dist/esm/pprof.d.ts +2 -0
  89. package/dist/esm/pprof.js +17 -5
  90. package/dist/esm/profiling.d.ts +31 -0
  91. package/dist/esm/profiling.js +66 -12
  92. package/dist/esm/redact-presets.d.ts +16 -0
  93. package/dist/esm/redact-presets.js +58 -0
  94. package/dist/esm/register.js +20 -7
  95. package/dist/esm/version.d.ts +1 -1
  96. package/dist/esm/version.js +1 -1
  97. package/package.json +42 -1
package/dist/esm/env.js CHANGED
@@ -1,3 +1,5 @@
1
+ import * as fs from 'node:fs';
2
+ import * as nodePath from 'node:path';
1
3
  // Standard OTEL_* environment variables, used as fallbacks after the WT_* ones.
2
4
  /** Parse `k=v,k2=v2` (OTEL_RESOURCE_ATTRIBUTES / OTEL_EXPORTER_OTLP_HEADERS); values are percent-decoded. */
3
5
  export function parseKv(s) {
@@ -31,3 +33,63 @@ export function otelEnv(env = process.env) {
31
33
  resource,
32
34
  };
33
35
  }
36
+ const CI_SHA_VARS = ['GITHUB_SHA', 'CI_COMMIT_SHA', 'VERCEL_GIT_COMMIT_SHA', 'SOURCE_VERSION', 'BUILD_SHA', 'RENDER_GIT_COMMIT', 'HEROKU_SLUG_COMMIT'];
37
+ const present = (v) => { const t = (v || '').trim(); return t && t.toLowerCase() !== 'unknown' ? t : undefined; };
38
+ /**
39
+ * The commit `HEAD` points at, found by walking up from `from` to a `.git` directory (a `.git` file of a worktree or submodule is
40
+ * followed through its `gitdir:` line). Reads `HEAD`, then the ref file or `packed-refs`; no child process. undefined when not found.
41
+ */
42
+ export function gitHeadSha(from) {
43
+ const sha = /^[0-9a-f]{40,64}$/i;
44
+ try {
45
+ for (let dir = nodePath.resolve(from);; dir = nodePath.dirname(dir)) {
46
+ let git = nodePath.join(dir, '.git');
47
+ if (fs.existsSync(git)) {
48
+ if (fs.statSync(git).isFile())
49
+ git = nodePath.resolve(dir, /^gitdir:\s*(.+)$/m.exec(fs.readFileSync(git, 'utf8'))?.[1]?.trim() ?? '');
50
+ const head = fs.readFileSync(nodePath.join(git, 'HEAD'), 'utf8').trim();
51
+ if (sha.test(head))
52
+ return head; // detached
53
+ const ref = /^ref:\s*(\S+)/.exec(head)?.[1];
54
+ if (!ref)
55
+ return undefined;
56
+ // a worktree keeps its branch refs in the common dir
57
+ const cd = nodePath.join(git, 'commondir');
58
+ const common = fs.existsSync(cd) ? nodePath.resolve(git, fs.readFileSync(cd, 'utf8').trim()) : git;
59
+ for (const d of [git, common]) {
60
+ try {
61
+ const v = fs.readFileSync(nodePath.join(d, ref), 'utf8').trim();
62
+ if (sha.test(v))
63
+ return v;
64
+ }
65
+ catch { /* not a loose ref */ }
66
+ try {
67
+ const line = fs.readFileSync(nodePath.join(d, 'packed-refs'), 'utf8').split('\n').find((l) => l.endsWith(` ${ref}`));
68
+ const v = line?.split(' ')[0];
69
+ if (v && sha.test(v))
70
+ return v;
71
+ }
72
+ catch { /* no packed-refs */ }
73
+ }
74
+ return undefined;
75
+ }
76
+ if (nodePath.dirname(dir) === dir)
77
+ return undefined;
78
+ }
79
+ }
80
+ catch {
81
+ return undefined;
82
+ }
83
+ }
84
+ /**
85
+ * service.version: explicit option, WT_RELEASE, OTEL_RESOURCE_ATTRIBUTES, the first CI commit-sha variable, the build stamp
86
+ * (`WT_BUILD_STAMP`, `BUILD_STAMP`), then the `.git` HEAD found upward from `gitFrom` (skipped when not given). Empty and `unknown` count as absent.
87
+ */
88
+ export function resolveRelease(opt, env, otelResource, gitFrom) {
89
+ for (const v of [opt, env.WT_RELEASE, otelResource['service.version'], ...CI_SHA_VARS.map((k) => env[k]), env.WT_BUILD_STAMP, env.BUILD_STAMP]) {
90
+ const r = present(v);
91
+ if (r)
92
+ return r;
93
+ }
94
+ return gitFrom ? gitHeadSha(gitFrom) : undefined;
95
+ }
@@ -9,12 +9,20 @@ import type { ReadableSpan, TracerProvider } from '@opentelemetry/sdk-trace';
9
9
  import type { LoggerProvider } from '@opentelemetry/sdk-logs';
10
10
  // @ts-ignore optional peer: absent -> any (keeps skipLibCheck:false consumers compiling)
11
11
  import type { MeterProvider } from '@opentelemetry/sdk-metrics';
12
+ import type { RedactHook } from './redact.js';
13
+ import type { RedactPreset } from './redact-presets.js';
14
+ import type { CheckInOptions, CheckInStatus, MonitorOptions } from './cron/monitor.js';
12
15
  import { captureException } from './errors.js';
13
16
  import type { CaptureContext } from './errors.js';
14
17
  import { RemoteSampler } from './sampling.js';
15
18
  import type { Profiler } from './profiling.js';
16
19
  export { captureException, metrics, trace };
20
+ export { instrument } from './instrument-for.js';
21
+ export type { Framework, InstrumentOptions } from './instrument-for.js';
22
+ export { droppedDbSpans } from './integrations/db.js';
17
23
  export { setCaptureRules, capturePayload } from './payload.js';
24
+ export type { MonitorOptions, MonitorConfig, CheckInOptions, CheckInStatus } from './cron/monitor.js';
25
+ export { instrumentNestSchedule, instrumentNodeCron } from './cron/schedulers.js';
18
26
  export type { CaptureRule } from './payload.js';
19
27
  export type { CaptureContext };
20
28
  export declare const DEFAULT_ENDPOINT = "http://localhost:4318";
@@ -26,7 +34,7 @@ export interface InitOptions {
26
34
  apiKey?: string;
27
35
  /** Defaults to WT_ENVIRONMENT. */
28
36
  environment?: string;
29
- /** Sent as `service.version`; the server links an issue's first release by matching it to an existing release's version. Defaults to WT_RELEASE, then `service.version` in OTEL_RESOURCE_ATTRIBUTES. */
37
+ /** Sent as `service.version`; the server links an issue's first release by matching it to an existing release's version. Defaults to WT_RELEASE, then `service.version` in OTEL_RESOURCE_ATTRIBUTES, then the first set of GITHUB_SHA, CI_COMMIT_SHA, VERCEL_GIT_COMMIT_SHA, SOURCE_VERSION, BUILD_SHA, RENDER_GIT_COMMIT, HEROKU_SLUG_COMMIT, then WT_BUILD_STAMP / BUILD_STAMP, then the commit `.git/HEAD` points at (searched upward from the working directory). */
30
38
  release?: string;
31
39
  /** Hook uncaughtException / unhandledRejection (default true). */
32
40
  captureProcessErrors?: boolean;
@@ -50,11 +58,15 @@ export interface InitOptions {
50
58
  /** Resource attributes (on top of OTEL_RESOURCE_ATTRIBUTES); keys starting with `wt.` are dropped. */
51
59
  resource?: Record<string, string | number | boolean>;
52
60
  /**
53
- * Called before export for every span attribute, every span event's and link's attributes (so `exception.message`
61
+ * Built-in redaction: `'strict'` (default) replaces string values of secret-shaped keys (authorization, cookie, set-cookie,
62
+ * password, secret, token, api_key, session, ...) with `[redacted]` on spans, span events, logs and errors, and URLs lose their
63
+ * query string and token-shaped path segments (`urlScrubbing`); `'pii'` also masks emails, phone numbers, Luhn-valid card numbers
64
+ * and IP addresses in every string; `'off'` disables the presets and URL scrubbing. The gateway applies the same rules again as
65
+ * a backstop. A function is a hook run after the `'strict'` preset. The hook is called before export for every span attribute, every span event's and link's attributes (so `exception.message`
54
66
  * and `exception.stacktrace`), the span name (key `span.name`), the status message (`status.message`) and each log
55
67
  * body (`log.body`) and log attribute; return the value to keep.
56
68
  */
57
- redact?: (key: string, value: unknown) => unknown;
69
+ redact?: RedactHook | RedactPreset;
58
70
  /** Directory for the offline buffer. */
59
71
  bufferDir?: string;
60
72
  /** Apply the project's head sampling rules in the SDK, polled from /v1/sampling; defaults to WT_REMOTE_SAMPLING=1. Spans dropped here never reach the gateway, so its exact request counts see only what is kept. */
@@ -71,9 +83,17 @@ export interface InitOptions {
71
83
  * Scrub URLs on every span, including spans other libraries create (Next.js, node:http, fetch): the query string is dropped
72
84
  * (`url.query` removed) and path segments that look like secrets (UUIDs, hex runs, JWTs, long digit runs, random tokens of >= 16 chars;
73
85
  * hosts, readable words, slugs and short ids stay) become `:redacted`, in `http.target`, `url.full`, `url.path`, `http.url` and URLs in span names.
74
- * `http.route` is kept. Default true; `false` records URLs as the instrumentation produced them.
86
+ * `http.route` is kept. Default true (false when `redact: 'off'`); `false` records URLs as the instrumentation produced them.
75
87
  */
76
88
  urlScrubbing?: boolean;
89
+ /**
90
+ * What to do when another library (Sentry, ...) already registered the global TracerProvider. `'auto'` (default): attach the
91
+ * Watchtower exporter to that provider as a span processor and leave its context manager and propagator alone; `'own'`: build and
92
+ * register our own provider (the old behaviour; the other one is kept as the global and a warning says so); `'attach'`: like
93
+ * `'auto'`, with a warning when there is no provider to attach to. Attached spans carry the other provider's resource
94
+ * (set `OTEL_SERVICE_NAME`), and `remoteSampling` is not applied.
95
+ */
96
+ otel?: 'auto' | 'own' | 'attach';
77
97
  /** Return false to drop a finished span before export (the `/next` entry uses it for `nextInternals`). */
78
98
  spanFilter?: (span: ReadableSpan) => boolean;
79
99
  /**
@@ -116,7 +136,15 @@ export interface ScoreOptions {
116
136
  }
117
137
  /** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
118
138
  export declare function safeResource(attrs?: Record<string, unknown>): Record<string, unknown>;
119
- export declare class Client {
139
+ /** What `init()` reports (also fields of the returned Client): per signal `ok`, `disabled` (switched off) or `blocked` (an OTel global was already registered and kept). */
140
+ export interface InitStatus {
141
+ traces: 'ok' | 'disabled' | 'blocked';
142
+ metrics: 'ok' | 'disabled' | 'blocked';
143
+ logs: 'ok' | 'disabled' | 'blocked';
144
+ blocked: string[];
145
+ warnings: string[];
146
+ }
147
+ export declare class Client implements InitStatus {
120
148
  readonly endpoint: string;
121
149
  readonly apiUrl: string;
122
150
  readonly project: string | undefined;
@@ -124,7 +152,17 @@ export declare class Client {
124
152
  Authorization: string;
125
153
  } & Record<string, string>;
126
154
  readonly fetch: typeof fetch;
127
- readonly redact: InitOptions['redact'];
155
+ readonly redact: RedactHook | undefined;
156
+ private readonly scrubUrls;
157
+ /** What `init` found: `blocked` lists the OTel globals that were already registered and kept. */
158
+ traces: 'ok' | 'disabled' | 'blocked';
159
+ metrics: 'ok' | 'disabled' | 'blocked';
160
+ logs: 'ok' | 'disabled' | 'blocked';
161
+ blocked: string[];
162
+ warnings: string[];
163
+ /** True when the exporter was attached to another provider's pipeline (`otel`). */
164
+ readonly attached: boolean;
165
+ private readonly spanProcessors;
128
166
  private readonly logCaller;
129
167
  readonly provider: TracerProvider;
130
168
  readonly resource: Resource;
@@ -165,6 +203,12 @@ export declare class Client {
165
203
  shutdown(): Promise<void>;
166
204
  }
167
205
  export declare function init(opts: InitOptions): Client;
206
+ /** Why telemetry is off (set by the preload when it cannot start); shown on the first dropped helper call. */
207
+ export declare const setOffReason: (r: string) => void;
208
+ /** One check-in for the monitor `slug`. Fire and forget by default (5 s cap, never throws); `{ wait: true }` awaits it. */
209
+ export declare const checkIn: (slug: string, status: CheckInStatus, o?: CheckInOptions) => Promise<void>;
210
+ /** `withMonitor(slug, fn, { schedule, timezone, checkinMargin, maxRuntime, failureThreshold, wait })`: `fn` as a monitored job; the monitor is created or updated from `schedule`. */
211
+ export declare const withMonitor: <T>(slug: string, fn: () => T | Promise<T>, o?: MonitorOptions) => Promise<T>;
168
212
  export declare const checkin: <T>(slug: string, fn: () => T | Promise<T>) => Promise<T>;
169
213
  export declare const flush: () => Promise<unknown>;
170
214
  export declare const log: (l: LogLevel, b: string, a?: Record<string, unknown>) => void;
package/dist/esm/index.js CHANGED
@@ -2,14 +2,18 @@
2
2
  // offline buffer, wt.*-safe resource attributes, redaction hook, getPrompt/score/evaluate/observe.
3
3
  import { inspect } from 'node:util';
4
4
  import { context, diag, metrics, propagation, trace } from '@opentelemetry/api';
5
- import { need as needPeer } from './load.js';
6
- import { otelEnv } from './env.js';
5
+ import { missingPeers, need as needPeer, STANDALONE_HINT } from './load.js';
6
+ import { otelEnv, resolveRelease } from './env.js';
7
+ import { dbRequireParent, setHttpServerActive } from './integrations/db.js';
7
8
  import { instrumentAll } from './instrument.js';
8
9
  import { LOG_BODY_KEY, redactor } from './redact.js';
9
- import { urlScrubber } from './scrub.js';
10
+ import { resolveRedact } from './redact-presets.js';
11
+ import { foreignProvider, attachProcessors } from './attach.js';
12
+ import { scrubUrl, urlScrubber } from './scrub.js';
10
13
  import { filterExporter } from './filter.js';
11
14
  import { nextInternalsFilter } from './next-filter.js';
12
15
  import { checkin as runCheckin } from './checkin.js';
16
+ import { checkIn as checkInFor, withMonitor as withMonitorFor } from './cron/monitor.js';
13
17
  import { installSignalFlush } from './shutdown.js';
14
18
  import { VERSION } from './version.js';
15
19
  import { SpoolExporter, spoolLogs } from './buffer.js';
@@ -21,7 +25,10 @@ import { startProfiling } from './profiling.js';
21
25
  import { RemoteSampler } from './sampling.js';
22
26
  import { codeOrigin } from './origin.js';
23
27
  export { captureException, metrics, trace };
28
+ export { instrument } from './instrument-for.js';
29
+ export { droppedDbSpans } from './integrations/db.js';
24
30
  export { setCaptureRules, capturePayload } from './payload.js';
31
+ export { instrumentNestSchedule, instrumentNodeCron } from './cron/schedulers.js';
25
32
  export const DEFAULT_ENDPOINT = 'http://localhost:4318';
26
33
  /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
27
34
  export function callerOf(stack) {
@@ -41,6 +48,16 @@ export class Client {
41
48
  headers;
42
49
  fetch;
43
50
  redact;
51
+ scrubUrls;
52
+ /** What `init` found: `blocked` lists the OTel globals that were already registered and kept. */
53
+ traces = 'ok';
54
+ metrics = 'ok';
55
+ logs = 'ok';
56
+ blocked = [];
57
+ warnings = [];
58
+ /** True when the exporter was attached to another provider's pipeline (`otel`). */
59
+ attached = false;
60
+ spanProcessors = [];
44
61
  logCaller;
45
62
  provider;
46
63
  resource;
@@ -68,7 +85,7 @@ export class Client {
68
85
  this.headers = { ...oe.headers, Authorization: `Bearer ${apiKey}` };
69
86
  this.fetch = o.fetch || globalThis.fetch;
70
87
  const environment = o.environment || env.WT_ENVIRONMENT || oe.resource['deployment.environment.name'];
71
- const release = o.release || env.WT_RELEASE || oe.resource['service.version'];
88
+ const release = resolveRelease(o.release, env, oe.resource, process.cwd());
72
89
  // the OTel SDK packages are optional peers, loaded here (not at import time) with a clear error when missing
73
90
  const { resourceFromAttributes, defaultResource } = needPeer('@opentelemetry/resources');
74
91
  // the SDK's own telemetry.sdk.* (name, language, version): a bare resourceFromAttributes does not add them
@@ -86,7 +103,9 @@ export class Client {
86
103
  ...safeResource(o.resource),
87
104
  });
88
105
  this.resource = resource;
89
- this.redact = o.redact;
106
+ const red = resolveRedact(o.redact);
107
+ this.redact = red.hook;
108
+ this.scrubUrls = o.urlScrubbing ?? red.preset !== 'off';
90
109
  this.logCaller = o.logCaller ?? env.WATCHTOWER_LOG_CALLER === '1';
91
110
  let exporter = o.exporter;
92
111
  if (!exporter) {
@@ -110,19 +129,30 @@ export class Client {
110
129
  });
111
130
  sampler = new ParentBasedSampler({ root: this.remoteSampler });
112
131
  }
113
- this.provider = new TP({
114
- resource,
115
- ...(sampler ? { sampler: sampler } : {}),
116
- spanProcessors: [
117
- ...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [codeOrigin()] : []),
118
- ...(o.urlScrubbing ?? true ? [urlScrubber()] : []),
119
- ...(o.redact ? [redactor(o.redact)] : []),
120
- new BatchSpanProcessor({ exporter }),
121
- ],
122
- });
132
+ const procs = [
133
+ ...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [codeOrigin()] : []),
134
+ ...(o.urlScrubbing ?? red.preset !== 'off' ? [urlScrubber()] : []),
135
+ ...(red.hook ? [redactor(red.hook)] : []),
136
+ new BatchSpanProcessor({ exporter }),
137
+ ];
138
+ const mode = o.otel ?? 'auto';
139
+ const foreign = mode !== 'own' && !state.ownsGlobals ? foreignProvider() : undefined;
140
+ if (foreign && attachProcessors(foreign, procs)) {
141
+ this.attached = true;
142
+ this.spanProcessors = procs;
143
+ this.provider = foreign;
144
+ if (sampler)
145
+ this.warnings.push('remoteSampling is not applied when attached to another tracer provider');
146
+ }
147
+ else {
148
+ if (foreign || mode === 'attach')
149
+ this.warnings.push(foreign ? 'otel: the existing tracer provider has no span-processor hook; using our own' : "otel: 'attach' but no tracer provider is registered yet; using our own");
150
+ this.provider = new TP({ resource, ...(sampler ? { sampler: sampler } : {}), spanProcessors: procs });
151
+ }
123
152
  this.tracer = this.provider.getTracer('watchtower');
124
153
  // an explicit exporter never switches the other signals off: only `logs` / `metrics` / the env switches do
125
154
  const logsOn = switchOn('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs ?? (o.logExporter ? true : undefined));
155
+ this.logs = logsOn ? 'ok' : 'disabled';
126
156
  let logExporter = logsOn ? o.logExporter : undefined;
127
157
  if (logsOn && !logExporter) {
128
158
  const url = `${this.endpoint}/v1/logs`;
@@ -142,6 +172,9 @@ export class Client {
142
172
  this.meterProvider = m.provider;
143
173
  this.stopRuntime = m.stop;
144
174
  }
175
+ else {
176
+ this.metrics = 'disabled';
177
+ }
145
178
  if (o.captureRules ?? env.WT_CAPTURE_RULES === '1') {
146
179
  void loadCaptureRules({ apiUrl: this.apiUrl, key: apiKey, service: String(resource.attributes['service.name']), fetch: this.fetch });
147
180
  }
@@ -151,9 +184,18 @@ export class Client {
151
184
  let attrs = dropReserved(attributes);
152
185
  const redact = this.redact;
153
186
  if (redact) {
154
- attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => [k, redact(k, v)]));
187
+ // dotted keys (flattened nested objects) also go through the hook under their leaf name, so a hook keyed on `url` sees `req.url`
188
+ attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => {
189
+ v = redact(k, v);
190
+ const leaf = k.slice(k.lastIndexOf('.') + 1);
191
+ return [k, leaf !== k ? redact(leaf, v) : v];
192
+ }));
155
193
  body = String(redact(LOG_BODY_KEY, body) ?? '');
156
194
  }
195
+ if (this.scrubUrls)
196
+ for (const k of Object.keys(attrs))
197
+ if (typeof attrs[k] === 'string' && /(^|\.)(url|uri|target)$/.test(k))
198
+ attrs[k] = scrubUrl(attrs[k]);
157
199
  if (this.logCaller)
158
200
  attrs = { ...attrs, ...callerOf(new Error().stack) };
159
201
  this.logger.emit({
@@ -168,7 +210,8 @@ export class Client {
168
210
  return runCheckin(this, monitorSlug, fn);
169
211
  }
170
212
  forceFlush() {
171
- return Promise.all([this.provider.forceFlush(), this.loggerProvider.forceFlush(), this.meterProvider?.forceFlush()]);
213
+ const spans = this.attached ? Promise.all(this.spanProcessors.map((p) => p.forceFlush())) : this.provider.forceFlush();
214
+ return Promise.all([spans, this.loggerProvider.forceFlush(), this.meterProvider?.forceFlush()]);
172
215
  }
173
216
  // keep the key out of console.log / JSON.stringify
174
217
  toJSON() {
@@ -268,7 +311,7 @@ export class Client {
268
311
  shutdown() {
269
312
  this.stopRuntime();
270
313
  this.remoteSampler?.stop();
271
- return Promise.all([this.profiler?.then((p) => p?.stop()), this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
314
+ return Promise.all([this.profiler?.then((p) => p?.stop()), this.attached ? Promise.all(this.spanProcessors.map((p) => p.shutdown())) : this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
272
315
  }
273
316
  }
274
317
  // One state per process, not per module copy: a CommonJS and an ESM copy of this package (a `-r` preload plus an `import`
@@ -280,25 +323,42 @@ const state = (globalThis[Symbol.for('watchtower.node.state')] ??= {});
280
323
  * left alone and a warning says so; only globals a previous `init` installed are torn down on a re-init.
281
324
  */
282
325
  function registerGlobals(c) {
326
+ const blocked = [];
283
327
  if (state.ownsGlobals) { // a re-init replaces what the previous init installed
284
328
  trace.disable();
285
329
  context.disable();
286
330
  propagation.disable();
287
331
  metrics.disable();
288
332
  }
289
- const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
290
- const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
291
- const ok = [
292
- trace.setGlobalTracerProvider(c.provider),
293
- context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()),
294
- propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })),
295
- ...(c.meterProvider ? [metrics.setGlobalMeterProvider(c.meterProvider)] : []),
296
- ];
297
- state.ownsGlobals = ok[0] === true;
298
- if (ok.includes(false))
299
- diag.warn('watchtower: an OpenTelemetry global (tracer provider, context manager, propagator or meter provider) was already registered and was kept; call init() before other OTel setup to own them');
333
+ if (!c.attached) { // attached: the other setup owns the provider, context manager and propagator
334
+ const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
335
+ const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
336
+ const ownTracer = trace.setGlobalTracerProvider(c.provider);
337
+ state.ownsGlobals = ownTracer === true;
338
+ if (!ownTracer)
339
+ blocked.push('tracer-provider');
340
+ if (!context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()))
341
+ blocked.push('context-manager');
342
+ if (!propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })))
343
+ blocked.push('propagator');
344
+ }
345
+ if (c.meterProvider && !metrics.setGlobalMeterProvider(c.meterProvider))
346
+ blocked.push('meter-provider');
347
+ c.blocked = blocked;
348
+ if (blocked.includes('tracer-provider'))
349
+ c.traces = 'blocked';
350
+ if (blocked.includes('meter-provider'))
351
+ c.metrics = 'blocked';
352
+ if (blocked.length) {
353
+ const w = `watchtower: OpenTelemetry global(s) already registered and kept: ${blocked.join(', ')}; call init() before other OTel setup to own them, or leave \`otel\` on 'auto' to attach to the existing provider`;
354
+ c.warnings.push(w);
355
+ diag.warn(w);
356
+ }
300
357
  }
301
358
  export function init(opts) {
359
+ const missing = missingPeers();
360
+ if (missing.length)
361
+ console.warn(`[watchtower] missing OpenTelemetry package(s): ${missing.join(', ')}. Next.js standalone: ${STANDALONE_HINT}. Otherwise: npm install ${missing.join(' ')}`);
302
362
  const nextMode = opts?.nextInternals ?? (process.env.NEXT_RUNTIME ? 'sample' : undefined);
303
363
  if (nextMode) {
304
364
  const own = nextInternalsFilter(nextMode, opts?.nextInternalsRate);
@@ -315,11 +375,12 @@ export function init(opts) {
315
375
  const mode = opts?.instrument ?? (env === 'auto' ? 'auto' : !['0', 'off', 'false', 'no'].includes(env));
316
376
  if (mode) {
317
377
  const names = instrumentAll({ tracerProvider: c.provider, meterProvider: c.meterProvider }, [c.endpoint, c.apiUrl], mode === 'auto');
378
+ setHttpServerActive(names.includes('@opentelemetry/instrumentation-http'));
318
379
  const logOn = opts?.instrumentLog ?? !['0', 'off', 'false', 'no'].includes((process.env.WT_INSTRUMENT_LOG || '').trim().toLowerCase());
319
380
  if (mode === 'auto' && logOn) {
320
381
  console.info(`[watchtower] instrumented: ${names.map((n) => n.replace('@opentelemetry/instrumentation-', '')).join(', ') || 'nothing (no supported package found)'}`);
321
382
  }
322
- if (logOn && !['0', 'false', 'off', 'no'].includes((process.env.WATCHTOWER_DB_REQUIRE_PARENT || '').trim().toLowerCase())) {
383
+ if (logOn && dbRequireParent(undefined)) {
323
384
  console.info('watchtower: requireParent on; queries outside a span are not recorded (WATCHTOWER_DB_REQUIRE_PARENT=0 to keep them)');
324
385
  }
325
386
  }
@@ -339,17 +400,26 @@ export function init(opts) {
339
400
  }
340
401
  return c;
341
402
  }
403
+ /** Every public helper is a no-op before `init()` (like `captureException`), so shared code can call them in any order. */
342
404
  function need() {
343
- if (!state.current)
344
- throw new Error('watchtower.init() has not been called');
405
+ if (!state.current && !state.offWarned && process.env.NODE_ENV !== 'test') {
406
+ state.offWarned = true; // once per process: a dropped call must not go unnoticed, nor flood the log
407
+ console.warn(`[watchtower] telemetry is OFF: ${state.offReason ?? 'init() was not called'}. Calls to log, observe, checkin and the other helpers are dropped.`);
408
+ }
345
409
  return state.current;
346
410
  }
347
- export const checkin = (slug, fn) => need().checkin(slug, fn);
411
+ /** Why telemetry is off (set by the preload when it cannot start); shown on the first dropped helper call. */
412
+ export const setOffReason = (r) => { state.offReason = r; };
413
+ /** One check-in for the monitor `slug`. Fire and forget by default (5 s cap, never throws); `{ wait: true }` awaits it. */
414
+ export const checkIn = (slug, status, o) => { const c = need(); return c ? checkInFor(c, slug, status, o) : Promise.resolve(); };
415
+ /** `withMonitor(slug, fn, { schedule, timezone, checkinMargin, maxRuntime, failureThreshold, wait })`: `fn` as a monitored job; the monitor is created or updated from `schedule`. */
416
+ export const withMonitor = (slug, fn, o) => { const c = need(); return c ? withMonitorFor(c, slug, fn, o) : Promise.resolve().then(fn); };
417
+ export const checkin = (slug, fn) => need()?.checkin(slug, fn) ?? Promise.resolve().then(fn);
348
418
  export const flush = () => (state.current ? state.current.forceFlush() : Promise.resolve());
349
- export const log = (l, b, a) => need().log(l, b, a);
350
- export const getPrompt = (id) => need().getPrompt(id);
351
- export const score = (t, v, o) => need().score(t, v, o);
352
- export const evaluate = (d, t, s, o) => need().evaluate(d, t, s, o);
353
- export const evaluateResume = (d, t, s, prev, o) => need().evaluateResume(d, t, s, prev, o);
354
- export const observe = (n, f) => need().observe(n, f);
419
+ export const log = (l, b, a) => need()?.log(l, b, a);
420
+ export const getPrompt = (id) => need()?.getPrompt(id) ?? Promise.resolve(undefined);
421
+ export const score = (t, v, o) => need()?.score(t, v, o) ?? Promise.resolve(undefined);
422
+ export const evaluate = (d, t, s, o) => need()?.evaluate(d, t, s, o) ?? Promise.resolve([]);
423
+ export const evaluateResume = (d, t, s, prev, o) => need()?.evaluateResume(d, t, s, prev, o) ?? Promise.resolve([]);
424
+ export const observe = (n, f) => need()?.observe(n, f) ?? f;
355
425
  export const shutdown = () => (state.current ? state.current.shutdown() : Promise.resolve());
@@ -0,0 +1,17 @@
1
+ export type Framework = 'nest' | 'express' | 'next' | 'bullmq' | 'pg' | 'postgres' | 'ioredis';
2
+ export interface InstrumentOptions {
3
+ /** URLs never traced as outgoing calls (our own endpoint); `init` passes its own. */
4
+ selfUrls?: string[];
5
+ }
6
+ /**
7
+ * `instrument('nest' | 'express' | 'next' | 'bullmq' | 'pg' | 'postgres' | 'ioredis')`. Call it after `init()`.
8
+ *
9
+ * - `nest`, `express`: node:http server instrumentation; server spans are renamed `METHOD /route/:param` from the matched route (Nest on Express).
10
+ * - `next`: http and undici (fetch); Next's own span filtering is `init`'s `nextInternals` (applied when `NEXT_RUNTIME` is set).
11
+ * - `bullmq`: producer and worker spans with trace context in the job (`instrumentBullMQ`); order does not matter, it patches the classes.
12
+ * - `postgres`: postgres.js through its diagnostics channel; order does not matter.
13
+ * - `pg`, `ioredis`: the OTel instrumentation, which hooks `require`: call before the first `require('pg')` / `import 'pg'` (best: the `/register` preload).
14
+ *
15
+ * Returns the packages or integrations enabled by this call (empty when already on, or its peer is not installed).
16
+ */
17
+ export declare function instrument(name: Framework, opts?: InstrumentOptions): string[];
@@ -0,0 +1,41 @@
1
+ // One call per framework: `instrument('nest')`. Applies the right instrumentation and returns what it enabled.
2
+ import { load } from './load.js';
3
+ import { instrumentOne } from './instrument.js';
4
+ import { setHttpServerActive } from './integrations/db.js';
5
+ import { instrumentPostgres } from './integrations/postgres.js';
6
+ import { instrumentBullMQ } from './integrations/bullmq.js';
7
+ const HTTP = '@opentelemetry/instrumentation-http';
8
+ const UNDICI = '@opentelemetry/instrumentation-undici';
9
+ /**
10
+ * `instrument('nest' | 'express' | 'next' | 'bullmq' | 'pg' | 'postgres' | 'ioredis')`. Call it after `init()`.
11
+ *
12
+ * - `nest`, `express`: node:http server instrumentation; server spans are renamed `METHOD /route/:param` from the matched route (Nest on Express).
13
+ * - `next`: http and undici (fetch); Next's own span filtering is `init`'s `nextInternals` (applied when `NEXT_RUNTIME` is set).
14
+ * - `bullmq`: producer and worker spans with trace context in the job (`instrumentBullMQ`); order does not matter, it patches the classes.
15
+ * - `postgres`: postgres.js through its diagnostics channel; order does not matter.
16
+ * - `pg`, `ioredis`: the OTel instrumentation, which hooks `require`: call before the first `require('pg')` / `import 'pg'` (best: the `/register` preload).
17
+ *
18
+ * Returns the packages or integrations enabled by this call (empty when already on, or its peer is not installed).
19
+ */
20
+ export function instrument(name, opts = {}) {
21
+ const self = opts.selfUrls ?? [];
22
+ const on = (pkgs) => {
23
+ const done = pkgs.filter((p) => instrumentOne(p, self));
24
+ if (pkgs.includes(HTTP) && done.includes(HTTP))
25
+ setHttpServerActive(true);
26
+ return done;
27
+ };
28
+ switch (name) {
29
+ case 'nest':
30
+ case 'express': return on([HTTP]);
31
+ case 'next': return on([HTTP, UNDICI]);
32
+ case 'pg': return on(['@opentelemetry/instrumentation-pg']);
33
+ case 'ioredis': return on(['@opentelemetry/instrumentation-ioredis']);
34
+ case 'postgres':
35
+ load('postgres');
36
+ instrumentPostgres();
37
+ return ['postgres.js'];
38
+ case 'bullmq': return instrumentBullMQ(load('bullmq')) ? ['bullmq'] : [];
39
+ default: throw new Error(`watchtower: instrument(${JSON.stringify(name)}): expected nest, express, next, bullmq, pg, postgres or ioredis`);
40
+ }
41
+ }
@@ -1,7 +1,11 @@
1
+ /** Instrumentation packages already registered in this process (by `init` or `instrument()`), so none is patched twice. */
2
+ export declare const registered: Set<string>;
1
3
  interface Providers {
2
4
  tracerProvider: unknown;
3
5
  meterProvider?: unknown;
4
6
  }
7
+ /** Register one entry by package name (for `instrument(name)`); false when already registered or not installed. Uses the global providers. */
8
+ export declare function instrumentOne(pkg: string, selfUrls?: string[]): boolean;
5
9
  /**
6
10
  * Register whichever of the OTel instrumentations is installed (http, undici, pg, ioredis; with `auto` also mysql2) and,
7
11
  * with `auto`, patch postgres.js and better-sqlite3 when installed. Returns the names that were enabled.
@@ -2,11 +2,15 @@
2
2
  import { load } from './load.js';
3
3
  import { instrumentBetterSqlite3 } from './integrations/sqlite.js';
4
4
  import { instrumentPostgres } from './integrations/postgres.js';
5
+ import { routeNamer } from './integrations/util.js';
6
+ /** Instrumentation packages already registered in this process (by `init` or `instrument()`), so none is patched twice. */
7
+ export const registered = new Set();
5
8
  const ENTRIES = [
6
9
  {
7
10
  pkg: '@opentelemetry/instrumentation-http', ctor: 'HttpInstrumentation',
8
11
  // our own exports and API calls must not become spans
9
- opts: (self) => ({ ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
12
+ // routeNamer: server spans become `GET /users/:id` from the matched Express / Nest route (http alone names them just `GET`)
13
+ opts: (self) => ({ applyCustomAttributesOnSpan: routeNamer, ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
10
14
  },
11
15
  {
12
16
  pkg: '@opentelemetry/instrumentation-undici', ctor: 'UndiciInstrumentation',
@@ -16,6 +20,30 @@ const ENTRIES = [
16
20
  { pkg: '@opentelemetry/instrumentation-ioredis', ctor: 'IORedisInstrumentation', opts: () => ({}) },
17
21
  { pkg: '@opentelemetry/instrumentation-mysql2', ctor: 'MySQL2Instrumentation', opts: () => ({}), auto: true },
18
22
  ];
23
+ /** Register one entry by package name (for `instrument(name)`); false when already registered or not installed. Uses the global providers. */
24
+ export function instrumentOne(pkg, selfUrls = []) {
25
+ const e = ENTRIES.find((x) => x.pkg === pkg);
26
+ if (!e || registered.has(pkg))
27
+ return false;
28
+ const self = selfUrls.flatMap((u) => { try {
29
+ return [new URL(u)];
30
+ }
31
+ catch {
32
+ return [];
33
+ } });
34
+ try {
35
+ const { registerInstrumentations } = load('@opentelemetry/instrumentation');
36
+ const Ctor = load(pkg)[e.ctor];
37
+ if (!Ctor)
38
+ return false;
39
+ registerInstrumentations({ instrumentations: [new Ctor(e.opts(self))] });
40
+ registered.add(pkg);
41
+ return true;
42
+ }
43
+ catch {
44
+ return false;
45
+ } // peer not installed
46
+ }
19
47
  /** Drivers patched in place (no OTel package): `instrument: 'auto'` only. Each loads the app's installed copy or is skipped. */
20
48
  const DRIVERS = [
21
49
  { name: 'postgres.js', run: () => { load('postgres'); instrumentPostgres(); } }, // only when `postgres` is installed
@@ -49,6 +77,7 @@ export function instrumentAll(p, selfUrls, auto = false) {
49
77
  if (Ctor) {
50
78
  found.push(new Ctor(e.opts(self)));
51
79
  names.push(e.pkg);
80
+ registered.add(e.pkg);
52
81
  }
53
82
  }
54
83
  catch {
@@ -8,12 +8,18 @@ interface BullMQLike {
8
8
  prototype: Record<string | symbol, unknown>;
9
9
  };
10
10
  }
11
+ export interface BullMQOptions {
12
+ /** Report a job's failure as an error when it is the last attempt (no retry left): queue, job name and id, attempt, attemptsMade and attempts as attributes. */
13
+ captureFailures?: boolean;
14
+ /** Run repeatable jobs (`repeat.pattern`/`cron`) as cron monitors named `<queue>-<job name>`; needs the `ingest:checkins` key scope. */
15
+ cronMonitors?: boolean;
16
+ }
11
17
  /**
12
18
  * `import * as bullmq from 'bullmq'; instrumentBullMQ(bullmq)` (ESM or CJS): PRODUCER spans on
13
19
  * `Queue.add`/`addBulk`, CONSUMER spans around each job a `Worker` processes, parented by the
14
20
  * producer's context carried in job data under `_wt` (stripped before your processor runs; object
15
21
  * data only). Call it once, before creating queues and workers. Returns false when this bullmq
16
- * has no `Worker.callProcessJob` (consumer spans off; producer spans still on).
22
+ * has no `Worker.callProcessJob` (consumer spans off; producer spans still on). Options: see `BullMQOptions`.
17
23
  * shortcut: sandboxed (file path) processors run in a child process and get no consumer span.
18
24
  */
19
- export declare function instrumentBullMQ(bullmq: BullMQLike): boolean;
25
+ export declare function instrumentBullMQ(bullmq: BullMQLike, options?: BullMQOptions): boolean;