@watchtowerobservability/node 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +39 -8
  2. package/dist/cjs/env.d.ts +12 -0
  3. package/dist/cjs/env.js +37 -0
  4. package/dist/cjs/errors.d.ts +8 -0
  5. package/dist/cjs/errors.js +23 -0
  6. package/dist/cjs/index.d.ts +29 -9
  7. package/dist/cjs/index.js +87 -49
  8. package/dist/cjs/instrument.d.ts +7 -0
  9. package/dist/cjs/instrument.js +51 -0
  10. package/dist/cjs/integrations/genai-agents.d.ts +4 -1
  11. package/dist/cjs/integrations/genai-agents.js +27 -24
  12. package/dist/cjs/integrations/genai-frameworks.d.ts +25 -0
  13. package/dist/cjs/integrations/genai-frameworks.js +211 -0
  14. package/dist/cjs/integrations/genai.d.ts +1 -0
  15. package/dist/cjs/integrations/genai.js +7 -2
  16. package/dist/cjs/integrations/next.d.ts +4 -1
  17. package/dist/cjs/integrations/next.js +13 -7
  18. package/dist/cjs/load.js +36 -5
  19. package/dist/cjs/origin.d.ts +9 -0
  20. package/dist/cjs/origin.js +43 -0
  21. package/dist/cjs/redact.d.ts +8 -0
  22. package/dist/cjs/redact.js +43 -0
  23. package/dist/cjs/register.d.ts +1 -0
  24. package/dist/cjs/register.js +29 -0
  25. package/dist/cjs/sampling.d.ts +53 -0
  26. package/dist/cjs/sampling.js +98 -0
  27. package/dist/esm/env.d.ts +12 -0
  28. package/dist/esm/env.js +33 -0
  29. package/dist/esm/errors.d.ts +8 -0
  30. package/dist/esm/errors.js +22 -0
  31. package/dist/esm/index.d.ts +29 -9
  32. package/dist/esm/index.js +87 -50
  33. package/dist/esm/instrument.d.ts +7 -0
  34. package/dist/esm/instrument.js +48 -0
  35. package/dist/esm/integrations/genai-agents.d.ts +4 -1
  36. package/dist/esm/integrations/genai-agents.js +4 -4
  37. package/dist/esm/integrations/genai-frameworks.d.ts +25 -0
  38. package/dist/esm/integrations/genai-frameworks.js +206 -0
  39. package/dist/esm/integrations/genai.d.ts +1 -0
  40. package/dist/esm/integrations/genai.js +3 -1
  41. package/dist/esm/integrations/next.d.ts +4 -1
  42. package/dist/esm/integrations/next.js +11 -6
  43. package/dist/esm/load.js +78 -2
  44. package/dist/esm/origin.d.ts +9 -0
  45. package/dist/esm/origin.js +39 -0
  46. package/dist/esm/redact.d.ts +8 -0
  47. package/dist/esm/redact.js +39 -0
  48. package/dist/esm/register.d.ts +1 -0
  49. package/dist/esm/register.js +27 -0
  50. package/dist/esm/sampling.d.ts +53 -0
  51. package/dist/esm/sampling.js +91 -0
  52. package/package.json +33 -2
package/README.md CHANGED
@@ -4,7 +4,7 @@
4
4
 
5
5
  ## Install
6
6
 
7
- GitHub Packages needs a token with `read:packages` even for public packages:
7
+ Published on npm:
8
8
 
9
9
  ```sh
10
10
  npm install @watchtowerobservability/node @opentelemetry/context-async-hooks @opentelemetry/core \
@@ -14,8 +14,8 @@ npm install @watchtowerobservability/node @opentelemetry/context-async-hooks @op
14
14
 
15
15
  `@opentelemetry/api` is the only hard dependency; the OpenTelemetry SDK packages above are optional peers (so your app controls their versions) and are loaded by `init`, which throws an error naming the missing ones.
16
16
 
17
- Or install the attested tarball from a release without a token:
18
- `npm install https://github.com/watchtowerobservability/watchtower-node/releases/download/v0.2.0/watchtowerobservability-node-0.2.0.tgz`.
17
+ Optional auto-instrumentation peers (installed ones are used, see below):
18
+ `npm install @opentelemetry/instrumentation @opentelemetry/instrumentation-http @opentelemetry/instrumentation-undici @opentelemetry/instrumentation-pg @opentelemetry/instrumentation-ioredis`.
19
19
 
20
20
  Written in TypeScript and shipped as both ESM and CommonJS (`import` and `require` both work); the `.d.ts`
21
21
  typings are generated from the source. A thin layer over the OpenTelemetry Node SDK (ruling L1): `Authorization: Bearer` header, endpoint default (`WT_OTLP_ENDPOINT`), disk offline buffer,
@@ -39,6 +39,32 @@ wt.log('info', 'started', { port: 3000 }); // OTLP log, linked to the active
39
39
  flushes and exits with code 1 like Node does, only when no other handler is registered.
40
40
  Request bodies, headers, query strings and query arguments are never recorded.
41
41
 
42
+ ## Auto-instrumentation and preload
43
+
44
+ `init` patches `node:http`, `fetch` (undici), `pg` and `ioredis` for each `@opentelemetry/instrumentation-*` package you have installed (the list above); pass `instrument: false` or set `WT_INSTRUMENT=0` to skip. Patching only reaches modules loaded after it runs, so load the SDK first:
45
+
46
+ ```sh
47
+ node --require @watchtowerobservability/node/register app.js # CommonJS app (also: node -r ...)
48
+ node --import @watchtowerobservability/node/register app.mjs # ESM app
49
+ ```
50
+
51
+ `register` calls `init({})` when `WT_INGEST_KEY` is set. For an ESM app it also registers OpenTelemetry's ESM loader hook (`@opentelemetry/instrumentation/hook.mjs`, resolved from your project, so install `@opentelemetry/instrumentation`) so `node:http` and other ES-module imports get patched; no extra flag is needed. The CommonJS and ESM copies of this package share one client (kept on `globalThis`), so `flush()` and `shutdown()` work from either. Start a CommonJS app with `--require` and an ES-module app with `--import`.
52
+
53
+ `init` registers the global tracer provider, context manager, propagator and meter provider only when none is registered yet (OpenTelemetry allows each once): call `init` before other OTel setup. It no longer disables globals set earlier; a warning (via the OTel `diag` logger) says when one was kept. A second `init` replaces only what the first one installed.
54
+
55
+ ## Next.js
56
+
57
+ ```ts
58
+ // instrumentation.ts
59
+ import { makeRegister } from '@watchtowerobservability/node/next';
60
+ export const register = makeRegister({ redact: (k, v) => v }); // any InitOptions; plain `register` takes env only
61
+ export { onRequestError } from '@watchtowerobservability/node/next';
62
+ ```
63
+
64
+ Install the OpenTelemetry peers above in the app itself (with pnpm, as direct dependencies, so Next's standalone tracer copies them). The SDK requires each peer by its literal name, so webpack and Next's file tracer can follow them; if the SDK arrives through a transpiled workspace package, add it to `transpilePackages`. Add `outputFileTracingIncludes` for the peers only if the standalone build still reports a missing `@opentelemetry/*` package.
65
+
66
+ Bundling with esbuild or webpack (ESM or CommonJS output) works: both the CommonJS and ESM builds name every optional peer with a literal `require`/`import()` inside try, so the bundler follows the peers you have installed and skips the rest.
67
+
42
68
  ## Integrations
43
69
 
44
70
  Each lives at its own subpath, so nothing is loaded that you do not import.
@@ -86,7 +112,10 @@ Each lives at its own subpath, so nothing is loaded that you do not import.
86
112
  | Mistral, Ollama | `.../genai` | `instrumentMistral(new Mistral())`, `instrumentOllama(ollama)` |
87
113
  | LlamaIndex.TS | `.../genai` | `instrumentLlamaIndex(Settings)` |
88
114
  | OpenAI Agents SDK | `.../genai` | `instrumentOpenAIAgents(addTraceProcessor)` |
89
- | Vercel AI SDK | `.../genai` | `generateText({ experimental_telemetry: vercelAiTelemetry() })` |
115
+ | LangChain.js / LangGraph.js | `.../genai` | `graph.invoke(x, { callbacks: [langchainCallbackHandler()] })` |
116
+ | Vercel AI SDK agent loop | `.../genai` | `const { generateText } = instrumentVercelAI({ generateText })` |
117
+ | Mastra | `.../genai` | `instrumentMastra(agent)` |
118
+ | Vercel AI SDK (telemetry only) | `.../genai` | `generateText({ experimental_telemetry: vercelAiTelemetry() })` |
90
119
 
91
120
  GenAI spans follow the OTel GenAI semantic conventions (model, token usage, finish reasons); prompt and
92
121
  completion text is not recorded. Streaming calls end their span at creation, without token usage.
@@ -269,14 +298,16 @@ See `docs/product/security/app-api-protection.md`.
269
298
  | Option | Env var | Default |
270
299
  | --- | --- | --- |
271
300
  | `apiKey` | `WT_INGEST_KEY` | required, sent as `Authorization: Bearer`, masked in output |
272
- | `endpoint` | `WT_OTLP_ENDPOINT` | `http://localhost:4318` |
273
- | `serviceName` | `WT_SERVICE` | `app` |
301
+ | `endpoint` | `WT_OTLP_ENDPOINT`, then `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4318` |
302
+ | `serviceName` | `WT_SERVICE`, then `OTEL_SERVICE_NAME` | `app` |
274
303
  | `environment` | `WT_ENVIRONMENT` | unset |
275
- | `release` | `WT_RELEASE` | unset |
304
+ | `release` | `WT_RELEASE`, then `service.version` in `OTEL_RESOURCE_ATTRIBUTES` | unset; sent as `service.version`, and an issue's first release is linked when a release with that version exists in the project |
305
+ | `resource` | `OTEL_RESOURCE_ATTRIBUTES` | none; `OTEL_EXPORTER_OTLP_HEADERS` adds extra headers, and its `Authorization=Bearer <key>` is used when `WT_INGEST_KEY` is unset |
306
+ | `instrument` | `WT_INSTRUMENT` | `on` for installed http / undici / pg / ioredis instrumentations |
276
307
  | `bufferDir` | | no offline buffer (spans, logs and metrics spool to `traces/`, `logs/` and `metrics/`, 500 batches each) |
277
308
  | `logs` | `WATCHTOWER_LOGS` (`OTEL_LOGS_EXPORTER=none` also turns it off) | `on`: OTLP logs (`log()`, pino, winston) |
278
309
  | `metrics` | `WATCHTOWER_METRICS` (`OTEL_METRICS_EXPORTER=none` also turns it off) | `on`: OTLP metrics, see below |
279
- | `redact` | | none; called for every span and log attribute |
310
+ | `redact` | | none; called for every span and log attribute, every span event's attributes (`exception.message`, `exception.stacktrace`), the span name (key `span.name`), the status message (`status.message`) and log bodies (`log.body`) |
280
311
 
281
312
  Metrics: `init` sets the global meter provider, so `metrics.getMeter('app')` (re-exported from
282
313
  `@opentelemetry/api`) records to Watchtower. A periodic reader exports to `/v1/metrics` every
@@ -0,0 +1,12 @@
1
+ /** Parse `k=v,k2=v2` (OTEL_RESOURCE_ATTRIBUTES / OTEL_EXPORTER_OTLP_HEADERS); values are percent-decoded. */
2
+ export declare function parseKv(s: string | undefined): Record<string, string>;
3
+ export interface OtelEnv {
4
+ endpoint?: string;
5
+ /** Header names are lower-cased by nobody: they are kept as written, except `Authorization` is matched case-insensitively. */
6
+ headers: Record<string, string>;
7
+ /** `Authorization: Bearer <key>` from OTEL_EXPORTER_OTLP_HEADERS, the key only. */
8
+ bearer?: string;
9
+ serviceName?: string;
10
+ resource: Record<string, string>;
11
+ }
12
+ export declare function otelEnv(env?: NodeJS.ProcessEnv): OtelEnv;
@@ -0,0 +1,37 @@
1
+ "use strict";
2
+ // Standard OTEL_* environment variables, used as fallbacks after the WT_* ones.
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.parseKv = parseKv;
5
+ exports.otelEnv = otelEnv;
6
+ /** Parse `k=v,k2=v2` (OTEL_RESOURCE_ATTRIBUTES / OTEL_EXPORTER_OTLP_HEADERS); values are percent-decoded. */
7
+ function parseKv(s) {
8
+ const out = {};
9
+ for (const part of (s || '').split(',')) {
10
+ const i = part.indexOf('=');
11
+ if (i <= 0)
12
+ continue;
13
+ const dec = (x) => { try {
14
+ return decodeURIComponent(x.trim());
15
+ }
16
+ catch {
17
+ return x.trim();
18
+ } };
19
+ out[dec(part.slice(0, i))] = dec(part.slice(i + 1));
20
+ }
21
+ return out;
22
+ }
23
+ function otelEnv(env = process.env) {
24
+ const headers = parseKv(env.OTEL_EXPORTER_OTLP_HEADERS);
25
+ const authKey = Object.keys(headers).find((k) => k.toLowerCase() === 'authorization');
26
+ const auth = authKey ? headers[authKey] : undefined;
27
+ if (authKey)
28
+ delete headers[authKey];
29
+ const resource = parseKv(env.OTEL_RESOURCE_ATTRIBUTES);
30
+ return {
31
+ endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT || undefined,
32
+ headers,
33
+ bearer: auth && /^bearer\s+/i.test(auth) ? auth.replace(/^bearer\s+/i, '') : undefined,
34
+ serviceName: env.OTEL_SERVICE_NAME || resource['service.name'] || undefined,
35
+ resource,
36
+ };
37
+ }
@@ -2,7 +2,15 @@ export interface CaptureContext {
2
2
  /** false for crashes (uncaught); default true. Keys starting with `wt.` are dropped. */
3
3
  handled?: boolean;
4
4
  attributes?: Record<string, string | number | boolean | undefined>;
5
+ /**
6
+ * Exception replay: variables to show on the throwing frame. Node cannot read another frame's locals, so
7
+ * the caller names them. Secret-looking names are masked, values are shortened; the project must also turn
8
+ * on "exception replay" in its issue settings or the server drops them.
9
+ */
10
+ locals?: Record<string, unknown>;
5
11
  }
12
+ /** `exception.locals` for the innermost stack frame of `e`: the function and line come from its stack, the values from `locals`. */
13
+ export declare function localsAttribute(e: Error, locals: Record<string, unknown>): string | undefined;
6
14
  /** The slice of `process` the hooks need; tests pass an EventEmitter. */
7
15
  export interface ProcessLike {
8
16
  on(event: string, listener: (arg: unknown) => void): unknown;
@@ -1,11 +1,31 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.localsAttribute = localsAttribute;
3
4
  exports.dropReserved = dropReserved;
4
5
  exports.captureException = captureException;
5
6
  exports.installProcessHandlers = installProcessHandlers;
6
7
  // Error capture: an exception becomes an OTel span event (exception.*) with trace and span ids.
7
8
  const api_1 = require("@opentelemetry/api");
8
9
  const captured = new WeakMap(); // error object -> traceId, so framework + process hooks never double-report
10
+ const SENSITIVE = /pass|secret|token|api[-_]?key|auth|cookie|session|credit|card|ssn|private|bearer/i;
11
+ /** `exception.locals` for the innermost stack frame of `e`: the function and line come from its stack, the values from `locals`. */
12
+ function localsAttribute(e, locals) {
13
+ const m = /^\s+at (?:(\S+) \()?(.*?):(\d+):\d+\)?$/m.exec(e.stack || '');
14
+ const vars = {};
15
+ for (const [k, v] of Object.entries(locals).slice(0, 20)) {
16
+ let text;
17
+ try {
18
+ text = typeof v === 'string' ? v : JSON.stringify(v) ?? String(v);
19
+ }
20
+ catch {
21
+ text = '<unrepresentable>';
22
+ }
23
+ vars[k] = SENSITIVE.test(k) ? '[Filtered]' : text.slice(0, 120);
24
+ }
25
+ if (!Object.keys(vars).length)
26
+ return undefined;
27
+ return JSON.stringify([{ function: m?.[1] ?? '', filename: m?.[2] ?? '', lineno: m ? Number(m[3]) : undefined, vars }]);
28
+ }
9
29
  function dropReserved(attrs) {
10
30
  return Object.fromEntries(Object.entries(attrs || {}).filter(([k]) => !k.startsWith('wt.')));
11
31
  }
@@ -20,7 +40,10 @@ function captureException(err, ctx) {
20
40
  const active = api_1.trace.getActiveSpan();
21
41
  const span = active && active.isRecording() ? active : api_1.trace.getTracer('watchtower').startSpan('exception');
22
42
  span.setAttributes(attrs);
43
+ const replay = o.locals ? localsAttribute(e, o.locals) : undefined;
23
44
  span.recordException(e);
45
+ if (replay)
46
+ span.setAttribute('exception.locals', replay); // the OTel JS exception event takes no attributes
24
47
  span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: e.message });
25
48
  const traceId = span.spanContext().traceId;
26
49
  if (span !== active)
@@ -7,6 +7,7 @@ import type { LoggerProvider } from '@opentelemetry/sdk-logs';
7
7
  import type { MeterProvider } from '@opentelemetry/sdk-metrics';
8
8
  import { captureException } from './errors.js';
9
9
  import type { CaptureContext } from './errors.js';
10
+ import { RemoteSampler } from './sampling.js';
10
11
  import type { Profiler } from './profiling.js';
11
12
  export { captureException, metrics, trace };
12
13
  export { setCaptureRules, capturePayload } from './payload.js';
@@ -14,40 +15,56 @@ export type { CaptureRule } from './payload.js';
14
15
  export type { CaptureContext };
15
16
  export declare const DEFAULT_ENDPOINT = "http://localhost:4318";
16
17
  export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
18
+ /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
19
+ export declare function callerOf(stack: string | undefined): Record<string, string | number>;
17
20
  export interface InitOptions {
18
- /** Ingest key; defaults to WT_INGEST_KEY. */
21
+ /** Ingest key; defaults to WT_INGEST_KEY, then a `Bearer` Authorization in OTEL_EXPORTER_OTLP_HEADERS. */
19
22
  apiKey?: string;
20
23
  /** Defaults to WT_ENVIRONMENT. */
21
24
  environment?: string;
22
- /** Defaults to WT_RELEASE. */
25
+ /** Sent as `service.version`; the server links an issue's first release by matching it to an existing release's version. Defaults to WT_RELEASE, then `service.version` in OTEL_RESOURCE_ATTRIBUTES. */
23
26
  release?: string;
24
27
  /** Hook uncaughtException / unhandledRejection (default true). */
25
28
  captureProcessErrors?: boolean;
26
- /** Replace the log exporter (tests). */
29
+ /** Replace the log exporter (tests); setting it turns logs on unless `logs` is false. Exported through a batch processor. */
27
30
  logExporter?: unknown;
28
31
  /** Export OTLP logs; defaults to WATCHTOWER_LOGS (on unless `off`; OTEL_LOGS_EXPORTER=none also turns it off). */
29
32
  logs?: boolean;
33
+ /** Stamp code.filepath/code.lineno of the `log()` caller (reads the stack); defaults to WATCHTOWER_LOG_CALLER=1. */
34
+ logCaller?: boolean;
30
35
  /** Export OTLP metrics; defaults to WATCHTOWER_METRICS (on unless `off`; OTEL_METRICS_EXPORTER=none also turns it off). */
31
36
  metrics?: boolean;
32
- /** Replace the metric exporter (tests). */
37
+ /** Replace the metric exporter (tests); setting it turns metrics on unless `metrics` is false. */
33
38
  metricExporter?: unknown;
34
- /** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT or http://localhost:4318. */
39
+ /** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT, then OTEL_EXPORTER_OTLP_ENDPOINT, then http://localhost:4318. */
35
40
  endpoint?: string;
36
41
  /** REST API base URL for getPrompt/score; defaults to WT_API_URL or endpoint. */
37
42
  apiUrl?: string;
38
43
  project?: string;
39
- /** Defaults to WT_SERVICE. */
44
+ /** Defaults to WT_SERVICE, then OTEL_SERVICE_NAME, then `service.name` in OTEL_RESOURCE_ATTRIBUTES, then `app`. */
40
45
  serviceName?: string;
41
- /** Resource attributes; keys starting with `wt.` are dropped. */
46
+ /** Resource attributes (on top of OTEL_RESOURCE_ATTRIBUTES); keys starting with `wt.` are dropped. */
42
47
  resource?: Record<string, string | number | boolean>;
43
- /** Called for every span attribute before export; return the value to keep. */
48
+ /**
49
+ * Called before export for every span attribute, every span event's and link's attributes (so `exception.message`
50
+ * and `exception.stacktrace`), the span name (key `span.name`), the status message (`status.message`) and each log
51
+ * body (`log.body`) and log attribute; return the value to keep.
52
+ */
44
53
  redact?: (key: string, value: unknown) => unknown;
45
54
  /** Directory for the offline buffer. */
46
55
  bufferDir?: string;
56
+ /** Apply the project's head sampling rules in the SDK, polled from /v1/sampling; defaults to WT_REMOTE_SAMPLING=1. Spans dropped here never reach the gateway, so its exact request counts see only what is kept. */
57
+ remoteSampling?: boolean;
58
+ /** Seconds between sampling polls; defaults to WT_SAMPLING_INTERVAL_S or 30. */
59
+ samplingIntervalS?: number;
60
+ /** Stamp code.filepath / code.lineno / code.function on spans (the code that started the span); defaults to WT_CODE_ORIGIN=1. */
61
+ codeOrigin?: boolean;
47
62
  /** Fetch the service's payload capture rules (APM-016) and record bodies/GraphQL where a rule matches; defaults to WT_CAPTURE_RULES=1. */
48
63
  captureRules?: boolean;
49
64
  /** Replace the span exporter (tests). */
50
65
  exporter?: unknown;
66
+ /** Patch node:http, fetch (undici), pg and ioredis for those whose `@opentelemetry/instrumentation-*` peer is installed; defaults to on unless WT_INSTRUMENT=0. `init` only: use the `/register` preload so it runs before the app imports them. */
67
+ instrument?: boolean;
51
68
  fetch?: typeof fetch;
52
69
  }
53
70
  export interface EvalRow {
@@ -77,9 +94,10 @@ export declare class Client {
77
94
  readonly project: string | undefined;
78
95
  readonly headers: {
79
96
  Authorization: string;
80
- };
97
+ } & Record<string, string>;
81
98
  readonly fetch: typeof fetch;
82
99
  readonly redact: InitOptions['redact'];
100
+ private readonly logCaller;
83
101
  readonly provider: TracerProvider;
84
102
  readonly resource: Resource;
85
103
  readonly tracer: Tracer;
@@ -89,6 +107,8 @@ export declare class Client {
89
107
  readonly meterProvider?: MeterProvider;
90
108
  private readonly stopRuntime;
91
109
  uninstall?: () => void;
110
+ /** Set when remote sampling is on. */
111
+ remoteSampler?: RemoteSampler;
92
112
  /** Set when WT_PROFILING=1 started the continuous profiler. */
93
113
  profiler?: Promise<Profiler | undefined>;
94
114
  constructor(opts: InitOptions);
package/dist/cjs/index.js CHANGED
@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.shutdown = exports.observe = exports.evaluateResume = exports.evaluate = exports.score = exports.getPrompt = exports.log = exports.flush = exports.Client = exports.DEFAULT_ENDPOINT = exports.capturePayload = exports.setCaptureRules = exports.trace = exports.metrics = exports.captureException = void 0;
37
+ exports.callerOf = callerOf;
37
38
  exports.safeResource = safeResource;
38
39
  exports.init = init;
39
40
  // Thin Watchtower layer over the OpenTelemetry Node SDK: key header, endpoint default,
@@ -43,6 +44,9 @@ const api_1 = require("@opentelemetry/api");
43
44
  Object.defineProperty(exports, "metrics", { enumerable: true, get: function () { return api_1.metrics; } });
44
45
  Object.defineProperty(exports, "trace", { enumerable: true, get: function () { return api_1.trace; } });
45
46
  const load_js_1 = require("./load.js");
47
+ const env_js_1 = require("./env.js");
48
+ const instrument_js_1 = require("./instrument.js");
49
+ const redact_js_1 = require("./redact.js");
46
50
  const buffer_js_1 = require("./buffer.js");
47
51
  const metrics_js_1 = require("./metrics.js");
48
52
  const nodePath = __importStar(require("node:path"));
@@ -50,10 +54,17 @@ const errors_js_1 = require("./errors.js");
50
54
  Object.defineProperty(exports, "captureException", { enumerable: true, get: function () { return errors_js_1.captureException; } });
51
55
  const payload_js_1 = require("./payload.js");
52
56
  const profiling_js_1 = require("./profiling.js");
57
+ const sampling_js_1 = require("./sampling.js");
58
+ const origin_js_1 = require("./origin.js");
53
59
  var payload_js_2 = require("./payload.js");
54
60
  Object.defineProperty(exports, "setCaptureRules", { enumerable: true, get: function () { return payload_js_2.setCaptureRules; } });
55
61
  Object.defineProperty(exports, "capturePayload", { enumerable: true, get: function () { return payload_js_2.capturePayload; } });
56
62
  exports.DEFAULT_ENDPOINT = 'http://localhost:4318';
63
+ /** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
64
+ function callerOf(stack) {
65
+ const m = /\(?((?:file:\/\/)?[^()\s]+):(\d+):\d+\)?$/.exec((stack ?? '').split('\n')[2] ?? '');
66
+ return m ? { 'code.filepath': m[1].replace(/^file:\/\//, ''), 'code.lineno': Number(m[2]) } : {};
67
+ }
57
68
  const SEVERITY = { trace: 1, debug: 5, info: 9, warn: 13, error: 17, fatal: 21 };
58
69
  /** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
59
70
  function safeResource(attrs) {
@@ -67,6 +78,7 @@ class Client {
67
78
  headers;
68
79
  fetch;
69
80
  redact;
81
+ logCaller;
70
82
  provider;
71
83
  resource;
72
84
  tracer;
@@ -76,31 +88,36 @@ class Client {
76
88
  meterProvider;
77
89
  stopRuntime = () => { };
78
90
  uninstall;
91
+ /** Set when remote sampling is on. */
92
+ remoteSampler;
79
93
  /** Set when WT_PROFILING=1 started the continuous profiler. */
80
94
  profiler;
81
95
  constructor(opts) {
82
96
  const o = opts || {};
83
97
  const env = process.env;
84
- const apiKey = o.apiKey || env.WT_INGEST_KEY;
98
+ const oe = (0, env_js_1.otelEnv)(env);
99
+ const apiKey = o.apiKey || env.WT_INGEST_KEY || oe.bearer;
85
100
  if (!apiKey)
86
101
  throw new Error('apiKey is required (or set WT_INGEST_KEY)');
87
- this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || exports.DEFAULT_ENDPOINT).replace(/\/+$/, '');
102
+ this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || oe.endpoint || exports.DEFAULT_ENDPOINT).replace(/\/+$/, '');
88
103
  this.apiUrl = (o.apiUrl || env.WT_API_URL || this.endpoint).replace(/\/+$/, '');
89
104
  this.project = o.project || env.WT_PROJECT_ID;
90
- this.headers = { Authorization: `Bearer ${apiKey}` };
105
+ this.headers = { ...oe.headers, Authorization: `Bearer ${apiKey}` };
91
106
  this.fetch = o.fetch || globalThis.fetch;
92
- const environment = o.environment || env.WT_ENVIRONMENT;
93
- const release = o.release || env.WT_RELEASE;
107
+ const environment = o.environment || env.WT_ENVIRONMENT || oe.resource['deployment.environment.name'];
108
+ const release = o.release || env.WT_RELEASE || oe.resource['service.version'];
94
109
  // the OTel SDK packages are optional peers, loaded here (not at import time) with a clear error when missing
95
110
  const { resourceFromAttributes } = (0, load_js_1.need)('@opentelemetry/resources');
96
111
  const resource = resourceFromAttributes({
97
- 'service.name': o.serviceName || env.WT_SERVICE || 'app',
112
+ ...safeResource(oe.resource),
113
+ 'service.name': o.serviceName || env.WT_SERVICE || oe.serviceName || 'app',
98
114
  ...(environment ? { 'deployment.environment.name': environment } : {}),
99
115
  ...(release ? { 'service.version': release } : {}),
100
116
  ...safeResource(o.resource),
101
117
  });
102
118
  this.resource = resource;
103
119
  this.redact = o.redact;
120
+ this.logCaller = o.logCaller ?? env.WATCHTOWER_LOG_CALLER === '1';
104
121
  let exporter = o.exporter;
105
122
  if (!exporter) {
106
123
  const url = `${this.endpoint}/v1/traces`;
@@ -112,30 +129,42 @@ class Client {
112
129
  }
113
130
  // the redactor runs first, so the batch processor exports redacted attributes
114
131
  const { BatchSpanProcessor, TracerProvider: TP } = (0, load_js_1.need)('@opentelemetry/sdk-trace');
132
+ let sampler;
133
+ if (o.remoteSampling ?? env.WT_REMOTE_SAMPLING === '1') {
134
+ const { ParentBasedSampler } = (0, load_js_1.need)('@opentelemetry/sdk-trace');
135
+ this.remoteSampler = new sampling_js_1.RemoteSampler({
136
+ endpoint: this.endpoint, headers: this.headers, service: String(resource.attributes['service.name']), env: environment,
137
+ intervalMs: (o.samplingIntervalS ?? (Number(env.WT_SAMPLING_INTERVAL_S) || 30)) * 1000, fetch: this.fetch,
138
+ });
139
+ sampler = new ParentBasedSampler({ root: this.remoteSampler });
140
+ }
115
141
  this.provider = new TP({
116
142
  resource,
117
- spanProcessors: [...(o.redact ? [redactor(o.redact)] : []), new BatchSpanProcessor({ exporter })],
143
+ ...(sampler ? { sampler: sampler } : {}),
144
+ spanProcessors: [
145
+ ...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [(0, origin_js_1.codeOrigin)()] : []),
146
+ ...(o.redact ? [(0, redact_js_1.redactor)(o.redact)] : []),
147
+ new BatchSpanProcessor({ exporter }),
148
+ ],
118
149
  });
119
150
  this.tracer = this.provider.getTracer('watchtower');
120
- const logsOn = (0, metrics_js_1.switchOn)('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs);
151
+ // an explicit exporter never switches the other signals off: only `logs` / `metrics` / the env switches do
152
+ const logsOn = (0, metrics_js_1.switchOn)('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs ?? (o.logExporter ? true : undefined));
121
153
  let logExporter = logsOn ? o.logExporter : undefined;
122
- if (logsOn && !logExporter && !o.exporter) {
154
+ if (logsOn && !logExporter) {
123
155
  const url = `${this.endpoint}/v1/logs`;
124
156
  const { OTLPLogExporter } = (0, load_js_1.need)('@opentelemetry/exporter-logs-otlp-http');
125
157
  logExporter = new OTLPLogExporter({ url, headers: this.headers });
126
158
  if (o.bufferDir)
127
159
  logExporter = (0, buffer_js_1.spoolLogs)(logExporter, nodePath.join(o.bufferDir, 'logs'), url, this.headers, this.fetch);
128
160
  }
129
- const { LoggerProvider: LP, SimpleLogRecordProcessor, BatchLogRecordProcessor } = (0, load_js_1.need)('@opentelemetry/sdk-logs');
161
+ const { LoggerProvider: LP, BatchLogRecordProcessor } = (0, load_js_1.need)('@opentelemetry/sdk-logs');
130
162
  this.loggerProvider = new LP({
131
163
  resource,
132
- processors: !logExporter ? [] : [o.logExporter
133
- ? new SimpleLogRecordProcessor({ exporter: logExporter })
134
- : new BatchLogRecordProcessor({ exporter: logExporter })],
164
+ processors: !logExporter ? [] : [new BatchLogRecordProcessor({ exporter: logExporter })],
135
165
  });
136
166
  this.logger = this.loggerProvider.getLogger('watchtower');
137
- // with a test span exporter only an explicit metric exporter is used, as for logs
138
- if ((0, metrics_js_1.switchOn)('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics) && (o.metricExporter || !o.exporter)) {
167
+ if ((0, metrics_js_1.switchOn)('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics ?? (o.metricExporter ? true : undefined))) {
139
168
  const m = (0, metrics_js_1.startMetrics)(this.endpoint, this.headers, resource, o.bufferDir, this.fetch, o.metricExporter);
140
169
  this.meterProvider = m.provider;
141
170
  this.stopRuntime = m.stop;
@@ -148,8 +177,12 @@ class Client {
148
177
  log(level, body, attributes) {
149
178
  let attrs = (0, errors_js_1.dropReserved)(attributes);
150
179
  const redact = this.redact;
151
- if (redact)
180
+ if (redact) {
152
181
  attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => [k, redact(k, v)]));
182
+ body = String(redact(redact_js_1.LOG_BODY_KEY, body) ?? '');
183
+ }
184
+ if (this.logCaller)
185
+ attrs = { ...attrs, ...callerOf(new Error().stack) };
153
186
  this.logger.emit({
154
187
  severityNumber: SEVERITY[level] || SEVERITY.info,
155
188
  severityText: String(level).toUpperCase(),
@@ -257,42 +290,47 @@ class Client {
257
290
  }
258
291
  shutdown() {
259
292
  this.stopRuntime();
293
+ this.remoteSampler?.stop();
260
294
  return Promise.all([this.profiler?.then((p) => p?.stop()), this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
261
295
  }
262
296
  }
263
297
  exports.Client = Client;
264
- function redactor(hook) {
265
- return {
266
- onStart() { },
267
- onEnd(span) {
268
- for (const k of Object.keys(span.attributes))
269
- span.attributes[k] = hook(k, span.attributes[k]);
270
- },
271
- shutdown: () => Promise.resolve(),
272
- forceFlush: () => Promise.resolve(),
273
- };
298
+ // One state per process, not per module copy: a CommonJS and an ESM copy of this package (a `-r` preload plus an `import`
299
+ // app, say) must share the client or flush()/shutdown() in one copy would miss the init() of the other.
300
+ const state = (globalThis[Symbol.for('watchtower.node.state')] ??= {});
301
+ /**
302
+ * Register the global tracer provider, context manager, propagator and meter provider. OTel allows each to be set once
303
+ * (a second set returns false and changes nothing), so a provider the app or an earlier instrumentation registered is
304
+ * left alone and a warning says so; only globals a previous `init` installed are torn down on a re-init.
305
+ */
306
+ function registerGlobals(c) {
307
+ if (state.ownsGlobals) { // a re-init replaces what the previous init installed
308
+ api_1.trace.disable();
309
+ api_1.context.disable();
310
+ api_1.propagation.disable();
311
+ api_1.metrics.disable();
312
+ }
313
+ const { AsyncLocalStorageContextManager } = (0, load_js_1.need)('@opentelemetry/context-async-hooks');
314
+ const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = (0, load_js_1.need)('@opentelemetry/core');
315
+ const ok = [
316
+ api_1.trace.setGlobalTracerProvider(c.provider),
317
+ api_1.context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()),
318
+ api_1.propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })),
319
+ ...(c.meterProvider ? [api_1.metrics.setGlobalMeterProvider(c.meterProvider)] : []),
320
+ ];
321
+ state.ownsGlobals = ok[0] === true;
322
+ if (ok.includes(false))
323
+ api_1.diag.warn('watchtower: an OpenTelemetry global (tracer provider, context manager, propagator or meter provider) was already registered and was kept; call init() before other OTel setup to own them');
274
324
  }
275
- let current;
276
325
  function init(opts) {
277
- if (current && current.uninstall)
278
- current.uninstall();
279
- void current?.profiler?.then((p) => p?.stop());
326
+ if (state.current && state.current.uninstall)
327
+ state.current.uninstall();
328
+ void state.current?.profiler?.then((p) => p?.stop());
280
329
  const c = new Client(opts);
281
- current = c;
282
- // Global tracer + async context so trace.getActiveSpan() works; a re-init replaces the previous client's.
283
- api_1.trace.disable();
284
- api_1.context.disable();
285
- api_1.propagation.disable();
286
- const { AsyncLocalStorageContextManager } = (0, load_js_1.need)('@opentelemetry/context-async-hooks');
287
- const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = (0, load_js_1.need)('@opentelemetry/core');
288
- api_1.context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
289
- api_1.propagation.setGlobalPropagator(new CompositePropagator({
290
- propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
291
- }));
292
- api_1.trace.setGlobalTracerProvider(c.provider);
293
- if (c.meterProvider) {
294
- api_1.metrics.disable(); // a re-init replaces the previous client's provider
295
- api_1.metrics.setGlobalMeterProvider(c.meterProvider);
330
+ state.current = c;
331
+ registerGlobals(c);
332
+ if (opts?.instrument ?? !['0', 'off', 'false', 'no'].includes((process.env.WT_INSTRUMENT || '').trim().toLowerCase())) {
333
+ (0, instrument_js_1.instrumentAll)({ tracerProvider: c.provider, meterProvider: c.meterProvider }, [c.endpoint, c.apiUrl]);
296
334
  }
297
335
  if (process.env.WT_PROFILING === '1') {
298
336
  const attrs = c.resource.attributes;
@@ -309,11 +347,11 @@ function init(opts) {
309
347
  return c;
310
348
  }
311
349
  function need() {
312
- if (!current)
350
+ if (!state.current)
313
351
  throw new Error('watchtower.init() has not been called');
314
- return current;
352
+ return state.current;
315
353
  }
316
- const flush = () => (current ? current.forceFlush() : Promise.resolve());
354
+ const flush = () => (state.current ? state.current.forceFlush() : Promise.resolve());
317
355
  exports.flush = flush;
318
356
  const log = (l, b, a) => need().log(l, b, a);
319
357
  exports.log = log;
@@ -327,5 +365,5 @@ const evaluateResume = (d, t, s, prev, o) => need().evaluateResume(d, t, s, prev
327
365
  exports.evaluateResume = evaluateResume;
328
366
  const observe = (n, f) => need().observe(n, f);
329
367
  exports.observe = observe;
330
- const shutdown = () => (current ? current.shutdown() : Promise.resolve());
368
+ const shutdown = () => (state.current ? state.current.shutdown() : Promise.resolve());
331
369
  exports.shutdown = shutdown;
@@ -0,0 +1,7 @@
1
+ interface Providers {
2
+ tracerProvider: unknown;
3
+ meterProvider?: unknown;
4
+ }
5
+ /** Register whichever of the four instrumentations is installed. Returns the package names that were enabled. */
6
+ export declare function instrumentAll(p: Providers, selfUrls: string[]): string[];
7
+ export {};
@@ -0,0 +1,51 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.instrumentAll = instrumentAll;
4
+ // Auto-instrumentation: node:http, fetch (undici), pg and ioredis, each only when its optional peer package is installed.
5
+ const load_js_1 = require("./load.js");
6
+ const ENTRIES = [
7
+ {
8
+ pkg: '@opentelemetry/instrumentation-http', ctor: 'HttpInstrumentation',
9
+ // our own exports and API calls must not become spans
10
+ opts: (self) => ({ ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
11
+ },
12
+ {
13
+ pkg: '@opentelemetry/instrumentation-undici', ctor: 'UndiciInstrumentation',
14
+ opts: (self) => ({ ignoreRequestHook: (r) => self.some((u) => r.origin === u.origin) }),
15
+ },
16
+ { pkg: '@opentelemetry/instrumentation-pg', ctor: 'PgInstrumentation', opts: () => ({}) },
17
+ { pkg: '@opentelemetry/instrumentation-ioredis', ctor: 'IORedisInstrumentation', opts: () => ({}) },
18
+ ];
19
+ /** Register whichever of the four instrumentations is installed. Returns the package names that were enabled. */
20
+ function instrumentAll(p, selfUrls) {
21
+ let registerInstrumentations;
22
+ try {
23
+ ({ registerInstrumentations } = (0, load_js_1.load)('@opentelemetry/instrumentation'));
24
+ }
25
+ catch {
26
+ return []; // optional peer not installed
27
+ }
28
+ const self = selfUrls.flatMap((u) => { try {
29
+ return [new URL(u)];
30
+ }
31
+ catch {
32
+ return [];
33
+ } });
34
+ const found = [];
35
+ const names = [];
36
+ for (const e of ENTRIES) {
37
+ try {
38
+ const Ctor = (0, load_js_1.load)(e.pkg)[e.ctor];
39
+ if (Ctor) {
40
+ found.push(new Ctor(e.opts(self)));
41
+ names.push(e.pkg);
42
+ }
43
+ }
44
+ catch {
45
+ // package not installed (or failed to construct): skip it, never break init
46
+ }
47
+ }
48
+ if (found.length)
49
+ registerInstrumentations({ instrumentations: found, tracerProvider: p.tracerProvider, meterProvider: p.meterProvider });
50
+ return names;
51
+ }
@@ -1,5 +1,8 @@
1
- import type { Span } from '@opentelemetry/api';
1
+ import type { Attributes, Span } from '@opentelemetry/api';
2
2
  type Obj = Record<string, unknown>;
3
+ export declare const obj: (v: unknown) => Obj;
4
+ export declare const s: (v: unknown) => string;
5
+ export declare function start(name: string, attrs: Attributes, parent?: Span, startTime?: number): Span;
3
6
  /** The part of LlamaIndex.TS `Settings.callbackManager` used here. */
4
7
  export interface CallbackManagerLike {
5
8
  on(event: string, handler: (e: {