@watchtowerobservability/node 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -8
- package/dist/cjs/env.d.ts +12 -0
- package/dist/cjs/env.js +37 -0
- package/dist/cjs/errors.d.ts +8 -0
- package/dist/cjs/errors.js +23 -0
- package/dist/cjs/index.d.ts +29 -9
- package/dist/cjs/index.js +87 -49
- package/dist/cjs/instrument.d.ts +7 -0
- package/dist/cjs/instrument.js +51 -0
- package/dist/cjs/integrations/genai-agents.d.ts +4 -1
- package/dist/cjs/integrations/genai-agents.js +27 -24
- package/dist/cjs/integrations/genai-frameworks.d.ts +25 -0
- package/dist/cjs/integrations/genai-frameworks.js +211 -0
- package/dist/cjs/integrations/genai.d.ts +1 -0
- package/dist/cjs/integrations/genai.js +7 -2
- package/dist/cjs/integrations/next.d.ts +4 -1
- package/dist/cjs/integrations/next.js +13 -7
- package/dist/cjs/load.js +36 -5
- package/dist/cjs/origin.d.ts +9 -0
- package/dist/cjs/origin.js +43 -0
- package/dist/cjs/redact.d.ts +8 -0
- package/dist/cjs/redact.js +43 -0
- package/dist/cjs/register.d.ts +1 -0
- package/dist/cjs/register.js +29 -0
- package/dist/cjs/sampling.d.ts +53 -0
- package/dist/cjs/sampling.js +98 -0
- package/dist/esm/env.d.ts +12 -0
- package/dist/esm/env.js +33 -0
- package/dist/esm/errors.d.ts +8 -0
- package/dist/esm/errors.js +22 -0
- package/dist/esm/index.d.ts +29 -9
- package/dist/esm/index.js +87 -50
- package/dist/esm/instrument.d.ts +7 -0
- package/dist/esm/instrument.js +48 -0
- package/dist/esm/integrations/genai-agents.d.ts +4 -1
- package/dist/esm/integrations/genai-agents.js +4 -4
- package/dist/esm/integrations/genai-frameworks.d.ts +25 -0
- package/dist/esm/integrations/genai-frameworks.js +206 -0
- package/dist/esm/integrations/genai.d.ts +1 -0
- package/dist/esm/integrations/genai.js +3 -1
- package/dist/esm/integrations/next.d.ts +4 -1
- package/dist/esm/integrations/next.js +11 -6
- package/dist/esm/load.js +78 -2
- package/dist/esm/origin.d.ts +9 -0
- package/dist/esm/origin.js +39 -0
- package/dist/esm/redact.d.ts +8 -0
- package/dist/esm/redact.js +39 -0
- package/dist/esm/register.d.ts +1 -0
- package/dist/esm/register.js +27 -0
- package/dist/esm/sampling.d.ts +53 -0
- package/dist/esm/sampling.js +91 -0
- package/package.json +33 -2
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { Attributes, Context, Link, SpanKind } from '@opentelemetry/api';
|
|
2
|
+
import type { Sampler, SamplingResult } from '@opentelemetry/sdk-trace';
|
|
3
|
+
export interface SamplingRule {
|
|
4
|
+
name?: string;
|
|
5
|
+
enabled?: boolean | null;
|
|
6
|
+
match?: {
|
|
7
|
+
service?: string;
|
|
8
|
+
env?: string;
|
|
9
|
+
span_name_prefix?: string;
|
|
10
|
+
span_name_glob?: string;
|
|
11
|
+
span_name_regex?: string;
|
|
12
|
+
attrs?: Record<string, string>;
|
|
13
|
+
min_duration_ms?: number | null;
|
|
14
|
+
status?: string;
|
|
15
|
+
};
|
|
16
|
+
action?: {
|
|
17
|
+
kind?: string;
|
|
18
|
+
ratio?: number | null;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export interface RemoteConfig {
|
|
22
|
+
strategyType?: string;
|
|
23
|
+
probabilisticSampling?: {
|
|
24
|
+
samplingRate: number;
|
|
25
|
+
};
|
|
26
|
+
rules?: SamplingRule[];
|
|
27
|
+
}
|
|
28
|
+
/** fnv64a of the trace id's bytes, as a fraction of 2^64; matches the gateway's `ratioKeeps`. */
|
|
29
|
+
export declare function traceFraction(traceIdHex: string): number;
|
|
30
|
+
export declare function globMatches(glob: string, text: string): boolean;
|
|
31
|
+
/** The keep ratio for a span under `cfg` (1 = keep). First matching ratio/drop rule wins. */
|
|
32
|
+
export declare function ratioFor(cfg: RemoteConfig, service: string, env: string, name: string, attrs: Attributes): number;
|
|
33
|
+
export interface RemoteSamplerOptions {
|
|
34
|
+
endpoint: string;
|
|
35
|
+
headers: Record<string, string>;
|
|
36
|
+
service: string;
|
|
37
|
+
env?: string;
|
|
38
|
+
intervalMs?: number;
|
|
39
|
+
fetch?: typeof fetch;
|
|
40
|
+
}
|
|
41
|
+
export declare class RemoteSampler implements Sampler {
|
|
42
|
+
private readonly o;
|
|
43
|
+
cfg: RemoteConfig;
|
|
44
|
+
private etag;
|
|
45
|
+
private timer?;
|
|
46
|
+
private readonly fetchFn;
|
|
47
|
+
constructor(o: RemoteSamplerOptions);
|
|
48
|
+
/** One poll; a failure or 304 keeps the current config. */
|
|
49
|
+
poll(): Promise<void>;
|
|
50
|
+
stop(): void;
|
|
51
|
+
shouldSample(_c: Context, traceId: string, name: string, _k: SpanKind, attributes: Attributes, _l: Link[]): SamplingResult;
|
|
52
|
+
toString(): string;
|
|
53
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RemoteSampler = void 0;
|
|
4
|
+
exports.traceFraction = traceFraction;
|
|
5
|
+
exports.globMatches = globMatches;
|
|
6
|
+
exports.ratioFor = ratioFor;
|
|
7
|
+
const NOT_RECORD = 0;
|
|
8
|
+
const RECORD_AND_SAMPLED = 2;
|
|
9
|
+
const FNV_OFFSET = 0xcbf29ce484222325n;
|
|
10
|
+
const FNV_PRIME = 0x100000001b3n;
|
|
11
|
+
const MASK = 0xffffffffffffffffn;
|
|
12
|
+
const TWO_64 = 18446744073709551616;
|
|
13
|
+
/** fnv64a of the trace id's bytes, as a fraction of 2^64; matches the gateway's `ratioKeeps`. */
|
|
14
|
+
function traceFraction(traceIdHex) {
|
|
15
|
+
let h = FNV_OFFSET;
|
|
16
|
+
for (let i = 0; i + 1 < traceIdHex.length; i += 2)
|
|
17
|
+
h = ((h ^ BigInt(parseInt(traceIdHex.slice(i, i + 2), 16))) * FNV_PRIME) & MASK;
|
|
18
|
+
return Number(h) / TWO_64;
|
|
19
|
+
}
|
|
20
|
+
function globMatches(glob, text) {
|
|
21
|
+
const re = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.');
|
|
22
|
+
return new RegExp(`^${re}$`, 's').test(text);
|
|
23
|
+
}
|
|
24
|
+
function ruleMatches(r, service, env, name, attrs) {
|
|
25
|
+
const m = r.match ?? {};
|
|
26
|
+
// duration and status are only known at the end of the span: the gateway decides those
|
|
27
|
+
if (m.min_duration_ms != null || m.status)
|
|
28
|
+
return false;
|
|
29
|
+
if ((m.service && m.service !== service) || (m.env && m.env !== env))
|
|
30
|
+
return false;
|
|
31
|
+
if (m.span_name_prefix && !name.startsWith(m.span_name_prefix))
|
|
32
|
+
return false;
|
|
33
|
+
if (m.span_name_glob && !globMatches(m.span_name_glob, name))
|
|
34
|
+
return false;
|
|
35
|
+
if (m.span_name_regex) {
|
|
36
|
+
try {
|
|
37
|
+
if (!new RegExp(m.span_name_regex).test(name))
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return Object.entries(m.attrs ?? {}).every(([k, v]) => String(attrs[k]) === v);
|
|
45
|
+
}
|
|
46
|
+
/** The keep ratio for a span under `cfg` (1 = keep). First matching ratio/drop rule wins. */
|
|
47
|
+
function ratioFor(cfg, service, env, name, attrs) {
|
|
48
|
+
if (cfg.strategyType === 'PROBABILISTIC')
|
|
49
|
+
return clamp(cfg.probabilisticSampling?.samplingRate);
|
|
50
|
+
for (const r of cfg.rules ?? []) {
|
|
51
|
+
if (r.enabled === false || !ruleMatches(r, service, env, name, attrs))
|
|
52
|
+
continue;
|
|
53
|
+
if (r.action?.kind === 'drop')
|
|
54
|
+
return 0;
|
|
55
|
+
if (r.action?.kind === 'ratio' && r.action.ratio != null)
|
|
56
|
+
return clamp(r.action.ratio);
|
|
57
|
+
return 1; // keep, tail, rate: decided later
|
|
58
|
+
}
|
|
59
|
+
return 1;
|
|
60
|
+
}
|
|
61
|
+
const clamp = (r) => (typeof r === 'number' && Number.isFinite(r) ? Math.min(1, Math.max(0, r)) : 1);
|
|
62
|
+
class RemoteSampler {
|
|
63
|
+
o;
|
|
64
|
+
cfg = {};
|
|
65
|
+
etag = '';
|
|
66
|
+
timer;
|
|
67
|
+
fetchFn;
|
|
68
|
+
constructor(o) {
|
|
69
|
+
this.o = o;
|
|
70
|
+
this.fetchFn = o.fetch ?? globalThis.fetch;
|
|
71
|
+
void this.poll();
|
|
72
|
+
this.timer = setInterval(() => void this.poll(), Math.max(1000, o.intervalMs ?? 30_000));
|
|
73
|
+
this.timer.unref?.();
|
|
74
|
+
}
|
|
75
|
+
/** One poll; a failure or 304 keeps the current config. */
|
|
76
|
+
async poll() {
|
|
77
|
+
try {
|
|
78
|
+
const q = new URLSearchParams({ service: this.o.service, ...(this.o.env ? { env: this.o.env } : {}) });
|
|
79
|
+
const res = await this.fetchFn(`${this.o.endpoint}/v1/sampling?${q}`, {
|
|
80
|
+
headers: { ...this.o.headers, ...(this.etag ? { 'If-None-Match': this.etag } : {}) },
|
|
81
|
+
});
|
|
82
|
+
if (res.status === 304 || !res.ok)
|
|
83
|
+
return;
|
|
84
|
+
this.cfg = (await res.json());
|
|
85
|
+
this.etag = res.headers.get('ETag') ?? '';
|
|
86
|
+
}
|
|
87
|
+
catch { /* fallback: keep the last good config */ }
|
|
88
|
+
}
|
|
89
|
+
stop() { if (this.timer)
|
|
90
|
+
clearInterval(this.timer); }
|
|
91
|
+
shouldSample(_c, traceId, name, _k, attributes, _l) {
|
|
92
|
+
const r = ratioFor(this.cfg, this.o.service, this.o.env ?? '', name, attributes);
|
|
93
|
+
const keep = r >= 1 || (r > 0 && traceFraction(traceId) < r);
|
|
94
|
+
return { decision: keep ? RECORD_AND_SAMPLED : NOT_RECORD };
|
|
95
|
+
}
|
|
96
|
+
toString() { return 'WatchtowerRemoteSampler'; }
|
|
97
|
+
}
|
|
98
|
+
exports.RemoteSampler = RemoteSampler;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Parse `k=v,k2=v2` (OTEL_RESOURCE_ATTRIBUTES / OTEL_EXPORTER_OTLP_HEADERS); values are percent-decoded. */
|
|
2
|
+
export declare function parseKv(s: string | undefined): Record<string, string>;
|
|
3
|
+
export interface OtelEnv {
|
|
4
|
+
endpoint?: string;
|
|
5
|
+
/** Header names are lower-cased by nobody: they are kept as written, except `Authorization` is matched case-insensitively. */
|
|
6
|
+
headers: Record<string, string>;
|
|
7
|
+
/** `Authorization: Bearer <key>` from OTEL_EXPORTER_OTLP_HEADERS, the key only. */
|
|
8
|
+
bearer?: string;
|
|
9
|
+
serviceName?: string;
|
|
10
|
+
resource: Record<string, string>;
|
|
11
|
+
}
|
|
12
|
+
export declare function otelEnv(env?: NodeJS.ProcessEnv): OtelEnv;
|
package/dist/esm/env.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Standard OTEL_* environment variables, used as fallbacks after the WT_* ones.
|
|
2
|
+
/** Parse `k=v,k2=v2` (OTEL_RESOURCE_ATTRIBUTES / OTEL_EXPORTER_OTLP_HEADERS); values are percent-decoded. */
|
|
3
|
+
export function parseKv(s) {
|
|
4
|
+
const out = {};
|
|
5
|
+
for (const part of (s || '').split(',')) {
|
|
6
|
+
const i = part.indexOf('=');
|
|
7
|
+
if (i <= 0)
|
|
8
|
+
continue;
|
|
9
|
+
const dec = (x) => { try {
|
|
10
|
+
return decodeURIComponent(x.trim());
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return x.trim();
|
|
14
|
+
} };
|
|
15
|
+
out[dec(part.slice(0, i))] = dec(part.slice(i + 1));
|
|
16
|
+
}
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
export function otelEnv(env = process.env) {
|
|
20
|
+
const headers = parseKv(env.OTEL_EXPORTER_OTLP_HEADERS);
|
|
21
|
+
const authKey = Object.keys(headers).find((k) => k.toLowerCase() === 'authorization');
|
|
22
|
+
const auth = authKey ? headers[authKey] : undefined;
|
|
23
|
+
if (authKey)
|
|
24
|
+
delete headers[authKey];
|
|
25
|
+
const resource = parseKv(env.OTEL_RESOURCE_ATTRIBUTES);
|
|
26
|
+
return {
|
|
27
|
+
endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT || undefined,
|
|
28
|
+
headers,
|
|
29
|
+
bearer: auth && /^bearer\s+/i.test(auth) ? auth.replace(/^bearer\s+/i, '') : undefined,
|
|
30
|
+
serviceName: env.OTEL_SERVICE_NAME || resource['service.name'] || undefined,
|
|
31
|
+
resource,
|
|
32
|
+
};
|
|
33
|
+
}
|
package/dist/esm/errors.d.ts
CHANGED
|
@@ -2,7 +2,15 @@ export interface CaptureContext {
|
|
|
2
2
|
/** false for crashes (uncaught); default true. Keys starting with `wt.` are dropped. */
|
|
3
3
|
handled?: boolean;
|
|
4
4
|
attributes?: Record<string, string | number | boolean | undefined>;
|
|
5
|
+
/**
|
|
6
|
+
* Exception replay: variables to show on the throwing frame. Node cannot read another frame's locals, so
|
|
7
|
+
* the caller names them. Secret-looking names are masked, values are shortened; the project must also turn
|
|
8
|
+
* on "exception replay" in its issue settings or the server drops them.
|
|
9
|
+
*/
|
|
10
|
+
locals?: Record<string, unknown>;
|
|
5
11
|
}
|
|
12
|
+
/** `exception.locals` for the innermost stack frame of `e`: the function and line come from its stack, the values from `locals`. */
|
|
13
|
+
export declare function localsAttribute(e: Error, locals: Record<string, unknown>): string | undefined;
|
|
6
14
|
/** The slice of `process` the hooks need; tests pass an EventEmitter. */
|
|
7
15
|
export interface ProcessLike {
|
|
8
16
|
on(event: string, listener: (arg: unknown) => void): unknown;
|
package/dist/esm/errors.js
CHANGED
|
@@ -1,6 +1,25 @@
|
|
|
1
1
|
// Error capture: an exception becomes an OTel span event (exception.*) with trace and span ids.
|
|
2
2
|
import { trace, SpanStatusCode } from '@opentelemetry/api';
|
|
3
3
|
const captured = new WeakMap(); // error object -> traceId, so framework + process hooks never double-report
|
|
4
|
+
const SENSITIVE = /pass|secret|token|api[-_]?key|auth|cookie|session|credit|card|ssn|private|bearer/i;
|
|
5
|
+
/** `exception.locals` for the innermost stack frame of `e`: the function and line come from its stack, the values from `locals`. */
|
|
6
|
+
export function localsAttribute(e, locals) {
|
|
7
|
+
const m = /^\s+at (?:(\S+) \()?(.*?):(\d+):\d+\)?$/m.exec(e.stack || '');
|
|
8
|
+
const vars = {};
|
|
9
|
+
for (const [k, v] of Object.entries(locals).slice(0, 20)) {
|
|
10
|
+
let text;
|
|
11
|
+
try {
|
|
12
|
+
text = typeof v === 'string' ? v : JSON.stringify(v) ?? String(v);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
text = '<unrepresentable>';
|
|
16
|
+
}
|
|
17
|
+
vars[k] = SENSITIVE.test(k) ? '[Filtered]' : text.slice(0, 120);
|
|
18
|
+
}
|
|
19
|
+
if (!Object.keys(vars).length)
|
|
20
|
+
return undefined;
|
|
21
|
+
return JSON.stringify([{ function: m?.[1] ?? '', filename: m?.[2] ?? '', lineno: m ? Number(m[3]) : undefined, vars }]);
|
|
22
|
+
}
|
|
4
23
|
export function dropReserved(attrs) {
|
|
5
24
|
return Object.fromEntries(Object.entries(attrs || {}).filter(([k]) => !k.startsWith('wt.')));
|
|
6
25
|
}
|
|
@@ -15,7 +34,10 @@ export function captureException(err, ctx) {
|
|
|
15
34
|
const active = trace.getActiveSpan();
|
|
16
35
|
const span = active && active.isRecording() ? active : trace.getTracer('watchtower').startSpan('exception');
|
|
17
36
|
span.setAttributes(attrs);
|
|
37
|
+
const replay = o.locals ? localsAttribute(e, o.locals) : undefined;
|
|
18
38
|
span.recordException(e);
|
|
39
|
+
if (replay)
|
|
40
|
+
span.setAttribute('exception.locals', replay); // the OTel JS exception event takes no attributes
|
|
19
41
|
span.setStatus({ code: SpanStatusCode.ERROR, message: e.message });
|
|
20
42
|
const traceId = span.spanContext().traceId;
|
|
21
43
|
if (span !== active)
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type { LoggerProvider } from '@opentelemetry/sdk-logs';
|
|
|
7
7
|
import type { MeterProvider } from '@opentelemetry/sdk-metrics';
|
|
8
8
|
import { captureException } from './errors.js';
|
|
9
9
|
import type { CaptureContext } from './errors.js';
|
|
10
|
+
import { RemoteSampler } from './sampling.js';
|
|
10
11
|
import type { Profiler } from './profiling.js';
|
|
11
12
|
export { captureException, metrics, trace };
|
|
12
13
|
export { setCaptureRules, capturePayload } from './payload.js';
|
|
@@ -14,40 +15,56 @@ export type { CaptureRule } from './payload.js';
|
|
|
14
15
|
export type { CaptureContext };
|
|
15
16
|
export declare const DEFAULT_ENDPOINT = "http://localhost:4318";
|
|
16
17
|
export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
|
|
18
|
+
/** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
|
|
19
|
+
export declare function callerOf(stack: string | undefined): Record<string, string | number>;
|
|
17
20
|
export interface InitOptions {
|
|
18
|
-
/** Ingest key; defaults to WT_INGEST_KEY. */
|
|
21
|
+
/** Ingest key; defaults to WT_INGEST_KEY, then a `Bearer` Authorization in OTEL_EXPORTER_OTLP_HEADERS. */
|
|
19
22
|
apiKey?: string;
|
|
20
23
|
/** Defaults to WT_ENVIRONMENT. */
|
|
21
24
|
environment?: string;
|
|
22
|
-
/** Defaults to WT_RELEASE. */
|
|
25
|
+
/** Sent as `service.version`; the server links an issue's first release by matching it to an existing release's version. Defaults to WT_RELEASE, then `service.version` in OTEL_RESOURCE_ATTRIBUTES. */
|
|
23
26
|
release?: string;
|
|
24
27
|
/** Hook uncaughtException / unhandledRejection (default true). */
|
|
25
28
|
captureProcessErrors?: boolean;
|
|
26
|
-
/** Replace the log exporter (tests). */
|
|
29
|
+
/** Replace the log exporter (tests); setting it turns logs on unless `logs` is false. Exported through a batch processor. */
|
|
27
30
|
logExporter?: unknown;
|
|
28
31
|
/** Export OTLP logs; defaults to WATCHTOWER_LOGS (on unless `off`; OTEL_LOGS_EXPORTER=none also turns it off). */
|
|
29
32
|
logs?: boolean;
|
|
33
|
+
/** Stamp code.filepath/code.lineno of the `log()` caller (reads the stack); defaults to WATCHTOWER_LOG_CALLER=1. */
|
|
34
|
+
logCaller?: boolean;
|
|
30
35
|
/** Export OTLP metrics; defaults to WATCHTOWER_METRICS (on unless `off`; OTEL_METRICS_EXPORTER=none also turns it off). */
|
|
31
36
|
metrics?: boolean;
|
|
32
|
-
/** Replace the metric exporter (tests). */
|
|
37
|
+
/** Replace the metric exporter (tests); setting it turns metrics on unless `metrics` is false. */
|
|
33
38
|
metricExporter?: unknown;
|
|
34
|
-
/** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT
|
|
39
|
+
/** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT, then OTEL_EXPORTER_OTLP_ENDPOINT, then http://localhost:4318. */
|
|
35
40
|
endpoint?: string;
|
|
36
41
|
/** REST API base URL for getPrompt/score; defaults to WT_API_URL or endpoint. */
|
|
37
42
|
apiUrl?: string;
|
|
38
43
|
project?: string;
|
|
39
|
-
/** Defaults to WT_SERVICE. */
|
|
44
|
+
/** Defaults to WT_SERVICE, then OTEL_SERVICE_NAME, then `service.name` in OTEL_RESOURCE_ATTRIBUTES, then `app`. */
|
|
40
45
|
serviceName?: string;
|
|
41
|
-
/** Resource attributes; keys starting with `wt.` are dropped. */
|
|
46
|
+
/** Resource attributes (on top of OTEL_RESOURCE_ATTRIBUTES); keys starting with `wt.` are dropped. */
|
|
42
47
|
resource?: Record<string, string | number | boolean>;
|
|
43
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Called before export for every span attribute, every span event's and link's attributes (so `exception.message`
|
|
50
|
+
* and `exception.stacktrace`), the span name (key `span.name`), the status message (`status.message`) and each log
|
|
51
|
+
* body (`log.body`) and log attribute; return the value to keep.
|
|
52
|
+
*/
|
|
44
53
|
redact?: (key: string, value: unknown) => unknown;
|
|
45
54
|
/** Directory for the offline buffer. */
|
|
46
55
|
bufferDir?: string;
|
|
56
|
+
/** Apply the project's head sampling rules in the SDK, polled from /v1/sampling; defaults to WT_REMOTE_SAMPLING=1. Spans dropped here never reach the gateway, so its exact request counts see only what is kept. */
|
|
57
|
+
remoteSampling?: boolean;
|
|
58
|
+
/** Seconds between sampling polls; defaults to WT_SAMPLING_INTERVAL_S or 30. */
|
|
59
|
+
samplingIntervalS?: number;
|
|
60
|
+
/** Stamp code.filepath / code.lineno / code.function on spans (the code that started the span); defaults to WT_CODE_ORIGIN=1. */
|
|
61
|
+
codeOrigin?: boolean;
|
|
47
62
|
/** Fetch the service's payload capture rules (APM-016) and record bodies/GraphQL where a rule matches; defaults to WT_CAPTURE_RULES=1. */
|
|
48
63
|
captureRules?: boolean;
|
|
49
64
|
/** Replace the span exporter (tests). */
|
|
50
65
|
exporter?: unknown;
|
|
66
|
+
/** Patch node:http, fetch (undici), pg and ioredis for those whose `@opentelemetry/instrumentation-*` peer is installed; defaults to on unless WT_INSTRUMENT=0. `init` only: use the `/register` preload so it runs before the app imports them. */
|
|
67
|
+
instrument?: boolean;
|
|
51
68
|
fetch?: typeof fetch;
|
|
52
69
|
}
|
|
53
70
|
export interface EvalRow {
|
|
@@ -77,9 +94,10 @@ export declare class Client {
|
|
|
77
94
|
readonly project: string | undefined;
|
|
78
95
|
readonly headers: {
|
|
79
96
|
Authorization: string;
|
|
80
|
-
}
|
|
97
|
+
} & Record<string, string>;
|
|
81
98
|
readonly fetch: typeof fetch;
|
|
82
99
|
readonly redact: InitOptions['redact'];
|
|
100
|
+
private readonly logCaller;
|
|
83
101
|
readonly provider: TracerProvider;
|
|
84
102
|
readonly resource: Resource;
|
|
85
103
|
readonly tracer: Tracer;
|
|
@@ -89,6 +107,8 @@ export declare class Client {
|
|
|
89
107
|
readonly meterProvider?: MeterProvider;
|
|
90
108
|
private readonly stopRuntime;
|
|
91
109
|
uninstall?: () => void;
|
|
110
|
+
/** Set when remote sampling is on. */
|
|
111
|
+
remoteSampler?: RemoteSampler;
|
|
92
112
|
/** Set when WT_PROFILING=1 started the continuous profiler. */
|
|
93
113
|
profiler?: Promise<Profiler | undefined>;
|
|
94
114
|
constructor(opts: InitOptions);
|
package/dist/esm/index.js
CHANGED
|
@@ -1,17 +1,27 @@
|
|
|
1
1
|
// Thin Watchtower layer over the OpenTelemetry Node SDK: key header, endpoint default,
|
|
2
2
|
// offline buffer, wt.*-safe resource attributes, redaction hook, getPrompt/score/evaluate/observe.
|
|
3
3
|
import { inspect } from 'node:util';
|
|
4
|
-
import { context, metrics, propagation, trace } from '@opentelemetry/api';
|
|
4
|
+
import { context, diag, metrics, propagation, trace } from '@opentelemetry/api';
|
|
5
5
|
import { need as needPeer } from './load.js';
|
|
6
|
+
import { otelEnv } from './env.js';
|
|
7
|
+
import { instrumentAll } from './instrument.js';
|
|
8
|
+
import { LOG_BODY_KEY, redactor } from './redact.js';
|
|
6
9
|
import { SpoolExporter, spoolLogs } from './buffer.js';
|
|
7
10
|
import { startMetrics, switchOn } from './metrics.js';
|
|
8
11
|
import * as nodePath from 'node:path';
|
|
9
12
|
import { captureException, installProcessHandlers, dropReserved } from './errors.js';
|
|
10
13
|
import { loadCaptureRules } from './payload.js';
|
|
11
14
|
import { startProfiling } from './profiling.js';
|
|
15
|
+
import { RemoteSampler } from './sampling.js';
|
|
16
|
+
import { codeOrigin } from './origin.js';
|
|
12
17
|
export { captureException, metrics, trace };
|
|
13
18
|
export { setCaptureRules, capturePayload } from './payload.js';
|
|
14
19
|
export const DEFAULT_ENDPOINT = 'http://localhost:4318';
|
|
20
|
+
/** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
|
|
21
|
+
export function callerOf(stack) {
|
|
22
|
+
const m = /\(?((?:file:\/\/)?[^()\s]+):(\d+):\d+\)?$/.exec((stack ?? '').split('\n')[2] ?? '');
|
|
23
|
+
return m ? { 'code.filepath': m[1].replace(/^file:\/\//, ''), 'code.lineno': Number(m[2]) } : {};
|
|
24
|
+
}
|
|
15
25
|
const SEVERITY = { trace: 1, debug: 5, info: 9, warn: 13, error: 17, fatal: 21 };
|
|
16
26
|
/** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
|
|
17
27
|
export function safeResource(attrs) {
|
|
@@ -25,6 +35,7 @@ export class Client {
|
|
|
25
35
|
headers;
|
|
26
36
|
fetch;
|
|
27
37
|
redact;
|
|
38
|
+
logCaller;
|
|
28
39
|
provider;
|
|
29
40
|
resource;
|
|
30
41
|
tracer;
|
|
@@ -34,31 +45,36 @@ export class Client {
|
|
|
34
45
|
meterProvider;
|
|
35
46
|
stopRuntime = () => { };
|
|
36
47
|
uninstall;
|
|
48
|
+
/** Set when remote sampling is on. */
|
|
49
|
+
remoteSampler;
|
|
37
50
|
/** Set when WT_PROFILING=1 started the continuous profiler. */
|
|
38
51
|
profiler;
|
|
39
52
|
constructor(opts) {
|
|
40
53
|
const o = opts || {};
|
|
41
54
|
const env = process.env;
|
|
42
|
-
const
|
|
55
|
+
const oe = otelEnv(env);
|
|
56
|
+
const apiKey = o.apiKey || env.WT_INGEST_KEY || oe.bearer;
|
|
43
57
|
if (!apiKey)
|
|
44
58
|
throw new Error('apiKey is required (or set WT_INGEST_KEY)');
|
|
45
|
-
this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || DEFAULT_ENDPOINT).replace(/\/+$/, '');
|
|
59
|
+
this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || oe.endpoint || DEFAULT_ENDPOINT).replace(/\/+$/, '');
|
|
46
60
|
this.apiUrl = (o.apiUrl || env.WT_API_URL || this.endpoint).replace(/\/+$/, '');
|
|
47
61
|
this.project = o.project || env.WT_PROJECT_ID;
|
|
48
|
-
this.headers = { Authorization: `Bearer ${apiKey}` };
|
|
62
|
+
this.headers = { ...oe.headers, Authorization: `Bearer ${apiKey}` };
|
|
49
63
|
this.fetch = o.fetch || globalThis.fetch;
|
|
50
|
-
const environment = o.environment || env.WT_ENVIRONMENT;
|
|
51
|
-
const release = o.release || env.WT_RELEASE;
|
|
64
|
+
const environment = o.environment || env.WT_ENVIRONMENT || oe.resource['deployment.environment.name'];
|
|
65
|
+
const release = o.release || env.WT_RELEASE || oe.resource['service.version'];
|
|
52
66
|
// the OTel SDK packages are optional peers, loaded here (not at import time) with a clear error when missing
|
|
53
67
|
const { resourceFromAttributes } = needPeer('@opentelemetry/resources');
|
|
54
68
|
const resource = resourceFromAttributes({
|
|
55
|
-
|
|
69
|
+
...safeResource(oe.resource),
|
|
70
|
+
'service.name': o.serviceName || env.WT_SERVICE || oe.serviceName || 'app',
|
|
56
71
|
...(environment ? { 'deployment.environment.name': environment } : {}),
|
|
57
72
|
...(release ? { 'service.version': release } : {}),
|
|
58
73
|
...safeResource(o.resource),
|
|
59
74
|
});
|
|
60
75
|
this.resource = resource;
|
|
61
76
|
this.redact = o.redact;
|
|
77
|
+
this.logCaller = o.logCaller ?? env.WATCHTOWER_LOG_CALLER === '1';
|
|
62
78
|
let exporter = o.exporter;
|
|
63
79
|
if (!exporter) {
|
|
64
80
|
const url = `${this.endpoint}/v1/traces`;
|
|
@@ -70,30 +86,42 @@ export class Client {
|
|
|
70
86
|
}
|
|
71
87
|
// the redactor runs first, so the batch processor exports redacted attributes
|
|
72
88
|
const { BatchSpanProcessor, TracerProvider: TP } = needPeer('@opentelemetry/sdk-trace');
|
|
89
|
+
let sampler;
|
|
90
|
+
if (o.remoteSampling ?? env.WT_REMOTE_SAMPLING === '1') {
|
|
91
|
+
const { ParentBasedSampler } = needPeer('@opentelemetry/sdk-trace');
|
|
92
|
+
this.remoteSampler = new RemoteSampler({
|
|
93
|
+
endpoint: this.endpoint, headers: this.headers, service: String(resource.attributes['service.name']), env: environment,
|
|
94
|
+
intervalMs: (o.samplingIntervalS ?? (Number(env.WT_SAMPLING_INTERVAL_S) || 30)) * 1000, fetch: this.fetch,
|
|
95
|
+
});
|
|
96
|
+
sampler = new ParentBasedSampler({ root: this.remoteSampler });
|
|
97
|
+
}
|
|
73
98
|
this.provider = new TP({
|
|
74
99
|
resource,
|
|
75
|
-
|
|
100
|
+
...(sampler ? { sampler: sampler } : {}),
|
|
101
|
+
spanProcessors: [
|
|
102
|
+
...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [codeOrigin()] : []),
|
|
103
|
+
...(o.redact ? [redactor(o.redact)] : []),
|
|
104
|
+
new BatchSpanProcessor({ exporter }),
|
|
105
|
+
],
|
|
76
106
|
});
|
|
77
107
|
this.tracer = this.provider.getTracer('watchtower');
|
|
78
|
-
|
|
108
|
+
// an explicit exporter never switches the other signals off: only `logs` / `metrics` / the env switches do
|
|
109
|
+
const logsOn = switchOn('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs ?? (o.logExporter ? true : undefined));
|
|
79
110
|
let logExporter = logsOn ? o.logExporter : undefined;
|
|
80
|
-
if (logsOn && !logExporter
|
|
111
|
+
if (logsOn && !logExporter) {
|
|
81
112
|
const url = `${this.endpoint}/v1/logs`;
|
|
82
113
|
const { OTLPLogExporter } = needPeer('@opentelemetry/exporter-logs-otlp-http');
|
|
83
114
|
logExporter = new OTLPLogExporter({ url, headers: this.headers });
|
|
84
115
|
if (o.bufferDir)
|
|
85
116
|
logExporter = spoolLogs(logExporter, nodePath.join(o.bufferDir, 'logs'), url, this.headers, this.fetch);
|
|
86
117
|
}
|
|
87
|
-
const { LoggerProvider: LP,
|
|
118
|
+
const { LoggerProvider: LP, BatchLogRecordProcessor } = needPeer('@opentelemetry/sdk-logs');
|
|
88
119
|
this.loggerProvider = new LP({
|
|
89
120
|
resource,
|
|
90
|
-
processors: !logExporter ? [] : [
|
|
91
|
-
? new SimpleLogRecordProcessor({ exporter: logExporter })
|
|
92
|
-
: new BatchLogRecordProcessor({ exporter: logExporter })],
|
|
121
|
+
processors: !logExporter ? [] : [new BatchLogRecordProcessor({ exporter: logExporter })],
|
|
93
122
|
});
|
|
94
123
|
this.logger = this.loggerProvider.getLogger('watchtower');
|
|
95
|
-
|
|
96
|
-
if (switchOn('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics) && (o.metricExporter || !o.exporter)) {
|
|
124
|
+
if (switchOn('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics ?? (o.metricExporter ? true : undefined))) {
|
|
97
125
|
const m = startMetrics(this.endpoint, this.headers, resource, o.bufferDir, this.fetch, o.metricExporter);
|
|
98
126
|
this.meterProvider = m.provider;
|
|
99
127
|
this.stopRuntime = m.stop;
|
|
@@ -106,8 +134,12 @@ export class Client {
|
|
|
106
134
|
log(level, body, attributes) {
|
|
107
135
|
let attrs = dropReserved(attributes);
|
|
108
136
|
const redact = this.redact;
|
|
109
|
-
if (redact)
|
|
137
|
+
if (redact) {
|
|
110
138
|
attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => [k, redact(k, v)]));
|
|
139
|
+
body = String(redact(LOG_BODY_KEY, body) ?? '');
|
|
140
|
+
}
|
|
141
|
+
if (this.logCaller)
|
|
142
|
+
attrs = { ...attrs, ...callerOf(new Error().stack) };
|
|
111
143
|
this.logger.emit({
|
|
112
144
|
severityNumber: SEVERITY[level] || SEVERITY.info,
|
|
113
145
|
severityText: String(level).toUpperCase(),
|
|
@@ -215,41 +247,46 @@ export class Client {
|
|
|
215
247
|
}
|
|
216
248
|
shutdown() {
|
|
217
249
|
this.stopRuntime();
|
|
250
|
+
this.remoteSampler?.stop();
|
|
218
251
|
return Promise.all([this.profiler?.then((p) => p?.stop()), this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
|
|
219
252
|
}
|
|
220
253
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
254
|
+
// One state per process, not per module copy: a CommonJS and an ESM copy of this package (a `-r` preload plus an `import`
|
|
255
|
+
// app, say) must share the client or flush()/shutdown() in one copy would miss the init() of the other.
|
|
256
|
+
const state = (globalThis[Symbol.for('watchtower.node.state')] ??= {});
|
|
257
|
+
/**
|
|
258
|
+
* Register the global tracer provider, context manager, propagator and meter provider. OTel allows each to be set once
|
|
259
|
+
* (a second set returns false and changes nothing), so a provider the app or an earlier instrumentation registered is
|
|
260
|
+
* left alone and a warning says so; only globals a previous `init` installed are torn down on a re-init.
|
|
261
|
+
*/
|
|
262
|
+
function registerGlobals(c) {
|
|
263
|
+
if (state.ownsGlobals) { // a re-init replaces what the previous init installed
|
|
264
|
+
trace.disable();
|
|
265
|
+
context.disable();
|
|
266
|
+
propagation.disable();
|
|
267
|
+
metrics.disable();
|
|
268
|
+
}
|
|
269
|
+
const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
|
|
270
|
+
const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
|
|
271
|
+
const ok = [
|
|
272
|
+
trace.setGlobalTracerProvider(c.provider),
|
|
273
|
+
context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()),
|
|
274
|
+
propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })),
|
|
275
|
+
...(c.meterProvider ? [metrics.setGlobalMeterProvider(c.meterProvider)] : []),
|
|
276
|
+
];
|
|
277
|
+
state.ownsGlobals = ok[0] === true;
|
|
278
|
+
if (ok.includes(false))
|
|
279
|
+
diag.warn('watchtower: an OpenTelemetry global (tracer provider, context manager, propagator or meter provider) was already registered and was kept; call init() before other OTel setup to own them');
|
|
231
280
|
}
|
|
232
|
-
let current;
|
|
233
281
|
export function init(opts) {
|
|
234
|
-
if (current && current.uninstall)
|
|
235
|
-
current.uninstall();
|
|
236
|
-
void current?.profiler?.then((p) => p?.stop());
|
|
282
|
+
if (state.current && state.current.uninstall)
|
|
283
|
+
state.current.uninstall();
|
|
284
|
+
void state.current?.profiler?.then((p) => p?.stop());
|
|
237
285
|
const c = new Client(opts);
|
|
238
|
-
current = c;
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
propagation.disable();
|
|
243
|
-
const { AsyncLocalStorageContextManager } = needPeer('@opentelemetry/context-async-hooks');
|
|
244
|
-
const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = needPeer('@opentelemetry/core');
|
|
245
|
-
context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
|
|
246
|
-
propagation.setGlobalPropagator(new CompositePropagator({
|
|
247
|
-
propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
|
|
248
|
-
}));
|
|
249
|
-
trace.setGlobalTracerProvider(c.provider);
|
|
250
|
-
if (c.meterProvider) {
|
|
251
|
-
metrics.disable(); // a re-init replaces the previous client's provider
|
|
252
|
-
metrics.setGlobalMeterProvider(c.meterProvider);
|
|
286
|
+
state.current = c;
|
|
287
|
+
registerGlobals(c);
|
|
288
|
+
if (opts?.instrument ?? !['0', 'off', 'false', 'no'].includes((process.env.WT_INSTRUMENT || '').trim().toLowerCase())) {
|
|
289
|
+
instrumentAll({ tracerProvider: c.provider, meterProvider: c.meterProvider }, [c.endpoint, c.apiUrl]);
|
|
253
290
|
}
|
|
254
291
|
if (process.env.WT_PROFILING === '1') {
|
|
255
292
|
const attrs = c.resource.attributes;
|
|
@@ -266,15 +303,15 @@ export function init(opts) {
|
|
|
266
303
|
return c;
|
|
267
304
|
}
|
|
268
305
|
function need() {
|
|
269
|
-
if (!current)
|
|
306
|
+
if (!state.current)
|
|
270
307
|
throw new Error('watchtower.init() has not been called');
|
|
271
|
-
return current;
|
|
308
|
+
return state.current;
|
|
272
309
|
}
|
|
273
|
-
export const flush = () => (current ? current.forceFlush() : Promise.resolve());
|
|
310
|
+
export const flush = () => (state.current ? state.current.forceFlush() : Promise.resolve());
|
|
274
311
|
export const log = (l, b, a) => need().log(l, b, a);
|
|
275
312
|
export const getPrompt = (id) => need().getPrompt(id);
|
|
276
313
|
export const score = (t, v, o) => need().score(t, v, o);
|
|
277
314
|
export const evaluate = (d, t, s, o) => need().evaluate(d, t, s, o);
|
|
278
315
|
export const evaluateResume = (d, t, s, prev, o) => need().evaluateResume(d, t, s, prev, o);
|
|
279
316
|
export const observe = (n, f) => need().observe(n, f);
|
|
280
|
-
export const shutdown = () => (current ? current.shutdown() : Promise.resolve());
|
|
317
|
+
export const shutdown = () => (state.current ? state.current.shutdown() : Promise.resolve());
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
interface Providers {
|
|
2
|
+
tracerProvider: unknown;
|
|
3
|
+
meterProvider?: unknown;
|
|
4
|
+
}
|
|
5
|
+
/** Register whichever of the four instrumentations is installed. Returns the package names that were enabled. */
|
|
6
|
+
export declare function instrumentAll(p: Providers, selfUrls: string[]): string[];
|
|
7
|
+
export {};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// Auto-instrumentation: node:http, fetch (undici), pg and ioredis, each only when its optional peer package is installed.
|
|
2
|
+
import { load } from './load.js';
|
|
3
|
+
const ENTRIES = [
|
|
4
|
+
{
|
|
5
|
+
pkg: '@opentelemetry/instrumentation-http', ctor: 'HttpInstrumentation',
|
|
6
|
+
// our own exports and API calls must not become spans
|
|
7
|
+
opts: (self) => ({ ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
pkg: '@opentelemetry/instrumentation-undici', ctor: 'UndiciInstrumentation',
|
|
11
|
+
opts: (self) => ({ ignoreRequestHook: (r) => self.some((u) => r.origin === u.origin) }),
|
|
12
|
+
},
|
|
13
|
+
{ pkg: '@opentelemetry/instrumentation-pg', ctor: 'PgInstrumentation', opts: () => ({}) },
|
|
14
|
+
{ pkg: '@opentelemetry/instrumentation-ioredis', ctor: 'IORedisInstrumentation', opts: () => ({}) },
|
|
15
|
+
];
|
|
16
|
+
/** Register whichever of the four instrumentations is installed. Returns the package names that were enabled. */
|
|
17
|
+
export function instrumentAll(p, selfUrls) {
|
|
18
|
+
let registerInstrumentations;
|
|
19
|
+
try {
|
|
20
|
+
({ registerInstrumentations } = load('@opentelemetry/instrumentation'));
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return []; // optional peer not installed
|
|
24
|
+
}
|
|
25
|
+
const self = selfUrls.flatMap((u) => { try {
|
|
26
|
+
return [new URL(u)];
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return [];
|
|
30
|
+
} });
|
|
31
|
+
const found = [];
|
|
32
|
+
const names = [];
|
|
33
|
+
for (const e of ENTRIES) {
|
|
34
|
+
try {
|
|
35
|
+
const Ctor = load(e.pkg)[e.ctor];
|
|
36
|
+
if (Ctor) {
|
|
37
|
+
found.push(new Ctor(e.opts(self)));
|
|
38
|
+
names.push(e.pkg);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
// package not installed (or failed to construct): skip it, never break init
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (found.length)
|
|
46
|
+
registerInstrumentations({ instrumentations: found, tracerProvider: p.tracerProvider, meterProvider: p.meterProvider });
|
|
47
|
+
return names;
|
|
48
|
+
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import type { Span } from '@opentelemetry/api';
|
|
1
|
+
import type { Attributes, Span } from '@opentelemetry/api';
|
|
2
2
|
type Obj = Record<string, unknown>;
|
|
3
|
+
export declare const obj: (v: unknown) => Obj;
|
|
4
|
+
export declare const s: (v: unknown) => string;
|
|
5
|
+
export declare function start(name: string, attrs: Attributes, parent?: Span, startTime?: number): Span;
|
|
3
6
|
/** The part of LlamaIndex.TS `Settings.callbackManager` used here. */
|
|
4
7
|
export interface CallbackManagerLike {
|
|
5
8
|
on(event: string, handler: (e: {
|