@watchtowerobservability/node 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -9
- package/dist/cjs/checkin.d.ts +7 -0
- package/dist/cjs/checkin.js +18 -0
- package/dist/cjs/env.d.ts +12 -0
- package/dist/cjs/env.js +37 -0
- package/dist/cjs/errors.d.ts +8 -0
- package/dist/cjs/errors.js +23 -0
- package/dist/cjs/filter.d.ts +2 -0
- package/dist/cjs/filter.js +16 -0
- package/dist/cjs/index.d.ts +48 -10
- package/dist/cjs/index.js +119 -53
- package/dist/cjs/instrument.d.ts +10 -0
- package/dist/cjs/instrument.js +74 -0
- package/dist/cjs/integrations/db.d.ts +3 -0
- package/dist/cjs/integrations/db.js +4 -1
- package/dist/cjs/integrations/genai-agents.d.ts +4 -1
- package/dist/cjs/integrations/genai-agents.js +27 -24
- package/dist/cjs/integrations/genai-frameworks.d.ts +25 -0
- package/dist/cjs/integrations/genai-frameworks.js +211 -0
- package/dist/cjs/integrations/genai.d.ts +1 -0
- package/dist/cjs/integrations/genai.js +7 -2
- package/dist/cjs/integrations/next-build.d.ts +31 -0
- package/dist/cjs/integrations/next-build.js +95 -0
- package/dist/cjs/integrations/next-tunnel.d.ts +14 -0
- package/dist/cjs/integrations/next-tunnel.js +52 -0
- package/dist/cjs/integrations/next.d.ts +24 -1
- package/dist/cjs/integrations/next.js +39 -7
- package/dist/cjs/load.js +37 -5
- package/dist/cjs/next-sourcemaps-cli.d.ts +2 -0
- package/dist/cjs/next-sourcemaps-cli.js +9 -0
- package/dist/cjs/origin.d.ts +9 -0
- package/dist/cjs/origin.js +43 -0
- package/dist/cjs/redact.d.ts +8 -0
- package/dist/cjs/redact.js +43 -0
- package/dist/cjs/register.d.ts +1 -0
- package/dist/cjs/register.js +29 -0
- package/dist/cjs/sampling.d.ts +53 -0
- package/dist/cjs/sampling.js +98 -0
- package/dist/cjs/scrub.d.ts +7 -0
- package/dist/cjs/scrub.js +39 -0
- package/dist/cjs/shutdown.d.ts +14 -0
- package/dist/cjs/shutdown.js +26 -0
- package/dist/cjs/sql.js +11 -1
- package/dist/cjs/version.d.ts +2 -0
- package/dist/cjs/version.js +5 -0
- package/dist/esm/checkin.d.ts +7 -0
- package/dist/esm/checkin.js +15 -0
- package/dist/esm/env.d.ts +12 -0
- package/dist/esm/env.js +33 -0
- package/dist/esm/errors.d.ts +8 -0
- package/dist/esm/errors.js +22 -0
- package/dist/esm/filter.d.ts +2 -0
- package/dist/esm/filter.js +13 -0
- package/dist/esm/index.d.ts +48 -10
- package/dist/esm/index.js +117 -53
- package/dist/esm/instrument.d.ts +10 -0
- package/dist/esm/instrument.js +71 -0
- package/dist/esm/integrations/db.d.ts +3 -0
- package/dist/esm/integrations/db.js +5 -2
- package/dist/esm/integrations/genai-agents.d.ts +4 -1
- package/dist/esm/integrations/genai-agents.js +4 -4
- package/dist/esm/integrations/genai-frameworks.d.ts +25 -0
- package/dist/esm/integrations/genai-frameworks.js +206 -0
- package/dist/esm/integrations/genai.d.ts +1 -0
- package/dist/esm/integrations/genai.js +3 -1
- package/dist/esm/integrations/next-build.d.ts +31 -0
- package/dist/esm/integrations/next-build.js +58 -0
- package/dist/esm/integrations/next-tunnel.d.ts +14 -0
- package/dist/esm/integrations/next-tunnel.js +49 -0
- package/dist/esm/integrations/next.d.ts +24 -1
- package/dist/esm/integrations/next.js +32 -6
- package/dist/esm/load.js +80 -2
- package/dist/esm/next-sourcemaps-cli.d.ts +2 -0
- package/dist/esm/next-sourcemaps-cli.js +7 -0
- package/dist/esm/origin.d.ts +9 -0
- package/dist/esm/origin.js +39 -0
- package/dist/esm/redact.d.ts +8 -0
- package/dist/esm/redact.js +39 -0
- package/dist/esm/register.d.ts +1 -0
- package/dist/esm/register.js +27 -0
- package/dist/esm/sampling.d.ts +53 -0
- package/dist/esm/sampling.js +91 -0
- package/dist/esm/scrub.d.ts +7 -0
- package/dist/esm/scrub.js +34 -0
- package/dist/esm/shutdown.d.ts +14 -0
- package/dist/esm/shutdown.js +23 -0
- package/dist/esm/sql.js +11 -1
- package/dist/esm/version.d.ts +2 -0
- package/dist/esm/version.js +2 -0
- package/package.json +40 -2
package/dist/cjs/index.js
CHANGED
|
@@ -33,7 +33,8 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.shutdown = exports.observe = exports.evaluateResume = exports.evaluate = exports.score = exports.getPrompt = exports.log = exports.flush = exports.Client = exports.DEFAULT_ENDPOINT = exports.capturePayload = exports.setCaptureRules = exports.trace = exports.metrics = exports.captureException = void 0;
|
|
36
|
+
exports.shutdown = exports.observe = exports.evaluateResume = exports.evaluate = exports.score = exports.getPrompt = exports.log = exports.flush = exports.checkin = exports.Client = exports.DEFAULT_ENDPOINT = exports.capturePayload = exports.setCaptureRules = exports.trace = exports.metrics = exports.captureException = void 0;
|
|
37
|
+
exports.callerOf = callerOf;
|
|
37
38
|
exports.safeResource = safeResource;
|
|
38
39
|
exports.init = init;
|
|
39
40
|
// Thin Watchtower layer over the OpenTelemetry Node SDK: key header, endpoint default,
|
|
@@ -43,6 +44,14 @@ const api_1 = require("@opentelemetry/api");
|
|
|
43
44
|
Object.defineProperty(exports, "metrics", { enumerable: true, get: function () { return api_1.metrics; } });
|
|
44
45
|
Object.defineProperty(exports, "trace", { enumerable: true, get: function () { return api_1.trace; } });
|
|
45
46
|
const load_js_1 = require("./load.js");
|
|
47
|
+
const env_js_1 = require("./env.js");
|
|
48
|
+
const instrument_js_1 = require("./instrument.js");
|
|
49
|
+
const redact_js_1 = require("./redact.js");
|
|
50
|
+
const scrub_js_1 = require("./scrub.js");
|
|
51
|
+
const filter_js_1 = require("./filter.js");
|
|
52
|
+
const checkin_js_1 = require("./checkin.js");
|
|
53
|
+
const shutdown_js_1 = require("./shutdown.js");
|
|
54
|
+
const version_js_1 = require("./version.js");
|
|
46
55
|
const buffer_js_1 = require("./buffer.js");
|
|
47
56
|
const metrics_js_1 = require("./metrics.js");
|
|
48
57
|
const nodePath = __importStar(require("node:path"));
|
|
@@ -50,10 +59,17 @@ const errors_js_1 = require("./errors.js");
|
|
|
50
59
|
Object.defineProperty(exports, "captureException", { enumerable: true, get: function () { return errors_js_1.captureException; } });
|
|
51
60
|
const payload_js_1 = require("./payload.js");
|
|
52
61
|
const profiling_js_1 = require("./profiling.js");
|
|
62
|
+
const sampling_js_1 = require("./sampling.js");
|
|
63
|
+
const origin_js_1 = require("./origin.js");
|
|
53
64
|
var payload_js_2 = require("./payload.js");
|
|
54
65
|
Object.defineProperty(exports, "setCaptureRules", { enumerable: true, get: function () { return payload_js_2.setCaptureRules; } });
|
|
55
66
|
Object.defineProperty(exports, "capturePayload", { enumerable: true, get: function () { return payload_js_2.capturePayload; } });
|
|
56
67
|
exports.DEFAULT_ENDPOINT = 'http://localhost:4318';
|
|
68
|
+
/** code.filepath/code.lineno of the frame that called `log()` (stack line 0 is "Error", 1 is `log`, 2 the caller). */
|
|
69
|
+
function callerOf(stack) {
|
|
70
|
+
const m = /\(?((?:file:\/\/)?[^()\s]+):(\d+):\d+\)?$/.exec((stack ?? '').split('\n')[2] ?? '');
|
|
71
|
+
return m ? { 'code.filepath': m[1].replace(/^file:\/\//, ''), 'code.lineno': Number(m[2]) } : {};
|
|
72
|
+
}
|
|
57
73
|
const SEVERITY = { trace: 1, debug: 5, info: 9, warn: 13, error: 17, fatal: 21 };
|
|
58
74
|
/** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
|
|
59
75
|
function safeResource(attrs) {
|
|
@@ -67,6 +83,7 @@ class Client {
|
|
|
67
83
|
headers;
|
|
68
84
|
fetch;
|
|
69
85
|
redact;
|
|
86
|
+
logCaller;
|
|
70
87
|
provider;
|
|
71
88
|
resource;
|
|
72
89
|
tracer;
|
|
@@ -76,31 +93,43 @@ class Client {
|
|
|
76
93
|
meterProvider;
|
|
77
94
|
stopRuntime = () => { };
|
|
78
95
|
uninstall;
|
|
96
|
+
/** Set when remote sampling is on. */
|
|
97
|
+
remoteSampler;
|
|
79
98
|
/** Set when WT_PROFILING=1 started the continuous profiler. */
|
|
80
99
|
profiler;
|
|
81
100
|
constructor(opts) {
|
|
82
101
|
const o = opts || {};
|
|
83
102
|
const env = process.env;
|
|
84
|
-
const
|
|
103
|
+
const oe = (0, env_js_1.otelEnv)(env);
|
|
104
|
+
const apiKey = o.apiKey || env.WT_INGEST_KEY || oe.bearer;
|
|
85
105
|
if (!apiKey)
|
|
86
106
|
throw new Error('apiKey is required (or set WT_INGEST_KEY)');
|
|
87
|
-
this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || exports.DEFAULT_ENDPOINT).replace(/\/+$/, '');
|
|
107
|
+
this.endpoint = (o.endpoint || env.WT_OTLP_ENDPOINT || oe.endpoint || exports.DEFAULT_ENDPOINT).replace(/\/+$/, '');
|
|
88
108
|
this.apiUrl = (o.apiUrl || env.WT_API_URL || this.endpoint).replace(/\/+$/, '');
|
|
89
109
|
this.project = o.project || env.WT_PROJECT_ID;
|
|
90
|
-
this.headers = { Authorization: `Bearer ${apiKey}` };
|
|
110
|
+
this.headers = { ...oe.headers, Authorization: `Bearer ${apiKey}` };
|
|
91
111
|
this.fetch = o.fetch || globalThis.fetch;
|
|
92
|
-
const environment = o.environment || env.WT_ENVIRONMENT;
|
|
93
|
-
const release = o.release || env.WT_RELEASE;
|
|
112
|
+
const environment = o.environment || env.WT_ENVIRONMENT || oe.resource['deployment.environment.name'];
|
|
113
|
+
const release = o.release || env.WT_RELEASE || oe.resource['service.version'];
|
|
94
114
|
// the OTel SDK packages are optional peers, loaded here (not at import time) with a clear error when missing
|
|
95
|
-
const { resourceFromAttributes } = (0, load_js_1.need)('@opentelemetry/resources');
|
|
115
|
+
const { resourceFromAttributes, defaultResource } = (0, load_js_1.need)('@opentelemetry/resources');
|
|
116
|
+
// the SDK's own telemetry.sdk.* (name, language, version): a bare resourceFromAttributes does not add them
|
|
117
|
+
const sdk = (defaultResource().attributes ?? {});
|
|
96
118
|
const resource = resourceFromAttributes({
|
|
97
|
-
'
|
|
119
|
+
'telemetry.sdk.name': String(sdk['telemetry.sdk.name'] ?? 'opentelemetry'),
|
|
120
|
+
'telemetry.sdk.language': String(sdk['telemetry.sdk.language'] ?? 'nodejs'),
|
|
121
|
+
'telemetry.sdk.version': String(sdk['telemetry.sdk.version'] ?? 'unknown'),
|
|
122
|
+
'telemetry.distro.name': 'watchtower',
|
|
123
|
+
'telemetry.distro.version': version_js_1.VERSION,
|
|
124
|
+
...safeResource(oe.resource),
|
|
125
|
+
'service.name': o.serviceName || env.WT_SERVICE || oe.serviceName || 'app',
|
|
98
126
|
...(environment ? { 'deployment.environment.name': environment } : {}),
|
|
99
127
|
...(release ? { 'service.version': release } : {}),
|
|
100
128
|
...safeResource(o.resource),
|
|
101
129
|
});
|
|
102
130
|
this.resource = resource;
|
|
103
131
|
this.redact = o.redact;
|
|
132
|
+
this.logCaller = o.logCaller ?? env.WATCHTOWER_LOG_CALLER === '1';
|
|
104
133
|
let exporter = o.exporter;
|
|
105
134
|
if (!exporter) {
|
|
106
135
|
const url = `${this.endpoint}/v1/traces`;
|
|
@@ -110,32 +139,47 @@ class Client {
|
|
|
110
139
|
if (o.bufferDir)
|
|
111
140
|
exporter = new buffer_js_1.SpoolExporter(exporter, nodePath.join(o.bufferDir, 'traces'), url, this.headers, this.fetch);
|
|
112
141
|
}
|
|
113
|
-
|
|
142
|
+
if (o.spanFilter)
|
|
143
|
+
exporter = (0, filter_js_1.filterExporter)(exporter, o.spanFilter);
|
|
144
|
+
// the scrubber and redactor run first, so the batch processor exports scrubbed attributes (the user's `redact` sees scrubbed URLs)
|
|
114
145
|
const { BatchSpanProcessor, TracerProvider: TP } = (0, load_js_1.need)('@opentelemetry/sdk-trace');
|
|
146
|
+
let sampler;
|
|
147
|
+
if (o.remoteSampling ?? env.WT_REMOTE_SAMPLING === '1') {
|
|
148
|
+
const { ParentBasedSampler } = (0, load_js_1.need)('@opentelemetry/sdk-trace');
|
|
149
|
+
this.remoteSampler = new sampling_js_1.RemoteSampler({
|
|
150
|
+
endpoint: this.endpoint, headers: this.headers, service: String(resource.attributes['service.name']), env: environment,
|
|
151
|
+
intervalMs: (o.samplingIntervalS ?? (Number(env.WT_SAMPLING_INTERVAL_S) || 30)) * 1000, fetch: this.fetch,
|
|
152
|
+
});
|
|
153
|
+
sampler = new ParentBasedSampler({ root: this.remoteSampler });
|
|
154
|
+
}
|
|
115
155
|
this.provider = new TP({
|
|
116
156
|
resource,
|
|
117
|
-
|
|
157
|
+
...(sampler ? { sampler: sampler } : {}),
|
|
158
|
+
spanProcessors: [
|
|
159
|
+
...(o.codeOrigin ?? env.WT_CODE_ORIGIN === '1' ? [(0, origin_js_1.codeOrigin)()] : []),
|
|
160
|
+
...(o.urlScrubbing ?? true ? [(0, scrub_js_1.urlScrubber)()] : []),
|
|
161
|
+
...(o.redact ? [(0, redact_js_1.redactor)(o.redact)] : []),
|
|
162
|
+
new BatchSpanProcessor({ exporter }),
|
|
163
|
+
],
|
|
118
164
|
});
|
|
119
165
|
this.tracer = this.provider.getTracer('watchtower');
|
|
120
|
-
|
|
166
|
+
// an explicit exporter never switches the other signals off: only `logs` / `metrics` / the env switches do
|
|
167
|
+
const logsOn = (0, metrics_js_1.switchOn)('WATCHTOWER_LOGS', 'OTEL_LOGS_EXPORTER', o.logs ?? (o.logExporter ? true : undefined));
|
|
121
168
|
let logExporter = logsOn ? o.logExporter : undefined;
|
|
122
|
-
if (logsOn && !logExporter
|
|
169
|
+
if (logsOn && !logExporter) {
|
|
123
170
|
const url = `${this.endpoint}/v1/logs`;
|
|
124
171
|
const { OTLPLogExporter } = (0, load_js_1.need)('@opentelemetry/exporter-logs-otlp-http');
|
|
125
172
|
logExporter = new OTLPLogExporter({ url, headers: this.headers });
|
|
126
173
|
if (o.bufferDir)
|
|
127
174
|
logExporter = (0, buffer_js_1.spoolLogs)(logExporter, nodePath.join(o.bufferDir, 'logs'), url, this.headers, this.fetch);
|
|
128
175
|
}
|
|
129
|
-
const { LoggerProvider: LP,
|
|
176
|
+
const { LoggerProvider: LP, BatchLogRecordProcessor } = (0, load_js_1.need)('@opentelemetry/sdk-logs');
|
|
130
177
|
this.loggerProvider = new LP({
|
|
131
178
|
resource,
|
|
132
|
-
processors: !logExporter ? [] : [
|
|
133
|
-
? new SimpleLogRecordProcessor({ exporter: logExporter })
|
|
134
|
-
: new BatchLogRecordProcessor({ exporter: logExporter })],
|
|
179
|
+
processors: !logExporter ? [] : [new BatchLogRecordProcessor({ exporter: logExporter })],
|
|
135
180
|
});
|
|
136
181
|
this.logger = this.loggerProvider.getLogger('watchtower');
|
|
137
|
-
|
|
138
|
-
if ((0, metrics_js_1.switchOn)('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics) && (o.metricExporter || !o.exporter)) {
|
|
182
|
+
if ((0, metrics_js_1.switchOn)('WATCHTOWER_METRICS', 'OTEL_METRICS_EXPORTER', o.metrics ?? (o.metricExporter ? true : undefined))) {
|
|
139
183
|
const m = (0, metrics_js_1.startMetrics)(this.endpoint, this.headers, resource, o.bufferDir, this.fetch, o.metricExporter);
|
|
140
184
|
this.meterProvider = m.provider;
|
|
141
185
|
this.stopRuntime = m.stop;
|
|
@@ -148,8 +192,12 @@ class Client {
|
|
|
148
192
|
log(level, body, attributes) {
|
|
149
193
|
let attrs = (0, errors_js_1.dropReserved)(attributes);
|
|
150
194
|
const redact = this.redact;
|
|
151
|
-
if (redact)
|
|
195
|
+
if (redact) {
|
|
152
196
|
attrs = Object.fromEntries(Object.entries(attrs).map(([k, v]) => [k, redact(k, v)]));
|
|
197
|
+
body = String(redact(redact_js_1.LOG_BODY_KEY, body) ?? '');
|
|
198
|
+
}
|
|
199
|
+
if (this.logCaller)
|
|
200
|
+
attrs = { ...attrs, ...callerOf(new Error().stack) };
|
|
153
201
|
this.logger.emit({
|
|
154
202
|
severityNumber: SEVERITY[level] || SEVERITY.info,
|
|
155
203
|
severityText: String(level).toUpperCase(),
|
|
@@ -157,6 +205,10 @@ class Client {
|
|
|
157
205
|
attributes: attrs,
|
|
158
206
|
});
|
|
159
207
|
}
|
|
208
|
+
/** Run `fn` as a cron / scheduler job with check-ins to the monitor `monitorSlug` (needs the `ingest:checkins` key scope): `in_progress`, then `ok` or `error` with the duration. */
|
|
209
|
+
checkin(monitorSlug, fn) {
|
|
210
|
+
return (0, checkin_js_1.checkin)(this, monitorSlug, fn);
|
|
211
|
+
}
|
|
160
212
|
forceFlush() {
|
|
161
213
|
return Promise.all([this.provider.forceFlush(), this.loggerProvider.forceFlush(), this.meterProvider?.forceFlush()]);
|
|
162
214
|
}
|
|
@@ -257,42 +309,52 @@ class Client {
|
|
|
257
309
|
}
|
|
258
310
|
shutdown() {
|
|
259
311
|
this.stopRuntime();
|
|
312
|
+
this.remoteSampler?.stop();
|
|
260
313
|
return Promise.all([this.profiler?.then((p) => p?.stop()), this.provider.shutdown(), this.loggerProvider.shutdown(), this.meterProvider?.shutdown()]).then(() => undefined);
|
|
261
314
|
}
|
|
262
315
|
}
|
|
263
316
|
exports.Client = Client;
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
317
|
+
// One state per process, not per module copy: a CommonJS and an ESM copy of this package (a `-r` preload plus an `import`
|
|
318
|
+
// app, say) must share the client or flush()/shutdown() in one copy would miss the init() of the other.
|
|
319
|
+
const state = (globalThis[Symbol.for('watchtower.node.state')] ??= {});
|
|
320
|
+
/**
|
|
321
|
+
* Register the global tracer provider, context manager, propagator and meter provider. OTel allows each to be set once
|
|
322
|
+
* (a second set returns false and changes nothing), so a provider the app or an earlier instrumentation registered is
|
|
323
|
+
* left alone and a warning says so; only globals a previous `init` installed are torn down on a re-init.
|
|
324
|
+
*/
|
|
325
|
+
function registerGlobals(c) {
|
|
326
|
+
if (state.ownsGlobals) { // a re-init replaces what the previous init installed
|
|
327
|
+
api_1.trace.disable();
|
|
328
|
+
api_1.context.disable();
|
|
329
|
+
api_1.propagation.disable();
|
|
330
|
+
api_1.metrics.disable();
|
|
331
|
+
}
|
|
332
|
+
const { AsyncLocalStorageContextManager } = (0, load_js_1.need)('@opentelemetry/context-async-hooks');
|
|
333
|
+
const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = (0, load_js_1.need)('@opentelemetry/core');
|
|
334
|
+
const ok = [
|
|
335
|
+
api_1.trace.setGlobalTracerProvider(c.provider),
|
|
336
|
+
api_1.context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()),
|
|
337
|
+
api_1.propagation.setGlobalPropagator(new CompositePropagator({ propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] })),
|
|
338
|
+
...(c.meterProvider ? [api_1.metrics.setGlobalMeterProvider(c.meterProvider)] : []),
|
|
339
|
+
];
|
|
340
|
+
state.ownsGlobals = ok[0] === true;
|
|
341
|
+
if (ok.includes(false))
|
|
342
|
+
api_1.diag.warn('watchtower: an OpenTelemetry global (tracer provider, context manager, propagator or meter provider) was already registered and was kept; call init() before other OTel setup to own them');
|
|
274
343
|
}
|
|
275
|
-
let current;
|
|
276
344
|
function init(opts) {
|
|
277
|
-
if (current && current.uninstall)
|
|
278
|
-
current.uninstall();
|
|
279
|
-
void current?.profiler?.then((p) => p?.stop());
|
|
345
|
+
if (state.current && state.current.uninstall)
|
|
346
|
+
state.current.uninstall();
|
|
347
|
+
void state.current?.profiler?.then((p) => p?.stop());
|
|
280
348
|
const c = new Client(opts);
|
|
281
|
-
current = c;
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
|
|
291
|
-
}));
|
|
292
|
-
api_1.trace.setGlobalTracerProvider(c.provider);
|
|
293
|
-
if (c.meterProvider) {
|
|
294
|
-
api_1.metrics.disable(); // a re-init replaces the previous client's provider
|
|
295
|
-
api_1.metrics.setGlobalMeterProvider(c.meterProvider);
|
|
349
|
+
state.current = c;
|
|
350
|
+
registerGlobals(c);
|
|
351
|
+
const env = (process.env.WT_INSTRUMENT || '').trim().toLowerCase();
|
|
352
|
+
const mode = opts?.instrument ?? (env === 'auto' ? 'auto' : !['0', 'off', 'false', 'no'].includes(env));
|
|
353
|
+
if (mode) {
|
|
354
|
+
const names = (0, instrument_js_1.instrumentAll)({ tracerProvider: c.provider, meterProvider: c.meterProvider }, [c.endpoint, c.apiUrl], mode === 'auto');
|
|
355
|
+
if (mode === 'auto' && (opts?.instrumentLog ?? !['0', 'off', 'false', 'no'].includes((process.env.WT_INSTRUMENT_LOG || '').trim().toLowerCase()))) {
|
|
356
|
+
console.info(`[watchtower] instrumented: ${names.map((n) => n.replace('@opentelemetry/instrumentation-', '')).join(', ') || 'nothing (no supported package found)'}`);
|
|
357
|
+
}
|
|
296
358
|
}
|
|
297
359
|
if (process.env.WT_PROFILING === '1') {
|
|
298
360
|
const attrs = c.resource.attributes;
|
|
@@ -304,16 +366,20 @@ function init(opts) {
|
|
|
304
366
|
}).catch(() => undefined); // best effort: never break init
|
|
305
367
|
}
|
|
306
368
|
if ((opts || {}).captureProcessErrors !== false) {
|
|
307
|
-
|
|
369
|
+
const offErrors = (0, errors_js_1.installProcessHandlers)(() => c.forceFlush());
|
|
370
|
+
const offSignals = (0, shutdown_js_1.installSignalFlush)(() => c.forceFlush());
|
|
371
|
+
c.uninstall = () => { offErrors(); offSignals(); };
|
|
308
372
|
}
|
|
309
373
|
return c;
|
|
310
374
|
}
|
|
311
375
|
function need() {
|
|
312
|
-
if (!current)
|
|
376
|
+
if (!state.current)
|
|
313
377
|
throw new Error('watchtower.init() has not been called');
|
|
314
|
-
return current;
|
|
378
|
+
return state.current;
|
|
315
379
|
}
|
|
316
|
-
const
|
|
380
|
+
const checkin = (slug, fn) => need().checkin(slug, fn);
|
|
381
|
+
exports.checkin = checkin;
|
|
382
|
+
const flush = () => (state.current ? state.current.forceFlush() : Promise.resolve());
|
|
317
383
|
exports.flush = flush;
|
|
318
384
|
const log = (l, b, a) => need().log(l, b, a);
|
|
319
385
|
exports.log = log;
|
|
@@ -327,5 +393,5 @@ const evaluateResume = (d, t, s, prev, o) => need().evaluateResume(d, t, s, prev
|
|
|
327
393
|
exports.evaluateResume = evaluateResume;
|
|
328
394
|
const observe = (n, f) => need().observe(n, f);
|
|
329
395
|
exports.observe = observe;
|
|
330
|
-
const shutdown = () => (current ? current.shutdown() : Promise.resolve());
|
|
396
|
+
const shutdown = () => (state.current ? state.current.shutdown() : Promise.resolve());
|
|
331
397
|
exports.shutdown = shutdown;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
interface Providers {
|
|
2
|
+
tracerProvider: unknown;
|
|
3
|
+
meterProvider?: unknown;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Register whichever of the OTel instrumentations is installed (http, undici, pg, ioredis; with `auto` also mysql2) and,
|
|
7
|
+
* with `auto`, patch postgres.js and better-sqlite3 when installed. Returns the names that were enabled.
|
|
8
|
+
*/
|
|
9
|
+
export declare function instrumentAll(p: Providers, selfUrls: string[], auto?: boolean): string[];
|
|
10
|
+
export {};
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.instrumentAll = instrumentAll;
|
|
4
|
+
// Auto-instrumentation: node:http, fetch (undici), pg and ioredis, each only when its optional peer package is installed.
|
|
5
|
+
const load_js_1 = require("./load.js");
|
|
6
|
+
const sqlite_js_1 = require("./integrations/sqlite.js");
|
|
7
|
+
const postgres_js_1 = require("./integrations/postgres.js");
|
|
8
|
+
const ENTRIES = [
|
|
9
|
+
{
|
|
10
|
+
pkg: '@opentelemetry/instrumentation-http', ctor: 'HttpInstrumentation',
|
|
11
|
+
// our own exports and API calls must not become spans
|
|
12
|
+
opts: (self) => ({ ignoreOutgoingRequestHook: (r) => self.some((u) => (r.hostname || r.host) === u.hostname && String(r.port ?? (u.protocol === 'https:' ? 443 : 80)) === String(u.port || (u.protocol === 'https:' ? 443 : 80))) }),
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
pkg: '@opentelemetry/instrumentation-undici', ctor: 'UndiciInstrumentation',
|
|
16
|
+
opts: (self) => ({ ignoreRequestHook: (r) => self.some((u) => r.origin === u.origin) }),
|
|
17
|
+
},
|
|
18
|
+
{ pkg: '@opentelemetry/instrumentation-pg', ctor: 'PgInstrumentation', opts: () => ({}) },
|
|
19
|
+
{ pkg: '@opentelemetry/instrumentation-ioredis', ctor: 'IORedisInstrumentation', opts: () => ({}) },
|
|
20
|
+
{ pkg: '@opentelemetry/instrumentation-mysql2', ctor: 'MySQL2Instrumentation', opts: () => ({}), auto: true },
|
|
21
|
+
];
|
|
22
|
+
/** Drivers patched in place (no OTel package): `instrument: 'auto'` only. Each loads the app's installed copy or is skipped. */
|
|
23
|
+
const DRIVERS = [
|
|
24
|
+
{ name: 'postgres.js', run: () => { (0, load_js_1.load)('postgres'); (0, postgres_js_1.instrumentPostgres)(); } }, // only when `postgres` is installed
|
|
25
|
+
{ name: 'better-sqlite3', run: () => (0, sqlite_js_1.instrumentBetterSqlite3)((0, load_js_1.load)('better-sqlite3')) },
|
|
26
|
+
];
|
|
27
|
+
/**
|
|
28
|
+
* Register whichever of the OTel instrumentations is installed (http, undici, pg, ioredis; with `auto` also mysql2) and,
|
|
29
|
+
* with `auto`, patch postgres.js and better-sqlite3 when installed. Returns the names that were enabled.
|
|
30
|
+
*/
|
|
31
|
+
function instrumentAll(p, selfUrls, auto = false) {
|
|
32
|
+
let registerInstrumentations;
|
|
33
|
+
try {
|
|
34
|
+
({ registerInstrumentations } = (0, load_js_1.load)('@opentelemetry/instrumentation'));
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return auto ? patchDrivers([]) : []; // optional peer not installed
|
|
38
|
+
}
|
|
39
|
+
const self = selfUrls.flatMap((u) => { try {
|
|
40
|
+
return [new URL(u)];
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return [];
|
|
44
|
+
} });
|
|
45
|
+
const found = [];
|
|
46
|
+
const names = [];
|
|
47
|
+
for (const e of ENTRIES) {
|
|
48
|
+
if (e.auto && !auto)
|
|
49
|
+
continue;
|
|
50
|
+
try {
|
|
51
|
+
const Ctor = (0, load_js_1.load)(e.pkg)[e.ctor];
|
|
52
|
+
if (Ctor) {
|
|
53
|
+
found.push(new Ctor(e.opts(self)));
|
|
54
|
+
names.push(e.pkg);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// package not installed (or failed to construct): skip it, never break init
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (found.length)
|
|
62
|
+
registerInstrumentations({ instrumentations: found, tracerProvider: p.tracerProvider, meterProvider: p.meterProvider });
|
|
63
|
+
return auto ? patchDrivers(names) : names;
|
|
64
|
+
}
|
|
65
|
+
function patchDrivers(names) {
|
|
66
|
+
for (const d of DRIVERS) {
|
|
67
|
+
try {
|
|
68
|
+
d.run();
|
|
69
|
+
names.push(d.name);
|
|
70
|
+
}
|
|
71
|
+
catch { /* driver not installed */ }
|
|
72
|
+
}
|
|
73
|
+
return names;
|
|
74
|
+
}
|
|
@@ -5,6 +5,9 @@ export interface DbOptions {
|
|
|
5
5
|
/** Flag a statement run more than this many times under one parent span. Default:
|
|
6
6
|
* `WATCHTOWER_DB_N_PLUS_ONE` or 5; 0 turns it off. */
|
|
7
7
|
nPlusOne?: number;
|
|
8
|
+
/** Record a query only when a span is active (a request, job or `observe` span). Default true (`WATCHTOWER_DB_REQUIRE_PARENT=0`
|
|
9
|
+
* turns it off): startup DDL, migrations and polling loops run outside any span and would otherwise each become a root trace. */
|
|
10
|
+
requireParent?: boolean;
|
|
8
11
|
}
|
|
9
12
|
export interface DbOp {
|
|
10
13
|
/** `db.system.name`, e.g. `sqlite`. */
|
|
@@ -76,6 +76,10 @@ function repeats(parent, key) {
|
|
|
76
76
|
}
|
|
77
77
|
/** A CLIENT span for `op` (not made active): summary, hash, N+1 hint. Prefer `traced`. */
|
|
78
78
|
function startDbSpan(op, opts, parent, startTime) {
|
|
79
|
+
const p = api_1.trace.getSpan(parent)?.spanContext();
|
|
80
|
+
const requireParent = opts?.requireParent ?? !['0', 'false', 'off', 'no'].includes((process.env.WATCHTOWER_DB_REQUIRE_PARENT || '').trim().toLowerCase());
|
|
81
|
+
if (requireParent && !(p && api_1.trace.isSpanContextValid(p)))
|
|
82
|
+
return api_1.trace.wrapSpanContext(api_1.INVALID_SPAN_CONTEXT); // non-recording: nothing is exported
|
|
79
83
|
const attrs = { 'db.system.name': op.system, ...op.attributes };
|
|
80
84
|
let { operation, collection } = op;
|
|
81
85
|
let name = [operation, collection].filter(Boolean).join(' ') || op.system;
|
|
@@ -98,7 +102,6 @@ function startDbSpan(op, opts, parent, startTime) {
|
|
|
98
102
|
attrs['db.collection.name'] = collection;
|
|
99
103
|
if (op.namespace)
|
|
100
104
|
attrs['db.namespace'] = op.namespace;
|
|
101
|
-
const p = api_1.trace.getSpan(parent)?.spanContext();
|
|
102
105
|
const max = threshold(opts);
|
|
103
106
|
if (p && max > 0) {
|
|
104
107
|
const n = repeats(p.spanId, key);
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import type { Span } from '@opentelemetry/api';
|
|
1
|
+
import type { Attributes, Span } from '@opentelemetry/api';
|
|
2
2
|
type Obj = Record<string, unknown>;
|
|
3
|
+
export declare const obj: (v: unknown) => Obj;
|
|
4
|
+
export declare const s: (v: unknown) => string;
|
|
5
|
+
export declare function start(name: string, attrs: Attributes, parent?: Span, startTime?: number): Span;
|
|
3
6
|
/** The part of LlamaIndex.TS `Settings.callbackManager` used here. */
|
|
4
7
|
export interface CallbackManagerLike {
|
|
5
8
|
on(event: string, handler: (e: {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.AgentsProcessor = void 0;
|
|
3
|
+
exports.AgentsProcessor = exports.s = exports.obj = void 0;
|
|
4
|
+
exports.start = start;
|
|
4
5
|
exports.instrumentLlamaIndex = instrumentLlamaIndex;
|
|
5
6
|
exports.instrumentOpenAIAgents = instrumentOpenAIAgents;
|
|
6
7
|
// GenAI spans for frameworks with their own hooks: LlamaIndex.TS (callback manager events) and the OpenAI Agents
|
|
@@ -10,41 +11,43 @@ exports.instrumentOpenAIAgents = instrumentOpenAIAgents;
|
|
|
10
11
|
const api_1 = require("@opentelemetry/api");
|
|
11
12
|
const genai_stream_js_1 = require("./genai-stream.js");
|
|
12
13
|
const obj = (v) => (v && typeof v === 'object' ? v : {});
|
|
14
|
+
exports.obj = obj;
|
|
13
15
|
const s = (v) => (typeof v === 'string' ? v : '');
|
|
14
|
-
|
|
16
|
+
exports.s = s;
|
|
17
|
+
function start(name, attrs, parent, startTime) {
|
|
15
18
|
const ctx = parent ? api_1.trace.setSpan(api_1.context.active(), parent) : api_1.context.active();
|
|
16
19
|
const clean = Object.fromEntries(Object.entries(attrs).filter(([, v]) => v !== undefined && v !== ''));
|
|
17
|
-
return api_1.trace.getTracer('watchtower').startSpan(name, { kind: api_1.SpanKind.INTERNAL, attributes: clean }, ctx);
|
|
20
|
+
return api_1.trace.getTracer('watchtower').startSpan(name, { kind: api_1.SpanKind.INTERNAL, attributes: clean, ...(startTime !== undefined ? { startTime } : {}) }, ctx);
|
|
18
21
|
}
|
|
19
22
|
/** instrumentLlamaIndex(Settings) (from `llamaindex`): llm-start/stream/end and tool call events as spans. */
|
|
20
23
|
function instrumentLlamaIndex(settings) {
|
|
21
24
|
const open = new Map(), t0 = new Map();
|
|
22
25
|
const cm = settings.callbackManager;
|
|
23
26
|
cm.on('llm-start', (e) => {
|
|
24
|
-
const id = s(obj(e.detail).id);
|
|
27
|
+
const id = (0, exports.s)((0, exports.obj)(e.detail).id);
|
|
25
28
|
open.set(id, start('chat', { 'gen_ai.operation.name': 'chat', 'gen_ai.system': 'llama_index' }));
|
|
26
29
|
t0.set(id, Date.now());
|
|
27
30
|
});
|
|
28
31
|
cm.on('llm-stream', (e) => {
|
|
29
|
-
const id = s(obj(e.detail).id), at = t0.get(id);
|
|
32
|
+
const id = (0, exports.s)((0, exports.obj)(e.detail).id), at = t0.get(id);
|
|
30
33
|
if (at !== undefined) {
|
|
31
34
|
t0.delete(id);
|
|
32
35
|
open.get(id)?.setAttribute('gen_ai.response.time_to_first_chunk', (Date.now() - at) / 1000);
|
|
33
36
|
}
|
|
34
37
|
});
|
|
35
38
|
cm.on('llm-end', (e) => {
|
|
36
|
-
const d = obj(e.detail), id = s(d.id), span = open.get(id);
|
|
39
|
+
const d = (0, exports.obj)(e.detail), id = (0, exports.s)(d.id), span = open.get(id);
|
|
37
40
|
if (!span)
|
|
38
41
|
return;
|
|
39
42
|
open.delete(id);
|
|
40
43
|
t0.delete(id);
|
|
41
|
-
const res = obj(d.response);
|
|
44
|
+
const res = (0, exports.obj)(d.response);
|
|
42
45
|
const x = (0, genai_stream_js_1.extract)(res.raw);
|
|
43
|
-
const msg = obj(res.message), opts = obj(msg.options);
|
|
46
|
+
const msg = (0, exports.obj)(res.message), opts = (0, exports.obj)(msg.options);
|
|
44
47
|
for (const tc of Array.isArray(opts.toolCall) ? opts.toolCall : []) {
|
|
45
|
-
const name = s(obj(tc).name);
|
|
48
|
+
const name = (0, exports.s)((0, exports.obj)(tc).name);
|
|
46
49
|
if (name)
|
|
47
|
-
x.tools.push([name, s(obj(tc).id) || undefined]);
|
|
50
|
+
x.tools.push([name, (0, exports.s)((0, exports.obj)(tc).id) || undefined]);
|
|
48
51
|
}
|
|
49
52
|
if (x.model)
|
|
50
53
|
span.updateName(`chat ${x.model}`);
|
|
@@ -52,7 +55,7 @@ function instrumentLlamaIndex(settings) {
|
|
|
52
55
|
span.end();
|
|
53
56
|
});
|
|
54
57
|
cm.on('llm-tool-call', (e) => {
|
|
55
|
-
const name = s(obj(obj(e.detail).toolCall).name);
|
|
58
|
+
const name = (0, exports.s)((0, exports.obj)((0, exports.obj)(e.detail).toolCall).name);
|
|
56
59
|
start(`execute_tool ${name}`.trim(), { 'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': name }).end();
|
|
57
60
|
});
|
|
58
61
|
return { open };
|
|
@@ -63,27 +66,27 @@ class AgentsProcessor {
|
|
|
63
66
|
async onTraceStart() { }
|
|
64
67
|
async onTraceEnd() { }
|
|
65
68
|
async onSpanStart(sp) {
|
|
66
|
-
const d = obj(sp.spanData), type = s(d.type), parent = sp.parentId ? this.open.get(sp.parentId) : undefined;
|
|
69
|
+
const d = (0, exports.obj)(sp.spanData), type = (0, exports.s)(d.type), parent = sp.parentId ? this.open.get(sp.parentId) : undefined;
|
|
67
70
|
let name = type || 'agents';
|
|
68
71
|
let attrs = {};
|
|
69
72
|
if (type === 'agent') {
|
|
70
|
-
name = `invoke_agent ${s(d.name)}`.trim();
|
|
71
|
-
attrs = { 'gen_ai.operation.name': 'invoke_agent', 'gen_ai.agent.name': s(d.name), 'gen_ai.system': 'openai' };
|
|
73
|
+
name = `invoke_agent ${(0, exports.s)(d.name)}`.trim();
|
|
74
|
+
attrs = { 'gen_ai.operation.name': 'invoke_agent', 'gen_ai.agent.name': (0, exports.s)(d.name), 'gen_ai.system': 'openai' };
|
|
72
75
|
}
|
|
73
76
|
else if (type === 'generation' || type === 'response') {
|
|
74
|
-
name = `chat ${s(d.model)}`.trim();
|
|
75
|
-
attrs = { 'gen_ai.operation.name': 'chat', 'gen_ai.system': 'openai', 'gen_ai.request.model': s(d.model) };
|
|
77
|
+
name = `chat ${(0, exports.s)(d.model)}`.trim();
|
|
78
|
+
attrs = { 'gen_ai.operation.name': 'chat', 'gen_ai.system': 'openai', 'gen_ai.request.model': (0, exports.s)(d.model) };
|
|
76
79
|
}
|
|
77
80
|
else if (type === 'function') {
|
|
78
|
-
name = `execute_tool ${s(d.name)}`.trim();
|
|
79
|
-
attrs = { 'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': s(d.name) };
|
|
81
|
+
name = `execute_tool ${(0, exports.s)(d.name)}`.trim();
|
|
82
|
+
attrs = { 'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': (0, exports.s)(d.name) };
|
|
80
83
|
}
|
|
81
84
|
else if (type === 'handoff') {
|
|
82
|
-
attrs = { 'gen_ai.operation.name': 'handoff', 'gen_ai.handoff.from_agent': s(d.from_agent), 'gen_ai.handoff.to_agent': s(d.to_agent) };
|
|
85
|
+
attrs = { 'gen_ai.operation.name': 'handoff', 'gen_ai.handoff.from_agent': (0, exports.s)(d.from_agent), 'gen_ai.handoff.to_agent': (0, exports.s)(d.to_agent) };
|
|
83
86
|
}
|
|
84
87
|
else if (type === 'guardrail') {
|
|
85
|
-
name = `guardrail ${s(d.name)}`.trim();
|
|
86
|
-
attrs = { 'gen_ai.guardrail.name': s(d.name) };
|
|
88
|
+
name = `guardrail ${(0, exports.s)(d.name)}`.trim();
|
|
89
|
+
attrs = { 'gen_ai.guardrail.name': (0, exports.s)(d.name) };
|
|
87
90
|
}
|
|
88
91
|
if (sp.spanId)
|
|
89
92
|
this.open.set(sp.spanId, start(name, attrs, parent));
|
|
@@ -93,10 +96,10 @@ class AgentsProcessor {
|
|
|
93
96
|
if (!span || !sp.spanId)
|
|
94
97
|
return;
|
|
95
98
|
this.open.delete(sp.spanId);
|
|
96
|
-
const d = obj(sp.spanData);
|
|
99
|
+
const d = (0, exports.obj)(sp.spanData);
|
|
97
100
|
if (d.type === 'generation') {
|
|
98
|
-
const u = obj(d.usage);
|
|
99
|
-
(0, genai_stream_js_1.applyExtract)(span, { model: s(d.model) || undefined, input: u.input_tokens, output: u.output_tokens, reasons: [], tools: [] });
|
|
101
|
+
const u = (0, exports.obj)(d.usage);
|
|
102
|
+
(0, genai_stream_js_1.applyExtract)(span, { model: (0, exports.s)(d.model) || undefined, input: u.input_tokens, output: u.output_tokens, reasons: [], tools: [] });
|
|
100
103
|
}
|
|
101
104
|
else if (d.type === 'response') {
|
|
102
105
|
(0, genai_stream_js_1.applyExtract)(span, (0, genai_stream_js_1.extract)(d._response ?? d.response));
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Span } from '@opentelemetry/api';
|
|
2
|
+
type Fn = (...a: unknown[]) => unknown;
|
|
3
|
+
/**
|
|
4
|
+
* langchainCallbackHandler(): pass it as `callbacks: [handler]` to a LangChain.js model, chain or LangGraph.js graph
|
|
5
|
+
* (`graph.invoke(input, { callbacks: [handler] })`). LangGraph nodes (chains whose metadata names `langgraph_node`)
|
|
6
|
+
* and the graph itself become `invoke_agent` spans, tools `execute_tool`, model calls `chat`, nested as they run;
|
|
7
|
+
* plain LCEL chains get no span. Returns a plain handler object (LangChain wraps it) with the open spans for tests.
|
|
8
|
+
*/
|
|
9
|
+
export declare function langchainCallbackHandler(): Record<string, unknown> & {
|
|
10
|
+
open: Map<string, Span>;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* instrumentVercelAI({ generateText, streamText }) with the functions from `ai`: returns wrapped copies (ES module
|
|
14
|
+
* exports cannot be patched). Pass `agentName` to name the run (default: the `experimental_telemetry.functionId`, else `agent`).
|
|
15
|
+
* const { generateText } = instrumentVercelAI({ generateText: ai.generateText });
|
|
16
|
+
* A multi-step loop (`maxSteps` / `stopWhen`) gives one `invoke_agent` span with a `chat` span per step and an
|
|
17
|
+
* `execute_tool` span per tool call.
|
|
18
|
+
*/
|
|
19
|
+
export declare function instrumentVercelAI<T extends {
|
|
20
|
+
generateText?: Fn;
|
|
21
|
+
streamText?: Fn;
|
|
22
|
+
}>(ai: T, agentName?: string): T;
|
|
23
|
+
/** instrumentMastra(agent): wraps `agent.generate` and `agent.stream` of a Mastra `Agent`; the run is named after the agent. Returns the agent. */
|
|
24
|
+
export declare function instrumentMastra<T>(agent: T): T;
|
|
25
|
+
export {};
|