@watchtowerobservability/node 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +304 -2
- package/dist/cjs/appsec.d.ts +95 -0
- package/dist/cjs/appsec.js +206 -0
- package/dist/cjs/buffer.d.ts +49 -0
- package/dist/cjs/buffer.js +134 -0
- package/dist/cjs/errors.d.ts +17 -0
- package/dist/cjs/errors.js +55 -0
- package/dist/cjs/index.d.ts +123 -0
- package/dist/cjs/index.js +315 -0
- package/dist/cjs/integrations/amqp.d.ts +13 -0
- package/dist/cjs/integrations/amqp.js +96 -0
- package/dist/cjs/integrations/azure-functions.d.ts +18 -0
- package/dist/cjs/integrations/azure-functions.js +97 -0
- package/dist/cjs/integrations/bullmq.d.ts +19 -0
- package/dist/cjs/integrations/bullmq.js +83 -0
- package/dist/cjs/integrations/cache.d.ts +13 -0
- package/dist/cjs/integrations/cache.js +51 -0
- package/dist/cjs/integrations/db.d.ts +62 -0
- package/dist/cjs/integrations/db.js +288 -0
- package/dist/cjs/integrations/duckdb.d.ts +22 -0
- package/dist/cjs/integrations/duckdb.js +66 -0
- package/dist/cjs/integrations/express.d.ts +25 -0
- package/dist/cjs/integrations/express.js +58 -0
- package/dist/cjs/integrations/fastify.d.ts +6 -0
- package/dist/cjs/integrations/fastify.js +44 -0
- package/dist/cjs/integrations/genai-agents.d.ts +36 -0
- package/dist/cjs/integrations/genai-agents.js +124 -0
- package/dist/cjs/integrations/genai-stream.d.ts +19 -0
- package/dist/cjs/integrations/genai-stream.js +135 -0
- package/dist/cjs/integrations/genai.d.ts +18 -0
- package/dist/cjs/integrations/genai.js +150 -0
- package/dist/cjs/integrations/graphile-worker.d.ts +22 -0
- package/dist/cjs/integrations/graphile-worker.js +50 -0
- package/dist/cjs/integrations/koa.d.ts +19 -0
- package/dist/cjs/integrations/koa.js +34 -0
- package/dist/cjs/integrations/lambda.d.ts +11 -0
- package/dist/cjs/integrations/lambda.js +68 -0
- package/dist/cjs/integrations/level.d.ts +18 -0
- package/dist/cjs/integrations/level.js +71 -0
- package/dist/cjs/integrations/lmdb.d.ts +26 -0
- package/dist/cjs/integrations/lmdb.js +78 -0
- package/dist/cjs/integrations/logs.d.ts +10 -0
- package/dist/cjs/integrations/logs.js +77 -0
- package/dist/cjs/integrations/messaging.d.ts +17 -0
- package/dist/cjs/integrations/messaging.js +73 -0
- package/dist/cjs/integrations/mqtt.d.ts +7 -0
- package/dist/cjs/integrations/mqtt.js +41 -0
- package/dist/cjs/integrations/nats.d.ts +17 -0
- package/dist/cjs/integrations/nats.js +73 -0
- package/dist/cjs/integrations/nestjs.d.ts +16 -0
- package/dist/cjs/integrations/nestjs.js +30 -0
- package/dist/cjs/integrations/next.d.ts +11 -0
- package/dist/cjs/integrations/next.js +59 -0
- package/dist/cjs/integrations/orm.d.ts +47 -0
- package/dist/cjs/integrations/orm.js +93 -0
- package/dist/cjs/integrations/pg-boss.d.ts +8 -0
- package/dist/cjs/integrations/pg-boss.js +63 -0
- package/dist/cjs/integrations/pools.d.ts +41 -0
- package/dist/cjs/integrations/pools.js +94 -0
- package/dist/cjs/integrations/postgres.d.ts +12 -0
- package/dist/cjs/integrations/postgres.js +92 -0
- package/dist/cjs/integrations/prisma.d.ts +3 -0
- package/dist/cjs/integrations/prisma.js +33 -0
- package/dist/cjs/integrations/sqlite.d.ts +36 -0
- package/dist/cjs/integrations/sqlite.js +125 -0
- package/dist/cjs/integrations/util.d.ts +9 -0
- package/dist/cjs/integrations/util.js +37 -0
- package/dist/cjs/jest.d.ts +21 -0
- package/dist/cjs/jest.js +45 -0
- package/dist/cjs/load.d.ts +1 -0
- package/dist/cjs/load.js +11 -0
- package/dist/cjs/metrics.d.ts +10 -0
- package/dist/cjs/metrics.js +81 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/payload.d.ts +42 -0
- package/dist/cjs/payload.js +145 -0
- package/dist/cjs/pprof.d.ts +6 -0
- package/dist/cjs/pprof.js +88 -0
- package/dist/cjs/probes.d.ts +97 -0
- package/dist/cjs/probes.js +422 -0
- package/dist/cjs/profiling.d.ts +84 -0
- package/dist/cjs/profiling.js +263 -0
- package/dist/cjs/reporter-core.d.ts +32 -0
- package/dist/cjs/reporter-core.js +102 -0
- package/dist/cjs/sql.d.ts +19 -0
- package/dist/cjs/sql.js +332 -0
- package/dist/cjs/test-plan.d.ts +102 -0
- package/dist/cjs/test-plan.js +352 -0
- package/dist/cjs/vitest.d.ts +27 -0
- package/dist/cjs/vitest.js +52 -0
- package/dist/esm/appsec.d.ts +95 -0
- package/dist/esm/appsec.js +193 -0
- package/dist/esm/buffer.d.ts +49 -0
- package/dist/esm/buffer.js +92 -0
- package/dist/esm/errors.d.ts +17 -0
- package/dist/esm/errors.js +50 -0
- package/dist/esm/index.d.ts +123 -0
- package/dist/esm/index.js +264 -0
- package/dist/esm/integrations/amqp.d.ts +13 -0
- package/dist/esm/integrations/amqp.js +92 -0
- package/dist/esm/integrations/azure-functions.d.ts +18 -0
- package/dist/esm/integrations/azure-functions.js +60 -0
- package/dist/esm/integrations/bullmq.d.ts +19 -0
- package/dist/esm/integrations/bullmq.js +79 -0
- package/dist/esm/integrations/cache.d.ts +13 -0
- package/dist/esm/integrations/cache.js +46 -0
- package/dist/esm/integrations/db.d.ts +62 -0
- package/dist/esm/integrations/db.js +239 -0
- package/dist/esm/integrations/duckdb.d.ts +22 -0
- package/dist/esm/integrations/duckdb.js +29 -0
- package/dist/esm/integrations/express.d.ts +25 -0
- package/dist/esm/integrations/express.js +54 -0
- package/dist/esm/integrations/fastify.d.ts +6 -0
- package/dist/esm/integrations/fastify.js +41 -0
- package/dist/esm/integrations/genai-agents.d.ts +36 -0
- package/dist/esm/integrations/genai-agents.js +118 -0
- package/dist/esm/integrations/genai-stream.d.ts +19 -0
- package/dist/esm/integrations/genai-stream.js +130 -0
- package/dist/esm/integrations/genai.d.ts +18 -0
- package/dist/esm/integrations/genai.js +138 -0
- package/dist/esm/integrations/graphile-worker.d.ts +22 -0
- package/dist/esm/integrations/graphile-worker.js +46 -0
- package/dist/esm/integrations/koa.d.ts +19 -0
- package/dist/esm/integrations/koa.js +31 -0
- package/dist/esm/integrations/lambda.d.ts +11 -0
- package/dist/esm/integrations/lambda.js +32 -0
- package/dist/esm/integrations/level.d.ts +18 -0
- package/dist/esm/integrations/level.js +35 -0
- package/dist/esm/integrations/lmdb.d.ts +26 -0
- package/dist/esm/integrations/lmdb.js +42 -0
- package/dist/esm/integrations/logs.d.ts +10 -0
- package/dist/esm/integrations/logs.js +40 -0
- package/dist/esm/integrations/messaging.d.ts +17 -0
- package/dist/esm/integrations/messaging.js +64 -0
- package/dist/esm/integrations/mqtt.d.ts +7 -0
- package/dist/esm/integrations/mqtt.js +38 -0
- package/dist/esm/integrations/nats.d.ts +17 -0
- package/dist/esm/integrations/nats.js +70 -0
- package/dist/esm/integrations/nestjs.d.ts +16 -0
- package/dist/esm/integrations/nestjs.js +26 -0
- package/dist/esm/integrations/next.d.ts +11 -0
- package/dist/esm/integrations/next.js +22 -0
- package/dist/esm/integrations/orm.d.ts +47 -0
- package/dist/esm/integrations/orm.js +86 -0
- package/dist/esm/integrations/pg-boss.d.ts +8 -0
- package/dist/esm/integrations/pg-boss.js +60 -0
- package/dist/esm/integrations/pools.d.ts +41 -0
- package/dist/esm/integrations/pools.js +89 -0
- package/dist/esm/integrations/postgres.d.ts +12 -0
- package/dist/esm/integrations/postgres.js +55 -0
- package/dist/esm/integrations/prisma.d.ts +3 -0
- package/dist/esm/integrations/prisma.js +30 -0
- package/dist/esm/integrations/sqlite.d.ts +36 -0
- package/dist/esm/integrations/sqlite.js +84 -0
- package/dist/esm/integrations/util.d.ts +9 -0
- package/dist/esm/integrations/util.js +32 -0
- package/dist/esm/jest.d.ts +21 -0
- package/dist/esm/jest.js +41 -0
- package/dist/esm/load.d.ts +1 -0
- package/dist/esm/load.js +5 -0
- package/dist/esm/metrics.d.ts +10 -0
- package/dist/esm/metrics.js +44 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/payload.d.ts +42 -0
- package/dist/esm/payload.js +131 -0
- package/dist/esm/pprof.d.ts +6 -0
- package/dist/esm/pprof.js +85 -0
- package/dist/esm/probes.d.ts +97 -0
- package/dist/esm/probes.js +412 -0
- package/dist/esm/profiling.d.ts +84 -0
- package/dist/esm/profiling.js +251 -0
- package/dist/esm/reporter-core.d.ts +32 -0
- package/dist/esm/reporter-core.js +63 -0
- package/dist/esm/sql.d.ts +19 -0
- package/dist/esm/sql.js +324 -0
- package/dist/esm/test-plan.d.ts +102 -0
- package/dist/esm/test-plan.js +336 -0
- package/dist/esm/vitest.d.ts +27 -0
- package/dist/esm/vitest.js +48 -0
- package/package.json +379 -4
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// App and API Protection in-app WAF (W52). `startAppsec()` polls the project's ruleset from the
|
|
2
|
+
// edge (`GET /v1/appsec/ruleset`, an ingestion key with scope `probes:read`, ETag, every `poll_s`):
|
|
3
|
+
// mode, enabled pattern rules and the denylist. `appsecMiddleware()` (Express, Connect, any
|
|
4
|
+
// `(req, res, next)` stack) or `handleRequest(req, res)` (plain `http`, Fastify `onRequest`, Koa)
|
|
5
|
+
// inspects each request before the handler: path, query, User-Agent, other header values and an
|
|
6
|
+
// already parsed body, percent-decoded once, against the same rules the worker runs over stored
|
|
7
|
+
// spans. A match is recorded on the active server span (`appsec.event`, `appsec.rule_id`,
|
|
8
|
+
// `appsec.category`, `appsec.blocked`); in `block` mode, and for denylisted IPs or users in any
|
|
9
|
+
// mode, the request is answered with 403 problem+json and the handler never runs.
|
|
10
|
+
// Nothing is fetched or inspected until `startAppsec` is called; with no ruleset yet every
|
|
11
|
+
// request passes. Rules are matched case-insensitively; a rule that does not compile is skipped.
|
|
12
|
+
import { trace } from '@opentelemetry/api';
|
|
13
|
+
/** Rule id recorded for denylist blocks (the worker reports them as their own signal). */
|
|
14
|
+
export const DENYLIST = 'denylist';
|
|
15
|
+
const BODY_MAX = 64 * 1024;
|
|
16
|
+
let mode = 'monitor';
|
|
17
|
+
let compiled = [];
|
|
18
|
+
let denyIp = new Map();
|
|
19
|
+
let denyUser = new Map();
|
|
20
|
+
const expiryMap = (list) => new Map((list || []).map((e) => [e.value, e.expires_at ? e.expires_at * 1000 : Infinity]));
|
|
21
|
+
/** Replace the active ruleset (what the poller calls; tests may call it directly). */
|
|
22
|
+
export function setRuleset(r) {
|
|
23
|
+
mode = r?.mode === 'block' ? 'block' : 'monitor';
|
|
24
|
+
compiled = [];
|
|
25
|
+
for (const rule of r?.rules || []) {
|
|
26
|
+
try {
|
|
27
|
+
compiled.push({ rule, re: new RegExp(rule.pattern, 'i') });
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// not portable to JavaScript: skipped here, the worker still evaluates it
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
denyIp = expiryMap(r?.denylist?.ip);
|
|
34
|
+
denyUser = expiryMap(r?.denylist?.user);
|
|
35
|
+
}
|
|
36
|
+
const listed = (m, v, now) => !!v && (m.get(v) ?? 0) > now;
|
|
37
|
+
/** Whether `id` is on the user denylist (call after authentication to refuse a listed user). */
|
|
38
|
+
export const isUserBlocked = (id, now = Date.now()) => listed(denyUser, id, now);
|
|
39
|
+
/** Judge one request. Denylisted callers are blocked in either mode. */
|
|
40
|
+
export function inspect(req, now = Date.now()) {
|
|
41
|
+
if (listed(denyIp, req.ip, now) || listed(denyUser, req.user, now)) {
|
|
42
|
+
return { action: 'block', rule_id: DENYLIST, category: 'abuse' };
|
|
43
|
+
}
|
|
44
|
+
for (const { rule, re } of compiled) {
|
|
45
|
+
for (const f of rule.fields) {
|
|
46
|
+
const v = req[f];
|
|
47
|
+
if (!v)
|
|
48
|
+
continue;
|
|
49
|
+
const m = re.exec(v);
|
|
50
|
+
if (m) {
|
|
51
|
+
return { action: mode === 'block' ? 'block' : 'monitor', rule_id: rule.id, category: rule.category, match: m[0].slice(0, 200) };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return { action: 'allow' };
|
|
56
|
+
}
|
|
57
|
+
const decode = (s, form = false) => {
|
|
58
|
+
const x = form ? s.replace(/\+/g, ' ') : s;
|
|
59
|
+
try {
|
|
60
|
+
return decodeURIComponent(x);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return x;
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
const first = (v) => (Array.isArray(v) ? v[0] : v);
|
|
67
|
+
/** Client IP: first `X-Forwarded-For` hop, else `X-Real-IP`, else the socket peer.
|
|
68
|
+
* shortcut: trusts the forwarding headers; run behind a proxy that overwrites them. */
|
|
69
|
+
export function clientIp(req) {
|
|
70
|
+
const xff = first(req.headers?.['x-forwarded-for']);
|
|
71
|
+
if (xff)
|
|
72
|
+
return xff.split(',')[0].trim();
|
|
73
|
+
return first(req.headers?.['x-real-ip']) || req.socket?.remoteAddress?.replace(/^::ffff:/, '');
|
|
74
|
+
}
|
|
75
|
+
/** Decode a Node request into the inspector's fields. */
|
|
76
|
+
export function fromNode(req, user) {
|
|
77
|
+
const url = req.originalUrl || req.url || '/';
|
|
78
|
+
const q = url.indexOf('?');
|
|
79
|
+
const headers = Object.entries(req.headers || {})
|
|
80
|
+
.filter(([k]) => k !== 'user-agent')
|
|
81
|
+
.map(([, v]) => (Array.isArray(v) ? v.join(', ') : v || ''))
|
|
82
|
+
.join('\n');
|
|
83
|
+
let body;
|
|
84
|
+
if (typeof req.body === 'string')
|
|
85
|
+
body = req.body.slice(0, BODY_MAX);
|
|
86
|
+
else if (req.body && typeof req.body === 'object') {
|
|
87
|
+
try {
|
|
88
|
+
body = JSON.stringify(req.body).slice(0, BODY_MAX);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
body = undefined;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
path: decode(q < 0 ? url : url.slice(0, q)),
|
|
96
|
+
query: q < 0 ? '' : decode(url.slice(q + 1), true),
|
|
97
|
+
user_agent: first(req.headers?.['user-agent']) || '',
|
|
98
|
+
headers,
|
|
99
|
+
body,
|
|
100
|
+
ip: clientIp(req),
|
|
101
|
+
user,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** Record a decision on the active span. */
|
|
105
|
+
export function annotate(d, ip) {
|
|
106
|
+
const span = trace.getActiveSpan();
|
|
107
|
+
if (!span)
|
|
108
|
+
return;
|
|
109
|
+
if (ip)
|
|
110
|
+
span.setAttribute('appsec.client_ip', ip);
|
|
111
|
+
if (d.action === 'allow')
|
|
112
|
+
return;
|
|
113
|
+
span.setAttributes({
|
|
114
|
+
'appsec.event': true,
|
|
115
|
+
'appsec.rule_id': d.rule_id || '',
|
|
116
|
+
'appsec.category': d.category || '',
|
|
117
|
+
'appsec.blocked': d.action === 'block',
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
const PROBLEM = JSON.stringify({
|
|
121
|
+
type: 'about:blank',
|
|
122
|
+
title: 'Forbidden',
|
|
123
|
+
status: 403,
|
|
124
|
+
detail: 'Request blocked by Watchtower App and API Protection',
|
|
125
|
+
});
|
|
126
|
+
/** Inspect and, when blocked, answer 403. Returns true when the request was answered. */
|
|
127
|
+
export function handleRequest(req, res, user) {
|
|
128
|
+
if (!compiled.length && !denyIp.size && !denyUser.size)
|
|
129
|
+
return false;
|
|
130
|
+
const r = fromNode(req, user);
|
|
131
|
+
const d = inspect(r);
|
|
132
|
+
annotate(d, r.ip);
|
|
133
|
+
if (d.action !== 'block')
|
|
134
|
+
return false;
|
|
135
|
+
res.statusCode = 403;
|
|
136
|
+
res.setHeader?.('content-type', 'application/problem+json');
|
|
137
|
+
res.end(PROBLEM);
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
/** Connect-style middleware; `user(req)` names the caller when the app already knows it. */
|
|
141
|
+
export function appsecMiddleware(o = {}) {
|
|
142
|
+
return (req, res, next) => {
|
|
143
|
+
if (!handleRequest(req, res, o.user?.(req)))
|
|
144
|
+
next();
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
/** Mark the current request as a login attempt; failed ones feed the account-takeover rules. */
|
|
148
|
+
export function trackLogin(login, success) {
|
|
149
|
+
const span = trace.getActiveSpan();
|
|
150
|
+
span?.setAttributes({ 'appsec.login': login, 'appsec.login.success': success });
|
|
151
|
+
if (success)
|
|
152
|
+
span?.setAttribute('appsec.auth', true);
|
|
153
|
+
}
|
|
154
|
+
/** Fetch the ruleset now and keep polling; `stop()` ends the polling. */
|
|
155
|
+
export function startAppsec(o = {}) {
|
|
156
|
+
const env = process.env;
|
|
157
|
+
const base = (o.apiUrl || env.WT_API_URL || 'https://api.watchtower.dev').replace(/\/$/, '');
|
|
158
|
+
const key = o.key || env.WT_INGEST_KEY || '';
|
|
159
|
+
const service = o.service || env.WT_SERVICE || env.OTEL_SERVICE_NAME || 'unknown';
|
|
160
|
+
const f = o.fetch || globalThis.fetch;
|
|
161
|
+
let etag = '';
|
|
162
|
+
let timer;
|
|
163
|
+
let stopped = false;
|
|
164
|
+
let pollS = 30;
|
|
165
|
+
const poll = async () => {
|
|
166
|
+
try {
|
|
167
|
+
const res = await f(`${base}/v1/appsec/ruleset?service=${encodeURIComponent(service)}`, {
|
|
168
|
+
headers: { Authorization: `Bearer ${key}`, ...(etag ? { 'If-None-Match': etag } : {}) },
|
|
169
|
+
});
|
|
170
|
+
if (res.status === 200) {
|
|
171
|
+
const body = (await res.json());
|
|
172
|
+
setRuleset(body);
|
|
173
|
+
etag = res.headers.get('etag') || '';
|
|
174
|
+
pollS = Math.max(10, body.poll_s || 30);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
// keep the last ruleset; try again on the next poll
|
|
179
|
+
}
|
|
180
|
+
if (!stopped) {
|
|
181
|
+
timer = setTimeout(() => void poll(), pollS * 1000);
|
|
182
|
+
timer.unref?.();
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
return {
|
|
186
|
+
ready: poll(),
|
|
187
|
+
stop() {
|
|
188
|
+
stopped = true;
|
|
189
|
+
if (timer)
|
|
190
|
+
clearTimeout(timer);
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { SpanExporter, ReadableSpan } from '@opentelemetry/sdk-trace-base';
|
|
2
|
+
import type { LogRecordExporter, ReadableLogRecord } from '@opentelemetry/sdk-logs';
|
|
3
|
+
import { AggregationTemporality } from '@opentelemetry/sdk-metrics';
|
|
4
|
+
import type { InstrumentType, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
|
|
5
|
+
type Done = Parameters<SpanExporter['export']>[1];
|
|
6
|
+
/** What the spool wraps: a span, log record or metric exporter; `B` is one export batch. */
|
|
7
|
+
export interface BatchExporter<B> {
|
|
8
|
+
export(batch: B, done: Done): void;
|
|
9
|
+
shutdown(): Promise<void>;
|
|
10
|
+
forceFlush?(): Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
export interface SpoolOptions<B = ReadableSpan[]> {
|
|
13
|
+
maxFiles?: number;
|
|
14
|
+
encode?: (batch: B) => Uint8Array;
|
|
15
|
+
}
|
|
16
|
+
/** OTLP/JSON encoders for the three signals. */
|
|
17
|
+
export declare const encodeSpans: (spans: ReadableSpan[]) => Uint8Array;
|
|
18
|
+
export declare const encodeLogs: (logs: ReadableLogRecord[]) => Uint8Array;
|
|
19
|
+
export declare const encodeMetrics: (rm: ResourceMetrics) => Uint8Array;
|
|
20
|
+
export declare class SpoolExporter<B = ReadableSpan[]> implements BatchExporter<B> {
|
|
21
|
+
private readonly inner;
|
|
22
|
+
private readonly dir;
|
|
23
|
+
private readonly url;
|
|
24
|
+
private readonly headers;
|
|
25
|
+
private readonly fetch;
|
|
26
|
+
private readonly maxFiles;
|
|
27
|
+
private readonly encode;
|
|
28
|
+
constructor(inner: BatchExporter<B>, dir: string, url: string, headers: Record<string, string>, fetch: typeof globalThis.fetch, opts?: SpoolOptions<B>);
|
|
29
|
+
private _files;
|
|
30
|
+
replay(): Promise<void>;
|
|
31
|
+
export(batch: B, done: Done): void;
|
|
32
|
+
private _spool;
|
|
33
|
+
shutdown(): Promise<void>;
|
|
34
|
+
forceFlush(): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
/** A spool for log records: same directory rules, `/v1/logs` replay. */
|
|
37
|
+
export declare const spoolLogs: (inner: LogRecordExporter, dir: string, url: string, headers: Record<string, string>, fetch: typeof globalThis.fetch, opts?: {
|
|
38
|
+
maxFiles?: number;
|
|
39
|
+
}) => SpoolExporter<ReadableLogRecord[]>;
|
|
40
|
+
/** A spool for metrics (`/v1/metrics` replay); the reader still asks the real exporter for temporality. */
|
|
41
|
+
export declare class MetricSpool extends SpoolExporter<ResourceMetrics> implements PushMetricExporter {
|
|
42
|
+
private readonly metricInner;
|
|
43
|
+
constructor(metricInner: PushMetricExporter, dir: string, url: string, headers: Record<string, string>, fetch: typeof globalThis.fetch, opts?: {
|
|
44
|
+
maxFiles?: number;
|
|
45
|
+
});
|
|
46
|
+
selectAggregationTemporality(t: InstrumentType): AggregationTemporality;
|
|
47
|
+
forceFlush(): Promise<void>;
|
|
48
|
+
}
|
|
49
|
+
export {};
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Offline buffer: failed OTLP batches (spans, log records or metrics) are spooled to disk, at most
|
|
2
|
+
// `maxFiles` batches per signal directory (oldest dropped), and replayed oldest first before the
|
|
3
|
+
// next export, so they go out as soon as the endpoint is reachable again.
|
|
4
|
+
import * as fs from 'node:fs';
|
|
5
|
+
import * as path from 'node:path';
|
|
6
|
+
import * as crypto from 'node:crypto';
|
|
7
|
+
import { AggregationTemporality } from '@opentelemetry/sdk-metrics';
|
|
8
|
+
import { JsonLogsSerializer, JsonMetricsSerializer, JsonTraceSerializer } from '@opentelemetry/otlp-transformer';
|
|
9
|
+
/** OTLP/JSON encoders for the three signals. */
|
|
10
|
+
export const encodeSpans = (spans) => Buffer.from(JsonTraceSerializer.serializeRequest(spans));
|
|
11
|
+
export const encodeLogs = (logs) => Buffer.from(JsonLogsSerializer.serializeRequest(logs));
|
|
12
|
+
export const encodeMetrics = (rm) => Buffer.from(JsonMetricsSerializer.serializeRequest(rm));
|
|
13
|
+
export class SpoolExporter {
|
|
14
|
+
inner;
|
|
15
|
+
dir;
|
|
16
|
+
url;
|
|
17
|
+
headers;
|
|
18
|
+
fetch;
|
|
19
|
+
maxFiles;
|
|
20
|
+
encode;
|
|
21
|
+
constructor(inner, dir, url, headers, fetch, opts) {
|
|
22
|
+
this.inner = inner;
|
|
23
|
+
this.dir = dir;
|
|
24
|
+
this.url = url;
|
|
25
|
+
this.headers = headers;
|
|
26
|
+
this.fetch = fetch;
|
|
27
|
+
this.maxFiles = (opts && opts.maxFiles) || 500;
|
|
28
|
+
this.encode = (opts && opts.encode) || encodeSpans;
|
|
29
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
30
|
+
}
|
|
31
|
+
_files() {
|
|
32
|
+
return fs.readdirSync(this.dir)
|
|
33
|
+
.map((n) => ({ n, t: fs.statSync(path.join(this.dir, n)).mtimeMs }))
|
|
34
|
+
.sort((a, b) => a.t - b.t || a.n.localeCompare(b.n))
|
|
35
|
+
.map((f) => f.n);
|
|
36
|
+
}
|
|
37
|
+
async replay() {
|
|
38
|
+
for (const n of this._files()) {
|
|
39
|
+
const p = path.join(this.dir, n);
|
|
40
|
+
try {
|
|
41
|
+
const res = await this.fetch(this.url, {
|
|
42
|
+
method: 'POST',
|
|
43
|
+
headers: { ...this.headers, 'Content-Type': 'application/json' },
|
|
44
|
+
body: fs.readFileSync(p),
|
|
45
|
+
});
|
|
46
|
+
if (!res.ok)
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return; // still offline; keep order
|
|
51
|
+
}
|
|
52
|
+
fs.unlinkSync(p);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export(batch, done) {
|
|
56
|
+
void this.replay().then(() => this.inner.export(batch, (r) => {
|
|
57
|
+
if (r.code !== 0)
|
|
58
|
+
this._spool(this.encode(batch));
|
|
59
|
+
done(r);
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
_spool(payload) {
|
|
63
|
+
const files = this._files();
|
|
64
|
+
for (const old of files.slice(0, Math.max(0, files.length - this.maxFiles + 1))) {
|
|
65
|
+
fs.unlinkSync(path.join(this.dir, old)); // drop-oldest
|
|
66
|
+
}
|
|
67
|
+
fs.writeFileSync(path.join(this.dir, `${crypto.randomUUID()}.json`), payload);
|
|
68
|
+
}
|
|
69
|
+
shutdown() {
|
|
70
|
+
return this.inner.shutdown();
|
|
71
|
+
}
|
|
72
|
+
forceFlush() {
|
|
73
|
+
return this.inner.forceFlush ? this.inner.forceFlush() : Promise.resolve();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/** A spool for log records: same directory rules, `/v1/logs` replay. */
|
|
77
|
+
export const spoolLogs = (inner, dir, url, headers, fetch, opts) => new SpoolExporter(inner, dir, url, headers, fetch, { ...opts, encode: encodeLogs });
|
|
78
|
+
/** A spool for metrics (`/v1/metrics` replay); the reader still asks the real exporter for temporality. */
|
|
79
|
+
export class MetricSpool extends SpoolExporter {
|
|
80
|
+
metricInner;
|
|
81
|
+
constructor(metricInner, dir, url, headers, fetch, opts) {
|
|
82
|
+
super(metricInner, dir, url, headers, fetch, { ...opts, encode: encodeMetrics });
|
|
83
|
+
this.metricInner = metricInner;
|
|
84
|
+
}
|
|
85
|
+
selectAggregationTemporality(t) {
|
|
86
|
+
const pick = this.metricInner.selectAggregationTemporality;
|
|
87
|
+
return pick ? pick.call(this.metricInner, t) : AggregationTemporality.CUMULATIVE; // the SDK default
|
|
88
|
+
}
|
|
89
|
+
forceFlush() {
|
|
90
|
+
return this.metricInner.forceFlush();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface CaptureContext {
|
|
2
|
+
/** false for crashes (uncaught); default true. Keys starting with `wt.` are dropped. */
|
|
3
|
+
handled?: boolean;
|
|
4
|
+
attributes?: Record<string, string | number | boolean | undefined>;
|
|
5
|
+
}
|
|
6
|
+
/** The slice of `process` the hooks need; tests pass an EventEmitter. */
|
|
7
|
+
export interface ProcessLike {
|
|
8
|
+
on(event: string, listener: (arg: unknown) => void): unknown;
|
|
9
|
+
off(event: string, listener: (arg: unknown) => void): unknown;
|
|
10
|
+
listenerCount(event: string): number;
|
|
11
|
+
exit(code?: number): unknown;
|
|
12
|
+
}
|
|
13
|
+
export declare function dropReserved<T>(attrs?: Record<string, T>): Record<string, T>;
|
|
14
|
+
/** Record err on the active span (or a new `exception` span). Returns the trace id. */
|
|
15
|
+
export declare function captureException(err: unknown, ctx?: CaptureContext): string;
|
|
16
|
+
/** uncaughtException / unhandledRejection hooks. Exits like Node would, but only when we are the sole listener. */
|
|
17
|
+
export declare function installProcessHandlers(flush: () => Promise<unknown>, proc?: ProcessLike): () => void;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Error capture: an exception becomes an OTel span event (exception.*) with trace and span ids.
|
|
2
|
+
import { trace, SpanStatusCode } from '@opentelemetry/api';
|
|
3
|
+
const captured = new WeakMap(); // error object -> traceId, so framework + process hooks never double-report
|
|
4
|
+
export function dropReserved(attrs) {
|
|
5
|
+
return Object.fromEntries(Object.entries(attrs || {}).filter(([k]) => !k.startsWith('wt.')));
|
|
6
|
+
}
|
|
7
|
+
/** Record err on the active span (or a new `exception` span). Returns the trace id. */
|
|
8
|
+
export function captureException(err, ctx) {
|
|
9
|
+
const o = ctx || {};
|
|
10
|
+
const e = err instanceof Error ? err : new Error(typeof err === 'string' ? err : JSON.stringify(err));
|
|
11
|
+
const seen = captured.get(e);
|
|
12
|
+
if (seen !== undefined)
|
|
13
|
+
return seen;
|
|
14
|
+
const attrs = { 'error.handled': o.handled !== false, ...dropReserved(o.attributes) };
|
|
15
|
+
const active = trace.getActiveSpan();
|
|
16
|
+
const span = active && active.isRecording() ? active : trace.getTracer('watchtower').startSpan('exception');
|
|
17
|
+
span.setAttributes(attrs);
|
|
18
|
+
span.recordException(e);
|
|
19
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: e.message });
|
|
20
|
+
const traceId = span.spanContext().traceId;
|
|
21
|
+
if (span !== active)
|
|
22
|
+
span.end();
|
|
23
|
+
captured.set(e, traceId);
|
|
24
|
+
return traceId;
|
|
25
|
+
}
|
|
26
|
+
/** uncaughtException / unhandledRejection hooks. Exits like Node would, but only when we are the sole listener. */
|
|
27
|
+
export function installProcessHandlers(flush, proc) {
|
|
28
|
+
const p = proc || process;
|
|
29
|
+
const exitAfterFlush = () => Promise.race([flush(), new Promise((r) => setTimeout(r, 2000).unref())]).finally(() => p.exit(1));
|
|
30
|
+
const onException = (e) => {
|
|
31
|
+
captureException(e, { handled: false, attributes: { 'error.mechanism': 'uncaughtException' } });
|
|
32
|
+
if (p.listenerCount('uncaughtException') > 1)
|
|
33
|
+
return;
|
|
34
|
+
console.error(e);
|
|
35
|
+
exitAfterFlush();
|
|
36
|
+
};
|
|
37
|
+
const onRejection = (reason) => {
|
|
38
|
+
captureException(reason, { handled: false, attributes: { 'error.mechanism': 'unhandledRejection' } });
|
|
39
|
+
if (p.listenerCount('unhandledRejection') > 1)
|
|
40
|
+
return;
|
|
41
|
+
console.error(reason);
|
|
42
|
+
exitAfterFlush();
|
|
43
|
+
};
|
|
44
|
+
p.on('uncaughtException', onException);
|
|
45
|
+
p.on('unhandledRejection', onRejection);
|
|
46
|
+
return () => {
|
|
47
|
+
p.off('uncaughtException', onException);
|
|
48
|
+
p.off('unhandledRejection', onRejection);
|
|
49
|
+
};
|
|
50
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
import { metrics, trace } from '@opentelemetry/api';
|
|
3
|
+
import type { Tracer } from '@opentelemetry/api';
|
|
4
|
+
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
5
|
+
import { LoggerProvider } from '@opentelemetry/sdk-logs';
|
|
6
|
+
import type { MeterProvider } from '@opentelemetry/sdk-metrics';
|
|
7
|
+
import { captureException } from './errors.js';
|
|
8
|
+
import type { CaptureContext } from './errors.js';
|
|
9
|
+
import type { Profiler } from './profiling.js';
|
|
10
|
+
export { captureException, metrics, trace };
|
|
11
|
+
export { setCaptureRules, capturePayload } from './payload.js';
|
|
12
|
+
export type { CaptureRule } from './payload.js';
|
|
13
|
+
export type { CaptureContext };
|
|
14
|
+
export declare const DEFAULT_ENDPOINT = "http://localhost:4318";
|
|
15
|
+
export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
|
|
16
|
+
export interface InitOptions {
|
|
17
|
+
/** Ingest key; defaults to WT_INGEST_KEY. */
|
|
18
|
+
apiKey?: string;
|
|
19
|
+
/** Defaults to WT_ENVIRONMENT. */
|
|
20
|
+
environment?: string;
|
|
21
|
+
/** Defaults to WT_RELEASE. */
|
|
22
|
+
release?: string;
|
|
23
|
+
/** Hook uncaughtException / unhandledRejection (default true). */
|
|
24
|
+
captureProcessErrors?: boolean;
|
|
25
|
+
/** Replace the log exporter (tests). */
|
|
26
|
+
logExporter?: unknown;
|
|
27
|
+
/** Export OTLP logs; defaults to WATCHTOWER_LOGS (on unless `off`; OTEL_LOGS_EXPORTER=none also turns it off). */
|
|
28
|
+
logs?: boolean;
|
|
29
|
+
/** Export OTLP metrics; defaults to WATCHTOWER_METRICS (on unless `off`; OTEL_METRICS_EXPORTER=none also turns it off). */
|
|
30
|
+
metrics?: boolean;
|
|
31
|
+
/** Replace the metric exporter (tests). */
|
|
32
|
+
metricExporter?: unknown;
|
|
33
|
+
/** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT or http://localhost:4318. */
|
|
34
|
+
endpoint?: string;
|
|
35
|
+
/** REST API base URL for getPrompt/score; defaults to WT_API_URL or endpoint. */
|
|
36
|
+
apiUrl?: string;
|
|
37
|
+
project?: string;
|
|
38
|
+
/** Defaults to WT_SERVICE. */
|
|
39
|
+
serviceName?: string;
|
|
40
|
+
/** Resource attributes; keys starting with `wt.` are dropped. */
|
|
41
|
+
resource?: Record<string, string | number | boolean>;
|
|
42
|
+
/** Called for every span attribute before export; return the value to keep. */
|
|
43
|
+
redact?: (key: string, value: unknown) => unknown;
|
|
44
|
+
/** Directory for the offline buffer. */
|
|
45
|
+
bufferDir?: string;
|
|
46
|
+
/** Fetch the service's payload capture rules (APM-016) and record bodies/GraphQL where a rule matches; defaults to WT_CAPTURE_RULES=1. */
|
|
47
|
+
captureRules?: boolean;
|
|
48
|
+
/** Replace the span exporter (tests). */
|
|
49
|
+
exporter?: unknown;
|
|
50
|
+
fetch?: typeof fetch;
|
|
51
|
+
}
|
|
52
|
+
export interface EvalRow {
|
|
53
|
+
case: number;
|
|
54
|
+
traceId: string;
|
|
55
|
+
scores: Record<string, unknown>;
|
|
56
|
+
errors: Record<string, string>;
|
|
57
|
+
}
|
|
58
|
+
export interface EvalOptions {
|
|
59
|
+
name?: string;
|
|
60
|
+
errorTolerance?: boolean;
|
|
61
|
+
/** Rows of an earlier run; cases that finished without errors are kept, not re-run. */
|
|
62
|
+
resume?: EvalRow[];
|
|
63
|
+
}
|
|
64
|
+
export type Scorers = Record<string, (testCase: any, output: any) => unknown | Promise<unknown>>;
|
|
65
|
+
export type EvalTask = (input: any) => unknown | Promise<unknown>;
|
|
66
|
+
export interface ScoreOptions {
|
|
67
|
+
name?: string;
|
|
68
|
+
spanId?: string;
|
|
69
|
+
comment?: string;
|
|
70
|
+
}
|
|
71
|
+
/** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
|
|
72
|
+
export declare function safeResource(attrs?: Record<string, unknown>): Record<string, unknown>;
|
|
73
|
+
export declare class Client {
|
|
74
|
+
readonly endpoint: string;
|
|
75
|
+
readonly apiUrl: string;
|
|
76
|
+
readonly project: string | undefined;
|
|
77
|
+
readonly headers: {
|
|
78
|
+
Authorization: string;
|
|
79
|
+
};
|
|
80
|
+
readonly fetch: typeof fetch;
|
|
81
|
+
readonly redact: InitOptions['redact'];
|
|
82
|
+
readonly provider: NodeTracerProvider;
|
|
83
|
+
readonly tracer: Tracer;
|
|
84
|
+
readonly loggerProvider: LoggerProvider;
|
|
85
|
+
readonly logger: ReturnType<LoggerProvider['getLogger']>;
|
|
86
|
+
/** Set when metrics are on; `init` makes it the global meter provider (`metrics.getMeter`). */
|
|
87
|
+
readonly meterProvider?: MeterProvider;
|
|
88
|
+
private readonly stopRuntime;
|
|
89
|
+
uninstall?: () => void;
|
|
90
|
+
/** Set when WT_PROFILING=1 started the continuous profiler. */
|
|
91
|
+
profiler?: Promise<Profiler | undefined>;
|
|
92
|
+
constructor(opts: InitOptions);
|
|
93
|
+
/** Emit an OTLP log record; trace and span ids come from the active context. */
|
|
94
|
+
log(level: LogLevel, body: string, attributes?: Record<string, unknown>): void;
|
|
95
|
+
forceFlush(): Promise<unknown>;
|
|
96
|
+
toJSON(): {
|
|
97
|
+
endpoint: string;
|
|
98
|
+
project: string | undefined;
|
|
99
|
+
headers: {
|
|
100
|
+
Authorization: string;
|
|
101
|
+
};
|
|
102
|
+
};
|
|
103
|
+
[inspect.custom](): string;
|
|
104
|
+
private _call;
|
|
105
|
+
getPrompt(promptId: string): Promise<unknown>;
|
|
106
|
+
score(traceId: string, value: number | boolean | string, o?: ScoreOptions): Promise<unknown>;
|
|
107
|
+
/** Run task per case under a span; record each scorer result as a score when a project is set. */
|
|
108
|
+
evaluate(dataset: unknown[], task: EvalTask, scorers: Scorers, o?: EvalOptions): Promise<EvalRow[]>;
|
|
109
|
+
/** Re-run only the cases that errored or are missing in `previous`. */
|
|
110
|
+
evaluateResume(dataset: unknown[], task: EvalTask, scorers: Scorers, previous: EvalRow[], o?: EvalOptions): Promise<EvalRow[]>;
|
|
111
|
+
/** Wrap fn in a span. */
|
|
112
|
+
observe<A extends unknown[], R>(name: string, fn: (...a: A) => R): (...a: A) => R;
|
|
113
|
+
shutdown(): Promise<void>;
|
|
114
|
+
}
|
|
115
|
+
export declare function init(opts: InitOptions): Client;
|
|
116
|
+
export declare const flush: () => Promise<unknown>;
|
|
117
|
+
export declare const log: (l: LogLevel, b: string, a?: Record<string, unknown>) => void;
|
|
118
|
+
export declare const getPrompt: (id: string) => Promise<unknown>;
|
|
119
|
+
export declare const score: (t: string, v: number | boolean | string, o?: ScoreOptions) => Promise<unknown>;
|
|
120
|
+
export declare const evaluate: (d: unknown[], t: EvalTask, s: Scorers, o?: EvalOptions) => Promise<EvalRow[]>;
|
|
121
|
+
export declare const evaluateResume: (d: unknown[], t: EvalTask, s: Scorers, prev: EvalRow[], o?: EvalOptions) => Promise<EvalRow[]>;
|
|
122
|
+
export declare const observe: <A extends unknown[], R>(n: string, f: (...a: A) => R) => ((...a: A) => R);
|
|
123
|
+
export declare const shutdown: () => Promise<void>;
|