@watchtowerobservability/node 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +121 -2
- package/dist/cjs/buffer.d.ts +49 -0
- package/dist/cjs/buffer.js +134 -0
- package/dist/cjs/errors.d.ts +17 -0
- package/dist/cjs/errors.js +55 -0
- package/dist/cjs/index.d.ts +120 -0
- package/dist/cjs/index.js +302 -0
- package/dist/cjs/integrations/azure-functions.d.ts +18 -0
- package/dist/cjs/integrations/azure-functions.js +97 -0
- package/dist/cjs/integrations/express.d.ts +25 -0
- package/dist/cjs/integrations/express.js +58 -0
- package/dist/cjs/integrations/fastify.d.ts +6 -0
- package/dist/cjs/integrations/fastify.js +44 -0
- package/dist/cjs/integrations/genai-agents.d.ts +36 -0
- package/dist/cjs/integrations/genai-agents.js +124 -0
- package/dist/cjs/integrations/genai-stream.d.ts +19 -0
- package/dist/cjs/integrations/genai-stream.js +135 -0
- package/dist/cjs/integrations/genai.d.ts +18 -0
- package/dist/cjs/integrations/genai.js +150 -0
- package/dist/cjs/integrations/koa.d.ts +19 -0
- package/dist/cjs/integrations/koa.js +34 -0
- package/dist/cjs/integrations/lambda.d.ts +11 -0
- package/dist/cjs/integrations/lambda.js +68 -0
- package/dist/cjs/integrations/logs.d.ts +10 -0
- package/dist/cjs/integrations/logs.js +77 -0
- package/dist/cjs/integrations/nestjs.d.ts +16 -0
- package/dist/cjs/integrations/nestjs.js +30 -0
- package/dist/cjs/integrations/next.d.ts +11 -0
- package/dist/cjs/integrations/next.js +59 -0
- package/dist/cjs/integrations/prisma.d.ts +3 -0
- package/dist/cjs/integrations/prisma.js +33 -0
- package/dist/cjs/integrations/util.d.ts +9 -0
- package/dist/cjs/integrations/util.js +37 -0
- package/dist/cjs/jest.d.ts +21 -0
- package/dist/cjs/jest.js +45 -0
- package/dist/cjs/load.d.ts +1 -0
- package/dist/cjs/load.js +11 -0
- package/dist/cjs/metrics.d.ts +10 -0
- package/dist/cjs/metrics.js +81 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/payload.d.ts +42 -0
- package/dist/cjs/payload.js +145 -0
- package/dist/cjs/probes.d.ts +97 -0
- package/dist/cjs/probes.js +422 -0
- package/dist/cjs/profiling.d.ts +81 -0
- package/dist/cjs/profiling.js +253 -0
- package/dist/cjs/reporter-core.d.ts +32 -0
- package/dist/cjs/reporter-core.js +102 -0
- package/dist/cjs/test-plan.d.ts +93 -0
- package/dist/cjs/test-plan.js +262 -0
- package/dist/cjs/vitest.d.ts +27 -0
- package/dist/cjs/vitest.js +52 -0
- package/dist/esm/buffer.d.ts +49 -0
- package/dist/esm/buffer.js +92 -0
- package/dist/esm/errors.d.ts +17 -0
- package/dist/esm/errors.js +50 -0
- package/dist/esm/index.d.ts +120 -0
- package/dist/esm/index.js +251 -0
- package/dist/esm/integrations/azure-functions.d.ts +18 -0
- package/dist/esm/integrations/azure-functions.js +60 -0
- package/dist/esm/integrations/express.d.ts +25 -0
- package/dist/esm/integrations/express.js +54 -0
- package/dist/esm/integrations/fastify.d.ts +6 -0
- package/dist/esm/integrations/fastify.js +41 -0
- package/dist/esm/integrations/genai-agents.d.ts +36 -0
- package/dist/esm/integrations/genai-agents.js +118 -0
- package/dist/esm/integrations/genai-stream.d.ts +19 -0
- package/dist/esm/integrations/genai-stream.js +130 -0
- package/dist/esm/integrations/genai.d.ts +18 -0
- package/dist/esm/integrations/genai.js +138 -0
- package/dist/esm/integrations/koa.d.ts +19 -0
- package/dist/esm/integrations/koa.js +31 -0
- package/dist/esm/integrations/lambda.d.ts +11 -0
- package/dist/esm/integrations/lambda.js +32 -0
- package/dist/esm/integrations/logs.d.ts +10 -0
- package/dist/esm/integrations/logs.js +40 -0
- package/dist/esm/integrations/nestjs.d.ts +16 -0
- package/dist/esm/integrations/nestjs.js +26 -0
- package/dist/esm/integrations/next.d.ts +11 -0
- package/dist/esm/integrations/next.js +22 -0
- package/dist/esm/integrations/prisma.d.ts +3 -0
- package/dist/esm/integrations/prisma.js +30 -0
- package/dist/esm/integrations/util.d.ts +9 -0
- package/dist/esm/integrations/util.js +32 -0
- package/dist/esm/jest.d.ts +21 -0
- package/dist/esm/jest.js +41 -0
- package/dist/esm/load.d.ts +1 -0
- package/dist/esm/load.js +5 -0
- package/dist/esm/metrics.d.ts +10 -0
- package/dist/esm/metrics.js +44 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/payload.d.ts +42 -0
- package/dist/esm/payload.js +131 -0
- package/dist/esm/probes.d.ts +97 -0
- package/dist/esm/probes.js +412 -0
- package/dist/esm/profiling.d.ts +81 -0
- package/dist/esm/profiling.js +242 -0
- package/dist/esm/reporter-core.d.ts +32 -0
- package/dist/esm/reporter-core.js +63 -0
- package/dist/esm/test-plan.d.ts +93 -0
- package/dist/esm/test-plan.js +248 -0
- package/dist/esm/vitest.d.ts +27 -0
- package/dist/esm/vitest.js +48 -0
- package/package.json +219 -4
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
// Continuous CPU profiler on V8's own sampler (`node:inspector`, no native addon): every
|
|
2
|
+
// `periodMs` the CPU profile is folded into stacks and posted to the gateway's
|
|
3
|
+
// Pyroscope-compatible `/ingest` as `<service>.cpu` (format `folded`). Best effort: a failed
|
|
4
|
+
// upload is counted and dropped. `withSpan` labels samples taken while its function runs, joined
|
|
5
|
+
// on V8's sample timestamps, so a trace links to its exact profile.
|
|
6
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
7
|
+
import { Session } from 'node:inspector';
|
|
8
|
+
import { promiseHooks } from 'node:v8';
|
|
9
|
+
import { trace } from '@opentelemetry/api';
|
|
10
|
+
const intervals = [];
|
|
11
|
+
const MAX_INTERVALS = 100_000;
|
|
12
|
+
function nowUs() {
|
|
13
|
+
return Number(process.hrtime.bigint() / 1000n);
|
|
14
|
+
}
|
|
15
|
+
/** The span of the running code, carried across `await` once async labels are on. */
|
|
16
|
+
const spanStore = new AsyncLocalStorage();
|
|
17
|
+
/** Promises created under a span, and that span. */
|
|
18
|
+
const tagged = new WeakMap();
|
|
19
|
+
/** The interval each `before` opened, for its `after`; `null` for an unlabeled continuation. */
|
|
20
|
+
const running = [];
|
|
21
|
+
let asyncSpans = false;
|
|
22
|
+
function open(prefix) {
|
|
23
|
+
if (intervals.length >= MAX_INTERVALS)
|
|
24
|
+
return null;
|
|
25
|
+
const iv = { start: nowUs(), end: Number.MAX_SAFE_INTEGER, prefix };
|
|
26
|
+
intervals.push(iv);
|
|
27
|
+
return iv;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Opt-in (overhead unmeasured): `withSpan` also labels its async function's code after each
|
|
31
|
+
* `await`. The span rides in an AsyncLocalStorage; a promise created under it is tagged, and each
|
|
32
|
+
* run of its continuation (V8 promise hooks `before` to `after`) is one more interval. Plain
|
|
33
|
+
* callbacks it starts (timers, events) are labeled only from their own `await`s on.
|
|
34
|
+
*/
|
|
35
|
+
export function enableAsyncSpanLabels() {
|
|
36
|
+
if (asyncSpans)
|
|
37
|
+
return;
|
|
38
|
+
asyncSpans = true;
|
|
39
|
+
promiseHooks.createHook({
|
|
40
|
+
init(promise) {
|
|
41
|
+
const p = spanStore.getStore(); // init runs synchronously, in the creating code's context
|
|
42
|
+
if (p)
|
|
43
|
+
tagged.set(promise, p);
|
|
44
|
+
},
|
|
45
|
+
before(promise) {
|
|
46
|
+
if (running.length >= 1_000)
|
|
47
|
+
running.length = 0; // afters lost to an uncaught exception
|
|
48
|
+
const p = tagged.get(promise);
|
|
49
|
+
running.push(p ? open(p) : null);
|
|
50
|
+
},
|
|
51
|
+
after() {
|
|
52
|
+
const iv = running.pop();
|
|
53
|
+
if (iv)
|
|
54
|
+
iv.end = nowUs();
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Labels samples taken while `fn` runs with a span (default: the active OpenTelemetry span). Only
|
|
60
|
+
* `fn`'s synchronous part, unless [[enableAsyncSpanLabels]] (or the profiler's `asyncSpans`) is on.
|
|
61
|
+
*/
|
|
62
|
+
export function withSpan(fn, ids) {
|
|
63
|
+
const sc = ids ?? trace.getActiveSpan()?.spanContext();
|
|
64
|
+
if (!sc || intervals.length >= MAX_INTERVALS)
|
|
65
|
+
return fn();
|
|
66
|
+
const prefix = `span_id=${sc.spanId},trace_id=${sc.traceId}\t`;
|
|
67
|
+
const iv = open(prefix);
|
|
68
|
+
try {
|
|
69
|
+
return asyncSpans ? spanStore.run(prefix, fn) : fn();
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
iv.end = nowUs();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Takes the intervals a profile up to `until` can need; open ones stay for the next profile. */
|
|
76
|
+
export function takeIntervals(until) {
|
|
77
|
+
const out = intervals.slice();
|
|
78
|
+
const keep = out.filter((iv) => iv.end > until);
|
|
79
|
+
intervals.length = 0;
|
|
80
|
+
intervals.push(...keep);
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
/** The innermost span at `t`; `sorted` is by start. */
|
|
84
|
+
function prefixAt(sorted, t) {
|
|
85
|
+
let lo = 0;
|
|
86
|
+
let hi = sorted.length;
|
|
87
|
+
while (lo < hi) {
|
|
88
|
+
const mid = (lo + hi) >> 1;
|
|
89
|
+
if (sorted[mid].start <= t)
|
|
90
|
+
lo = mid + 1;
|
|
91
|
+
else
|
|
92
|
+
hi = mid;
|
|
93
|
+
}
|
|
94
|
+
for (let i = lo - 1; i >= 0 && i >= lo - 16; i--)
|
|
95
|
+
if (sorted[i].end >= t)
|
|
96
|
+
return sorted[i].prefix;
|
|
97
|
+
return '';
|
|
98
|
+
}
|
|
99
|
+
const MAX_DEPTH = 128;
|
|
100
|
+
function frameName(f) {
|
|
101
|
+
const file = f.url ? f.url.split('/').pop() : '';
|
|
102
|
+
const name = f.functionName || '(anonymous)';
|
|
103
|
+
return (file ? `${name} (${file}:${f.lineNumber + 1})` : name).replace(/;/g, ':');
|
|
104
|
+
}
|
|
105
|
+
/** Root-first folded stacks with their sample counts; V8's synthetic `(root)` is left out. */
|
|
106
|
+
export function foldProfile(p, spans = []) {
|
|
107
|
+
const parent = new Map();
|
|
108
|
+
const byId = new Map();
|
|
109
|
+
for (const n of p.nodes) {
|
|
110
|
+
byId.set(n.id, n);
|
|
111
|
+
for (const c of n.children ?? [])
|
|
112
|
+
parent.set(c, n.id);
|
|
113
|
+
}
|
|
114
|
+
const counts = new Map();
|
|
115
|
+
const sorted = [...spans].sort((a, b) => a.start - b.start);
|
|
116
|
+
let t = p.startTime ?? 0;
|
|
117
|
+
for (const [i, id] of (p.samples ?? []).entries()) {
|
|
118
|
+
t += p.timeDeltas?.[i] ?? 0;
|
|
119
|
+
const names = [];
|
|
120
|
+
for (let cur = id; cur !== undefined && names.length < MAX_DEPTH; cur = parent.get(cur)) {
|
|
121
|
+
const n = byId.get(cur);
|
|
122
|
+
if (!n || n.callFrame.functionName === '(root)')
|
|
123
|
+
break;
|
|
124
|
+
if (n.callFrame.functionName === '(idle)' || n.callFrame.functionName === '(program)') {
|
|
125
|
+
names.length = 0;
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
names.push(frameName(n.callFrame));
|
|
129
|
+
}
|
|
130
|
+
if (names.length === 0)
|
|
131
|
+
continue;
|
|
132
|
+
const key = prefixAt(sorted, t) + names.reverse().join(';');
|
|
133
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
134
|
+
}
|
|
135
|
+
return counts;
|
|
136
|
+
}
|
|
137
|
+
export function foldedBody(counts) {
|
|
138
|
+
let out = '';
|
|
139
|
+
for (const [k, n] of counts)
|
|
140
|
+
out += `${k} ${n}\n`;
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
export function ingestUrl(endpoint, service, tags, fromS, untilS, hz) {
|
|
144
|
+
const labels = Object.entries(tags).filter(([, v]) => v).sort(([a], [b]) => a.localeCompare(b))
|
|
145
|
+
.map(([k, v]) => `${k}=${v}`).join(',');
|
|
146
|
+
const q = new URLSearchParams({
|
|
147
|
+
name: `${service}.cpu{${labels}}`, from: String(Math.floor(fromS)), until: String(Math.floor(untilS)),
|
|
148
|
+
format: 'folded', sampleRate: String(hz), spyName: 'watchtower-node', units: 'samples',
|
|
149
|
+
});
|
|
150
|
+
return `${endpoint.replace(/\/+$/, '')}/ingest?${q}`;
|
|
151
|
+
}
|
|
152
|
+
export class Profiler {
|
|
153
|
+
uploaded = 0;
|
|
154
|
+
dropped = 0;
|
|
155
|
+
key;
|
|
156
|
+
endpoint;
|
|
157
|
+
service;
|
|
158
|
+
tags;
|
|
159
|
+
intervalUs;
|
|
160
|
+
periodMs;
|
|
161
|
+
session;
|
|
162
|
+
timer;
|
|
163
|
+
from = 0;
|
|
164
|
+
constructor(o = {}) {
|
|
165
|
+
const key = o.apiKey ?? process.env.WT_INGEST_KEY;
|
|
166
|
+
if (!key)
|
|
167
|
+
throw new Error('apiKey is required');
|
|
168
|
+
this.key = key;
|
|
169
|
+
this.endpoint = o.endpoint ?? process.env.WT_OTLP_ENDPOINT ?? 'http://localhost:4318';
|
|
170
|
+
this.service = o.service ?? process.env.WT_SERVICE ?? 'app';
|
|
171
|
+
this.tags = { env: process.env.WT_ENVIRONMENT, version: process.env.WT_RELEASE, ...o.tags };
|
|
172
|
+
this.intervalUs = Math.min(Math.max(o.intervalUs ?? 10_000, 1_000), 1_000_000);
|
|
173
|
+
this.periodMs = Math.max(o.periodMs ?? 15_000, 1_000);
|
|
174
|
+
if (o.asyncSpans)
|
|
175
|
+
enableAsyncSpanLabels();
|
|
176
|
+
}
|
|
177
|
+
/** Not printable with the key. */
|
|
178
|
+
[Symbol.for('nodejs.util.inspect.custom')]() {
|
|
179
|
+
return `Profiler { service: '${this.service}' }`;
|
|
180
|
+
}
|
|
181
|
+
post(method, params) {
|
|
182
|
+
return new Promise((resolve, reject) => {
|
|
183
|
+
this.session.post(method, params ?? {}, (err, res) => (err ? reject(err) : resolve(res)));
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
async start() {
|
|
187
|
+
if (this.session)
|
|
188
|
+
return this;
|
|
189
|
+
this.session = new Session();
|
|
190
|
+
this.session.connect();
|
|
191
|
+
await this.post('Profiler.enable');
|
|
192
|
+
await this.post('Profiler.setSamplingInterval', { interval: this.intervalUs });
|
|
193
|
+
await this.post('Profiler.start');
|
|
194
|
+
this.from = Date.now() / 1000;
|
|
195
|
+
this.timer = setInterval(() => void this.flush(true), this.periodMs);
|
|
196
|
+
this.timer.unref();
|
|
197
|
+
return this;
|
|
198
|
+
}
|
|
199
|
+
/** Stop the current profile, upload it and (when `restart`) start the next one. */
|
|
200
|
+
async flush(restart) {
|
|
201
|
+
if (!this.session)
|
|
202
|
+
return;
|
|
203
|
+
const until = Date.now() / 1000;
|
|
204
|
+
const { profile } = await this.post('Profiler.stop');
|
|
205
|
+
const from = this.from;
|
|
206
|
+
if (restart) {
|
|
207
|
+
await this.post('Profiler.start');
|
|
208
|
+
this.from = until;
|
|
209
|
+
}
|
|
210
|
+
const counts = foldProfile(profile, takeIntervals(nowUs()));
|
|
211
|
+
if (counts.size)
|
|
212
|
+
await this.upload(counts, from, until);
|
|
213
|
+
}
|
|
214
|
+
async upload(counts, fromS, untilS) {
|
|
215
|
+
try {
|
|
216
|
+
const hz = Math.round(1_000_000 / this.intervalUs);
|
|
217
|
+
const r = await fetch(ingestUrl(this.endpoint, this.service, this.tags, fromS, untilS, hz), {
|
|
218
|
+
method: 'POST', body: foldedBody(counts),
|
|
219
|
+
headers: { authorization: `Bearer ${this.key}`, 'content-type': 'text/plain' },
|
|
220
|
+
signal: AbortSignal.timeout(10_000),
|
|
221
|
+
});
|
|
222
|
+
if (r.ok)
|
|
223
|
+
this.uploaded++;
|
|
224
|
+
else
|
|
225
|
+
this.dropped++;
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
this.dropped++;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
async stop() {
|
|
232
|
+
if (this.timer)
|
|
233
|
+
clearInterval(this.timer);
|
|
234
|
+
await this.flush(false).catch(() => undefined);
|
|
235
|
+
this.session?.disconnect();
|
|
236
|
+
this.session = undefined;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
/** Start a [[Profiler]] with the SDK's environment defaults. */
|
|
240
|
+
export function startProfiling(o = {}) {
|
|
241
|
+
return new Profiler(o).start();
|
|
242
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Client, InitOptions } from './index.js';
|
|
2
|
+
export interface ReporterOptions {
|
|
3
|
+
client?: Client;
|
|
4
|
+
apiKey?: string;
|
|
5
|
+
serviceName?: string;
|
|
6
|
+
/** Extra options passed to `init` when the reporter creates its own client. */
|
|
7
|
+
init?: InitOptions;
|
|
8
|
+
}
|
|
9
|
+
export type TestOutcome = 'passed' | 'failed' | 'skipped';
|
|
10
|
+
export interface TestRecord {
|
|
11
|
+
name: string;
|
|
12
|
+
outcome: TestOutcome;
|
|
13
|
+
durationMs?: number;
|
|
14
|
+
message?: string;
|
|
15
|
+
endMs?: number;
|
|
16
|
+
/** `suite::file > describe > test` (see test-plan.ts) */
|
|
17
|
+
id?: string;
|
|
18
|
+
/** ticket of a quarantined test; its failure did not fail the run */
|
|
19
|
+
quarantined?: string;
|
|
20
|
+
/** a test the project had never seen (early flake detection) */
|
|
21
|
+
isNew?: boolean;
|
|
22
|
+
/** failed attempts before the final result (retries) */
|
|
23
|
+
attempts?: number;
|
|
24
|
+
}
|
|
25
|
+
/** Client from `opts.client`, else `WT_INGEST_KEY` (+ `WT_PROJECT_ID`); `null` when neither is set. */
|
|
26
|
+
export declare function clientFor(opts?: ReporterOptions): Client | null;
|
|
27
|
+
/**
|
|
28
|
+
* tests: [{ name, outcome: 'passed'|'failed'|'skipped', durationMs, message?, endMs? }]
|
|
29
|
+
* Spans are created after the fact, so start/end times are set explicitly.
|
|
30
|
+
*/
|
|
31
|
+
export declare function reportTests(client: Client | null, tests: TestRecord[], framework?: string): Promise<number>;
|
|
32
|
+
export declare const outcome: (s?: string) => TestOutcome;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Shared by the Jest and Vitest reporters: every finished test becomes a span named
|
|
2
|
+
// `test <full name>` with `test.outcome` / `test.duration_ms`, and, with a project, a
|
|
3
|
+
// `test_pass` score (1 passed, 0 failed; skipped tests get no score). Telemetry failures never
|
|
4
|
+
// fail the test run.
|
|
5
|
+
import { SpanStatusCode } from '@opentelemetry/api';
|
|
6
|
+
import * as wt from './index.js';
|
|
7
|
+
/** Client from `opts.client`, else `WT_INGEST_KEY` (+ `WT_PROJECT_ID`); `null` when neither is set. */
|
|
8
|
+
export function clientFor(opts) {
|
|
9
|
+
const o = opts || {};
|
|
10
|
+
if (o.client)
|
|
11
|
+
return o.client;
|
|
12
|
+
const apiKey = o.apiKey || process.env.WT_INGEST_KEY;
|
|
13
|
+
if (!apiKey)
|
|
14
|
+
return null;
|
|
15
|
+
return wt.init({ apiKey, serviceName: o.serviceName || 'tests', ...o.init });
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* tests: [{ name, outcome: 'passed'|'failed'|'skipped', durationMs, message?, endMs? }]
|
|
19
|
+
* Spans are created after the fact, so start/end times are set explicitly.
|
|
20
|
+
*/
|
|
21
|
+
export async function reportTests(client, tests, framework = 'test') {
|
|
22
|
+
if (!client)
|
|
23
|
+
return 0;
|
|
24
|
+
let n = 0;
|
|
25
|
+
for (const t of tests) {
|
|
26
|
+
try {
|
|
27
|
+
const end = t.endMs || Date.now();
|
|
28
|
+
const span = client.tracer.startSpan(`test ${t.name}`, { startTime: end - (t.durationMs || 0) });
|
|
29
|
+
span.setAttribute('test.outcome', t.outcome);
|
|
30
|
+
span.setAttribute('test.duration_ms', t.durationMs || 0);
|
|
31
|
+
// OTel test semconv plus the attributes the pytest plugin sets
|
|
32
|
+
const id = t.id || t.name;
|
|
33
|
+
span.setAttribute('test.framework', framework);
|
|
34
|
+
span.setAttribute('test.suite.name', id.includes('::') ? id.slice(0, id.indexOf('::')) : framework);
|
|
35
|
+
span.setAttribute('test.case.name', id);
|
|
36
|
+
span.setAttribute('test.case.result.status', t.outcome === 'passed' ? 'pass' : t.outcome === 'failed' ? 'fail' : 'skip');
|
|
37
|
+
if (t.quarantined)
|
|
38
|
+
span.setAttribute('test.quarantined', true);
|
|
39
|
+
if (t.isNew) {
|
|
40
|
+
span.setAttribute('test.is_new', true);
|
|
41
|
+
span.setAttribute('test.efd.retries', t.attempts || 0);
|
|
42
|
+
span.setAttribute('test.is_flaky', (t.attempts || 0) > 0 && t.outcome === 'passed');
|
|
43
|
+
}
|
|
44
|
+
if (t.outcome === 'failed') {
|
|
45
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: String(t.message || 'failed').slice(0, 500) });
|
|
46
|
+
}
|
|
47
|
+
const traceId = span.spanContext().traceId;
|
|
48
|
+
span.end(end);
|
|
49
|
+
n += 1;
|
|
50
|
+
if (client.project && t.outcome !== 'skipped') {
|
|
51
|
+
await client.score(traceId, t.outcome === 'passed' ? 1 : 0, { name: 'test_pass', comment: t.name });
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
catch (_e) { /* telemetry down must not fail tests */ }
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
await client.provider.forceFlush();
|
|
58
|
+
}
|
|
59
|
+
catch (_e) { /* same */ }
|
|
60
|
+
return n;
|
|
61
|
+
}
|
|
62
|
+
const OUTCOME = { passed: 'passed', pass: 'passed', failed: 'failed', fail: 'failed' };
|
|
63
|
+
export const outcome = (s) => OUTCOME[s] || 'skipped';
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { TestRecord } from './reporter-core.js';
|
|
2
|
+
import type { Client } from './index.js';
|
|
3
|
+
export interface TestPlan {
|
|
4
|
+
quarantined: {
|
|
5
|
+
id: string;
|
|
6
|
+
ticket_url?: string | null;
|
|
7
|
+
owner?: string | null;
|
|
8
|
+
}[];
|
|
9
|
+
disabled: string[];
|
|
10
|
+
skippable: string[];
|
|
11
|
+
new_tests: string[];
|
|
12
|
+
efd_retries: number;
|
|
13
|
+
}
|
|
14
|
+
/** What the plan says about one test. */
|
|
15
|
+
export interface TestDecision {
|
|
16
|
+
skip?: 'disabled' | 'unaffected';
|
|
17
|
+
/** ticket URL (or "no ticket") of a quarantined test */
|
|
18
|
+
quarantined?: string;
|
|
19
|
+
isNew?: boolean;
|
|
20
|
+
retries: number;
|
|
21
|
+
}
|
|
22
|
+
export declare const suiteName: (framework: string) => string;
|
|
23
|
+
export declare function testId(suite: string, file: string | undefined, names: string[]): string;
|
|
24
|
+
export declare function changedFiles(): string[];
|
|
25
|
+
/** The session plan; `null` without a project or when the API is unreachable (then everything runs). */
|
|
26
|
+
export declare function fetchTestPlan(client: Client | null, tests?: string[]): Promise<TestPlan | null>;
|
|
27
|
+
/** The plan the global setup wrote (workers and reporters), cached per process. */
|
|
28
|
+
export declare function loadPlan(): TestPlan | null;
|
|
29
|
+
/** Vitest/Jest global setup: fetch the plan once and publish it to the workers (which inherit the env). */
|
|
30
|
+
export default function globalSetup(): Promise<void>;
|
|
31
|
+
/** Decides per test id; with no plan nothing is skipped or retried. */
|
|
32
|
+
export declare function decider(plan: TestPlan | null): (id: string) => TestDecision;
|
|
33
|
+
/** The slice of a Vitest task the setup reads and changes. */
|
|
34
|
+
export interface PlanTask {
|
|
35
|
+
type?: string;
|
|
36
|
+
name: string;
|
|
37
|
+
mode?: string;
|
|
38
|
+
retry?: number;
|
|
39
|
+
filepath?: string;
|
|
40
|
+
tasks?: PlanTask[];
|
|
41
|
+
meta?: Record<string, unknown>;
|
|
42
|
+
result?: {
|
|
43
|
+
state?: string;
|
|
44
|
+
errors?: {
|
|
45
|
+
message?: string;
|
|
46
|
+
}[];
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/** Vitest's `beforeAll`/`afterEach`, passed in so this package does not import Vitest. */
|
|
50
|
+
export interface VitestHooks {
|
|
51
|
+
beforeAll(fn: (suite: PlanTask) => unknown): void;
|
|
52
|
+
afterEach(fn: (ctx: {
|
|
53
|
+
task: PlanTask;
|
|
54
|
+
skip(note?: string): void;
|
|
55
|
+
}) => unknown): void;
|
|
56
|
+
}
|
|
57
|
+
/** Applies the plan in a Vitest setup file. Decisions go to `task.meta.wt` for the reporter. */
|
|
58
|
+
export declare function vitestSetup(hooks: VitestHooks): void;
|
|
59
|
+
interface CircusBlock {
|
|
60
|
+
name: string;
|
|
61
|
+
parent?: CircusBlock;
|
|
62
|
+
children?: ({
|
|
63
|
+
type?: string;
|
|
64
|
+
name: string;
|
|
65
|
+
} & Partial<CircusTest>)[];
|
|
66
|
+
}
|
|
67
|
+
interface CircusTest {
|
|
68
|
+
name: string;
|
|
69
|
+
mode?: string;
|
|
70
|
+
status?: string;
|
|
71
|
+
errors: unknown[];
|
|
72
|
+
parent?: CircusBlock;
|
|
73
|
+
}
|
|
74
|
+
export interface CircusEvent {
|
|
75
|
+
name: string;
|
|
76
|
+
test?: CircusTest;
|
|
77
|
+
describeBlock?: CircusBlock;
|
|
78
|
+
}
|
|
79
|
+
type JestEnvBase = new (...args: any[]) => {
|
|
80
|
+
global: Record<string | symbol, unknown>;
|
|
81
|
+
handleTestEvent?(event: CircusEvent, state: unknown): unknown;
|
|
82
|
+
};
|
|
83
|
+
/**
|
|
84
|
+
* Wraps a Jest test environment class so it applies the plan: skips disabled/unaffected tests on
|
|
85
|
+
* `test_start`, raises `jest.retryTimes` for describe blocks holding a new test, and turns a
|
|
86
|
+
* quarantined test's failure into a skip that keeps its failure message (the reporter reads it).
|
|
87
|
+
*/
|
|
88
|
+
export declare function jestEnvironment<B extends JestEnvBase>(Base: B): B;
|
|
89
|
+
/** One <testsuite>; a test retried k times gets k failed attempts ahead of its final result. */
|
|
90
|
+
export declare function toJunit(suite: string, tests: TestRecord[]): string;
|
|
91
|
+
/** Posts the results as JUnit when WT_CI_RUN_ID is set; never throws. */
|
|
92
|
+
export declare function uploadResults(client: Client | null, suite: string, tests: TestRecord[]): Promise<boolean>;
|
|
93
|
+
export {};
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
// Test optimization for Vitest and Jest, the same contract as the pytest plugin. Opt in with
|
|
2
|
+
// WT_TEST_OPTIMIZATION=1 (plus WT_INGEST_KEY, WT_PROJECT_ID and WT_API_URL; WT_API_KEY, when set,
|
|
3
|
+
// authenticates the project API calls instead of the ingest key):
|
|
4
|
+
// - a global setup asks `POST /v1/projects/{prj}/tests/plan` once and hands the plan to the
|
|
5
|
+
// workers through a file;
|
|
6
|
+
// - disabled tests are skipped, and so are tests Test Impact Analysis marks unaffected by the
|
|
7
|
+
// changed files (WT_CHANGED_FILES comma-separated, or `git diff --name-only $WT_GIT_BASE...HEAD`);
|
|
8
|
+
// - quarantined tests run, but a failure is turned into a skip (the run stays green) and the
|
|
9
|
+
// reporter records it as a quarantined failure;
|
|
10
|
+
// - a test the project has never seen is retried up to `efd_retries` times (early flake detection).
|
|
11
|
+
// The reporters upload the results as JUnit (`POST /v1/projects/{prj}/test-runs`) when
|
|
12
|
+
// WT_CI_RUN_ID is set, which feeds flaky-test detection and auto-quarantine.
|
|
13
|
+
//
|
|
14
|
+
// Ids are `<suite>::<file relative to cwd> > <describe>... > <test>`, suite = WT_TEST_SUITE or
|
|
15
|
+
// the framework name. Vitest:
|
|
16
|
+
// globalSetup: ['./wt-global.mjs'] // export { default } from '@watchtowerobservability/node/test-plan'
|
|
17
|
+
// setupFiles: ['./wt-setup.ts'] // import { beforeAll, afterEach } from 'vitest';
|
|
18
|
+
// // import { vitestSetup } from '@watchtowerobservability/node/test-plan';
|
|
19
|
+
// // vitestSetup({ beforeAll, afterEach });
|
|
20
|
+
// Jest: globalSetup as above, and testEnvironment: './wt-env.js' with
|
|
21
|
+
// module.exports = require('@watchtowerobservability/node/test-plan').jestEnvironment(require('jest-environment-node').TestEnvironment);
|
|
22
|
+
import { execFileSync } from 'node:child_process';
|
|
23
|
+
import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
|
|
24
|
+
import { tmpdir } from 'node:os';
|
|
25
|
+
import { join, relative, sep } from 'node:path';
|
|
26
|
+
import { clientFor } from './reporter-core.js';
|
|
27
|
+
const PLAN_ENV = 'WT_TEST_PLAN_FILE';
|
|
28
|
+
export const suiteName = (framework) => process.env.WT_TEST_SUITE || framework;
|
|
29
|
+
export function testId(suite, file, names) {
|
|
30
|
+
const f = file ? relative(process.cwd(), file).split(sep).join('/') : '';
|
|
31
|
+
return `${suite}::${[f, ...names].filter(Boolean).join(' > ')}`;
|
|
32
|
+
}
|
|
33
|
+
export function changedFiles() {
|
|
34
|
+
const env = process.env.WT_CHANGED_FILES;
|
|
35
|
+
if (env)
|
|
36
|
+
return env.split(',').map((f) => f.trim()).filter(Boolean);
|
|
37
|
+
const base = process.env.WT_GIT_BASE;
|
|
38
|
+
if (!base || base.startsWith('-'))
|
|
39
|
+
return [];
|
|
40
|
+
try {
|
|
41
|
+
const out = execFileSync('git', ['diff', '--name-only', `${base}...HEAD`, '--'], { encoding: 'utf8', timeout: 30_000 });
|
|
42
|
+
return out.split(/\s+/).filter(Boolean);
|
|
43
|
+
}
|
|
44
|
+
catch (_e) {
|
|
45
|
+
return [];
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
function apiHeaders(client) {
|
|
49
|
+
const key = process.env.WT_API_KEY;
|
|
50
|
+
return key ? { Authorization: `Bearer ${key}` } : { ...client.headers };
|
|
51
|
+
}
|
|
52
|
+
const projectUrl = (client, path) => `${client.apiUrl}/v1/projects/${encodeURIComponent(client.project || '')}/${path}`;
|
|
53
|
+
/** The session plan; `null` without a project or when the API is unreachable (then everything runs). */
|
|
54
|
+
export async function fetchTestPlan(client, tests = []) {
|
|
55
|
+
if (!client || !client.project)
|
|
56
|
+
return null;
|
|
57
|
+
try {
|
|
58
|
+
const res = await client.fetch(projectUrl(client, 'tests/plan'), {
|
|
59
|
+
method: 'POST',
|
|
60
|
+
headers: { ...apiHeaders(client), 'Content-Type': 'application/json' },
|
|
61
|
+
body: JSON.stringify({ changed_files: changedFiles(), tests }),
|
|
62
|
+
});
|
|
63
|
+
return res.ok ? (await res.json()) : null;
|
|
64
|
+
}
|
|
65
|
+
catch (_e) {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
let cached;
|
|
70
|
+
/** The plan the global setup wrote (workers and reporters), cached per process. */
|
|
71
|
+
export function loadPlan() {
|
|
72
|
+
if (cached === undefined) {
|
|
73
|
+
const f = process.env[PLAN_ENV];
|
|
74
|
+
try {
|
|
75
|
+
cached = f ? JSON.parse(readFileSync(f, 'utf8')) : null;
|
|
76
|
+
}
|
|
77
|
+
catch (_e) {
|
|
78
|
+
cached = null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return cached;
|
|
82
|
+
}
|
|
83
|
+
/** Vitest/Jest global setup: fetch the plan once and publish it to the workers (which inherit the env). */
|
|
84
|
+
export default async function globalSetup() {
|
|
85
|
+
if (process.env.WT_TEST_OPTIMIZATION !== '1')
|
|
86
|
+
return;
|
|
87
|
+
const client = clientFor({});
|
|
88
|
+
const plan = await fetchTestPlan(client);
|
|
89
|
+
if (client)
|
|
90
|
+
await client.shutdown().catch(() => { });
|
|
91
|
+
if (!plan)
|
|
92
|
+
return;
|
|
93
|
+
// shortcut: the file (a few KB) stays in the OS temp dir; Jest has no teardown return to remove it
|
|
94
|
+
const file = join(mkdtempSync(join(tmpdir(), 'wt-plan-')), 'plan.json');
|
|
95
|
+
writeFileSync(file, JSON.stringify(plan));
|
|
96
|
+
process.env[PLAN_ENV] = file;
|
|
97
|
+
cached = plan;
|
|
98
|
+
}
|
|
99
|
+
/** Decides per test id; with no plan nothing is skipped or retried. */
|
|
100
|
+
export function decider(plan) {
|
|
101
|
+
if (!plan)
|
|
102
|
+
return () => ({ retries: 0 });
|
|
103
|
+
const disabled = new Set(plan.disabled || []);
|
|
104
|
+
const skippable = new Set(plan.skippable || []);
|
|
105
|
+
const fresh = new Set(plan.new_tests || []);
|
|
106
|
+
const quarantined = new Map((plan.quarantined || []).map((q) => [q.id, q.ticket_url || 'no ticket']));
|
|
107
|
+
return (id) => {
|
|
108
|
+
const d = { retries: 0 };
|
|
109
|
+
const q = quarantined.get(id);
|
|
110
|
+
if (disabled.has(id))
|
|
111
|
+
d.skip = 'disabled';
|
|
112
|
+
else if (skippable.has(id) && q === undefined)
|
|
113
|
+
d.skip = 'unaffected';
|
|
114
|
+
if (q !== undefined)
|
|
115
|
+
d.quarantined = q;
|
|
116
|
+
if (fresh.has(id)) {
|
|
117
|
+
d.isNew = true;
|
|
118
|
+
d.retries = Math.max(0, plan.efd_retries || 0);
|
|
119
|
+
}
|
|
120
|
+
return d;
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/** Applies the plan in a Vitest setup file. Decisions go to `task.meta.wt` for the reporter. */
|
|
124
|
+
export function vitestSetup(hooks) {
|
|
125
|
+
const plan = loadPlan();
|
|
126
|
+
if (!plan)
|
|
127
|
+
return;
|
|
128
|
+
const decide = decider(plan);
|
|
129
|
+
const suite = suiteName('vitest');
|
|
130
|
+
const walk = (t, file, names) => {
|
|
131
|
+
for (const c of t.tasks || []) {
|
|
132
|
+
const path = [...names, c.name];
|
|
133
|
+
if (c.tasks) {
|
|
134
|
+
walk(c, file, path);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
const d = decide(testId(suite, file, path));
|
|
138
|
+
c.meta = { ...c.meta, wt: d };
|
|
139
|
+
if (d.skip && (c.mode === 'run' || c.mode === 'queued'))
|
|
140
|
+
c.mode = 'skip';
|
|
141
|
+
if (d.retries > (c.retry || 0))
|
|
142
|
+
c.retry = d.retries;
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
// a setup file's beforeAll runs on the file suite after collection, before any test
|
|
146
|
+
hooks.beforeAll((file) => walk(file, file.filepath, []));
|
|
147
|
+
hooks.afterEach((ctx) => {
|
|
148
|
+
const d = ctx.task.meta?.wt;
|
|
149
|
+
if (!d?.quarantined || ctx.task.result?.state !== 'fail')
|
|
150
|
+
return;
|
|
151
|
+
const msg = ctx.task.result.errors?.[0]?.message || 'failed';
|
|
152
|
+
ctx.task.meta = { ...ctx.task.meta, wtQuarantinedFailure: msg.slice(0, 2000) };
|
|
153
|
+
ctx.skip(`quarantined in Watchtower (${d.quarantined})`);
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
const RETRY_TIMES = Symbol.for('RETRY_TIMES');
|
|
157
|
+
/**
|
|
158
|
+
* Wraps a Jest test environment class so it applies the plan: skips disabled/unaffected tests on
|
|
159
|
+
* `test_start`, raises `jest.retryTimes` for describe blocks holding a new test, and turns a
|
|
160
|
+
* quarantined test's failure into a skip that keeps its failure message (the reporter reads it).
|
|
161
|
+
*/
|
|
162
|
+
export function jestEnvironment(Base) {
|
|
163
|
+
return class WatchtowerEnvironment extends Base {
|
|
164
|
+
wtFile;
|
|
165
|
+
wtDecide = decider(loadPlan());
|
|
166
|
+
wtSuite = suiteName('jest');
|
|
167
|
+
wtRetrySaved = [];
|
|
168
|
+
// any[]: see JestEnvBase
|
|
169
|
+
constructor(...args) {
|
|
170
|
+
super(...args);
|
|
171
|
+
this.wtFile = args[1]?.testPath || '';
|
|
172
|
+
}
|
|
173
|
+
wtId(test) {
|
|
174
|
+
const names = [test.name];
|
|
175
|
+
for (let b = test.parent; b && b.parent; b = b.parent)
|
|
176
|
+
names.unshift(b.name);
|
|
177
|
+
return testId(this.wtSuite, this.wtFile, names);
|
|
178
|
+
}
|
|
179
|
+
async handleTestEvent(event, state) {
|
|
180
|
+
if (super.handleTestEvent)
|
|
181
|
+
await super.handleTestEvent(event, state);
|
|
182
|
+
const t = event.test;
|
|
183
|
+
if (event.name === 'test_start' && t) {
|
|
184
|
+
if (this.wtDecide(this.wtId(t)).skip)
|
|
185
|
+
t.mode = 'skip';
|
|
186
|
+
}
|
|
187
|
+
else if (event.name === 'test_done' && t && t.errors.length > 0 && this.wtDecide(this.wtId(t)).quarantined) {
|
|
188
|
+
t.status = 'skip'; // reported as pending; the errors stay as failureMessages
|
|
189
|
+
}
|
|
190
|
+
else if (event.name === 'run_describe_start' && event.describeBlock) {
|
|
191
|
+
const b = event.describeBlock;
|
|
192
|
+
const prev = this.global[RETRY_TIMES];
|
|
193
|
+
this.wtRetrySaved.push(prev);
|
|
194
|
+
// shortcut: Jest retries per describe block, so other failing tests in a block holding a new
|
|
195
|
+
// test are retried too; fine for flake detection, upgrade if circus gets per-test retries
|
|
196
|
+
const n = Math.max(0, ...(b.children || []).filter((c) => c.type === 'test').map((c) => this.wtDecide(this.wtId({ name: c.name, parent: b })).retries));
|
|
197
|
+
if (n > (Number(prev) || 0))
|
|
198
|
+
this.global[RETRY_TIMES] = n;
|
|
199
|
+
}
|
|
200
|
+
else if (event.name === 'run_describe_finish') {
|
|
201
|
+
this.global[RETRY_TIMES] = this.wtRetrySaved.pop();
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
// ---------------------------------------------------------------- JUnit upload
|
|
207
|
+
const esc = (s) => s.replace(/[&<>"']/g, (c) => `&#${c.charCodeAt(0)};`);
|
|
208
|
+
/** One <testsuite>; a test retried k times gets k failed attempts ahead of its final result. */
|
|
209
|
+
export function toJunit(suite, tests) {
|
|
210
|
+
const prefix = `${suite}::`;
|
|
211
|
+
const cases = [];
|
|
212
|
+
for (const t of tests) {
|
|
213
|
+
const id = t.id || `${prefix}${t.name}`;
|
|
214
|
+
const name = esc(id.startsWith(prefix) ? id.slice(prefix.length) : id);
|
|
215
|
+
const time = ((t.durationMs || 0) / 1000).toFixed(3);
|
|
216
|
+
const fail = `<failure message="${esc(String(t.message || 'failed').slice(0, 500))}"/>`;
|
|
217
|
+
for (let i = 0; i < (t.attempts || 0); i++)
|
|
218
|
+
cases.push(`<testcase name="${name}" time="0">${fail}</testcase>`);
|
|
219
|
+
const body = t.outcome === 'failed' ? fail : t.outcome === 'skipped' ? '<skipped/>' : '';
|
|
220
|
+
cases.push(`<testcase name="${name}" time="${time}">${body}</testcase>`);
|
|
221
|
+
}
|
|
222
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites><testsuite name="${esc(suite)}">${cases.join('')}</testsuite></testsuites>\n`;
|
|
223
|
+
}
|
|
224
|
+
/** Posts the results as JUnit when WT_CI_RUN_ID is set; never throws. */
|
|
225
|
+
export async function uploadResults(client, suite, tests) {
|
|
226
|
+
const env = process.env;
|
|
227
|
+
const runId = env.WT_CI_RUN_ID;
|
|
228
|
+
if (!client || !client.project || !runId || tests.length === 0)
|
|
229
|
+
return false;
|
|
230
|
+
const form = new FormData();
|
|
231
|
+
const fields = [
|
|
232
|
+
['run_id', runId],
|
|
233
|
+
['branch', env.WT_BRANCH || env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME || env.CI_COMMIT_REF_NAME],
|
|
234
|
+
['sha', env.WT_SHA || env.GITHUB_SHA || env.CI_COMMIT_SHA],
|
|
235
|
+
['suite', suite],
|
|
236
|
+
];
|
|
237
|
+
for (const [k, v] of fields)
|
|
238
|
+
if (v)
|
|
239
|
+
form.append(k, v.slice(0, 200));
|
|
240
|
+
form.append('file', new Blob([toJunit(suite, tests)], { type: 'application/xml' }), 'junit.xml');
|
|
241
|
+
try {
|
|
242
|
+
const res = await client.fetch(projectUrl(client, 'test-runs'), { method: 'POST', headers: apiHeaders(client), body: form });
|
|
243
|
+
return res.ok;
|
|
244
|
+
}
|
|
245
|
+
catch (_e) {
|
|
246
|
+
return false;
|
|
247
|
+
}
|
|
248
|
+
}
|