@watchtowerobservability/node 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +121 -2
- package/dist/cjs/buffer.d.ts +49 -0
- package/dist/cjs/buffer.js +134 -0
- package/dist/cjs/errors.d.ts +17 -0
- package/dist/cjs/errors.js +55 -0
- package/dist/cjs/index.d.ts +120 -0
- package/dist/cjs/index.js +302 -0
- package/dist/cjs/integrations/azure-functions.d.ts +18 -0
- package/dist/cjs/integrations/azure-functions.js +97 -0
- package/dist/cjs/integrations/express.d.ts +25 -0
- package/dist/cjs/integrations/express.js +58 -0
- package/dist/cjs/integrations/fastify.d.ts +6 -0
- package/dist/cjs/integrations/fastify.js +44 -0
- package/dist/cjs/integrations/genai-agents.d.ts +36 -0
- package/dist/cjs/integrations/genai-agents.js +124 -0
- package/dist/cjs/integrations/genai-stream.d.ts +19 -0
- package/dist/cjs/integrations/genai-stream.js +135 -0
- package/dist/cjs/integrations/genai.d.ts +18 -0
- package/dist/cjs/integrations/genai.js +150 -0
- package/dist/cjs/integrations/koa.d.ts +19 -0
- package/dist/cjs/integrations/koa.js +34 -0
- package/dist/cjs/integrations/lambda.d.ts +11 -0
- package/dist/cjs/integrations/lambda.js +68 -0
- package/dist/cjs/integrations/logs.d.ts +10 -0
- package/dist/cjs/integrations/logs.js +77 -0
- package/dist/cjs/integrations/nestjs.d.ts +16 -0
- package/dist/cjs/integrations/nestjs.js +30 -0
- package/dist/cjs/integrations/next.d.ts +11 -0
- package/dist/cjs/integrations/next.js +59 -0
- package/dist/cjs/integrations/prisma.d.ts +3 -0
- package/dist/cjs/integrations/prisma.js +33 -0
- package/dist/cjs/integrations/util.d.ts +9 -0
- package/dist/cjs/integrations/util.js +37 -0
- package/dist/cjs/jest.d.ts +21 -0
- package/dist/cjs/jest.js +45 -0
- package/dist/cjs/load.d.ts +1 -0
- package/dist/cjs/load.js +11 -0
- package/dist/cjs/metrics.d.ts +10 -0
- package/dist/cjs/metrics.js +81 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/payload.d.ts +42 -0
- package/dist/cjs/payload.js +145 -0
- package/dist/cjs/probes.d.ts +97 -0
- package/dist/cjs/probes.js +422 -0
- package/dist/cjs/profiling.d.ts +81 -0
- package/dist/cjs/profiling.js +253 -0
- package/dist/cjs/reporter-core.d.ts +32 -0
- package/dist/cjs/reporter-core.js +102 -0
- package/dist/cjs/test-plan.d.ts +93 -0
- package/dist/cjs/test-plan.js +262 -0
- package/dist/cjs/vitest.d.ts +27 -0
- package/dist/cjs/vitest.js +52 -0
- package/dist/esm/buffer.d.ts +49 -0
- package/dist/esm/buffer.js +92 -0
- package/dist/esm/errors.d.ts +17 -0
- package/dist/esm/errors.js +50 -0
- package/dist/esm/index.d.ts +120 -0
- package/dist/esm/index.js +251 -0
- package/dist/esm/integrations/azure-functions.d.ts +18 -0
- package/dist/esm/integrations/azure-functions.js +60 -0
- package/dist/esm/integrations/express.d.ts +25 -0
- package/dist/esm/integrations/express.js +54 -0
- package/dist/esm/integrations/fastify.d.ts +6 -0
- package/dist/esm/integrations/fastify.js +41 -0
- package/dist/esm/integrations/genai-agents.d.ts +36 -0
- package/dist/esm/integrations/genai-agents.js +118 -0
- package/dist/esm/integrations/genai-stream.d.ts +19 -0
- package/dist/esm/integrations/genai-stream.js +130 -0
- package/dist/esm/integrations/genai.d.ts +18 -0
- package/dist/esm/integrations/genai.js +138 -0
- package/dist/esm/integrations/koa.d.ts +19 -0
- package/dist/esm/integrations/koa.js +31 -0
- package/dist/esm/integrations/lambda.d.ts +11 -0
- package/dist/esm/integrations/lambda.js +32 -0
- package/dist/esm/integrations/logs.d.ts +10 -0
- package/dist/esm/integrations/logs.js +40 -0
- package/dist/esm/integrations/nestjs.d.ts +16 -0
- package/dist/esm/integrations/nestjs.js +26 -0
- package/dist/esm/integrations/next.d.ts +11 -0
- package/dist/esm/integrations/next.js +22 -0
- package/dist/esm/integrations/prisma.d.ts +3 -0
- package/dist/esm/integrations/prisma.js +30 -0
- package/dist/esm/integrations/util.d.ts +9 -0
- package/dist/esm/integrations/util.js +32 -0
- package/dist/esm/jest.d.ts +21 -0
- package/dist/esm/jest.js +41 -0
- package/dist/esm/load.d.ts +1 -0
- package/dist/esm/load.js +5 -0
- package/dist/esm/metrics.d.ts +10 -0
- package/dist/esm/metrics.js +44 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/payload.d.ts +42 -0
- package/dist/esm/payload.js +131 -0
- package/dist/esm/probes.d.ts +97 -0
- package/dist/esm/probes.js +412 -0
- package/dist/esm/profiling.d.ts +81 -0
- package/dist/esm/profiling.js +242 -0
- package/dist/esm/reporter-core.d.ts +32 -0
- package/dist/esm/reporter-core.js +63 -0
- package/dist/esm/test-plan.d.ts +93 -0
- package/dist/esm/test-plan.js +248 -0
- package/dist/esm/vitest.d.ts +27 -0
- package/dist/esm/vitest.js +48 -0
- package/package.json +219 -4
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.suiteName = void 0;
|
|
4
|
+
exports.testId = testId;
|
|
5
|
+
exports.changedFiles = changedFiles;
|
|
6
|
+
exports.fetchTestPlan = fetchTestPlan;
|
|
7
|
+
exports.loadPlan = loadPlan;
|
|
8
|
+
exports.default = globalSetup;
|
|
9
|
+
exports.decider = decider;
|
|
10
|
+
exports.vitestSetup = vitestSetup;
|
|
11
|
+
exports.jestEnvironment = jestEnvironment;
|
|
12
|
+
exports.toJunit = toJunit;
|
|
13
|
+
exports.uploadResults = uploadResults;
|
|
14
|
+
// Test optimization for Vitest and Jest, the same contract as the pytest plugin. Opt in with
|
|
15
|
+
// WT_TEST_OPTIMIZATION=1 (plus WT_INGEST_KEY, WT_PROJECT_ID and WT_API_URL; WT_API_KEY, when set,
|
|
16
|
+
// authenticates the project API calls instead of the ingest key):
|
|
17
|
+
// - a global setup asks `POST /v1/projects/{prj}/tests/plan` once and hands the plan to the
|
|
18
|
+
// workers through a file;
|
|
19
|
+
// - disabled tests are skipped, and so are tests Test Impact Analysis marks unaffected by the
|
|
20
|
+
// changed files (WT_CHANGED_FILES comma-separated, or `git diff --name-only $WT_GIT_BASE...HEAD`);
|
|
21
|
+
// - quarantined tests run, but a failure is turned into a skip (the run stays green) and the
|
|
22
|
+
// reporter records it as a quarantined failure;
|
|
23
|
+
// - a test the project has never seen is retried up to `efd_retries` times (early flake detection).
|
|
24
|
+
// The reporters upload the results as JUnit (`POST /v1/projects/{prj}/test-runs`) when
|
|
25
|
+
// WT_CI_RUN_ID is set, which feeds flaky-test detection and auto-quarantine.
|
|
26
|
+
//
|
|
27
|
+
// Ids are `<suite>::<file relative to cwd> > <describe>... > <test>`, suite = WT_TEST_SUITE or
|
|
28
|
+
// the framework name. Vitest:
|
|
29
|
+
// globalSetup: ['./wt-global.mjs'] // export { default } from '@watchtowerobservability/node/test-plan'
|
|
30
|
+
// setupFiles: ['./wt-setup.ts'] // import { beforeAll, afterEach } from 'vitest';
|
|
31
|
+
// // import { vitestSetup } from '@watchtowerobservability/node/test-plan';
|
|
32
|
+
// // vitestSetup({ beforeAll, afterEach });
|
|
33
|
+
// Jest: globalSetup as above, and testEnvironment: './wt-env.js' with
|
|
34
|
+
// module.exports = require('@watchtowerobservability/node/test-plan').jestEnvironment(require('jest-environment-node').TestEnvironment);
|
|
35
|
+
const node_child_process_1 = require("node:child_process");
|
|
36
|
+
const node_fs_1 = require("node:fs");
|
|
37
|
+
const node_os_1 = require("node:os");
|
|
38
|
+
const node_path_1 = require("node:path");
|
|
39
|
+
const reporter_core_js_1 = require("./reporter-core.js");
|
|
40
|
+
const PLAN_ENV = 'WT_TEST_PLAN_FILE';
|
|
41
|
+
const suiteName = (framework) => process.env.WT_TEST_SUITE || framework;
|
|
42
|
+
exports.suiteName = suiteName;
|
|
43
|
+
function testId(suite, file, names) {
|
|
44
|
+
const f = file ? (0, node_path_1.relative)(process.cwd(), file).split(node_path_1.sep).join('/') : '';
|
|
45
|
+
return `${suite}::${[f, ...names].filter(Boolean).join(' > ')}`;
|
|
46
|
+
}
|
|
47
|
+
function changedFiles() {
|
|
48
|
+
const env = process.env.WT_CHANGED_FILES;
|
|
49
|
+
if (env)
|
|
50
|
+
return env.split(',').map((f) => f.trim()).filter(Boolean);
|
|
51
|
+
const base = process.env.WT_GIT_BASE;
|
|
52
|
+
if (!base || base.startsWith('-'))
|
|
53
|
+
return [];
|
|
54
|
+
try {
|
|
55
|
+
const out = (0, node_child_process_1.execFileSync)('git', ['diff', '--name-only', `${base}...HEAD`, '--'], { encoding: 'utf8', timeout: 30_000 });
|
|
56
|
+
return out.split(/\s+/).filter(Boolean);
|
|
57
|
+
}
|
|
58
|
+
catch (_e) {
|
|
59
|
+
return [];
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
function apiHeaders(client) {
|
|
63
|
+
const key = process.env.WT_API_KEY;
|
|
64
|
+
return key ? { Authorization: `Bearer ${key}` } : { ...client.headers };
|
|
65
|
+
}
|
|
66
|
+
const projectUrl = (client, path) => `${client.apiUrl}/v1/projects/${encodeURIComponent(client.project || '')}/${path}`;
|
|
67
|
+
/** The session plan; `null` without a project or when the API is unreachable (then everything runs). */
|
|
68
|
+
async function fetchTestPlan(client, tests = []) {
|
|
69
|
+
if (!client || !client.project)
|
|
70
|
+
return null;
|
|
71
|
+
try {
|
|
72
|
+
const res = await client.fetch(projectUrl(client, 'tests/plan'), {
|
|
73
|
+
method: 'POST',
|
|
74
|
+
headers: { ...apiHeaders(client), 'Content-Type': 'application/json' },
|
|
75
|
+
body: JSON.stringify({ changed_files: changedFiles(), tests }),
|
|
76
|
+
});
|
|
77
|
+
return res.ok ? (await res.json()) : null;
|
|
78
|
+
}
|
|
79
|
+
catch (_e) {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
let cached;
|
|
84
|
+
/** The plan the global setup wrote (workers and reporters), cached per process. */
|
|
85
|
+
function loadPlan() {
|
|
86
|
+
if (cached === undefined) {
|
|
87
|
+
const f = process.env[PLAN_ENV];
|
|
88
|
+
try {
|
|
89
|
+
cached = f ? JSON.parse((0, node_fs_1.readFileSync)(f, 'utf8')) : null;
|
|
90
|
+
}
|
|
91
|
+
catch (_e) {
|
|
92
|
+
cached = null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return cached;
|
|
96
|
+
}
|
|
97
|
+
/** Vitest/Jest global setup: fetch the plan once and publish it to the workers (which inherit the env). */
|
|
98
|
+
async function globalSetup() {
|
|
99
|
+
if (process.env.WT_TEST_OPTIMIZATION !== '1')
|
|
100
|
+
return;
|
|
101
|
+
const client = (0, reporter_core_js_1.clientFor)({});
|
|
102
|
+
const plan = await fetchTestPlan(client);
|
|
103
|
+
if (client)
|
|
104
|
+
await client.shutdown().catch(() => { });
|
|
105
|
+
if (!plan)
|
|
106
|
+
return;
|
|
107
|
+
// shortcut: the file (a few KB) stays in the OS temp dir; Jest has no teardown return to remove it
|
|
108
|
+
const file = (0, node_path_1.join)((0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), 'wt-plan-')), 'plan.json');
|
|
109
|
+
(0, node_fs_1.writeFileSync)(file, JSON.stringify(plan));
|
|
110
|
+
process.env[PLAN_ENV] = file;
|
|
111
|
+
cached = plan;
|
|
112
|
+
}
|
|
113
|
+
/** Decides per test id; with no plan nothing is skipped or retried. */
|
|
114
|
+
function decider(plan) {
|
|
115
|
+
if (!plan)
|
|
116
|
+
return () => ({ retries: 0 });
|
|
117
|
+
const disabled = new Set(plan.disabled || []);
|
|
118
|
+
const skippable = new Set(plan.skippable || []);
|
|
119
|
+
const fresh = new Set(plan.new_tests || []);
|
|
120
|
+
const quarantined = new Map((plan.quarantined || []).map((q) => [q.id, q.ticket_url || 'no ticket']));
|
|
121
|
+
return (id) => {
|
|
122
|
+
const d = { retries: 0 };
|
|
123
|
+
const q = quarantined.get(id);
|
|
124
|
+
if (disabled.has(id))
|
|
125
|
+
d.skip = 'disabled';
|
|
126
|
+
else if (skippable.has(id) && q === undefined)
|
|
127
|
+
d.skip = 'unaffected';
|
|
128
|
+
if (q !== undefined)
|
|
129
|
+
d.quarantined = q;
|
|
130
|
+
if (fresh.has(id)) {
|
|
131
|
+
d.isNew = true;
|
|
132
|
+
d.retries = Math.max(0, plan.efd_retries || 0);
|
|
133
|
+
}
|
|
134
|
+
return d;
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
/** Applies the plan in a Vitest setup file. Decisions go to `task.meta.wt` for the reporter. */
|
|
138
|
+
function vitestSetup(hooks) {
|
|
139
|
+
const plan = loadPlan();
|
|
140
|
+
if (!plan)
|
|
141
|
+
return;
|
|
142
|
+
const decide = decider(plan);
|
|
143
|
+
const suite = (0, exports.suiteName)('vitest');
|
|
144
|
+
const walk = (t, file, names) => {
|
|
145
|
+
for (const c of t.tasks || []) {
|
|
146
|
+
const path = [...names, c.name];
|
|
147
|
+
if (c.tasks) {
|
|
148
|
+
walk(c, file, path);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
const d = decide(testId(suite, file, path));
|
|
152
|
+
c.meta = { ...c.meta, wt: d };
|
|
153
|
+
if (d.skip && (c.mode === 'run' || c.mode === 'queued'))
|
|
154
|
+
c.mode = 'skip';
|
|
155
|
+
if (d.retries > (c.retry || 0))
|
|
156
|
+
c.retry = d.retries;
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
// a setup file's beforeAll runs on the file suite after collection, before any test
|
|
160
|
+
hooks.beforeAll((file) => walk(file, file.filepath, []));
|
|
161
|
+
hooks.afterEach((ctx) => {
|
|
162
|
+
const d = ctx.task.meta?.wt;
|
|
163
|
+
if (!d?.quarantined || ctx.task.result?.state !== 'fail')
|
|
164
|
+
return;
|
|
165
|
+
const msg = ctx.task.result.errors?.[0]?.message || 'failed';
|
|
166
|
+
ctx.task.meta = { ...ctx.task.meta, wtQuarantinedFailure: msg.slice(0, 2000) };
|
|
167
|
+
ctx.skip(`quarantined in Watchtower (${d.quarantined})`);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
const RETRY_TIMES = Symbol.for('RETRY_TIMES');
|
|
171
|
+
/**
|
|
172
|
+
* Wraps a Jest test environment class so it applies the plan: skips disabled/unaffected tests on
|
|
173
|
+
* `test_start`, raises `jest.retryTimes` for describe blocks holding a new test, and turns a
|
|
174
|
+
* quarantined test's failure into a skip that keeps its failure message (the reporter reads it).
|
|
175
|
+
*/
|
|
176
|
+
function jestEnvironment(Base) {
|
|
177
|
+
return class WatchtowerEnvironment extends Base {
|
|
178
|
+
wtFile;
|
|
179
|
+
wtDecide = decider(loadPlan());
|
|
180
|
+
wtSuite = (0, exports.suiteName)('jest');
|
|
181
|
+
wtRetrySaved = [];
|
|
182
|
+
// any[]: see JestEnvBase
|
|
183
|
+
constructor(...args) {
|
|
184
|
+
super(...args);
|
|
185
|
+
this.wtFile = args[1]?.testPath || '';
|
|
186
|
+
}
|
|
187
|
+
wtId(test) {
|
|
188
|
+
const names = [test.name];
|
|
189
|
+
for (let b = test.parent; b && b.parent; b = b.parent)
|
|
190
|
+
names.unshift(b.name);
|
|
191
|
+
return testId(this.wtSuite, this.wtFile, names);
|
|
192
|
+
}
|
|
193
|
+
async handleTestEvent(event, state) {
|
|
194
|
+
if (super.handleTestEvent)
|
|
195
|
+
await super.handleTestEvent(event, state);
|
|
196
|
+
const t = event.test;
|
|
197
|
+
if (event.name === 'test_start' && t) {
|
|
198
|
+
if (this.wtDecide(this.wtId(t)).skip)
|
|
199
|
+
t.mode = 'skip';
|
|
200
|
+
}
|
|
201
|
+
else if (event.name === 'test_done' && t && t.errors.length > 0 && this.wtDecide(this.wtId(t)).quarantined) {
|
|
202
|
+
t.status = 'skip'; // reported as pending; the errors stay as failureMessages
|
|
203
|
+
}
|
|
204
|
+
else if (event.name === 'run_describe_start' && event.describeBlock) {
|
|
205
|
+
const b = event.describeBlock;
|
|
206
|
+
const prev = this.global[RETRY_TIMES];
|
|
207
|
+
this.wtRetrySaved.push(prev);
|
|
208
|
+
// shortcut: Jest retries per describe block, so other failing tests in a block holding a new
|
|
209
|
+
// test are retried too; fine for flake detection, upgrade if circus gets per-test retries
|
|
210
|
+
const n = Math.max(0, ...(b.children || []).filter((c) => c.type === 'test').map((c) => this.wtDecide(this.wtId({ name: c.name, parent: b })).retries));
|
|
211
|
+
if (n > (Number(prev) || 0))
|
|
212
|
+
this.global[RETRY_TIMES] = n;
|
|
213
|
+
}
|
|
214
|
+
else if (event.name === 'run_describe_finish') {
|
|
215
|
+
this.global[RETRY_TIMES] = this.wtRetrySaved.pop();
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
// ---------------------------------------------------------------- JUnit upload
|
|
221
|
+
const esc = (s) => s.replace(/[&<>"']/g, (c) => `&#${c.charCodeAt(0)};`);
|
|
222
|
+
/** One <testsuite>; a test retried k times gets k failed attempts ahead of its final result. */
|
|
223
|
+
function toJunit(suite, tests) {
|
|
224
|
+
const prefix = `${suite}::`;
|
|
225
|
+
const cases = [];
|
|
226
|
+
for (const t of tests) {
|
|
227
|
+
const id = t.id || `${prefix}${t.name}`;
|
|
228
|
+
const name = esc(id.startsWith(prefix) ? id.slice(prefix.length) : id);
|
|
229
|
+
const time = ((t.durationMs || 0) / 1000).toFixed(3);
|
|
230
|
+
const fail = `<failure message="${esc(String(t.message || 'failed').slice(0, 500))}"/>`;
|
|
231
|
+
for (let i = 0; i < (t.attempts || 0); i++)
|
|
232
|
+
cases.push(`<testcase name="${name}" time="0">${fail}</testcase>`);
|
|
233
|
+
const body = t.outcome === 'failed' ? fail : t.outcome === 'skipped' ? '<skipped/>' : '';
|
|
234
|
+
cases.push(`<testcase name="${name}" time="${time}">${body}</testcase>`);
|
|
235
|
+
}
|
|
236
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites><testsuite name="${esc(suite)}">${cases.join('')}</testsuite></testsuites>\n`;
|
|
237
|
+
}
|
|
238
|
+
/** Posts the results as JUnit when WT_CI_RUN_ID is set; never throws. */
|
|
239
|
+
async function uploadResults(client, suite, tests) {
|
|
240
|
+
const env = process.env;
|
|
241
|
+
const runId = env.WT_CI_RUN_ID;
|
|
242
|
+
if (!client || !client.project || !runId || tests.length === 0)
|
|
243
|
+
return false;
|
|
244
|
+
const form = new FormData();
|
|
245
|
+
const fields = [
|
|
246
|
+
['run_id', runId],
|
|
247
|
+
['branch', env.WT_BRANCH || env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME || env.CI_COMMIT_REF_NAME],
|
|
248
|
+
['sha', env.WT_SHA || env.GITHUB_SHA || env.CI_COMMIT_SHA],
|
|
249
|
+
['suite', suite],
|
|
250
|
+
];
|
|
251
|
+
for (const [k, v] of fields)
|
|
252
|
+
if (v)
|
|
253
|
+
form.append(k, v.slice(0, 200));
|
|
254
|
+
form.append('file', new Blob([toJunit(suite, tests)], { type: 'application/xml' }), 'junit.xml');
|
|
255
|
+
try {
|
|
256
|
+
const res = await client.fetch(projectUrl(client, 'test-runs'), { method: 'POST', headers: apiHeaders(client), body: form });
|
|
257
|
+
return res.ok;
|
|
258
|
+
}
|
|
259
|
+
catch (_e) {
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { ReporterOptions } from './reporter-core.js';
|
|
2
|
+
import type { TestDecision } from './test-plan.js';
|
|
3
|
+
/** The slice of Vitest's task tree this reporter reads. */
|
|
4
|
+
export interface VitestTask {
|
|
5
|
+
type?: string;
|
|
6
|
+
name: string;
|
|
7
|
+
filepath?: string;
|
|
8
|
+
mode?: string;
|
|
9
|
+
meta?: {
|
|
10
|
+
wt?: TestDecision;
|
|
11
|
+
wtQuarantinedFailure?: string;
|
|
12
|
+
};
|
|
13
|
+
result?: {
|
|
14
|
+
state?: string;
|
|
15
|
+
duration?: number;
|
|
16
|
+
retryCount?: number;
|
|
17
|
+
errors?: {
|
|
18
|
+
message?: string;
|
|
19
|
+
}[];
|
|
20
|
+
};
|
|
21
|
+
tasks?: VitestTask[];
|
|
22
|
+
}
|
|
23
|
+
export default class WatchtowerVitestReporter {
|
|
24
|
+
private readonly client;
|
|
25
|
+
constructor(options?: ReporterOptions);
|
|
26
|
+
onFinished(files?: VitestTask[]): Promise<void>;
|
|
27
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
// Vitest reporter: `reporters: ['default', ['@watchtowerobservability/node/vitest', { /* apiKey, client, ... */ }]]`
|
|
4
|
+
// (or `new WatchtowerVitestReporter(opts)` in vitest.config). Reads the file/task tree handed to
|
|
5
|
+
// `onFinished`, which every Vitest version still calls. With the test-plan setup (test-plan.ts) it
|
|
6
|
+
// also reports quarantined failures and retries, and uploads JUnit when WT_CI_RUN_ID is set.
|
|
7
|
+
const reporter_core_js_1 = require("./reporter-core.js");
|
|
8
|
+
const test_plan_js_1 = require("./test-plan.js");
|
|
9
|
+
function collect(task, names, file, out, end) {
|
|
10
|
+
// a file task's name is its path; keep it out of test names
|
|
11
|
+
const path = task.filepath ? [] : [...names, task.name];
|
|
12
|
+
const f = task.filepath || file;
|
|
13
|
+
if (task.type === 'test' || task.type === 'custom') {
|
|
14
|
+
const r = task.result || {};
|
|
15
|
+
const wt = task.meta?.wt;
|
|
16
|
+
const qFail = task.meta?.wtQuarantinedFailure;
|
|
17
|
+
const retried = r.retryCount || 0;
|
|
18
|
+
out.push({
|
|
19
|
+
name: path.join(' '),
|
|
20
|
+
id: (0, test_plan_js_1.testId)((0, test_plan_js_1.suiteName)('vitest'), f, path),
|
|
21
|
+
outcome: qFail ? 'failed' : (0, reporter_core_js_1.outcome)(task.mode === 'skip' || task.mode === 'todo' ? 'skip' : r.state),
|
|
22
|
+
durationMs: Math.round(r.duration || 0),
|
|
23
|
+
message: qFail || (r.errors && r.errors[0] && r.errors[0].message),
|
|
24
|
+
endMs: end,
|
|
25
|
+
quarantined: qFail ? wt?.quarantined || 'no ticket' : undefined,
|
|
26
|
+
isNew: wt?.isNew,
|
|
27
|
+
// every retry before the final attempt was a failure
|
|
28
|
+
attempts: r.state === 'fail' || r.state === 'pass' ? retried : 0,
|
|
29
|
+
});
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
for (const t of task.tasks || [])
|
|
33
|
+
collect(t, path, f, out, end);
|
|
34
|
+
}
|
|
35
|
+
class WatchtowerVitestReporter {
|
|
36
|
+
client;
|
|
37
|
+
constructor(options) {
|
|
38
|
+
this.client = (0, reporter_core_js_1.clientFor)(options);
|
|
39
|
+
}
|
|
40
|
+
async onFinished(files) {
|
|
41
|
+
const tests = [];
|
|
42
|
+
const end = Date.now();
|
|
43
|
+
for (const f of files || [])
|
|
44
|
+
collect(f, [], undefined, tests, end);
|
|
45
|
+
await (0, reporter_core_js_1.reportTests)(this.client, tests, 'vitest');
|
|
46
|
+
await (0, test_plan_js_1.uploadResults)(this.client, (0, test_plan_js_1.suiteName)('vitest'), tests);
|
|
47
|
+
if (this.client)
|
|
48
|
+
await this.client.shutdown().catch(() => { });
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
exports.default = WatchtowerVitestReporter;
|
|
52
|
+
module.exports = Object.assign(exports.default, { default: exports.default });
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { SpanExporter, ReadableSpan } from '@opentelemetry/sdk-trace-base';
|
|
2
|
+
import type { LogRecordExporter, ReadableLogRecord } from '@opentelemetry/sdk-logs';
|
|
3
|
+
import { AggregationTemporality } from '@opentelemetry/sdk-metrics';
|
|
4
|
+
import type { InstrumentType, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
|
|
5
|
+
type Done = Parameters<SpanExporter['export']>[1];
|
|
6
|
+
/** What the spool wraps: a span, log record or metric exporter; `B` is one export batch. */
|
|
7
|
+
export interface BatchExporter<B> {
|
|
8
|
+
export(batch: B, done: Done): void;
|
|
9
|
+
shutdown(): Promise<void>;
|
|
10
|
+
forceFlush?(): Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
export interface SpoolOptions<B = ReadableSpan[]> {
|
|
13
|
+
maxFiles?: number;
|
|
14
|
+
encode?: (batch: B) => Uint8Array;
|
|
15
|
+
}
|
|
16
|
+
/** OTLP/JSON encoders for the three signals. */
|
|
17
|
+
export declare const encodeSpans: (spans: ReadableSpan[]) => Uint8Array;
|
|
18
|
+
export declare const encodeLogs: (logs: ReadableLogRecord[]) => Uint8Array;
|
|
19
|
+
export declare const encodeMetrics: (rm: ResourceMetrics) => Uint8Array;
|
|
20
|
+
export declare class SpoolExporter<B = ReadableSpan[]> implements BatchExporter<B> {
|
|
21
|
+
private readonly inner;
|
|
22
|
+
private readonly dir;
|
|
23
|
+
private readonly url;
|
|
24
|
+
private readonly headers;
|
|
25
|
+
private readonly fetch;
|
|
26
|
+
private readonly maxFiles;
|
|
27
|
+
private readonly encode;
|
|
28
|
+
constructor(inner: BatchExporter<B>, dir: string, url: string, headers: Record<string, string>, fetch: typeof globalThis.fetch, opts?: SpoolOptions<B>);
|
|
29
|
+
private _files;
|
|
30
|
+
replay(): Promise<void>;
|
|
31
|
+
export(batch: B, done: Done): void;
|
|
32
|
+
private _spool;
|
|
33
|
+
shutdown(): Promise<void>;
|
|
34
|
+
forceFlush(): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
/** A spool for log records: same directory rules, `/v1/logs` replay. */
|
|
37
|
+
export declare const spoolLogs: (inner: LogRecordExporter, dir: string, url: string, headers: Record<string, string>, fetch: typeof globalThis.fetch, opts?: {
|
|
38
|
+
maxFiles?: number;
|
|
39
|
+
}) => SpoolExporter<ReadableLogRecord[]>;
|
|
40
|
+
/** A spool for metrics (`/v1/metrics` replay); the reader still asks the real exporter for temporality. */
|
|
41
|
+
export declare class MetricSpool extends SpoolExporter<ResourceMetrics> implements PushMetricExporter {
|
|
42
|
+
private readonly metricInner;
|
|
43
|
+
constructor(metricInner: PushMetricExporter, dir: string, url: string, headers: Record<string, string>, fetch: typeof globalThis.fetch, opts?: {
|
|
44
|
+
maxFiles?: number;
|
|
45
|
+
});
|
|
46
|
+
selectAggregationTemporality(t: InstrumentType): AggregationTemporality;
|
|
47
|
+
forceFlush(): Promise<void>;
|
|
48
|
+
}
|
|
49
|
+
export {};
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Offline buffer: failed OTLP batches (spans, log records or metrics) are spooled to disk, at most
|
|
2
|
+
// `maxFiles` batches per signal directory (oldest dropped), and replayed oldest first before the
|
|
3
|
+
// next export, so they go out as soon as the endpoint is reachable again.
|
|
4
|
+
import * as fs from 'node:fs';
|
|
5
|
+
import * as path from 'node:path';
|
|
6
|
+
import * as crypto from 'node:crypto';
|
|
7
|
+
import { AggregationTemporality } from '@opentelemetry/sdk-metrics';
|
|
8
|
+
import { JsonLogsSerializer, JsonMetricsSerializer, JsonTraceSerializer } from '@opentelemetry/otlp-transformer';
|
|
9
|
+
/** OTLP/JSON encoders for the three signals. */
|
|
10
|
+
export const encodeSpans = (spans) => Buffer.from(JsonTraceSerializer.serializeRequest(spans));
|
|
11
|
+
export const encodeLogs = (logs) => Buffer.from(JsonLogsSerializer.serializeRequest(logs));
|
|
12
|
+
export const encodeMetrics = (rm) => Buffer.from(JsonMetricsSerializer.serializeRequest(rm));
|
|
13
|
+
export class SpoolExporter {
|
|
14
|
+
inner;
|
|
15
|
+
dir;
|
|
16
|
+
url;
|
|
17
|
+
headers;
|
|
18
|
+
fetch;
|
|
19
|
+
maxFiles;
|
|
20
|
+
encode;
|
|
21
|
+
constructor(inner, dir, url, headers, fetch, opts) {
|
|
22
|
+
this.inner = inner;
|
|
23
|
+
this.dir = dir;
|
|
24
|
+
this.url = url;
|
|
25
|
+
this.headers = headers;
|
|
26
|
+
this.fetch = fetch;
|
|
27
|
+
this.maxFiles = (opts && opts.maxFiles) || 500;
|
|
28
|
+
this.encode = (opts && opts.encode) || encodeSpans;
|
|
29
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
30
|
+
}
|
|
31
|
+
_files() {
|
|
32
|
+
return fs.readdirSync(this.dir)
|
|
33
|
+
.map((n) => ({ n, t: fs.statSync(path.join(this.dir, n)).mtimeMs }))
|
|
34
|
+
.sort((a, b) => a.t - b.t || a.n.localeCompare(b.n))
|
|
35
|
+
.map((f) => f.n);
|
|
36
|
+
}
|
|
37
|
+
async replay() {
|
|
38
|
+
for (const n of this._files()) {
|
|
39
|
+
const p = path.join(this.dir, n);
|
|
40
|
+
try {
|
|
41
|
+
const res = await this.fetch(this.url, {
|
|
42
|
+
method: 'POST',
|
|
43
|
+
headers: { ...this.headers, 'Content-Type': 'application/json' },
|
|
44
|
+
body: fs.readFileSync(p),
|
|
45
|
+
});
|
|
46
|
+
if (!res.ok)
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return; // still offline; keep order
|
|
51
|
+
}
|
|
52
|
+
fs.unlinkSync(p);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export(batch, done) {
|
|
56
|
+
void this.replay().then(() => this.inner.export(batch, (r) => {
|
|
57
|
+
if (r.code !== 0)
|
|
58
|
+
this._spool(this.encode(batch));
|
|
59
|
+
done(r);
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
_spool(payload) {
|
|
63
|
+
const files = this._files();
|
|
64
|
+
for (const old of files.slice(0, Math.max(0, files.length - this.maxFiles + 1))) {
|
|
65
|
+
fs.unlinkSync(path.join(this.dir, old)); // drop-oldest
|
|
66
|
+
}
|
|
67
|
+
fs.writeFileSync(path.join(this.dir, `${crypto.randomUUID()}.json`), payload);
|
|
68
|
+
}
|
|
69
|
+
shutdown() {
|
|
70
|
+
return this.inner.shutdown();
|
|
71
|
+
}
|
|
72
|
+
forceFlush() {
|
|
73
|
+
return this.inner.forceFlush ? this.inner.forceFlush() : Promise.resolve();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/** A spool for log records: same directory rules, `/v1/logs` replay. */
|
|
77
|
+
export const spoolLogs = (inner, dir, url, headers, fetch, opts) => new SpoolExporter(inner, dir, url, headers, fetch, { ...opts, encode: encodeLogs });
|
|
78
|
+
/** A spool for metrics (`/v1/metrics` replay); the reader still asks the real exporter for temporality. */
|
|
79
|
+
export class MetricSpool extends SpoolExporter {
|
|
80
|
+
metricInner;
|
|
81
|
+
constructor(metricInner, dir, url, headers, fetch, opts) {
|
|
82
|
+
super(metricInner, dir, url, headers, fetch, { ...opts, encode: encodeMetrics });
|
|
83
|
+
this.metricInner = metricInner;
|
|
84
|
+
}
|
|
85
|
+
selectAggregationTemporality(t) {
|
|
86
|
+
const pick = this.metricInner.selectAggregationTemporality;
|
|
87
|
+
return pick ? pick.call(this.metricInner, t) : AggregationTemporality.CUMULATIVE; // the SDK default
|
|
88
|
+
}
|
|
89
|
+
forceFlush() {
|
|
90
|
+
return this.metricInner.forceFlush();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface CaptureContext {
|
|
2
|
+
/** false for crashes (uncaught); default true. Keys starting with `wt.` are dropped. */
|
|
3
|
+
handled?: boolean;
|
|
4
|
+
attributes?: Record<string, string | number | boolean | undefined>;
|
|
5
|
+
}
|
|
6
|
+
/** The slice of `process` the hooks need; tests pass an EventEmitter. */
|
|
7
|
+
export interface ProcessLike {
|
|
8
|
+
on(event: string, listener: (arg: unknown) => void): unknown;
|
|
9
|
+
off(event: string, listener: (arg: unknown) => void): unknown;
|
|
10
|
+
listenerCount(event: string): number;
|
|
11
|
+
exit(code?: number): unknown;
|
|
12
|
+
}
|
|
13
|
+
export declare function dropReserved<T>(attrs?: Record<string, T>): Record<string, T>;
|
|
14
|
+
/** Record err on the active span (or a new `exception` span). Returns the trace id. */
|
|
15
|
+
export declare function captureException(err: unknown, ctx?: CaptureContext): string;
|
|
16
|
+
/** uncaughtException / unhandledRejection hooks. Exits like Node would, but only when we are the sole listener. */
|
|
17
|
+
export declare function installProcessHandlers(flush: () => Promise<unknown>, proc?: ProcessLike): () => void;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Error capture: an exception becomes an OTel span event (exception.*) with trace and span ids.
|
|
2
|
+
import { trace, SpanStatusCode } from '@opentelemetry/api';
|
|
3
|
+
const captured = new WeakMap(); // error object -> traceId, so framework + process hooks never double-report
|
|
4
|
+
export function dropReserved(attrs) {
|
|
5
|
+
return Object.fromEntries(Object.entries(attrs || {}).filter(([k]) => !k.startsWith('wt.')));
|
|
6
|
+
}
|
|
7
|
+
/** Record err on the active span (or a new `exception` span). Returns the trace id. */
|
|
8
|
+
export function captureException(err, ctx) {
|
|
9
|
+
const o = ctx || {};
|
|
10
|
+
const e = err instanceof Error ? err : new Error(typeof err === 'string' ? err : JSON.stringify(err));
|
|
11
|
+
const seen = captured.get(e);
|
|
12
|
+
if (seen !== undefined)
|
|
13
|
+
return seen;
|
|
14
|
+
const attrs = { 'error.handled': o.handled !== false, ...dropReserved(o.attributes) };
|
|
15
|
+
const active = trace.getActiveSpan();
|
|
16
|
+
const span = active && active.isRecording() ? active : trace.getTracer('watchtower').startSpan('exception');
|
|
17
|
+
span.setAttributes(attrs);
|
|
18
|
+
span.recordException(e);
|
|
19
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: e.message });
|
|
20
|
+
const traceId = span.spanContext().traceId;
|
|
21
|
+
if (span !== active)
|
|
22
|
+
span.end();
|
|
23
|
+
captured.set(e, traceId);
|
|
24
|
+
return traceId;
|
|
25
|
+
}
|
|
26
|
+
/** uncaughtException / unhandledRejection hooks. Exits like Node would, but only when we are the sole listener. */
|
|
27
|
+
export function installProcessHandlers(flush, proc) {
|
|
28
|
+
const p = proc || process;
|
|
29
|
+
const exitAfterFlush = () => Promise.race([flush(), new Promise((r) => setTimeout(r, 2000).unref())]).finally(() => p.exit(1));
|
|
30
|
+
const onException = (e) => {
|
|
31
|
+
captureException(e, { handled: false, attributes: { 'error.mechanism': 'uncaughtException' } });
|
|
32
|
+
if (p.listenerCount('uncaughtException') > 1)
|
|
33
|
+
return;
|
|
34
|
+
console.error(e);
|
|
35
|
+
exitAfterFlush();
|
|
36
|
+
};
|
|
37
|
+
const onRejection = (reason) => {
|
|
38
|
+
captureException(reason, { handled: false, attributes: { 'error.mechanism': 'unhandledRejection' } });
|
|
39
|
+
if (p.listenerCount('unhandledRejection') > 1)
|
|
40
|
+
return;
|
|
41
|
+
console.error(reason);
|
|
42
|
+
exitAfterFlush();
|
|
43
|
+
};
|
|
44
|
+
p.on('uncaughtException', onException);
|
|
45
|
+
p.on('unhandledRejection', onRejection);
|
|
46
|
+
return () => {
|
|
47
|
+
p.off('uncaughtException', onException);
|
|
48
|
+
p.off('unhandledRejection', onRejection);
|
|
49
|
+
};
|
|
50
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
import { metrics, trace } from '@opentelemetry/api';
|
|
3
|
+
import type { Tracer } from '@opentelemetry/api';
|
|
4
|
+
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
5
|
+
import { LoggerProvider } from '@opentelemetry/sdk-logs';
|
|
6
|
+
import type { MeterProvider } from '@opentelemetry/sdk-metrics';
|
|
7
|
+
import { captureException } from './errors.js';
|
|
8
|
+
import type { CaptureContext } from './errors.js';
|
|
9
|
+
export { captureException, metrics, trace };
|
|
10
|
+
export { setCaptureRules, capturePayload } from './payload.js';
|
|
11
|
+
export type { CaptureRule } from './payload.js';
|
|
12
|
+
export type { CaptureContext };
|
|
13
|
+
export declare const DEFAULT_ENDPOINT = "http://localhost:4318";
|
|
14
|
+
export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
|
|
15
|
+
export interface InitOptions {
|
|
16
|
+
/** Ingest key; defaults to WT_INGEST_KEY. */
|
|
17
|
+
apiKey?: string;
|
|
18
|
+
/** Defaults to WT_ENVIRONMENT. */
|
|
19
|
+
environment?: string;
|
|
20
|
+
/** Defaults to WT_RELEASE. */
|
|
21
|
+
release?: string;
|
|
22
|
+
/** Hook uncaughtException / unhandledRejection (default true). */
|
|
23
|
+
captureProcessErrors?: boolean;
|
|
24
|
+
/** Replace the log exporter (tests). */
|
|
25
|
+
logExporter?: unknown;
|
|
26
|
+
/** Export OTLP logs; defaults to WATCHTOWER_LOGS (on unless `off`; OTEL_LOGS_EXPORTER=none also turns it off). */
|
|
27
|
+
logs?: boolean;
|
|
28
|
+
/** Export OTLP metrics; defaults to WATCHTOWER_METRICS (on unless `off`; OTEL_METRICS_EXPORTER=none also turns it off). */
|
|
29
|
+
metrics?: boolean;
|
|
30
|
+
/** Replace the metric exporter (tests). */
|
|
31
|
+
metricExporter?: unknown;
|
|
32
|
+
/** OTLP/HTTP base URL; defaults to WT_OTLP_ENDPOINT or http://localhost:4318. */
|
|
33
|
+
endpoint?: string;
|
|
34
|
+
/** REST API base URL for getPrompt/score; defaults to WT_API_URL or endpoint. */
|
|
35
|
+
apiUrl?: string;
|
|
36
|
+
project?: string;
|
|
37
|
+
/** Defaults to WT_SERVICE. */
|
|
38
|
+
serviceName?: string;
|
|
39
|
+
/** Resource attributes; keys starting with `wt.` are dropped. */
|
|
40
|
+
resource?: Record<string, string | number | boolean>;
|
|
41
|
+
/** Called for every span attribute before export; return the value to keep. */
|
|
42
|
+
redact?: (key: string, value: unknown) => unknown;
|
|
43
|
+
/** Directory for the offline buffer. */
|
|
44
|
+
bufferDir?: string;
|
|
45
|
+
/** Fetch the service's payload capture rules (APM-016) and record bodies/GraphQL where a rule matches; defaults to WT_CAPTURE_RULES=1. */
|
|
46
|
+
captureRules?: boolean;
|
|
47
|
+
/** Replace the span exporter (tests). */
|
|
48
|
+
exporter?: unknown;
|
|
49
|
+
fetch?: typeof fetch;
|
|
50
|
+
}
|
|
51
|
+
export interface EvalRow {
|
|
52
|
+
case: number;
|
|
53
|
+
traceId: string;
|
|
54
|
+
scores: Record<string, unknown>;
|
|
55
|
+
errors: Record<string, string>;
|
|
56
|
+
}
|
|
57
|
+
export interface EvalOptions {
|
|
58
|
+
name?: string;
|
|
59
|
+
errorTolerance?: boolean;
|
|
60
|
+
/** Rows of an earlier run; cases that finished without errors are kept, not re-run. */
|
|
61
|
+
resume?: EvalRow[];
|
|
62
|
+
}
|
|
63
|
+
export type Scorers = Record<string, (testCase: any, output: any) => unknown | Promise<unknown>>;
|
|
64
|
+
export type EvalTask = (input: any) => unknown | Promise<unknown>;
|
|
65
|
+
export interface ScoreOptions {
|
|
66
|
+
name?: string;
|
|
67
|
+
spanId?: string;
|
|
68
|
+
comment?: string;
|
|
69
|
+
}
|
|
70
|
+
/** Drop caller-supplied wt.* keys: tenant identity comes only from the key. */
|
|
71
|
+
export declare function safeResource(attrs?: Record<string, unknown>): Record<string, unknown>;
|
|
72
|
+
export declare class Client {
|
|
73
|
+
readonly endpoint: string;
|
|
74
|
+
readonly apiUrl: string;
|
|
75
|
+
readonly project: string | undefined;
|
|
76
|
+
readonly headers: {
|
|
77
|
+
Authorization: string;
|
|
78
|
+
};
|
|
79
|
+
readonly fetch: typeof fetch;
|
|
80
|
+
readonly redact: InitOptions['redact'];
|
|
81
|
+
readonly provider: NodeTracerProvider;
|
|
82
|
+
readonly tracer: Tracer;
|
|
83
|
+
readonly loggerProvider: LoggerProvider;
|
|
84
|
+
readonly logger: ReturnType<LoggerProvider['getLogger']>;
|
|
85
|
+
/** Set when metrics are on; `init` makes it the global meter provider (`metrics.getMeter`). */
|
|
86
|
+
readonly meterProvider?: MeterProvider;
|
|
87
|
+
private readonly stopRuntime;
|
|
88
|
+
uninstall?: () => void;
|
|
89
|
+
constructor(opts: InitOptions);
|
|
90
|
+
/** Emit an OTLP log record; trace and span ids come from the active context. */
|
|
91
|
+
log(level: LogLevel, body: string, attributes?: Record<string, unknown>): void;
|
|
92
|
+
forceFlush(): Promise<unknown>;
|
|
93
|
+
toJSON(): {
|
|
94
|
+
endpoint: string;
|
|
95
|
+
project: string | undefined;
|
|
96
|
+
headers: {
|
|
97
|
+
Authorization: string;
|
|
98
|
+
};
|
|
99
|
+
};
|
|
100
|
+
[inspect.custom](): string;
|
|
101
|
+
private _call;
|
|
102
|
+
getPrompt(promptId: string): Promise<unknown>;
|
|
103
|
+
score(traceId: string, value: number | boolean | string, o?: ScoreOptions): Promise<unknown>;
|
|
104
|
+
/** Run task per case under a span; record each scorer result as a score when a project is set. */
|
|
105
|
+
evaluate(dataset: unknown[], task: EvalTask, scorers: Scorers, o?: EvalOptions): Promise<EvalRow[]>;
|
|
106
|
+
/** Re-run only the cases that errored or are missing in `previous`. */
|
|
107
|
+
evaluateResume(dataset: unknown[], task: EvalTask, scorers: Scorers, previous: EvalRow[], o?: EvalOptions): Promise<EvalRow[]>;
|
|
108
|
+
/** Wrap fn in a span. */
|
|
109
|
+
observe<A extends unknown[], R>(name: string, fn: (...a: A) => R): (...a: A) => R;
|
|
110
|
+
shutdown(): Promise<void>;
|
|
111
|
+
}
|
|
112
|
+
export declare function init(opts: InitOptions): Client;
|
|
113
|
+
export declare const flush: () => Promise<unknown>;
|
|
114
|
+
export declare const log: (l: LogLevel, b: string, a?: Record<string, unknown>) => void;
|
|
115
|
+
export declare const getPrompt: (id: string) => Promise<unknown>;
|
|
116
|
+
export declare const score: (t: string, v: number | boolean | string, o?: ScoreOptions) => Promise<unknown>;
|
|
117
|
+
export declare const evaluate: (d: unknown[], t: EvalTask, s: Scorers, o?: EvalOptions) => Promise<EvalRow[]>;
|
|
118
|
+
export declare const evaluateResume: (d: unknown[], t: EvalTask, s: Scorers, prev: EvalRow[], o?: EvalOptions) => Promise<EvalRow[]>;
|
|
119
|
+
export declare const observe: <A extends unknown[], R>(n: string, f: (...a: A) => R) => ((...a: A) => R);
|
|
120
|
+
export declare const shutdown: () => Promise<void>;
|