@attest-ai/core 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/cjs/index.cjs +2484 -0
  2. package/dist/cjs/index.cjs.map +1 -0
  3. package/dist/cjs/index.d.cts +1085 -0
  4. package/dist/cjs/proto/index.cjs +208 -0
  5. package/dist/cjs/proto/index.cjs.map +1 -0
  6. package/dist/cjs/proto/index.d.cts +228 -0
  7. package/dist/client.d.ts +7 -3
  8. package/dist/client.d.ts.map +1 -1
  9. package/dist/client.js +48 -9
  10. package/dist/client.js.map +1 -1
  11. package/dist/config.d.ts +21 -0
  12. package/dist/config.d.ts.map +1 -0
  13. package/dist/config.js +31 -0
  14. package/dist/config.js.map +1 -0
  15. package/dist/delegate.d.ts.map +1 -1
  16. package/dist/delegate.js +1 -7
  17. package/dist/delegate.js.map +1 -1
  18. package/dist/esm/cli.d.ts +9 -0
  19. package/dist/esm/cli.js +154 -0
  20. package/dist/esm/cli.js.map +1 -0
  21. package/dist/esm/index.d.ts +1085 -0
  22. package/dist/esm/index.js +2385 -0
  23. package/dist/esm/index.js.map +1 -0
  24. package/dist/esm/proto/index.d.ts +228 -0
  25. package/dist/esm/proto/index.js +152 -0
  26. package/dist/esm/proto/index.js.map +1 -0
  27. package/dist/expect.d.ts +8 -2
  28. package/dist/expect.d.ts.map +1 -1
  29. package/dist/expect.js +12 -1
  30. package/dist/expect.js.map +1 -1
  31. package/dist/index.d.ts +1 -0
  32. package/dist/index.d.ts.map +1 -1
  33. package/dist/index.js +1 -0
  34. package/dist/index.js.map +1 -1
  35. package/dist/proto/errors.d.ts +5 -0
  36. package/dist/proto/errors.d.ts.map +1 -1
  37. package/dist/proto/errors.js +11 -0
  38. package/dist/proto/errors.js.map +1 -1
  39. package/dist/simulation.d.ts +8 -0
  40. package/dist/simulation.d.ts.map +1 -0
  41. package/dist/simulation.js +21 -0
  42. package/dist/simulation.js.map +1 -0
  43. package/dist/trace.d.ts +1 -0
  44. package/dist/trace.d.ts.map +1 -1
  45. package/dist/trace.js +3 -0
  46. package/dist/trace.js.map +1 -1
  47. package/dist/version.d.ts +1 -1
  48. package/dist/version.js +1 -1
  49. package/package.json +34 -8
@@ -0,0 +1,2385 @@
1
+ // src/proto/constants.ts
2
+ var STEP_LLM_CALL = "llm_call";
3
+ var STEP_TOOL_CALL = "tool_call";
4
+ var STEP_RETRIEVAL = "retrieval";
5
+ var STEP_AGENT_CALL = "agent_call";
6
+ var STATUS_PASS = "pass";
7
+ var STATUS_SOFT_FAIL = "soft_fail";
8
+ var STATUS_HARD_FAIL = "hard_fail";
9
+ var TYPE_SCHEMA = "schema";
10
+ var TYPE_CONSTRAINT = "constraint";
11
+ var TYPE_TRACE = "trace";
12
+ var TYPE_CONTENT = "content";
13
+ var TYPE_EMBEDDING = "embedding";
14
+ var TYPE_LLM_JUDGE = "llm_judge";
15
+ var TYPE_TRACE_TREE = "trace_tree";
16
+ var TYPE_PLUGIN = "plugin";
17
+ var ERR_INVALID_TRACE = 1001;
18
+ var ERR_ASSERTION_ERROR = 1002;
19
+ var ERR_PROVIDER_ERROR = 2001;
20
+ var ERR_ENGINE_ERROR = 3001;
21
+ var ERR_TIMEOUT = 3002;
22
+ var ERR_SESSION_ERROR = 3003;
23
+
24
+ // src/proto/types.ts
25
+ function traceId(id) {
26
+ return id;
27
+ }
28
+ function assertionId(id) {
29
+ return id;
30
+ }
31
+ function agentId(id) {
32
+ return id;
33
+ }
34
+
35
+ // src/proto/errors.ts
36
+ var ProtocolError = class extends Error {
37
+ code;
38
+ errorMessage;
39
+ data;
40
+ constructor(code, message, data) {
41
+ super(message);
42
+ this.name = "ProtocolError";
43
+ this.code = code;
44
+ this.errorMessage = message;
45
+ this.data = data;
46
+ }
47
+ };
48
+ var EngineTimeoutError = class extends Error {
49
+ method;
50
+ timeoutMs;
51
+ constructor(method, timeoutMs) {
52
+ super(
53
+ `Engine request '${method}' timed out after ${timeoutMs}ms. Set ATTEST_ENGINE_TIMEOUT to increase the limit.`
54
+ );
55
+ this.name = "EngineTimeoutError";
56
+ this.method = method;
57
+ this.timeoutMs = timeoutMs;
58
+ }
59
+ };
60
+
61
+ // src/proto/codec.ts
62
+ function encodeRequest(id, method, params) {
63
+ const msg = {
64
+ jsonrpc: "2.0",
65
+ id,
66
+ method,
67
+ params
68
+ };
69
+ return JSON.stringify(msg) + "\n";
70
+ }
71
+ function decodeResponse(line) {
72
+ const text = line.trim();
73
+ if (!text) {
74
+ throw new Error("empty response line");
75
+ }
76
+ let data;
77
+ try {
78
+ data = JSON.parse(text);
79
+ } catch (e) {
80
+ throw new Error(`malformed JSON response: ${e instanceof Error ? e.message : String(e)}`);
81
+ }
82
+ if (typeof data !== "object" || data === null || Array.isArray(data)) {
83
+ throw new Error(`expected JSON object, got ${Array.isArray(data) ? "array" : typeof data}`);
84
+ }
85
+ const obj = data;
86
+ if (obj.jsonrpc !== "2.0") {
87
+ throw new Error(`invalid jsonrpc version: ${String(obj.jsonrpc)}`);
88
+ }
89
+ if (obj.error != null) {
90
+ const err = obj.error;
91
+ let errorData;
92
+ if (err.data != null) {
93
+ const raw = err.data;
94
+ errorData = {
95
+ error_type: raw.error_type ?? "",
96
+ retryable: raw.retryable ?? false,
97
+ detail: raw.detail ?? ""
98
+ };
99
+ }
100
+ throw new ProtocolError(
101
+ err.code ?? -1,
102
+ err.message ?? "unknown error",
103
+ errorData
104
+ );
105
+ }
106
+ return obj;
107
+ }
108
+ function extractResult(response) {
109
+ if (response.result === void 0) {
110
+ throw new Error("response missing 'result' field");
111
+ }
112
+ return response.result;
113
+ }
114
+ function extractId(response) {
115
+ if (response.id === void 0) {
116
+ throw new Error("response missing 'id' field");
117
+ }
118
+ return Number(response.id);
119
+ }
120
+
121
+ // src/version.ts
122
+ var VERSION = "0.5.0";
123
+ var ENGINE_VERSION = "0.5.0";
124
+
125
+ // src/engine-manager.ts
126
+ import { spawn, execSync } from "child_process";
127
+ import { createInterface } from "readline";
128
+ import { existsSync as existsSync2, accessSync, constants as fsConstants } from "fs";
129
+ import { resolve, join as join2, dirname } from "path";
130
+ import { fileURLToPath } from "url";
131
+
132
+ // src/engine-downloader.ts
133
+ import * as https from "https";
134
+ import * as http from "http";
135
+ import * as fs from "fs";
136
+ import * as path from "path";
137
+ import * as os from "os";
138
+ import * as crypto from "crypto";
139
+ var RELEASE_BASE = `https://github.com/attest-framework/attest/releases/download/v${ENGINE_VERSION}`;
140
+ var CHECKSUMS_URL = `${RELEASE_BASE}/checksums-sha256.txt`;
141
+ var MAX_REDIRECTS = 5;
142
+ function attestBinDir() {
143
+ const dir = path.join(os.homedir(), ".attest", "bin");
144
+ fs.mkdirSync(dir, { recursive: true });
145
+ return dir;
146
+ }
147
+ function platformKey() {
148
+ const platform2 = os.platform();
149
+ const arch2 = os.arch();
150
+ const mapping = {
151
+ darwin: { arm64: "darwin-arm64", x64: "darwin-amd64" },
152
+ linux: { arm64: "linux-arm64", x64: "linux-amd64" },
153
+ win32: { x64: "windows-amd64", arm64: "windows-arm64" }
154
+ };
155
+ const osMap = mapping[platform2];
156
+ if (!osMap) {
157
+ throw new Error(
158
+ `Unsupported platform: ${platform2}. Supported: darwin, linux, win32.`
159
+ );
160
+ }
161
+ const key = osMap[arch2];
162
+ if (!key) {
163
+ throw new Error(
164
+ `Unsupported architecture '${arch2}' on platform '${platform2}'. Supported architectures: ${Object.keys(osMap).join(", ")}.`
165
+ );
166
+ }
167
+ return key;
168
+ }
169
+ function binaryFilename() {
170
+ return os.platform() === "win32" ? "attest-engine.exe" : "attest-engine";
171
+ }
172
+ function cachedEnginePath() {
173
+ const dir = path.join(os.homedir(), ".attest", "bin");
174
+ const binPath = path.join(dir, binaryFilename());
175
+ const versionFile = path.join(dir, ".engine-version");
176
+ if (!fs.existsSync(binPath)) {
177
+ return null;
178
+ }
179
+ if (!fs.existsSync(versionFile)) {
180
+ return null;
181
+ }
182
+ const cachedVersion = fs.readFileSync(versionFile, "utf-8").trim();
183
+ if (cachedVersion !== ENGINE_VERSION) {
184
+ return null;
185
+ }
186
+ return binPath;
187
+ }
188
+ function parseChecksums(text) {
189
+ const checksums = /* @__PURE__ */ new Map();
190
+ for (const line of text.split("\n")) {
191
+ const trimmed = line.trim();
192
+ if (!trimmed) continue;
193
+ const match = trimmed.match(/^([a-f0-9]{64})\s+(.+)$/);
194
+ if (match) {
195
+ checksums.set(match[2], match[1]);
196
+ }
197
+ }
198
+ return checksums;
199
+ }
200
+ function httpsGet(url, redirectCount = 0) {
201
+ if (redirectCount > MAX_REDIRECTS) {
202
+ return Promise.reject(
203
+ new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
204
+ );
205
+ }
206
+ return new Promise((resolve2, reject) => {
207
+ const client = url.startsWith("https:") ? https : http;
208
+ client.get(url, (res) => {
209
+ const status = res.statusCode ?? 0;
210
+ if (status === 301 || status === 302) {
211
+ const location = res.headers.location;
212
+ if (!location) {
213
+ reject(new Error(`Redirect ${status} without Location header from ${url}`));
214
+ return;
215
+ }
216
+ res.resume();
217
+ resolve2(httpsGet(location, redirectCount + 1));
218
+ return;
219
+ }
220
+ if (status < 200 || status >= 300) {
221
+ res.resume();
222
+ reject(new Error(`HTTP ${status} fetching ${url}`));
223
+ return;
224
+ }
225
+ const chunks = [];
226
+ res.on("data", (chunk) => chunks.push(chunk));
227
+ res.on("end", () => resolve2(Buffer.concat(chunks)));
228
+ res.on("error", reject);
229
+ }).on("error", reject);
230
+ });
231
+ }
232
+ async function downloadEngine() {
233
+ const key = platformKey();
234
+ const filename = `attest-engine-${key}`;
235
+ const binaryUrl = `${RELEASE_BASE}/${filename}`;
236
+ process.stderr.write(`[attest] Downloading engine v${ENGINE_VERSION} for ${key}...
237
+ `);
238
+ process.stderr.write(`[attest] Fetching checksums...
239
+ `);
240
+ const checksumsText = (await httpsGet(CHECKSUMS_URL)).toString("utf-8");
241
+ const checksums = parseChecksums(checksumsText);
242
+ const expectedHash = checksums.get(filename);
243
+ if (!expectedHash) {
244
+ throw new Error(
245
+ `No checksum found for '${filename}' in checksums-sha256.txt. Available files: ${[...checksums.keys()].join(", ")}`
246
+ );
247
+ }
248
+ process.stderr.write(`[attest] Downloading ${binaryUrl}...
249
+ `);
250
+ const data = await httpsGet(binaryUrl);
251
+ const actualHash = crypto.createHash("sha256").update(data).digest("hex");
252
+ if (actualHash !== expectedHash) {
253
+ throw new Error(
254
+ `SHA256 mismatch for ${filename}. Expected: ${expectedHash}, got: ${actualHash}. The download may be corrupted. Delete ~/.attest/bin/ and retry.`
255
+ );
256
+ }
257
+ process.stderr.write(`[attest] Checksum verified.
258
+ `);
259
+ const dir = attestBinDir();
260
+ const destFilename = binaryFilename();
261
+ const destPath = path.join(dir, destFilename);
262
+ const tmpPath = path.join(dir, `.${destFilename}.tmp.${process.pid}`);
263
+ fs.writeFileSync(tmpPath, data);
264
+ fs.chmodSync(tmpPath, 493);
265
+ fs.renameSync(tmpPath, destPath);
266
+ fs.writeFileSync(path.join(dir, ".engine-version"), ENGINE_VERSION);
267
+ process.stderr.write(`[attest] Engine installed at ${destPath}
268
+ `);
269
+ return destPath;
270
+ }
271
+
272
+ // src/engine-manager.ts
273
+ var ENGINE_BINARY_NAME = "attest-engine";
274
+ async function findEngineBinary() {
275
+ const envPath = process.env["ATTEST_ENGINE_PATH"];
276
+ if (envPath) {
277
+ if (!existsSync2(envPath)) {
278
+ throw new Error(
279
+ `ATTEST_ENGINE_PATH is set to '${envPath}' but the file does not exist.`
280
+ );
281
+ }
282
+ return envPath;
283
+ }
284
+ try {
285
+ const cmd = process.platform === "win32" ? "where" : "which";
286
+ const found = execSync(`${cmd} ${ENGINE_BINARY_NAME}`, { encoding: "utf-8" }).trim();
287
+ if (found) return found;
288
+ } catch {
289
+ }
290
+ const cached = cachedEnginePath();
291
+ if (cached) return cached;
292
+ const candidates = [
293
+ join2(dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "..", "..", "bin", ENGINE_BINARY_NAME),
294
+ join2(process.cwd(), "bin", ENGINE_BINARY_NAME)
295
+ ];
296
+ for (const candidate of candidates) {
297
+ const resolved = resolve(candidate);
298
+ try {
299
+ accessSync(resolved, fsConstants.X_OK);
300
+ if (existsSync2(resolved)) return resolved;
301
+ } catch {
302
+ }
303
+ }
304
+ const noDownload = process.env["ATTEST_ENGINE_NO_DOWNLOAD"];
305
+ if (noDownload === "1" || noDownload === "true" || noDownload === "yes") {
306
+ throw new Error(
307
+ `Cannot find '${ENGINE_BINARY_NAME}' binary and auto-download is disabled (ATTEST_ENGINE_NO_DOWNLOAD is set). Install the engine manually:
308
+ 1. Download from https://github.com/attest-framework/attest/releases
309
+ 2. Place the binary on your PATH or set ATTEST_ENGINE_PATH`
310
+ );
311
+ }
312
+ return downloadEngine();
313
+ }
314
+ var EngineManager = class {
315
+ enginePath;
316
+ logLevel;
317
+ process = null;
318
+ reader = null;
319
+ initialized = false;
320
+ requestId = 0;
321
+ initResult = null;
322
+ constructor(options) {
323
+ this.enginePath = options?.enginePath ?? "";
324
+ this.logLevel = options?.logLevel ?? "warn";
325
+ }
326
+ async start() {
327
+ if (!this.enginePath) {
328
+ this.enginePath = await findEngineBinary();
329
+ }
330
+ this.process = spawn(this.enginePath, [`--log-level=${this.logLevel}`], {
331
+ stdio: ["pipe", "pipe", "pipe"]
332
+ });
333
+ this.reader = createInterface({ input: this.process.stdout });
334
+ const result = await this.sendRequestInternal("initialize", {
335
+ sdk_name: "attest-typescript",
336
+ sdk_version: VERSION,
337
+ protocol_version: 1,
338
+ required_capabilities: ["layers_1_4"],
339
+ preferred_encoding: "json"
340
+ });
341
+ this.initResult = result;
342
+ if (!this.initResult.compatible) {
343
+ throw new Error(
344
+ `Engine incompatible. Missing capabilities: ${JSON.stringify(this.initResult.missing)}`
345
+ );
346
+ }
347
+ this.initialized = true;
348
+ return this.initResult;
349
+ }
350
+ async stop() {
351
+ if (this.process === null) return;
352
+ if (this.initialized) {
353
+ try {
354
+ await this.sendRequestInternal("shutdown", {});
355
+ } catch {
356
+ }
357
+ }
358
+ if (this.reader) {
359
+ this.reader.close();
360
+ this.reader = null;
361
+ }
362
+ if (this.process.exitCode === null) {
363
+ this.process.kill("SIGTERM");
364
+ await new Promise((res) => {
365
+ const timer = setTimeout(() => {
366
+ this.process?.kill("SIGKILL");
367
+ }, 5e3);
368
+ this.process.on("exit", () => {
369
+ clearTimeout(timer);
370
+ res();
371
+ });
372
+ });
373
+ }
374
+ this.initialized = false;
375
+ this.process = null;
376
+ }
377
+ async sendRequest(method, params) {
378
+ if (!this.initialized && method !== "initialize") {
379
+ throw new Error("Engine not initialized. Call start() first.");
380
+ }
381
+ return this.sendRequestInternal(method, params);
382
+ }
383
+ async sendRequestInternal(method, params) {
384
+ if (this.process === null || this.process.stdin === null || this.reader === null) {
385
+ throw new Error("Engine process not started.");
386
+ }
387
+ this.requestId += 1;
388
+ const requestStr = encodeRequest(this.requestId, method, params);
389
+ return new Promise((resolve2, reject) => {
390
+ const onLine = (line) => {
391
+ try {
392
+ const response = decodeResponse(line);
393
+ resolve2(extractResult(response));
394
+ } catch (err) {
395
+ reject(err);
396
+ }
397
+ };
398
+ this.reader.once("line", onLine);
399
+ const ok = this.process.stdin.write(requestStr, (err) => {
400
+ if (err) {
401
+ this.reader.removeListener("line", onLine);
402
+ reject(err);
403
+ }
404
+ });
405
+ if (!ok) {
406
+ this.process.stdin.once("drain", () => {
407
+ });
408
+ }
409
+ });
410
+ }
411
+ get isRunning() {
412
+ return this.process !== null && this.process.exitCode === null;
413
+ }
414
+ /** Expose process for AttestClient reader loop access */
415
+ get childProcess() {
416
+ return this.process;
417
+ }
418
+ /** Expose readline interface for AttestClient reader loop */
419
+ get readlineInterface() {
420
+ return this.reader;
421
+ }
422
+ };
423
+
424
+ // src/config.ts
425
+ var _simulationMode = false;
426
+ function config(options) {
427
+ if (options.simulation !== void 0) {
428
+ _simulationMode = options.simulation;
429
+ }
430
+ }
431
+ function isSimulationMode() {
432
+ return _simulationMode || process.env["ATTEST_SIMULATION"] === "1" || process.env["ATTEST_SIMULATION"] === "true" || process.env["ATTEST_SIMULATION"] === "yes";
433
+ }
434
+ function resetConfig() {
435
+ _simulationMode = false;
436
+ }
437
+
438
+ // src/simulation.ts
439
+ function simulationEvaluateBatch(assertions) {
440
+ const results = assertions.map((assertion) => ({
441
+ assertion_id: assertion.assertion_id,
442
+ status: "pass",
443
+ score: 1,
444
+ explanation: `[simulation] assertion '${assertion.assertion_id}' passed (type: ${assertion.type})`,
445
+ cost: 0,
446
+ duration_ms: 0
447
+ }));
448
+ return {
449
+ results,
450
+ total_cost: 0,
451
+ total_duration_ms: 0
452
+ };
453
+ }
454
+
455
+ // src/client.ts
456
+ var DEFAULT_TIMEOUT_MS = 3e4;
457
+ function resolveTimeoutMs(override) {
458
+ if (override !== void 0) return override;
459
+ const env = process.env["ATTEST_ENGINE_TIMEOUT"];
460
+ if (env !== void 0 && env !== "") {
461
+ const parsed = Number(env);
462
+ if (!Number.isFinite(parsed) || parsed <= 0) {
463
+ throw new Error(
464
+ `ATTEST_ENGINE_TIMEOUT must be a positive number of milliseconds, got: '${env}'`
465
+ );
466
+ }
467
+ return parsed;
468
+ }
469
+ return DEFAULT_TIMEOUT_MS;
470
+ }
471
+ function makeTimeoutPromise(method, timeoutMs) {
472
+ return new Promise((_, reject) => {
473
+ setTimeout(() => {
474
+ reject(new EngineTimeoutError(method, timeoutMs));
475
+ }, timeoutMs);
476
+ });
477
+ }
478
+ var AttestClient = class {
479
+ engine;
480
+ requestId = 0;
481
+ pending = /* @__PURE__ */ new Map();
482
+ writeChain = Promise.resolve();
483
+ readerActive = false;
484
+ lineHandler = null;
485
+ constructor(engine) {
486
+ this.engine = engine;
487
+ }
488
+ startReader() {
489
+ if (this.readerActive) return;
490
+ const rl = this.engine.readlineInterface;
491
+ if (rl === null) {
492
+ throw new Error("Engine readline interface not available. Call engine.start() first.");
493
+ }
494
+ this.readerActive = true;
495
+ this.lineHandler = (line) => this.handleLine(line);
496
+ rl.on("line", this.lineHandler);
497
+ rl.on("close", () => {
498
+ this.failAll(new Error("Engine closed stdout."));
499
+ this.readerActive = false;
500
+ });
501
+ }
502
+ stopReader() {
503
+ if (!this.readerActive) return;
504
+ const rl = this.engine.readlineInterface;
505
+ if (rl !== null && this.lineHandler !== null) {
506
+ rl.removeListener("line", this.lineHandler);
507
+ }
508
+ this.lineHandler = null;
509
+ this.readerActive = false;
510
+ }
511
+ handleLine(line) {
512
+ let reqId;
513
+ try {
514
+ const response = decodeResponse(line);
515
+ reqId = extractId(response);
516
+ const pending = this.pending.get(reqId);
517
+ if (pending === void 0) return;
518
+ this.pending.delete(reqId);
519
+ try {
520
+ const result = extractResult(response);
521
+ pending.resolve(result);
522
+ } catch (err) {
523
+ pending.reject(err);
524
+ }
525
+ } catch (err) {
526
+ if (err instanceof ProtocolError) {
527
+ try {
528
+ const raw = JSON.parse(line.trim());
529
+ reqId = Number(raw.id ?? -1);
530
+ } catch {
531
+ reqId = -1;
532
+ }
533
+ const pending = this.pending.get(reqId);
534
+ if (pending !== void 0) {
535
+ this.pending.delete(reqId);
536
+ pending.reject(err);
537
+ }
538
+ }
539
+ }
540
+ }
541
+ failAll(err) {
542
+ for (const pending of this.pending.values()) {
543
+ pending.reject(err);
544
+ }
545
+ this.pending.clear();
546
+ }
547
+ async sendRequest(method, params, timeoutMs) {
548
+ const resolvedTimeout = resolveTimeoutMs(timeoutMs);
549
+ if (!this.readerActive) {
550
+ return Promise.race([
551
+ this.engine.sendRequest(method, params),
552
+ makeTimeoutPromise(method, resolvedTimeout)
553
+ ]);
554
+ }
555
+ const cp = this.engine.childProcess;
556
+ if (cp === null || cp.stdin === null) {
557
+ throw new Error("Engine process not started.");
558
+ }
559
+ let capturedReqId;
560
+ const requestPromise = new Promise((resolve2, reject) => {
561
+ this.writeChain = this.writeChain.then(() => {
562
+ this.requestId += 1;
563
+ const reqId = this.requestId;
564
+ capturedReqId = reqId;
565
+ this.pending.set(reqId, { resolve: resolve2, reject });
566
+ const requestStr = encodeRequest(reqId, method, params);
567
+ cp.stdin.write(requestStr, (err) => {
568
+ if (err) {
569
+ this.pending.delete(reqId);
570
+ reject(err);
571
+ }
572
+ });
573
+ });
574
+ });
575
+ const timeoutPromise = makeTimeoutPromise(method, resolvedTimeout);
576
+ return Promise.race([requestPromise, timeoutPromise]).catch((err) => {
577
+ if (err instanceof EngineTimeoutError && capturedReqId !== void 0) {
578
+ this.pending.delete(capturedReqId);
579
+ }
580
+ throw err;
581
+ });
582
+ }
583
+ async evaluateBatch(trace, assertions, options) {
584
+ if (isSimulationMode()) {
585
+ return simulationEvaluateBatch(assertions);
586
+ }
587
+ const params = {
588
+ trace,
589
+ assertions: [...assertions]
590
+ };
591
+ const raw = await this.sendRequest(
592
+ "evaluate_batch",
593
+ params,
594
+ options?.timeout
595
+ );
596
+ return raw;
597
+ }
598
+ async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options) {
599
+ const result = {
600
+ assertion_id: assertionId2,
601
+ status,
602
+ score,
603
+ explanation,
604
+ cost: 0,
605
+ duration_ms: 0
606
+ };
607
+ const params = {
608
+ trace_id: traceId2,
609
+ plugin_name: pluginName,
610
+ assertion_id: assertionId2,
611
+ result
612
+ };
613
+ const raw = await this.sendRequest(
614
+ "submit_plugin_result",
615
+ params,
616
+ options?.timeout
617
+ );
618
+ return Boolean(raw?.accepted ?? false);
619
+ }
620
+ };
621
+
622
+ // src/trace.ts
623
+ import { randomUUID } from "crypto";
624
+ var TraceBuilder = class {
625
+ traceId;
626
+ agentId;
627
+ input;
628
+ steps = [];
629
+ output;
630
+ metadata;
631
+ parentTraceId;
632
+ constructor(agentId2) {
633
+ this.traceId = `trc_${randomUUID().replace(/-/g, "").slice(0, 12)}`;
634
+ this.agentId = agentId2;
635
+ }
636
+ setTraceId(traceId2) {
637
+ this.traceId = traceId2;
638
+ return this;
639
+ }
640
+ setInput(input) {
641
+ this.input = input;
642
+ return this;
643
+ }
644
+ addLlmCall(name, options) {
645
+ this.steps.push({
646
+ type: STEP_LLM_CALL,
647
+ name,
648
+ args: options?.args,
649
+ result: options?.result,
650
+ metadata: options?.metadata
651
+ });
652
+ return this;
653
+ }
654
+ addToolCall(name, options) {
655
+ this.steps.push({
656
+ type: STEP_TOOL_CALL,
657
+ name,
658
+ args: options?.args,
659
+ result: options?.result,
660
+ metadata: options?.metadata
661
+ });
662
+ return this;
663
+ }
664
+ addRetrieval(name, options) {
665
+ this.steps.push({
666
+ type: STEP_RETRIEVAL,
667
+ name,
668
+ args: options?.args,
669
+ result: options?.result,
670
+ metadata: options?.metadata
671
+ });
672
+ return this;
673
+ }
674
+ addStep(step) {
675
+ this.steps.push(step);
676
+ return this;
677
+ }
678
+ setOutput(output) {
679
+ this.output = output;
680
+ return this;
681
+ }
682
+ setMetadata(metadata) {
683
+ this.metadata = metadata;
684
+ return this;
685
+ }
686
+ getTraceId() {
687
+ return this.traceId;
688
+ }
689
+ setParentTraceId(parentId) {
690
+ this.parentTraceId = parentId;
691
+ return this;
692
+ }
693
+ build() {
694
+ if (this.output === void 0) {
695
+ throw new Error("Trace output is required. Call setOutput() before build().");
696
+ }
697
+ return {
698
+ trace_id: this.traceId,
699
+ output: this.output,
700
+ schema_version: 1,
701
+ agent_id: this.agentId,
702
+ input: this.input,
703
+ steps: [...this.steps],
704
+ metadata: this.metadata,
705
+ parent_trace_id: this.parentTraceId
706
+ };
707
+ }
708
+ };
709
+
710
+ // src/result.ts
711
+ var AgentResult = class {
712
+ trace;
713
+ assertionResults;
714
+ totalCost;
715
+ totalDurationMs;
716
+ constructor(trace, assertionResults = [], totalCost = 0, totalDurationMs = 0) {
717
+ this.trace = trace;
718
+ this.assertionResults = assertionResults;
719
+ this.totalCost = totalCost;
720
+ this.totalDurationMs = totalDurationMs;
721
+ }
722
+ get passed() {
723
+ return this.assertionResults.every((r) => r.status === STATUS_PASS);
724
+ }
725
+ get failedAssertions() {
726
+ return this.assertionResults.filter((r) => r.status !== STATUS_PASS);
727
+ }
728
+ get hardFailures() {
729
+ return this.assertionResults.filter((r) => r.status === STATUS_HARD_FAIL);
730
+ }
731
+ get softFailures() {
732
+ return this.assertionResults.filter((r) => r.status === STATUS_SOFT_FAIL);
733
+ }
734
+ get passCount() {
735
+ return this.assertionResults.filter((r) => r.status === STATUS_PASS).length;
736
+ }
737
+ get failCount() {
738
+ return this.assertionResults.length - this.passCount;
739
+ }
740
+ };
741
+
742
+ // src/trace-tree.ts
743
+ var TraceTree = class {
744
+ root;
745
+ constructor(root) {
746
+ this.root = root;
747
+ }
748
+ get agents() {
749
+ const result = [];
750
+ this.collectAgents(this.root, result);
751
+ return result;
752
+ }
753
+ collectAgents(trace, acc) {
754
+ if (trace.agent_id != null) {
755
+ acc.push(trace.agent_id);
756
+ }
757
+ for (const step of trace.steps) {
758
+ if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
759
+ this.collectAgents(step.sub_trace, acc);
760
+ }
761
+ }
762
+ }
763
+ findAgent(agentId2) {
764
+ return this.findAgentIn(this.root, agentId2);
765
+ }
766
+ findAgentIn(trace, agentId2) {
767
+ if (trace.agent_id === agentId2) return trace;
768
+ for (const step of trace.steps) {
769
+ if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
770
+ const found = this.findAgentIn(step.sub_trace, agentId2);
771
+ if (found !== void 0) return found;
772
+ }
773
+ }
774
+ return void 0;
775
+ }
776
+ get depth() {
777
+ return this.computeDepth(this.root);
778
+ }
779
+ computeDepth(trace) {
780
+ let maxChild = -1;
781
+ for (const step of trace.steps) {
782
+ if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
783
+ const childDepth = this.computeDepth(step.sub_trace);
784
+ if (childDepth > maxChild) maxChild = childDepth;
785
+ }
786
+ }
787
+ return maxChild >= 0 ? maxChild + 1 : 0;
788
+ }
789
+ flatten() {
790
+ const result = [];
791
+ this.flattenInto(this.root, result);
792
+ return result;
793
+ }
794
+ flattenInto(trace, acc) {
795
+ acc.push(trace);
796
+ for (const step of trace.steps) {
797
+ if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
798
+ this.flattenInto(step.sub_trace, acc);
799
+ }
800
+ }
801
+ }
802
+ get delegations() {
803
+ const result = [];
804
+ this.collectDelegations(this.root, result);
805
+ return result;
806
+ }
807
+ collectDelegations(trace, acc) {
808
+ const parentId = trace.agent_id ?? "";
809
+ for (const step of trace.steps) {
810
+ if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
811
+ const childId = step.sub_trace.agent_id ?? "";
812
+ acc.push([parentId, childId]);
813
+ this.collectDelegations(step.sub_trace, acc);
814
+ }
815
+ }
816
+ }
817
+ allToolCalls() {
818
+ return this.flatten().flatMap(
819
+ (t) => t.steps.filter((step) => step.type === STEP_TOOL_CALL)
820
+ );
821
+ }
822
+ get aggregateTokens() {
823
+ return this.flatten().reduce(
824
+ (sum, t) => sum + (t.metadata?.total_tokens ?? 0),
825
+ 0
826
+ );
827
+ }
828
+ get aggregateCost() {
829
+ return this.flatten().reduce(
830
+ (sum, t) => sum + (t.metadata?.cost_usd ?? 0),
831
+ 0
832
+ );
833
+ }
834
+ get aggregateLatency() {
835
+ return this.flatten().reduce(
836
+ (sum, t) => sum + (t.metadata?.latency_ms ?? 0),
837
+ 0
838
+ );
839
+ }
840
+ };
841
+
842
+ // src/expect.ts
843
+ import { randomUUID as randomUUID2 } from "crypto";
844
+ var ExpectChain = class {
845
+ _result;
846
+ _assertions = [];
847
+ constructor(result) {
848
+ this._result = result;
849
+ }
850
+ get assertions() {
851
+ return [...this._assertions];
852
+ }
853
+ get trace() {
854
+ return this._result.trace;
855
+ }
856
+ add(assertionType, spec) {
857
+ this._assertions.push({
858
+ assertion_id: `assert_${randomUUID2().replace(/-/g, "").slice(0, 8)}`,
859
+ type: assertionType,
860
+ spec
861
+ });
862
+ return this;
863
+ }
864
+ // -- Layer 1: Schema --
865
+ /**
866
+ * Assert that `output.structured` conforms to the given JSON Schema.
867
+ *
868
+ * @param schema - A JSON Schema object to validate against.
869
+ * @returns The chain for fluent composition.
870
+ *
871
+ * @example
872
+ * ```ts
873
+ * expect(result).outputMatchesSchema({ type: "object", required: ["answer"] });
874
+ * ```
875
+ */
876
+ outputMatchesSchema(schema) {
877
+ return this.add(TYPE_SCHEMA, { target: "output.structured", schema });
878
+ }
879
+ /**
880
+ * Assert that a specific output field conforms to the given JSON Schema.
881
+ *
882
+ * @param field - Dot-delimited path under `output` (e.g. `"structured.items"`).
883
+ * @param schema - A JSON Schema object to validate against.
884
+ * @returns The chain for fluent composition.
885
+ *
886
+ * @example
887
+ * ```ts
888
+ * expect(result).outputFieldMatchesSchema("structured.items", { type: "array" });
889
+ * ```
890
+ */
891
+ outputFieldMatchesSchema(field, schema) {
892
+ return this.add(TYPE_SCHEMA, { target: `output.${field}`, schema });
893
+ }
894
+ /**
895
+ * Assert that a named tool's arguments conform to the given JSON Schema.
896
+ *
897
+ * @param toolName - The tool name to match in `steps`.
898
+ * @param schema - A JSON Schema object to validate against.
899
+ * @returns The chain for fluent composition.
900
+ *
901
+ * @example
902
+ * ```ts
903
+ * expect(result).toolArgsMatchSchema("search", { type: "object", required: ["query"] });
904
+ * ```
905
+ */
906
+ toolArgsMatchSchema(toolName, schema) {
907
+ return this.add(TYPE_SCHEMA, {
908
+ target: `steps[?name=='${toolName}'].args`,
909
+ schema
910
+ });
911
+ }
912
+ /**
913
+ * Assert that a named tool's result conforms to the given JSON Schema.
914
+ *
915
+ * @param toolName - The tool name to match in `steps`.
916
+ * @param schema - A JSON Schema object to validate against.
917
+ * @returns The chain for fluent composition.
918
+ *
919
+ * @example
920
+ * ```ts
921
+ * expect(result).toolResultMatchesSchema("search", { type: "object" });
922
+ * ```
923
+ */
924
+ toolResultMatchesSchema(toolName, schema) {
925
+ return this.add(TYPE_SCHEMA, {
926
+ target: `steps[?name=='${toolName}'].result`,
927
+ schema
928
+ });
929
+ }
930
+ // -- Layer 2: Constraints --
931
+ /**
932
+ * Assert total cost (in USD) is at or below `maxCost`.
933
+ *
934
+ * @param maxCost - Maximum allowed cost in USD.
935
+ * @param opts.soft - When `true` the failure is non-blocking.
936
+ * @returns The chain for fluent composition.
937
+ *
938
+ * @example
939
+ * ```ts
940
+ * expect(result).costUnder(0.05);
941
+ * expect(result).costUnder(0.05, { soft: true });
942
+ * ```
943
+ */
944
+ costUnder(maxCost, opts) {
945
+ return this.add(TYPE_CONSTRAINT, {
946
+ field: "metadata.cost_usd",
947
+ operator: "lte",
948
+ value: maxCost,
949
+ soft: opts?.soft ?? false
950
+ });
951
+ }
952
+ /**
953
+ * Assert total latency is at or below `maxMs` milliseconds.
954
+ *
955
+ * @param maxMs - Maximum allowed latency in milliseconds.
956
+ * @param opts.soft - When `true` the failure is non-blocking.
957
+ * @returns The chain for fluent composition.
958
+ *
959
+ * @example
960
+ * ```ts
961
+ * expect(result).latencyUnder(2000);
962
+ * ```
963
+ */
964
+ latencyUnder(maxMs, opts) {
965
+ return this.add(TYPE_CONSTRAINT, {
966
+ field: "metadata.latency_ms",
967
+ operator: "lte",
968
+ value: maxMs,
969
+ soft: opts?.soft ?? false
970
+ });
971
+ }
972
+ /**
973
+ * Assert total token usage is at or below `maxTokens`.
974
+ *
975
+ * @param maxTokens - Maximum allowed token count.
976
+ * @param opts.soft - When `true` the failure is non-blocking.
977
+ * @returns The chain for fluent composition.
978
+ *
979
+ * @example
980
+ * ```ts
981
+ * expect(result).tokensUnder(500);
982
+ * ```
983
+ */
984
+ tokensUnder(maxTokens, opts) {
985
+ return this.add(TYPE_CONSTRAINT, {
986
+ field: "metadata.total_tokens",
987
+ operator: "lte",
988
+ value: maxTokens,
989
+ soft: opts?.soft ?? false
990
+ });
991
+ }
992
+ /**
993
+ * Assert total token usage is between `minTokens` and `maxTokens` (inclusive).
994
+ *
995
+ * @param minTokens - Minimum allowed token count.
996
+ * @param maxTokens - Maximum allowed token count.
997
+ * @param opts.soft - When `true` the failure is non-blocking.
998
+ * @returns The chain for fluent composition.
999
+ *
1000
+ * @example
1001
+ * ```ts
1002
+ * expect(result).tokensBetween(100, 500);
1003
+ * ```
1004
+ */
1005
+ tokensBetween(minTokens, maxTokens, opts) {
1006
+ return this.add(TYPE_CONSTRAINT, {
1007
+ field: "metadata.total_tokens",
1008
+ operator: "between",
1009
+ min: minTokens,
1010
+ max: maxTokens,
1011
+ soft: opts?.soft ?? false
1012
+ });
1013
+ }
1014
+ /**
1015
+ * Assert the number of steps satisfies the given comparison.
1016
+ *
1017
+ * @param operator - Comparison operator (e.g. `"eq"`, `"lte"`, `"gte"`).
1018
+ * @param value - The reference count to compare against.
1019
+ * @param opts.soft - When `true` the failure is non-blocking.
1020
+ * @returns The chain for fluent composition.
1021
+ *
1022
+ * @example
1023
+ * ```ts
1024
+ * expect(result).stepCount("lte", 5);
1025
+ * ```
1026
+ */
1027
+ stepCount(operator, value, opts) {
1028
+ return this.add(TYPE_CONSTRAINT, {
1029
+ field: "steps.length",
1030
+ operator,
1031
+ value,
1032
+ soft: opts?.soft ?? false
1033
+ });
1034
+ }
1035
+ /**
1036
+ * Assert the number of tool-call steps satisfies the given comparison.
1037
+ *
1038
+ * @param operator - Comparison operator (e.g. `"eq"`, `"lte"`, `"gte"`).
1039
+ * @param value - The reference count to compare against.
1040
+ * @param opts.soft - When `true` the failure is non-blocking.
1041
+ * @returns The chain for fluent composition.
1042
+ *
1043
+ * @example
1044
+ * ```ts
1045
+ * expect(result).toolCallCount("gte", 1);
1046
+ * ```
1047
+ */
1048
+ toolCallCount(operator, value, opts) {
1049
+ return this.add(TYPE_CONSTRAINT, {
1050
+ field: "steps[?type=='tool_call'].length",
1051
+ operator,
1052
+ value,
1053
+ soft: opts?.soft ?? false
1054
+ });
1055
+ }
1056
+ /**
1057
+ * Generic constraint assertion on any field.
1058
+ *
1059
+ * @param field - Dot-delimited path into the trace (e.g. `"metadata.cost_usd"`).
1060
+ * @param operator - Comparison operator.
1061
+ * @param opts.value - Reference value for single-operand operators.
1062
+ * @param opts.min - Minimum for `"between"` operator.
1063
+ * @param opts.max - Maximum for `"between"` operator.
1064
+ * @param opts.soft - When `true` the failure is non-blocking.
1065
+ * @returns The chain for fluent composition.
1066
+ *
1067
+ * @example
1068
+ * ```ts
1069
+ * expect(result).constraint("metadata.cost_usd", "lte", { value: 0.1 });
1070
+ * ```
1071
+ */
1072
+ constraint(field, operator, opts) {
1073
+ const spec = {
1074
+ field,
1075
+ operator,
1076
+ soft: opts?.soft ?? false
1077
+ };
1078
+ if (opts?.value !== void 0) spec.value = opts.value;
1079
+ if (opts?.min !== void 0) spec.min = opts.min;
1080
+ if (opts?.max !== void 0) spec.max = opts.max;
1081
+ return this.add(TYPE_CONSTRAINT, spec);
1082
+ }
1083
+ // -- Layer 3: Trace --
1084
+ /**
1085
+ * Assert that tools were called in the given order (non-contiguous allowed).
1086
+ *
1087
+ * @param tools - Ordered list of tool names expected.
1088
+ * @param opts.soft - When `true` the failure is non-blocking.
1089
+ * @returns The chain for fluent composition.
1090
+ *
1091
+ * @example
1092
+ * ```ts
1093
+ * expect(result).toolsCalledInOrder(["search", "write"]);
1094
+ * ```
1095
+ */
1096
+ toolsCalledInOrder(tools, opts) {
1097
+ return this.add(TYPE_TRACE, {
1098
+ check: "contains_in_order",
1099
+ tools,
1100
+ soft: opts?.soft ?? false
1101
+ });
1102
+ }
1103
+ /**
1104
+ * Assert that tools were called in exact contiguous order.
1105
+ *
1106
+ * @param tools - Exact ordered list of tool names.
1107
+ * @param opts.soft - When `true` the failure is non-blocking.
1108
+ * @returns The chain for fluent composition.
1109
+ *
1110
+ * @example
1111
+ * ```ts
1112
+ * expect(result).toolsCalledExactly(["search", "summarize", "respond"]);
1113
+ * ```
1114
+ */
1115
+ toolsCalledExactly(tools, opts) {
1116
+ return this.add(TYPE_TRACE, {
1117
+ check: "exact_order",
1118
+ tools,
1119
+ soft: opts?.soft ?? false
1120
+ });
1121
+ }
1122
+ /**
1123
+ * Assert a tool is not called more than `maxRepetitions` times consecutively.
1124
+ *
1125
+ * @param tool - The tool name to check.
1126
+ * @param maxRepetitions - Maximum allowed consecutive calls (default `1`).
1127
+ * @param opts.soft - When `true` the failure is non-blocking.
1128
+ * @returns The chain for fluent composition.
1129
+ *
1130
+ * @example
1131
+ * ```ts
1132
+ * expect(result).noToolLoops("fetch", 2);
1133
+ * ```
1134
+ */
1135
+ noToolLoops(tool, maxRepetitions = 1, opts) {
1136
+ return this.add(TYPE_TRACE, {
1137
+ check: "loop_detection",
1138
+ tool,
1139
+ max_repetitions: maxRepetitions,
1140
+ soft: opts?.soft ?? false
1141
+ });
1142
+ }
1143
+ /**
1144
+ * Assert no tool is called more than once.
1145
+ *
1146
+ * @param opts.soft - When `true` the failure is non-blocking.
1147
+ * @returns The chain for fluent composition.
1148
+ *
1149
+ * @example
1150
+ * ```ts
1151
+ * expect(result).noDuplicateTools();
1152
+ * ```
1153
+ */
1154
+ noDuplicateTools(opts) {
1155
+ return this.add(TYPE_TRACE, { check: "no_duplicates", soft: opts?.soft ?? false });
1156
+ }
1157
+ /**
1158
+ * Assert that all listed tools were called at least once.
1159
+ *
1160
+ * @param tools - Tool names that must appear in the trace.
1161
+ * @param opts.soft - When `true` the failure is non-blocking.
1162
+ * @returns The chain for fluent composition.
1163
+ *
1164
+ * @example
1165
+ * ```ts
1166
+ * expect(result).requiredTools(["search", "respond"]);
1167
+ * ```
1168
+ */
1169
+ requiredTools(tools, opts) {
1170
+ return this.add(TYPE_TRACE, {
1171
+ check: "required_tools",
1172
+ tools,
1173
+ soft: opts?.soft ?? false
1174
+ });
1175
+ }
1176
+ /**
1177
+ * Assert that none of the listed tools were called.
1178
+ *
1179
+ * @param tools - Tool names that must not appear in the trace.
1180
+ * @param opts.soft - When `true` the failure is non-blocking.
1181
+ * @returns The chain for fluent composition.
1182
+ *
1183
+ * @example
1184
+ * ```ts
1185
+ * expect(result).forbiddenTools(["delete_user", "drop_table"]);
1186
+ * ```
1187
+ */
1188
+ forbiddenTools(tools, opts) {
1189
+ return this.add(TYPE_TRACE, {
1190
+ check: "forbidden_tools",
1191
+ tools,
1192
+ soft: opts?.soft ?? false
1193
+ });
1194
+ }
1195
+ // -- Layer 4: Content --
1196
+ /**
1197
+ * Assert the output message contains the given string.
1198
+ *
1199
+ * @param value - Substring to search for.
1200
+ * @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
1201
+ * @param opts.soft - When `true` the failure is non-blocking.
1202
+ * @returns The chain for fluent composition.
1203
+ *
1204
+ * @example
1205
+ * ```ts
1206
+ * expect(result).outputContains("refund approved");
1207
+ * ```
1208
+ */
1209
+ outputContains(value, opts) {
1210
+ return this.add(TYPE_CONTENT, {
1211
+ target: "output.message",
1212
+ check: "contains",
1213
+ value,
1214
+ case_sensitive: opts?.caseSensitive ?? false,
1215
+ soft: opts?.soft ?? false
1216
+ });
1217
+ }
1218
+ /**
1219
+ * Assert the output message does not contain the given string.
1220
+ *
1221
+ * @param value - Substring that must be absent.
1222
+ * @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
1223
+ * @param opts.soft - When `true` the failure is non-blocking.
1224
+ * @returns The chain for fluent composition.
1225
+ *
1226
+ * @example
1227
+ * ```ts
1228
+ * expect(result).outputNotContains("internal error");
1229
+ * ```
1230
+ */
1231
+ outputNotContains(value, opts) {
1232
+ return this.add(TYPE_CONTENT, {
1233
+ target: "output.message",
1234
+ check: "not_contains",
1235
+ value,
1236
+ case_sensitive: opts?.caseSensitive ?? false,
1237
+ soft: opts?.soft ?? false
1238
+ });
1239
+ }
1240
+ /**
1241
+ * Assert the output message matches the given regular expression.
1242
+ *
1243
+ * @param pattern - Regex pattern string.
1244
+ * @param opts.soft - When `true` the failure is non-blocking.
1245
+ * @returns The chain for fluent composition.
1246
+ *
1247
+ * @example
1248
+ * ```ts
1249
+ * expect(result).outputMatchesRegex("order #\\d{5}");
1250
+ * ```
1251
+ */
1252
+ outputMatchesRegex(pattern, opts) {
1253
+ return this.add(TYPE_CONTENT, {
1254
+ target: "output.message",
1255
+ check: "regex_match",
1256
+ value: pattern,
1257
+ soft: opts?.soft ?? false
1258
+ });
1259
+ }
1260
+ /**
1261
+ * Assert the output message contains all of the given keywords.
1262
+ *
1263
+ * @param keywords - Keywords that must all be present.
1264
+ * @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
1265
+ * @param opts.soft - When `true` the failure is non-blocking.
1266
+ * @returns The chain for fluent composition.
1267
+ *
1268
+ * @example
1269
+ * ```ts
1270
+ * expect(result).outputHasAllKeywords(["refund", "processed"]);
1271
+ * ```
1272
+ */
1273
+ outputHasAllKeywords(keywords, opts) {
1274
+ return this.add(TYPE_CONTENT, {
1275
+ target: "output.message",
1276
+ check: "keyword_all",
1277
+ values: keywords,
1278
+ case_sensitive: opts?.caseSensitive ?? false,
1279
+ soft: opts?.soft ?? false
1280
+ });
1281
+ }
1282
+ /**
1283
+ * Assert the output message contains at least one of the given keywords.
1284
+ *
1285
+ * @param keywords - Keywords where at least one must be present.
1286
+ * @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
1287
+ * @param opts.soft - When `true` the failure is non-blocking.
1288
+ * @returns The chain for fluent composition.
1289
+ *
1290
+ * @example
1291
+ * ```ts
1292
+ * expect(result).outputHasAnyKeyword(["yes", "confirmed"]);
1293
+ * ```
1294
+ */
1295
+ outputHasAnyKeyword(keywords, opts) {
1296
+ return this.add(TYPE_CONTENT, {
1297
+ target: "output.message",
1298
+ check: "keyword_any",
1299
+ values: keywords,
1300
+ case_sensitive: opts?.caseSensitive ?? false,
1301
+ soft: opts?.soft ?? false
1302
+ });
1303
+ }
1304
+ /**
1305
+ * Assert the output message contains none of the forbidden terms.
1306
+ *
1307
+ * @param terms - Terms that must not appear in the output.
1308
+ * @returns The chain for fluent composition.
1309
+ *
1310
+ * @example
1311
+ * ```ts
1312
+ * expect(result).outputForbids(["password", "api_key"]);
1313
+ * ```
1314
+ */
1315
+ outputForbids(terms) {
1316
+ return this.add(TYPE_CONTENT, {
1317
+ target: "output.message",
1318
+ check: "forbidden",
1319
+ values: terms
1320
+ });
1321
+ }
1322
+ /**
1323
+ * Generic content-contains check on any target path.
1324
+ *
1325
+ * @param target - Dot-delimited path into the trace.
1326
+ * @param value - Substring to search for.
1327
+ * @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
1328
+ * @param opts.soft - When `true` the failure is non-blocking.
1329
+ * @returns The chain for fluent composition.
1330
+ *
1331
+ * @example
1332
+ * ```ts
1333
+ * expect(result).contentContains("steps[0].result.text", "success");
1334
+ * ```
1335
+ */
1336
+ contentContains(target, value, opts) {
1337
+ return this.add(TYPE_CONTENT, {
1338
+ target,
1339
+ check: "contains",
1340
+ value,
1341
+ case_sensitive: opts?.caseSensitive ?? false,
1342
+ soft: opts?.soft ?? false
1343
+ });
1344
+ }
1345
+ // -- Layer 5: Embedding Similarity --
1346
+ /**
1347
+ * Assert the output is semantically similar to a reference string via embeddings.
1348
+ *
1349
+ * @param reference - The reference text to compare against.
1350
+ * @param opts.threshold - Minimum cosine similarity (default `0.8`).
1351
+ * @param opts.model - Override the embedding model.
1352
+ * @param opts.soft - When `true` the failure is non-blocking.
1353
+ * @returns The chain for fluent composition.
1354
+ *
1355
+ * @example
1356
+ * ```ts
1357
+ * expect(result).outputSimilarTo("The order has been refunded", { threshold: 0.85 });
1358
+ * ```
1359
+ */
1360
+ outputSimilarTo(reference, opts) {
1361
+ return this.add(TYPE_EMBEDDING, {
1362
+ target: "output.message",
1363
+ reference,
1364
+ threshold: opts?.threshold ?? 0.8,
1365
+ model: opts?.model,
1366
+ soft: opts?.soft ?? false
1367
+ });
1368
+ }
1369
+ // -- Layer 6: LLM Judge --
1370
+ /**
1371
+ * Assert the output passes LLM judge evaluation against given criteria.
1372
+ *
1373
+ * @param criteria - Natural language description of what makes a good response.
1374
+ * @param opts.rubric - Rubric name (default `"default"`).
1375
+ * @param opts.threshold - Minimum passing score (default `0.8`).
1376
+ * @param opts.model - Override the judge model.
1377
+ * @param opts.soft - When `true` the failure is non-blocking.
1378
+ * @returns The chain for fluent composition.
1379
+ *
1380
+ * @example
1381
+ * ```ts
1382
+ * expect(result).passesJudge("Response is polite and helpful", { threshold: 0.9 });
1383
+ * ```
1384
+ */
1385
+ passesJudge(criteria, opts) {
1386
+ return this.add(TYPE_LLM_JUDGE, {
1387
+ target: "output.message",
1388
+ criteria,
1389
+ rubric: opts?.rubric ?? "default",
1390
+ threshold: opts?.threshold ?? 0.8,
1391
+ model: opts?.model,
1392
+ soft: opts?.soft ?? false
1393
+ });
1394
+ }
1395
+ // -- Layer 7: Trace Tree (Multi-Agent) --
1396
+ /**
1397
+ * Assert that a specific agent was called in the trace tree.
1398
+ *
1399
+ * @param agentId - The agent identifier to look for.
1400
+ * @param opts.soft - When `true` the failure is non-blocking.
1401
+ * @returns The chain for fluent composition.
1402
+ *
1403
+ * @example
1404
+ * ```ts
1405
+ * expect(result).agentCalled("summarizer");
1406
+ * ```
1407
+ */
1408
+ agentCalled(agentId2, opts) {
1409
+ return this.add(TYPE_TRACE_TREE, {
1410
+ check: "agent_called",
1411
+ agent_id: agentId2,
1412
+ soft: opts?.soft ?? false
1413
+ });
1414
+ }
1415
+ /**
1416
+ * Assert the trace tree delegation depth does not exceed `maxDepth`.
1417
+ *
1418
+ * @param maxDepth - Maximum allowed delegation depth.
1419
+ * @param opts.soft - When `true` the failure is non-blocking.
1420
+ * @returns The chain for fluent composition.
1421
+ *
1422
+ * @example
1423
+ * ```ts
1424
+ * expect(result).delegationDepth(3);
1425
+ * ```
1426
+ */
1427
+ delegationDepth(maxDepth, opts) {
1428
+ return this.add(TYPE_TRACE_TREE, {
1429
+ check: "delegation_depth",
1430
+ max_depth: maxDepth,
1431
+ soft: opts?.soft ?? false
1432
+ });
1433
+ }
1434
+ /**
1435
+ * Assert a sub-agent's output contains the given string.
1436
+ *
1437
+ * @param agentId - The agent whose output to check.
1438
+ * @param value - Substring to search for.
1439
+ * @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
1440
+ * @param opts.soft - When `true` the failure is non-blocking.
1441
+ * @returns The chain for fluent composition.
1442
+ *
1443
+ * @example
1444
+ * ```ts
1445
+ * expect(result).agentOutputContains("summarizer", "key findings");
1446
+ * ```
1447
+ */
1448
+ agentOutputContains(agentId2, value, opts) {
1449
+ return this.add(TYPE_TRACE_TREE, {
1450
+ check: "agent_output_contains",
1451
+ agent_id: agentId2,
1452
+ value,
1453
+ case_sensitive: opts?.caseSensitive ?? false,
1454
+ soft: opts?.soft ?? false
1455
+ });
1456
+ }
1457
+ /**
1458
+ * Assert data flows from one agent's output to another agent's input.
1459
+ *
1460
+ * @param fromAgent - Source agent ID.
1461
+ * @param toAgent - Destination agent ID.
1462
+ * @param field - The data field that must flow between agents.
1463
+ * @param opts.soft - When `true` the failure is non-blocking.
1464
+ * @returns The chain for fluent composition.
1465
+ *
1466
+ * @example
1467
+ * ```ts
1468
+ * expect(result).crossAgentDataFlow("researcher", "writer", "findings");
1469
+ * ```
1470
+ */
1471
+ crossAgentDataFlow(fromAgent, toAgent, field, opts) {
1472
+ return this.add(TYPE_TRACE_TREE, {
1473
+ check: "cross_agent_data_flow",
1474
+ from_agent: fromAgent,
1475
+ to_agent: toAgent,
1476
+ field,
1477
+ soft: opts?.soft ?? false
1478
+ });
1479
+ }
1480
+ /**
1481
+ * Assert agent delegations follow the specified transitions.
1482
+ *
1483
+ * @param transitions - Array of `[fromAgent, toAgent]` pairs.
1484
+ * @param opts.soft - When `true` the failure is non-blocking.
1485
+ * @returns The chain for fluent composition.
1486
+ *
1487
+ * @example
1488
+ * ```ts
1489
+ * expect(result).followsTransitions([["root", "planner"], ["planner", "executor"]]);
1490
+ * ```
1491
+ */
1492
+ followsTransitions(transitions, opts) {
1493
+ return this.add(TYPE_TRACE_TREE, {
1494
+ check: "follows_transitions",
1495
+ transitions,
1496
+ soft: opts?.soft ?? false
1497
+ });
1498
+ }
1499
+ /**
1500
+ * Assert aggregate cost across the entire trace tree is at or below `maxCost`.
1501
+ *
1502
+ * @param maxCost - Maximum allowed aggregate cost in USD.
1503
+ * @param opts.soft - When `true` the failure is non-blocking.
1504
+ * @returns The chain for fluent composition.
1505
+ *
1506
+ * @example
1507
+ * ```ts
1508
+ * expect(result).aggregateCostUnder(0.50);
1509
+ * ```
1510
+ */
1511
+ aggregateCostUnder(maxCost, opts) {
1512
+ return this.add(TYPE_TRACE_TREE, {
1513
+ check: "aggregate_cost",
1514
+ operator: "lte",
1515
+ value: maxCost,
1516
+ soft: opts?.soft ?? false
1517
+ });
1518
+ }
1519
+ /**
1520
+ * Assert aggregate tokens across the entire trace tree is at or below `maxTokens`.
1521
+ *
1522
+ * @param maxTokens - Maximum allowed aggregate token count.
1523
+ * @param opts.soft - When `true` the failure is non-blocking.
1524
+ * @returns The chain for fluent composition.
1525
+ *
1526
+ * @example
1527
+ * ```ts
1528
+ * expect(result).aggregateTokensUnder(10000);
1529
+ * ```
1530
+ */
1531
+ aggregateTokensUnder(maxTokens, opts) {
1532
+ return this.add(TYPE_TRACE_TREE, {
1533
+ check: "aggregate_tokens",
1534
+ operator: "lte",
1535
+ value: maxTokens,
1536
+ soft: opts?.soft ?? false
1537
+ });
1538
+ }
1539
+ /**
1540
+ * Assert `agentA` started before `agentB` in the trace tree.
1541
+ *
1542
+ * @param agentA - The agent expected to run first.
1543
+ * @param agentB - The agent expected to run second.
1544
+ * @param opts.soft - When `true` the failure is non-blocking.
1545
+ * @returns The chain for fluent composition.
1546
+ *
1547
+ * @example
1548
+ * ```ts
1549
+ * expect(result).agentOrderedBefore("planner", "executor");
1550
+ * ```
1551
+ */
1552
+ agentOrderedBefore(agentA, agentB, opts) {
1553
+ return this.add(TYPE_TRACE_TREE, {
1554
+ check: "agent_ordered_before",
1555
+ agent_a: agentA,
1556
+ agent_b: agentB,
1557
+ soft: opts?.soft ?? false
1558
+ });
1559
+ }
1560
+ /**
1561
+ * Assert `agentA` and `agentB` ran concurrently (overlapping wall-clock time).
1562
+ *
1563
+ * @param agentA - First agent ID.
1564
+ * @param agentB - Second agent ID.
1565
+ * @param opts.soft - When `true` the failure is non-blocking.
1566
+ * @returns The chain for fluent composition.
1567
+ *
1568
+ * @example
1569
+ * ```ts
1570
+ * expect(result).agentsOverlap("worker-1", "worker-2");
1571
+ * ```
1572
+ */
1573
+ agentsOverlap(agentA, agentB, opts) {
1574
+ return this.add(TYPE_TRACE_TREE, {
1575
+ check: "agents_overlap",
1576
+ agent_a: agentA,
1577
+ agent_b: agentB,
1578
+ soft: opts?.soft ?? false
1579
+ });
1580
+ }
1581
+ /**
1582
+ * Assert a specific agent's wall-clock duration is under `maxMs` milliseconds.
1583
+ *
1584
+ * @param agentId - The agent to check.
1585
+ * @param maxMs - Maximum allowed wall-clock time in milliseconds.
1586
+ * @param opts.soft - When `true` the failure is non-blocking.
1587
+ * @returns The chain for fluent composition.
1588
+ *
1589
+ * @example
1590
+ * ```ts
1591
+ * expect(result).agentWallTimeUnder("summarizer", 5000);
1592
+ * ```
1593
+ */
1594
+ agentWallTimeUnder(agentId2, maxMs, opts) {
1595
+ return this.add(TYPE_TRACE_TREE, {
1596
+ check: "agent_wall_time_under",
1597
+ agent_id: agentId2,
1598
+ max_ms: maxMs,
1599
+ soft: opts?.soft ?? false
1600
+ });
1601
+ }
1602
+ /**
1603
+ * Assert agents ran in ordered groups (parallel within each group, sequential across groups).
1604
+ *
1605
+ * @param groups - Array of agent-ID arrays. Each sub-array is a parallel group.
1606
+ * @param opts.soft - When `true` the failure is non-blocking.
1607
+ * @returns The chain for fluent composition.
1608
+ *
1609
+ * @example
1610
+ * ```ts
1611
+ * expect(result).orderedAgents([["planner"], ["worker-1", "worker-2"], ["aggregator"]]);
1612
+ * ```
1613
+ */
1614
+ orderedAgents(groups, opts) {
1615
+ return this.add(TYPE_TRACE_TREE, {
1616
+ check: "ordered_agents",
1617
+ groups,
1618
+ soft: opts?.soft ?? false
1619
+ });
1620
+ }
1621
+ // -- Layer 8: Plugin --
1622
+ /**
1623
+ * Assert via a registered plugin (Layer 8).
1624
+ *
1625
+ * @param pluginId - Identifier of the plugin registered with the engine.
1626
+ * @param config - Plugin-specific configuration passed at evaluation time.
1627
+ * @param opts.soft - When `true` the failure is non-blocking.
1628
+ * @returns The chain for fluent composition.
1629
+ *
1630
+ * @example
1631
+ * ```ts
1632
+ * expect(result).plugin("custom-safety-check", { threshold: 0.95 });
1633
+ * ```
1634
+ */
1635
+ plugin(pluginId, config2, opts) {
1636
+ const spec = { plugin_id: pluginId, soft: opts?.soft ?? false };
1637
+ if (config2 !== void 0) spec.config = config2;
1638
+ return this.add(TYPE_PLUGIN, spec);
1639
+ }
1640
+ // -- TraceTree Analytics (P9 parity) --
1641
+ /**
1642
+ * Assert aggregate latency across the trace tree is at or below `maxMs`.
1643
+ *
1644
+ * @param maxMs - Maximum allowed aggregate latency in milliseconds.
1645
+ * @param opts.soft - When `true` the failure is non-blocking.
1646
+ * @returns The chain for fluent composition.
1647
+ *
1648
+ * @example
1649
+ * ```ts
1650
+ * expect(result).aggregateLatencyUnder(5000);
1651
+ * ```
1652
+ */
1653
+ aggregateLatencyUnder(maxMs, opts) {
1654
+ return this.add(TYPE_TRACE_TREE, {
1655
+ check: "aggregate_latency",
1656
+ operator: "lte",
1657
+ value: maxMs,
1658
+ soft: opts?.soft ?? false
1659
+ });
1660
+ }
1661
+ /**
1662
+ * Assert that all specified tools were called across the entire trace tree.
1663
+ *
1664
+ * @param toolNames - Tool names that must appear somewhere in the tree.
1665
+ * @param opts.soft - When `true` the failure is non-blocking.
1666
+ * @returns The chain for fluent composition.
1667
+ *
1668
+ * @example
1669
+ * ```ts
1670
+ * expect(result).allToolsCalled(["search", "summarize"]);
1671
+ * ```
1672
+ */
1673
+ allToolsCalled(toolNames, opts) {
1674
+ return this.add(TYPE_TRACE_TREE, {
1675
+ check: "all_tools_called",
1676
+ tools: toolNames,
1677
+ soft: opts?.soft ?? false
1678
+ });
1679
+ }
1680
+ };
1681
+ function attestExpect(result) {
1682
+ if (result instanceof AgentResult) {
1683
+ return new ExpectChain(result);
1684
+ }
1685
+ return new ExpectChain(new AgentResult(result));
1686
+ }
1687
+
1688
+ // src/context.ts
1689
+ import { AsyncLocalStorage } from "async_hooks";
1690
+ var activeBuilder = new AsyncLocalStorage();
1691
+
1692
+ // src/agent.ts
1693
+ var Agent = class {
1694
+ name;
1695
+ fn;
1696
+ constructor(name, fn) {
1697
+ this.name = name;
1698
+ this.fn = fn;
1699
+ }
1700
+ run(args = {}) {
1701
+ if (this.fn === void 0) {
1702
+ throw new Error("No agent function provided. Pass fn to Agent().");
1703
+ }
1704
+ const builder = new TraceBuilder(this.name);
1705
+ builder.setInput(args);
1706
+ const output = activeBuilder.run(builder, () => this.fn(builder, args));
1707
+ this.setOutput(builder, output);
1708
+ return new AgentResult(builder.build());
1709
+ }
1710
+ async arun(args = {}) {
1711
+ if (this.fn === void 0) {
1712
+ throw new Error("No agent function provided. Pass fn to Agent().");
1713
+ }
1714
+ const builder = new TraceBuilder(this.name);
1715
+ builder.setInput(args);
1716
+ const output = await activeBuilder.run(builder, async () => this.fn(builder, args));
1717
+ this.setOutput(builder, output);
1718
+ return new AgentResult(builder.build());
1719
+ }
1720
+ setOutput(builder, output) {
1721
+ if (output !== null && output !== void 0 && typeof output === "object" && !Array.isArray(output)) {
1722
+ builder.setOutput(output);
1723
+ } else if (output !== void 0) {
1724
+ builder.setOutput({ result: output });
1725
+ } else {
1726
+ builder.setOutput({ result: null });
1727
+ }
1728
+ }
1729
+ withTrace(trace) {
1730
+ return new AgentResult(trace);
1731
+ }
1732
+ };
1733
+ function agent(name, fn) {
1734
+ const wrapped = new Agent(name, fn);
1735
+ const wrapper = (args) => {
1736
+ return wrapped.run(args ?? {});
1737
+ };
1738
+ wrapper.agent = wrapped;
1739
+ return wrapper;
1740
+ }
1741
+
1742
+ // src/delegate.ts
1743
+ async function delegate(agentId2, fn) {
1744
+ const parent = activeBuilder.getStore();
1745
+ if (parent === void 0) {
1746
+ throw new Error(
1747
+ "delegate() must be used within an Agent.run() context. No active TraceBuilder found."
1748
+ );
1749
+ }
1750
+ const child = new TraceBuilder(agentId2);
1751
+ child.setParentTraceId(parent.getTraceId());
1752
+ await activeBuilder.run(child, async () => {
1753
+ await fn(child);
1754
+ });
1755
+ const childTrace = child.build();
1756
+ parent.addStep({
1757
+ type: STEP_AGENT_CALL,
1758
+ name: agentId2,
1759
+ args: void 0,
1760
+ result: childTrace.output,
1761
+ sub_trace: childTrace
1762
+ });
1763
+ }
1764
+
1765
+ // src/tier.ts
1766
+ var TIER_1 = 1;
1767
+ var TIER_2 = 2;
1768
+ var TIER_3 = 3;
1769
+ function tier(level) {
1770
+ return (fn) => {
1771
+ fn._attest_tier = level;
1772
+ return fn;
1773
+ };
1774
+ }
1775
+
1776
+ // src/plugins.ts
1777
+ var PluginRegistry = class {
1778
+ plugins = /* @__PURE__ */ new Map();
1779
+ register(name, plugin) {
1780
+ const pluginType = plugin.pluginType;
1781
+ let typeMap = this.plugins.get(pluginType);
1782
+ if (typeMap === void 0) {
1783
+ typeMap = /* @__PURE__ */ new Map();
1784
+ this.plugins.set(pluginType, typeMap);
1785
+ }
1786
+ typeMap.set(name, plugin);
1787
+ }
1788
+ get(pluginType, name) {
1789
+ return this.plugins.get(pluginType)?.get(name);
1790
+ }
1791
+ listPlugins(pluginType) {
1792
+ if (pluginType !== void 0) {
1793
+ const typeMap = this.plugins.get(pluginType);
1794
+ return typeMap !== void 0 ? [...typeMap.keys()] : [];
1795
+ }
1796
+ const result = [];
1797
+ for (const [ptype, typeMap] of this.plugins) {
1798
+ for (const pname of typeMap.keys()) {
1799
+ result.push(`${ptype}/${pname}`);
1800
+ }
1801
+ }
1802
+ return result;
1803
+ }
1804
+ };
1805
+ async function executePluginAssertion(plugin, trace, spec, client, traceId2, assertionId2, timeoutMs = 3e4) {
1806
+ const resultPromise = Promise.resolve(plugin.execute(trace, spec));
1807
+ const timeoutPromise = new Promise((_, reject) => {
1808
+ setTimeout(() => reject(new Error(`Plugin '${plugin.name}' timed out after ${timeoutMs}ms`)), timeoutMs);
1809
+ });
1810
+ const result = await Promise.race([resultPromise, timeoutPromise]);
1811
+ await client.submitPluginResult(
1812
+ traceId2,
1813
+ plugin.name,
1814
+ assertionId2,
1815
+ result.status,
1816
+ result.score,
1817
+ result.explanation
1818
+ );
1819
+ return result;
1820
+ }
1821
+
1822
+ // src/continuous.ts
1823
+ import { EventEmitter } from "events";
1824
+ var DEFAULT_QUEUE_SIZE = 1e3;
1825
+ function resolveQueueSize() {
1826
+ const raw = process.env["ATTEST_CONTINUOUS_QUEUE_SIZE"];
1827
+ if (raw !== void 0 && raw !== "") {
1828
+ const parsed = Number(raw);
1829
+ if (Number.isFinite(parsed) && parsed > 0) return parsed;
1830
+ }
1831
+ return DEFAULT_QUEUE_SIZE;
1832
+ }
1833
+ var Sampler = class {
1834
+ rate;
1835
+ constructor(rate) {
1836
+ if (rate < 0 || rate > 1) {
1837
+ throw new Error(`sample_rate must be in [0.0, 1.0], got ${rate}`);
1838
+ }
1839
+ this.rate = rate;
1840
+ }
1841
+ shouldSample() {
1842
+ return Math.random() < this.rate;
1843
+ }
1844
+ };
1845
+ var AlertDispatcher = class extends EventEmitter {
1846
+ webhookUrl;
1847
+ slackUrl;
1848
+ constructor(webhookUrl, slackUrl) {
1849
+ super();
1850
+ this.webhookUrl = webhookUrl;
1851
+ this.slackUrl = slackUrl;
1852
+ }
1853
+ async dispatch(alert) {
1854
+ this.emit("alert", alert);
1855
+ const tasks = [];
1856
+ if (this.webhookUrl) {
1857
+ tasks.push(this.postJson(this.webhookUrl, alert));
1858
+ }
1859
+ if (this.slackUrl) {
1860
+ const text = `[attest] drift alert -- type=${alert.drift_type} score=${alert.score} trace_id=${alert.trace_id}`;
1861
+ tasks.push(this.postJson(this.slackUrl, { text }));
1862
+ }
1863
+ const results = await Promise.allSettled(tasks);
1864
+ for (const result of results) {
1865
+ if (result.status === "rejected") {
1866
+ this.emit("alertError", result.reason);
1867
+ }
1868
+ }
1869
+ }
1870
+ async postJson(url, payload) {
1871
+ const response = await fetch(url, {
1872
+ method: "POST",
1873
+ headers: { "Content-Type": "application/json" },
1874
+ body: JSON.stringify(payload),
1875
+ signal: AbortSignal.timeout(1e4)
1876
+ });
1877
+ if (!response.ok) {
1878
+ throw new Error(`POST to ${url} returned ${response.status}`);
1879
+ }
1880
+ }
1881
+ };
1882
+ var ContinuousEvalRunner = class {
1883
+ client;
1884
+ assertions;
1885
+ sampler;
1886
+ dispatcher;
1887
+ queue = [];
1888
+ maxSize;
1889
+ running = false;
1890
+ loopTimer = null;
1891
+ constructor(client, assertions, options) {
1892
+ this.client = client;
1893
+ this.assertions = assertions;
1894
+ this.sampler = new Sampler(options?.sampleRate ?? 1);
1895
+ this.dispatcher = new AlertDispatcher(options?.alertWebhook, options?.alertSlackUrl);
1896
+ this.maxSize = options?.maxSize ?? resolveQueueSize();
1897
+ }
1898
+ get alertEmitter() {
1899
+ return this.dispatcher;
1900
+ }
1901
+ async evaluateTrace(trace) {
1902
+ if (!this.sampler.shouldSample()) return null;
1903
+ return this.client.evaluateBatch(trace, this.assertions);
1904
+ }
1905
+ submit(trace) {
1906
+ if (this.queue.length >= this.maxSize) {
1907
+ return false;
1908
+ }
1909
+ this.queue.push(trace);
1910
+ return true;
1911
+ }
1912
+ start() {
1913
+ if (this.running) return;
1914
+ this.running = true;
1915
+ this.scheduleNext();
1916
+ }
1917
+ stop() {
1918
+ this.running = false;
1919
+ if (this.loopTimer !== null) {
1920
+ clearTimeout(this.loopTimer);
1921
+ this.loopTimer = null;
1922
+ }
1923
+ }
1924
+ get queueLength() {
1925
+ return this.queue.length;
1926
+ }
1927
+ get isRunning() {
1928
+ return this.running;
1929
+ }
1930
+ scheduleNext() {
1931
+ if (!this.running) return;
1932
+ this.loopTimer = setTimeout(async () => {
1933
+ if (!this.running) return;
1934
+ const trace = this.queue.shift();
1935
+ if (trace !== void 0) {
1936
+ try {
1937
+ await this.evaluateTrace(trace);
1938
+ } catch {
1939
+ }
1940
+ }
1941
+ this.scheduleNext();
1942
+ }, this.queue.length > 0 ? 0 : 100);
1943
+ }
1944
+ };
1945
+
1946
+ // src/adapters/manual.ts
1947
+ var ManualAdapter = class {
1948
+ agentId;
1949
+ constructor(agentId2) {
1950
+ this.agentId = agentId2;
1951
+ }
1952
+ capture(builderFn) {
1953
+ const builder = new TraceBuilder(this.agentId);
1954
+ builderFn(builder);
1955
+ return builder.build();
1956
+ }
1957
+ createBuilder() {
1958
+ return new TraceBuilder(this.agentId);
1959
+ }
1960
+ };
1961
+
1962
+ // src/adapters/openai.ts
1963
+ var OpenAIAdapter = class {
1964
+ agentId;
1965
+ constructor(agentId2) {
1966
+ this.agentId = agentId2;
1967
+ }
1968
+ traceFromResponse(response, options) {
1969
+ const builder = new TraceBuilder(this.agentId);
1970
+ if (options?.inputMessages) {
1971
+ builder.setInput({ messages: options.inputMessages });
1972
+ }
1973
+ const message = response.choices[0].message;
1974
+ const completionText = message.content ?? "";
1975
+ const stepArgs = {};
1976
+ if (response.model !== void 0) stepArgs.model = response.model;
1977
+ const stepResult = { completion: completionText };
1978
+ if (response.usage?.total_tokens !== void 0) {
1979
+ stepResult.tokens = response.usage.total_tokens;
1980
+ }
1981
+ builder.addLlmCall("completion", { args: stepArgs, result: stepResult });
1982
+ if (message.tool_calls) {
1983
+ for (const tc of message.tool_calls) {
1984
+ builder.addToolCall(tc.function.name, {
1985
+ args: { arguments: tc.function.arguments }
1986
+ });
1987
+ }
1988
+ }
1989
+ builder.setOutput({
1990
+ message: completionText,
1991
+ structured: options?.structuredOutput ?? {}
1992
+ });
1993
+ builder.setMetadata({
1994
+ total_tokens: response.usage?.total_tokens,
1995
+ cost_usd: options?.costUsd,
1996
+ latency_ms: options?.latencyMs,
1997
+ model: response.model
1998
+ });
1999
+ return builder.build();
2000
+ }
2001
+ };
2002
+
2003
+ // src/adapters/anthropic.ts
2004
+ var AnthropicAdapter = class {
2005
+ agentId;
2006
+ constructor(agentId2) {
2007
+ this.agentId = agentId2;
2008
+ }
2009
+ traceFromResponse(response, options) {
2010
+ const builder = new TraceBuilder(this.agentId);
2011
+ if (options?.inputMessages) {
2012
+ builder.setInput({ messages: options.inputMessages });
2013
+ }
2014
+ const completionParts = [];
2015
+ for (const block of response.content) {
2016
+ if (block.type === "text") {
2017
+ completionParts.push(block.text);
2018
+ } else if (block.type === "tool_use") {
2019
+ builder.addToolCall(block.name, {
2020
+ args: block.input
2021
+ });
2022
+ }
2023
+ }
2024
+ const completionText = completionParts.join("\n");
2025
+ const stepArgs = {};
2026
+ if (response.model !== void 0) stepArgs.model = response.model;
2027
+ builder.addLlmCall("completion", { args: stepArgs, result: { completion: completionText } });
2028
+ builder.setOutput({ message: completionText });
2029
+ let totalTokens;
2030
+ if (response.usage) {
2031
+ totalTokens = response.usage.input_tokens + response.usage.output_tokens;
2032
+ }
2033
+ builder.setMetadata({
2034
+ total_tokens: totalTokens,
2035
+ cost_usd: options?.costUsd,
2036
+ latency_ms: options?.latencyMs,
2037
+ model: response.model
2038
+ });
2039
+ return builder.build();
2040
+ }
2041
+ };
2042
+
2043
+ // src/adapters/gemini.ts
2044
+ var GeminiAdapter = class {
2045
+ agentId;
2046
+ constructor(agentId2) {
2047
+ this.agentId = agentId2;
2048
+ }
2049
+ traceFromResponse(response, options) {
2050
+ const builder = new TraceBuilder(this.agentId);
2051
+ if (options?.inputText) {
2052
+ builder.setInput({ text: options.inputText });
2053
+ }
2054
+ let completionText = "";
2055
+ if (response.text != null) {
2056
+ completionText = response.text;
2057
+ } else if (response.candidates && response.candidates.length > 0) {
2058
+ completionText = response.candidates[0].content.parts.filter((p) => p.text != null).map((p) => p.text).join("");
2059
+ }
2060
+ if (response.candidates && response.candidates.length > 0) {
2061
+ for (const part of response.candidates[0].content.parts) {
2062
+ if (part.functionCall) {
2063
+ builder.addToolCall(part.functionCall.name, {
2064
+ args: part.functionCall.args ?? {}
2065
+ });
2066
+ }
2067
+ }
2068
+ }
2069
+ builder.addLlmCall("completion", { result: { completion: completionText } });
2070
+ builder.setOutput({ message: completionText });
2071
+ builder.setMetadata({
2072
+ cost_usd: options?.costUsd,
2073
+ latency_ms: options?.latencyMs,
2074
+ model: options?.model
2075
+ });
2076
+ return builder.build();
2077
+ }
2078
+ };
2079
+
2080
+ // src/adapters/ollama.ts
2081
+ var OllamaAdapter = class {
2082
+ agentId;
2083
+ constructor(agentId2) {
2084
+ this.agentId = agentId2;
2085
+ }
2086
+ traceFromResponse(response, options) {
2087
+ const builder = new TraceBuilder(this.agentId);
2088
+ if (options?.inputMessages) {
2089
+ builder.setInput({ messages: options.inputMessages });
2090
+ }
2091
+ const completionText = response.message?.content ?? "";
2092
+ builder.addLlmCall("completion", {
2093
+ args: { model: response.model ?? "" },
2094
+ result: { completion: completionText }
2095
+ });
2096
+ builder.setOutput({ message: completionText });
2097
+ let totalTokens;
2098
+ if (response.eval_count !== void 0 && response.prompt_eval_count !== void 0) {
2099
+ totalTokens = response.eval_count + response.prompt_eval_count;
2100
+ }
2101
+ builder.setMetadata({
2102
+ total_tokens: totalTokens,
2103
+ latency_ms: options?.latencyMs,
2104
+ model: response.model
2105
+ });
2106
+ return builder.build();
2107
+ }
2108
+ };
2109
+
2110
+ // src/adapters/otel.ts
2111
+ var OTelAdapter = class _OTelAdapter {
2112
+ agentId;
2113
+ constructor(agentId2) {
2114
+ this.agentId = agentId2;
2115
+ }
2116
+ static fromSpans(spans, agentId2) {
2117
+ const adapter = new _OTelAdapter(agentId2);
2118
+ return adapter.buildTrace(spans);
2119
+ }
2120
+ buildTrace(spans) {
2121
+ const sorted = [...spans].sort((a, b) => {
2122
+ const aTime = a.startTime[0] * 1e9 + a.startTime[1];
2123
+ const bTime = b.startTime[0] * 1e9 + b.startTime[1];
2124
+ return aTime - bTime;
2125
+ });
2126
+ const builder = new TraceBuilder(this.agentId);
2127
+ const rootSpan = this.findRootSpan(sorted);
2128
+ if (rootSpan) {
2129
+ const traceId2 = rootSpan.spanContext().traceId;
2130
+ if (traceId2) {
2131
+ builder.setTraceId(`otel_${traceId2.slice(0, 16)}`);
2132
+ }
2133
+ }
2134
+ let outputMessage = "";
2135
+ let totalTokens;
2136
+ let latencyMs;
2137
+ let model;
2138
+ for (const span of sorted) {
2139
+ const attrs = span.attributes ?? {};
2140
+ const stepType = this.classifySpan(attrs, span.name);
2141
+ if (stepType === "llm_call") {
2142
+ const [stepArgs, stepResult] = this.extractLlmStep(attrs);
2143
+ builder.addLlmCall(span.name, {
2144
+ args: stepArgs,
2145
+ result: stepResult,
2146
+ metadata: this.spanMetadata(span)
2147
+ });
2148
+ const completion = attrs["gen_ai.completion"];
2149
+ if (completion) outputMessage = String(completion);
2150
+ const inputTokens = Number(attrs["gen_ai.usage.input_tokens"] ?? 0);
2151
+ const outputTokens = Number(attrs["gen_ai.usage.output_tokens"] ?? 0);
2152
+ const spanTokens = inputTokens + outputTokens;
2153
+ if (spanTokens > 0) totalTokens = (totalTokens ?? 0) + spanTokens;
2154
+ if (model === void 0) {
2155
+ const responseModel = attrs["gen_ai.response.model"];
2156
+ const requestModel = attrs["gen_ai.request.model"];
2157
+ if (responseModel) model = String(responseModel);
2158
+ else if (requestModel) model = String(requestModel);
2159
+ }
2160
+ } else if (stepType === "tool_call") {
2161
+ const [stepArgs, stepResult] = this.extractToolStep(attrs);
2162
+ const toolName = String(attrs["gen_ai.tool.name"] ?? span.name);
2163
+ builder.addToolCall(toolName, {
2164
+ args: stepArgs,
2165
+ result: stepResult,
2166
+ metadata: this.spanMetadata(span)
2167
+ });
2168
+ }
2169
+ }
2170
+ if (rootSpan) {
2171
+ const startNs = rootSpan.startTime[0] * 1e9 + rootSpan.startTime[1];
2172
+ const endNs = rootSpan.endTime[0] * 1e9 + rootSpan.endTime[1];
2173
+ latencyMs = Math.floor((endNs - startNs) / 1e6);
2174
+ }
2175
+ builder.setOutput({ message: outputMessage });
2176
+ builder.setMetadata({ total_tokens: totalTokens, latency_ms: latencyMs, model });
2177
+ return builder.build();
2178
+ }
2179
+ findRootSpan(spans) {
2180
+ for (const span of spans) {
2181
+ if (!span.parentSpanId) return span;
2182
+ }
2183
+ return spans[0];
2184
+ }
2185
+ classifySpan(attrs, name) {
2186
+ const op = String(attrs["gen_ai.operation.name"] ?? "");
2187
+ if (op === "chat" || op === "completion" || op === "generate_content" || "gen_ai.completion" in attrs) {
2188
+ return "llm_call";
2189
+ }
2190
+ if (op === "tool" || "gen_ai.tool.name" in attrs) {
2191
+ return "tool_call";
2192
+ }
2193
+ const lower = name.toLowerCase();
2194
+ if (lower.includes("completion") || lower.includes("chat")) return "llm_call";
2195
+ if (lower.includes("tool")) return "tool_call";
2196
+ return void 0;
2197
+ }
2198
+ extractLlmStep(attrs) {
2199
+ const args = {};
2200
+ const result = {};
2201
+ if ("gen_ai.request.model" in attrs) args.model = String(attrs["gen_ai.request.model"]);
2202
+ if ("gen_ai.system" in attrs) args.system = String(attrs["gen_ai.system"]);
2203
+ if ("gen_ai.prompt" in attrs) args.prompt = String(attrs["gen_ai.prompt"]);
2204
+ if ("gen_ai.completion" in attrs) result.completion = String(attrs["gen_ai.completion"]);
2205
+ if ("gen_ai.usage.input_tokens" in attrs) result.input_tokens = Number(attrs["gen_ai.usage.input_tokens"]);
2206
+ if ("gen_ai.usage.output_tokens" in attrs) result.output_tokens = Number(attrs["gen_ai.usage.output_tokens"]);
2207
+ if ("gen_ai.response.model" in attrs) result.model = String(attrs["gen_ai.response.model"]);
2208
+ return [args, result];
2209
+ }
2210
+ extractToolStep(attrs) {
2211
+ const args = {};
2212
+ const result = {};
2213
+ if ("gen_ai.tool.call.id" in attrs) args.call_id = String(attrs["gen_ai.tool.call.id"]);
2214
+ if ("gen_ai.tool.parameters" in attrs) args.parameters = attrs["gen_ai.tool.parameters"];
2215
+ if ("gen_ai.tool.output" in attrs) result.output = attrs["gen_ai.tool.output"];
2216
+ return [args, result];
2217
+ }
2218
+ spanMetadata(span) {
2219
+ const startNs = span.startTime[0] * 1e9 + span.startTime[1];
2220
+ const endNs = span.endTime[0] * 1e9 + span.endTime[1];
2221
+ return { duration_ms: Math.floor((endNs - startNs) / 1e6) };
2222
+ }
2223
+ };
2224
+
2225
+ // src/adapters/langchain.ts
2226
+ var LangChainAdapter = class {
2227
+ agentId;
2228
+ llmCalls = [];
2229
+ toolCalls = [];
2230
+ chainOutput = "";
2231
+ startTime = 0;
2232
+ constructor(agentId2) {
2233
+ this.agentId = agentId2;
2234
+ }
2235
+ /**
2236
+ * Reset accumulated state for a new trace capture session.
2237
+ */
2238
+ reset() {
2239
+ this.llmCalls.length = 0;
2240
+ this.toolCalls.length = 0;
2241
+ this.chainOutput = "";
2242
+ this.startTime = 0;
2243
+ }
2244
+ /**
2245
+ * Call this when a chain starts (maps to `handleChainStart`).
2246
+ */
2247
+ handleChainStart() {
2248
+ if (this.startTime === 0) {
2249
+ this.startTime = Date.now();
2250
+ }
2251
+ }
2252
+ /**
2253
+ * Call this when an LLM call completes (maps to `handleLLMEnd`).
2254
+ */
2255
+ handleLLMEnd(output) {
2256
+ const text = output.generations[0]?.[0]?.text ?? "";
2257
+ const tokens = output.llmOutput?.["tokenUsage"];
2258
+ const model = output.llmOutput?.["modelName"];
2259
+ this.llmCalls.push({
2260
+ name: "completion",
2261
+ text,
2262
+ tokens: tokens?.totalTokens,
2263
+ model
2264
+ });
2265
+ }
2266
+ /**
2267
+ * Call this when a tool call completes (maps to `handleToolEnd`).
2268
+ */
2269
+ handleToolEnd(output, metadata) {
2270
+ this.toolCalls.push({
2271
+ name: metadata?.name ?? "unknown_tool",
2272
+ output
2273
+ });
2274
+ }
2275
+ /**
2276
+ * Call this when a chain completes (maps to `handleChainEnd`).
2277
+ */
2278
+ handleChainEnd(output) {
2279
+ const text = output["output"] ?? output["text"] ?? output["result"];
2280
+ if (typeof text === "string") {
2281
+ this.chainOutput = text;
2282
+ }
2283
+ }
2284
+ /**
2285
+ * Build a Trace from accumulated callback events.
2286
+ *
2287
+ * @param options.costUsd - Total cost override.
2288
+ * @param options.model - Model name override.
2289
+ * @returns A finalized Trace object.
2290
+ */
2291
+ buildTrace(options) {
2292
+ const builder = new TraceBuilder(this.agentId);
2293
+ const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
2294
+ let totalTokens;
2295
+ for (const llm of this.llmCalls) {
2296
+ const args = {};
2297
+ if (llm.model ?? options?.model) args.model = llm.model ?? options?.model;
2298
+ const result = { completion: llm.text };
2299
+ if (llm.tokens !== void 0) {
2300
+ result.tokens = llm.tokens;
2301
+ totalTokens = (totalTokens ?? 0) + llm.tokens;
2302
+ }
2303
+ builder.addLlmCall(llm.name, { args, result });
2304
+ }
2305
+ for (const tool of this.toolCalls) {
2306
+ builder.addToolCall(tool.name, {
2307
+ result: { output: tool.output }
2308
+ });
2309
+ }
2310
+ const outputMessage = this.chainOutput || (this.llmCalls.length > 0 ? this.llmCalls[this.llmCalls.length - 1].text : "");
2311
+ builder.setOutput({ message: outputMessage });
2312
+ builder.setMetadata({
2313
+ total_tokens: totalTokens,
2314
+ cost_usd: options?.costUsd,
2315
+ latency_ms: latencyMs,
2316
+ model: options?.model ?? this.llmCalls[0]?.model
2317
+ });
2318
+ return builder.build();
2319
+ }
2320
+ };
2321
+ export {
2322
+ Agent,
2323
+ AgentResult,
2324
+ AlertDispatcher,
2325
+ AnthropicAdapter,
2326
+ AttestClient,
2327
+ ContinuousEvalRunner,
2328
+ ENGINE_VERSION,
2329
+ ERR_ASSERTION_ERROR,
2330
+ ERR_ENGINE_ERROR,
2331
+ ERR_INVALID_TRACE,
2332
+ ERR_PROVIDER_ERROR,
2333
+ ERR_SESSION_ERROR,
2334
+ ERR_TIMEOUT,
2335
+ EngineManager,
2336
+ EngineTimeoutError,
2337
+ ExpectChain,
2338
+ GeminiAdapter,
2339
+ LangChainAdapter,
2340
+ ManualAdapter,
2341
+ OTelAdapter,
2342
+ OllamaAdapter,
2343
+ OpenAIAdapter,
2344
+ PluginRegistry,
2345
+ ProtocolError,
2346
+ STATUS_HARD_FAIL,
2347
+ STATUS_PASS,
2348
+ STATUS_SOFT_FAIL,
2349
+ STEP_AGENT_CALL,
2350
+ STEP_LLM_CALL,
2351
+ STEP_RETRIEVAL,
2352
+ STEP_TOOL_CALL,
2353
+ Sampler,
2354
+ TIER_1,
2355
+ TIER_2,
2356
+ TIER_3,
2357
+ TYPE_CONSTRAINT,
2358
+ TYPE_CONTENT,
2359
+ TYPE_EMBEDDING,
2360
+ TYPE_LLM_JUDGE,
2361
+ TYPE_PLUGIN,
2362
+ TYPE_SCHEMA,
2363
+ TYPE_TRACE,
2364
+ TYPE_TRACE_TREE,
2365
+ TraceBuilder,
2366
+ TraceTree,
2367
+ VERSION,
2368
+ activeBuilder,
2369
+ agent,
2370
+ agentId,
2371
+ assertionId,
2372
+ attestExpect,
2373
+ config,
2374
+ decodeResponse,
2375
+ delegate,
2376
+ encodeRequest,
2377
+ executePluginAssertion,
2378
+ extractId,
2379
+ extractResult,
2380
+ isSimulationMode,
2381
+ resetConfig,
2382
+ tier,
2383
+ traceId
2384
+ };
2385
+ //# sourceMappingURL=index.js.map