@attest-ai/core 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +2595 -0
- package/dist/cjs/index.cjs.map +1 -0
- package/dist/cjs/index.d.cts +1111 -0
- package/dist/cjs/proto/index.cjs +208 -0
- package/dist/cjs/proto/index.cjs.map +1 -0
- package/dist/cjs/proto/index.d.cts +269 -0
- package/dist/esm/cli.d.ts +9 -0
- package/dist/esm/cli.js +154 -0
- package/dist/esm/cli.js.map +1 -0
- package/dist/esm/index.d.ts +1111 -0
- package/dist/esm/index.js +2490 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/proto/index.d.ts +269 -0
- package/dist/esm/proto/index.js +152 -0
- package/dist/esm/proto/index.js.map +1 -0
- package/package.json +37 -11
- package/dist/adapters/anthropic.d.ts +0 -31
- package/dist/adapters/anthropic.d.ts.map +0 -1
- package/dist/adapters/anthropic.js +0 -42
- package/dist/adapters/anthropic.js.map +0 -1
- package/dist/adapters/gemini.d.ts +0 -30
- package/dist/adapters/gemini.d.ts.map +0 -1
- package/dist/adapters/gemini.js +0 -41
- package/dist/adapters/gemini.js.map +0 -1
- package/dist/adapters/index.d.ts +0 -11
- package/dist/adapters/index.d.ts.map +0 -1
- package/dist/adapters/index.js +0 -7
- package/dist/adapters/index.js.map +0 -1
- package/dist/adapters/manual.d.ts +0 -9
- package/dist/adapters/manual.d.ts.map +0 -1
- package/dist/adapters/manual.js +0 -16
- package/dist/adapters/manual.js.map +0 -1
- package/dist/adapters/ollama.d.ts +0 -19
- package/dist/adapters/ollama.d.ts.map +0 -1
- package/dist/adapters/ollama.js +0 -30
- package/dist/adapters/ollama.js.map +0 -1
- package/dist/adapters/openai.d.ts +0 -34
- package/dist/adapters/openai.d.ts.map +0 -1
- package/dist/adapters/openai.js +0 -42
- package/dist/adapters/openai.js.map +0 -1
- package/dist/adapters/otel.d.ts +0 -25
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/adapters/otel.js +0 -133
- package/dist/adapters/otel.js.map +0 -1
- package/dist/agent.d.ts +0 -14
- package/dist/agent.d.ts.map +0 -1
- package/dist/agent.js +0 -54
- package/dist/agent.js.map +0 -1
- package/dist/client.d.ts +0 -19
- package/dist/client.d.ts.map +0 -1
- package/dist/client.js +0 -132
- package/dist/client.js.map +0 -1
- package/dist/context.d.ts +0 -4
- package/dist/context.d.ts.map +0 -1
- package/dist/context.js +0 -3
- package/dist/context.js.map +0 -1
- package/dist/delegate.d.ts +0 -3
- package/dist/delegate.d.ts.map +0 -1
- package/dist/delegate.js +0 -30
- package/dist/delegate.js.map +0 -1
- package/dist/engine-downloader.d.ts +0 -7
- package/dist/engine-downloader.d.ts.map +0 -1
- package/dist/engine-downloader.js +0 -136
- package/dist/engine-downloader.js.map +0 -1
- package/dist/engine-manager.d.ts +0 -26
- package/dist/engine-manager.d.ts.map +0 -1
- package/dist/engine-manager.js +0 -172
- package/dist/engine-manager.js.map +0 -1
- package/dist/expect.d.ts +0 -133
- package/dist/expect.d.ts.map +0 -1
- package/dist/expect.js +0 -330
- package/dist/expect.js.map +0 -1
- package/dist/index.d.ts +0 -15
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -14
- package/dist/index.js.map +0 -1
- package/dist/proto/codec.d.ts +0 -20
- package/dist/proto/codec.d.ts.map +0 -1
- package/dist/proto/codec.js +0 -71
- package/dist/proto/codec.js.map +0 -1
- package/dist/proto/constants.d.ts +0 -21
- package/dist/proto/constants.d.ts.map +0 -1
- package/dist/proto/constants.js +0 -25
- package/dist/proto/constants.js.map +0 -1
- package/dist/proto/errors.d.ts +0 -8
- package/dist/proto/errors.d.ts.map +0 -1
- package/dist/proto/errors.js +0 -13
- package/dist/proto/errors.js.map +0 -1
- package/dist/proto/index.d.ts +0 -5
- package/dist/proto/index.d.ts.map +0 -1
- package/dist/proto/index.js +0 -5
- package/dist/proto/index.js.map +0 -1
- package/dist/proto/types.d.ts +0 -104
- package/dist/proto/types.d.ts.map +0 -1
- package/dist/proto/types.js +0 -2
- package/dist/proto/types.js.map +0 -1
- package/dist/result.d.ts +0 -15
- package/dist/result.d.ts.map +0 -1
- package/dist/result.js +0 -32
- package/dist/result.js.map +0 -1
- package/dist/tier.d.ts +0 -5
- package/dist/tier.d.ts.map +0 -1
- package/dist/tier.js +0 -10
- package/dist/tier.js.map +0 -1
- package/dist/trace-tree.d.ts +0 -20
- package/dist/trace-tree.d.ts.map +0 -1
- package/dist/trace-tree.js +0 -92
- package/dist/trace-tree.js.map +0 -1
- package/dist/trace.d.ts +0 -40
- package/dist/trace.d.ts.map +0 -1
- package/dist/trace.js +0 -85
- package/dist/trace.js.map +0 -1
- package/dist/version.d.ts +0 -3
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js +0 -3
- package/dist/version.js.map +0 -1
|
@@ -0,0 +1,2490 @@
|
|
|
1
|
+
// src/proto/constants.ts
|
|
2
|
+
var STEP_LLM_CALL = "llm_call";
|
|
3
|
+
var STEP_TOOL_CALL = "tool_call";
|
|
4
|
+
var STEP_RETRIEVAL = "retrieval";
|
|
5
|
+
var STEP_AGENT_CALL = "agent_call";
|
|
6
|
+
var STATUS_PASS = "pass";
|
|
7
|
+
var STATUS_SOFT_FAIL = "soft_fail";
|
|
8
|
+
var STATUS_HARD_FAIL = "hard_fail";
|
|
9
|
+
var TYPE_SCHEMA = "schema";
|
|
10
|
+
var TYPE_CONSTRAINT = "constraint";
|
|
11
|
+
var TYPE_TRACE = "trace";
|
|
12
|
+
var TYPE_CONTENT = "content";
|
|
13
|
+
var TYPE_EMBEDDING = "embedding";
|
|
14
|
+
var TYPE_LLM_JUDGE = "llm_judge";
|
|
15
|
+
var TYPE_TRACE_TREE = "trace_tree";
|
|
16
|
+
var TYPE_PLUGIN = "plugin";
|
|
17
|
+
var ERR_INVALID_TRACE = 1001;
|
|
18
|
+
var ERR_ASSERTION_ERROR = 1002;
|
|
19
|
+
var ERR_PROVIDER_ERROR = 2001;
|
|
20
|
+
var ERR_ENGINE_ERROR = 3001;
|
|
21
|
+
var ERR_TIMEOUT = 3002;
|
|
22
|
+
var ERR_SESSION_ERROR = 3003;
|
|
23
|
+
|
|
24
|
+
// src/proto/types.ts
|
|
25
|
+
function traceId(id) {
|
|
26
|
+
return id;
|
|
27
|
+
}
|
|
28
|
+
function assertionId(id) {
|
|
29
|
+
return id;
|
|
30
|
+
}
|
|
31
|
+
function agentId(id) {
|
|
32
|
+
return id;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// src/proto/errors.ts
|
|
36
|
+
var ProtocolError = class extends Error {
|
|
37
|
+
code;
|
|
38
|
+
errorMessage;
|
|
39
|
+
data;
|
|
40
|
+
constructor(code, message, data) {
|
|
41
|
+
super(message);
|
|
42
|
+
this.name = "ProtocolError";
|
|
43
|
+
this.code = code;
|
|
44
|
+
this.errorMessage = message;
|
|
45
|
+
this.data = data;
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
var EngineTimeoutError = class extends Error {
|
|
49
|
+
method;
|
|
50
|
+
timeoutMs;
|
|
51
|
+
constructor(method, timeoutMs) {
|
|
52
|
+
super(
|
|
53
|
+
`Engine request '${method}' timed out after ${timeoutMs}ms. Set ATTEST_ENGINE_TIMEOUT to increase the limit.`
|
|
54
|
+
);
|
|
55
|
+
this.name = "EngineTimeoutError";
|
|
56
|
+
this.method = method;
|
|
57
|
+
this.timeoutMs = timeoutMs;
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
// src/proto/codec.ts
|
|
62
|
+
function encodeRequest(id, method, params) {
|
|
63
|
+
const msg = {
|
|
64
|
+
jsonrpc: "2.0",
|
|
65
|
+
id,
|
|
66
|
+
method,
|
|
67
|
+
params
|
|
68
|
+
};
|
|
69
|
+
return JSON.stringify(msg) + "\n";
|
|
70
|
+
}
|
|
71
|
+
function decodeResponse(line) {
|
|
72
|
+
const text = line.trim();
|
|
73
|
+
if (!text) {
|
|
74
|
+
throw new Error("empty response line");
|
|
75
|
+
}
|
|
76
|
+
let data;
|
|
77
|
+
try {
|
|
78
|
+
data = JSON.parse(text);
|
|
79
|
+
} catch (e) {
|
|
80
|
+
throw new Error(`malformed JSON response: ${e instanceof Error ? e.message : String(e)}`);
|
|
81
|
+
}
|
|
82
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) {
|
|
83
|
+
throw new Error(`expected JSON object, got ${Array.isArray(data) ? "array" : typeof data}`);
|
|
84
|
+
}
|
|
85
|
+
const obj = data;
|
|
86
|
+
if (obj.jsonrpc !== "2.0") {
|
|
87
|
+
throw new Error(`invalid jsonrpc version: ${String(obj.jsonrpc)}`);
|
|
88
|
+
}
|
|
89
|
+
if (obj.error != null) {
|
|
90
|
+
const err = obj.error;
|
|
91
|
+
let errorData;
|
|
92
|
+
if (err.data != null) {
|
|
93
|
+
const raw = err.data;
|
|
94
|
+
errorData = {
|
|
95
|
+
error_type: raw.error_type ?? "",
|
|
96
|
+
retryable: raw.retryable ?? false,
|
|
97
|
+
detail: raw.detail ?? ""
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
throw new ProtocolError(
|
|
101
|
+
err.code ?? -1,
|
|
102
|
+
err.message ?? "unknown error",
|
|
103
|
+
errorData
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
return obj;
|
|
107
|
+
}
|
|
108
|
+
function extractResult(response) {
|
|
109
|
+
if (response.result === void 0) {
|
|
110
|
+
throw new Error("response missing 'result' field");
|
|
111
|
+
}
|
|
112
|
+
return response.result;
|
|
113
|
+
}
|
|
114
|
+
function extractId(response) {
|
|
115
|
+
if (response.id === void 0) {
|
|
116
|
+
throw new Error("response missing 'id' field");
|
|
117
|
+
}
|
|
118
|
+
return Number(response.id);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// src/version.ts
|
|
122
|
+
var VERSION = "0.6.0";
|
|
123
|
+
var ENGINE_VERSION = "0.6.0";
|
|
124
|
+
|
|
125
|
+
// src/engine-manager.ts
|
|
126
|
+
import { spawn, execSync } from "child_process";
|
|
127
|
+
import { createInterface } from "readline";
|
|
128
|
+
import { existsSync as existsSync2, accessSync, constants as fsConstants } from "fs";
|
|
129
|
+
import { resolve, join as join2, dirname } from "path";
|
|
130
|
+
import { fileURLToPath } from "url";
|
|
131
|
+
|
|
132
|
+
// src/engine-downloader.ts
|
|
133
|
+
import * as https from "https";
|
|
134
|
+
import * as http from "http";
|
|
135
|
+
import * as fs from "fs";
|
|
136
|
+
import * as path from "path";
|
|
137
|
+
import * as os from "os";
|
|
138
|
+
import * as crypto from "crypto";
|
|
139
|
+
var RELEASE_BASE = `https://github.com/attest-framework/attest/releases/download/v${ENGINE_VERSION}`;
|
|
140
|
+
var CHECKSUMS_URL = `${RELEASE_BASE}/checksums-sha256.txt`;
|
|
141
|
+
var MAX_REDIRECTS = 5;
|
|
142
|
+
function attestBinDir() {
|
|
143
|
+
const dir = path.join(os.homedir(), ".attest", "bin");
|
|
144
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
145
|
+
return dir;
|
|
146
|
+
}
|
|
147
|
+
function platformKey() {
|
|
148
|
+
const platform2 = os.platform();
|
|
149
|
+
const arch2 = os.arch();
|
|
150
|
+
const mapping = {
|
|
151
|
+
darwin: { arm64: "darwin-arm64", x64: "darwin-amd64" },
|
|
152
|
+
linux: { arm64: "linux-arm64", x64: "linux-amd64" },
|
|
153
|
+
win32: { x64: "windows-amd64", arm64: "windows-arm64" }
|
|
154
|
+
};
|
|
155
|
+
const osMap = mapping[platform2];
|
|
156
|
+
if (!osMap) {
|
|
157
|
+
throw new Error(
|
|
158
|
+
`Unsupported platform: ${platform2}. Supported: darwin, linux, win32.`
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
const key = osMap[arch2];
|
|
162
|
+
if (!key) {
|
|
163
|
+
throw new Error(
|
|
164
|
+
`Unsupported architecture '${arch2}' on platform '${platform2}'. Supported architectures: ${Object.keys(osMap).join(", ")}.`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
return key;
|
|
168
|
+
}
|
|
169
|
+
function binaryFilename() {
|
|
170
|
+
return os.platform() === "win32" ? "attest-engine.exe" : "attest-engine";
|
|
171
|
+
}
|
|
172
|
+
function cachedEnginePath() {
|
|
173
|
+
const dir = path.join(os.homedir(), ".attest", "bin");
|
|
174
|
+
const binPath = path.join(dir, binaryFilename());
|
|
175
|
+
const versionFile = path.join(dir, ".engine-version");
|
|
176
|
+
if (!fs.existsSync(binPath)) {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
if (!fs.existsSync(versionFile)) {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
const cachedVersion = fs.readFileSync(versionFile, "utf-8").trim();
|
|
183
|
+
if (cachedVersion !== ENGINE_VERSION) {
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
return binPath;
|
|
187
|
+
}
|
|
188
|
+
function parseChecksums(text) {
|
|
189
|
+
const checksums = /* @__PURE__ */ new Map();
|
|
190
|
+
for (const line of text.split("\n")) {
|
|
191
|
+
const trimmed = line.trim();
|
|
192
|
+
if (!trimmed) continue;
|
|
193
|
+
const match = trimmed.match(/^([a-f0-9]{64})\s+(.+)$/);
|
|
194
|
+
if (match) {
|
|
195
|
+
checksums.set(match[2], match[1]);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return checksums;
|
|
199
|
+
}
|
|
200
|
+
function httpsGet(url, redirectCount = 0) {
|
|
201
|
+
if (redirectCount > MAX_REDIRECTS) {
|
|
202
|
+
return Promise.reject(
|
|
203
|
+
new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
return new Promise((resolve2, reject) => {
|
|
207
|
+
const client = url.startsWith("https:") ? https : http;
|
|
208
|
+
client.get(url, (res) => {
|
|
209
|
+
const status = res.statusCode ?? 0;
|
|
210
|
+
if (status === 301 || status === 302) {
|
|
211
|
+
const location = res.headers.location;
|
|
212
|
+
if (!location) {
|
|
213
|
+
reject(new Error(`Redirect ${status} without Location header from ${url}`));
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
res.resume();
|
|
217
|
+
resolve2(httpsGet(location, redirectCount + 1));
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
if (status < 200 || status >= 300) {
|
|
221
|
+
res.resume();
|
|
222
|
+
reject(new Error(`HTTP ${status} fetching ${url}`));
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
const chunks = [];
|
|
226
|
+
res.on("data", (chunk) => chunks.push(chunk));
|
|
227
|
+
res.on("end", () => resolve2(Buffer.concat(chunks)));
|
|
228
|
+
res.on("error", reject);
|
|
229
|
+
}).on("error", reject);
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
async function downloadEngine() {
|
|
233
|
+
const key = platformKey();
|
|
234
|
+
const filename = `attest-engine-${key}`;
|
|
235
|
+
const binaryUrl = `${RELEASE_BASE}/${filename}`;
|
|
236
|
+
process.stderr.write(`[attest] Downloading engine v${ENGINE_VERSION} for ${key}...
|
|
237
|
+
`);
|
|
238
|
+
process.stderr.write(`[attest] Fetching checksums...
|
|
239
|
+
`);
|
|
240
|
+
const checksumsText = (await httpsGet(CHECKSUMS_URL)).toString("utf-8");
|
|
241
|
+
const checksums = parseChecksums(checksumsText);
|
|
242
|
+
const expectedHash = checksums.get(filename);
|
|
243
|
+
if (!expectedHash) {
|
|
244
|
+
throw new Error(
|
|
245
|
+
`No checksum found for '${filename}' in checksums-sha256.txt. Available files: ${[...checksums.keys()].join(", ")}`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
process.stderr.write(`[attest] Downloading ${binaryUrl}...
|
|
249
|
+
`);
|
|
250
|
+
const data = await httpsGet(binaryUrl);
|
|
251
|
+
const actualHash = crypto.createHash("sha256").update(data).digest("hex");
|
|
252
|
+
if (actualHash !== expectedHash) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
`SHA256 mismatch for ${filename}. Expected: ${expectedHash}, got: ${actualHash}. The download may be corrupted. Delete ~/.attest/bin/ and retry.`
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
process.stderr.write(`[attest] Checksum verified.
|
|
258
|
+
`);
|
|
259
|
+
const dir = attestBinDir();
|
|
260
|
+
const destFilename = binaryFilename();
|
|
261
|
+
const destPath = path.join(dir, destFilename);
|
|
262
|
+
const tmpPath = path.join(dir, `.${destFilename}.tmp.${process.pid}`);
|
|
263
|
+
fs.writeFileSync(tmpPath, data);
|
|
264
|
+
fs.chmodSync(tmpPath, 493);
|
|
265
|
+
fs.renameSync(tmpPath, destPath);
|
|
266
|
+
fs.writeFileSync(path.join(dir, ".engine-version"), ENGINE_VERSION);
|
|
267
|
+
process.stderr.write(`[attest] Engine installed at ${destPath}
|
|
268
|
+
`);
|
|
269
|
+
return destPath;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// src/engine-manager.ts
|
|
273
|
+
var ENGINE_BINARY_NAME = "attest-engine";
|
|
274
|
+
async function findEngineBinary() {
|
|
275
|
+
const envPath = process.env["ATTEST_ENGINE_PATH"];
|
|
276
|
+
if (envPath) {
|
|
277
|
+
if (!existsSync2(envPath)) {
|
|
278
|
+
throw new Error(
|
|
279
|
+
`ATTEST_ENGINE_PATH is set to '${envPath}' but the file does not exist.`
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
return envPath;
|
|
283
|
+
}
|
|
284
|
+
try {
|
|
285
|
+
const cmd = process.platform === "win32" ? "where" : "which";
|
|
286
|
+
const found = execSync(`${cmd} ${ENGINE_BINARY_NAME}`, { encoding: "utf-8" }).trim();
|
|
287
|
+
if (found) return found;
|
|
288
|
+
} catch {
|
|
289
|
+
}
|
|
290
|
+
const cached = cachedEnginePath();
|
|
291
|
+
if (cached) return cached;
|
|
292
|
+
const candidates = [
|
|
293
|
+
join2(dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "..", "..", "bin", ENGINE_BINARY_NAME),
|
|
294
|
+
join2(process.cwd(), "bin", ENGINE_BINARY_NAME)
|
|
295
|
+
];
|
|
296
|
+
for (const candidate of candidates) {
|
|
297
|
+
const resolved = resolve(candidate);
|
|
298
|
+
try {
|
|
299
|
+
accessSync(resolved, fsConstants.X_OK);
|
|
300
|
+
if (existsSync2(resolved)) return resolved;
|
|
301
|
+
} catch {
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
const noDownload = process.env["ATTEST_ENGINE_NO_DOWNLOAD"];
|
|
305
|
+
if (noDownload === "1" || noDownload === "true" || noDownload === "yes") {
|
|
306
|
+
throw new Error(
|
|
307
|
+
`Cannot find '${ENGINE_BINARY_NAME}' binary and auto-download is disabled (ATTEST_ENGINE_NO_DOWNLOAD is set). Install the engine manually:
|
|
308
|
+
1. Download from https://github.com/attest-framework/attest/releases
|
|
309
|
+
2. Place the binary on your PATH or set ATTEST_ENGINE_PATH`
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
return downloadEngine();
|
|
313
|
+
}
|
|
314
|
+
var EngineManager = class {
|
|
315
|
+
enginePath;
|
|
316
|
+
logLevel;
|
|
317
|
+
process = null;
|
|
318
|
+
reader = null;
|
|
319
|
+
initialized = false;
|
|
320
|
+
requestId = 0;
|
|
321
|
+
initResult = null;
|
|
322
|
+
constructor(options) {
|
|
323
|
+
this.enginePath = options?.enginePath ?? "";
|
|
324
|
+
this.logLevel = options?.logLevel ?? "warn";
|
|
325
|
+
}
|
|
326
|
+
async start() {
|
|
327
|
+
if (!this.enginePath) {
|
|
328
|
+
this.enginePath = await findEngineBinary();
|
|
329
|
+
}
|
|
330
|
+
this.process = spawn(this.enginePath, [`--log-level=${this.logLevel}`], {
|
|
331
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
332
|
+
});
|
|
333
|
+
this.reader = createInterface({ input: this.process.stdout });
|
|
334
|
+
const result = await this.sendRequestInternal("initialize", {
|
|
335
|
+
sdk_name: "attest-typescript",
|
|
336
|
+
sdk_version: VERSION,
|
|
337
|
+
protocol_version: 1,
|
|
338
|
+
required_capabilities: ["layers_1_4"],
|
|
339
|
+
preferred_encoding: "json"
|
|
340
|
+
});
|
|
341
|
+
this.initResult = result;
|
|
342
|
+
if (!this.initResult.compatible) {
|
|
343
|
+
throw new Error(
|
|
344
|
+
`Engine incompatible. Missing capabilities: ${JSON.stringify(this.initResult.missing)}`
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
this.initialized = true;
|
|
348
|
+
return this.initResult;
|
|
349
|
+
}
|
|
350
|
+
async stop() {
|
|
351
|
+
if (this.process === null) return;
|
|
352
|
+
if (this.initialized) {
|
|
353
|
+
try {
|
|
354
|
+
await this.sendRequestInternal("shutdown", {});
|
|
355
|
+
} catch {
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
if (this.reader) {
|
|
359
|
+
this.reader.close();
|
|
360
|
+
this.reader = null;
|
|
361
|
+
}
|
|
362
|
+
if (this.process.exitCode === null) {
|
|
363
|
+
this.process.kill("SIGTERM");
|
|
364
|
+
await new Promise((res) => {
|
|
365
|
+
const timer = setTimeout(() => {
|
|
366
|
+
this.process?.kill("SIGKILL");
|
|
367
|
+
}, 5e3);
|
|
368
|
+
this.process.on("exit", () => {
|
|
369
|
+
clearTimeout(timer);
|
|
370
|
+
res();
|
|
371
|
+
});
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
this.initialized = false;
|
|
375
|
+
this.process = null;
|
|
376
|
+
}
|
|
377
|
+
async sendRequest(method, params) {
|
|
378
|
+
if (!this.initialized && method !== "initialize") {
|
|
379
|
+
throw new Error("Engine not initialized. Call start() first.");
|
|
380
|
+
}
|
|
381
|
+
return this.sendRequestInternal(method, params);
|
|
382
|
+
}
|
|
383
|
+
async sendRequestInternal(method, params) {
|
|
384
|
+
if (this.process === null || this.process.stdin === null || this.reader === null) {
|
|
385
|
+
throw new Error("Engine process not started.");
|
|
386
|
+
}
|
|
387
|
+
this.requestId += 1;
|
|
388
|
+
const requestStr = encodeRequest(this.requestId, method, params);
|
|
389
|
+
return new Promise((resolve2, reject) => {
|
|
390
|
+
const onLine = (line) => {
|
|
391
|
+
try {
|
|
392
|
+
const response = decodeResponse(line);
|
|
393
|
+
resolve2(extractResult(response));
|
|
394
|
+
} catch (err) {
|
|
395
|
+
reject(err);
|
|
396
|
+
}
|
|
397
|
+
};
|
|
398
|
+
this.reader.once("line", onLine);
|
|
399
|
+
const ok = this.process.stdin.write(requestStr, (err) => {
|
|
400
|
+
if (err) {
|
|
401
|
+
this.reader.removeListener("line", onLine);
|
|
402
|
+
reject(err);
|
|
403
|
+
}
|
|
404
|
+
});
|
|
405
|
+
if (!ok) {
|
|
406
|
+
this.process.stdin.once("drain", () => {
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
get isRunning() {
|
|
412
|
+
return this.process !== null && this.process.exitCode === null;
|
|
413
|
+
}
|
|
414
|
+
/** Expose process for AttestClient reader loop access */
|
|
415
|
+
get childProcess() {
|
|
416
|
+
return this.process;
|
|
417
|
+
}
|
|
418
|
+
/** Expose readline interface for AttestClient reader loop */
|
|
419
|
+
get readlineInterface() {
|
|
420
|
+
return this.reader;
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
// src/config.ts
|
|
425
|
+
var _simulationMode = false;
|
|
426
|
+
var _sampleRate = 0;
|
|
427
|
+
function config(options) {
|
|
428
|
+
if (options.simulation !== void 0) {
|
|
429
|
+
_simulationMode = options.simulation;
|
|
430
|
+
}
|
|
431
|
+
if (options.sampleRate !== void 0) {
|
|
432
|
+
if (options.sampleRate < 0 || options.sampleRate > 1) {
|
|
433
|
+
throw new RangeError(
|
|
434
|
+
`sampleRate must be between 0.0 and 1.0, got ${options.sampleRate}`
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
_sampleRate = options.sampleRate;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
function isSimulationMode() {
|
|
441
|
+
return _simulationMode || process.env["ATTEST_SIMULATION"] === "1" || process.env["ATTEST_SIMULATION"] === "true" || process.env["ATTEST_SIMULATION"] === "yes";
|
|
442
|
+
}
|
|
443
|
+
function getSampleRate() {
|
|
444
|
+
if (_sampleRate > 0) return _sampleRate;
|
|
445
|
+
const env = process.env["ATTEST_SAMPLE_RATE"];
|
|
446
|
+
if (env !== void 0 && env !== "") {
|
|
447
|
+
const parsed = Number(env);
|
|
448
|
+
if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
|
|
449
|
+
throw new RangeError(
|
|
450
|
+
`ATTEST_SAMPLE_RATE must be between 0.0 and 1.0, got '${env}'`
|
|
451
|
+
);
|
|
452
|
+
}
|
|
453
|
+
return parsed;
|
|
454
|
+
}
|
|
455
|
+
return 0;
|
|
456
|
+
}
|
|
457
|
+
function resetConfig() {
|
|
458
|
+
_simulationMode = false;
|
|
459
|
+
_sampleRate = 0;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// src/simulation.ts
|
|
463
|
+
function simulationEvaluateBatch(assertions) {
|
|
464
|
+
const results = assertions.map((assertion) => ({
|
|
465
|
+
assertion_id: assertion.assertion_id,
|
|
466
|
+
status: "pass",
|
|
467
|
+
score: 1,
|
|
468
|
+
explanation: `[simulation] assertion '${assertion.assertion_id}' passed (type: ${assertion.type})`,
|
|
469
|
+
cost: 0,
|
|
470
|
+
duration_ms: 0
|
|
471
|
+
}));
|
|
472
|
+
return {
|
|
473
|
+
results,
|
|
474
|
+
total_cost: 0,
|
|
475
|
+
total_duration_ms: 0
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// src/client.ts
|
|
480
|
+
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
481
|
+
function resolveTimeoutMs(override) {
|
|
482
|
+
if (override !== void 0) return override;
|
|
483
|
+
const env = process.env["ATTEST_ENGINE_TIMEOUT"];
|
|
484
|
+
if (env !== void 0 && env !== "") {
|
|
485
|
+
const parsed = Number(env);
|
|
486
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
487
|
+
throw new Error(
|
|
488
|
+
`ATTEST_ENGINE_TIMEOUT must be a positive number of milliseconds, got: '${env}'`
|
|
489
|
+
);
|
|
490
|
+
}
|
|
491
|
+
return parsed;
|
|
492
|
+
}
|
|
493
|
+
return DEFAULT_TIMEOUT_MS;
|
|
494
|
+
}
|
|
495
|
+
function makeTimeoutPromise(method, timeoutMs) {
|
|
496
|
+
return new Promise((_, reject) => {
|
|
497
|
+
setTimeout(() => {
|
|
498
|
+
reject(new EngineTimeoutError(method, timeoutMs));
|
|
499
|
+
}, timeoutMs);
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
var AttestClient = class {
|
|
503
|
+
engine;
|
|
504
|
+
requestId = 0;
|
|
505
|
+
pending = /* @__PURE__ */ new Map();
|
|
506
|
+
writeChain = Promise.resolve();
|
|
507
|
+
readerActive = false;
|
|
508
|
+
lineHandler = null;
|
|
509
|
+
constructor(engine) {
|
|
510
|
+
this.engine = engine;
|
|
511
|
+
}
|
|
512
|
+
startReader() {
|
|
513
|
+
if (this.readerActive) return;
|
|
514
|
+
const rl = this.engine.readlineInterface;
|
|
515
|
+
if (rl === null) {
|
|
516
|
+
throw new Error("Engine readline interface not available. Call engine.start() first.");
|
|
517
|
+
}
|
|
518
|
+
this.readerActive = true;
|
|
519
|
+
this.lineHandler = (line) => this.handleLine(line);
|
|
520
|
+
rl.on("line", this.lineHandler);
|
|
521
|
+
rl.on("close", () => {
|
|
522
|
+
this.failAll(new Error("Engine closed stdout."));
|
|
523
|
+
this.readerActive = false;
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
stopReader() {
|
|
527
|
+
if (!this.readerActive) return;
|
|
528
|
+
const rl = this.engine.readlineInterface;
|
|
529
|
+
if (rl !== null && this.lineHandler !== null) {
|
|
530
|
+
rl.removeListener("line", this.lineHandler);
|
|
531
|
+
}
|
|
532
|
+
this.lineHandler = null;
|
|
533
|
+
this.readerActive = false;
|
|
534
|
+
}
|
|
535
|
+
handleLine(line) {
|
|
536
|
+
let reqId;
|
|
537
|
+
try {
|
|
538
|
+
const response = decodeResponse(line);
|
|
539
|
+
reqId = extractId(response);
|
|
540
|
+
const pending = this.pending.get(reqId);
|
|
541
|
+
if (pending === void 0) return;
|
|
542
|
+
this.pending.delete(reqId);
|
|
543
|
+
try {
|
|
544
|
+
const result = extractResult(response);
|
|
545
|
+
pending.resolve(result);
|
|
546
|
+
} catch (err) {
|
|
547
|
+
pending.reject(err);
|
|
548
|
+
}
|
|
549
|
+
} catch (err) {
|
|
550
|
+
if (err instanceof ProtocolError) {
|
|
551
|
+
try {
|
|
552
|
+
const raw = JSON.parse(line.trim());
|
|
553
|
+
reqId = Number(raw.id ?? -1);
|
|
554
|
+
} catch {
|
|
555
|
+
reqId = -1;
|
|
556
|
+
}
|
|
557
|
+
const pending = this.pending.get(reqId);
|
|
558
|
+
if (pending !== void 0) {
|
|
559
|
+
this.pending.delete(reqId);
|
|
560
|
+
pending.reject(err);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
failAll(err) {
|
|
566
|
+
for (const pending of this.pending.values()) {
|
|
567
|
+
pending.reject(err);
|
|
568
|
+
}
|
|
569
|
+
this.pending.clear();
|
|
570
|
+
}
|
|
571
|
+
async sendRequest(method, params, timeoutMs) {
|
|
572
|
+
const resolvedTimeout = resolveTimeoutMs(timeoutMs);
|
|
573
|
+
if (!this.readerActive) {
|
|
574
|
+
return Promise.race([
|
|
575
|
+
this.engine.sendRequest(method, params),
|
|
576
|
+
makeTimeoutPromise(method, resolvedTimeout)
|
|
577
|
+
]);
|
|
578
|
+
}
|
|
579
|
+
const cp = this.engine.childProcess;
|
|
580
|
+
if (cp === null || cp.stdin === null) {
|
|
581
|
+
throw new Error("Engine process not started.");
|
|
582
|
+
}
|
|
583
|
+
let capturedReqId;
|
|
584
|
+
const requestPromise = new Promise((resolve2, reject) => {
|
|
585
|
+
this.writeChain = this.writeChain.then(() => {
|
|
586
|
+
this.requestId += 1;
|
|
587
|
+
const reqId = this.requestId;
|
|
588
|
+
capturedReqId = reqId;
|
|
589
|
+
this.pending.set(reqId, { resolve: resolve2, reject });
|
|
590
|
+
const requestStr = encodeRequest(reqId, method, params);
|
|
591
|
+
cp.stdin.write(requestStr, (err) => {
|
|
592
|
+
if (err) {
|
|
593
|
+
this.pending.delete(reqId);
|
|
594
|
+
reject(err);
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
});
|
|
598
|
+
});
|
|
599
|
+
const timeoutPromise = makeTimeoutPromise(method, resolvedTimeout);
|
|
600
|
+
return Promise.race([requestPromise, timeoutPromise]).catch((err) => {
|
|
601
|
+
if (err instanceof EngineTimeoutError && capturedReqId !== void 0) {
|
|
602
|
+
this.pending.delete(capturedReqId);
|
|
603
|
+
}
|
|
604
|
+
throw err;
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
async evaluateBatch(trace, assertions, options) {
|
|
608
|
+
if (isSimulationMode()) {
|
|
609
|
+
return simulationEvaluateBatch(assertions);
|
|
610
|
+
}
|
|
611
|
+
const params = {
|
|
612
|
+
trace,
|
|
613
|
+
assertions: [...assertions]
|
|
614
|
+
};
|
|
615
|
+
const raw = await this.sendRequest(
|
|
616
|
+
"evaluate_batch",
|
|
617
|
+
params,
|
|
618
|
+
options?.timeout
|
|
619
|
+
);
|
|
620
|
+
return raw;
|
|
621
|
+
}
|
|
622
|
+
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options) {
|
|
623
|
+
const result = {
|
|
624
|
+
assertion_id: assertionId2,
|
|
625
|
+
status,
|
|
626
|
+
score,
|
|
627
|
+
explanation,
|
|
628
|
+
cost: 0,
|
|
629
|
+
duration_ms: 0
|
|
630
|
+
};
|
|
631
|
+
const params = {
|
|
632
|
+
trace_id: traceId2,
|
|
633
|
+
plugin_name: pluginName,
|
|
634
|
+
assertion_id: assertionId2,
|
|
635
|
+
result
|
|
636
|
+
};
|
|
637
|
+
const raw = await this.sendRequest(
|
|
638
|
+
"submit_plugin_result",
|
|
639
|
+
params,
|
|
640
|
+
options?.timeout
|
|
641
|
+
);
|
|
642
|
+
return Boolean(raw?.accepted ?? false);
|
|
643
|
+
}
|
|
644
|
+
async queryDrift(assertionId2, windowSize = 50, options) {
|
|
645
|
+
if (isSimulationMode()) {
|
|
646
|
+
return {
|
|
647
|
+
assertion_id: assertionId2,
|
|
648
|
+
mean: 1,
|
|
649
|
+
stddev: 0,
|
|
650
|
+
count: 0,
|
|
651
|
+
latest_score: 1,
|
|
652
|
+
deviation: 0,
|
|
653
|
+
status: "no_data"
|
|
654
|
+
};
|
|
655
|
+
}
|
|
656
|
+
const raw = await this.sendRequest(
|
|
657
|
+
"query_drift",
|
|
658
|
+
{ assertion_id: assertionId2, window_size: windowSize },
|
|
659
|
+
options?.timeout
|
|
660
|
+
);
|
|
661
|
+
return raw.report;
|
|
662
|
+
}
|
|
663
|
+
async generateUserMessage(persona, conversationHistory, faultConfig, options) {
|
|
664
|
+
if (isSimulationMode()) {
|
|
665
|
+
return `[simulation] Hello from ${persona.name}`;
|
|
666
|
+
}
|
|
667
|
+
const params = {
|
|
668
|
+
persona,
|
|
669
|
+
conversation_history: conversationHistory
|
|
670
|
+
};
|
|
671
|
+
if (faultConfig !== void 0) {
|
|
672
|
+
params.fault_config = faultConfig;
|
|
673
|
+
}
|
|
674
|
+
const raw = await this.sendRequest(
|
|
675
|
+
"generate_user_message",
|
|
676
|
+
params,
|
|
677
|
+
options?.timeout
|
|
678
|
+
);
|
|
679
|
+
return String(raw.message);
|
|
680
|
+
}
|
|
681
|
+
};
|
|
682
|
+
|
|
683
|
+
// src/trace.ts
|
|
684
|
+
import { randomUUID } from "crypto";
|
|
685
|
+
var TraceBuilder = class {
|
|
686
|
+
traceId;
|
|
687
|
+
agentId;
|
|
688
|
+
input;
|
|
689
|
+
steps = [];
|
|
690
|
+
output;
|
|
691
|
+
metadata;
|
|
692
|
+
parentTraceId;
|
|
693
|
+
constructor(agentId2) {
|
|
694
|
+
this.traceId = `trc_${randomUUID().replace(/-/g, "").slice(0, 12)}`;
|
|
695
|
+
this.agentId = agentId2;
|
|
696
|
+
}
|
|
697
|
+
setTraceId(traceId2) {
|
|
698
|
+
this.traceId = traceId2;
|
|
699
|
+
return this;
|
|
700
|
+
}
|
|
701
|
+
setInput(input) {
|
|
702
|
+
this.input = input;
|
|
703
|
+
return this;
|
|
704
|
+
}
|
|
705
|
+
addLlmCall(name, options) {
|
|
706
|
+
this.steps.push({
|
|
707
|
+
type: STEP_LLM_CALL,
|
|
708
|
+
name,
|
|
709
|
+
args: options?.args,
|
|
710
|
+
result: options?.result,
|
|
711
|
+
metadata: options?.metadata
|
|
712
|
+
});
|
|
713
|
+
return this;
|
|
714
|
+
}
|
|
715
|
+
addToolCall(name, options) {
|
|
716
|
+
this.steps.push({
|
|
717
|
+
type: STEP_TOOL_CALL,
|
|
718
|
+
name,
|
|
719
|
+
args: options?.args,
|
|
720
|
+
result: options?.result,
|
|
721
|
+
metadata: options?.metadata
|
|
722
|
+
});
|
|
723
|
+
return this;
|
|
724
|
+
}
|
|
725
|
+
addRetrieval(name, options) {
|
|
726
|
+
this.steps.push({
|
|
727
|
+
type: STEP_RETRIEVAL,
|
|
728
|
+
name,
|
|
729
|
+
args: options?.args,
|
|
730
|
+
result: options?.result,
|
|
731
|
+
metadata: options?.metadata
|
|
732
|
+
});
|
|
733
|
+
return this;
|
|
734
|
+
}
|
|
735
|
+
addStep(step) {
|
|
736
|
+
this.steps.push(step);
|
|
737
|
+
return this;
|
|
738
|
+
}
|
|
739
|
+
setOutput(output) {
|
|
740
|
+
this.output = output;
|
|
741
|
+
return this;
|
|
742
|
+
}
|
|
743
|
+
setMetadata(metadata) {
|
|
744
|
+
this.metadata = metadata;
|
|
745
|
+
return this;
|
|
746
|
+
}
|
|
747
|
+
getTraceId() {
|
|
748
|
+
return this.traceId;
|
|
749
|
+
}
|
|
750
|
+
setParentTraceId(parentId) {
|
|
751
|
+
this.parentTraceId = parentId;
|
|
752
|
+
return this;
|
|
753
|
+
}
|
|
754
|
+
build() {
|
|
755
|
+
if (this.output === void 0) {
|
|
756
|
+
throw new Error("Trace output is required. Call setOutput() before build().");
|
|
757
|
+
}
|
|
758
|
+
return {
|
|
759
|
+
trace_id: this.traceId,
|
|
760
|
+
output: this.output,
|
|
761
|
+
schema_version: 1,
|
|
762
|
+
agent_id: this.agentId,
|
|
763
|
+
input: this.input,
|
|
764
|
+
steps: [...this.steps],
|
|
765
|
+
metadata: this.metadata,
|
|
766
|
+
parent_trace_id: this.parentTraceId
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
};
|
|
770
|
+
|
|
771
|
+
// src/result.ts
|
|
772
|
+
var AgentResult = class {
|
|
773
|
+
trace;
|
|
774
|
+
assertionResults;
|
|
775
|
+
totalCost;
|
|
776
|
+
totalDurationMs;
|
|
777
|
+
constructor(trace, assertionResults = [], totalCost = 0, totalDurationMs = 0) {
|
|
778
|
+
this.trace = trace;
|
|
779
|
+
this.assertionResults = assertionResults;
|
|
780
|
+
this.totalCost = totalCost;
|
|
781
|
+
this.totalDurationMs = totalDurationMs;
|
|
782
|
+
}
|
|
783
|
+
get passed() {
|
|
784
|
+
return this.assertionResults.every((r) => r.status === STATUS_PASS);
|
|
785
|
+
}
|
|
786
|
+
get failedAssertions() {
|
|
787
|
+
return this.assertionResults.filter((r) => r.status !== STATUS_PASS);
|
|
788
|
+
}
|
|
789
|
+
get hardFailures() {
|
|
790
|
+
return this.assertionResults.filter((r) => r.status === STATUS_HARD_FAIL);
|
|
791
|
+
}
|
|
792
|
+
get softFailures() {
|
|
793
|
+
return this.assertionResults.filter((r) => r.status === STATUS_SOFT_FAIL);
|
|
794
|
+
}
|
|
795
|
+
get passCount() {
|
|
796
|
+
return this.assertionResults.filter((r) => r.status === STATUS_PASS).length;
|
|
797
|
+
}
|
|
798
|
+
get failCount() {
|
|
799
|
+
return this.assertionResults.length - this.passCount;
|
|
800
|
+
}
|
|
801
|
+
};
|
|
802
|
+
|
|
803
|
+
// src/trace-tree.ts
|
|
804
|
+
var TraceTree = class {
|
|
805
|
+
root;
|
|
806
|
+
constructor(root) {
|
|
807
|
+
this.root = root;
|
|
808
|
+
}
|
|
809
|
+
get agents() {
|
|
810
|
+
const result = [];
|
|
811
|
+
this.collectAgents(this.root, result);
|
|
812
|
+
return result;
|
|
813
|
+
}
|
|
814
|
+
collectAgents(trace, acc) {
|
|
815
|
+
if (trace.agent_id != null) {
|
|
816
|
+
acc.push(trace.agent_id);
|
|
817
|
+
}
|
|
818
|
+
for (const step of trace.steps) {
|
|
819
|
+
if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
|
|
820
|
+
this.collectAgents(step.sub_trace, acc);
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
findAgent(agentId2) {
|
|
825
|
+
return this.findAgentIn(this.root, agentId2);
|
|
826
|
+
}
|
|
827
|
+
findAgentIn(trace, agentId2) {
|
|
828
|
+
if (trace.agent_id === agentId2) return trace;
|
|
829
|
+
for (const step of trace.steps) {
|
|
830
|
+
if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
|
|
831
|
+
const found = this.findAgentIn(step.sub_trace, agentId2);
|
|
832
|
+
if (found !== void 0) return found;
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
return void 0;
|
|
836
|
+
}
|
|
837
|
+
get depth() {
|
|
838
|
+
return this.computeDepth(this.root);
|
|
839
|
+
}
|
|
840
|
+
computeDepth(trace) {
|
|
841
|
+
let maxChild = -1;
|
|
842
|
+
for (const step of trace.steps) {
|
|
843
|
+
if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
|
|
844
|
+
const childDepth = this.computeDepth(step.sub_trace);
|
|
845
|
+
if (childDepth > maxChild) maxChild = childDepth;
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
return maxChild >= 0 ? maxChild + 1 : 0;
|
|
849
|
+
}
|
|
850
|
+
flatten() {
|
|
851
|
+
const result = [];
|
|
852
|
+
this.flattenInto(this.root, result);
|
|
853
|
+
return result;
|
|
854
|
+
}
|
|
855
|
+
flattenInto(trace, acc) {
|
|
856
|
+
acc.push(trace);
|
|
857
|
+
for (const step of trace.steps) {
|
|
858
|
+
if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
|
|
859
|
+
this.flattenInto(step.sub_trace, acc);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
get delegations() {
|
|
864
|
+
const result = [];
|
|
865
|
+
this.collectDelegations(this.root, result);
|
|
866
|
+
return result;
|
|
867
|
+
}
|
|
868
|
+
collectDelegations(trace, acc) {
|
|
869
|
+
const parentId = trace.agent_id ?? "";
|
|
870
|
+
for (const step of trace.steps) {
|
|
871
|
+
if (step.type === STEP_AGENT_CALL && step.sub_trace != null) {
|
|
872
|
+
const childId = step.sub_trace.agent_id ?? "";
|
|
873
|
+
acc.push([parentId, childId]);
|
|
874
|
+
this.collectDelegations(step.sub_trace, acc);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
allToolCalls() {
|
|
879
|
+
return this.flatten().flatMap(
|
|
880
|
+
(t) => t.steps.filter((step) => step.type === STEP_TOOL_CALL)
|
|
881
|
+
);
|
|
882
|
+
}
|
|
883
|
+
get aggregateTokens() {
|
|
884
|
+
return this.flatten().reduce(
|
|
885
|
+
(sum, t) => sum + (t.metadata?.total_tokens ?? 0),
|
|
886
|
+
0
|
|
887
|
+
);
|
|
888
|
+
}
|
|
889
|
+
get aggregateCost() {
|
|
890
|
+
return this.flatten().reduce(
|
|
891
|
+
(sum, t) => sum + (t.metadata?.cost_usd ?? 0),
|
|
892
|
+
0
|
|
893
|
+
);
|
|
894
|
+
}
|
|
895
|
+
get aggregateLatency() {
|
|
896
|
+
return this.flatten().reduce(
|
|
897
|
+
(sum, t) => sum + (t.metadata?.latency_ms ?? 0),
|
|
898
|
+
0
|
|
899
|
+
);
|
|
900
|
+
}
|
|
901
|
+
};
|
|
902
|
+
|
|
903
|
+
// src/expect.ts
|
|
904
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
905
|
+
var ExpectChain = class {
|
|
906
|
+
_result;
|
|
907
|
+
_assertions = [];
|
|
908
|
+
constructor(result) {
|
|
909
|
+
this._result = result;
|
|
910
|
+
}
|
|
911
|
+
get assertions() {
|
|
912
|
+
return [...this._assertions];
|
|
913
|
+
}
|
|
914
|
+
get trace() {
|
|
915
|
+
return this._result.trace;
|
|
916
|
+
}
|
|
917
|
+
add(assertionType, spec) {
|
|
918
|
+
this._assertions.push({
|
|
919
|
+
assertion_id: `assert_${randomUUID2().replace(/-/g, "").slice(0, 8)}`,
|
|
920
|
+
type: assertionType,
|
|
921
|
+
spec
|
|
922
|
+
});
|
|
923
|
+
return this;
|
|
924
|
+
}
|
|
925
|
+
// -- Layer 1: Schema --
|
|
926
|
+
/**
|
|
927
|
+
* Assert that `output.structured` conforms to the given JSON Schema.
|
|
928
|
+
*
|
|
929
|
+
* @param schema - A JSON Schema object to validate against.
|
|
930
|
+
* @returns The chain for fluent composition.
|
|
931
|
+
*
|
|
932
|
+
* @example
|
|
933
|
+
* ```ts
|
|
934
|
+
* expect(result).outputMatchesSchema({ type: "object", required: ["answer"] });
|
|
935
|
+
* ```
|
|
936
|
+
*/
|
|
937
|
+
outputMatchesSchema(schema) {
|
|
938
|
+
return this.add(TYPE_SCHEMA, { target: "output.structured", schema });
|
|
939
|
+
}
|
|
940
|
+
/**
|
|
941
|
+
* Assert that a specific output field conforms to the given JSON Schema.
|
|
942
|
+
*
|
|
943
|
+
* @param field - Dot-delimited path under `output` (e.g. `"structured.items"`).
|
|
944
|
+
* @param schema - A JSON Schema object to validate against.
|
|
945
|
+
* @returns The chain for fluent composition.
|
|
946
|
+
*
|
|
947
|
+
* @example
|
|
948
|
+
* ```ts
|
|
949
|
+
* expect(result).outputFieldMatchesSchema("structured.items", { type: "array" });
|
|
950
|
+
* ```
|
|
951
|
+
*/
|
|
952
|
+
outputFieldMatchesSchema(field, schema) {
|
|
953
|
+
return this.add(TYPE_SCHEMA, { target: `output.${field}`, schema });
|
|
954
|
+
}
|
|
955
|
+
/**
|
|
956
|
+
* Assert that a named tool's arguments conform to the given JSON Schema.
|
|
957
|
+
*
|
|
958
|
+
* @param toolName - The tool name to match in `steps`.
|
|
959
|
+
* @param schema - A JSON Schema object to validate against.
|
|
960
|
+
* @returns The chain for fluent composition.
|
|
961
|
+
*
|
|
962
|
+
* @example
|
|
963
|
+
* ```ts
|
|
964
|
+
* expect(result).toolArgsMatchSchema("search", { type: "object", required: ["query"] });
|
|
965
|
+
* ```
|
|
966
|
+
*/
|
|
967
|
+
toolArgsMatchSchema(toolName, schema) {
|
|
968
|
+
return this.add(TYPE_SCHEMA, {
|
|
969
|
+
target: `steps[?name=='${toolName}'].args`,
|
|
970
|
+
schema
|
|
971
|
+
});
|
|
972
|
+
}
|
|
973
|
+
/**
|
|
974
|
+
* Assert that a named tool's result conforms to the given JSON Schema.
|
|
975
|
+
*
|
|
976
|
+
* @param toolName - The tool name to match in `steps`.
|
|
977
|
+
* @param schema - A JSON Schema object to validate against.
|
|
978
|
+
* @returns The chain for fluent composition.
|
|
979
|
+
*
|
|
980
|
+
* @example
|
|
981
|
+
* ```ts
|
|
982
|
+
* expect(result).toolResultMatchesSchema("search", { type: "object" });
|
|
983
|
+
* ```
|
|
984
|
+
*/
|
|
985
|
+
toolResultMatchesSchema(toolName, schema) {
|
|
986
|
+
return this.add(TYPE_SCHEMA, {
|
|
987
|
+
target: `steps[?name=='${toolName}'].result`,
|
|
988
|
+
schema
|
|
989
|
+
});
|
|
990
|
+
}
|
|
991
|
+
// -- Layer 2: Constraints --
|
|
992
|
+
/**
|
|
993
|
+
* Assert total cost (in USD) is at or below `maxCost`.
|
|
994
|
+
*
|
|
995
|
+
* @param maxCost - Maximum allowed cost in USD.
|
|
996
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
997
|
+
* @returns The chain for fluent composition.
|
|
998
|
+
*
|
|
999
|
+
* @example
|
|
1000
|
+
* ```ts
|
|
1001
|
+
* expect(result).costUnder(0.05);
|
|
1002
|
+
* expect(result).costUnder(0.05, { soft: true });
|
|
1003
|
+
* ```
|
|
1004
|
+
*/
|
|
1005
|
+
costUnder(maxCost, opts) {
|
|
1006
|
+
return this.add(TYPE_CONSTRAINT, {
|
|
1007
|
+
field: "metadata.cost_usd",
|
|
1008
|
+
operator: "lte",
|
|
1009
|
+
value: maxCost,
|
|
1010
|
+
soft: opts?.soft ?? false
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Assert total latency is at or below `maxMs` milliseconds.
|
|
1015
|
+
*
|
|
1016
|
+
* @param maxMs - Maximum allowed latency in milliseconds.
|
|
1017
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1018
|
+
* @returns The chain for fluent composition.
|
|
1019
|
+
*
|
|
1020
|
+
* @example
|
|
1021
|
+
* ```ts
|
|
1022
|
+
* expect(result).latencyUnder(2000);
|
|
1023
|
+
* ```
|
|
1024
|
+
*/
|
|
1025
|
+
latencyUnder(maxMs, opts) {
|
|
1026
|
+
return this.add(TYPE_CONSTRAINT, {
|
|
1027
|
+
field: "metadata.latency_ms",
|
|
1028
|
+
operator: "lte",
|
|
1029
|
+
value: maxMs,
|
|
1030
|
+
soft: opts?.soft ?? false
|
|
1031
|
+
});
|
|
1032
|
+
}
|
|
1033
|
+
/**
|
|
1034
|
+
* Assert total token usage is at or below `maxTokens`.
|
|
1035
|
+
*
|
|
1036
|
+
* @param maxTokens - Maximum allowed token count.
|
|
1037
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1038
|
+
* @returns The chain for fluent composition.
|
|
1039
|
+
*
|
|
1040
|
+
* @example
|
|
1041
|
+
* ```ts
|
|
1042
|
+
* expect(result).tokensUnder(500);
|
|
1043
|
+
* ```
|
|
1044
|
+
*/
|
|
1045
|
+
tokensUnder(maxTokens, opts) {
|
|
1046
|
+
return this.add(TYPE_CONSTRAINT, {
|
|
1047
|
+
field: "metadata.total_tokens",
|
|
1048
|
+
operator: "lte",
|
|
1049
|
+
value: maxTokens,
|
|
1050
|
+
soft: opts?.soft ?? false
|
|
1051
|
+
});
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Assert total token usage is between `minTokens` and `maxTokens` (inclusive).
|
|
1055
|
+
*
|
|
1056
|
+
* @param minTokens - Minimum allowed token count.
|
|
1057
|
+
* @param maxTokens - Maximum allowed token count.
|
|
1058
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1059
|
+
* @returns The chain for fluent composition.
|
|
1060
|
+
*
|
|
1061
|
+
* @example
|
|
1062
|
+
* ```ts
|
|
1063
|
+
* expect(result).tokensBetween(100, 500);
|
|
1064
|
+
* ```
|
|
1065
|
+
*/
|
|
1066
|
+
tokensBetween(minTokens, maxTokens, opts) {
|
|
1067
|
+
return this.add(TYPE_CONSTRAINT, {
|
|
1068
|
+
field: "metadata.total_tokens",
|
|
1069
|
+
operator: "between",
|
|
1070
|
+
min: minTokens,
|
|
1071
|
+
max: maxTokens,
|
|
1072
|
+
soft: opts?.soft ?? false
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
/**
|
|
1076
|
+
* Assert the number of steps satisfies the given comparison.
|
|
1077
|
+
*
|
|
1078
|
+
* @param operator - Comparison operator (e.g. `"eq"`, `"lte"`, `"gte"`).
|
|
1079
|
+
* @param value - The reference count to compare against.
|
|
1080
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1081
|
+
* @returns The chain for fluent composition.
|
|
1082
|
+
*
|
|
1083
|
+
* @example
|
|
1084
|
+
* ```ts
|
|
1085
|
+
* expect(result).stepCount("lte", 5);
|
|
1086
|
+
* ```
|
|
1087
|
+
*/
|
|
1088
|
+
stepCount(operator, value, opts) {
|
|
1089
|
+
return this.add(TYPE_CONSTRAINT, {
|
|
1090
|
+
field: "steps.length",
|
|
1091
|
+
operator,
|
|
1092
|
+
value,
|
|
1093
|
+
soft: opts?.soft ?? false
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
/**
|
|
1097
|
+
* Assert the number of tool-call steps satisfies the given comparison.
|
|
1098
|
+
*
|
|
1099
|
+
* @param operator - Comparison operator (e.g. `"eq"`, `"lte"`, `"gte"`).
|
|
1100
|
+
* @param value - The reference count to compare against.
|
|
1101
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1102
|
+
* @returns The chain for fluent composition.
|
|
1103
|
+
*
|
|
1104
|
+
* @example
|
|
1105
|
+
* ```ts
|
|
1106
|
+
* expect(result).toolCallCount("gte", 1);
|
|
1107
|
+
* ```
|
|
1108
|
+
*/
|
|
1109
|
+
toolCallCount(operator, value, opts) {
|
|
1110
|
+
return this.add(TYPE_CONSTRAINT, {
|
|
1111
|
+
field: "steps[?type=='tool_call'].length",
|
|
1112
|
+
operator,
|
|
1113
|
+
value,
|
|
1114
|
+
soft: opts?.soft ?? false
|
|
1115
|
+
});
|
|
1116
|
+
}
|
|
1117
|
+
/**
|
|
1118
|
+
* Generic constraint assertion on any field.
|
|
1119
|
+
*
|
|
1120
|
+
* @param field - Dot-delimited path into the trace (e.g. `"metadata.cost_usd"`).
|
|
1121
|
+
* @param operator - Comparison operator.
|
|
1122
|
+
* @param opts.value - Reference value for single-operand operators.
|
|
1123
|
+
* @param opts.min - Minimum for `"between"` operator.
|
|
1124
|
+
* @param opts.max - Maximum for `"between"` operator.
|
|
1125
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1126
|
+
* @returns The chain for fluent composition.
|
|
1127
|
+
*
|
|
1128
|
+
* @example
|
|
1129
|
+
* ```ts
|
|
1130
|
+
* expect(result).constraint("metadata.cost_usd", "lte", { value: 0.1 });
|
|
1131
|
+
* ```
|
|
1132
|
+
*/
|
|
1133
|
+
constraint(field, operator, opts) {
|
|
1134
|
+
const spec = {
|
|
1135
|
+
field,
|
|
1136
|
+
operator,
|
|
1137
|
+
soft: opts?.soft ?? false
|
|
1138
|
+
};
|
|
1139
|
+
if (opts?.value !== void 0) spec.value = opts.value;
|
|
1140
|
+
if (opts?.min !== void 0) spec.min = opts.min;
|
|
1141
|
+
if (opts?.max !== void 0) spec.max = opts.max;
|
|
1142
|
+
return this.add(TYPE_CONSTRAINT, spec);
|
|
1143
|
+
}
|
|
1144
|
+
// -- Layer 3: Trace --
|
|
1145
|
+
/**
|
|
1146
|
+
* Assert that tools were called in the given order (non-contiguous allowed).
|
|
1147
|
+
*
|
|
1148
|
+
* @param tools - Ordered list of tool names expected.
|
|
1149
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1150
|
+
* @returns The chain for fluent composition.
|
|
1151
|
+
*
|
|
1152
|
+
* @example
|
|
1153
|
+
* ```ts
|
|
1154
|
+
* expect(result).toolsCalledInOrder(["search", "write"]);
|
|
1155
|
+
* ```
|
|
1156
|
+
*/
|
|
1157
|
+
toolsCalledInOrder(tools, opts) {
|
|
1158
|
+
return this.add(TYPE_TRACE, {
|
|
1159
|
+
check: "contains_in_order",
|
|
1160
|
+
tools,
|
|
1161
|
+
soft: opts?.soft ?? false
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Assert that tools were called in exact contiguous order.
|
|
1166
|
+
*
|
|
1167
|
+
* @param tools - Exact ordered list of tool names.
|
|
1168
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1169
|
+
* @returns The chain for fluent composition.
|
|
1170
|
+
*
|
|
1171
|
+
* @example
|
|
1172
|
+
* ```ts
|
|
1173
|
+
* expect(result).toolsCalledExactly(["search", "summarize", "respond"]);
|
|
1174
|
+
* ```
|
|
1175
|
+
*/
|
|
1176
|
+
toolsCalledExactly(tools, opts) {
|
|
1177
|
+
return this.add(TYPE_TRACE, {
|
|
1178
|
+
check: "exact_order",
|
|
1179
|
+
tools,
|
|
1180
|
+
soft: opts?.soft ?? false
|
|
1181
|
+
});
|
|
1182
|
+
}
|
|
1183
|
+
/**
|
|
1184
|
+
* Assert a tool is not called more than `maxRepetitions` times consecutively.
|
|
1185
|
+
*
|
|
1186
|
+
* @param tool - The tool name to check.
|
|
1187
|
+
* @param maxRepetitions - Maximum allowed consecutive calls (default `1`).
|
|
1188
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1189
|
+
* @returns The chain for fluent composition.
|
|
1190
|
+
*
|
|
1191
|
+
* @example
|
|
1192
|
+
* ```ts
|
|
1193
|
+
* expect(result).noToolLoops("fetch", 2);
|
|
1194
|
+
* ```
|
|
1195
|
+
*/
|
|
1196
|
+
noToolLoops(tool, maxRepetitions = 1, opts) {
|
|
1197
|
+
return this.add(TYPE_TRACE, {
|
|
1198
|
+
check: "loop_detection",
|
|
1199
|
+
tool,
|
|
1200
|
+
max_repetitions: maxRepetitions,
|
|
1201
|
+
soft: opts?.soft ?? false
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
/**
|
|
1205
|
+
* Assert no tool is called more than once.
|
|
1206
|
+
*
|
|
1207
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1208
|
+
* @returns The chain for fluent composition.
|
|
1209
|
+
*
|
|
1210
|
+
* @example
|
|
1211
|
+
* ```ts
|
|
1212
|
+
* expect(result).noDuplicateTools();
|
|
1213
|
+
* ```
|
|
1214
|
+
*/
|
|
1215
|
+
noDuplicateTools(opts) {
|
|
1216
|
+
return this.add(TYPE_TRACE, { check: "no_duplicates", soft: opts?.soft ?? false });
|
|
1217
|
+
}
|
|
1218
|
+
/**
|
|
1219
|
+
* Assert that all listed tools were called at least once.
|
|
1220
|
+
*
|
|
1221
|
+
* @param tools - Tool names that must appear in the trace.
|
|
1222
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1223
|
+
* @returns The chain for fluent composition.
|
|
1224
|
+
*
|
|
1225
|
+
* @example
|
|
1226
|
+
* ```ts
|
|
1227
|
+
* expect(result).requiredTools(["search", "respond"]);
|
|
1228
|
+
* ```
|
|
1229
|
+
*/
|
|
1230
|
+
requiredTools(tools, opts) {
|
|
1231
|
+
return this.add(TYPE_TRACE, {
|
|
1232
|
+
check: "required_tools",
|
|
1233
|
+
tools,
|
|
1234
|
+
soft: opts?.soft ?? false
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Assert that none of the listed tools were called.
|
|
1239
|
+
*
|
|
1240
|
+
* @param tools - Tool names that must not appear in the trace.
|
|
1241
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1242
|
+
* @returns The chain for fluent composition.
|
|
1243
|
+
*
|
|
1244
|
+
* @example
|
|
1245
|
+
* ```ts
|
|
1246
|
+
* expect(result).forbiddenTools(["delete_user", "drop_table"]);
|
|
1247
|
+
* ```
|
|
1248
|
+
*/
|
|
1249
|
+
forbiddenTools(tools, opts) {
|
|
1250
|
+
return this.add(TYPE_TRACE, {
|
|
1251
|
+
check: "forbidden_tools",
|
|
1252
|
+
tools,
|
|
1253
|
+
soft: opts?.soft ?? false
|
|
1254
|
+
});
|
|
1255
|
+
}
|
|
1256
|
+
// -- Layer 4: Content --
|
|
1257
|
+
/**
|
|
1258
|
+
* Assert the output message contains the given string.
|
|
1259
|
+
*
|
|
1260
|
+
* @param value - Substring to search for.
|
|
1261
|
+
* @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
|
|
1262
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1263
|
+
* @returns The chain for fluent composition.
|
|
1264
|
+
*
|
|
1265
|
+
* @example
|
|
1266
|
+
* ```ts
|
|
1267
|
+
* expect(result).outputContains("refund approved");
|
|
1268
|
+
* ```
|
|
1269
|
+
*/
|
|
1270
|
+
outputContains(value, opts) {
|
|
1271
|
+
return this.add(TYPE_CONTENT, {
|
|
1272
|
+
target: "output.message",
|
|
1273
|
+
check: "contains",
|
|
1274
|
+
value,
|
|
1275
|
+
case_sensitive: opts?.caseSensitive ?? false,
|
|
1276
|
+
soft: opts?.soft ?? false
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
/**
|
|
1280
|
+
* Assert the output message does not contain the given string.
|
|
1281
|
+
*
|
|
1282
|
+
* @param value - Substring that must be absent.
|
|
1283
|
+
* @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
|
|
1284
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1285
|
+
* @returns The chain for fluent composition.
|
|
1286
|
+
*
|
|
1287
|
+
* @example
|
|
1288
|
+
* ```ts
|
|
1289
|
+
* expect(result).outputNotContains("internal error");
|
|
1290
|
+
* ```
|
|
1291
|
+
*/
|
|
1292
|
+
outputNotContains(value, opts) {
|
|
1293
|
+
return this.add(TYPE_CONTENT, {
|
|
1294
|
+
target: "output.message",
|
|
1295
|
+
check: "not_contains",
|
|
1296
|
+
value,
|
|
1297
|
+
case_sensitive: opts?.caseSensitive ?? false,
|
|
1298
|
+
soft: opts?.soft ?? false
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
/**
|
|
1302
|
+
* Assert the output message matches the given regular expression.
|
|
1303
|
+
*
|
|
1304
|
+
* @param pattern - Regex pattern string.
|
|
1305
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1306
|
+
* @returns The chain for fluent composition.
|
|
1307
|
+
*
|
|
1308
|
+
* @example
|
|
1309
|
+
* ```ts
|
|
1310
|
+
* expect(result).outputMatchesRegex("order #\\d{5}");
|
|
1311
|
+
* ```
|
|
1312
|
+
*/
|
|
1313
|
+
outputMatchesRegex(pattern, opts) {
|
|
1314
|
+
return this.add(TYPE_CONTENT, {
|
|
1315
|
+
target: "output.message",
|
|
1316
|
+
check: "regex_match",
|
|
1317
|
+
value: pattern,
|
|
1318
|
+
soft: opts?.soft ?? false
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
/**
|
|
1322
|
+
* Assert the output message contains all of the given keywords.
|
|
1323
|
+
*
|
|
1324
|
+
* @param keywords - Keywords that must all be present.
|
|
1325
|
+
* @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
|
|
1326
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1327
|
+
* @returns The chain for fluent composition.
|
|
1328
|
+
*
|
|
1329
|
+
* @example
|
|
1330
|
+
* ```ts
|
|
1331
|
+
* expect(result).outputHasAllKeywords(["refund", "processed"]);
|
|
1332
|
+
* ```
|
|
1333
|
+
*/
|
|
1334
|
+
outputHasAllKeywords(keywords, opts) {
|
|
1335
|
+
return this.add(TYPE_CONTENT, {
|
|
1336
|
+
target: "output.message",
|
|
1337
|
+
check: "keyword_all",
|
|
1338
|
+
values: keywords,
|
|
1339
|
+
case_sensitive: opts?.caseSensitive ?? false,
|
|
1340
|
+
soft: opts?.soft ?? false
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* Assert the output message contains at least one of the given keywords.
|
|
1345
|
+
*
|
|
1346
|
+
* @param keywords - Keywords where at least one must be present.
|
|
1347
|
+
* @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
|
|
1348
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1349
|
+
* @returns The chain for fluent composition.
|
|
1350
|
+
*
|
|
1351
|
+
* @example
|
|
1352
|
+
* ```ts
|
|
1353
|
+
* expect(result).outputHasAnyKeyword(["yes", "confirmed"]);
|
|
1354
|
+
* ```
|
|
1355
|
+
*/
|
|
1356
|
+
outputHasAnyKeyword(keywords, opts) {
|
|
1357
|
+
return this.add(TYPE_CONTENT, {
|
|
1358
|
+
target: "output.message",
|
|
1359
|
+
check: "keyword_any",
|
|
1360
|
+
values: keywords,
|
|
1361
|
+
case_sensitive: opts?.caseSensitive ?? false,
|
|
1362
|
+
soft: opts?.soft ?? false
|
|
1363
|
+
});
|
|
1364
|
+
}
|
|
1365
|
+
/**
|
|
1366
|
+
* Assert the output message contains none of the forbidden terms.
|
|
1367
|
+
*
|
|
1368
|
+
* @param terms - Terms that must not appear in the output.
|
|
1369
|
+
* @returns The chain for fluent composition.
|
|
1370
|
+
*
|
|
1371
|
+
* @example
|
|
1372
|
+
* ```ts
|
|
1373
|
+
* expect(result).outputForbids(["password", "api_key"]);
|
|
1374
|
+
* ```
|
|
1375
|
+
*/
|
|
1376
|
+
outputForbids(terms) {
|
|
1377
|
+
return this.add(TYPE_CONTENT, {
|
|
1378
|
+
target: "output.message",
|
|
1379
|
+
check: "forbidden",
|
|
1380
|
+
values: terms
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Generic content-contains check on any target path.
|
|
1385
|
+
*
|
|
1386
|
+
* @param target - Dot-delimited path into the trace.
|
|
1387
|
+
* @param value - Substring to search for.
|
|
1388
|
+
* @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
|
|
1389
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1390
|
+
* @returns The chain for fluent composition.
|
|
1391
|
+
*
|
|
1392
|
+
* @example
|
|
1393
|
+
* ```ts
|
|
1394
|
+
* expect(result).contentContains("steps[0].result.text", "success");
|
|
1395
|
+
* ```
|
|
1396
|
+
*/
|
|
1397
|
+
contentContains(target, value, opts) {
|
|
1398
|
+
return this.add(TYPE_CONTENT, {
|
|
1399
|
+
target,
|
|
1400
|
+
check: "contains",
|
|
1401
|
+
value,
|
|
1402
|
+
case_sensitive: opts?.caseSensitive ?? false,
|
|
1403
|
+
soft: opts?.soft ?? false
|
|
1404
|
+
});
|
|
1405
|
+
}
|
|
1406
|
+
// -- Layer 5: Embedding Similarity --
|
|
1407
|
+
/**
|
|
1408
|
+
* Assert the output is semantically similar to a reference string via embeddings.
|
|
1409
|
+
*
|
|
1410
|
+
* @param reference - The reference text to compare against.
|
|
1411
|
+
* @param opts.threshold - Minimum cosine similarity (default `0.8`).
|
|
1412
|
+
* @param opts.model - Override the embedding model.
|
|
1413
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1414
|
+
* @returns The chain for fluent composition.
|
|
1415
|
+
*
|
|
1416
|
+
* @example
|
|
1417
|
+
* ```ts
|
|
1418
|
+
* expect(result).outputSimilarTo("The order has been refunded", { threshold: 0.85 });
|
|
1419
|
+
* ```
|
|
1420
|
+
*/
|
|
1421
|
+
outputSimilarTo(reference, opts) {
|
|
1422
|
+
return this.add(TYPE_EMBEDDING, {
|
|
1423
|
+
target: "output.message",
|
|
1424
|
+
reference,
|
|
1425
|
+
threshold: opts?.threshold ?? 0.8,
|
|
1426
|
+
model: opts?.model,
|
|
1427
|
+
soft: opts?.soft ?? false
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
// -- Layer 6: LLM Judge --
|
|
1431
|
+
/**
|
|
1432
|
+
* Assert the output passes LLM judge evaluation against given criteria.
|
|
1433
|
+
*
|
|
1434
|
+
* @param criteria - Natural language description of what makes a good response.
|
|
1435
|
+
* @param opts.rubric - Rubric name (default `"default"`).
|
|
1436
|
+
* @param opts.threshold - Minimum passing score (default `0.8`).
|
|
1437
|
+
* @param opts.model - Override the judge model.
|
|
1438
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1439
|
+
* @returns The chain for fluent composition.
|
|
1440
|
+
*
|
|
1441
|
+
* @example
|
|
1442
|
+
* ```ts
|
|
1443
|
+
* expect(result).passesJudge("Response is polite and helpful", { threshold: 0.9 });
|
|
1444
|
+
* ```
|
|
1445
|
+
*/
|
|
1446
|
+
passesJudge(criteria, opts) {
|
|
1447
|
+
return this.add(TYPE_LLM_JUDGE, {
|
|
1448
|
+
target: "output.message",
|
|
1449
|
+
criteria,
|
|
1450
|
+
rubric: opts?.rubric ?? "default",
|
|
1451
|
+
threshold: opts?.threshold ?? 0.8,
|
|
1452
|
+
model: opts?.model,
|
|
1453
|
+
soft: opts?.soft ?? false
|
|
1454
|
+
});
|
|
1455
|
+
}
|
|
1456
|
+
// -- Layer 7: Trace Tree (Multi-Agent) --
|
|
1457
|
+
/**
|
|
1458
|
+
* Assert that a specific agent was called in the trace tree.
|
|
1459
|
+
*
|
|
1460
|
+
* @param agentId - The agent identifier to look for.
|
|
1461
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1462
|
+
* @returns The chain for fluent composition.
|
|
1463
|
+
*
|
|
1464
|
+
* @example
|
|
1465
|
+
* ```ts
|
|
1466
|
+
* expect(result).agentCalled("summarizer");
|
|
1467
|
+
* ```
|
|
1468
|
+
*/
|
|
1469
|
+
agentCalled(agentId2, opts) {
|
|
1470
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1471
|
+
check: "agent_called",
|
|
1472
|
+
agent_id: agentId2,
|
|
1473
|
+
soft: opts?.soft ?? false
|
|
1474
|
+
});
|
|
1475
|
+
}
|
|
1476
|
+
/**
|
|
1477
|
+
* Assert the trace tree delegation depth does not exceed `maxDepth`.
|
|
1478
|
+
*
|
|
1479
|
+
* @param maxDepth - Maximum allowed delegation depth.
|
|
1480
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1481
|
+
* @returns The chain for fluent composition.
|
|
1482
|
+
*
|
|
1483
|
+
* @example
|
|
1484
|
+
* ```ts
|
|
1485
|
+
* expect(result).delegationDepth(3);
|
|
1486
|
+
* ```
|
|
1487
|
+
*/
|
|
1488
|
+
delegationDepth(maxDepth, opts) {
|
|
1489
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1490
|
+
check: "delegation_depth",
|
|
1491
|
+
max_depth: maxDepth,
|
|
1492
|
+
soft: opts?.soft ?? false
|
|
1493
|
+
});
|
|
1494
|
+
}
|
|
1495
|
+
/**
|
|
1496
|
+
* Assert a sub-agent's output contains the given string.
|
|
1497
|
+
*
|
|
1498
|
+
* @param agentId - The agent whose output to check.
|
|
1499
|
+
* @param value - Substring to search for.
|
|
1500
|
+
* @param opts.caseSensitive - Enable case-sensitive matching (default `false`).
|
|
1501
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1502
|
+
* @returns The chain for fluent composition.
|
|
1503
|
+
*
|
|
1504
|
+
* @example
|
|
1505
|
+
* ```ts
|
|
1506
|
+
* expect(result).agentOutputContains("summarizer", "key findings");
|
|
1507
|
+
* ```
|
|
1508
|
+
*/
|
|
1509
|
+
agentOutputContains(agentId2, value, opts) {
|
|
1510
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1511
|
+
check: "agent_output_contains",
|
|
1512
|
+
agent_id: agentId2,
|
|
1513
|
+
value,
|
|
1514
|
+
case_sensitive: opts?.caseSensitive ?? false,
|
|
1515
|
+
soft: opts?.soft ?? false
|
|
1516
|
+
});
|
|
1517
|
+
}
|
|
1518
|
+
/**
|
|
1519
|
+
* Assert data flows from one agent's output to another agent's input.
|
|
1520
|
+
*
|
|
1521
|
+
* @param fromAgent - Source agent ID.
|
|
1522
|
+
* @param toAgent - Destination agent ID.
|
|
1523
|
+
* @param field - The data field that must flow between agents.
|
|
1524
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1525
|
+
* @returns The chain for fluent composition.
|
|
1526
|
+
*
|
|
1527
|
+
* @example
|
|
1528
|
+
* ```ts
|
|
1529
|
+
* expect(result).crossAgentDataFlow("researcher", "writer", "findings");
|
|
1530
|
+
* ```
|
|
1531
|
+
*/
|
|
1532
|
+
crossAgentDataFlow(fromAgent, toAgent, field, opts) {
|
|
1533
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1534
|
+
check: "cross_agent_data_flow",
|
|
1535
|
+
from_agent: fromAgent,
|
|
1536
|
+
to_agent: toAgent,
|
|
1537
|
+
field,
|
|
1538
|
+
soft: opts?.soft ?? false
|
|
1539
|
+
});
|
|
1540
|
+
}
|
|
1541
|
+
/**
|
|
1542
|
+
* Assert agent delegations follow the specified transitions.
|
|
1543
|
+
*
|
|
1544
|
+
* @param transitions - Array of `[fromAgent, toAgent]` pairs.
|
|
1545
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1546
|
+
* @returns The chain for fluent composition.
|
|
1547
|
+
*
|
|
1548
|
+
* @example
|
|
1549
|
+
* ```ts
|
|
1550
|
+
* expect(result).followsTransitions([["root", "planner"], ["planner", "executor"]]);
|
|
1551
|
+
* ```
|
|
1552
|
+
*/
|
|
1553
|
+
followsTransitions(transitions, opts) {
|
|
1554
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1555
|
+
check: "follows_transitions",
|
|
1556
|
+
transitions,
|
|
1557
|
+
soft: opts?.soft ?? false
|
|
1558
|
+
});
|
|
1559
|
+
}
|
|
1560
|
+
/**
|
|
1561
|
+
* Assert aggregate cost across the entire trace tree is at or below `maxCost`.
|
|
1562
|
+
*
|
|
1563
|
+
* @param maxCost - Maximum allowed aggregate cost in USD.
|
|
1564
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1565
|
+
* @returns The chain for fluent composition.
|
|
1566
|
+
*
|
|
1567
|
+
* @example
|
|
1568
|
+
* ```ts
|
|
1569
|
+
* expect(result).aggregateCostUnder(0.50);
|
|
1570
|
+
* ```
|
|
1571
|
+
*/
|
|
1572
|
+
aggregateCostUnder(maxCost, opts) {
|
|
1573
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1574
|
+
check: "aggregate_cost",
|
|
1575
|
+
operator: "lte",
|
|
1576
|
+
value: maxCost,
|
|
1577
|
+
soft: opts?.soft ?? false
|
|
1578
|
+
});
|
|
1579
|
+
}
|
|
1580
|
+
/**
|
|
1581
|
+
* Assert aggregate tokens across the entire trace tree is at or below `maxTokens`.
|
|
1582
|
+
*
|
|
1583
|
+
* @param maxTokens - Maximum allowed aggregate token count.
|
|
1584
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1585
|
+
* @returns The chain for fluent composition.
|
|
1586
|
+
*
|
|
1587
|
+
* @example
|
|
1588
|
+
* ```ts
|
|
1589
|
+
* expect(result).aggregateTokensUnder(10000);
|
|
1590
|
+
* ```
|
|
1591
|
+
*/
|
|
1592
|
+
aggregateTokensUnder(maxTokens, opts) {
|
|
1593
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1594
|
+
check: "aggregate_tokens",
|
|
1595
|
+
operator: "lte",
|
|
1596
|
+
value: maxTokens,
|
|
1597
|
+
soft: opts?.soft ?? false
|
|
1598
|
+
});
|
|
1599
|
+
}
|
|
1600
|
+
/**
|
|
1601
|
+
* Assert `agentA` started before `agentB` in the trace tree.
|
|
1602
|
+
*
|
|
1603
|
+
* @param agentA - The agent expected to run first.
|
|
1604
|
+
* @param agentB - The agent expected to run second.
|
|
1605
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1606
|
+
* @returns The chain for fluent composition.
|
|
1607
|
+
*
|
|
1608
|
+
* @example
|
|
1609
|
+
* ```ts
|
|
1610
|
+
* expect(result).agentOrderedBefore("planner", "executor");
|
|
1611
|
+
* ```
|
|
1612
|
+
*/
|
|
1613
|
+
agentOrderedBefore(agentA, agentB, opts) {
|
|
1614
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1615
|
+
check: "agent_ordered_before",
|
|
1616
|
+
agent_a: agentA,
|
|
1617
|
+
agent_b: agentB,
|
|
1618
|
+
soft: opts?.soft ?? false
|
|
1619
|
+
});
|
|
1620
|
+
}
|
|
1621
|
+
/**
|
|
1622
|
+
* Assert `agentA` and `agentB` ran concurrently (overlapping wall-clock time).
|
|
1623
|
+
*
|
|
1624
|
+
* @param agentA - First agent ID.
|
|
1625
|
+
* @param agentB - Second agent ID.
|
|
1626
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1627
|
+
* @returns The chain for fluent composition.
|
|
1628
|
+
*
|
|
1629
|
+
* @example
|
|
1630
|
+
* ```ts
|
|
1631
|
+
* expect(result).agentsOverlap("worker-1", "worker-2");
|
|
1632
|
+
* ```
|
|
1633
|
+
*/
|
|
1634
|
+
agentsOverlap(agentA, agentB, opts) {
|
|
1635
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1636
|
+
check: "agents_overlap",
|
|
1637
|
+
agent_a: agentA,
|
|
1638
|
+
agent_b: agentB,
|
|
1639
|
+
soft: opts?.soft ?? false
|
|
1640
|
+
});
|
|
1641
|
+
}
|
|
1642
|
+
/**
|
|
1643
|
+
* Assert a specific agent's wall-clock duration is under `maxMs` milliseconds.
|
|
1644
|
+
*
|
|
1645
|
+
* @param agentId - The agent to check.
|
|
1646
|
+
* @param maxMs - Maximum allowed wall-clock time in milliseconds.
|
|
1647
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1648
|
+
* @returns The chain for fluent composition.
|
|
1649
|
+
*
|
|
1650
|
+
* @example
|
|
1651
|
+
* ```ts
|
|
1652
|
+
* expect(result).agentWallTimeUnder("summarizer", 5000);
|
|
1653
|
+
* ```
|
|
1654
|
+
*/
|
|
1655
|
+
agentWallTimeUnder(agentId2, maxMs, opts) {
|
|
1656
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1657
|
+
check: "agent_wall_time_under",
|
|
1658
|
+
agent_id: agentId2,
|
|
1659
|
+
max_ms: maxMs,
|
|
1660
|
+
soft: opts?.soft ?? false
|
|
1661
|
+
});
|
|
1662
|
+
}
|
|
1663
|
+
/**
|
|
1664
|
+
* Assert agents ran in ordered groups (parallel within each group, sequential across groups).
|
|
1665
|
+
*
|
|
1666
|
+
* @param groups - Array of agent-ID arrays. Each sub-array is a parallel group.
|
|
1667
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1668
|
+
* @returns The chain for fluent composition.
|
|
1669
|
+
*
|
|
1670
|
+
* @example
|
|
1671
|
+
* ```ts
|
|
1672
|
+
* expect(result).orderedAgents([["planner"], ["worker-1", "worker-2"], ["aggregator"]]);
|
|
1673
|
+
* ```
|
|
1674
|
+
*/
|
|
1675
|
+
orderedAgents(groups, opts) {
|
|
1676
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1677
|
+
check: "ordered_agents",
|
|
1678
|
+
groups,
|
|
1679
|
+
soft: opts?.soft ?? false
|
|
1680
|
+
});
|
|
1681
|
+
}
|
|
1682
|
+
// -- Layer 8: Plugin --
|
|
1683
|
+
/**
|
|
1684
|
+
* Assert via a registered plugin (Layer 8).
|
|
1685
|
+
*
|
|
1686
|
+
* @param pluginId - Identifier of the plugin registered with the engine.
|
|
1687
|
+
* @param config - Plugin-specific configuration passed at evaluation time.
|
|
1688
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1689
|
+
* @returns The chain for fluent composition.
|
|
1690
|
+
*
|
|
1691
|
+
* @example
|
|
1692
|
+
* ```ts
|
|
1693
|
+
* expect(result).plugin("custom-safety-check", { threshold: 0.95 });
|
|
1694
|
+
* ```
|
|
1695
|
+
*/
|
|
1696
|
+
plugin(pluginId, config2, opts) {
|
|
1697
|
+
const spec = { plugin_id: pluginId, soft: opts?.soft ?? false };
|
|
1698
|
+
if (config2 !== void 0) spec.config = config2;
|
|
1699
|
+
return this.add(TYPE_PLUGIN, spec);
|
|
1700
|
+
}
|
|
1701
|
+
// -- TraceTree Analytics (P9 parity) --
|
|
1702
|
+
/**
|
|
1703
|
+
* Assert aggregate latency across the trace tree is at or below `maxMs`.
|
|
1704
|
+
*
|
|
1705
|
+
* @param maxMs - Maximum allowed aggregate latency in milliseconds.
|
|
1706
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1707
|
+
* @returns The chain for fluent composition.
|
|
1708
|
+
*
|
|
1709
|
+
* @example
|
|
1710
|
+
* ```ts
|
|
1711
|
+
* expect(result).aggregateLatencyUnder(5000);
|
|
1712
|
+
* ```
|
|
1713
|
+
*/
|
|
1714
|
+
aggregateLatencyUnder(maxMs, opts) {
|
|
1715
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1716
|
+
check: "aggregate_latency",
|
|
1717
|
+
operator: "lte",
|
|
1718
|
+
value: maxMs,
|
|
1719
|
+
soft: opts?.soft ?? false
|
|
1720
|
+
});
|
|
1721
|
+
}
|
|
1722
|
+
/**
|
|
1723
|
+
* Assert that all specified tools were called across the entire trace tree.
|
|
1724
|
+
*
|
|
1725
|
+
* @param toolNames - Tool names that must appear somewhere in the tree.
|
|
1726
|
+
* @param opts.soft - When `true` the failure is non-blocking.
|
|
1727
|
+
* @returns The chain for fluent composition.
|
|
1728
|
+
*
|
|
1729
|
+
* @example
|
|
1730
|
+
* ```ts
|
|
1731
|
+
* expect(result).allToolsCalled(["search", "summarize"]);
|
|
1732
|
+
* ```
|
|
1733
|
+
*/
|
|
1734
|
+
allToolsCalled(toolNames, opts) {
|
|
1735
|
+
return this.add(TYPE_TRACE_TREE, {
|
|
1736
|
+
check: "all_tools_called",
|
|
1737
|
+
tools: toolNames,
|
|
1738
|
+
soft: opts?.soft ?? false
|
|
1739
|
+
});
|
|
1740
|
+
}
|
|
1741
|
+
};
|
|
1742
|
+
function attestExpect(result) {
|
|
1743
|
+
if (result instanceof AgentResult) {
|
|
1744
|
+
return new ExpectChain(result);
|
|
1745
|
+
}
|
|
1746
|
+
return new ExpectChain(new AgentResult(result));
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
// src/context.ts
|
|
1750
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
1751
|
+
var activeBuilder = new AsyncLocalStorage();
|
|
1752
|
+
|
|
1753
|
+
// src/agent.ts
|
|
1754
|
+
var Agent = class {
|
|
1755
|
+
name;
|
|
1756
|
+
fn;
|
|
1757
|
+
constructor(name, fn) {
|
|
1758
|
+
this.name = name;
|
|
1759
|
+
this.fn = fn;
|
|
1760
|
+
}
|
|
1761
|
+
run(args = {}) {
|
|
1762
|
+
if (this.fn === void 0) {
|
|
1763
|
+
throw new Error("No agent function provided. Pass fn to Agent().");
|
|
1764
|
+
}
|
|
1765
|
+
const builder = new TraceBuilder(this.name);
|
|
1766
|
+
builder.setInput(args);
|
|
1767
|
+
const output = activeBuilder.run(builder, () => this.fn(builder, args));
|
|
1768
|
+
this.setOutput(builder, output);
|
|
1769
|
+
return new AgentResult(builder.build());
|
|
1770
|
+
}
|
|
1771
|
+
async arun(args = {}) {
|
|
1772
|
+
if (this.fn === void 0) {
|
|
1773
|
+
throw new Error("No agent function provided. Pass fn to Agent().");
|
|
1774
|
+
}
|
|
1775
|
+
const builder = new TraceBuilder(this.name);
|
|
1776
|
+
builder.setInput(args);
|
|
1777
|
+
const output = await activeBuilder.run(builder, async () => this.fn(builder, args));
|
|
1778
|
+
this.setOutput(builder, output);
|
|
1779
|
+
return new AgentResult(builder.build());
|
|
1780
|
+
}
|
|
1781
|
+
setOutput(builder, output) {
|
|
1782
|
+
if (output !== null && output !== void 0 && typeof output === "object" && !Array.isArray(output)) {
|
|
1783
|
+
builder.setOutput(output);
|
|
1784
|
+
} else if (output !== void 0) {
|
|
1785
|
+
builder.setOutput({ result: output });
|
|
1786
|
+
} else {
|
|
1787
|
+
builder.setOutput({ result: null });
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
withTrace(trace) {
|
|
1791
|
+
return new AgentResult(trace);
|
|
1792
|
+
}
|
|
1793
|
+
};
|
|
1794
|
+
function agent(name, fn) {
|
|
1795
|
+
const wrapped = new Agent(name, fn);
|
|
1796
|
+
const wrapper = (args) => {
|
|
1797
|
+
return wrapped.run(args ?? {});
|
|
1798
|
+
};
|
|
1799
|
+
wrapper.agent = wrapped;
|
|
1800
|
+
return wrapper;
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
// src/delegate.ts
|
|
1804
|
+
async function delegate(agentId2, fn) {
|
|
1805
|
+
const parent = activeBuilder.getStore();
|
|
1806
|
+
if (parent === void 0) {
|
|
1807
|
+
throw new Error(
|
|
1808
|
+
"delegate() must be used within an Agent.run() context. No active TraceBuilder found."
|
|
1809
|
+
);
|
|
1810
|
+
}
|
|
1811
|
+
const child = new TraceBuilder(agentId2);
|
|
1812
|
+
child.setParentTraceId(parent.getTraceId());
|
|
1813
|
+
await activeBuilder.run(child, async () => {
|
|
1814
|
+
await fn(child);
|
|
1815
|
+
});
|
|
1816
|
+
const childTrace = child.build();
|
|
1817
|
+
parent.addStep({
|
|
1818
|
+
type: STEP_AGENT_CALL,
|
|
1819
|
+
name: agentId2,
|
|
1820
|
+
args: void 0,
|
|
1821
|
+
result: childTrace.output,
|
|
1822
|
+
sub_trace: childTrace
|
|
1823
|
+
});
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
// src/tier.ts
|
|
1827
|
+
var TIER_1 = 1;
|
|
1828
|
+
var TIER_2 = 2;
|
|
1829
|
+
var TIER_3 = 3;
|
|
1830
|
+
function tier(level) {
|
|
1831
|
+
return (fn) => {
|
|
1832
|
+
fn._attest_tier = level;
|
|
1833
|
+
return fn;
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
// src/plugins.ts
|
|
1838
|
+
var PluginRegistry = class {
|
|
1839
|
+
plugins = /* @__PURE__ */ new Map();
|
|
1840
|
+
register(name, plugin) {
|
|
1841
|
+
const pluginType = plugin.pluginType;
|
|
1842
|
+
let typeMap = this.plugins.get(pluginType);
|
|
1843
|
+
if (typeMap === void 0) {
|
|
1844
|
+
typeMap = /* @__PURE__ */ new Map();
|
|
1845
|
+
this.plugins.set(pluginType, typeMap);
|
|
1846
|
+
}
|
|
1847
|
+
typeMap.set(name, plugin);
|
|
1848
|
+
}
|
|
1849
|
+
get(pluginType, name) {
|
|
1850
|
+
return this.plugins.get(pluginType)?.get(name);
|
|
1851
|
+
}
|
|
1852
|
+
listPlugins(pluginType) {
|
|
1853
|
+
if (pluginType !== void 0) {
|
|
1854
|
+
const typeMap = this.plugins.get(pluginType);
|
|
1855
|
+
return typeMap !== void 0 ? [...typeMap.keys()] : [];
|
|
1856
|
+
}
|
|
1857
|
+
const result = [];
|
|
1858
|
+
for (const [ptype, typeMap] of this.plugins) {
|
|
1859
|
+
for (const pname of typeMap.keys()) {
|
|
1860
|
+
result.push(`${ptype}/${pname}`);
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
return result;
|
|
1864
|
+
}
|
|
1865
|
+
};
|
|
1866
|
+
async function executePluginAssertion(plugin, trace, spec, client, traceId2, assertionId2, timeoutMs = 3e4) {
|
|
1867
|
+
const resultPromise = Promise.resolve(plugin.execute(trace, spec));
|
|
1868
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
1869
|
+
setTimeout(() => reject(new Error(`Plugin '${plugin.name}' timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
1870
|
+
});
|
|
1871
|
+
const result = await Promise.race([resultPromise, timeoutPromise]);
|
|
1872
|
+
await client.submitPluginResult(
|
|
1873
|
+
traceId2,
|
|
1874
|
+
plugin.name,
|
|
1875
|
+
assertionId2,
|
|
1876
|
+
result.status,
|
|
1877
|
+
result.score,
|
|
1878
|
+
result.explanation
|
|
1879
|
+
);
|
|
1880
|
+
return result;
|
|
1881
|
+
}
|
|
1882
|
+
|
|
1883
|
+
// src/continuous.ts
|
|
1884
|
+
import { EventEmitter } from "events";
|
|
1885
|
+
var DEFAULT_QUEUE_SIZE = 1e3;
|
|
1886
|
+
function resolveQueueSize() {
|
|
1887
|
+
const raw = process.env["ATTEST_CONTINUOUS_QUEUE_SIZE"];
|
|
1888
|
+
if (raw !== void 0 && raw !== "") {
|
|
1889
|
+
const parsed = Number(raw);
|
|
1890
|
+
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
|
1891
|
+
}
|
|
1892
|
+
return DEFAULT_QUEUE_SIZE;
|
|
1893
|
+
}
|
|
1894
|
+
var Sampler = class {
|
|
1895
|
+
rate;
|
|
1896
|
+
constructor(rate) {
|
|
1897
|
+
if (rate < 0 || rate > 1) {
|
|
1898
|
+
throw new Error(`sample_rate must be in [0.0, 1.0], got ${rate}`);
|
|
1899
|
+
}
|
|
1900
|
+
this.rate = rate;
|
|
1901
|
+
}
|
|
1902
|
+
shouldSample() {
|
|
1903
|
+
return Math.random() < this.rate;
|
|
1904
|
+
}
|
|
1905
|
+
};
|
|
1906
|
+
var AlertDispatcher = class extends EventEmitter {
|
|
1907
|
+
webhookUrl;
|
|
1908
|
+
slackUrl;
|
|
1909
|
+
constructor(webhookUrl, slackUrl) {
|
|
1910
|
+
super();
|
|
1911
|
+
this.webhookUrl = webhookUrl;
|
|
1912
|
+
this.slackUrl = slackUrl;
|
|
1913
|
+
}
|
|
1914
|
+
async dispatch(alert) {
|
|
1915
|
+
this.emit("alert", alert);
|
|
1916
|
+
const tasks = [];
|
|
1917
|
+
if (this.webhookUrl) {
|
|
1918
|
+
tasks.push(this.postJson(this.webhookUrl, alert));
|
|
1919
|
+
}
|
|
1920
|
+
if (this.slackUrl) {
|
|
1921
|
+
const text = `[attest] drift alert -- type=${alert.drift_type} score=${alert.score} trace_id=${alert.trace_id}`;
|
|
1922
|
+
tasks.push(this.postJson(this.slackUrl, { text }));
|
|
1923
|
+
}
|
|
1924
|
+
const results = await Promise.allSettled(tasks);
|
|
1925
|
+
for (const result of results) {
|
|
1926
|
+
if (result.status === "rejected") {
|
|
1927
|
+
this.emit("alertError", result.reason);
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
async postJson(url, payload) {
|
|
1932
|
+
const response = await fetch(url, {
|
|
1933
|
+
method: "POST",
|
|
1934
|
+
headers: { "Content-Type": "application/json" },
|
|
1935
|
+
body: JSON.stringify(payload),
|
|
1936
|
+
signal: AbortSignal.timeout(1e4)
|
|
1937
|
+
});
|
|
1938
|
+
if (!response.ok) {
|
|
1939
|
+
throw new Error(`POST to ${url} returned ${response.status}`);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
};
|
|
1943
|
+
var ContinuousEvalRunner = class {
|
|
1944
|
+
client;
|
|
1945
|
+
assertions;
|
|
1946
|
+
sampler;
|
|
1947
|
+
dispatcher;
|
|
1948
|
+
queue = [];
|
|
1949
|
+
maxSize;
|
|
1950
|
+
running = false;
|
|
1951
|
+
loopTimer = null;
|
|
1952
|
+
constructor(client, assertions, options) {
|
|
1953
|
+
this.client = client;
|
|
1954
|
+
this.assertions = assertions;
|
|
1955
|
+
this.sampler = new Sampler(options?.sampleRate ?? 1);
|
|
1956
|
+
this.dispatcher = new AlertDispatcher(options?.alertWebhook, options?.alertSlackUrl);
|
|
1957
|
+
this.maxSize = options?.maxSize ?? resolveQueueSize();
|
|
1958
|
+
}
|
|
1959
|
+
get alertEmitter() {
|
|
1960
|
+
return this.dispatcher;
|
|
1961
|
+
}
|
|
1962
|
+
async evaluateTrace(trace) {
|
|
1963
|
+
if (!this.sampler.shouldSample()) return null;
|
|
1964
|
+
return this.client.evaluateBatch(trace, this.assertions);
|
|
1965
|
+
}
|
|
1966
|
+
submit(trace) {
|
|
1967
|
+
if (this.queue.length >= this.maxSize) {
|
|
1968
|
+
return false;
|
|
1969
|
+
}
|
|
1970
|
+
this.queue.push(trace);
|
|
1971
|
+
return true;
|
|
1972
|
+
}
|
|
1973
|
+
start() {
|
|
1974
|
+
if (this.running) return;
|
|
1975
|
+
this.running = true;
|
|
1976
|
+
this.scheduleNext();
|
|
1977
|
+
}
|
|
1978
|
+
stop() {
|
|
1979
|
+
this.running = false;
|
|
1980
|
+
if (this.loopTimer !== null) {
|
|
1981
|
+
clearTimeout(this.loopTimer);
|
|
1982
|
+
this.loopTimer = null;
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
get queueLength() {
|
|
1986
|
+
return this.queue.length;
|
|
1987
|
+
}
|
|
1988
|
+
get isRunning() {
|
|
1989
|
+
return this.running;
|
|
1990
|
+
}
|
|
1991
|
+
scheduleNext() {
|
|
1992
|
+
if (!this.running) return;
|
|
1993
|
+
this.loopTimer = setTimeout(async () => {
|
|
1994
|
+
if (!this.running) return;
|
|
1995
|
+
const trace = this.queue.shift();
|
|
1996
|
+
if (trace !== void 0) {
|
|
1997
|
+
try {
|
|
1998
|
+
await this.evaluateTrace(trace);
|
|
1999
|
+
} catch {
|
|
2000
|
+
}
|
|
2001
|
+
}
|
|
2002
|
+
this.scheduleNext();
|
|
2003
|
+
}, this.queue.length > 0 ? 0 : 100);
|
|
2004
|
+
}
|
|
2005
|
+
};
|
|
2006
|
+
|
|
2007
|
+
// src/personas.ts
|
|
2008
|
+
var FRIENDLY_USER = {
|
|
2009
|
+
name: "FriendlyUser",
|
|
2010
|
+
system_prompt: `You are a friendly, cooperative user interacting with an AI assistant.
|
|
2011
|
+
You make clear, well-formed requests. You respond positively to helpful answers and
|
|
2012
|
+
ask straightforward follow-up questions. Keep responses concise (1-3 sentences).`,
|
|
2013
|
+
style: "friendly",
|
|
2014
|
+
temperature: 0.7,
|
|
2015
|
+
max_tokens: 200
|
|
2016
|
+
};
|
|
2017
|
+
var ADVERSARIAL_USER = {
|
|
2018
|
+
name: "AdversarialUser",
|
|
2019
|
+
system_prompt: `You are an adversarial user testing the limits of an AI assistant.
|
|
2020
|
+
You ask edge case questions, make ambiguous requests, and probe boundary conditions.
|
|
2021
|
+
Try to find inconsistencies or unexpected behaviors. Keep responses concise (1-3 sentences).`,
|
|
2022
|
+
style: "adversarial",
|
|
2023
|
+
temperature: 0.9,
|
|
2024
|
+
max_tokens: 200
|
|
2025
|
+
};
|
|
2026
|
+
var CONFUSED_USER = {
|
|
2027
|
+
name: "ConfusedUser",
|
|
2028
|
+
system_prompt: `You are a confused user who has trouble articulating what you want.
|
|
2029
|
+
You make vague requests, sometimes contradict yourself, and frequently ask for clarification.
|
|
2030
|
+
You may misunderstand the assistant's responses. Keep responses concise (1-3 sentences).`,
|
|
2031
|
+
style: "confused",
|
|
2032
|
+
temperature: 0.8,
|
|
2033
|
+
max_tokens: 200
|
|
2034
|
+
};
|
|
2035
|
+
var COOPERATIVE_USER = {
|
|
2036
|
+
name: "CooperativeUser",
|
|
2037
|
+
system_prompt: `You are a cooperative user who actively helps the AI assistant succeed.
|
|
2038
|
+
You provide clear context, answer clarifying questions thoroughly, and confirm when
|
|
2039
|
+
the assistant's response meets your needs. Keep responses concise (1-3 sentences).`,
|
|
2040
|
+
style: "cooperative",
|
|
2041
|
+
temperature: 0.6,
|
|
2042
|
+
max_tokens: 200
|
|
2043
|
+
};
|
|
2044
|
+
|
|
2045
|
+
// src/adapters/manual.ts
|
|
2046
|
+
var ManualAdapter = class {
|
|
2047
|
+
agentId;
|
|
2048
|
+
constructor(agentId2) {
|
|
2049
|
+
this.agentId = agentId2;
|
|
2050
|
+
}
|
|
2051
|
+
capture(builderFn) {
|
|
2052
|
+
const builder = new TraceBuilder(this.agentId);
|
|
2053
|
+
builderFn(builder);
|
|
2054
|
+
return builder.build();
|
|
2055
|
+
}
|
|
2056
|
+
createBuilder() {
|
|
2057
|
+
return new TraceBuilder(this.agentId);
|
|
2058
|
+
}
|
|
2059
|
+
};
|
|
2060
|
+
|
|
2061
|
+
// src/adapters/openai.ts
|
|
2062
|
+
var OpenAIAdapter = class {
|
|
2063
|
+
agentId;
|
|
2064
|
+
constructor(agentId2) {
|
|
2065
|
+
this.agentId = agentId2;
|
|
2066
|
+
}
|
|
2067
|
+
traceFromResponse(response, options) {
|
|
2068
|
+
const builder = new TraceBuilder(this.agentId);
|
|
2069
|
+
if (options?.inputMessages) {
|
|
2070
|
+
builder.setInput({ messages: options.inputMessages });
|
|
2071
|
+
}
|
|
2072
|
+
const message = response.choices[0].message;
|
|
2073
|
+
const completionText = message.content ?? "";
|
|
2074
|
+
const stepArgs = {};
|
|
2075
|
+
if (response.model !== void 0) stepArgs.model = response.model;
|
|
2076
|
+
const stepResult = { completion: completionText };
|
|
2077
|
+
if (response.usage?.total_tokens !== void 0) {
|
|
2078
|
+
stepResult.tokens = response.usage.total_tokens;
|
|
2079
|
+
}
|
|
2080
|
+
builder.addLlmCall("completion", { args: stepArgs, result: stepResult });
|
|
2081
|
+
if (message.tool_calls) {
|
|
2082
|
+
for (const tc of message.tool_calls) {
|
|
2083
|
+
builder.addToolCall(tc.function.name, {
|
|
2084
|
+
args: { arguments: tc.function.arguments }
|
|
2085
|
+
});
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
builder.setOutput({
|
|
2089
|
+
message: completionText,
|
|
2090
|
+
structured: options?.structuredOutput ?? {}
|
|
2091
|
+
});
|
|
2092
|
+
builder.setMetadata({
|
|
2093
|
+
total_tokens: response.usage?.total_tokens,
|
|
2094
|
+
cost_usd: options?.costUsd,
|
|
2095
|
+
latency_ms: options?.latencyMs,
|
|
2096
|
+
model: response.model
|
|
2097
|
+
});
|
|
2098
|
+
return builder.build();
|
|
2099
|
+
}
|
|
2100
|
+
};
|
|
2101
|
+
|
|
2102
|
+
// src/adapters/anthropic.ts
|
|
2103
|
+
var AnthropicAdapter = class {
|
|
2104
|
+
agentId;
|
|
2105
|
+
constructor(agentId2) {
|
|
2106
|
+
this.agentId = agentId2;
|
|
2107
|
+
}
|
|
2108
|
+
traceFromResponse(response, options) {
|
|
2109
|
+
const builder = new TraceBuilder(this.agentId);
|
|
2110
|
+
if (options?.inputMessages) {
|
|
2111
|
+
builder.setInput({ messages: options.inputMessages });
|
|
2112
|
+
}
|
|
2113
|
+
const completionParts = [];
|
|
2114
|
+
for (const block of response.content) {
|
|
2115
|
+
if (block.type === "text") {
|
|
2116
|
+
completionParts.push(block.text);
|
|
2117
|
+
} else if (block.type === "tool_use") {
|
|
2118
|
+
builder.addToolCall(block.name, {
|
|
2119
|
+
args: block.input
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
const completionText = completionParts.join("\n");
|
|
2124
|
+
const stepArgs = {};
|
|
2125
|
+
if (response.model !== void 0) stepArgs.model = response.model;
|
|
2126
|
+
builder.addLlmCall("completion", { args: stepArgs, result: { completion: completionText } });
|
|
2127
|
+
builder.setOutput({ message: completionText });
|
|
2128
|
+
let totalTokens;
|
|
2129
|
+
if (response.usage) {
|
|
2130
|
+
totalTokens = response.usage.input_tokens + response.usage.output_tokens;
|
|
2131
|
+
}
|
|
2132
|
+
builder.setMetadata({
|
|
2133
|
+
total_tokens: totalTokens,
|
|
2134
|
+
cost_usd: options?.costUsd,
|
|
2135
|
+
latency_ms: options?.latencyMs,
|
|
2136
|
+
model: response.model
|
|
2137
|
+
});
|
|
2138
|
+
return builder.build();
|
|
2139
|
+
}
|
|
2140
|
+
};
|
|
2141
|
+
|
|
2142
|
+
// src/adapters/gemini.ts
|
|
2143
|
+
var GeminiAdapter = class {
|
|
2144
|
+
agentId;
|
|
2145
|
+
constructor(agentId2) {
|
|
2146
|
+
this.agentId = agentId2;
|
|
2147
|
+
}
|
|
2148
|
+
traceFromResponse(response, options) {
|
|
2149
|
+
const builder = new TraceBuilder(this.agentId);
|
|
2150
|
+
if (options?.inputText) {
|
|
2151
|
+
builder.setInput({ text: options.inputText });
|
|
2152
|
+
}
|
|
2153
|
+
let completionText = "";
|
|
2154
|
+
if (response.text != null) {
|
|
2155
|
+
completionText = response.text;
|
|
2156
|
+
} else if (response.candidates && response.candidates.length > 0) {
|
|
2157
|
+
completionText = response.candidates[0].content.parts.filter((p) => p.text != null).map((p) => p.text).join("");
|
|
2158
|
+
}
|
|
2159
|
+
if (response.candidates && response.candidates.length > 0) {
|
|
2160
|
+
for (const part of response.candidates[0].content.parts) {
|
|
2161
|
+
if (part.functionCall) {
|
|
2162
|
+
builder.addToolCall(part.functionCall.name, {
|
|
2163
|
+
args: part.functionCall.args ?? {}
|
|
2164
|
+
});
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
builder.addLlmCall("completion", { result: { completion: completionText } });
|
|
2169
|
+
builder.setOutput({ message: completionText });
|
|
2170
|
+
builder.setMetadata({
|
|
2171
|
+
cost_usd: options?.costUsd,
|
|
2172
|
+
latency_ms: options?.latencyMs,
|
|
2173
|
+
model: options?.model
|
|
2174
|
+
});
|
|
2175
|
+
return builder.build();
|
|
2176
|
+
}
|
|
2177
|
+
};
|
|
2178
|
+
|
|
2179
|
+
// src/adapters/ollama.ts
|
|
2180
|
+
var OllamaAdapter = class {
|
|
2181
|
+
agentId;
|
|
2182
|
+
constructor(agentId2) {
|
|
2183
|
+
this.agentId = agentId2;
|
|
2184
|
+
}
|
|
2185
|
+
traceFromResponse(response, options) {
|
|
2186
|
+
const builder = new TraceBuilder(this.agentId);
|
|
2187
|
+
if (options?.inputMessages) {
|
|
2188
|
+
builder.setInput({ messages: options.inputMessages });
|
|
2189
|
+
}
|
|
2190
|
+
const completionText = response.message?.content ?? "";
|
|
2191
|
+
builder.addLlmCall("completion", {
|
|
2192
|
+
args: { model: response.model ?? "" },
|
|
2193
|
+
result: { completion: completionText }
|
|
2194
|
+
});
|
|
2195
|
+
builder.setOutput({ message: completionText });
|
|
2196
|
+
let totalTokens;
|
|
2197
|
+
if (response.eval_count !== void 0 && response.prompt_eval_count !== void 0) {
|
|
2198
|
+
totalTokens = response.eval_count + response.prompt_eval_count;
|
|
2199
|
+
}
|
|
2200
|
+
builder.setMetadata({
|
|
2201
|
+
total_tokens: totalTokens,
|
|
2202
|
+
latency_ms: options?.latencyMs,
|
|
2203
|
+
model: response.model
|
|
2204
|
+
});
|
|
2205
|
+
return builder.build();
|
|
2206
|
+
}
|
|
2207
|
+
};
|
|
2208
|
+
|
|
2209
|
+
// src/adapters/otel.ts
|
|
2210
|
+
var OTelAdapter = class _OTelAdapter {
|
|
2211
|
+
agentId;
|
|
2212
|
+
constructor(agentId2) {
|
|
2213
|
+
this.agentId = agentId2;
|
|
2214
|
+
}
|
|
2215
|
+
static fromSpans(spans, agentId2) {
|
|
2216
|
+
const adapter = new _OTelAdapter(agentId2);
|
|
2217
|
+
return adapter.buildTrace(spans);
|
|
2218
|
+
}
|
|
2219
|
+
buildTrace(spans) {
|
|
2220
|
+
const sorted = [...spans].sort((a, b) => {
|
|
2221
|
+
const aTime = a.startTime[0] * 1e9 + a.startTime[1];
|
|
2222
|
+
const bTime = b.startTime[0] * 1e9 + b.startTime[1];
|
|
2223
|
+
return aTime - bTime;
|
|
2224
|
+
});
|
|
2225
|
+
const builder = new TraceBuilder(this.agentId);
|
|
2226
|
+
const rootSpan = this.findRootSpan(sorted);
|
|
2227
|
+
if (rootSpan) {
|
|
2228
|
+
const traceId2 = rootSpan.spanContext().traceId;
|
|
2229
|
+
if (traceId2) {
|
|
2230
|
+
builder.setTraceId(`otel_${traceId2.slice(0, 16)}`);
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
2233
|
+
let outputMessage = "";
|
|
2234
|
+
let totalTokens;
|
|
2235
|
+
let latencyMs;
|
|
2236
|
+
let model;
|
|
2237
|
+
for (const span of sorted) {
|
|
2238
|
+
const attrs = span.attributes ?? {};
|
|
2239
|
+
const stepType = this.classifySpan(attrs, span.name);
|
|
2240
|
+
if (stepType === "llm_call") {
|
|
2241
|
+
const [stepArgs, stepResult] = this.extractLlmStep(attrs);
|
|
2242
|
+
builder.addLlmCall(span.name, {
|
|
2243
|
+
args: stepArgs,
|
|
2244
|
+
result: stepResult,
|
|
2245
|
+
metadata: this.spanMetadata(span)
|
|
2246
|
+
});
|
|
2247
|
+
const completion = attrs["gen_ai.completion"];
|
|
2248
|
+
if (completion) outputMessage = String(completion);
|
|
2249
|
+
const inputTokens = Number(attrs["gen_ai.usage.input_tokens"] ?? 0);
|
|
2250
|
+
const outputTokens = Number(attrs["gen_ai.usage.output_tokens"] ?? 0);
|
|
2251
|
+
const spanTokens = inputTokens + outputTokens;
|
|
2252
|
+
if (spanTokens > 0) totalTokens = (totalTokens ?? 0) + spanTokens;
|
|
2253
|
+
if (model === void 0) {
|
|
2254
|
+
const responseModel = attrs["gen_ai.response.model"];
|
|
2255
|
+
const requestModel = attrs["gen_ai.request.model"];
|
|
2256
|
+
if (responseModel) model = String(responseModel);
|
|
2257
|
+
else if (requestModel) model = String(requestModel);
|
|
2258
|
+
}
|
|
2259
|
+
} else if (stepType === "tool_call") {
|
|
2260
|
+
const [stepArgs, stepResult] = this.extractToolStep(attrs);
|
|
2261
|
+
const toolName = String(attrs["gen_ai.tool.name"] ?? span.name);
|
|
2262
|
+
builder.addToolCall(toolName, {
|
|
2263
|
+
args: stepArgs,
|
|
2264
|
+
result: stepResult,
|
|
2265
|
+
metadata: this.spanMetadata(span)
|
|
2266
|
+
});
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
if (rootSpan) {
|
|
2270
|
+
const startNs = rootSpan.startTime[0] * 1e9 + rootSpan.startTime[1];
|
|
2271
|
+
const endNs = rootSpan.endTime[0] * 1e9 + rootSpan.endTime[1];
|
|
2272
|
+
latencyMs = Math.floor((endNs - startNs) / 1e6);
|
|
2273
|
+
}
|
|
2274
|
+
builder.setOutput({ message: outputMessage });
|
|
2275
|
+
builder.setMetadata({ total_tokens: totalTokens, latency_ms: latencyMs, model });
|
|
2276
|
+
return builder.build();
|
|
2277
|
+
}
|
|
2278
|
+
findRootSpan(spans) {
|
|
2279
|
+
for (const span of spans) {
|
|
2280
|
+
if (!span.parentSpanId) return span;
|
|
2281
|
+
}
|
|
2282
|
+
return spans[0];
|
|
2283
|
+
}
|
|
2284
|
+
classifySpan(attrs, name) {
|
|
2285
|
+
const op = String(attrs["gen_ai.operation.name"] ?? "");
|
|
2286
|
+
if (op === "chat" || op === "completion" || op === "generate_content" || "gen_ai.completion" in attrs) {
|
|
2287
|
+
return "llm_call";
|
|
2288
|
+
}
|
|
2289
|
+
if (op === "tool" || "gen_ai.tool.name" in attrs) {
|
|
2290
|
+
return "tool_call";
|
|
2291
|
+
}
|
|
2292
|
+
const lower = name.toLowerCase();
|
|
2293
|
+
if (lower.includes("completion") || lower.includes("chat")) return "llm_call";
|
|
2294
|
+
if (lower.includes("tool")) return "tool_call";
|
|
2295
|
+
return void 0;
|
|
2296
|
+
}
|
|
2297
|
+
extractLlmStep(attrs) {
|
|
2298
|
+
const args = {};
|
|
2299
|
+
const result = {};
|
|
2300
|
+
if ("gen_ai.request.model" in attrs) args.model = String(attrs["gen_ai.request.model"]);
|
|
2301
|
+
if ("gen_ai.system" in attrs) args.system = String(attrs["gen_ai.system"]);
|
|
2302
|
+
if ("gen_ai.prompt" in attrs) args.prompt = String(attrs["gen_ai.prompt"]);
|
|
2303
|
+
if ("gen_ai.completion" in attrs) result.completion = String(attrs["gen_ai.completion"]);
|
|
2304
|
+
if ("gen_ai.usage.input_tokens" in attrs) result.input_tokens = Number(attrs["gen_ai.usage.input_tokens"]);
|
|
2305
|
+
if ("gen_ai.usage.output_tokens" in attrs) result.output_tokens = Number(attrs["gen_ai.usage.output_tokens"]);
|
|
2306
|
+
if ("gen_ai.response.model" in attrs) result.model = String(attrs["gen_ai.response.model"]);
|
|
2307
|
+
return [args, result];
|
|
2308
|
+
}
|
|
2309
|
+
extractToolStep(attrs) {
|
|
2310
|
+
const args = {};
|
|
2311
|
+
const result = {};
|
|
2312
|
+
if ("gen_ai.tool.call.id" in attrs) args.call_id = String(attrs["gen_ai.tool.call.id"]);
|
|
2313
|
+
if ("gen_ai.tool.parameters" in attrs) args.parameters = attrs["gen_ai.tool.parameters"];
|
|
2314
|
+
if ("gen_ai.tool.output" in attrs) result.output = attrs["gen_ai.tool.output"];
|
|
2315
|
+
return [args, result];
|
|
2316
|
+
}
|
|
2317
|
+
spanMetadata(span) {
|
|
2318
|
+
const startNs = span.startTime[0] * 1e9 + span.startTime[1];
|
|
2319
|
+
const endNs = span.endTime[0] * 1e9 + span.endTime[1];
|
|
2320
|
+
return { duration_ms: Math.floor((endNs - startNs) / 1e6) };
|
|
2321
|
+
}
|
|
2322
|
+
};
|
|
2323
|
+
|
|
2324
|
+
// src/adapters/langchain.ts
|
|
2325
|
+
var LangChainAdapter = class {
|
|
2326
|
+
agentId;
|
|
2327
|
+
llmCalls = [];
|
|
2328
|
+
toolCalls = [];
|
|
2329
|
+
chainOutput = "";
|
|
2330
|
+
startTime = 0;
|
|
2331
|
+
constructor(agentId2) {
|
|
2332
|
+
this.agentId = agentId2;
|
|
2333
|
+
}
|
|
2334
|
+
/**
|
|
2335
|
+
* Reset accumulated state for a new trace capture session.
|
|
2336
|
+
*/
|
|
2337
|
+
reset() {
|
|
2338
|
+
this.llmCalls.length = 0;
|
|
2339
|
+
this.toolCalls.length = 0;
|
|
2340
|
+
this.chainOutput = "";
|
|
2341
|
+
this.startTime = 0;
|
|
2342
|
+
}
|
|
2343
|
+
/**
|
|
2344
|
+
* Call this when a chain starts (maps to `handleChainStart`).
|
|
2345
|
+
*/
|
|
2346
|
+
handleChainStart() {
|
|
2347
|
+
if (this.startTime === 0) {
|
|
2348
|
+
this.startTime = Date.now();
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
/**
|
|
2352
|
+
* Call this when an LLM call completes (maps to `handleLLMEnd`).
|
|
2353
|
+
*/
|
|
2354
|
+
handleLLMEnd(output) {
|
|
2355
|
+
const text = output.generations[0]?.[0]?.text ?? "";
|
|
2356
|
+
const tokens = output.llmOutput?.["tokenUsage"];
|
|
2357
|
+
const model = output.llmOutput?.["modelName"];
|
|
2358
|
+
this.llmCalls.push({
|
|
2359
|
+
name: "completion",
|
|
2360
|
+
text,
|
|
2361
|
+
tokens: tokens?.totalTokens,
|
|
2362
|
+
model
|
|
2363
|
+
});
|
|
2364
|
+
}
|
|
2365
|
+
/**
|
|
2366
|
+
* Call this when a tool call completes (maps to `handleToolEnd`).
|
|
2367
|
+
*/
|
|
2368
|
+
handleToolEnd(output, metadata) {
|
|
2369
|
+
this.toolCalls.push({
|
|
2370
|
+
name: metadata?.name ?? "unknown_tool",
|
|
2371
|
+
output
|
|
2372
|
+
});
|
|
2373
|
+
}
|
|
2374
|
+
/**
|
|
2375
|
+
* Call this when a chain completes (maps to `handleChainEnd`).
|
|
2376
|
+
*/
|
|
2377
|
+
handleChainEnd(output) {
|
|
2378
|
+
const text = output["output"] ?? output["text"] ?? output["result"];
|
|
2379
|
+
if (typeof text === "string") {
|
|
2380
|
+
this.chainOutput = text;
|
|
2381
|
+
}
|
|
2382
|
+
}
|
|
2383
|
+
/**
|
|
2384
|
+
* Build a Trace from accumulated callback events.
|
|
2385
|
+
*
|
|
2386
|
+
* @param options.costUsd - Total cost override.
|
|
2387
|
+
* @param options.model - Model name override.
|
|
2388
|
+
* @returns A finalized Trace object.
|
|
2389
|
+
*/
|
|
2390
|
+
buildTrace(options) {
|
|
2391
|
+
const builder = new TraceBuilder(this.agentId);
|
|
2392
|
+
const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
|
|
2393
|
+
let totalTokens;
|
|
2394
|
+
for (const llm of this.llmCalls) {
|
|
2395
|
+
const args = {};
|
|
2396
|
+
if (llm.model ?? options?.model) args.model = llm.model ?? options?.model;
|
|
2397
|
+
const result = { completion: llm.text };
|
|
2398
|
+
if (llm.tokens !== void 0) {
|
|
2399
|
+
result.tokens = llm.tokens;
|
|
2400
|
+
totalTokens = (totalTokens ?? 0) + llm.tokens;
|
|
2401
|
+
}
|
|
2402
|
+
builder.addLlmCall(llm.name, { args, result });
|
|
2403
|
+
}
|
|
2404
|
+
for (const tool of this.toolCalls) {
|
|
2405
|
+
builder.addToolCall(tool.name, {
|
|
2406
|
+
result: { output: tool.output }
|
|
2407
|
+
});
|
|
2408
|
+
}
|
|
2409
|
+
const outputMessage = this.chainOutput || (this.llmCalls.length > 0 ? this.llmCalls[this.llmCalls.length - 1].text : "");
|
|
2410
|
+
builder.setOutput({ message: outputMessage });
|
|
2411
|
+
builder.setMetadata({
|
|
2412
|
+
total_tokens: totalTokens,
|
|
2413
|
+
cost_usd: options?.costUsd,
|
|
2414
|
+
latency_ms: latencyMs,
|
|
2415
|
+
model: options?.model ?? this.llmCalls[0]?.model
|
|
2416
|
+
});
|
|
2417
|
+
return builder.build();
|
|
2418
|
+
}
|
|
2419
|
+
};
|
|
2420
|
+
export {
|
|
2421
|
+
ADVERSARIAL_USER,
|
|
2422
|
+
Agent,
|
|
2423
|
+
AgentResult,
|
|
2424
|
+
AlertDispatcher,
|
|
2425
|
+
AnthropicAdapter,
|
|
2426
|
+
AttestClient,
|
|
2427
|
+
CONFUSED_USER,
|
|
2428
|
+
COOPERATIVE_USER,
|
|
2429
|
+
ContinuousEvalRunner,
|
|
2430
|
+
ENGINE_VERSION,
|
|
2431
|
+
ERR_ASSERTION_ERROR,
|
|
2432
|
+
ERR_ENGINE_ERROR,
|
|
2433
|
+
ERR_INVALID_TRACE,
|
|
2434
|
+
ERR_PROVIDER_ERROR,
|
|
2435
|
+
ERR_SESSION_ERROR,
|
|
2436
|
+
ERR_TIMEOUT,
|
|
2437
|
+
EngineManager,
|
|
2438
|
+
EngineTimeoutError,
|
|
2439
|
+
ExpectChain,
|
|
2440
|
+
FRIENDLY_USER,
|
|
2441
|
+
GeminiAdapter,
|
|
2442
|
+
LangChainAdapter,
|
|
2443
|
+
ManualAdapter,
|
|
2444
|
+
OTelAdapter,
|
|
2445
|
+
OllamaAdapter,
|
|
2446
|
+
OpenAIAdapter,
|
|
2447
|
+
PluginRegistry,
|
|
2448
|
+
ProtocolError,
|
|
2449
|
+
STATUS_HARD_FAIL,
|
|
2450
|
+
STATUS_PASS,
|
|
2451
|
+
STATUS_SOFT_FAIL,
|
|
2452
|
+
STEP_AGENT_CALL,
|
|
2453
|
+
STEP_LLM_CALL,
|
|
2454
|
+
STEP_RETRIEVAL,
|
|
2455
|
+
STEP_TOOL_CALL,
|
|
2456
|
+
Sampler,
|
|
2457
|
+
TIER_1,
|
|
2458
|
+
TIER_2,
|
|
2459
|
+
TIER_3,
|
|
2460
|
+
TYPE_CONSTRAINT,
|
|
2461
|
+
TYPE_CONTENT,
|
|
2462
|
+
TYPE_EMBEDDING,
|
|
2463
|
+
TYPE_LLM_JUDGE,
|
|
2464
|
+
TYPE_PLUGIN,
|
|
2465
|
+
TYPE_SCHEMA,
|
|
2466
|
+
TYPE_TRACE,
|
|
2467
|
+
TYPE_TRACE_TREE,
|
|
2468
|
+
TraceBuilder,
|
|
2469
|
+
TraceTree,
|
|
2470
|
+
VERSION,
|
|
2471
|
+
activeBuilder,
|
|
2472
|
+
agent,
|
|
2473
|
+
agentId,
|
|
2474
|
+
assertionId,
|
|
2475
|
+
attestExpect,
|
|
2476
|
+
config,
|
|
2477
|
+
decodeResponse,
|
|
2478
|
+
delegate,
|
|
2479
|
+
encodeRequest,
|
|
2480
|
+
executePluginAssertion,
|
|
2481
|
+
extractId,
|
|
2482
|
+
extractResult,
|
|
2483
|
+
getSampleRate,
|
|
2484
|
+
isSimulationMode,
|
|
2485
|
+
resetConfig,
|
|
2486
|
+
simulationEvaluateBatch,
|
|
2487
|
+
tier,
|
|
2488
|
+
traceId
|
|
2489
|
+
};
|
|
2490
|
+
//# sourceMappingURL=index.js.map
|