@attest-ai/core 1.0.0-rc.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +359 -33
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +154 -6
- package/dist/esm/cli.js +12 -3
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +154 -6
- package/dist/esm/index.js +349 -33
- package/dist/esm/index.js.map +1 -1
- package/package.json +1 -1
package/dist/cjs/index.cjs
CHANGED
|
@@ -56,6 +56,7 @@ __export(index_exports, {
|
|
|
56
56
|
GeminiAdapter: () => GeminiAdapter,
|
|
57
57
|
LangChainAdapter: () => LangChainAdapter,
|
|
58
58
|
ManualAdapter: () => ManualAdapter,
|
|
59
|
+
MockToolRegistry: () => MockToolRegistry,
|
|
59
60
|
OTLPConversionError: () => OTLPConversionError,
|
|
60
61
|
OTelAdapter: () => OTelAdapter,
|
|
61
62
|
OUTCOME_FAIL: () => OUTCOME_FAIL,
|
|
@@ -75,6 +76,7 @@ __export(index_exports, {
|
|
|
75
76
|
REASON_REPLAY_MISS: () => REASON_REPLAY_MISS,
|
|
76
77
|
REASON_SHORT_CIRCUITED: () => REASON_SHORT_CIRCUITED,
|
|
77
78
|
REASON_SIMULATION: () => REASON_SIMULATION,
|
|
79
|
+
RepeatResult: () => RepeatResult,
|
|
78
80
|
ReplayError: () => ReplayError,
|
|
79
81
|
ReplayMissError: () => ReplayMissError,
|
|
80
82
|
ReplayNotEvaluatedError: () => ReplayNotEvaluatedError,
|
|
@@ -87,6 +89,7 @@ __export(index_exports, {
|
|
|
87
89
|
STEP_RETRIEVAL: () => STEP_RETRIEVAL,
|
|
88
90
|
STEP_TOOL_CALL: () => STEP_TOOL_CALL,
|
|
89
91
|
Sampler: () => Sampler,
|
|
92
|
+
ScenarioResult: () => ScenarioResult,
|
|
90
93
|
TIER_1: () => TIER_1,
|
|
91
94
|
TIER_2: () => TIER_2,
|
|
92
95
|
TIER_3: () => TIER_3,
|
|
@@ -121,6 +124,7 @@ __export(index_exports, {
|
|
|
121
124
|
executePluginAssertion: () => executePluginAssertion,
|
|
122
125
|
extractId: () => extractId,
|
|
123
126
|
extractResult: () => extractResult,
|
|
127
|
+
faultInject: () => faultInject,
|
|
124
128
|
firstMiss: () => firstMiss,
|
|
125
129
|
formatUsage: () => formatUsage,
|
|
126
130
|
getSampleRate: () => getSampleRate,
|
|
@@ -132,17 +136,23 @@ __export(index_exports, {
|
|
|
132
136
|
loadFixture: () => loadFixture,
|
|
133
137
|
loadLabelsCSV: () => loadLabelsCSV,
|
|
134
138
|
loadLabelsJSONL: () => loadLabelsJSONL,
|
|
139
|
+
loadPackagePlugins: () => loadPackagePlugins,
|
|
140
|
+
mockTool: () => mockTool,
|
|
135
141
|
parseUsage: () => parseUsage,
|
|
136
142
|
previewLine: () => previewLine,
|
|
137
143
|
promptHash: () => promptHash,
|
|
138
144
|
record: () => record,
|
|
145
|
+
registerPlugin: () => registerPlugin,
|
|
139
146
|
renderAssertionFailure: () => renderAssertionFailure,
|
|
140
147
|
renderDiagnostics: () => renderDiagnostics,
|
|
141
148
|
renderSummary: () => renderSummary,
|
|
149
|
+
renderVerdictDiagnostics: () => renderVerdictDiagnostics,
|
|
142
150
|
renderVitest: () => renderVitest,
|
|
151
|
+
repeat: () => repeat,
|
|
143
152
|
requireSimulationCapability: () => requireSimulationCapability,
|
|
144
153
|
resetConfig: () => resetConfig,
|
|
145
154
|
resolveMode: () => resolveMode,
|
|
155
|
+
scenario: () => scenario,
|
|
146
156
|
simulationEnvEnabled: () => simulationEnvEnabled,
|
|
147
157
|
startReplay: () => startReplay,
|
|
148
158
|
startWorld: () => startWorld,
|
|
@@ -377,8 +387,8 @@ function extractId(response) {
|
|
|
377
387
|
}
|
|
378
388
|
|
|
379
389
|
// src/version.ts
|
|
380
|
-
var VERSION = "1.0.0
|
|
381
|
-
var ENGINE_VERSION = "1.0.0
|
|
390
|
+
var VERSION = "1.0.0";
|
|
391
|
+
var ENGINE_VERSION = "1.0.0";
|
|
382
392
|
|
|
383
393
|
// src/engine-manager.ts
|
|
384
394
|
var import_node_child_process = require("child_process");
|
|
@@ -446,7 +456,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
446
456
|
new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
|
|
447
457
|
);
|
|
448
458
|
}
|
|
449
|
-
return new Promise((
|
|
459
|
+
return new Promise((resolve5, reject) => {
|
|
450
460
|
const client = url.startsWith("https:") ? https : http;
|
|
451
461
|
client.get(url, (res) => {
|
|
452
462
|
const status = res.statusCode ?? 0;
|
|
@@ -457,7 +467,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
457
467
|
return;
|
|
458
468
|
}
|
|
459
469
|
res.resume();
|
|
460
|
-
|
|
470
|
+
resolve5(httpsGet(location, redirectCount + 1));
|
|
461
471
|
return;
|
|
462
472
|
}
|
|
463
473
|
if (status < 200 || status >= 300) {
|
|
@@ -467,7 +477,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
467
477
|
}
|
|
468
478
|
const chunks = [];
|
|
469
479
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
470
|
-
res.on("end", () =>
|
|
480
|
+
res.on("end", () => resolve5(Buffer.concat(chunks)));
|
|
471
481
|
res.on("error", reject);
|
|
472
482
|
}).on("error", reject);
|
|
473
483
|
});
|
|
@@ -694,11 +704,11 @@ var EngineManager = class {
|
|
|
694
704
|
}
|
|
695
705
|
this.requestId += 1;
|
|
696
706
|
const requestStr = encodeRequest(this.requestId, method, params);
|
|
697
|
-
return new Promise((
|
|
707
|
+
return new Promise((resolve5, reject) => {
|
|
698
708
|
const onLine = (line) => {
|
|
699
709
|
try {
|
|
700
710
|
const response = decodeResponse(line);
|
|
701
|
-
|
|
711
|
+
resolve5(extractResult(response));
|
|
702
712
|
} catch (err) {
|
|
703
713
|
reject(err);
|
|
704
714
|
}
|
|
@@ -1065,12 +1075,12 @@ var AttestClient = class {
|
|
|
1065
1075
|
throw new Error("Engine process not started.");
|
|
1066
1076
|
}
|
|
1067
1077
|
let capturedReqId;
|
|
1068
|
-
const requestPromise = new Promise((
|
|
1078
|
+
const requestPromise = new Promise((resolve5, reject) => {
|
|
1069
1079
|
this.writeChain = this.writeChain.then(() => {
|
|
1070
1080
|
this.requestId += 1;
|
|
1071
1081
|
const reqId = this.requestId;
|
|
1072
1082
|
capturedReqId = reqId;
|
|
1073
|
-
this.pending.set(reqId, { resolve:
|
|
1083
|
+
this.pending.set(reqId, { resolve: resolve5, reject });
|
|
1074
1084
|
const requestStr = encodeRequest(reqId, method, params);
|
|
1075
1085
|
cp.stdin.write(requestStr, (err) => {
|
|
1076
1086
|
if (err) {
|
|
@@ -1185,6 +1195,26 @@ function requireSimulationCapability(init) {
|
|
|
1185
1195
|
|
|
1186
1196
|
// src/trace.ts
|
|
1187
1197
|
var import_node_crypto = require("crypto");
|
|
1198
|
+
|
|
1199
|
+
// src/simulation/context.ts
|
|
1200
|
+
var import_node_async_hooks = require("async_hooks");
|
|
1201
|
+
var activeMockRegistry = new import_node_async_hooks.AsyncLocalStorage();
|
|
1202
|
+
function createRng(seed) {
|
|
1203
|
+
if (seed === void 0) return Math.random;
|
|
1204
|
+
let state = seed >>> 0;
|
|
1205
|
+
return () => {
|
|
1206
|
+
state = state + 1831565813 >>> 0;
|
|
1207
|
+
let t = state;
|
|
1208
|
+
t = Math.imul(t ^ t >>> 15, t | 1);
|
|
1209
|
+
t ^= t + Math.imul(t ^ t >>> 7, t | 61);
|
|
1210
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
1211
|
+
};
|
|
1212
|
+
}
|
|
1213
|
+
function sleep(ms) {
|
|
1214
|
+
return new Promise((resolve5) => setTimeout(resolve5, ms));
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
// src/trace.ts
|
|
1188
1218
|
var TraceBuilder = class {
|
|
1189
1219
|
traceId;
|
|
1190
1220
|
agentId;
|
|
@@ -1216,11 +1246,22 @@ var TraceBuilder = class {
|
|
|
1216
1246
|
return this;
|
|
1217
1247
|
}
|
|
1218
1248
|
addToolCall(name, options2) {
|
|
1249
|
+
const mock = activeMockRegistry.getStore()?.get(name);
|
|
1250
|
+
let result = options2?.result;
|
|
1251
|
+
if (mock !== void 0) {
|
|
1252
|
+
const mocked = mock(options2?.args ?? {});
|
|
1253
|
+
if (typeof mocked?.then === "function") {
|
|
1254
|
+
throw new Error(`mock for tool '${name}' returned a Promise; mock tools must be synchronous`);
|
|
1255
|
+
}
|
|
1256
|
+
if (typeof mocked === "object" && mocked !== null && !Array.isArray(mocked)) {
|
|
1257
|
+
result = mocked;
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1219
1260
|
this.steps.push({
|
|
1220
1261
|
type: STEP_TOOL_CALL,
|
|
1221
1262
|
name,
|
|
1222
1263
|
args: options2?.args,
|
|
1223
|
-
result
|
|
1264
|
+
result,
|
|
1224
1265
|
metadata: options2?.metadata
|
|
1225
1266
|
});
|
|
1226
1267
|
return this;
|
|
@@ -2875,18 +2916,18 @@ function serve(engine, args) {
|
|
|
2875
2916
|
let stdout = "";
|
|
2876
2917
|
let stderr = "";
|
|
2877
2918
|
let resolveReady = () => void 0;
|
|
2878
|
-
const ready = new Promise((
|
|
2879
|
-
resolveReady =
|
|
2919
|
+
const ready = new Promise((resolve5) => {
|
|
2920
|
+
resolveReady = resolve5;
|
|
2880
2921
|
});
|
|
2881
|
-
const exited = new Promise((
|
|
2922
|
+
const exited = new Promise((resolve5) => {
|
|
2882
2923
|
child.on("close", (code, signal) => {
|
|
2883
2924
|
resolveReady(null);
|
|
2884
|
-
|
|
2925
|
+
resolve5({ code, signal });
|
|
2885
2926
|
});
|
|
2886
2927
|
child.on("error", (err) => {
|
|
2887
2928
|
stderr += err.message;
|
|
2888
2929
|
resolveReady(null);
|
|
2889
|
-
|
|
2930
|
+
resolve5({ code: null, signal: null });
|
|
2890
2931
|
});
|
|
2891
2932
|
});
|
|
2892
2933
|
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
@@ -2978,11 +3019,14 @@ async function recordExclusive(name, fn, opts) {
|
|
|
2978
3019
|
throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
|
|
2979
3020
|
}
|
|
2980
3021
|
const outcome = JSON.parse(lines[lines.length - 1] ?? "");
|
|
2981
|
-
if (exit.code !== 0 ||
|
|
3022
|
+
if (exit.code !== 0 || outcome.outcome === "not_verified") {
|
|
2982
3023
|
throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
|
|
2983
3024
|
}
|
|
3025
|
+
if (outcome.outcome === "nothing_to_verify") {
|
|
3026
|
+
return { value, fixture: outcome.fixture, verified: false, outcome: "nothing_to_verify", testFile: null };
|
|
3027
|
+
}
|
|
2984
3028
|
const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
|
|
2985
|
-
return { value, fixture: outcome.fixture, verified: true, testFile };
|
|
3029
|
+
return { value, fixture: outcome.fixture, verified: true, outcome: "verified", testFile };
|
|
2986
3030
|
}
|
|
2987
3031
|
|
|
2988
3032
|
// src/replay.ts
|
|
@@ -3143,18 +3187,18 @@ async function start(name, mode, opts) {
|
|
|
3143
3187
|
let stdout = "";
|
|
3144
3188
|
let stderr = "";
|
|
3145
3189
|
let resolveReady = () => void 0;
|
|
3146
|
-
const ready = new Promise((
|
|
3147
|
-
resolveReady =
|
|
3190
|
+
const ready = new Promise((resolve5) => {
|
|
3191
|
+
resolveReady = resolve5;
|
|
3148
3192
|
});
|
|
3149
|
-
const exited = new Promise((
|
|
3193
|
+
const exited = new Promise((resolve5) => {
|
|
3150
3194
|
child.on("close", (code) => {
|
|
3151
3195
|
resolveReady(null);
|
|
3152
|
-
|
|
3196
|
+
resolve5(code);
|
|
3153
3197
|
});
|
|
3154
3198
|
child.on("error", (err) => {
|
|
3155
3199
|
stderr += err.message;
|
|
3156
3200
|
resolveReady(null);
|
|
3157
|
-
|
|
3201
|
+
resolve5(null);
|
|
3158
3202
|
});
|
|
3159
3203
|
});
|
|
3160
3204
|
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
@@ -3279,18 +3323,18 @@ async function start2(file, opts) {
|
|
|
3279
3323
|
let stdout = "";
|
|
3280
3324
|
let stderr = "";
|
|
3281
3325
|
let resolveReady = () => void 0;
|
|
3282
|
-
const ready = new Promise((
|
|
3283
|
-
resolveReady =
|
|
3326
|
+
const ready = new Promise((resolve5) => {
|
|
3327
|
+
resolveReady = resolve5;
|
|
3284
3328
|
});
|
|
3285
|
-
const exited = new Promise((
|
|
3329
|
+
const exited = new Promise((resolve5) => {
|
|
3286
3330
|
child.on("close", (code) => {
|
|
3287
3331
|
resolveReady(null);
|
|
3288
|
-
|
|
3332
|
+
resolve5(code);
|
|
3289
3333
|
});
|
|
3290
3334
|
child.on("error", (err) => {
|
|
3291
3335
|
stderr += err.message;
|
|
3292
3336
|
resolveReady(null);
|
|
3293
|
-
|
|
3337
|
+
resolve5(null);
|
|
3294
3338
|
});
|
|
3295
3339
|
});
|
|
3296
3340
|
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
@@ -3371,8 +3415,8 @@ function parseFinal(out) {
|
|
|
3371
3415
|
}
|
|
3372
3416
|
|
|
3373
3417
|
// src/context.ts
|
|
3374
|
-
var
|
|
3375
|
-
var activeBuilder = new
|
|
3418
|
+
var import_node_async_hooks2 = require("async_hooks");
|
|
3419
|
+
var activeBuilder = new import_node_async_hooks2.AsyncLocalStorage();
|
|
3376
3420
|
|
|
3377
3421
|
// src/agent.ts
|
|
3378
3422
|
var Agent = class {
|
|
@@ -3459,6 +3503,9 @@ function tier(level) {
|
|
|
3459
3503
|
}
|
|
3460
3504
|
|
|
3461
3505
|
// src/plugins.ts
|
|
3506
|
+
var import_node_fs2 = require("fs");
|
|
3507
|
+
var import_node_path2 = require("path");
|
|
3508
|
+
var import_node_url2 = require("url");
|
|
3462
3509
|
var PluginRegistry = class {
|
|
3463
3510
|
plugins = /* @__PURE__ */ new Map();
|
|
3464
3511
|
register(name, plugin) {
|
|
@@ -3489,10 +3536,19 @@ var PluginRegistry = class {
|
|
|
3489
3536
|
};
|
|
3490
3537
|
async function executePluginAssertion(plugin, trace, spec, client, traceId2, assertionId2, timeoutMs = 3e4) {
|
|
3491
3538
|
const resultPromise = Promise.resolve(plugin.execute(trace, spec));
|
|
3539
|
+
let timer;
|
|
3492
3540
|
const timeoutPromise = new Promise((_, reject) => {
|
|
3493
|
-
|
|
3541
|
+
timer = setTimeout(
|
|
3542
|
+
() => reject(new Error(`Plugin '${plugin.name}' timed out after ${timeoutMs}ms`)),
|
|
3543
|
+
timeoutMs
|
|
3544
|
+
);
|
|
3494
3545
|
});
|
|
3495
|
-
|
|
3546
|
+
let result;
|
|
3547
|
+
try {
|
|
3548
|
+
result = await Promise.race([resultPromise, timeoutPromise]);
|
|
3549
|
+
} finally {
|
|
3550
|
+
clearTimeout(timer);
|
|
3551
|
+
}
|
|
3496
3552
|
await client.submitPluginResult(
|
|
3497
3553
|
traceId2,
|
|
3498
3554
|
plugin.name,
|
|
@@ -3503,6 +3559,100 @@ async function executePluginAssertion(plugin, trace, spec, client, traceId2, ass
|
|
|
3503
3559
|
);
|
|
3504
3560
|
return result;
|
|
3505
3561
|
}
|
|
3562
|
+
function registerPlugin(registry, name, plugin) {
|
|
3563
|
+
registry.register(name, plugin);
|
|
3564
|
+
}
|
|
3565
|
+
function isAttestPlugin(value) {
|
|
3566
|
+
if (typeof value !== "object" || value === null) return false;
|
|
3567
|
+
const v = value;
|
|
3568
|
+
return typeof v["name"] === "string" && typeof v["pluginType"] === "string" && typeof v["execute"] === "function";
|
|
3569
|
+
}
|
|
3570
|
+
function readJson(path5) {
|
|
3571
|
+
return JSON.parse((0, import_node_fs2.readFileSync)(path5, "utf8"));
|
|
3572
|
+
}
|
|
3573
|
+
function findInstalledPackage(projectRoot, name) {
|
|
3574
|
+
let dir = (0, import_node_path2.resolve)(projectRoot);
|
|
3575
|
+
for (; ; ) {
|
|
3576
|
+
const candidate = (0, import_node_path2.join)(dir, "node_modules", name);
|
|
3577
|
+
if ((0, import_node_fs2.existsSync)((0, import_node_path2.join)(candidate, "package.json"))) return candidate;
|
|
3578
|
+
const parent = (0, import_node_path2.dirname)(dir);
|
|
3579
|
+
if (parent === dir) return void 0;
|
|
3580
|
+
dir = parent;
|
|
3581
|
+
}
|
|
3582
|
+
}
|
|
3583
|
+
function declaredPluginPaths(pkgJson, dep) {
|
|
3584
|
+
const attest = pkgJson["attest"];
|
|
3585
|
+
if (attest === void 0) return [];
|
|
3586
|
+
const plugins = typeof attest === "object" && attest !== null ? attest["plugins"] : void 0;
|
|
3587
|
+
if (plugins === void 0) return [];
|
|
3588
|
+
if (!Array.isArray(plugins) || !plugins.every((p) => typeof p === "string")) {
|
|
3589
|
+
throw new Error(`'${dep}': "attest.plugins" must be an array of strings`);
|
|
3590
|
+
}
|
|
3591
|
+
return plugins;
|
|
3592
|
+
}
|
|
3593
|
+
function pluginsOf(mod) {
|
|
3594
|
+
const def = mod["default"];
|
|
3595
|
+
const nested = typeof def === "object" && def !== null ? def : void 0;
|
|
3596
|
+
for (const candidate of [def, nested?.["default"]]) {
|
|
3597
|
+
if (isAttestPlugin(candidate)) return [candidate];
|
|
3598
|
+
}
|
|
3599
|
+
return [.../* @__PURE__ */ new Set([...Object.values(mod), ...Object.values(nested ?? {})])].filter(isAttestPlugin);
|
|
3600
|
+
}
|
|
3601
|
+
function isInside(root, file) {
|
|
3602
|
+
const within = (r, f) => {
|
|
3603
|
+
const rel = (0, import_node_path2.relative)(r, f);
|
|
3604
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${import_node_path2.sep}`) && !(0, import_node_path2.isAbsolute)(rel);
|
|
3605
|
+
};
|
|
3606
|
+
return within(root, file) && within((0, import_node_fs2.realpathSync)(root), (0, import_node_fs2.realpathSync)(file));
|
|
3607
|
+
}
|
|
3608
|
+
async function loadPackagePlugins(registry, projectRoot = process.cwd()) {
|
|
3609
|
+
let project;
|
|
3610
|
+
try {
|
|
3611
|
+
project = readJson((0, import_node_path2.join)(projectRoot, "package.json"));
|
|
3612
|
+
} catch (err) {
|
|
3613
|
+
console.error(`attest: cannot read ${(0, import_node_path2.join)(projectRoot, "package.json")}; no plugins loaded:`, err);
|
|
3614
|
+
return 0;
|
|
3615
|
+
}
|
|
3616
|
+
const depNames = /* @__PURE__ */ new Set();
|
|
3617
|
+
for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) {
|
|
3618
|
+
const deps = project[field];
|
|
3619
|
+
if (typeof deps === "object" && deps !== null) {
|
|
3620
|
+
for (const dep of Object.keys(deps)) depNames.add(dep);
|
|
3621
|
+
}
|
|
3622
|
+
}
|
|
3623
|
+
let loaded = 0;
|
|
3624
|
+
for (const dep of depNames) {
|
|
3625
|
+
const pkgDir = findInstalledPackage(projectRoot, dep);
|
|
3626
|
+
if (pkgDir === void 0) continue;
|
|
3627
|
+
let entries;
|
|
3628
|
+
try {
|
|
3629
|
+
entries = declaredPluginPaths(readJson((0, import_node_path2.join)(pkgDir, "package.json")), dep);
|
|
3630
|
+
} catch (err) {
|
|
3631
|
+
console.error(`attest: failed to read plugins declared by '${dep}':`, err);
|
|
3632
|
+
continue;
|
|
3633
|
+
}
|
|
3634
|
+
for (const entry of entries) {
|
|
3635
|
+
try {
|
|
3636
|
+
const file = (0, import_node_path2.resolve)(pkgDir, entry);
|
|
3637
|
+
if (!isInside(pkgDir, file)) {
|
|
3638
|
+
throw new Error(`plugin path '${entry}' escapes package '${dep}'`);
|
|
3639
|
+
}
|
|
3640
|
+
const mod = await import((0, import_node_url2.pathToFileURL)(file).href);
|
|
3641
|
+
const plugins = pluginsOf(mod);
|
|
3642
|
+
if (plugins.length === 0) {
|
|
3643
|
+
throw new Error(`'${entry}' exports no plugin (name, pluginType, execute)`);
|
|
3644
|
+
}
|
|
3645
|
+
for (const plugin of plugins) {
|
|
3646
|
+
registry.register(plugin.name, plugin);
|
|
3647
|
+
loaded++;
|
|
3648
|
+
}
|
|
3649
|
+
} catch (err) {
|
|
3650
|
+
console.error(`attest: failed to load plugin '${entry}' from '${dep}':`, err);
|
|
3651
|
+
}
|
|
3652
|
+
}
|
|
3653
|
+
}
|
|
3654
|
+
return loaded;
|
|
3655
|
+
}
|
|
3506
3656
|
|
|
3507
3657
|
// src/continuous.ts
|
|
3508
3658
|
var import_node_events = require("events");
|
|
@@ -3882,6 +4032,172 @@ the assistant's response meets your needs. Keep responses concise (1-3 sentences
|
|
|
3882
4032
|
max_tokens: 200
|
|
3883
4033
|
};
|
|
3884
4034
|
|
|
4035
|
+
// src/simulation/mock-tools.ts
|
|
4036
|
+
var MockToolRegistry = class {
|
|
4037
|
+
tools = /* @__PURE__ */ new Map();
|
|
4038
|
+
register(name, fn) {
|
|
4039
|
+
this.tools.set(name, fn);
|
|
4040
|
+
}
|
|
4041
|
+
/**
|
|
4042
|
+
* Run `fn` with the registered mocks active. `TraceBuilder.addToolCall`
|
|
4043
|
+
* returns the mock's result for a registered tool name. Scopes are
|
|
4044
|
+
* isolated per async call chain (AsyncLocalStorage).
|
|
4045
|
+
*/
|
|
4046
|
+
run(fn) {
|
|
4047
|
+
return activeMockRegistry.run(new Map(this.tools), fn);
|
|
4048
|
+
}
|
|
4049
|
+
};
|
|
4050
|
+
function mockTool(name, fn) {
|
|
4051
|
+
const wrapper = ((args) => fn(args));
|
|
4052
|
+
return Object.assign(wrapper, { mockToolName: name });
|
|
4053
|
+
}
|
|
4054
|
+
|
|
4055
|
+
// src/simulation/fault-inject.ts
|
|
4056
|
+
function faultInject(options2, fn) {
|
|
4057
|
+
const errorRate = options2.errorRate ?? 0;
|
|
4058
|
+
const latencyJitterMs = options2.latencyJitterMs ?? 0;
|
|
4059
|
+
const rng = createRng(options2.seed);
|
|
4060
|
+
return async (...args) => {
|
|
4061
|
+
if (errorRate > 0 && rng() < errorRate) {
|
|
4062
|
+
throw new Error(`Injected fault in ${fn.name || "anonymous"}`);
|
|
4063
|
+
}
|
|
4064
|
+
if (latencyJitterMs > 0) {
|
|
4065
|
+
await sleep(rng() * latencyJitterMs);
|
|
4066
|
+
}
|
|
4067
|
+
return fn(...args);
|
|
4068
|
+
};
|
|
4069
|
+
}
|
|
4070
|
+
|
|
4071
|
+
// src/simulation/verdict.ts
|
|
4072
|
+
function aggregateOutcomes(outcomes) {
|
|
4073
|
+
if (outcomes.includes(OUTCOME_FAIL)) return OUTCOME_FAIL;
|
|
4074
|
+
if (outcomes.includes(OUTCOME_SKIP)) return OUTCOME_SKIP;
|
|
4075
|
+
return OUTCOME_PASS;
|
|
4076
|
+
}
|
|
4077
|
+
function sumNotEvaluatedReasons(results) {
|
|
4078
|
+
const counts = {};
|
|
4079
|
+
for (const r of results) {
|
|
4080
|
+
for (const [reason, n] of Object.entries(r.notEvaluatedReasons)) {
|
|
4081
|
+
counts[reason] = (counts[reason] ?? 0) + n;
|
|
4082
|
+
}
|
|
4083
|
+
}
|
|
4084
|
+
return counts;
|
|
4085
|
+
}
|
|
4086
|
+
function formatSkipReason(counts) {
|
|
4087
|
+
const parts = Object.entries(counts).map(([reason, n]) => n === 1 ? reason : `${reason} \xD7${n}`);
|
|
4088
|
+
return `not evaluated: ${parts.join(", ")}`;
|
|
4089
|
+
}
|
|
4090
|
+
function renderVerdictDiagnostics(verdict, color = false) {
|
|
4091
|
+
return verdict instanceof AgentResult ? renderDiagnostics(verdict, { color }) : verdict.renderDiagnostics?.({ color }) ?? "";
|
|
4092
|
+
}
|
|
4093
|
+
function verdictsDiagnostics(results, label, color = false) {
|
|
4094
|
+
const blocks = results.flatMap(
|
|
4095
|
+
(r, i) => r.outcome() === OUTCOME_PASS ? [] : [`${label} ${i + 1}/${results.length}:${renderVerdictDiagnostics(r, color)}`]
|
|
4096
|
+
);
|
|
4097
|
+
return blocks.length > 0 ? blocks.join("\n") : ` (no failing ${label}s)
|
|
4098
|
+
`;
|
|
4099
|
+
}
|
|
4100
|
+
|
|
4101
|
+
// src/simulation/repeat.ts
|
|
4102
|
+
var RepeatResult = class {
|
|
4103
|
+
results;
|
|
4104
|
+
constructor(results = []) {
|
|
4105
|
+
this.results = results;
|
|
4106
|
+
}
|
|
4107
|
+
get count() {
|
|
4108
|
+
return this.results.length;
|
|
4109
|
+
}
|
|
4110
|
+
/** Trials skipped because an assertion was not evaluated. */
|
|
4111
|
+
get notEvaluated() {
|
|
4112
|
+
return this.results.filter((r) => r.outcome() === OUTCOME_SKIP).length;
|
|
4113
|
+
}
|
|
4114
|
+
/** Passing / evaluated trials; null when no trial was evaluated. */
|
|
4115
|
+
get passRate() {
|
|
4116
|
+
const evaluated = this.count - this.notEvaluated;
|
|
4117
|
+
if (evaluated === 0) return null;
|
|
4118
|
+
return this.results.filter((r) => r.outcome() === OUTCOME_PASS).length / evaluated;
|
|
4119
|
+
}
|
|
4120
|
+
/** True when every trial's outcome is pass (soft failures included). */
|
|
4121
|
+
get allPassed() {
|
|
4122
|
+
return this.results.every((r) => r.outcome() === OUTCOME_PASS);
|
|
4123
|
+
}
|
|
4124
|
+
outcome(options2) {
|
|
4125
|
+
return aggregateOutcomes(this.results.map((r) => r.outcome(options2)));
|
|
4126
|
+
}
|
|
4127
|
+
get notEvaluatedReasons() {
|
|
4128
|
+
return sumNotEvaluatedReasons(this.results);
|
|
4129
|
+
}
|
|
4130
|
+
skipReason() {
|
|
4131
|
+
return formatSkipReason(this.notEvaluatedReasons);
|
|
4132
|
+
}
|
|
4133
|
+
renderDiagnostics(options2) {
|
|
4134
|
+
return verdictsDiagnostics(this.results, "trial", options2?.color);
|
|
4135
|
+
}
|
|
4136
|
+
};
|
|
4137
|
+
function repeat(n, fn) {
|
|
4138
|
+
return async (...args) => {
|
|
4139
|
+
const results = [];
|
|
4140
|
+
for (let i = 0; i < n; i++) {
|
|
4141
|
+
results.push(await fn(...args));
|
|
4142
|
+
}
|
|
4143
|
+
return new RepeatResult(results);
|
|
4144
|
+
};
|
|
4145
|
+
}
|
|
4146
|
+
|
|
4147
|
+
// src/simulation/scenario.ts
|
|
4148
|
+
var ScenarioResult = class {
|
|
4149
|
+
config;
|
|
4150
|
+
results;
|
|
4151
|
+
constructor(config2, results = []) {
|
|
4152
|
+
this.config = config2;
|
|
4153
|
+
this.results = results;
|
|
4154
|
+
}
|
|
4155
|
+
/** True when the aggregate `outcome()` is pass. */
|
|
4156
|
+
get passed() {
|
|
4157
|
+
return this.outcome() === OUTCOME_PASS;
|
|
4158
|
+
}
|
|
4159
|
+
outcome(options2) {
|
|
4160
|
+
return aggregateOutcomes(this.results.map((r) => r.outcome(options2)));
|
|
4161
|
+
}
|
|
4162
|
+
get notEvaluatedReasons() {
|
|
4163
|
+
return sumNotEvaluatedReasons(this.results);
|
|
4164
|
+
}
|
|
4165
|
+
skipReason() {
|
|
4166
|
+
return formatSkipReason(this.notEvaluatedReasons);
|
|
4167
|
+
}
|
|
4168
|
+
renderDiagnostics(options2) {
|
|
4169
|
+
return verdictsDiagnostics(this.results, "turn", options2?.color);
|
|
4170
|
+
}
|
|
4171
|
+
};
|
|
4172
|
+
function scenario(options2, fn) {
|
|
4173
|
+
const config2 = {
|
|
4174
|
+
persona: options2.persona,
|
|
4175
|
+
mockTools: { ...options2.mockTools },
|
|
4176
|
+
faultErrorRate: options2.faultErrorRate ?? 0,
|
|
4177
|
+
faultLatencyJitterMs: options2.faultLatencyJitterMs ?? 0,
|
|
4178
|
+
maxTurns: options2.maxTurns ?? 1,
|
|
4179
|
+
seed: options2.seed
|
|
4180
|
+
};
|
|
4181
|
+
return async (...args) => {
|
|
4182
|
+
const run = async () => {
|
|
4183
|
+
const results = [];
|
|
4184
|
+
const rng = createRng(config2.seed);
|
|
4185
|
+
for (let turn = 0; turn < config2.maxTurns; turn++) {
|
|
4186
|
+
if (config2.faultErrorRate > 0 && rng() < config2.faultErrorRate) {
|
|
4187
|
+
throw new Error(`Injected fault at turn ${turn}`);
|
|
4188
|
+
}
|
|
4189
|
+
if (config2.faultLatencyJitterMs > 0) {
|
|
4190
|
+
await sleep(rng() * config2.faultLatencyJitterMs);
|
|
4191
|
+
}
|
|
4192
|
+
results.push(await fn({ persona: config2.persona, turn }, ...args));
|
|
4193
|
+
}
|
|
4194
|
+
return new ScenarioResult(config2, results);
|
|
4195
|
+
};
|
|
4196
|
+
const mocks = new Map(Object.entries(config2.mockTools));
|
|
4197
|
+
return mocks.size > 0 ? activeMockRegistry.run(mocks, run) : run();
|
|
4198
|
+
};
|
|
4199
|
+
}
|
|
4200
|
+
|
|
3885
4201
|
// src/adapters/manual.ts
|
|
3886
4202
|
var ManualAdapter = class {
|
|
3887
4203
|
agentId;
|
|
@@ -4093,7 +4409,7 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
4093
4409
|
}
|
|
4094
4410
|
};
|
|
4095
4411
|
function runConvert(engine, args, payload) {
|
|
4096
|
-
return new Promise((
|
|
4412
|
+
return new Promise((resolve5, reject) => {
|
|
4097
4413
|
const child = (0, import_node_child_process5.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
4098
4414
|
let timer;
|
|
4099
4415
|
child.on("error", (e) => {
|
|
@@ -4146,7 +4462,7 @@ function runConvert(engine, args, payload) {
|
|
|
4146
4462
|
);
|
|
4147
4463
|
return;
|
|
4148
4464
|
}
|
|
4149
|
-
|
|
4465
|
+
resolve5(Buffer.concat(out).toString("utf8"));
|
|
4150
4466
|
});
|
|
4151
4467
|
stdin.end(payload);
|
|
4152
4468
|
});
|
|
@@ -4314,6 +4630,7 @@ var LangChainAdapter = class {
|
|
|
4314
4630
|
GeminiAdapter,
|
|
4315
4631
|
LangChainAdapter,
|
|
4316
4632
|
ManualAdapter,
|
|
4633
|
+
MockToolRegistry,
|
|
4317
4634
|
OTLPConversionError,
|
|
4318
4635
|
OTelAdapter,
|
|
4319
4636
|
OUTCOME_FAIL,
|
|
@@ -4333,6 +4650,7 @@ var LangChainAdapter = class {
|
|
|
4333
4650
|
REASON_REPLAY_MISS,
|
|
4334
4651
|
REASON_SHORT_CIRCUITED,
|
|
4335
4652
|
REASON_SIMULATION,
|
|
4653
|
+
RepeatResult,
|
|
4336
4654
|
ReplayError,
|
|
4337
4655
|
ReplayMissError,
|
|
4338
4656
|
ReplayNotEvaluatedError,
|
|
@@ -4345,6 +4663,7 @@ var LangChainAdapter = class {
|
|
|
4345
4663
|
STEP_RETRIEVAL,
|
|
4346
4664
|
STEP_TOOL_CALL,
|
|
4347
4665
|
Sampler,
|
|
4666
|
+
ScenarioResult,
|
|
4348
4667
|
TIER_1,
|
|
4349
4668
|
TIER_2,
|
|
4350
4669
|
TIER_3,
|
|
@@ -4379,6 +4698,7 @@ var LangChainAdapter = class {
|
|
|
4379
4698
|
executePluginAssertion,
|
|
4380
4699
|
extractId,
|
|
4381
4700
|
extractResult,
|
|
4701
|
+
faultInject,
|
|
4382
4702
|
firstMiss,
|
|
4383
4703
|
formatUsage,
|
|
4384
4704
|
getSampleRate,
|
|
@@ -4390,17 +4710,23 @@ var LangChainAdapter = class {
|
|
|
4390
4710
|
loadFixture,
|
|
4391
4711
|
loadLabelsCSV,
|
|
4392
4712
|
loadLabelsJSONL,
|
|
4713
|
+
loadPackagePlugins,
|
|
4714
|
+
mockTool,
|
|
4393
4715
|
parseUsage,
|
|
4394
4716
|
previewLine,
|
|
4395
4717
|
promptHash,
|
|
4396
4718
|
record,
|
|
4719
|
+
registerPlugin,
|
|
4397
4720
|
renderAssertionFailure,
|
|
4398
4721
|
renderDiagnostics,
|
|
4399
4722
|
renderSummary,
|
|
4723
|
+
renderVerdictDiagnostics,
|
|
4400
4724
|
renderVitest,
|
|
4725
|
+
repeat,
|
|
4401
4726
|
requireSimulationCapability,
|
|
4402
4727
|
resetConfig,
|
|
4403
4728
|
resolveMode,
|
|
4729
|
+
scenario,
|
|
4404
4730
|
simulationEnvEnabled,
|
|
4405
4731
|
startReplay,
|
|
4406
4732
|
startWorld,
|