@attest-ai/core 1.0.0-rc.4 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -56,6 +56,7 @@ __export(index_exports, {
56
56
  GeminiAdapter: () => GeminiAdapter,
57
57
  LangChainAdapter: () => LangChainAdapter,
58
58
  ManualAdapter: () => ManualAdapter,
59
+ MockToolRegistry: () => MockToolRegistry,
59
60
  OTLPConversionError: () => OTLPConversionError,
60
61
  OTelAdapter: () => OTelAdapter,
61
62
  OUTCOME_FAIL: () => OUTCOME_FAIL,
@@ -75,6 +76,7 @@ __export(index_exports, {
75
76
  REASON_REPLAY_MISS: () => REASON_REPLAY_MISS,
76
77
  REASON_SHORT_CIRCUITED: () => REASON_SHORT_CIRCUITED,
77
78
  REASON_SIMULATION: () => REASON_SIMULATION,
79
+ RepeatResult: () => RepeatResult,
78
80
  ReplayError: () => ReplayError,
79
81
  ReplayMissError: () => ReplayMissError,
80
82
  ReplayNotEvaluatedError: () => ReplayNotEvaluatedError,
@@ -87,6 +89,7 @@ __export(index_exports, {
87
89
  STEP_RETRIEVAL: () => STEP_RETRIEVAL,
88
90
  STEP_TOOL_CALL: () => STEP_TOOL_CALL,
89
91
  Sampler: () => Sampler,
92
+ ScenarioResult: () => ScenarioResult,
90
93
  TIER_1: () => TIER_1,
91
94
  TIER_2: () => TIER_2,
92
95
  TIER_3: () => TIER_3,
@@ -121,6 +124,7 @@ __export(index_exports, {
121
124
  executePluginAssertion: () => executePluginAssertion,
122
125
  extractId: () => extractId,
123
126
  extractResult: () => extractResult,
127
+ faultInject: () => faultInject,
124
128
  firstMiss: () => firstMiss,
125
129
  formatUsage: () => formatUsage,
126
130
  getSampleRate: () => getSampleRate,
@@ -132,17 +136,23 @@ __export(index_exports, {
132
136
  loadFixture: () => loadFixture,
133
137
  loadLabelsCSV: () => loadLabelsCSV,
134
138
  loadLabelsJSONL: () => loadLabelsJSONL,
139
+ loadPackagePlugins: () => loadPackagePlugins,
140
+ mockTool: () => mockTool,
135
141
  parseUsage: () => parseUsage,
136
142
  previewLine: () => previewLine,
137
143
  promptHash: () => promptHash,
138
144
  record: () => record,
145
+ registerPlugin: () => registerPlugin,
139
146
  renderAssertionFailure: () => renderAssertionFailure,
140
147
  renderDiagnostics: () => renderDiagnostics,
141
148
  renderSummary: () => renderSummary,
149
+ renderVerdictDiagnostics: () => renderVerdictDiagnostics,
142
150
  renderVitest: () => renderVitest,
151
+ repeat: () => repeat,
143
152
  requireSimulationCapability: () => requireSimulationCapability,
144
153
  resetConfig: () => resetConfig,
145
154
  resolveMode: () => resolveMode,
155
+ scenario: () => scenario,
146
156
  simulationEnvEnabled: () => simulationEnvEnabled,
147
157
  startReplay: () => startReplay,
148
158
  startWorld: () => startWorld,
@@ -377,8 +387,8 @@ function extractId(response) {
377
387
  }
378
388
 
379
389
  // src/version.ts
380
- var VERSION = "1.0.0-rc.4";
381
- var ENGINE_VERSION = "1.0.0-rc.4";
390
+ var VERSION = "1.0.0";
391
+ var ENGINE_VERSION = "1.0.0";
382
392
 
383
393
  // src/engine-manager.ts
384
394
  var import_node_child_process = require("child_process");
@@ -446,7 +456,7 @@ function httpsGet(url, redirectCount = 0) {
446
456
  new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
447
457
  );
448
458
  }
449
- return new Promise((resolve4, reject) => {
459
+ return new Promise((resolve5, reject) => {
450
460
  const client = url.startsWith("https:") ? https : http;
451
461
  client.get(url, (res) => {
452
462
  const status = res.statusCode ?? 0;
@@ -457,7 +467,7 @@ function httpsGet(url, redirectCount = 0) {
457
467
  return;
458
468
  }
459
469
  res.resume();
460
- resolve4(httpsGet(location, redirectCount + 1));
470
+ resolve5(httpsGet(location, redirectCount + 1));
461
471
  return;
462
472
  }
463
473
  if (status < 200 || status >= 300) {
@@ -467,7 +477,7 @@ function httpsGet(url, redirectCount = 0) {
467
477
  }
468
478
  const chunks = [];
469
479
  res.on("data", (chunk) => chunks.push(chunk));
470
- res.on("end", () => resolve4(Buffer.concat(chunks)));
480
+ res.on("end", () => resolve5(Buffer.concat(chunks)));
471
481
  res.on("error", reject);
472
482
  }).on("error", reject);
473
483
  });
@@ -694,11 +704,11 @@ var EngineManager = class {
694
704
  }
695
705
  this.requestId += 1;
696
706
  const requestStr = encodeRequest(this.requestId, method, params);
697
- return new Promise((resolve4, reject) => {
707
+ return new Promise((resolve5, reject) => {
698
708
  const onLine = (line) => {
699
709
  try {
700
710
  const response = decodeResponse(line);
701
- resolve4(extractResult(response));
711
+ resolve5(extractResult(response));
702
712
  } catch (err) {
703
713
  reject(err);
704
714
  }
@@ -1065,12 +1075,12 @@ var AttestClient = class {
1065
1075
  throw new Error("Engine process not started.");
1066
1076
  }
1067
1077
  let capturedReqId;
1068
- const requestPromise = new Promise((resolve4, reject) => {
1078
+ const requestPromise = new Promise((resolve5, reject) => {
1069
1079
  this.writeChain = this.writeChain.then(() => {
1070
1080
  this.requestId += 1;
1071
1081
  const reqId = this.requestId;
1072
1082
  capturedReqId = reqId;
1073
- this.pending.set(reqId, { resolve: resolve4, reject });
1083
+ this.pending.set(reqId, { resolve: resolve5, reject });
1074
1084
  const requestStr = encodeRequest(reqId, method, params);
1075
1085
  cp.stdin.write(requestStr, (err) => {
1076
1086
  if (err) {
@@ -1185,6 +1195,26 @@ function requireSimulationCapability(init) {
1185
1195
 
1186
1196
  // src/trace.ts
1187
1197
  var import_node_crypto = require("crypto");
1198
+
1199
+ // src/simulation/context.ts
1200
+ var import_node_async_hooks = require("async_hooks");
1201
+ var activeMockRegistry = new import_node_async_hooks.AsyncLocalStorage();
1202
+ function createRng(seed) {
1203
+ if (seed === void 0) return Math.random;
1204
+ let state = seed >>> 0;
1205
+ return () => {
1206
+ state = state + 1831565813 >>> 0;
1207
+ let t = state;
1208
+ t = Math.imul(t ^ t >>> 15, t | 1);
1209
+ t ^= t + Math.imul(t ^ t >>> 7, t | 61);
1210
+ return ((t ^ t >>> 14) >>> 0) / 4294967296;
1211
+ };
1212
+ }
1213
+ function sleep(ms) {
1214
+ return new Promise((resolve5) => setTimeout(resolve5, ms));
1215
+ }
1216
+
1217
+ // src/trace.ts
1188
1218
  var TraceBuilder = class {
1189
1219
  traceId;
1190
1220
  agentId;
@@ -1216,11 +1246,22 @@ var TraceBuilder = class {
1216
1246
  return this;
1217
1247
  }
1218
1248
  addToolCall(name, options2) {
1249
+ const mock = activeMockRegistry.getStore()?.get(name);
1250
+ let result = options2?.result;
1251
+ if (mock !== void 0) {
1252
+ const mocked = mock(options2?.args ?? {});
1253
+ if (typeof mocked?.then === "function") {
1254
+ throw new Error(`mock for tool '${name}' returned a Promise; mock tools must be synchronous`);
1255
+ }
1256
+ if (typeof mocked === "object" && mocked !== null && !Array.isArray(mocked)) {
1257
+ result = mocked;
1258
+ }
1259
+ }
1219
1260
  this.steps.push({
1220
1261
  type: STEP_TOOL_CALL,
1221
1262
  name,
1222
1263
  args: options2?.args,
1223
- result: options2?.result,
1264
+ result,
1224
1265
  metadata: options2?.metadata
1225
1266
  });
1226
1267
  return this;
@@ -2875,18 +2916,18 @@ function serve(engine, args) {
2875
2916
  let stdout = "";
2876
2917
  let stderr = "";
2877
2918
  let resolveReady = () => void 0;
2878
- const ready = new Promise((resolve4) => {
2879
- resolveReady = resolve4;
2919
+ const ready = new Promise((resolve5) => {
2920
+ resolveReady = resolve5;
2880
2921
  });
2881
- const exited = new Promise((resolve4) => {
2922
+ const exited = new Promise((resolve5) => {
2882
2923
  child.on("close", (code, signal) => {
2883
2924
  resolveReady(null);
2884
- resolve4({ code, signal });
2925
+ resolve5({ code, signal });
2885
2926
  });
2886
2927
  child.on("error", (err) => {
2887
2928
  stderr += err.message;
2888
2929
  resolveReady(null);
2889
- resolve4({ code: null, signal: null });
2930
+ resolve5({ code: null, signal: null });
2890
2931
  });
2891
2932
  });
2892
2933
  child.stdout.setEncoding("utf8").on("data", (chunk) => {
@@ -2978,11 +3019,14 @@ async function recordExclusive(name, fn, opts) {
2978
3019
  throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
2979
3020
  }
2980
3021
  const outcome = JSON.parse(lines[lines.length - 1] ?? "");
2981
- if (exit.code !== 0 || !outcome.verified) {
3022
+ if (exit.code !== 0 || outcome.outcome === "not_verified") {
2982
3023
  throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
2983
3024
  }
3025
+ if (outcome.outcome === "nothing_to_verify") {
3026
+ return { value, fixture: outcome.fixture, verified: false, outcome: "nothing_to_verify", testFile: null };
3027
+ }
2984
3028
  const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
2985
- return { value, fixture: outcome.fixture, verified: true, testFile };
3029
+ return { value, fixture: outcome.fixture, verified: true, outcome: "verified", testFile };
2986
3030
  }
2987
3031
 
2988
3032
  // src/replay.ts
@@ -3143,18 +3187,18 @@ async function start(name, mode, opts) {
3143
3187
  let stdout = "";
3144
3188
  let stderr = "";
3145
3189
  let resolveReady = () => void 0;
3146
- const ready = new Promise((resolve4) => {
3147
- resolveReady = resolve4;
3190
+ const ready = new Promise((resolve5) => {
3191
+ resolveReady = resolve5;
3148
3192
  });
3149
- const exited = new Promise((resolve4) => {
3193
+ const exited = new Promise((resolve5) => {
3150
3194
  child.on("close", (code) => {
3151
3195
  resolveReady(null);
3152
- resolve4(code);
3196
+ resolve5(code);
3153
3197
  });
3154
3198
  child.on("error", (err) => {
3155
3199
  stderr += err.message;
3156
3200
  resolveReady(null);
3157
- resolve4(null);
3201
+ resolve5(null);
3158
3202
  });
3159
3203
  });
3160
3204
  child.stdout.setEncoding("utf8").on("data", (chunk) => {
@@ -3279,18 +3323,18 @@ async function start2(file, opts) {
3279
3323
  let stdout = "";
3280
3324
  let stderr = "";
3281
3325
  let resolveReady = () => void 0;
3282
- const ready = new Promise((resolve4) => {
3283
- resolveReady = resolve4;
3326
+ const ready = new Promise((resolve5) => {
3327
+ resolveReady = resolve5;
3284
3328
  });
3285
- const exited = new Promise((resolve4) => {
3329
+ const exited = new Promise((resolve5) => {
3286
3330
  child.on("close", (code) => {
3287
3331
  resolveReady(null);
3288
- resolve4(code);
3332
+ resolve5(code);
3289
3333
  });
3290
3334
  child.on("error", (err) => {
3291
3335
  stderr += err.message;
3292
3336
  resolveReady(null);
3293
- resolve4(null);
3337
+ resolve5(null);
3294
3338
  });
3295
3339
  });
3296
3340
  child.stdout.setEncoding("utf8").on("data", (chunk) => {
@@ -3371,8 +3415,8 @@ function parseFinal(out) {
3371
3415
  }
3372
3416
 
3373
3417
  // src/context.ts
3374
- var import_node_async_hooks = require("async_hooks");
3375
- var activeBuilder = new import_node_async_hooks.AsyncLocalStorage();
3418
+ var import_node_async_hooks2 = require("async_hooks");
3419
+ var activeBuilder = new import_node_async_hooks2.AsyncLocalStorage();
3376
3420
 
3377
3421
  // src/agent.ts
3378
3422
  var Agent = class {
@@ -3459,6 +3503,9 @@ function tier(level) {
3459
3503
  }
3460
3504
 
3461
3505
  // src/plugins.ts
3506
+ var import_node_fs2 = require("fs");
3507
+ var import_node_path2 = require("path");
3508
+ var import_node_url2 = require("url");
3462
3509
  var PluginRegistry = class {
3463
3510
  plugins = /* @__PURE__ */ new Map();
3464
3511
  register(name, plugin) {
@@ -3489,10 +3536,19 @@ var PluginRegistry = class {
3489
3536
  };
3490
3537
  async function executePluginAssertion(plugin, trace, spec, client, traceId2, assertionId2, timeoutMs = 3e4) {
3491
3538
  const resultPromise = Promise.resolve(plugin.execute(trace, spec));
3539
+ let timer;
3492
3540
  const timeoutPromise = new Promise((_, reject) => {
3493
- setTimeout(() => reject(new Error(`Plugin '${plugin.name}' timed out after ${timeoutMs}ms`)), timeoutMs);
3541
+ timer = setTimeout(
3542
+ () => reject(new Error(`Plugin '${plugin.name}' timed out after ${timeoutMs}ms`)),
3543
+ timeoutMs
3544
+ );
3494
3545
  });
3495
- const result = await Promise.race([resultPromise, timeoutPromise]);
3546
+ let result;
3547
+ try {
3548
+ result = await Promise.race([resultPromise, timeoutPromise]);
3549
+ } finally {
3550
+ clearTimeout(timer);
3551
+ }
3496
3552
  await client.submitPluginResult(
3497
3553
  traceId2,
3498
3554
  plugin.name,
@@ -3503,6 +3559,100 @@ async function executePluginAssertion(plugin, trace, spec, client, traceId2, ass
3503
3559
  );
3504
3560
  return result;
3505
3561
  }
3562
+ function registerPlugin(registry, name, plugin) {
3563
+ registry.register(name, plugin);
3564
+ }
3565
+ function isAttestPlugin(value) {
3566
+ if (typeof value !== "object" || value === null) return false;
3567
+ const v = value;
3568
+ return typeof v["name"] === "string" && typeof v["pluginType"] === "string" && typeof v["execute"] === "function";
3569
+ }
3570
+ function readJson(path5) {
3571
+ return JSON.parse((0, import_node_fs2.readFileSync)(path5, "utf8"));
3572
+ }
3573
+ function findInstalledPackage(projectRoot, name) {
3574
+ let dir = (0, import_node_path2.resolve)(projectRoot);
3575
+ for (; ; ) {
3576
+ const candidate = (0, import_node_path2.join)(dir, "node_modules", name);
3577
+ if ((0, import_node_fs2.existsSync)((0, import_node_path2.join)(candidate, "package.json"))) return candidate;
3578
+ const parent = (0, import_node_path2.dirname)(dir);
3579
+ if (parent === dir) return void 0;
3580
+ dir = parent;
3581
+ }
3582
+ }
3583
+ function declaredPluginPaths(pkgJson, dep) {
3584
+ const attest = pkgJson["attest"];
3585
+ if (attest === void 0) return [];
3586
+ const plugins = typeof attest === "object" && attest !== null ? attest["plugins"] : void 0;
3587
+ if (plugins === void 0) return [];
3588
+ if (!Array.isArray(plugins) || !plugins.every((p) => typeof p === "string")) {
3589
+ throw new Error(`'${dep}': "attest.plugins" must be an array of strings`);
3590
+ }
3591
+ return plugins;
3592
+ }
3593
+ function pluginsOf(mod) {
3594
+ const def = mod["default"];
3595
+ const nested = typeof def === "object" && def !== null ? def : void 0;
3596
+ for (const candidate of [def, nested?.["default"]]) {
3597
+ if (isAttestPlugin(candidate)) return [candidate];
3598
+ }
3599
+ return [.../* @__PURE__ */ new Set([...Object.values(mod), ...Object.values(nested ?? {})])].filter(isAttestPlugin);
3600
+ }
3601
+ function isInside(root, file) {
3602
+ const within = (r, f) => {
3603
+ const rel = (0, import_node_path2.relative)(r, f);
3604
+ return rel !== "" && rel !== ".." && !rel.startsWith(`..${import_node_path2.sep}`) && !(0, import_node_path2.isAbsolute)(rel);
3605
+ };
3606
+ return within(root, file) && within((0, import_node_fs2.realpathSync)(root), (0, import_node_fs2.realpathSync)(file));
3607
+ }
3608
+ async function loadPackagePlugins(registry, projectRoot = process.cwd()) {
3609
+ let project;
3610
+ try {
3611
+ project = readJson((0, import_node_path2.join)(projectRoot, "package.json"));
3612
+ } catch (err) {
3613
+ console.error(`attest: cannot read ${(0, import_node_path2.join)(projectRoot, "package.json")}; no plugins loaded:`, err);
3614
+ return 0;
3615
+ }
3616
+ const depNames = /* @__PURE__ */ new Set();
3617
+ for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) {
3618
+ const deps = project[field];
3619
+ if (typeof deps === "object" && deps !== null) {
3620
+ for (const dep of Object.keys(deps)) depNames.add(dep);
3621
+ }
3622
+ }
3623
+ let loaded = 0;
3624
+ for (const dep of depNames) {
3625
+ const pkgDir = findInstalledPackage(projectRoot, dep);
3626
+ if (pkgDir === void 0) continue;
3627
+ let entries;
3628
+ try {
3629
+ entries = declaredPluginPaths(readJson((0, import_node_path2.join)(pkgDir, "package.json")), dep);
3630
+ } catch (err) {
3631
+ console.error(`attest: failed to read plugins declared by '${dep}':`, err);
3632
+ continue;
3633
+ }
3634
+ for (const entry of entries) {
3635
+ try {
3636
+ const file = (0, import_node_path2.resolve)(pkgDir, entry);
3637
+ if (!isInside(pkgDir, file)) {
3638
+ throw new Error(`plugin path '${entry}' escapes package '${dep}'`);
3639
+ }
3640
+ const mod = await import((0, import_node_url2.pathToFileURL)(file).href);
3641
+ const plugins = pluginsOf(mod);
3642
+ if (plugins.length === 0) {
3643
+ throw new Error(`'${entry}' exports no plugin (name, pluginType, execute)`);
3644
+ }
3645
+ for (const plugin of plugins) {
3646
+ registry.register(plugin.name, plugin);
3647
+ loaded++;
3648
+ }
3649
+ } catch (err) {
3650
+ console.error(`attest: failed to load plugin '${entry}' from '${dep}':`, err);
3651
+ }
3652
+ }
3653
+ }
3654
+ return loaded;
3655
+ }
3506
3656
 
3507
3657
  // src/continuous.ts
3508
3658
  var import_node_events = require("events");
@@ -3882,6 +4032,172 @@ the assistant's response meets your needs. Keep responses concise (1-3 sentences
3882
4032
  max_tokens: 200
3883
4033
  };
3884
4034
 
4035
+ // src/simulation/mock-tools.ts
4036
+ var MockToolRegistry = class {
4037
+ tools = /* @__PURE__ */ new Map();
4038
+ register(name, fn) {
4039
+ this.tools.set(name, fn);
4040
+ }
4041
+ /**
4042
+ * Run `fn` with the registered mocks active. `TraceBuilder.addToolCall`
4043
+ * returns the mock's result for a registered tool name. Scopes are
4044
+ * isolated per async call chain (AsyncLocalStorage).
4045
+ */
4046
+ run(fn) {
4047
+ return activeMockRegistry.run(new Map(this.tools), fn);
4048
+ }
4049
+ };
4050
+ function mockTool(name, fn) {
4051
+ const wrapper = ((args) => fn(args));
4052
+ return Object.assign(wrapper, { mockToolName: name });
4053
+ }
4054
+
4055
+ // src/simulation/fault-inject.ts
4056
+ function faultInject(options2, fn) {
4057
+ const errorRate = options2.errorRate ?? 0;
4058
+ const latencyJitterMs = options2.latencyJitterMs ?? 0;
4059
+ const rng = createRng(options2.seed);
4060
+ return async (...args) => {
4061
+ if (errorRate > 0 && rng() < errorRate) {
4062
+ throw new Error(`Injected fault in ${fn.name || "anonymous"}`);
4063
+ }
4064
+ if (latencyJitterMs > 0) {
4065
+ await sleep(rng() * latencyJitterMs);
4066
+ }
4067
+ return fn(...args);
4068
+ };
4069
+ }
4070
+
4071
+ // src/simulation/verdict.ts
4072
+ function aggregateOutcomes(outcomes) {
4073
+ if (outcomes.includes(OUTCOME_FAIL)) return OUTCOME_FAIL;
4074
+ if (outcomes.includes(OUTCOME_SKIP)) return OUTCOME_SKIP;
4075
+ return OUTCOME_PASS;
4076
+ }
4077
+ function sumNotEvaluatedReasons(results) {
4078
+ const counts = {};
4079
+ for (const r of results) {
4080
+ for (const [reason, n] of Object.entries(r.notEvaluatedReasons)) {
4081
+ counts[reason] = (counts[reason] ?? 0) + n;
4082
+ }
4083
+ }
4084
+ return counts;
4085
+ }
4086
+ function formatSkipReason(counts) {
4087
+ const parts = Object.entries(counts).map(([reason, n]) => n === 1 ? reason : `${reason} \xD7${n}`);
4088
+ return `not evaluated: ${parts.join(", ")}`;
4089
+ }
4090
+ function renderVerdictDiagnostics(verdict, color = false) {
4091
+ return verdict instanceof AgentResult ? renderDiagnostics(verdict, { color }) : verdict.renderDiagnostics?.({ color }) ?? "";
4092
+ }
4093
+ function verdictsDiagnostics(results, label, color = false) {
4094
+ const blocks = results.flatMap(
4095
+ (r, i) => r.outcome() === OUTCOME_PASS ? [] : [`${label} ${i + 1}/${results.length}:${renderVerdictDiagnostics(r, color)}`]
4096
+ );
4097
+ return blocks.length > 0 ? blocks.join("\n") : ` (no failing ${label}s)
4098
+ `;
4099
+ }
4100
+
4101
+ // src/simulation/repeat.ts
4102
+ var RepeatResult = class {
4103
+ results;
4104
+ constructor(results = []) {
4105
+ this.results = results;
4106
+ }
4107
+ get count() {
4108
+ return this.results.length;
4109
+ }
4110
+ /** Trials skipped because an assertion was not evaluated. */
4111
+ get notEvaluated() {
4112
+ return this.results.filter((r) => r.outcome() === OUTCOME_SKIP).length;
4113
+ }
4114
+ /** Passing / evaluated trials; null when no trial was evaluated. */
4115
+ get passRate() {
4116
+ const evaluated = this.count - this.notEvaluated;
4117
+ if (evaluated === 0) return null;
4118
+ return this.results.filter((r) => r.outcome() === OUTCOME_PASS).length / evaluated;
4119
+ }
4120
+ /** True when every trial's outcome is pass (soft failures included). */
4121
+ get allPassed() {
4122
+ return this.results.every((r) => r.outcome() === OUTCOME_PASS);
4123
+ }
4124
+ outcome(options2) {
4125
+ return aggregateOutcomes(this.results.map((r) => r.outcome(options2)));
4126
+ }
4127
+ get notEvaluatedReasons() {
4128
+ return sumNotEvaluatedReasons(this.results);
4129
+ }
4130
+ skipReason() {
4131
+ return formatSkipReason(this.notEvaluatedReasons);
4132
+ }
4133
+ renderDiagnostics(options2) {
4134
+ return verdictsDiagnostics(this.results, "trial", options2?.color);
4135
+ }
4136
+ };
4137
+ function repeat(n, fn) {
4138
+ return async (...args) => {
4139
+ const results = [];
4140
+ for (let i = 0; i < n; i++) {
4141
+ results.push(await fn(...args));
4142
+ }
4143
+ return new RepeatResult(results);
4144
+ };
4145
+ }
4146
+
4147
+ // src/simulation/scenario.ts
4148
+ var ScenarioResult = class {
4149
+ config;
4150
+ results;
4151
+ constructor(config2, results = []) {
4152
+ this.config = config2;
4153
+ this.results = results;
4154
+ }
4155
+ /** True when the aggregate `outcome()` is pass. */
4156
+ get passed() {
4157
+ return this.outcome() === OUTCOME_PASS;
4158
+ }
4159
+ outcome(options2) {
4160
+ return aggregateOutcomes(this.results.map((r) => r.outcome(options2)));
4161
+ }
4162
+ get notEvaluatedReasons() {
4163
+ return sumNotEvaluatedReasons(this.results);
4164
+ }
4165
+ skipReason() {
4166
+ return formatSkipReason(this.notEvaluatedReasons);
4167
+ }
4168
+ renderDiagnostics(options2) {
4169
+ return verdictsDiagnostics(this.results, "turn", options2?.color);
4170
+ }
4171
+ };
4172
+ function scenario(options2, fn) {
4173
+ const config2 = {
4174
+ persona: options2.persona,
4175
+ mockTools: { ...options2.mockTools },
4176
+ faultErrorRate: options2.faultErrorRate ?? 0,
4177
+ faultLatencyJitterMs: options2.faultLatencyJitterMs ?? 0,
4178
+ maxTurns: options2.maxTurns ?? 1,
4179
+ seed: options2.seed
4180
+ };
4181
+ return async (...args) => {
4182
+ const run = async () => {
4183
+ const results = [];
4184
+ const rng = createRng(config2.seed);
4185
+ for (let turn = 0; turn < config2.maxTurns; turn++) {
4186
+ if (config2.faultErrorRate > 0 && rng() < config2.faultErrorRate) {
4187
+ throw new Error(`Injected fault at turn ${turn}`);
4188
+ }
4189
+ if (config2.faultLatencyJitterMs > 0) {
4190
+ await sleep(rng() * config2.faultLatencyJitterMs);
4191
+ }
4192
+ results.push(await fn({ persona: config2.persona, turn }, ...args));
4193
+ }
4194
+ return new ScenarioResult(config2, results);
4195
+ };
4196
+ const mocks = new Map(Object.entries(config2.mockTools));
4197
+ return mocks.size > 0 ? activeMockRegistry.run(mocks, run) : run();
4198
+ };
4199
+ }
4200
+
3885
4201
  // src/adapters/manual.ts
3886
4202
  var ManualAdapter = class {
3887
4203
  agentId;
@@ -4093,7 +4409,7 @@ var OTelAdapter = class _OTelAdapter {
4093
4409
  }
4094
4410
  };
4095
4411
  function runConvert(engine, args, payload) {
4096
- return new Promise((resolve4, reject) => {
4412
+ return new Promise((resolve5, reject) => {
4097
4413
  const child = (0, import_node_child_process5.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
4098
4414
  let timer;
4099
4415
  child.on("error", (e) => {
@@ -4146,7 +4462,7 @@ function runConvert(engine, args, payload) {
4146
4462
  );
4147
4463
  return;
4148
4464
  }
4149
- resolve4(Buffer.concat(out).toString("utf8"));
4465
+ resolve5(Buffer.concat(out).toString("utf8"));
4150
4466
  });
4151
4467
  stdin.end(payload);
4152
4468
  });
@@ -4314,6 +4630,7 @@ var LangChainAdapter = class {
4314
4630
  GeminiAdapter,
4315
4631
  LangChainAdapter,
4316
4632
  ManualAdapter,
4633
+ MockToolRegistry,
4317
4634
  OTLPConversionError,
4318
4635
  OTelAdapter,
4319
4636
  OUTCOME_FAIL,
@@ -4333,6 +4650,7 @@ var LangChainAdapter = class {
4333
4650
  REASON_REPLAY_MISS,
4334
4651
  REASON_SHORT_CIRCUITED,
4335
4652
  REASON_SIMULATION,
4653
+ RepeatResult,
4336
4654
  ReplayError,
4337
4655
  ReplayMissError,
4338
4656
  ReplayNotEvaluatedError,
@@ -4345,6 +4663,7 @@ var LangChainAdapter = class {
4345
4663
  STEP_RETRIEVAL,
4346
4664
  STEP_TOOL_CALL,
4347
4665
  Sampler,
4666
+ ScenarioResult,
4348
4667
  TIER_1,
4349
4668
  TIER_2,
4350
4669
  TIER_3,
@@ -4379,6 +4698,7 @@ var LangChainAdapter = class {
4379
4698
  executePluginAssertion,
4380
4699
  extractId,
4381
4700
  extractResult,
4701
+ faultInject,
4382
4702
  firstMiss,
4383
4703
  formatUsage,
4384
4704
  getSampleRate,
@@ -4390,17 +4710,23 @@ var LangChainAdapter = class {
4390
4710
  loadFixture,
4391
4711
  loadLabelsCSV,
4392
4712
  loadLabelsJSONL,
4713
+ loadPackagePlugins,
4714
+ mockTool,
4393
4715
  parseUsage,
4394
4716
  previewLine,
4395
4717
  promptHash,
4396
4718
  record,
4719
+ registerPlugin,
4397
4720
  renderAssertionFailure,
4398
4721
  renderDiagnostics,
4399
4722
  renderSummary,
4723
+ renderVerdictDiagnostics,
4400
4724
  renderVitest,
4725
+ repeat,
4401
4726
  requireSimulationCapability,
4402
4727
  resetConfig,
4403
4728
  resolveMode,
4729
+ scenario,
4404
4730
  simulationEnvEnabled,
4405
4731
  startReplay,
4406
4732
  startWorld,