@attest-ai/core 1.0.0-rc.3 → 1.0.0-rc.5

package/dist/cjs/index.cjs

@@ -56,6 +56,7 @@ __export(index_exports, {
   GeminiAdapter: () => GeminiAdapter,
   LangChainAdapter: () => LangChainAdapter,
   ManualAdapter: () => ManualAdapter,
+  MockToolRegistry: () => MockToolRegistry,
   OTLPConversionError: () => OTLPConversionError,
   OTelAdapter: () => OTelAdapter,
   OUTCOME_FAIL: () => OUTCOME_FAIL,
@@ -75,6 +76,7 @@ __export(index_exports, {
   REASON_REPLAY_MISS: () => REASON_REPLAY_MISS,
   REASON_SHORT_CIRCUITED: () => REASON_SHORT_CIRCUITED,
   REASON_SIMULATION: () => REASON_SIMULATION,
+  RepeatResult: () => RepeatResult,
   ReplayError: () => ReplayError,
   ReplayMissError: () => ReplayMissError,
   ReplayNotEvaluatedError: () => ReplayNotEvaluatedError,
@@ -87,6 +89,7 @@ __export(index_exports, {
   STEP_RETRIEVAL: () => STEP_RETRIEVAL,
   STEP_TOOL_CALL: () => STEP_TOOL_CALL,
   Sampler: () => Sampler,
+  ScenarioResult: () => ScenarioResult,
   TIER_1: () => TIER_1,
   TIER_2: () => TIER_2,
   TIER_3: () => TIER_3,
@@ -121,6 +124,7 @@ __export(index_exports, {
   executePluginAssertion: () => executePluginAssertion,
   extractId: () => extractId,
   extractResult: () => extractResult,
+  faultInject: () => faultInject,
   firstMiss: () => firstMiss,
   formatUsage: () => formatUsage,
   getSampleRate: () => getSampleRate,
@@ -132,17 +136,23 @@ __export(index_exports, {
   loadFixture: () => loadFixture,
   loadLabelsCSV: () => loadLabelsCSV,
   loadLabelsJSONL: () => loadLabelsJSONL,
+  loadPackagePlugins: () => loadPackagePlugins,
+  mockTool: () => mockTool,
   parseUsage: () => parseUsage,
   previewLine: () => previewLine,
   promptHash: () => promptHash,
   record: () => record,
+  registerPlugin: () => registerPlugin,
   renderAssertionFailure: () => renderAssertionFailure,
   renderDiagnostics: () => renderDiagnostics,
   renderSummary: () => renderSummary,
+  renderVerdictDiagnostics: () => renderVerdictDiagnostics,
   renderVitest: () => renderVitest,
+  repeat: () => repeat,
   requireSimulationCapability: () => requireSimulationCapability,
   resetConfig: () => resetConfig,
   resolveMode: () => resolveMode,
+  scenario: () => scenario,
   simulationEnvEnabled: () => simulationEnvEnabled,
   startReplay: () => startReplay,
   startWorld: () => startWorld,
@@ -377,8 +387,8 @@ function extractId(response) {
 }
 
 // src/version.ts
-var VERSION = "1.0.0-rc.3";
-var ENGINE_VERSION = "1.0.0-rc.3";
+var VERSION = "1.0.0-rc.5";
+var ENGINE_VERSION = "1.0.0-rc.5";
 
 // src/engine-manager.ts
 var import_node_child_process = require("child_process");
@@ -446,7 +456,7 @@ function httpsGet(url, redirectCount = 0) {
       new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
     );
   }
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const client = url.startsWith("https:") ? https : http;
     client.get(url, (res) => {
       const status = res.statusCode ?? 0;
@@ -457,7 +467,7 @@ function httpsGet(url, redirectCount = 0) {
           return;
         }
         res.resume();
-        resolve4(httpsGet(location, redirectCount + 1));
+        resolve5(httpsGet(location, redirectCount + 1));
         return;
       }
       if (status < 200 || status >= 300) {
@@ -467,7 +477,7 @@ function httpsGet(url, redirectCount = 0) {
       }
       const chunks = [];
       res.on("data", (chunk) => chunks.push(chunk));
-      res.on("end", () => resolve4(Buffer.concat(chunks)));
+      res.on("end", () => resolve5(Buffer.concat(chunks)));
       res.on("error", reject);
     }).on("error", reject);
   });
@@ -694,11 +704,11 @@ var EngineManager = class {
     }
     this.requestId += 1;
     const requestStr = encodeRequest(this.requestId, method, params);
-    return new Promise((resolve4, reject) => {
+    return new Promise((resolve5, reject) => {
       const onLine = (line) => {
         try {
           const response = decodeResponse(line);
-          resolve4(extractResult(response));
+          resolve5(extractResult(response));
         } catch (err) {
           reject(err);
         }
@@ -1065,12 +1075,12 @@ var AttestClient = class {
       throw new Error("Engine process not started.");
     }
     let capturedReqId;
-    const requestPromise = new Promise((resolve4, reject) => {
+    const requestPromise = new Promise((resolve5, reject) => {
       this.writeChain = this.writeChain.then(() => {
         this.requestId += 1;
         const reqId = this.requestId;
         capturedReqId = reqId;
-        this.pending.set(reqId, { resolve: resolve4, reject });
+        this.pending.set(reqId, { resolve: resolve5, reject });
         const requestStr = encodeRequest(reqId, method, params);
         cp.stdin.write(requestStr, (err) => {
           if (err) {
@@ -1095,7 +1105,8 @@ var AttestClient = class {
     const params = {
       trace,
       assertions: [...assertions],
-      ...options2?.budget !== void 0 ? { budget: options2.budget } : {}
+      ...options2?.budget !== void 0 ? { budget: options2.budget } : {},
+      ...options2?.context !== void 0 ? { context: options2.context } : {}
     };
     const raw = await this.sendRequest(
       "evaluate_batch",
@@ -1184,6 +1195,26 @@ function requireSimulationCapability(init) {
 
 // src/trace.ts
 var import_node_crypto = require("crypto");
+
+// src/simulation/context.ts
+var import_node_async_hooks = require("async_hooks");
+var activeMockRegistry = new import_node_async_hooks.AsyncLocalStorage();
+function createRng(seed) {
+  if (seed === void 0) return Math.random;
+  let state = seed >>> 0;
+  return () => {
+    state = state + 1831565813 >>> 0;
+    let t = state;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+function sleep(ms) {
+  return new Promise((resolve5) => setTimeout(resolve5, ms));
+}
+
+// src/trace.ts
 var TraceBuilder = class {
   traceId;
   agentId;
@@ -1215,11 +1246,22 @@ var TraceBuilder = class {
     return this;
   }
   addToolCall(name, options2) {
+    const mock = activeMockRegistry.getStore()?.get(name);
+    let result = options2?.result;
+    if (mock !== void 0) {
+      const mocked = mock(options2?.args ?? {});
+      if (typeof mocked?.then === "function") {
+        throw new Error(`mock for tool '${name}' returned a Promise; mock tools must be synchronous`);
+      }
+      if (typeof mocked === "object" && mocked !== null && !Array.isArray(mocked)) {
+        result = mocked;
+      }
+    }
     this.steps.push({
       type: STEP_TOOL_CALL,
       name,
       args: options2?.args,
-      result: options2?.result,
+      result,
       metadata: options2?.metadata
     });
     return this;
@@ -2874,18 +2916,18 @@ function serve(engine, args) {
   let stdout = "";
   let stderr = "";
   let resolveReady = () => void 0;
-  const ready = new Promise((resolve4) => {
-    resolveReady = resolve4;
+  const ready = new Promise((resolve5) => {
+    resolveReady = resolve5;
   });
-  const exited = new Promise((resolve4) => {
+  const exited = new Promise((resolve5) => {
     child.on("close", (code, signal) => {
       resolveReady(null);
-      resolve4({ code, signal });
+      resolve5({ code, signal });
     });
     child.on("error", (err) => {
       stderr += err.message;
       resolveReady(null);
-      resolve4({ code: null, signal: null });
+      resolve5({ code: null, signal: null });
     });
   });
   child.stdout.setEncoding("utf8").on("data", (chunk) => {
@@ -3142,18 +3184,18 @@ async function start(name, mode, opts) {
   let stdout = "";
   let stderr = "";
   let resolveReady = () => void 0;
-  const ready = new Promise((resolve4) => {
-    resolveReady = resolve4;
+  const ready = new Promise((resolve5) => {
+    resolveReady = resolve5;
   });
-  const exited = new Promise((resolve4) => {
+  const exited = new Promise((resolve5) => {
     child.on("close", (code) => {
       resolveReady(null);
-      resolve4(code);
+      resolve5(code);
     });
     child.on("error", (err) => {
       stderr += err.message;
       resolveReady(null);
-      resolve4(null);
+      resolve5(null);
     });
   });
   child.stdout.setEncoding("utf8").on("data", (chunk) => {
@@ -3278,18 +3320,18 @@ async function start2(file, opts) {
   let stdout = "";
   let stderr = "";
   let resolveReady = () => void 0;
-  const ready = new Promise((resolve4) => {
-    resolveReady = resolve4;
+  const ready = new Promise((resolve5) => {
+    resolveReady = resolve5;
   });
-  const exited = new Promise((resolve4) => {
+  const exited = new Promise((resolve5) => {
     child.on("close", (code) => {
       resolveReady(null);
-      resolve4(code);
+      resolve5(code);
     });
     child.on("error", (err) => {
       stderr += err.message;
       resolveReady(null);
-      resolve4(null);
+      resolve5(null);
     });
   });
   child.stdout.setEncoding("utf8").on("data", (chunk) => {
@@ -3370,8 +3412,8 @@ function parseFinal(out) {
 }
 
 // src/context.ts
-var import_node_async_hooks = require("async_hooks");
-var activeBuilder = new import_node_async_hooks.AsyncLocalStorage();
+var import_node_async_hooks2 = require("async_hooks");
+var activeBuilder = new import_node_async_hooks2.AsyncLocalStorage();
 
 // src/agent.ts
 var Agent = class {
@@ -3458,6 +3500,9 @@ function tier(level) {
 }
 
 // src/plugins.ts
+var import_node_fs2 = require("fs");
+var import_node_path2 = require("path");
+var import_node_url2 = require("url");
 var PluginRegistry = class {
   plugins = /* @__PURE__ */ new Map();
   register(name, plugin) {
@@ -3488,10 +3533,19 @@ var PluginRegistry = class {
 };
 async function executePluginAssertion(plugin, trace, spec, client, traceId2, assertionId2, timeoutMs = 3e4) {
   const resultPromise = Promise.resolve(plugin.execute(trace, spec));
+  let timer;
   const timeoutPromise = new Promise((_, reject) => {
-    setTimeout(() => reject(new Error(`Plugin '${plugin.name}' timed out after ${timeoutMs}ms`)), timeoutMs);
+    timer = setTimeout(
+      () => reject(new Error(`Plugin '${plugin.name}' timed out after ${timeoutMs}ms`)),
+      timeoutMs
+    );
   });
-  const result = await Promise.race([resultPromise, timeoutPromise]);
+  let result;
+  try {
+    result = await Promise.race([resultPromise, timeoutPromise]);
+  } finally {
+    clearTimeout(timer);
+  }
   await client.submitPluginResult(
     traceId2,
     plugin.name,
@@ -3502,6 +3556,100 @@ async function executePluginAssertion(plugin, trace, spec, client, traceId2, ass
   );
   return result;
 }
+function registerPlugin(registry, name, plugin) {
+  registry.register(name, plugin);
+}
+function isAttestPlugin(value) {
+  if (typeof value !== "object" || value === null) return false;
+  const v = value;
+  return typeof v["name"] === "string" && typeof v["pluginType"] === "string" && typeof v["execute"] === "function";
+}
+function readJson(path5) {
+  return JSON.parse((0, import_node_fs2.readFileSync)(path5, "utf8"));
+}
+function findInstalledPackage(projectRoot, name) {
+  let dir = (0, import_node_path2.resolve)(projectRoot);
+  for (; ; ) {
+    const candidate = (0, import_node_path2.join)(dir, "node_modules", name);
+    if ((0, import_node_fs2.existsSync)((0, import_node_path2.join)(candidate, "package.json"))) return candidate;
+    const parent = (0, import_node_path2.dirname)(dir);
+    if (parent === dir) return void 0;
+    dir = parent;
+  }
+}
+function declaredPluginPaths(pkgJson, dep) {
+  const attest = pkgJson["attest"];
+  if (attest === void 0) return [];
+  const plugins = typeof attest === "object" && attest !== null ? attest["plugins"] : void 0;
+  if (plugins === void 0) return [];
+  if (!Array.isArray(plugins) || !plugins.every((p) => typeof p === "string")) {
+    throw new Error(`'${dep}': "attest.plugins" must be an array of strings`);
+  }
+  return plugins;
+}
+function pluginsOf(mod) {
+  const def = mod["default"];
+  const nested = typeof def === "object" && def !== null ? def : void 0;
+  for (const candidate of [def, nested?.["default"]]) {
+    if (isAttestPlugin(candidate)) return [candidate];
+  }
+  return [.../* @__PURE__ */ new Set([...Object.values(mod), ...Object.values(nested ?? {})])].filter(isAttestPlugin);
+}
+function isInside(root, file) {
+  const within = (r, f) => {
+    const rel = (0, import_node_path2.relative)(r, f);
+    return rel !== "" && rel !== ".." && !rel.startsWith(`..${import_node_path2.sep}`) && !(0, import_node_path2.isAbsolute)(rel);
+  };
+  return within(root, file) && within((0, import_node_fs2.realpathSync)(root), (0, import_node_fs2.realpathSync)(file));
+}
+async function loadPackagePlugins(registry, projectRoot = process.cwd()) {
+  let project;
+  try {
+    project = readJson((0, import_node_path2.join)(projectRoot, "package.json"));
+  } catch (err) {
+    console.error(`attest: cannot read ${(0, import_node_path2.join)(projectRoot, "package.json")}; no plugins loaded:`, err);
+    return 0;
+  }
+  const depNames = /* @__PURE__ */ new Set();
+  for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) {
+    const deps = project[field];
+    if (typeof deps === "object" && deps !== null) {
+      for (const dep of Object.keys(deps)) depNames.add(dep);
+    }
+  }
+  let loaded = 0;
+  for (const dep of depNames) {
+    const pkgDir = findInstalledPackage(projectRoot, dep);
+    if (pkgDir === void 0) continue;
+    let entries;
+    try {
+      entries = declaredPluginPaths(readJson((0, import_node_path2.join)(pkgDir, "package.json")), dep);
+    } catch (err) {
+      console.error(`attest: failed to read plugins declared by '${dep}':`, err);
+      continue;
+    }
+    for (const entry of entries) {
+      try {
+        const file = (0, import_node_path2.resolve)(pkgDir, entry);
+        if (!isInside(pkgDir, file)) {
+          throw new Error(`plugin path '${entry}' escapes package '${dep}'`);
+        }
+        const mod = await import((0, import_node_url2.pathToFileURL)(file).href);
+        const plugins = pluginsOf(mod);
+        if (plugins.length === 0) {
+          throw new Error(`'${entry}' exports no plugin (name, pluginType, execute)`);
+        }
+        for (const plugin of plugins) {
+          registry.register(plugin.name, plugin);
+          loaded++;
+        }
+      } catch (err) {
+        console.error(`attest: failed to load plugin '${entry}' from '${dep}':`, err);
+      }
+    }
+  }
+  return loaded;
+}
 
 // src/continuous.ts
 var import_node_events = require("events");
@@ -3881,6 +4029,172 @@ the assistant's response meets your needs. Keep responses concise (1-3 sentences
   max_tokens: 200
 };
 
+// src/simulation/mock-tools.ts
+var MockToolRegistry = class {
+  tools = /* @__PURE__ */ new Map();
+  register(name, fn) {
+    this.tools.set(name, fn);
+  }
+  /**
+   * Run `fn` with the registered mocks active. `TraceBuilder.addToolCall`
+   * returns the mock's result for a registered tool name. Scopes are
+   * isolated per async call chain (AsyncLocalStorage).
+   */
+  run(fn) {
+    return activeMockRegistry.run(new Map(this.tools), fn);
+  }
+};
+function mockTool(name, fn) {
+  const wrapper = ((args) => fn(args));
+  return Object.assign(wrapper, { mockToolName: name });
+}
+
+// src/simulation/fault-inject.ts
+function faultInject(options2, fn) {
+  const errorRate = options2.errorRate ?? 0;
+  const latencyJitterMs = options2.latencyJitterMs ?? 0;
+  const rng = createRng(options2.seed);
+  return async (...args) => {
+    if (errorRate > 0 && rng() < errorRate) {
+      throw new Error(`Injected fault in ${fn.name || "anonymous"}`);
+    }
+    if (latencyJitterMs > 0) {
+      await sleep(rng() * latencyJitterMs);
+    }
+    return fn(...args);
+  };
+}
+
+// src/simulation/verdict.ts
+function aggregateOutcomes(outcomes) {
+  if (outcomes.includes(OUTCOME_FAIL)) return OUTCOME_FAIL;
+  if (outcomes.includes(OUTCOME_SKIP)) return OUTCOME_SKIP;
+  return OUTCOME_PASS;
+}
+function sumNotEvaluatedReasons(results) {
+  const counts = {};
+  for (const r of results) {
+    for (const [reason, n] of Object.entries(r.notEvaluatedReasons)) {
+      counts[reason] = (counts[reason] ?? 0) + n;
+    }
+  }
+  return counts;
+}
+function formatSkipReason(counts) {
+  const parts = Object.entries(counts).map(([reason, n]) => n === 1 ? reason : `${reason} \xD7${n}`);
+  return `not evaluated: ${parts.join(", ")}`;
+}
+function renderVerdictDiagnostics(verdict, color = false) {
+  return verdict instanceof AgentResult ? renderDiagnostics(verdict, { color }) : verdict.renderDiagnostics?.({ color }) ?? "";
+}
+function verdictsDiagnostics(results, label, color = false) {
+  const blocks = results.flatMap(
+    (r, i) => r.outcome() === OUTCOME_PASS ? [] : [`${label} ${i + 1}/${results.length}:${renderVerdictDiagnostics(r, color)}`]
+  );
+  return blocks.length > 0 ? blocks.join("\n") : `  (no failing ${label}s)
+`;
+}
+
+// src/simulation/repeat.ts
+var RepeatResult = class {
+  results;
+  constructor(results = []) {
+    this.results = results;
+  }
+  get count() {
+    return this.results.length;
+  }
+  /** Trials skipped because an assertion was not evaluated. */
+  get notEvaluated() {
+    return this.results.filter((r) => r.outcome() === OUTCOME_SKIP).length;
+  }
+  /** Passing / evaluated trials; null when no trial was evaluated. */
+  get passRate() {
+    const evaluated = this.count - this.notEvaluated;
+    if (evaluated === 0) return null;
+    return this.results.filter((r) => r.outcome() === OUTCOME_PASS).length / evaluated;
+  }
+  /** True when every trial's outcome is pass (soft failures included). */
+  get allPassed() {
+    return this.results.every((r) => r.outcome() === OUTCOME_PASS);
+  }
+  outcome(options2) {
+    return aggregateOutcomes(this.results.map((r) => r.outcome(options2)));
+  }
+  get notEvaluatedReasons() {
+    return sumNotEvaluatedReasons(this.results);
+  }
+  skipReason() {
+    return formatSkipReason(this.notEvaluatedReasons);
+  }
+  renderDiagnostics(options2) {
+    return verdictsDiagnostics(this.results, "trial", options2?.color);
+  }
+};
+function repeat(n, fn) {
+  return async (...args) => {
+    const results = [];
+    for (let i = 0; i < n; i++) {
+      results.push(await fn(...args));
+    }
+    return new RepeatResult(results);
+  };
+}
+
+// src/simulation/scenario.ts
+var ScenarioResult = class {
+  config;
+  results;
+  constructor(config2, results = []) {
+    this.config = config2;
+    this.results = results;
+  }
+  /** True when the aggregate `outcome()` is pass. */
+  get passed() {
+    return this.outcome() === OUTCOME_PASS;
+  }
+  outcome(options2) {
+    return aggregateOutcomes(this.results.map((r) => r.outcome(options2)));
+  }
+  get notEvaluatedReasons() {
+    return sumNotEvaluatedReasons(this.results);
+  }
+  skipReason() {
+    return formatSkipReason(this.notEvaluatedReasons);
+  }
+  renderDiagnostics(options2) {
+    return verdictsDiagnostics(this.results, "turn", options2?.color);
+  }
+};
+function scenario(options2, fn) {
+  const config2 = {
+    persona: options2.persona,
+    mockTools: { ...options2.mockTools },
+    faultErrorRate: options2.faultErrorRate ?? 0,
+    faultLatencyJitterMs: options2.faultLatencyJitterMs ?? 0,
+    maxTurns: options2.maxTurns ?? 1,
+    seed: options2.seed
+  };
+  return async (...args) => {
+    const run = async () => {
+      const results = [];
+      const rng = createRng(config2.seed);
+      for (let turn = 0; turn < config2.maxTurns; turn++) {
+        if (config2.faultErrorRate > 0 && rng() < config2.faultErrorRate) {
+          throw new Error(`Injected fault at turn ${turn}`);
+        }
+        if (config2.faultLatencyJitterMs > 0) {
+          await sleep(rng() * config2.faultLatencyJitterMs);
+        }
+        results.push(await fn({ persona: config2.persona, turn }, ...args));
+      }
+      return new ScenarioResult(config2, results);
+    };
+    const mocks = new Map(Object.entries(config2.mockTools));
+    return mocks.size > 0 ? activeMockRegistry.run(mocks, run) : run();
+  };
+}
+
 // src/adapters/manual.ts
 var ManualAdapter = class {
   agentId;
@@ -4092,7 +4406,7 @@ var OTelAdapter = class _OTelAdapter {
   }
 };
 function runConvert(engine, args, payload) {
-  return new Promise((resolve4, reject) => {
+  return new Promise((resolve5, reject) => {
     const child = (0, import_node_child_process5.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
     let timer;
     child.on("error", (e) => {
@@ -4145,7 +4459,7 @@ function runConvert(engine, args, payload) {
         );
         return;
       }
-      resolve4(Buffer.concat(out).toString("utf8"));
+      resolve5(Buffer.concat(out).toString("utf8"));
     });
     stdin.end(payload);
   });
@@ -4313,6 +4627,7 @@ var LangChainAdapter = class {
   GeminiAdapter,
   LangChainAdapter,
   ManualAdapter,
+  MockToolRegistry,
   OTLPConversionError,
   OTelAdapter,
   OUTCOME_FAIL,
@@ -4332,6 +4647,7 @@ var LangChainAdapter = class {
   REASON_REPLAY_MISS,
   REASON_SHORT_CIRCUITED,
   REASON_SIMULATION,
+  RepeatResult,
   ReplayError,
   ReplayMissError,
   ReplayNotEvaluatedError,
@@ -4344,6 +4660,7 @@ var LangChainAdapter = class {
   STEP_RETRIEVAL,
   STEP_TOOL_CALL,
   Sampler,
+  ScenarioResult,
   TIER_1,
   TIER_2,
   TIER_3,
@@ -4378,6 +4695,7 @@ var LangChainAdapter = class {
   executePluginAssertion,
   extractId,
   extractResult,
+  faultInject,
   firstMiss,
   formatUsage,
   getSampleRate,
@@ -4389,17 +4707,23 @@ var LangChainAdapter = class {
   loadFixture,
   loadLabelsCSV,
   loadLabelsJSONL,
+  loadPackagePlugins,
+  mockTool,
   parseUsage,
   previewLine,
   promptHash,
   record,
+  registerPlugin,
   renderAssertionFailure,
   renderDiagnostics,
   renderSummary,
+  renderVerdictDiagnostics,
   renderVitest,
+  repeat,
   requireSimulationCapability,
   resetConfig,
   resolveMode,
+  scenario,
   simulationEnvEnabled,
   startReplay,
   startWorld,