@attest-ai/core 0.6.0 → 0.9.0

package/dist/esm/index.js

@@ -31,6 +31,27 @@ function assertionId(id) {
 function agentId(id) {
   return id;
 }
+function layerForType(assertionType) {
+  switch (assertionType) {
+    case "schema":
+      return 1;
+    case "constraint":
+      return 2;
+    case "trace":
+    case "trace_tree":
+      return 3;
+    case "content":
+      return 4;
+    case "embedding":
+      return 5;
+    case "llm_judge":
+      return 6;
+    case "plugin":
+      return 8;
+    default:
+      return 0;
+  }
+}
 
 // src/proto/errors.ts
 var ProtocolError = class extends Error {
@@ -57,6 +78,58 @@ var EngineTimeoutError = class extends Error {
     this.timeoutMs = timeoutMs;
   }
 };
+var ProtocolDesyncError = class extends Error {
+  diagnostics;
+  constructor(message, diagnostics) {
+    super(message);
+    this.name = "ProtocolDesyncError";
+    this.diagnostics = diagnostics;
+  }
+};
+
+// src/proto/diagnostics.ts
+var DEFAULT_BUFFER_SIZE = 32;
+var MAX_LINE_PREVIEW = 512;
+var ProtocolDiagnosticBuffer = class {
+  capacity;
+  entries = [];
+  constructor(capacity = DEFAULT_BUFFER_SIZE) {
+    if (capacity <= 0) {
+      throw new Error(
+        `ProtocolDiagnosticBuffer capacity must be positive, got ${capacity}`
+      );
+    }
+    this.capacity = capacity;
+  }
+  push(diagnostic) {
+    if (this.entries.length >= this.capacity) {
+      this.entries.shift();
+    }
+    this.entries.push(diagnostic);
+  }
+  snapshot() {
+    return this.entries.slice();
+  }
+  countWithin(windowMs, nowMs) {
+    const cutoff = nowMs - windowMs;
+    let count = 0;
+    for (let i = this.entries.length - 1; i >= 0; i -= 1) {
+      if (this.entries[i].timestampMs >= cutoff) {
+        count += 1;
+      } else {
+        break;
+      }
+    }
+    return count;
+  }
+  clear() {
+    this.entries.length = 0;
+  }
+};
+function previewLine(line) {
+  if (line.length <= MAX_LINE_PREVIEW) return line;
+  return line.slice(0, MAX_LINE_PREVIEW) + "\u2026";
+}
 
 // src/proto/codec.ts
 function encodeRequest(id, method, params) {
@@ -119,8 +192,8 @@ function extractId(response) {
 }
 
 // src/version.ts
-var VERSION = "0.6.0";
-var ENGINE_VERSION = "0.6.0";
+var VERSION = "0.9.0";
+var ENGINE_VERSION = "0.9.0";
 
 // src/engine-manager.ts
 import { spawn, execSync } from "child_process";
@@ -472,12 +545,16 @@ function simulationEvaluateBatch(assertions) {
   return {
     results,
     total_cost: 0,
-    total_duration_ms: 0
+    total_duration_ms: 0,
+    simulated: true
   };
 }
 
 // src/client.ts
 var DEFAULT_TIMEOUT_MS = 3e4;
+var DEFAULT_DIAGNOSTIC_BUFFER = 32;
+var DEFAULT_DESYNC_THRESHOLD = 3;
+var DEFAULT_DESYNC_WINDOW_MS = 1e3;
 function resolveTimeoutMs(override) {
   if (override !== void 0) return override;
   const env = process.env["ATTEST_ENGINE_TIMEOUT"];
@@ -499,6 +576,24 @@ function makeTimeoutPromise(method, timeoutMs) {
     }, timeoutMs);
   });
 }
+function debugProtocolEnabled() {
+  const flag = process.env["ATTEST_DEBUG_PROTOCOL"];
+  return flag === "1" || flag === "true" || flag === "yes";
+}
+function envLoggerDefault() {
+  if (debugProtocolEnabled()) {
+    return {
+      warn: (msg, ...args) => console.warn(msg, ...args),
+      error: (msg, ...args) => console.error(msg, ...args)
+    };
+  }
+  return {
+    warn: () => {
+    },
+    error: () => {
+    }
+  };
+}
 var AttestClient = class {
   engine;
   requestId = 0;
@@ -506,8 +601,34 @@ var AttestClient = class {
   writeChain = Promise.resolve();
   readerActive = false;
   lineHandler = null;
-  constructor(engine) {
+  logger;
+  diagnosticBuffer;
+  diagnosticListeners = /* @__PURE__ */ new Set();
+  desyncThreshold;
+  desyncWindowMs;
+  desynced = false;
+  constructor(engine, options) {
     this.engine = engine;
+    this.logger = options?.logger ?? envLoggerDefault();
+    this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
+      options?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
+    );
+    this.desyncThreshold = options?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
+    this.desyncWindowMs = options?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
+    if (options?.onDiagnostic !== void 0) {
+      this.diagnosticListeners.add(options.onDiagnostic);
+    }
+  }
+  /** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
+  onProtocolDiagnostic(listener) {
+    this.diagnosticListeners.add(listener);
+    return () => {
+      this.diagnosticListeners.delete(listener);
+    };
+  }
+  /** Snapshot of the diagnostic ring buffer (most recent last). */
+  protocolDiagnostics() {
+    return this.diagnosticBuffer.snapshot();
   }
   startReader() {
     if (this.readerActive) return;
@@ -532,35 +653,101 @@ var AttestClient = class {
     this.lineHandler = null;
     this.readerActive = false;
   }
+  recordDiagnostic(kind, message, rawLine) {
+    const diagnostic = {
+      kind,
+      message,
+      rawLine: previewLine(rawLine),
+      timestampMs: Date.now()
+    };
+    this.diagnosticBuffer.push(diagnostic);
+    this.logger.warn(`[attest.protocol] ${kind}: ${message}`);
+    for (const listener of this.diagnosticListeners) {
+      try {
+        listener(diagnostic);
+      } catch (err) {
+        this.logger.error(
+          `[attest.protocol] diagnostic listener threw: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
+    this.maybeTriggerDesync(diagnostic.timestampMs);
+  }
+  maybeTriggerDesync(nowMs) {
+    if (this.desynced) return;
+    const recent = this.diagnosticBuffer.countWithin(this.desyncWindowMs, nowMs);
+    if (recent >= this.desyncThreshold) {
+      this.desynced = true;
+      const snapshot = this.diagnosticBuffer.snapshot();
+      const err = new ProtocolDesyncError(
+        `Engine protocol desync: ${recent} unroutable response lines within ${this.desyncWindowMs}ms.`,
+        snapshot
+      );
+      this.logger.error(`[attest.protocol] ${err.message}`);
+      this.failAll(err);
+    }
+  }
   handleLine(line) {
+    let response;
+    try {
+      response = decodeResponse(line);
+    } catch (err) {
+      this.handleDecodeFailure(line, err);
+      return;
+    }
     let reqId;
     try {
-      const response = decodeResponse(line);
       reqId = extractId(response);
-      const pending = this.pending.get(reqId);
-      if (pending === void 0) return;
-      this.pending.delete(reqId);
-      try {
-        const result = extractResult(response);
-        pending.resolve(result);
-      } catch (err) {
-        pending.reject(err);
-      }
     } catch (err) {
-      if (err instanceof ProtocolError) {
-        try {
-          const raw = JSON.parse(line.trim());
-          reqId = Number(raw.id ?? -1);
-        } catch {
-          reqId = -1;
-        }
-        const pending = this.pending.get(reqId);
-        if (pending !== void 0) {
-          this.pending.delete(reqId);
-          pending.reject(err);
+      const message = err instanceof Error ? err.message : String(err);
+      this.recordDiagnostic("missing_id", message, line);
+      return;
+    }
+    const pending = this.pending.get(reqId);
+    if (pending === void 0) {
+      this.recordDiagnostic(
+        "unknown_id",
+        `no pending request for id=${reqId}`,
+        line
+      );
+      return;
+    }
+    this.pending.delete(reqId);
+    try {
+      const result = extractResult(response);
+      pending.resolve(result);
+    } catch (err) {
+      pending.reject(err);
+    }
+  }
+  handleDecodeFailure(line, err) {
+    if (err instanceof ProtocolError) {
+      let reqId = -1;
+      try {
+        const raw = JSON.parse(line.trim());
+        const idCandidate = raw["id"];
+        if (typeof idCandidate === "number" && Number.isFinite(idCandidate)) {
+          reqId = idCandidate;
         }
+      } catch {
+        reqId = -1;
+      }
+      const pending = this.pending.get(reqId);
+      if (pending !== void 0) {
+        this.pending.delete(reqId);
+        pending.reject(err);
+        return;
       }
+      this.recordDiagnostic(
+        "non_routable_error",
+        `engine error code=${err.code} (${err.errorMessage}) had no matching pending id`,
+        line
+      );
+      return;
     }
+    const message = err instanceof Error ? err.message : String(err);
+    const kind = classifyDecodeError(message);
+    this.recordDiagnostic(kind, message, line);
   }
   failAll(err) {
     for (const pending of this.pending.values()) {
@@ -569,6 +756,12 @@ var AttestClient = class {
     this.pending.clear();
   }
   async sendRequest(method, params, timeoutMs) {
+    if (this.desynced) {
+      throw new ProtocolDesyncError(
+        "Engine protocol is desynced; AttestClient will not send further requests.",
+        this.diagnosticBuffer.snapshot()
+      );
+    }
     const resolvedTimeout = resolveTimeoutMs(timeoutMs);
     if (!this.readerActive) {
       return Promise.race([
@@ -679,6 +872,13 @@ var AttestClient = class {
     return String(raw.message);
   }
 };
+function classifyDecodeError(message) {
+  if (message.startsWith("malformed JSON")) return "malformed_json";
+  if (message.startsWith("expected JSON object")) return "non_object_response";
+  if (message.startsWith("invalid jsonrpc version")) return "invalid_jsonrpc_version";
+  if (message.startsWith("empty response line")) return "malformed_json";
+  return "malformed_json";
+}
 
 // src/trace.ts
 import { randomUUID } from "crypto";
@@ -800,6 +1000,173 @@ var AgentResult = class {
   }
 };
 
+// src/diagnostics.ts
+var ANSI_RESET = "\x1B[0m";
+var ANSI_RED = "\x1B[31m";
+var ANSI_YELLOW = "\x1B[33m";
+var ANSI_GREEN = "\x1B[32m";
+var ANSI_DIM = "\x1B[2m";
+var ANSI_BOLD = "\x1B[1m";
+var LAYER_NAMES = {
+  1: "schema",
+  2: "constraint",
+  3: "trace",
+  4: "content",
+  5: "embedding",
+  6: "llm_judge",
+  7: "trace_tree",
+  8: "plugin"
+};
+function layerLabel(layer, assertionType) {
+  if (layer === 0 && assertionType) {
+    return `L? ${assertionType}`;
+  }
+  const name = LAYER_NAMES[layer] ?? assertionType ?? "uncategorized";
+  return `L${layer} ${name}`;
+}
+function statusGlyph(status, color) {
+  let glyph;
+  let c;
+  if (status === STATUS_PASS) {
+    glyph = "PASS";
+    c = ANSI_GREEN;
+  } else if (status === STATUS_SOFT_FAIL) {
+    glyph = "SOFT";
+    c = ANSI_YELLOW;
+  } else if (status === STATUS_HARD_FAIL) {
+    glyph = "FAIL";
+    c = ANSI_RED;
+  } else {
+    glyph = status.toUpperCase() || "????";
+    c = ANSI_DIM;
+  }
+  return color ? `${c}${glyph}${ANSI_RESET}` : glyph;
+}
+function bold(s, color) {
+  return color ? `${ANSI_BOLD}${s}${ANSI_RESET}` : s;
+}
+function dim(s, color) {
+  return color ? `${ANSI_DIM}${s}${ANSI_RESET}` : s;
+}
+function truncate(value, limit = 280) {
+  if (value.length <= limit) {
+    return value;
+  }
+  return value.slice(0, limit - 3) + "...";
+}
+function renderAssertionFailure(result, options = {}) {
+  const color = options.color ?? false;
+  const testFile = options.testFile;
+  const layer = result.layer ?? 0;
+  const label = layerLabel(layer, result.type);
+  const lines = [];
+  lines.push(
+    `  ${statusGlyph(result.status, color)} ${bold(result.assertion_id, color)} ${dim(`\u2014 ${label}`, color)}`
+  );
+  if (result.trace_node_path) {
+    lines.push(`      trace path: ${result.trace_node_path}`);
+  }
+  lines.push(`      score:      ${result.score.toFixed(3)}`);
+  if (result.expected) {
+    lines.push(`      expected:   ${truncate(result.expected)}`);
+  }
+  if (result.actual) {
+    lines.push(`      actual:     ${truncate(result.actual)}`);
+  }
+  if (result.explanation) {
+    lines.push(`      detail:     ${truncate(result.explanation)}`);
+  }
+  if (result.threshold_source && result.threshold_source !== "static") {
+    lines.push(`      threshold:  ${result.threshold_source}`);
+  }
+  if (result.judge_metadata) {
+    appendJudgeMetadata(lines, result.judge_metadata);
+  }
+  if (result.suggested_action) {
+    lines.push(`      hint:       ${truncate(result.suggested_action)}`);
+  }
+  if ((result.cost ?? 0) > 0 || (result.duration_ms ?? 0) > 0) {
+    const cost = result.cost ?? 0;
+    const durMs = result.duration_ms ?? 0;
+    lines.push(`      cost/lat:   $${cost.toFixed(6)} / ${durMs}ms`);
+  }
+  if (testFile) {
+    lines.push(`      source:     ${testFile}`);
+  }
+  return lines.join("\n") + "\n";
+}
+function appendJudgeMetadata(lines, meta) {
+  if (meta.model) {
+    let rubric = meta.rubric_name ?? "default";
+    if (meta.rubric_version) {
+      rubric = `${rubric} @ ${meta.rubric_version}`;
+    }
+    lines.push(`      judge:      ${meta.model} / ${rubric}`);
+  }
+  if (meta.prompt_hash) {
+    lines.push(`      prompt:     #${meta.prompt_hash}`);
+  }
+  const samples = meta.sample_scores ?? [];
+  if (samples.length > 1) {
+    const formatted = samples.map((s) => s.toFixed(2)).join(", ");
+    const mean = (meta.score_mean ?? 0).toFixed(2);
+    const stddev = (meta.score_stddev ?? 0).toFixed(2);
+    const flag = meta.high_variance ? " \u26A0 HIGH VARIANCE" : "";
+    lines.push(
+      `      samples:    [${formatted}] mean=${mean} stddev=${stddev}${flag}`
+    );
+  }
+  const probes = meta.bias_probes ?? [];
+  if (probes.length > 0) {
+    const formatted = probes.map((p) => `${p.name} \u0394${p.delta >= 0 ? "+" : ""}${p.delta.toFixed(2)}`).join(", ");
+    lines.push(`      bias:       ${formatted}`);
+  }
+  if (meta.calibration) {
+    const c = meta.calibration;
+    lines.push(
+      `      calibrated: ${c.label_count} labels, agreement=${c.agreement.toFixed(2)}, \u03BA=${c.cohen_kappa.toFixed(2)}`
+    );
+  }
+}
+function renderSummary(result, options = {}) {
+  const color = options.color ?? false;
+  const passN = result.passCount;
+  const soft = result.assertionResults.filter(
+    (r) => r.status === STATUS_SOFT_FAIL
+  ).length;
+  const hard = result.assertionResults.filter(
+    (r) => r.status === STATUS_HARD_FAIL
+  ).length;
+  const parts = [
+    `${statusGlyph(STATUS_PASS, color)} ${passN}`,
+    `${statusGlyph(STATUS_SOFT_FAIL, color)} ${soft}`,
+    `${statusGlyph(STATUS_HARD_FAIL, color)} ${hard}`,
+    `cost $${result.totalCost.toFixed(6)}`,
+    `dur ${result.totalDurationMs}ms`
+  ];
+  return parts.join(" | ");
+}
+function renderDiagnostics(result, options = {}) {
+  const color = options.color ?? false;
+  const includePassing = options.includePassing ?? false;
+  const rows = includePassing ? result.assertionResults : result.failedAssertions;
+  if (rows.length === 0) {
+    return dim("  (no failures)\n", color);
+  }
+  const out = [""];
+  out.push(
+    bold(
+      `Attest diagnostic \u2014 ${rows.length} of ${result.assertionResults.length} assertions failed:`,
+      color
+    )
+  );
+  for (const r of rows) {
+    out.push(renderAssertionFailure(r, options));
+  }
+  out.push(dim(`  Summary: ${renderSummary(result, { color })}`, color));
+  return out.join("\n");
+}
+
 // src/trace-tree.ts
 var TraceTree = class {
   root;
@@ -2004,6 +2371,222 @@ var ContinuousEvalRunner = class {
   }
 };
 
+// src/calibration.ts
+import { createHash as createHash2 } from "crypto";
+function computeAgreement(pairs, threshold) {
+  if (pairs.length === 0) {
+    throw new Error("no labeled pairs available");
+  }
+  if (threshold <= 0 || threshold >= 1) {
+    throw new Error("threshold must be in (0, 1)");
+  }
+  const n = pairs.length;
+  let humanPos = 0;
+  let judgePos = 0;
+  let bothPos = 0;
+  let bothNeg = 0;
+  for (const p of pairs) {
+    const hPos = p.human >= threshold;
+    const jPos = p.judge >= threshold;
+    if (hPos) humanPos++;
+    if (jPos) judgePos++;
+    if (hPos && jPos) bothPos++;
+    if (!hPos && !jPos) bothNeg++;
+  }
+  const humanNeg = n - humanPos;
+  const judgeNeg = n - judgePos;
+  const agreement = (bothPos + bothNeg) / n;
+  const expected = (humanPos * judgePos + humanNeg * judgeNeg) / (n * n);
+  const cohenKappa = expected < 1 ? (agreement - expected) / (1 - expected) : 0;
+  const rocAuc = rocAUC(pairs, threshold);
+  return { threshold, n, agreement, cohenKappa, rocAuc };
+}
+function rocAUC(pairs, threshold) {
+  const rows = pairs.map((p) => ({
+    score: p.judge,
+    pos: p.human >= threshold
+  }));
+  const pos = rows.filter((r) => r.pos).length;
+  const neg = rows.length - pos;
+  if (pos === 0 || neg === 0) return 0;
+  rows.sort((a, b) => a.score - b.score);
+  let rankSumPos = 0;
+  let i = 0;
+  while (i < rows.length) {
+    let j = i;
+    while (j < rows.length && rows[j].score === rows[i].score) {
+      j++;
+    }
+    const avgRank = (i + j + 1) / 2;
+    for (let k = i; k < j; k++) {
+      if (rows[k].pos) rankSumPos += avgRank;
+    }
+    i = j;
+  }
+  const auc = (rankSumPos - pos * (pos + 1) / 2) / (pos * neg);
+  if (Number.isNaN(auc) || !Number.isFinite(auc)) return 0;
+  return auc;
+}
+function loadLabelsCSV(text) {
+  const rows = parseCSV(text);
+  if (rows.length === 0) {
+    throw new Error("empty CSV");
+  }
+  let start = 0;
+  if (rows[0].length >= 2) {
+    const firstHuman = parseFloatStrict(rows[0][1]);
+    if (firstHuman === null) start = 1;
+  }
+  const out = [];
+  for (let idx = start; idx < rows.length; idx++) {
+    const row = rows[idx];
+    const lineNum = idx + 1;
+    if (row.length === 0) continue;
+    if (row[0].trimStart().startsWith("#")) continue;
+    if (row.length < 2) {
+      throw new Error(
+        `CSV line ${lineNum}: want at least 2 columns (input, human_label)`
+      );
+    }
+    const human = parseFloatStrict(row[1]);
+    if (human === null) {
+      throw new Error(`CSV line ${lineNum}: human_label not a float: ${row[1]}`);
+    }
+    const rec = {
+      input: row[0],
+      humanLabel: human,
+      judgeScore: 0,
+      judgeKnown: false
+    };
+    if (row.length >= 3 && row[2].trim() !== "") {
+      const judge = parseFloatStrict(row[2]);
+      if (judge === null) {
+        throw new Error(
+          `CSV line ${lineNum}: judge_score not a float: ${row[2]}`
+        );
+      }
+      rec.judgeScore = judge;
+      rec.judgeKnown = true;
+    }
+    out.push(rec);
+  }
+  if (out.length === 0) {
+    throw new Error("CSV contained no labeled rows");
+  }
+  return out;
+}
+function loadLabelsJSONL(text) {
+  const out = [];
+  const lines = text.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const stripped = lines[i].trim();
+    if (!stripped) continue;
+    let obj;
+    try {
+      obj = JSON.parse(stripped);
+    } catch (err) {
+      throw new Error(`JSONL line ${i + 1}: ${err.message}`);
+    }
+    if (Object.keys(obj).length === 0) continue;
+    if (obj["human_label"] === void 0) {
+      throw new Error(`JSONL line ${i + 1}: missing human_label`);
+    }
+    const human = parseJSONNumber(obj["human_label"]);
+    if (human === null) {
+      throw new Error(`JSONL line ${i + 1}: human_label is not a number`);
+    }
+    const rec = {
+      input: typeof obj["input"] === "string" ? obj["input"] : "",
+      humanLabel: human,
+      judgeScore: 0,
+      judgeKnown: false
+    };
+    if (obj["judge_score"] !== void 0 && obj["judge_score"] !== null) {
+      const judge = parseJSONNumber(obj["judge_score"]);
+      if (judge === null) {
+        throw new Error(`JSONL line ${i + 1}: judge_score is not a number`);
+      }
+      rec.judgeScore = judge;
+      rec.judgeKnown = true;
+    }
+    out.push(rec);
+  }
+  if (out.length === 0) {
+    throw new Error("JSONL contained no labeled rows");
+  }
+  return out;
+}
+function promptHash(text) {
+  return createHash2("sha256").update(text).digest("hex").slice(0, 16);
+}
+function parseFloatStrict(raw) {
+  const trimmed = raw.trim();
+  if (trimmed === "") return null;
+  const n = Number(trimmed);
+  if (Number.isNaN(n)) return null;
+  return n;
+}
+function parseJSONNumber(value) {
+  if (typeof value !== "number") return null;
+  if (Number.isNaN(value)) return null;
+  return value;
+}
+function parseCSV(text) {
+  const rows = [];
+  let current = [];
+  let field = "";
+  let inQuotes = false;
+  let i = 0;
+  while (i < text.length) {
+    const ch = text[i];
+    if (inQuotes) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i += 2;
+          continue;
+        }
+        inQuotes = false;
+        i++;
+        continue;
+      }
+      field += ch;
+      i++;
+      continue;
+    }
+    if (ch === '"') {
+      inQuotes = true;
+      i++;
+      continue;
+    }
+    if (ch === ",") {
+      current.push(field);
+      field = "";
+      i++;
+      continue;
+    }
+    if (ch === "\n") {
+      current.push(field);
+      rows.push(current);
+      current = [];
+      field = "";
+      i++;
+      continue;
+    }
+    if (ch === "\r") {
+      i++;
+      continue;
+    }
+    field += ch;
+    i++;
+  }
+  if (field.length > 0 || current.length > 0) {
+    current.push(field);
+    rows.push(current);
+  }
+  return rows;
+}
+
 // src/personas.ts
 var FRIENDLY_USER = {
   name: "FriendlyUser",
@@ -2445,6 +3028,8 @@ export {
   OllamaAdapter,
   OpenAIAdapter,
   PluginRegistry,
+  ProtocolDesyncError,
+  ProtocolDiagnosticBuffer,
   ProtocolError,
   STATUS_HARD_FAIL,
   STATUS_PASS,
@@ -2473,6 +3058,7 @@ export {
   agentId,
   assertionId,
   attestExpect,
+  computeAgreement,
   config,
   decodeResponse,
   delegate,
@@ -2482,6 +3068,14 @@ export {
   extractResult,
   getSampleRate,
   isSimulationMode,
+  layerForType,
+  loadLabelsCSV,
+  loadLabelsJSONL,
+  previewLine,
+  promptHash,
+  renderAssertionFailure,
+  renderDiagnostics,
+  renderSummary,
   resetConfig,
   simulationEvaluateBatch,
   tier,