@botpress/evals 1.18.3

package/dist/graders/llm.js

@@ -0,0 +1,125 @@
+// src/graders/llm.ts
+import { Client } from "@botpress/client";
+import { Cognitive } from "@botpress/cognitive";
+var DEFAULT_PASS_THRESHOLD = 3;
+function buildJudgeSystemPrompt(passThreshold) {
+  return `You are an evaluation judge for a chatbot. You will be given:
+- The user's message
+- The bot's response
+- Grading criteria
+
+Your job is to determine if the bot's response meets the criteria.
+
+Respond with a JSON object:
+{
+  "pass": true/false,
+  "score": 1-5,
+  "reason": "brief explanation"
+}
+
+Scoring guide:
+- 5: Fully meets criteria, excellent response
+- 4: Mostly meets criteria, minor issues
+- 3: Partially meets criteria, notable gaps
+- 2: Barely meets criteria, significant issues
+- 1: Does not meet criteria
+
+A score of ${passThreshold} or above is a pass.`;
+}
+var MODEL_ALIASES = ["fast", "best"];
+var _cognitive = null;
+var _availableModels = [];
+var _judgeModel = "fast";
+function _isModelValid(model) {
+  if (MODEL_ALIASES.includes(model)) return true;
+  if (_availableModels.length === 0) return false;
+  return _availableModels.includes(model);
+}
+async function initLLMJudge(credentials, options) {
+  const log = options?.logger ?? console;
+  const client = new Client({
+    token: credentials.token,
+    apiUrl: credentials.apiUrl,
+    botId: credentials.botId,
+    ...credentials.workspaceId !== void 0 ? { workspaceId: credentials.workspaceId } : {}
+  });
+  _cognitive = new Cognitive({ client, __experimental_beta: true });
+  try {
+    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys());
+  } catch (err) {
+    log.warn(`Failed to fetch available models: ${err.message}. LLM judge will be unavailable.`);
+    _availableModels = [];
+  }
+  _judgeModel = options?.model ?? "fast";
+  if (_availableModels.length === 0) {
+    log.warn("No available Cognitive models: cannot validate judge model.");
+  } else if (!_isModelValid(_judgeModel)) {
+    log.warn(`Configured LLM judge model "${_judgeModel}" is invalid. Run "adk models" to list available models.`);
+  }
+}
+async function gradeLLMJudge(botResponse, criteria, context) {
+  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
+  try {
+    if (!_cognitive) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: true,
+        expected: criteria,
+        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+      };
+    }
+    if (!_isModelValid(_judgeModel)) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: false,
+        expected: criteria,
+        actual: "FAILED \u2014 invalid LLM judge model"
+      };
+    }
+    const { output } = await _cognitive.generateContent({
+      model: _judgeModel,
+      temperature: 0,
+      responseFormat: "json_object",
+      systemPrompt: buildJudgeSystemPrompt(threshold),
+      messages: [
+        {
+          role: "user",
+          content: `User message: ${context.userMessage}
+
+Bot response: ${botResponse}
+
+Criteria: ${criteria}`
+        }
+      ]
+    });
+    const rawContent = output?.choices?.[0]?.content;
+    const content = typeof rawContent === "string" ? rawContent : null;
+    if (!content) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: true,
+        expected: criteria,
+        actual: "SKIPPED \u2014 LLM judge returned empty response"
+      };
+    }
+    const verdict = JSON.parse(content);
+    return {
+      assertion: `llm_judge: "${criteria}"`,
+      pass: verdict.score >= threshold,
+      expected: criteria,
+      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
+    };
+  } catch (err) {
+    return {
+      assertion: `llm_judge: "${criteria}"`,
+      pass: true,
+      expected: criteria,
+      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+    };
+  }
+}
+export {
+  gradeLLMJudge,
+  initLLMJudge
+};
+//# sourceMappingURL=llm.js.map

package/dist/graders/llm.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/llm.ts"],
+  "sourcesContent": ["/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? console\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    if (!_cognitive) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: true,\n      expected: criteria,\n      actual: `SKIPPED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n"],
+  "mappings": ";AAKA,SAAS,cAAc;AACvB,SAAS,iBAAiB;AAG1B,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAE1B,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAMA,eAAsB,aACpB,aACA,SACA;AACA,QAAM,MAAM,SAAS,UAAU;AAE/B,QAAM,SAAS,IAAI,OAAO;AAAA,IACxB,OAAO,YAAY;AAAA,IACnB,QAAQ,YAAY;AAAA,IACpB,OAAO,YAAY;AAAA,IACnB,GAAI,YAAY,gBAAgB,SAAY,EAAE,aAAa,YAAY,YAAY,IAAI,CAAC;AAAA,EAC1F,CAAC;AACD,eAAa,IAAI,UAAU,EAAE,QAAQ,qBAAqB,KAAK,CAAC;AAEhE,MAAI;AACF,uBAAmB,MAAM,MAAM,MAAM,WAAW,kBAAkB,GAAG,KAAK,CAAC;AAAA,EAC7E,SAAS,KAAK;AACZ,QAAI,KAAK,qCAAsC,IAAc,OAAO,kCAAkC;AACtG,uBAAmB,CAAC;AAAA,EACtB;AAEA,gBAAc,SAAS,SAAS;AAEhC,MAAI,iBAAiB,WAAW,GAAG;AACjC,QAAI,KAAK,6DAA6D;AAAA,EACxE,WAAW,CAAC,cAAc,WAAW,GAAG;AACtC,QAAI,KAAK,+BAA+B,WAAW,0DAA0D;AAAA,EAC/G;AACF;AAEA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AACF,QAAI,CAAC,YAAY;AACf,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,yCAAqC,IAAc,OAAO;AAAA,IACpE;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/graders/match.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Shared match utilities for assertion grading.
+ * Used by tool, state, and table graders.
+ */
+import type { MatchOperator } from '../types';
+/**
+ * Evaluate a single match operator against a value.
+ */
+export declare function matchValue(operator: MatchOperator, actual: unknown): boolean;
+export declare function operatorToString(operator: MatchOperator): string;
+//# sourceMappingURL=match.d.ts.map

package/dist/graders/match.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"match.d.ts","sourceRoot":"","sources":["../../src/graders/match.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAE7C;;GAEG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAuC5E;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,aAAa,GAAG,MAAM,CAWhE"}

package/dist/graders/match.js

@@ -0,0 +1,48 @@
+// src/graders/match.ts
+function matchValue(operator, actual) {
+  if (typeof operator === "string") {
+    return String(actual) === operator;
+  }
+  if ("equals" in operator) {
+    return actual === operator.equals;
+  }
+  if ("contains" in operator) {
+    return String(actual).toLowerCase().includes(operator.contains.toLowerCase());
+  }
+  if ("not_contains" in operator) {
+    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase());
+  }
+  if ("matches" in operator) {
+    return new RegExp(operator.matches, "i").test(String(actual));
+  }
+  if ("in" in operator) {
+    return operator.in.includes(actual);
+  }
+  if ("exists" in operator) {
+    return operator.exists ? actual !== void 0 && actual !== null : actual === void 0 || actual === null;
+  }
+  if ("gte" in operator) {
+    return Number(actual) >= operator.gte;
+  }
+  if ("lte" in operator) {
+    return Number(actual) <= operator.lte;
+  }
+  return false;
+}
+function operatorToString(operator) {
+  if (typeof operator === "string") return `equals "${operator}"`;
+  if ("equals" in operator) return `equals ${JSON.stringify(operator.equals)}`;
+  if ("contains" in operator) return `contains "${operator.contains}"`;
+  if ("not_contains" in operator) return `not_contains "${operator.not_contains}"`;
+  if ("matches" in operator) return `matches /${operator.matches}/`;
+  if ("in" in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(", ")}]`;
+  if ("exists" in operator) return operator.exists ? "exists" : "does not exist";
+  if ("gte" in operator) return `>= ${operator.gte}`;
+  if ("lte" in operator) return `<= ${operator.lte}`;
+  return JSON.stringify(operator);
+}
+export {
+  matchValue,
+  operatorToString
+};
+//# sourceMappingURL=match.js.map

package/dist/graders/match.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/match.ts"],
+  "sourcesContent": ["/**\n * Shared match utilities for assertion grading.\n * Used by tool, state, and table graders.\n */\n\nimport type { MatchOperator } from '../types'\n\n/**\n * Evaluate a single match operator against a value.\n */\nexport function matchValue(operator: MatchOperator, actual: unknown): boolean {\n  // Shorthand: string means exact match\n  if (typeof operator === 'string') {\n    return String(actual) === operator\n  }\n\n  if ('equals' in operator) {\n    return actual === operator.equals\n  }\n\n  if ('contains' in operator) {\n    return String(actual).toLowerCase().includes(operator.contains.toLowerCase())\n  }\n\n  if ('not_contains' in operator) {\n    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase())\n  }\n\n  if ('matches' in operator) {\n    return new RegExp(operator.matches, 'i').test(String(actual))\n  }\n\n  if ('in' in operator) {\n    return operator.in.includes(actual)\n  }\n\n  if ('exists' in operator) {\n    return operator.exists ? actual !== undefined && actual !== null : actual === undefined || actual === null\n  }\n\n  if ('gte' in operator) {\n    return Number(actual) >= operator.gte\n  }\n\n  if ('lte' in operator) {\n    return Number(actual) <= operator.lte\n  }\n\n  return false\n}\n\nexport function operatorToString(operator: MatchOperator): string {\n  if (typeof operator === 'string') return `equals \"${operator}\"`\n  if ('equals' in operator) return `equals ${JSON.stringify(operator.equals)}`\n  if ('contains' in operator) return `contains \"${operator.contains}\"`\n  if ('not_contains' in operator) return `not_contains \"${operator.not_contains}\"`\n  if ('matches' in operator) return `matches /${operator.matches}/`\n  if ('in' in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(', ')}]`\n  if ('exists' in operator) return operator.exists ? 'exists' : 'does not exist'\n  if ('gte' in operator) return `>= ${operator.gte}`\n  if ('lte' in operator) return `<= ${operator.lte}`\n  return JSON.stringify(operator)\n}\n"],
+  "mappings": ";AAUO,SAAS,WAAW,UAAyB,QAA0B;AAE5E,MAAI,OAAO,aAAa,UAAU;AAChC,WAAO,OAAO,MAAM,MAAM;AAAA,EAC5B;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,MAAI,cAAc,UAAU;AAC1B,WAAO,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,SAAS,YAAY,CAAC;AAAA,EAC9E;AAEA,MAAI,kBAAkB,UAAU;AAC9B,WAAO,CAAC,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,aAAa,YAAY,CAAC;AAAA,EACnF;AAEA,MAAI,aAAa,UAAU;AACzB,WAAO,IAAI,OAAO,SAAS,SAAS,GAAG,EAAE,KAAK,OAAO,MAAM,CAAC;AAAA,EAC9D;AAEA,MAAI,QAAQ,UAAU;AACpB,WAAO,SAAS,GAAG,SAAS,MAAM;AAAA,EACpC;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,SAAS,SAAS,WAAW,UAAa,WAAW,OAAO,WAAW,UAAa,WAAW;AAAA,EACxG;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,SAAO;AACT;AAEO,SAAS,iBAAiB,UAAiC;AAChE,MAAI,OAAO,aAAa,SAAU,QAAO,WAAW,QAAQ;AAC5D,MAAI,YAAY,SAAU,QAAO,UAAU,KAAK,UAAU,SAAS,MAAM,CAAC;AAC1E,MAAI,cAAc,SAAU,QAAO,aAAa,SAAS,QAAQ;AACjE,MAAI,kBAAkB,SAAU,QAAO,iBAAiB,SAAS,YAAY;AAC7E,MAAI,aAAa,SAAU,QAAO,YAAY,SAAS,OAAO;AAC9D,MAAI,QAAQ,SAAU,QAAO,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACxF,MAAI,YAAY,SAAU,QAAO,SAAS,SAAS,WAAW;AAC9D,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,SAAO,KAAK,UAAU,QAAQ;AAChC;",
+  "names": []
+}

package/dist/graders/outcome.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Outcome assertion orchestrator.
+ * Runs state and workflow graders after all conversation turns complete.
+ */
+import type { EvalDefinition, GraderResult, StateMutation, WorkflowSpan } from '../types';
+/**
+ * Grade all outcome assertions after the conversation completes.
+ * Takes transformer output (StateMutation[], WorkflowSpan[]) — no raw spans, no API calls.
+ */
+export declare function gradeOutcome(mutations: StateMutation[], evalDef: EvalDefinition, workflowSpans: WorkflowSpan[]): GraderResult[];
+//# sourceMappingURL=outcome.d.ts.map

package/dist/graders/outcome.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outcome.d.ts","sourceRoot":"","sources":["../../src/graders/outcome.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAIzF;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,SAAS,EAAE,aAAa,EAAE,EAC1B,OAAO,EAAE,cAAc,EACvB,aAAa,EAAE,YAAY,EAAE,GAC5B,YAAY,EAAE,CAiBhB"}

package/dist/graders/outcome.js

@@ -0,0 +1,148 @@
+// src/graders/state.ts
+function parseStatePath(path) {
+  const dot = path.indexOf(".");
+  if (dot === -1) {
+    throw new Error(`Invalid state path "${path}" \u2014 expected "type.field" format`);
+  }
+  const type = path.slice(0, dot);
+  const field = path.slice(dot + 1);
+  if (type !== "bot" && type !== "user" && type !== "conversation") {
+    throw new Error(`Unknown state type "${type}" in path "${path}" \u2014 expected bot, user, or conversation`);
+  }
+  return { type, field };
+}
+function gradeState(mutations, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const parsed = parseStatePath(assertion.path);
+    const matchingMutations = mutations.filter((m) => m.type === parsed.type);
+    if (assertion.equals !== void 0) {
+      if (matchingMutations.length === 0) {
+        results.push({
+          assertion: `state: ${assertion.path} equals`,
+          pass: false,
+          expected: JSON.stringify(assertion.equals),
+          actual: `No state.save span found for "${parsed.type}" \u2014 the bot did not mutate this state during the turn. If this state was seeded via setup.state, assert on the tool/action that reads it, or use a conversation turn that triggers a state write.`
+        });
+        continue;
+      }
+      const lastMutation = matchingMutations[matchingMutations.length - 1];
+      if (lastMutation.swappedToFile) {
+        results.push({
+          assertion: `state: ${assertion.path} equals`,
+          pass: false,
+          expected: JSON.stringify(assertion.equals),
+          actual: "State was too large for trace-based assertion (swapped to file)"
+        });
+        continue;
+      }
+      const currentState = lastMutation.current;
+      const actualValue = currentState ? currentState[parsed.field] : void 0;
+      const pass = deepEqual(actualValue, assertion.equals);
+      results.push({
+        assertion: `state: ${assertion.path} equals`,
+        pass,
+        expected: JSON.stringify(assertion.equals),
+        actual: JSON.stringify(actualValue)
+      });
+    }
+    if (assertion.changed !== void 0) {
+      if (matchingMutations.length === 0) {
+        results.push({
+          assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+          pass: !assertion.changed,
+          expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+          actual: 'No state mutation found for type "' + parsed.type + '"'
+        });
+        continue;
+      }
+      const hasSwapped = matchingMutations.some((m) => m.swappedToFile);
+      if (hasSwapped) {
+        results.push({
+          assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+          pass: false,
+          expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+          actual: "State was too large for trace-based assertion (swapped to file)"
+        });
+        continue;
+      }
+      const fieldChanged = matchingMutations.some((m) => m.changedKeys.includes(parsed.field));
+      const pass = assertion.changed ? fieldChanged : !fieldChanged;
+      results.push({
+        assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+        pass,
+        expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+        actual: fieldChanged ? `Field "${parsed.field}" was changed` : `Field "${parsed.field}" was not changed`
+      });
+    }
+  }
+  return results;
+}
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (a === null || b === null) return false;
+  if (typeof a !== typeof b) return false;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((v, i) => deepEqual(v, b[i]));
+  }
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  if (typeof a === "object" && typeof b === "object") {
+    const keysA = Object.keys(a);
+    const keysB = Object.keys(b);
+    if (keysA.length !== keysB.length) return false;
+    return keysA.every((k) => deepEqual(a[k], b[k]));
+  }
+  return false;
+}
+
+// src/graders/workflow.ts
+function gradeWorkflows(workflows, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const matching = workflows.filter((wf) => wf.name === assertion.name);
+    if (assertion.entered !== void 0) {
+      const wasEntered = matching.length > 0;
+      const pass = assertion.entered ? wasEntered : !wasEntered;
+      results.push({
+        assertion: `workflow: ${assertion.name} ${assertion.entered ? "entered" : "not entered"}`,
+        pass,
+        expected: assertion.entered ? `Workflow "${assertion.name}" was entered` : `Workflow "${assertion.name}" was not entered`,
+        actual: wasEntered ? `Found ${matching.length} workflow span(s)` : `No workflow spans found`
+      });
+    }
+    if (assertion.completed !== void 0) {
+      const didComplete = matching.some(
+        (wf) => wf.statusFinal === "completed" || wf.status === "ok" || wf.status === "error"
+      );
+      const pass = assertion.completed ? didComplete : !didComplete;
+      results.push({
+        assertion: `workflow: ${assertion.name} ${assertion.completed ? "completed" : "not completed"}`,
+        pass,
+        expected: assertion.completed ? `Workflow "${assertion.name}" completed` : `Workflow "${assertion.name}" did not complete`,
+        actual: didComplete ? `Completed` : `Not completed`
+      });
+    }
+  }
+  return results;
+}
+
+// src/graders/outcome.ts
+function gradeOutcome(mutations, evalDef, workflowSpans) {
+  const outcome = evalDef.outcome;
+  if (!outcome) return [];
+  const results = [];
+  if (outcome.state && outcome.state.length > 0) {
+    const stateResults = gradeState(mutations, outcome.state);
+    results.push(...stateResults);
+  }
+  if (outcome.workflow && outcome.workflow.length > 0) {
+    const workflowResults = gradeWorkflows(workflowSpans, outcome.workflow);
+    results.push(...workflowResults);
+  }
+  return results;
+}
+export {
+  gradeOutcome
+};
+//# sourceMappingURL=outcome.js.map

package/dist/graders/outcome.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/state.ts", "../../src/graders/workflow.ts", "../../src/graders/outcome.ts"],
+  "sourcesContent": ["/**\n * State assertion graders.\n * Pure functions that grade state assertions against StateMutation[] from the transformer.\n * No API calls, no retries, no BpClient.\n */\n\nimport type { StateMutation, StateAssertion, GraderResult } from '../types'\n\n/**\n * Parse a state path like \"conversation.lastCity\" into its components.\n *   bot.*            -> type: \"bot\",          field: \"*\"\n *   user.*           -> type: \"user\",         field: \"*\"\n *   conversation.*   -> type: \"conversation\", field: \"*\"\n */\nfunction parseStatePath(path: string): { type: string; field: string } {\n  const dot = path.indexOf('.')\n  if (dot === -1) {\n    throw new Error(`Invalid state path \"${path}\" \u2014 expected \"type.field\" format`)\n  }\n\n  const type = path.slice(0, dot)\n  const field = path.slice(dot + 1)\n\n  if (type !== 'bot' && type !== 'user' && type !== 'conversation') {\n    throw new Error(`Unknown state type \"${type}\" in path \"${path}\" \u2014 expected bot, user, or conversation`)\n  }\n\n  return { type, field }\n}\n\n/**\n * Grade state assertions using StateMutation[] from the transformer.\n *\n * - `equals`: find the LAST mutation matching the state type, extract field from `current`, deep-compare\n * - `changed: true`: check if the field appears in ANY mutation's `changedKeys` for that state type\n * - `changed: false`: check that the field appears in NO mutation's `changedKeys`\n * - Swapped state: when `swappedToFile` is true, current/previous are undefined \u2014 report a clear message\n */\nexport function gradeState(mutations: StateMutation[], assertions: StateAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const parsed = parseStatePath(assertion.path)\n    const matchingMutations = mutations.filter((m) => m.type === parsed.type)\n\n    if (assertion.equals !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual:\n            `No state.save span found for \"${parsed.type}\" \u2014 the bot did not mutate this state during the turn. ` +\n            'If this state was seeded via setup.state, assert on the tool/action that reads it, or use a conversation turn that triggers a state write.',\n        })\n        continue\n      }\n\n      const lastMutation = matchingMutations[matchingMutations.length - 1]!\n\n      if (lastMutation.swappedToFile) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const currentState = lastMutation.current as Record<string, unknown> | undefined\n      const actualValue = currentState ? currentState[parsed.field] : undefined\n      const pass = deepEqual(actualValue, assertion.equals)\n\n      results.push({\n        assertion: `state: ${assertion.path} equals`,\n        pass,\n        expected: JSON.stringify(assertion.equals),\n        actual: JSON.stringify(actualValue),\n      })\n    }\n\n    if (assertion.changed !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: !assertion.changed,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'No state mutation found for type \"' + parsed.type + '\"',\n        })\n        continue\n      }\n\n      // Check for swapped state in any matching mutation\n      const hasSwapped = matchingMutations.some((m) => m.swappedToFile)\n      if (hasSwapped) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: false,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const fieldChanged = matchingMutations.some((m) => m.changedKeys.includes(parsed.field))\n      const pass = assertion.changed ? fieldChanged : !fieldChanged\n\n      results.push({\n        assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n        pass,\n        expected: assertion.changed\n          ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n          : `Field \"${parsed.field}\" unchanged`,\n        actual: fieldChanged ? `Field \"${parsed.field}\" was changed` : `Field \"${parsed.field}\" was not changed`,\n      })\n    }\n  }\n\n  return results\n}\n\nfunction deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true\n  if (a === null || b === null) return false\n  if (typeof a !== typeof b) return false\n\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false\n    return a.every((v, i) => deepEqual(v, b[i]))\n  }\n\n  if (Array.isArray(a) !== Array.isArray(b)) return false\n\n  if (typeof a === 'object' && typeof b === 'object') {\n    const keysA = Object.keys(a as object)\n    const keysB = Object.keys(b as object)\n    if (keysA.length !== keysB.length) return false\n    return keysA.every((k) => deepEqual((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k]))\n  }\n\n  return false\n}\n", "/**\n * Workflow assertion graders.\n * Checks workflow execution via transformer-produced WorkflowSpan data.\n */\n\nimport type { WorkflowAssertion, WorkflowSpan, GraderResult } from '../types'\n\nexport function gradeWorkflows(workflows: WorkflowSpan[], assertions: WorkflowAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const matching = workflows.filter((wf) => wf.name === assertion.name)\n\n    // entered \u2014 was the workflow triggered at all?\n    if (assertion.entered !== undefined) {\n      const wasEntered = matching.length > 0\n      const pass = assertion.entered ? wasEntered : !wasEntered\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.entered ? 'entered' : 'not entered'}`,\n        pass,\n        expected: assertion.entered\n          ? `Workflow \"${assertion.name}\" was entered`\n          : `Workflow \"${assertion.name}\" was not entered`,\n        actual: wasEntered ? `Found ${matching.length} workflow span(s)` : `No workflow spans found`,\n      })\n    }\n\n    // completed \u2014 did the workflow complete?\n    if (assertion.completed !== undefined) {\n      const didComplete = matching.some(\n        (wf) => wf.statusFinal === 'completed' || wf.status === 'ok' || wf.status === 'error'\n      )\n      const pass = assertion.completed ? didComplete : !didComplete\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.completed ? 'completed' : 'not completed'}`,\n        pass,\n        expected: assertion.completed\n          ? `Workflow \"${assertion.name}\" completed`\n          : `Workflow \"${assertion.name}\" did not complete`,\n        actual: didComplete ? `Completed` : `Not completed`,\n      })\n    }\n  }\n\n  return results\n}\n", "/**\n * Outcome assertion orchestrator.\n * Runs state and workflow graders after all conversation turns complete.\n */\n\nimport type { EvalDefinition, GraderResult, StateMutation, WorkflowSpan } from '../types'\nimport { gradeState } from './state'\nimport { gradeWorkflows } from './workflow'\n\n/**\n * Grade all outcome assertions after the conversation completes.\n * Takes transformer output (StateMutation[], WorkflowSpan[]) \u2014 no raw spans, no API calls.\n */\nexport function gradeOutcome(\n  mutations: StateMutation[],\n  evalDef: EvalDefinition,\n  workflowSpans: WorkflowSpan[]\n): GraderResult[] {\n  const outcome = evalDef.outcome\n  if (!outcome) return []\n\n  const results: GraderResult[] = []\n\n  if (outcome.state && outcome.state.length > 0) {\n    const stateResults = gradeState(mutations, outcome.state)\n    results.push(...stateResults)\n  }\n\n  if (outcome.workflow && outcome.workflow.length > 0) {\n    const workflowResults = gradeWorkflows(workflowSpans, outcome.workflow)\n    results.push(...workflowResults)\n  }\n\n  return results\n}\n"],
+  "mappings": ";AAcA,SAAS,eAAe,MAA+C;AACrE,QAAM,MAAM,KAAK,QAAQ,GAAG;AAC5B,MAAI,QAAQ,IAAI;AACd,UAAM,IAAI,MAAM,uBAAuB,IAAI,uCAAkC;AAAA,EAC/E;AAEA,QAAM,OAAO,KAAK,MAAM,GAAG,GAAG;AAC9B,QAAM,QAAQ,KAAK,MAAM,MAAM,CAAC;AAEhC,MAAI,SAAS,SAAS,SAAS,UAAU,SAAS,gBAAgB;AAChE,UAAM,IAAI,MAAM,uBAAuB,IAAI,cAAc,IAAI,8CAAyC;AAAA,EACxG;AAEA,SAAO,EAAE,MAAM,MAAM;AACvB;AAUO,SAAS,WAAW,WAA4B,YAA8C;AACnG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,SAAS,eAAe,UAAU,IAAI;AAC5C,UAAM,oBAAoB,UAAU,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,IAAI;AAExE,QAAI,UAAU,WAAW,QAAW;AAClC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QACE,iCAAiC,OAAO,IAAI;AAAA,QAEhD,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,kBAAkB,SAAS,CAAC;AAEnE,UAAI,aAAa,eAAe;AAC9B,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,aAAa;AAClC,YAAM,cAAc,eAAe,aAAa,OAAO,KAAK,IAAI;AAChE,YAAM,OAAO,UAAU,aAAa,UAAU,MAAM;AAEpD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI;AAAA,QACnC;AAAA,QACA,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,QACzC,QAAQ,KAAK,UAAU,WAAW;AAAA,MACpC,CAAC;AAAA,IACH;AAEA,QAAI,UAAU,YAAY,QAAW;AACnC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM,CAAC,UAAU;AAAA,UACjB,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ,uCAAuC,OAAO,OAAO;AAAA,QAC/D,CAAC;AACD;AAAA,MACF;AAGA,YAAM,aAAa,kBAAkB,KAAK,CAAC,MAAM,EAAE,aAAa;AAChE,UAAI,YAAY;AACd,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM;AAAA,UACN,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,KAAK,CAAC,MAAM,EAAE,YAAY,SAAS,OAAO,KAAK,CAAC;AACvF,YAAM,OAAO,UAAU,UAAU,eAAe,CAAC;AAEjD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,QAClF;AAAA,QACA,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,QAC1B,QAAQ,eAAe,UAAU,OAAO,KAAK,kBAAkB,UAAU,OAAO,KAAK;AAAA,MACvF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,UAAU,GAAY,GAAqB;AAClD,MAAI,MAAM,EAAG,QAAO;AACpB,MAAI,MAAM,QAAQ,MAAM,KAAM,QAAO;AACrC,MAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAElC,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,WAAO,EAAE,MAAM,CAAC,GAAG,MAAM,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;AAAA,EAC7C;AAEA,MAAI,MAAM,QAAQ,CAAC,MAAM,MAAM,QAAQ,CAAC,EAAG,QAAO;AAElD,MAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;AAClD,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,QAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,WAAO,MAAM,MAAM,CAAC,MAAM,UAAW,EAA8B,CAAC,GAAI,EAA8B,CAAC,CAAC,CAAC;AAAA,EAC3G;AAEA,SAAO;AACT;;;AC3IO,SAAS,eAAe,WAA2B,YAAiD;AACzG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,WAAW,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,UAAU,IAAI;AAGpE,QAAI,UAAU,YAAY,QAAW;AACnC,YAAM,aAAa,SAAS,SAAS;AACrC,YAAM,OAAO,UAAU,UAAU,aAAa,CAAC;AAE/C,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,aAAa;AAAA,QACvF;AAAA,QACA,UAAU,UAAU,UAChB,aAAa,UAAU,IAAI,kBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,aAAa,SAAS,SAAS,MAAM,sBAAsB;AAAA,MACrE,CAAC;AAAA,IACH;AAGA,QAAI,UAAU,cAAc,QAAW;AACrC,YAAM,cAAc,SAAS;AAAA,QAC3B,CAAC,OAAO,GAAG,gBAAgB,eAAe,GAAG,WAAW,QAAQ,GAAG,WAAW;AAAA,MAChF;AACA,YAAM,OAAO,UAAU,YAAY,cAAc,CAAC;AAElD,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,YAAY,cAAc,eAAe;AAAA,QAC7F;AAAA,QACA,UAAU,UAAU,YAChB,aAAa,UAAU,IAAI,gBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,cAAc,cAAc;AAAA,MACtC,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;;;AClCO,SAAS,aACd,WACA,SACA,eACgB;AAChB,QAAM,UAAU,QAAQ;AACxB,MAAI,CAAC,QAAS,QAAO,CAAC;AAEtB,QAAM,UAA0B,CAAC;AAEjC,MAAI,QAAQ,SAAS,QAAQ,MAAM,SAAS,GAAG;AAC7C,UAAM,eAAe,WAAW,WAAW,QAAQ,KAAK;AACxD,YAAQ,KAAK,GAAG,YAAY;AAAA,EAC9B;AAEA,MAAI,QAAQ,YAAY,QAAQ,SAAS,SAAS,GAAG;AACnD,UAAM,kBAAkB,eAAe,eAAe,QAAQ,QAAQ;AACtE,YAAQ,KAAK,GAAG,eAAe;AAAA,EACjC;AAEA,SAAO;AACT;",
+  "names": []
+}

package/dist/graders/response.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Response assertion graders.
+ * Checks bot response text against expected assertions.
+ */
+import type { ResponseAssertion, GraderResult } from '../types';
+export declare function gradeResponse(botResponse: string, assertions: ResponseAssertion[], context: {
+    userMessage: string;
+    judgePassThreshold?: number;
+}): Promise<GraderResult[]>;
+//# sourceMappingURL=response.d.ts.map

package/dist/graders/response.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"response.d.ts","sourceRoot":"","sources":["../../src/graders/response.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAG/D,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,iBAAiB,EAAE,EAC/B,OAAO,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5D,OAAO,CAAC,YAAY,EAAE,CAAC,CAmEzB"}

package/dist/graders/response.js

@@ -0,0 +1,155 @@
+// src/graders/llm.ts
+import { Client } from "@botpress/client";
+import { Cognitive } from "@botpress/cognitive";
+var DEFAULT_PASS_THRESHOLD = 3;
+function buildJudgeSystemPrompt(passThreshold) {
+  return `You are an evaluation judge for a chatbot. You will be given:
+- The user's message
+- The bot's response
+- Grading criteria
+
+Your job is to determine if the bot's response meets the criteria.
+
+Respond with a JSON object:
+{
+  "pass": true/false,
+  "score": 1-5,
+  "reason": "brief explanation"
+}
+
+Scoring guide:
+- 5: Fully meets criteria, excellent response
+- 4: Mostly meets criteria, minor issues
+- 3: Partially meets criteria, notable gaps
+- 2: Barely meets criteria, significant issues
+- 1: Does not meet criteria
+
+A score of ${passThreshold} or above is a pass.`;
+}
+var MODEL_ALIASES = ["fast", "best"];
+var _cognitive = null;
+var _availableModels = [];
+var _judgeModel = "fast";
+function _isModelValid(model) {
+  if (MODEL_ALIASES.includes(model)) return true;
+  if (_availableModels.length === 0) return false;
+  return _availableModels.includes(model);
+}
+async function gradeLLMJudge(botResponse, criteria, context) {
+  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
+  try {
+    if (!_cognitive) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: true,
+        expected: criteria,
+        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+      };
+    }
+    if (!_isModelValid(_judgeModel)) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: false,
+        expected: criteria,
+        actual: "FAILED \u2014 invalid LLM judge model"
+      };
+    }
+    const { output } = await _cognitive.generateContent({
+      model: _judgeModel,
+      temperature: 0,
+      responseFormat: "json_object",
+      systemPrompt: buildJudgeSystemPrompt(threshold),
+      messages: [
+        {
+          role: "user",
+          content: `User message: ${context.userMessage}
+
+Bot response: ${botResponse}
+
+Criteria: ${criteria}`
+        }
+      ]
+    });
+    const rawContent = output?.choices?.[0]?.content;
+    const content = typeof rawContent === "string" ? rawContent : null;
+    if (!content) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: true,
+        expected: criteria,
+        actual: "SKIPPED \u2014 LLM judge returned empty response"
+      };
+    }
+    const verdict = JSON.parse(content);
+    return {
+      assertion: `llm_judge: "${criteria}"`,
+      pass: verdict.score >= threshold,
+      expected: criteria,
+      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
+    };
+  } catch (err) {
+    return {
+      assertion: `llm_judge: "${criteria}"`,
+      pass: true,
+      expected: criteria,
+      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+    };
+  }
+}
+
+// src/graders/response.ts
+async function gradeResponse(botResponse, assertions, context) {
+  const results = [];
+  for (const assertion of assertions) {
+    if ("contains" in assertion) {
+      const pass = botResponse.toLowerCase().includes(assertion.contains.toLowerCase());
+      results.push({
+        assertion: `contains "${assertion.contains}"`,
+        pass,
+        expected: `Response contains "${assertion.contains}"`,
+        actual: pass ? `Found in response` : `Not found in response`
+      });
+      continue;
+    }
+    if ("not_contains" in assertion) {
+      const pass = !botResponse.toLowerCase().includes(assertion.not_contains.toLowerCase());
+      results.push({
+        assertion: `not_contains "${assertion.not_contains}"`,
+        pass,
+        expected: `Response does not contain "${assertion.not_contains}"`,
+        actual: pass ? `Not found in response` : `Found in response`
+      });
+      continue;
+    }
+    if ("matches" in assertion) {
+      const regex = new RegExp(assertion.matches, "i");
+      const pass = regex.test(botResponse);
+      results.push({
+        assertion: `matches ${assertion.matches}`,
+        pass,
+        expected: `Response matches /${assertion.matches}/`,
+        actual: pass ? `Matched` : `No match`
+      });
+      continue;
+    }
+    if ("llm_judge" in assertion) {
+      const result = await gradeLLMJudge(botResponse, assertion.llm_judge, {
+        userMessage: context.userMessage,
+        ...context.judgePassThreshold !== void 0 ? { passThreshold: context.judgePassThreshold } : {}
+      });
+      results.push(result);
+      continue;
+    }
+    results.push({
+      assertion: "unknown",
+      pass: false,
+      expected: "known assertion type",
+      actual: `Unknown assertion: ${JSON.stringify(assertion)}`
+    });
+  }
+  return results;
+}
+export {
+  gradeResponse
+};
+//# sourceMappingURL=response.js.map

package/dist/graders/response.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/llm.ts", "../../src/graders/response.ts"],
+  "sourcesContent": ["/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? console\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    if (!_cognitive) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: true,\n      expected: criteria,\n      actual: `SKIPPED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n", "/**\n * Response assertion graders.\n * Checks bot response text against expected assertions.\n */\n\nimport type { ResponseAssertion, GraderResult } from '../types'\nimport { gradeLLMJudge } from './llm'\n\nexport async function gradeResponse(\n  botResponse: string,\n  assertions: ResponseAssertion[],\n  context: { userMessage: string; judgePassThreshold?: number }\n): Promise<GraderResult[]> {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    if ('contains' in assertion) {\n      const pass = botResponse.toLowerCase().includes(assertion.contains.toLowerCase())\n      results.push({\n        assertion: `contains \"${assertion.contains}\"`,\n        pass,\n        expected: `Response contains \"${assertion.contains}\"`,\n        actual: pass ? `Found in response` : `Not found in response`,\n      })\n      continue\n    }\n\n    if ('not_contains' in assertion) {\n      const pass = !botResponse.toLowerCase().includes(assertion.not_contains.toLowerCase())\n      results.push({\n        assertion: `not_contains \"${assertion.not_contains}\"`,\n        pass,\n        expected: `Response does not contain \"${assertion.not_contains}\"`,\n        actual: pass ? `Not found in response` : `Found in response`,\n      })\n      continue\n    }\n\n    if ('matches' in assertion) {\n      const regex = new RegExp(assertion.matches, 'i')\n      const pass = regex.test(botResponse)\n      results.push({\n        assertion: `matches ${assertion.matches}`,\n        pass,\n        expected: `Response matches /${assertion.matches}/`,\n        actual: pass ? `Matched` : `No match`,\n      })\n      continue\n    }\n\n    if ('llm_judge' in assertion) {\n      const result = await gradeLLMJudge(botResponse, assertion.llm_judge, {\n        userMessage: context.userMessage,\n        ...(context.judgePassThreshold !== undefined ? { passThreshold: context.judgePassThreshold } : {}),\n      })\n      results.push(result)\n      continue\n    }\n\n    // TODO: uncomment this once we support it\n    // if ('similar_to' in assertion) {\n    //   results.push({\n    //     assertion: `similar_to: \"${assertion.similar_to}\"`,\n    //     pass: true,\n    //     expected: assertion.similar_to,\n    //     actual: 'SKIPPED \u2014 similar_to not yet implemented',\n    //   })\n    //   continue\n    // }\n\n    results.push({\n      assertion: 'unknown',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown assertion: ${JSON.stringify(assertion)}`,\n    })\n  }\n\n  return results\n}\n"],
+  "mappings": ";AAKA,SAAS,cAAc;AACvB,SAAS,iBAAiB;AAG1B,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAE1B,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAoCA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AACF,QAAI,CAAC,YAAY;AACf,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,yCAAqC,IAAc,OAAO;AAAA,IACpE;AAAA,EACF;AACF;;;ACnJA,eAAsB,cACpB,aACA,YACA,SACyB;AACzB,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,YAAY,YAAY,EAAE,SAAS,UAAU,SAAS,YAAY,CAAC;AAChF,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,QAAQ;AAAA,QAC1C;AAAA,QACA,UAAU,sBAAsB,UAAU,QAAQ;AAAA,QAClD,QAAQ,OAAO,sBAAsB;AAAA,MACvC,CAAC;AACD;AAAA,IACF;AAEA,QAAI,kBAAkB,WAAW;AAC/B,YAAM,OAAO,CAAC,YAAY,YAAY,EAAE,SAAS,UAAU,aAAa,YAAY,CAAC;AACrF,cAAQ,KAAK;AAAA,QACX,WAAW,iBAAiB,UAAU,YAAY;AAAA,QAClD;AAAA,QACA,UAAU,8BAA8B,UAAU,YAAY;AAAA,QAC9D,QAAQ,OAAO,0BAA0B;AAAA,MAC3C,CAAC;AACD;AAAA,IACF;AAEA,QAAI,aAAa,WAAW;AAC1B,YAAM,QAAQ,IAAI,OAAO,UAAU,SAAS,GAAG;AAC/C,YAAM,OAAO,MAAM,KAAK,WAAW;AACnC,cAAQ,KAAK;AAAA,QACX,WAAW,WAAW,UAAU,OAAO;AAAA,QACvC;AAAA,QACA,UAAU,qBAAqB,UAAU,OAAO;AAAA,QAChD,QAAQ,OAAO,YAAY;AAAA,MAC7B,CAAC;AACD;AAAA,IACF;AAEA,QAAI,eAAe,WAAW;AAC5B,YAAM,SAAS,MAAM,cAAc,aAAa,UAAU,WAAW;AAAA,QACnE,aAAa,QAAQ;AAAA,QACrB,GAAI,QAAQ,uBAAuB,SAAY,EAAE,eAAe,QAAQ,mBAAmB,IAAI,CAAC;AAAA,MAClG,CAAC;AACD,cAAQ,KAAK,MAAM;AACnB;AAAA,IACF;AAaA,YAAQ,KAAK;AAAA,MACX,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,sBAAsB,KAAK,UAAU,SAAS,CAAC;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;",
+  "names": []
+}

package/dist/graders/state.d.ts

@@ -0,0 +1,16 @@
+/**
+ * State assertion graders.
+ * Pure functions that grade state assertions against StateMutation[] from the transformer.
+ * No API calls, no retries, no BpClient.
+ */
+import type { StateMutation, StateAssertion, GraderResult } from '../types';
+/**
+ * Grade state assertions using StateMutation[] from the transformer.
+ *
+ * - `equals`: find the LAST mutation matching the state type, extract field from `current`, deep-compare
+ * - `changed: true`: check if the field appears in ANY mutation's `changedKeys` for that state type
+ * - `changed: false`: check that the field appears in NO mutation's `changedKeys`
+ * - Swapped state: when `swappedToFile` is true, current/previous are undefined — report a clear message
+ */
+export declare function gradeState(mutations: StateMutation[], assertions: StateAssertion[]): GraderResult[];
+//# sourceMappingURL=state.d.ts.map

package/dist/graders/state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/graders/state.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAwB3E;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,SAAS,EAAE,aAAa,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE,GAAG,YAAY,EAAE,CAsFnG"}