@botpress/evals 1.18.3

package/dist/graders/state.js

@@ -0,0 +1,101 @@
+// src/graders/state.ts
+function parseStatePath(path) {
+  const dot = path.indexOf(".");
+  if (dot === -1) {
+    throw new Error(`Invalid state path "${path}" \u2014 expected "type.field" format`);
+  }
+  const type = path.slice(0, dot);
+  const field = path.slice(dot + 1);
+  if (type !== "bot" && type !== "user" && type !== "conversation") {
+    throw new Error(`Unknown state type "${type}" in path "${path}" \u2014 expected bot, user, or conversation`);
+  }
+  return { type, field };
+}
+function gradeState(mutations, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const parsed = parseStatePath(assertion.path);
+    const matchingMutations = mutations.filter((m) => m.type === parsed.type);
+    if (assertion.equals !== void 0) {
+      if (matchingMutations.length === 0) {
+        results.push({
+          assertion: `state: ${assertion.path} equals`,
+          pass: false,
+          expected: JSON.stringify(assertion.equals),
+          actual: `No state.save span found for "${parsed.type}" \u2014 the bot did not mutate this state during the turn. If this state was seeded via setup.state, assert on the tool/action that reads it, or use a conversation turn that triggers a state write.`
+        });
+        continue;
+      }
+      const lastMutation = matchingMutations[matchingMutations.length - 1];
+      if (lastMutation.swappedToFile) {
+        results.push({
+          assertion: `state: ${assertion.path} equals`,
+          pass: false,
+          expected: JSON.stringify(assertion.equals),
+          actual: "State was too large for trace-based assertion (swapped to file)"
+        });
+        continue;
+      }
+      const currentState = lastMutation.current;
+      const actualValue = currentState ? currentState[parsed.field] : void 0;
+      const pass = deepEqual(actualValue, assertion.equals);
+      results.push({
+        assertion: `state: ${assertion.path} equals`,
+        pass,
+        expected: JSON.stringify(assertion.equals),
+        actual: JSON.stringify(actualValue)
+      });
+    }
+    if (assertion.changed !== void 0) {
+      if (matchingMutations.length === 0) {
+        results.push({
+          assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+          pass: !assertion.changed,
+          expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+          actual: 'No state mutation found for type "' + parsed.type + '"'
+        });
+        continue;
+      }
+      const hasSwapped = matchingMutations.some((m) => m.swappedToFile);
+      if (hasSwapped) {
+        results.push({
+          assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+          pass: false,
+          expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+          actual: "State was too large for trace-based assertion (swapped to file)"
+        });
+        continue;
+      }
+      const fieldChanged = matchingMutations.some((m) => m.changedKeys.includes(parsed.field));
+      const pass = assertion.changed ? fieldChanged : !fieldChanged;
+      results.push({
+        assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+        pass,
+        expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+        actual: fieldChanged ? `Field "${parsed.field}" was changed` : `Field "${parsed.field}" was not changed`
+      });
+    }
+  }
+  return results;
+}
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (a === null || b === null) return false;
+  if (typeof a !== typeof b) return false;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((v, i) => deepEqual(v, b[i]));
+  }
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  if (typeof a === "object" && typeof b === "object") {
+    const keysA = Object.keys(a);
+    const keysB = Object.keys(b);
+    if (keysA.length !== keysB.length) return false;
+    return keysA.every((k) => deepEqual(a[k], b[k]));
+  }
+  return false;
+}
+export {
+  gradeState
+};
+//# sourceMappingURL=state.js.map

package/dist/graders/state.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/state.ts"],
+  "sourcesContent": ["/**\n * State assertion graders.\n * Pure functions that grade state assertions against StateMutation[] from the transformer.\n * No API calls, no retries, no BpClient.\n */\n\nimport type { StateMutation, StateAssertion, GraderResult } from '../types'\n\n/**\n * Parse a state path like \"conversation.lastCity\" into its components.\n *   bot.*            -> type: \"bot\",          field: \"*\"\n *   user.*           -> type: \"user\",         field: \"*\"\n *   conversation.*   -> type: \"conversation\", field: \"*\"\n */\nfunction parseStatePath(path: string): { type: string; field: string } {\n  const dot = path.indexOf('.')\n  if (dot === -1) {\n    throw new Error(`Invalid state path \"${path}\" \u2014 expected \"type.field\" format`)\n  }\n\n  const type = path.slice(0, dot)\n  const field = path.slice(dot + 1)\n\n  if (type !== 'bot' && type !== 'user' && type !== 'conversation') {\n    throw new Error(`Unknown state type \"${type}\" in path \"${path}\" \u2014 expected bot, user, or conversation`)\n  }\n\n  return { type, field }\n}\n\n/**\n * Grade state assertions using StateMutation[] from the transformer.\n *\n * - `equals`: find the LAST mutation matching the state type, extract field from `current`, deep-compare\n * - `changed: true`: check if the field appears in ANY mutation's `changedKeys` for that state type\n * - `changed: false`: check that the field appears in NO mutation's `changedKeys`\n * - Swapped state: when `swappedToFile` is true, current/previous are undefined \u2014 report a clear message\n */\nexport function gradeState(mutations: StateMutation[], assertions: StateAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const parsed = parseStatePath(assertion.path)\n    const matchingMutations = mutations.filter((m) => m.type === parsed.type)\n\n    if (assertion.equals !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual:\n            `No state.save span found for \"${parsed.type}\" \u2014 the bot did not mutate this state during the turn. ` +\n            'If this state was seeded via setup.state, assert on the tool/action that reads it, or use a conversation turn that triggers a state write.',\n        })\n        continue\n      }\n\n      const lastMutation = matchingMutations[matchingMutations.length - 1]!\n\n      if (lastMutation.swappedToFile) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const currentState = lastMutation.current as Record<string, unknown> | undefined\n      const actualValue = currentState ? currentState[parsed.field] : undefined\n      const pass = deepEqual(actualValue, assertion.equals)\n\n      results.push({\n        assertion: `state: ${assertion.path} equals`,\n        pass,\n        expected: JSON.stringify(assertion.equals),\n        actual: JSON.stringify(actualValue),\n      })\n    }\n\n    if (assertion.changed !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: !assertion.changed,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'No state mutation found for type \"' + parsed.type + '\"',\n        })\n        continue\n      }\n\n      // Check for swapped state in any matching mutation\n      const hasSwapped = matchingMutations.some((m) => m.swappedToFile)\n      if (hasSwapped) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: false,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const fieldChanged = matchingMutations.some((m) => m.changedKeys.includes(parsed.field))\n      const pass = assertion.changed ? fieldChanged : !fieldChanged\n\n      results.push({\n        assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n        pass,\n        expected: assertion.changed\n          ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n          : `Field \"${parsed.field}\" unchanged`,\n        actual: fieldChanged ? `Field \"${parsed.field}\" was changed` : `Field \"${parsed.field}\" was not changed`,\n      })\n    }\n  }\n\n  return results\n}\n\nfunction deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true\n  if (a === null || b === null) return false\n  if (typeof a !== typeof b) return false\n\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false\n    return a.every((v, i) => deepEqual(v, b[i]))\n  }\n\n  if (Array.isArray(a) !== Array.isArray(b)) return false\n\n  if (typeof a === 'object' && typeof b === 'object') {\n    const keysA = Object.keys(a as object)\n    const keysB = Object.keys(b as object)\n    if (keysA.length !== keysB.length) return false\n    return keysA.every((k) => deepEqual((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k]))\n  }\n\n  return false\n}\n"],
+  "mappings": ";AAcA,SAAS,eAAe,MAA+C;AACrE,QAAM,MAAM,KAAK,QAAQ,GAAG;AAC5B,MAAI,QAAQ,IAAI;AACd,UAAM,IAAI,MAAM,uBAAuB,IAAI,uCAAkC;AAAA,EAC/E;AAEA,QAAM,OAAO,KAAK,MAAM,GAAG,GAAG;AAC9B,QAAM,QAAQ,KAAK,MAAM,MAAM,CAAC;AAEhC,MAAI,SAAS,SAAS,SAAS,UAAU,SAAS,gBAAgB;AAChE,UAAM,IAAI,MAAM,uBAAuB,IAAI,cAAc,IAAI,8CAAyC;AAAA,EACxG;AAEA,SAAO,EAAE,MAAM,MAAM;AACvB;AAUO,SAAS,WAAW,WAA4B,YAA8C;AACnG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,SAAS,eAAe,UAAU,IAAI;AAC5C,UAAM,oBAAoB,UAAU,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,IAAI;AAExE,QAAI,UAAU,WAAW,QAAW;AAClC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QACE,iCAAiC,OAAO,IAAI;AAAA,QAEhD,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,kBAAkB,SAAS,CAAC;AAEnE,UAAI,aAAa,eAAe;AAC9B,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,aAAa;AAClC,YAAM,cAAc,eAAe,aAAa,OAAO,KAAK,IAAI;AAChE,YAAM,OAAO,UAAU,aAAa,UAAU,MAAM;AAEpD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI;AAAA,QACnC;AAAA,QACA,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,QACzC,QAAQ,KAAK,UAAU,WAAW;AAAA,MACpC,CAAC;AAAA,IACH;AAEA,QAAI,UAAU,YAAY,QAAW;AACnC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM,CAAC,UAAU;AAAA,UACjB,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ,uCAAuC,OAAO,OAAO;AAAA,QAC/D,CAAC;AACD;AAAA,MACF;AAGA,YAAM,aAAa,kBAAkB,KAAK,CAAC,MAAM,EAAE,aAAa;AAChE,UAAI,YAAY;AACd,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM;AAAA,UACN,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,KAAK,CAAC,MAAM,EAAE,YAAY,SAAS,OAAO,KAAK,CAAC;AACvF,YAAM,OAAO,UAAU,UAAU,eAAe,CAAC;AAEjD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,QAClF;AAAA,QACA,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,QAC1B,QAAQ,eAAe,UAAU,OAAO,KAAK,kBAAkB,UAAU,OAAO,KAAK;AAAA,MACvF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,UAAU,GAAY,GAAqB;AAClD,MAAI,MAAM,EAAG,QAAO;AACpB,MAAI,MAAM,QAAQ,MAAM,KAAM,QAAO;AACrC,MAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAElC,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,WAAO,EAAE,MAAM,CAAC,GAAG,MAAM,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;AAAA,EAC7C;AAEA,MAAI,MAAM,QAAQ,CAAC,MAAM,MAAM,QAAQ,CAAC,EAAG,QAAO;AAElD,MAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;AAClD,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,QAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,WAAO,MAAM,MAAM,CAAC,MAAM,UAAW,EAA8B,CAAC,GAAI,EAA8B,CAAC,CAAC,CAAC;AAAA,EAC3G;AAEA,SAAO;AACT;",
+  "names": []
+}

package/dist/graders/timing.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Timing assertion graders.
+ * Checks bot response duration against expected thresholds.
+ */
+import type { TimingAssertion, GraderResult } from '../types';
+export declare function gradeTiming(botDuration: number, assertions: TimingAssertion[]): GraderResult[];
+//# sourceMappingURL=timing.d.ts.map

package/dist/graders/timing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timing.d.ts","sourceRoot":"","sources":["../../src/graders/timing.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAG7D,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,YAAY,EAAE,CAgB9F"}

package/dist/graders/timing.js

@@ -0,0 +1,63 @@
+// src/graders/match.ts
+function matchValue(operator, actual) {
+  if (typeof operator === "string") {
+    return String(actual) === operator;
+  }
+  if ("equals" in operator) {
+    return actual === operator.equals;
+  }
+  if ("contains" in operator) {
+    return String(actual).toLowerCase().includes(operator.contains.toLowerCase());
+  }
+  if ("not_contains" in operator) {
+    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase());
+  }
+  if ("matches" in operator) {
+    return new RegExp(operator.matches, "i").test(String(actual));
+  }
+  if ("in" in operator) {
+    return operator.in.includes(actual);
+  }
+  if ("exists" in operator) {
+    return operator.exists ? actual !== void 0 && actual !== null : actual === void 0 || actual === null;
+  }
+  if ("gte" in operator) {
+    return Number(actual) >= operator.gte;
+  }
+  if ("lte" in operator) {
+    return Number(actual) <= operator.lte;
+  }
+  return false;
+}
+function operatorToString(operator) {
+  if (typeof operator === "string") return `equals "${operator}"`;
+  if ("equals" in operator) return `equals ${JSON.stringify(operator.equals)}`;
+  if ("contains" in operator) return `contains "${operator.contains}"`;
+  if ("not_contains" in operator) return `not_contains "${operator.not_contains}"`;
+  if ("matches" in operator) return `matches /${operator.matches}/`;
+  if ("in" in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(", ")}]`;
+  if ("exists" in operator) return operator.exists ? "exists" : "does not exist";
+  if ("gte" in operator) return `>= ${operator.gte}`;
+  if ("lte" in operator) return `<= ${operator.lte}`;
+  return JSON.stringify(operator);
+}
+
+// src/graders/timing.ts
+function gradeTiming(botDuration, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const pass = matchValue(assertion.response_time, botDuration);
+    const expected = operatorToString(assertion.response_time);
+    results.push({
+      assertion: `response_time ${expected}`,
+      pass,
+      expected: `Response time ${expected}`,
+      actual: `${botDuration}ms`
+    });
+  }
+  return results;
+}
+export {
+  gradeTiming
+};
+//# sourceMappingURL=timing.js.map

package/dist/graders/timing.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/match.ts", "../../src/graders/timing.ts"],
+  "sourcesContent": ["/**\n * Shared match utilities for assertion grading.\n * Used by tool, state, and table graders.\n */\n\nimport type { MatchOperator } from '../types'\n\n/**\n * Evaluate a single match operator against a value.\n */\nexport function matchValue(operator: MatchOperator, actual: unknown): boolean {\n  // Shorthand: string means exact match\n  if (typeof operator === 'string') {\n    return String(actual) === operator\n  }\n\n  if ('equals' in operator) {\n    return actual === operator.equals\n  }\n\n  if ('contains' in operator) {\n    return String(actual).toLowerCase().includes(operator.contains.toLowerCase())\n  }\n\n  if ('not_contains' in operator) {\n    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase())\n  }\n\n  if ('matches' in operator) {\n    return new RegExp(operator.matches, 'i').test(String(actual))\n  }\n\n  if ('in' in operator) {\n    return operator.in.includes(actual)\n  }\n\n  if ('exists' in operator) {\n    return operator.exists ? actual !== undefined && actual !== null : actual === undefined || actual === null\n  }\n\n  if ('gte' in operator) {\n    return Number(actual) >= operator.gte\n  }\n\n  if ('lte' in operator) {\n    return Number(actual) <= operator.lte\n  }\n\n  return false\n}\n\nexport function operatorToString(operator: MatchOperator): string {\n  if (typeof operator === 'string') return `equals \"${operator}\"`\n  if ('equals' in operator) return `equals ${JSON.stringify(operator.equals)}`\n  if ('contains' in operator) return `contains \"${operator.contains}\"`\n  if ('not_contains' in operator) return `not_contains \"${operator.not_contains}\"`\n  if ('matches' in operator) return `matches /${operator.matches}/`\n  if ('in' in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(', ')}]`\n  if ('exists' in operator) return operator.exists ? 'exists' : 'does not exist'\n  if ('gte' in operator) return `>= ${operator.gte}`\n  if ('lte' in operator) return `<= ${operator.lte}`\n  return JSON.stringify(operator)\n}\n", "/**\n * Timing assertion graders.\n * Checks bot response duration against expected thresholds.\n */\n\nimport type { TimingAssertion, GraderResult } from '../types'\nimport { matchValue, operatorToString } from './match'\n\nexport function gradeTiming(botDuration: number, assertions: TimingAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const pass = matchValue(assertion.response_time, botDuration)\n    const expected = operatorToString(assertion.response_time)\n\n    results.push({\n      assertion: `response_time ${expected}`,\n      pass,\n      expected: `Response time ${expected}`,\n      actual: `${botDuration}ms`,\n    })\n  }\n\n  return results\n}\n"],
+  "mappings": ";AAUO,SAAS,WAAW,UAAyB,QAA0B;AAE5E,MAAI,OAAO,aAAa,UAAU;AAChC,WAAO,OAAO,MAAM,MAAM;AAAA,EAC5B;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,MAAI,cAAc,UAAU;AAC1B,WAAO,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,SAAS,YAAY,CAAC;AAAA,EAC9E;AAEA,MAAI,kBAAkB,UAAU;AAC9B,WAAO,CAAC,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,aAAa,YAAY,CAAC;AAAA,EACnF;AAEA,MAAI,aAAa,UAAU;AACzB,WAAO,IAAI,OAAO,SAAS,SAAS,GAAG,EAAE,KAAK,OAAO,MAAM,CAAC;AAAA,EAC9D;AAEA,MAAI,QAAQ,UAAU;AACpB,WAAO,SAAS,GAAG,SAAS,MAAM;AAAA,EACpC;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,SAAS,SAAS,WAAW,UAAa,WAAW,OAAO,WAAW,UAAa,WAAW;AAAA,EACxG;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,SAAO;AACT;AAEO,SAAS,iBAAiB,UAAiC;AAChE,MAAI,OAAO,aAAa,SAAU,QAAO,WAAW,QAAQ;AAC5D,MAAI,YAAY,SAAU,QAAO,UAAU,KAAK,UAAU,SAAS,MAAM,CAAC;AAC1E,MAAI,cAAc,SAAU,QAAO,aAAa,SAAS,QAAQ;AACjE,MAAI,kBAAkB,SAAU,QAAO,iBAAiB,SAAS,YAAY;AAC7E,MAAI,aAAa,SAAU,QAAO,YAAY,SAAS,OAAO;AAC9D,MAAI,QAAQ,SAAU,QAAO,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACxF,MAAI,YAAY,SAAU,QAAO,SAAS,SAAS,WAAW;AAC9D,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,SAAO,KAAK,UAAU,QAAQ;AAChC;;;ACtDO,SAAS,YAAY,aAAqB,YAA+C;AAC9F,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,OAAO,WAAW,UAAU,eAAe,WAAW;AAC5D,UAAM,WAAW,iBAAiB,UAAU,aAAa;AAEzD,YAAQ,KAAK;AAAA,MACX,WAAW,iBAAiB,QAAQ;AAAA,MACpC;AAAA,MACA,UAAU,iBAAiB,QAAQ;AAAA,MACnC,QAAQ,GAAG,WAAW;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AACT;",
+  "names": []
+}

package/dist/graders/tools.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Tool assertion graders.
+ * Checks tool calls from traces against expected assertions.
+ */
+import type { ToolAssertion, ToolCall, GraderResult } from '../types';
+export declare function gradeTools(toolCalls: ToolCall[], assertions: ToolAssertion[]): GraderResult[];
+//# sourceMappingURL=tools.d.ts.map

package/dist/graders/tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/graders/tools.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAGrE,wBAAgB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG,YAAY,EAAE,CA6F7F"}

package/dist/graders/tools.js

@@ -0,0 +1,122 @@
+// src/graders/match.ts
+function matchValue(operator, actual) {
+  if (typeof operator === "string") {
+    return String(actual) === operator;
+  }
+  if ("equals" in operator) {
+    return actual === operator.equals;
+  }
+  if ("contains" in operator) {
+    return String(actual).toLowerCase().includes(operator.contains.toLowerCase());
+  }
+  if ("not_contains" in operator) {
+    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase());
+  }
+  if ("matches" in operator) {
+    return new RegExp(operator.matches, "i").test(String(actual));
+  }
+  if ("in" in operator) {
+    return operator.in.includes(actual);
+  }
+  if ("exists" in operator) {
+    return operator.exists ? actual !== void 0 && actual !== null : actual === void 0 || actual === null;
+  }
+  if ("gte" in operator) {
+    return Number(actual) >= operator.gte;
+  }
+  if ("lte" in operator) {
+    return Number(actual) <= operator.lte;
+  }
+  return false;
+}
+function operatorToString(operator) {
+  if (typeof operator === "string") return `equals "${operator}"`;
+  if ("equals" in operator) return `equals ${JSON.stringify(operator.equals)}`;
+  if ("contains" in operator) return `contains "${operator.contains}"`;
+  if ("not_contains" in operator) return `not_contains "${operator.not_contains}"`;
+  if ("matches" in operator) return `matches /${operator.matches}/`;
+  if ("in" in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(", ")}]`;
+  if ("exists" in operator) return operator.exists ? "exists" : "does not exist";
+  if ("gte" in operator) return `>= ${operator.gte}`;
+  if ("lte" in operator) return `<= ${operator.lte}`;
+  return JSON.stringify(operator);
+}
+
+// src/graders/tools.ts
+function gradeTools(toolCalls, assertions) {
+  return assertions.map((assertion) => {
+    if ("called" in assertion && !("not_called" in assertion) && !("call_order" in assertion)) {
+      const matches = toolCalls.filter((tc) => tc.name === assertion.called);
+      const wasCalled = matches.length > 0;
+      if (!wasCalled) {
+        return {
+          assertion: `tool called: ${assertion.called}`,
+          pass: false,
+          expected: `${assertion.called} was called`,
+          actual: `Not called. Tools called: [${toolCalls.map((tc) => tc.name).join(", ") || "none"}]`
+        };
+      }
+      if (assertion.params) {
+        const paramResults = [];
+        for (const [key, operator] of Object.entries(assertion.params)) {
+          const anyMatch = matches.some((tc) => matchValue(operator, tc.input[key]));
+          paramResults.push({
+            key,
+            pass: anyMatch,
+            detail: anyMatch ? `matched` : `expected ${key} ${operatorToString(operator)}, got ${JSON.stringify(matches.map((tc) => tc.input[key]))}`
+          });
+        }
+        const allParamsPass = paramResults.every((p) => p.pass);
+        const failedParams = paramResults.filter((p) => !p.pass);
+        return {
+          assertion: `tool called: ${assertion.called} with params`,
+          pass: allParamsPass,
+          expected: `${assertion.called} called with ${Object.entries(assertion.params).map(([k, v]) => `${k} ${operatorToString(v)}`).join(", ")}`,
+          actual: allParamsPass ? `Matched` : failedParams.map((p) => p.detail).join("; ")
+        };
+      }
+      return {
+        assertion: `tool called: ${assertion.called}`,
+        pass: true,
+        expected: `${assertion.called} was called`,
+        actual: `Called ${matches.length} time(s)`
+      };
+    }
+    if ("not_called" in assertion) {
+      const wasCalled = toolCalls.some((tc) => tc.name === assertion.not_called);
+      return {
+        assertion: `tool not_called: ${assertion.not_called}`,
+        pass: !wasCalled,
+        expected: `${assertion.not_called} was NOT called`,
+        actual: wasCalled ? `Was called` : `Not called`
+      };
+    }
+    if ("call_order" in assertion) {
+      const calledNames = toolCalls.map((tc) => tc.name);
+      const expectedOrder = assertion.call_order;
+      let cursor = 0;
+      for (const name of calledNames) {
+        if (cursor < expectedOrder.length && name === expectedOrder[cursor]) {
+          cursor++;
+        }
+      }
+      const inOrder = cursor === expectedOrder.length;
+      return {
+        assertion: `call_order: [${expectedOrder.join(" \u2192 ")}]`,
+        pass: inOrder,
+        expected: `Tools called in order: [${expectedOrder.join(" \u2192 ")}]`,
+        actual: `Actual order: [${calledNames.join(" \u2192 ") || "none"}]`
+      };
+    }
+    return {
+      assertion: "unknown tool assertion",
+      pass: false,
+      expected: "known assertion type",
+      actual: `Unknown: ${JSON.stringify(assertion)}`
+    };
+  });
+}
+export {
+  gradeTools
+};
+//# sourceMappingURL=tools.js.map

package/dist/graders/tools.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/match.ts", "../../src/graders/tools.ts"],
+  "sourcesContent": ["/**\n * Shared match utilities for assertion grading.\n * Used by tool, state, and table graders.\n */\n\nimport type { MatchOperator } from '../types'\n\n/**\n * Evaluate a single match operator against a value.\n */\nexport function matchValue(operator: MatchOperator, actual: unknown): boolean {\n  // Shorthand: string means exact match\n  if (typeof operator === 'string') {\n    return String(actual) === operator\n  }\n\n  if ('equals' in operator) {\n    return actual === operator.equals\n  }\n\n  if ('contains' in operator) {\n    return String(actual).toLowerCase().includes(operator.contains.toLowerCase())\n  }\n\n  if ('not_contains' in operator) {\n    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase())\n  }\n\n  if ('matches' in operator) {\n    return new RegExp(operator.matches, 'i').test(String(actual))\n  }\n\n  if ('in' in operator) {\n    return operator.in.includes(actual)\n  }\n\n  if ('exists' in operator) {\n    return operator.exists ? actual !== undefined && actual !== null : actual === undefined || actual === null\n  }\n\n  if ('gte' in operator) {\n    return Number(actual) >= operator.gte\n  }\n\n  if ('lte' in operator) {\n    return Number(actual) <= operator.lte\n  }\n\n  return false\n}\n\nexport function operatorToString(operator: MatchOperator): string {\n  if (typeof operator === 'string') return `equals \"${operator}\"`\n  if ('equals' in operator) return `equals ${JSON.stringify(operator.equals)}`\n  if ('contains' in operator) return `contains \"${operator.contains}\"`\n  if ('not_contains' in operator) return `not_contains \"${operator.not_contains}\"`\n  if ('matches' in operator) return `matches /${operator.matches}/`\n  if ('in' in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(', ')}]`\n  if ('exists' in operator) return operator.exists ? 'exists' : 'does not exist'\n  if ('gte' in operator) return `>= ${operator.gte}`\n  if ('lte' in operator) return `<= ${operator.lte}`\n  return JSON.stringify(operator)\n}\n", "/**\n * Tool assertion graders.\n * Checks tool calls from traces against expected assertions.\n */\n\nimport type { ToolAssertion, ToolCall, GraderResult } from '../types'\nimport { matchValue, operatorToString } from './match'\n\nexport function gradeTools(toolCalls: ToolCall[], assertions: ToolAssertion[]): GraderResult[] {\n  return assertions.map((assertion) => {\n    // --- called ---\n    if ('called' in assertion && !('not_called' in assertion) && !('call_order' in assertion)) {\n      const matches = toolCalls.filter((tc) => tc.name === assertion.called)\n      const wasCalled = matches.length > 0\n\n      if (!wasCalled) {\n        return {\n          assertion: `tool called: ${assertion.called}`,\n          pass: false,\n          expected: `${assertion.called} was called`,\n          actual: `Not called. Tools called: [${toolCalls.map((tc) => tc.name).join(', ') || 'none'}]`,\n        }\n      }\n\n      // Check params if specified\n      if (assertion.params) {\n        const paramResults: { key: string; pass: boolean; detail: string }[] = []\n\n        for (const [key, operator] of Object.entries(assertion.params)) {\n          // Check across all matching calls \u2014 pass if any call matches\n          const anyMatch = matches.some((tc) => matchValue(operator, tc.input[key]))\n          paramResults.push({\n            key,\n            pass: anyMatch,\n            detail: anyMatch\n              ? `matched`\n              : `expected ${key} ${operatorToString(operator)}, got ${JSON.stringify(matches.map((tc) => tc.input[key]))}`,\n          })\n        }\n\n        const allParamsPass = paramResults.every((p) => p.pass)\n        const failedParams = paramResults.filter((p) => !p.pass)\n\n        return {\n          assertion: `tool called: ${assertion.called} with params`,\n          pass: allParamsPass,\n          expected: `${assertion.called} called with ${Object.entries(assertion.params)\n            .map(([k, v]) => `${k} ${operatorToString(v)}`)\n            .join(', ')}`,\n          actual: allParamsPass ? `Matched` : failedParams.map((p) => p.detail).join('; '),\n        }\n      }\n\n      return {\n        assertion: `tool called: ${assertion.called}`,\n        pass: true,\n        expected: `${assertion.called} was called`,\n        actual: `Called ${matches.length} time(s)`,\n      }\n    }\n\n    // --- not_called ---\n    if ('not_called' in assertion) {\n      const wasCalled = toolCalls.some((tc) => tc.name === assertion.not_called)\n      return {\n        assertion: `tool not_called: ${assertion.not_called}`,\n        pass: !wasCalled,\n        expected: `${assertion.not_called} was NOT called`,\n        actual: wasCalled ? `Was called` : `Not called`,\n      }\n    }\n\n    // --- call_order ---\n    if ('call_order' in assertion) {\n      const calledNames = toolCalls.map((tc) => tc.name)\n      const expectedOrder = assertion.call_order as string[]\n\n      // Check that the expected sequence appears in order (not necessarily contiguous)\n      let cursor = 0\n      for (const name of calledNames) {\n        if (cursor < expectedOrder.length && name === expectedOrder[cursor]) {\n          cursor++\n        }\n      }\n      const inOrder = cursor === expectedOrder.length\n\n      return {\n        assertion: `call_order: [${expectedOrder.join(' \u2192 ')}]`,\n        pass: inOrder,\n        expected: `Tools called in order: [${expectedOrder.join(' \u2192 ')}]`,\n        actual: `Actual order: [${calledNames.join(' \u2192 ') || 'none'}]`,\n      }\n    }\n\n    return {\n      assertion: 'unknown tool assertion',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown: ${JSON.stringify(assertion)}`,\n    }\n  })\n}\n"],
+  "mappings": ";AAUO,SAAS,WAAW,UAAyB,QAA0B;AAE5E,MAAI,OAAO,aAAa,UAAU;AAChC,WAAO,OAAO,MAAM,MAAM;AAAA,EAC5B;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,MAAI,cAAc,UAAU;AAC1B,WAAO,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,SAAS,YAAY,CAAC;AAAA,EAC9E;AAEA,MAAI,kBAAkB,UAAU;AAC9B,WAAO,CAAC,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,aAAa,YAAY,CAAC;AAAA,EACnF;AAEA,MAAI,aAAa,UAAU;AACzB,WAAO,IAAI,OAAO,SAAS,SAAS,GAAG,EAAE,KAAK,OAAO,MAAM,CAAC;AAAA,EAC9D;AAEA,MAAI,QAAQ,UAAU;AACpB,WAAO,SAAS,GAAG,SAAS,MAAM;AAAA,EACpC;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,SAAS,SAAS,WAAW,UAAa,WAAW,OAAO,WAAW,UAAa,WAAW;AAAA,EACxG;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,SAAO;AACT;AAEO,SAAS,iBAAiB,UAAiC;AAChE,MAAI,OAAO,aAAa,SAAU,QAAO,WAAW,QAAQ;AAC5D,MAAI,YAAY,SAAU,QAAO,UAAU,KAAK,UAAU,SAAS,MAAM,CAAC;AAC1E,MAAI,cAAc,SAAU,QAAO,aAAa,SAAS,QAAQ;AACjE,MAAI,kBAAkB,SAAU,QAAO,iBAAiB,SAAS,YAAY;AAC7E,MAAI,aAAa,SAAU,QAAO,YAAY,SAAS,OAAO;AAC9D,MAAI,QAAQ,SAAU,QAAO,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACxF,MAAI,YAAY,SAAU,QAAO,SAAS,SAAS,WAAW;AAC9D,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,SAAO,KAAK,UAAU,QAAQ;AAChC;;;ACtDO,SAAS,WAAW,WAAuB,YAA6C;AAC7F,SAAO,WAAW,IAAI,CAAC,cAAc;AAEnC,QAAI,YAAY,aAAa,EAAE,gBAAgB,cAAc,EAAE,gBAAgB,YAAY;AACzF,YAAM,UAAU,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,UAAU,MAAM;AACrE,YAAM,YAAY,QAAQ,SAAS;AAEnC,UAAI,CAAC,WAAW;AACd,eAAO;AAAA,UACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,UAC3C,MAAM;AAAA,UACN,UAAU,GAAG,UAAU,MAAM;AAAA,UAC7B,QAAQ,8BAA8B,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,KAAK,MAAM;AAAA,QAC3F;AAAA,MACF;AAGA,UAAI,UAAU,QAAQ;AACpB,cAAM,eAAiE,CAAC;AAExE,mBAAW,CAAC,KAAK,QAAQ,KAAK,OAAO,QAAQ,UAAU,MAAM,GAAG;AAE9D,gBAAM,WAAW,QAAQ,KAAK,CAAC,OAAO,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AACzE,uBAAa,KAAK;AAAA,YAChB;AAAA,YACA,MAAM;AAAA,YACN,QAAQ,WACJ,YACA,YAAY,GAAG,IAAI,iBAAiB,QAAQ,CAAC,SAAS,KAAK,UAAU,QAAQ,IAAI,CAAC,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;AAAA,UAC9G,CAAC;AAAA,QACH;AAEA,cAAM,gBAAgB,aAAa,MAAM,CAAC,MAAM,EAAE,IAAI;AACtD,cAAM,eAAe,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;AAEvD,eAAO;AAAA,UACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,UAC3C,MAAM;AAAA,UACN,UAAU,GAAG,UAAU,MAAM,gBAAgB,OAAO,QAAQ,UAAU,MAAM,EACzE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,CAAC,CAAC,EAAE,EAC7C,KAAK,IAAI,CAAC;AAAA,UACb,QAAQ,gBAAgB,YAAY,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI;AAAA,QACjF;AAAA,MACF;AAEA,aAAO;AAAA,QACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,QAC3C,MAAM;AAAA,QACN,UAAU,GAAG,UAAU,MAAM;AAAA,QAC7B,QAAQ,UAAU,QAAQ,MAAM;AAAA,MAClC;AAAA,IACF;AAGA,QAAI,gBAAgB,WAAW;AAC7B,YAAM,YAAY,UAAU,KAAK,CAAC,OAAO,GAAG,SAAS,UAAU,UAAU;AACzE,aAAO;AAAA,QACL,WAAW,oBAAoB,UAAU,UAAU;AAAA,QACnD,MAAM,CAAC;AAAA,QACP,UAAU,GAAG,UAAU,UAAU;AAAA,QACjC,QAAQ,YAAY,eAAe;AAAA,MACrC;AAAA,IACF;AAGA,QAAI,gBAAgB,WAAW;AAC7B,YAAM,cAAc,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI;AACjD,YAAM,gBAAgB,UAAU;AAGhC,UAAI,SAAS;AACb,iBAAW,QAAQ,aAAa;AAC9B,YAAI,SAAS,cAAc,UAAU,SAAS,cAAc,MAAM,GAAG;AACnE;AAAA,QACF;AAAA,MACF;AACA,YAAM,UAAU,WAAW,cAAc;AAEzC,aAAO;AAAA,QACL,WAAW,gBAAgB,cAAc,KAAK,UAAK,CAAC;AAAA,QACpD,MAAM;AAAA,QACN,UAAU,2BAA2B,cAAc,KAAK,UAAK,CAAC;AAAA,QAC9D,QAAQ,kBAAkB,YAAY,KAAK,UAAK,KAAK,MAAM;AAAA,MAC7D;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,YAAY,KAAK,UAAU,SAAS,CAAC;AAAA,IAC/C;AAAA,EACF,CAAC;AACH;",
+  "names": []
+}

package/dist/graders/workflow.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Workflow assertion graders.
+ * Checks workflow execution via transformer-produced WorkflowSpan data.
+ */
+import type { WorkflowAssertion, WorkflowSpan, GraderResult } from '../types';
+export declare function gradeWorkflows(workflows: WorkflowSpan[], assertions: WorkflowAssertion[]): GraderResult[];
+//# sourceMappingURL=workflow.d.ts.map

package/dist/graders/workflow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../../src/graders/workflow.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAE7E,wBAAgB,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,UAAU,EAAE,iBAAiB,EAAE,GAAG,YAAY,EAAE,CAwCzG"}

package/dist/graders/workflow.js

@@ -0,0 +1,34 @@
+// src/graders/workflow.ts
+function gradeWorkflows(workflows, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const matching = workflows.filter((wf) => wf.name === assertion.name);
+    if (assertion.entered !== void 0) {
+      const wasEntered = matching.length > 0;
+      const pass = assertion.entered ? wasEntered : !wasEntered;
+      results.push({
+        assertion: `workflow: ${assertion.name} ${assertion.entered ? "entered" : "not entered"}`,
+        pass,
+        expected: assertion.entered ? `Workflow "${assertion.name}" was entered` : `Workflow "${assertion.name}" was not entered`,
+        actual: wasEntered ? `Found ${matching.length} workflow span(s)` : `No workflow spans found`
+      });
+    }
+    if (assertion.completed !== void 0) {
+      const didComplete = matching.some(
+        (wf) => wf.statusFinal === "completed" || wf.status === "ok" || wf.status === "error"
+      );
+      const pass = assertion.completed ? didComplete : !didComplete;
+      results.push({
+        assertion: `workflow: ${assertion.name} ${assertion.completed ? "completed" : "not completed"}`,
+        pass,
+        expected: assertion.completed ? `Workflow "${assertion.name}" completed` : `Workflow "${assertion.name}" did not complete`,
+        actual: didComplete ? `Completed` : `Not completed`
+      });
+    }
+  }
+  return results;
+}
+export {
+  gradeWorkflows
+};
+//# sourceMappingURL=workflow.js.map

package/dist/graders/workflow.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/graders/workflow.ts"],
+  "sourcesContent": ["/**\n * Workflow assertion graders.\n * Checks workflow execution via transformer-produced WorkflowSpan data.\n */\n\nimport type { WorkflowAssertion, WorkflowSpan, GraderResult } from '../types'\n\nexport function gradeWorkflows(workflows: WorkflowSpan[], assertions: WorkflowAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const matching = workflows.filter((wf) => wf.name === assertion.name)\n\n    // entered \u2014 was the workflow triggered at all?\n    if (assertion.entered !== undefined) {\n      const wasEntered = matching.length > 0\n      const pass = assertion.entered ? wasEntered : !wasEntered\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.entered ? 'entered' : 'not entered'}`,\n        pass,\n        expected: assertion.entered\n          ? `Workflow \"${assertion.name}\" was entered`\n          : `Workflow \"${assertion.name}\" was not entered`,\n        actual: wasEntered ? `Found ${matching.length} workflow span(s)` : `No workflow spans found`,\n      })\n    }\n\n    // completed \u2014 did the workflow complete?\n    if (assertion.completed !== undefined) {\n      const didComplete = matching.some(\n        (wf) => wf.statusFinal === 'completed' || wf.status === 'ok' || wf.status === 'error'\n      )\n      const pass = assertion.completed ? didComplete : !didComplete\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.completed ? 'completed' : 'not completed'}`,\n        pass,\n        expected: assertion.completed\n          ? `Workflow \"${assertion.name}\" completed`\n          : `Workflow \"${assertion.name}\" did not complete`,\n        actual: didComplete ? `Completed` : `Not completed`,\n      })\n    }\n  }\n\n  return results\n}\n"],
+  "mappings": ";AAOO,SAAS,eAAe,WAA2B,YAAiD;AACzG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,WAAW,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,UAAU,IAAI;AAGpE,QAAI,UAAU,YAAY,QAAW;AACnC,YAAM,aAAa,SAAS,SAAS;AACrC,YAAM,OAAO,UAAU,UAAU,aAAa,CAAC;AAE/C,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,aAAa;AAAA,QACvF;AAAA,QACA,UAAU,UAAU,UAChB,aAAa,UAAU,IAAI,kBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,aAAa,SAAS,SAAS,MAAM,sBAAsB;AAAA,MACrE,CAAC;AAAA,IACH;AAGA,QAAI,UAAU,cAAc,QAAW;AACrC,YAAM,cAAc,SAAS;AAAA,QAC3B,CAAC,OAAO,GAAG,gBAAgB,eAAe,GAAG,WAAW,QAAQ,GAAG,WAAW;AAAA,MAChF;AACA,YAAM,OAAO,UAAU,YAAY,cAAc,CAAC;AAElD,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,YAAY,cAAc,eAAe;AAAA,QAC7F;AAAA,QACA,UAAU,UAAU,YAChB,aAAa,UAAU,IAAI,gBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,cAAc,cAAc;AAAA,MACtC,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;",
+  "names": []
+}

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * ADK eval authoring API.
+ *
+ * Runtime-heavy execution pieces are available from subpaths such as
+ * `@botpress/evals/runner`, `@botpress/evals/loader`, and
+ * `@botpress/evals/graders`.
+ */
+export { Eval } from './definition';
+export type { BotConnection, ChatClient, ChatClientConnection, ConversationTurn, EvalDefinition, EvalFilter, EvalLogger, EvalProgressEvent, EvalReport, EvalRunReport, EvalRunnerConfig, EvalSetup, GraderResult, MatchOperator, NumericOperator, OutcomeAssertions, ResponseAssertion, Span, SpanStatus, StateAssertion, StateMutation, TimingAssertion, ToolAssertion, ToolCall, TurnAssertions, TurnData, TurnReport, WorkflowAssertion, WorkflowSpan, } from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAA;AAEnC,YAAY,EACV,aAAa,EACb,UAAU,EACV,oBAAoB,EACpB,gBAAgB,EAChB,cAAc,EACd,UAAU,EACV,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,SAAS,EACT,YAAY,EACZ,aAAa,EACb,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,IAAI,EACJ,UAAU,EACV,cAAc,EACd,aAAa,EACb,eAAe,EACf,aAAa,EACb,QAAQ,EACR,cAAc,EACd,QAAQ,EACR,UAAU,EACV,iBAAiB,EACjB,YAAY,GACb,MAAM,SAAS,CAAA"}

package/dist/index.js

@@ -0,0 +1,25 @@
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+export {
+  Eval
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/definition.ts"],
+  "sourcesContent": ["export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { type: string; payload?: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n"],
+  "mappings": ";AAuFO,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAKT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;",
+  "names": []
+}

package/dist/loader.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Eval file loader.
+ * Loads *.eval.ts files from a directory using dynamic imports.
+ * Supports both default and named exports, and multiple evals per file.
+ */
+import type { EvalDefinition, EvalFilter } from './types';
+export declare function loadEvalFile(filePath: string): Promise<EvalDefinition[]>;
+export declare function loadEvalsFromDir(dirPath: string): Promise<EvalDefinition[]>;
+export declare function loadEvalByName(dirPath: string, name: string): Promise<EvalDefinition | null>;
+export declare function filterEvals(evals: EvalDefinition[], filter?: EvalFilter): EvalDefinition[];
+//# sourceMappingURL=loader.d.ts.map

package/dist/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAazD,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAkC9E;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAuBjF;AAED,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAalG;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,CAe1F"}

package/dist/loader.js

@@ -0,0 +1,114 @@
+// src/loader.ts
+import { readdirSync, existsSync } from "fs";
+import { resolve } from "path";
+
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+
+// src/loader.ts
+function isEvalDefinition(value) {
+  return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
+}
+async function loadEvalFile(filePath) {
+  const absPath = resolve(filePath);
+  let mod;
+  try {
+    mod = await import(absPath);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `Failed to load eval file ${filePath}: ${msg}
+
+Make sure your eval file:
+  - Has no syntax or type errors
+  - Exports one or more \`new Eval({...})\` instances
+  - Has all dependencies installed (\`bun install\`)`,
+      { cause: err }
+    );
+  }
+  const results = [];
+  for (const [key, value] of Object.entries(mod)) {
+    if (key === "__esModule") continue;
+    if (value instanceof Eval || isEvalDefinition(value)) {
+      results.push(value);
+    }
+  }
+  if (results.length === 0) {
+    throw new Error(
+      `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`
+    );
+  }
+  return results;
+}
+async function loadEvalsFromDir(dirPath) {
+  const absDir = resolve(dirPath);
+  if (!existsSync(absDir)) {
+    return [];
+  }
+  const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts"));
+  const evals = [];
+  for (const f of files) {
+    const defs = await loadEvalFile(`${absDir}/${f}`);
+    evals.push(...defs);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const e of evals) {
+    if (seen.has(e.name)) {
+      throw new Error(`Duplicate eval name "${e.name}" found in ${dirPath}`);
+    }
+    seen.add(e.name);
+  }
+  return evals;
+}
+async function loadEvalByName(dirPath, name) {
+  const absDir = resolve(dirPath);
+  if (!existsSync(absDir)) return null;
+  const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts"));
+  for (const f of files) {
+    const defs = await loadEvalFile(`${absDir}/${f}`);
+    const found = defs.find((d) => d.name === name);
+    if (found) return found;
+  }
+  return null;
+}
+function filterEvals(evals, filter) {
+  if (!filter) return evals;
+  return evals.filter((e) => {
+    if (filter.names && filter.names.length > 0) {
+      if (!filter.names.includes(e.name)) return false;
+    }
+    if (filter.tags && filter.tags.length > 0) {
+      if (!e.tags || !filter.tags.some((t) => e.tags.includes(t))) return false;
+    }
+    if (filter.type) {
+      if (e.type !== filter.type) return false;
+    }
+    return true;
+  });
+}
+export {
+  filterEvals,
+  loadEvalByName,
+  loadEvalFile,
+  loadEvalsFromDir
+};
+//# sourceMappingURL=loader.js.map