@cogitator-ai/evals 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +202 -48
  2. package/dist/datasets/csv-loader.d.ts +5 -0
  3. package/dist/datasets/csv-loader.d.ts.map +1 -1
  4. package/dist/datasets/csv-loader.js +18 -0
  5. package/dist/datasets/csv-loader.js.map +1 -1
  6. package/dist/eval-builder.d.ts +3 -0
  7. package/dist/eval-builder.d.ts.map +1 -1
  8. package/dist/eval-builder.js +7 -0
  9. package/dist/eval-builder.js.map +1 -1
  10. package/dist/eval-comparison.d.ts +5 -0
  11. package/dist/eval-comparison.d.ts.map +1 -1
  12. package/dist/eval-comparison.js +20 -4
  13. package/dist/eval-comparison.js.map +1 -1
  14. package/dist/eval-suite.d.ts +48 -5
  15. package/dist/eval-suite.d.ts.map +1 -1
  16. package/dist/eval-suite.js +219 -57
  17. package/dist/eval-suite.js.map +1 -1
  18. package/dist/index.d.ts +14 -9
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +6 -3
  21. package/dist/index.js.map +1 -1
  22. package/dist/matchers.d.ts +68 -0
  23. package/dist/matchers.d.ts.map +1 -0
  24. package/dist/matchers.js +112 -0
  25. package/dist/matchers.js.map +1 -0
  26. package/dist/metrics/custom.d.ts +19 -7
  27. package/dist/metrics/custom.d.ts.map +1 -1
  28. package/dist/metrics/custom.js +5 -0
  29. package/dist/metrics/custom.js.map +1 -1
  30. package/dist/metrics/deterministic.d.ts +1 -0
  31. package/dist/metrics/deterministic.d.ts.map +1 -1
  32. package/dist/metrics/deterministic.js +49 -4
  33. package/dist/metrics/deterministic.js.map +1 -1
  34. package/dist/metrics/index.d.ts +9 -5
  35. package/dist/metrics/index.d.ts.map +1 -1
  36. package/dist/metrics/index.js +5 -3
  37. package/dist/metrics/index.js.map +1 -1
  38. package/dist/metrics/llm-judge.d.ts +20 -1
  39. package/dist/metrics/llm-judge.d.ts.map +1 -1
  40. package/dist/metrics/llm-judge.js +64 -10
  41. package/dist/metrics/llm-judge.js.map +1 -1
  42. package/dist/metrics/matching.d.ts +28 -0
  43. package/dist/metrics/matching.d.ts.map +1 -0
  44. package/dist/metrics/matching.js +124 -0
  45. package/dist/metrics/matching.js.map +1 -0
  46. package/dist/metrics/retrieval.d.ts +21 -0
  47. package/dist/metrics/retrieval.d.ts.map +1 -0
  48. package/dist/metrics/retrieval.js +108 -0
  49. package/dist/metrics/retrieval.js.map +1 -0
  50. package/dist/metrics/statistical.d.ts +28 -0
  51. package/dist/metrics/statistical.d.ts.map +1 -1
  52. package/dist/metrics/statistical.js +61 -0
  53. package/dist/metrics/statistical.js.map +1 -1
  54. package/dist/metrics/tool-calls.d.ts +43 -0
  55. package/dist/metrics/tool-calls.d.ts.map +1 -0
  56. package/dist/metrics/tool-calls.js +106 -0
  57. package/dist/metrics/tool-calls.js.map +1 -0
  58. package/dist/metrics/types.d.ts +47 -5
  59. package/dist/metrics/types.d.ts.map +1 -1
  60. package/dist/metrics/types.js.map +1 -1
  61. package/dist/reporters/console.d.ts.map +1 -1
  62. package/dist/reporters/console.js +6 -2
  63. package/dist/reporters/console.js.map +1 -1
  64. package/dist/reporters/format.d.ts +6 -0
  65. package/dist/reporters/format.d.ts.map +1 -1
  66. package/dist/reporters/format.js +4 -0
  67. package/dist/reporters/format.js.map +1 -1
  68. package/dist/reporters/github.d.ts.map +1 -1
  69. package/dist/reporters/github.js +3 -2
  70. package/dist/reporters/github.js.map +1 -1
  71. package/dist/reporters/index.d.ts +34 -21
  72. package/dist/reporters/index.d.ts.map +1 -1
  73. package/dist/reporters/index.js.map +1 -1
  74. package/dist/schema.d.ts +14 -0
  75. package/dist/schema.d.ts.map +1 -1
  76. package/dist/schema.js +13 -0
  77. package/dist/schema.js.map +1 -1
  78. package/package.json +8 -3
package/dist/schema.js CHANGED
@@ -1,14 +1,26 @@
1
1
  import { z } from 'zod';
2
+ /** A tool call a case expects: the tool's name and, optionally, arguments to compare */
3
+ export const ExpectedToolCallSchema = z.object({
4
+ name: z.string().min(1),
5
+ arguments: z.record(z.string(), z.unknown()).optional(),
6
+ });
2
7
  export const EvalCaseSchema = z.object({
3
8
  input: z.string(),
4
9
  expected: z.string().optional(),
5
10
  context: z.record(z.string(), z.unknown()).optional(),
6
11
  metadata: z.record(z.string(), z.unknown()).optional(),
12
+ /** Further user turns after `input`, run in order on the same conversation */
13
+ turns: z.array(z.string()).optional(),
14
+ /** The tool calls the agent should make over the whole conversation, in order */
15
+ expectedToolCalls: z.array(ExpectedToolCallSchema).optional(),
16
+ /** Ids or texts of the passages the retrieval should surface */
17
+ expectedPassages: z.array(z.string().min(1)).optional(),
7
18
  });
8
19
  export const EvalSuiteConfigSchema = z.object({
9
20
  concurrency: z.number().int().min(1).default(5),
10
21
  timeout: z.number().int().min(1000).default(30000),
11
22
  retries: z.number().int().min(0).max(10).default(0),
23
+ trials: z.number().int().min(1).default(1),
12
24
  });
13
25
  function isJudgeCache(value) {
14
26
  return (typeof value === 'object' &&
@@ -42,5 +54,6 @@ export const EvalComparisonConfigSchema = z.object({
42
54
  concurrency: z.number().int().min(1).default(5),
43
55
  timeout: z.number().int().min(1000).default(30000),
44
56
  retries: z.number().int().min(0).max(10).default(0),
57
+ trials: z.number().int().min(1).default(1),
45
58
  });
46
59
  //# sourceMappingURL=schema.js.map
package/dist/schema.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,iFAAiF;IACjF,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IAC7D,gEAAgE;IAChE,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cogitator-ai/evals",
3
- "version": "0.9.0",
3
+ "version": "0.11.0",
4
4
  "description": "Evaluation framework for Cogitator AI agents",
5
5
  "keywords": [
6
6
  "cogitator",
@@ -26,6 +26,11 @@
26
26
  "types": "./dist/vitest.d.ts",
27
27
  "import": "./dist/vitest.js",
28
28
  "default": "./dist/vitest.js"
29
+ },
30
+ "./matchers": {
31
+ "types": "./dist/matchers.d.ts",
32
+ "import": "./dist/matchers.js",
33
+ "default": "./dist/matchers.js"
29
34
  }
30
35
  },
31
36
  "sideEffects": false,
@@ -39,7 +44,7 @@
39
44
  "zod": "^4.6.5"
40
45
  },
41
46
  "peerDependencies": {
42
- "@cogitator-ai/core": ">=0.56.4",
47
+ "@cogitator-ai/core": ">=0.58.0",
43
48
  "vitest": ">=3.0.0"
44
49
  },
45
50
  "peerDependenciesMeta": {
@@ -54,7 +59,7 @@
54
59
  "papaparse": "^5.7.0"
55
60
  },
56
61
  "devDependencies": {
57
- "@cogitator-ai/core": "0.56.4",
62
+ "@cogitator-ai/core": "0.58.0",
58
63
  "@types/papaparse": "^5.3.15",
59
64
  "typescript": "^6.0.3",
60
65
  "vitest": "^5.0.3"