@cogitator-ai/evals 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -175,6 +175,12 @@ const suite = new EvalSuite({
|
|
|
175
175
|
});
|
|
176
176
|
```
|
|
177
177
|
|
|
178
|
+
The judge reads the case's `context` too, so `faithfulness()` can check a response against the source documents of a case without putting them into `input`:
|
|
179
|
+
|
|
180
|
+
```typescript
|
|
181
|
+
Dataset.from([{ input: 'Write the lede', context: { dossier: dossierText } }]);
|
|
182
|
+
```
|
|
183
|
+
|
|
178
184
|
The `judge` option is a `JudgeConfigInput`: `{ model, temperature?, maxTokens?, cogitator? }`, with `temperature` defaulting to `0`.
|
|
179
185
|
|
|
180
186
|
### Statistical
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,QAAQ,EAA+B,MAAM,SAAS,CAAC;AAErE,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QAAE,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAC7E,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAiCD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAiCjG;
|
|
1
|
+
{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,QAAQ,EAA+B,MAAM,SAAS,CAAC;AAErE,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QAAE,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAC7E,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAiCD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAiCjG;AAoCD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CAmCrF;AAED,wBAAgB,YAAY,IAAI,WAAW,CAK1C;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,SAAS,IAAI,WAAW,CAKvC;AAED,wBAAgB,WAAW,IAAI,WAAW,CAKzC;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}
|
|
@@ -67,9 +67,20 @@ function createJudgeMetric(name, systemPrompt) {
|
|
|
67
67
|
Object.defineProperty(fn, '__judgeName', { value: name, writable: false });
|
|
68
68
|
return fn;
|
|
69
69
|
}
|
|
70
|
+
function contextText(context) {
|
|
71
|
+
return Object.entries(context)
|
|
72
|
+
.map(([key, value]) => `${key}: ${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}`)
|
|
73
|
+
.join('\n');
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* What the judge reads: the input, the case's context when it has one (source documents, a
|
|
77
|
+
* dossier, retrieved passages), the expected answer and the response.
|
|
78
|
+
*/
|
|
70
79
|
function buildUserMessage(result) {
|
|
71
80
|
const expected = result.case.expected || 'N/A';
|
|
72
|
-
|
|
81
|
+
const context = result.case.context;
|
|
82
|
+
const contextBlock = context && Object.keys(context).length > 0 ? `\nContext:\n${contextText(context)}` : '';
|
|
83
|
+
return `Input: ${result.case.input}${contextBlock}\nExpected: ${expected}\nResponse: ${result.output}`;
|
|
73
84
|
}
|
|
74
85
|
export function bindJudgeContext(metric, context) {
|
|
75
86
|
const name = metric.__judgeName;
|
|
@@ -102,7 +113,7 @@ export function bindJudgeContext(metric, context) {
|
|
|
102
113
|
return bound;
|
|
103
114
|
}
|
|
104
115
|
export function faithfulness() {
|
|
105
|
-
return createJudgeMetric('faithfulness', 'You are evaluating the faithfulness of an AI assistant\'s response.\n\nGiven the input and the response, rate how faithful the response is to the facts and information in the input.\n\nScore from 0.0 (completely unfaithful) to 1.0 (perfectly faithful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
|
|
116
|
+
return createJudgeMetric('faithfulness', 'You are evaluating the faithfulness of an AI assistant\'s response.\n\nGiven the input, the context when there is one, and the response, rate how faithful the response is to the facts and information in the input and the context. Claims that neither supports count against it.\n\nScore from 0.0 (completely unfaithful) to 1.0 (perfectly faithful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
|
|
106
117
|
}
|
|
107
118
|
export function relevance() {
|
|
108
119
|
return createJudgeMetric('relevance', 'You are evaluating the relevance of an AI assistant\'s response.\n\nGiven the input and the response, rate how relevant the response is to the question asked.\n\nScore from 0.0 (completely irrelevant) to 1.0 (perfectly relevant).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,MAAM,WAAW,GAAG,8BAA8B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAElC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,IAAwB;IACzC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IAC7C,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1F,IAAI,IAAI,EAAE,EAAE;QAAE,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,OAAO;QACL,WAAW;QACX,SAAS,EAAE;YACT,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/E,OAAO;oBACL,MAAM,EACJ,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;iBACtF,CAAC;YACJ,CAAC;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,MAAM,WAAW,GAAG,8BAA8B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAElC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,IAAwB;IACzC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IAC7C,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1F,IAAI,IAAI,EAAE,EAAE;QAAE,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,OAAO;QACL,WAAW;QACX,SAAS,EAAE;YACT,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/E,OAAO;oBACL,MAAM,EACJ,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;iBACtF,CAAC;YACJ,CAAC;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,OAAgC;IACnD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,GAAG,KAAK,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAClF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;IACpC,MAAM,YAAY,GAChB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,YAAY,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AACzG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;YACvE,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;aACrE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAiB,GAAa,CAAC,OAAO,EAAE;aAClD,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,iBAAiB,CACtB,cAAc,EACd,qaAAqa,CACta,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,iBAAiB,CACtB,WAAW,EACX,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,iBAAiB,CACtB,aAAa,EACb,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Evaluation framework for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"zod": "^4.6.5"
|
|
22
22
|
},
|
|
23
23
|
"peerDependencies": {
|
|
24
|
-
"@cogitator-ai/core": ">=0.
|
|
24
|
+
"@cogitator-ai/core": ">=0.28.0"
|
|
25
25
|
},
|
|
26
26
|
"peerDependenciesMeta": {
|
|
27
27
|
"@cogitator-ai/core": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"papaparse": "^5.7.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
|
-
"@cogitator-ai/core": "0.
|
|
35
|
+
"@cogitator-ai/core": "0.28.0",
|
|
36
36
|
"@types/papaparse": "^5.3.15",
|
|
37
37
|
"typescript": "^6.0.3",
|
|
38
38
|
"vitest": "^5.0.3"
|