@fabric-harness/evals 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -1
- package/dist/index.d.ts +60 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +95 -0
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
package/README.md
CHANGED
|
@@ -10,9 +10,41 @@ Define test cases, run them against your agents, and track quality over time.
|
|
|
10
10
|
npm install @fabric-harness/evals @fabric-harness/sdk
|
|
11
11
|
```
|
|
12
12
|
|
|
13
|
+
## Quick start
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { runEvalSuite, exactMatchScorer, llmAsJudgeScorer, jsonShapeMatchScorer } from '@fabric-harness/evals';
|
|
17
|
+
|
|
18
|
+
const result = await runEvalSuite({
|
|
19
|
+
name: 'triage-quality',
|
|
20
|
+
cases: [
|
|
21
|
+
{ id: 'sev-high', input: { issue: 'data loss in prod' }, expected: { severity: 'high' } },
|
|
22
|
+
{ id: 'sev-low', input: { issue: 'typo in docs' }, expected: { severity: 'low' } },
|
|
23
|
+
],
|
|
24
|
+
runner: async ({ case: c }) => triageAgent(c.input),
|
|
25
|
+
scorers: [
|
|
26
|
+
jsonShapeMatchScorer({ severity: 'string', summary: 'string' }),
|
|
27
|
+
llmAsJudgeScorer({
|
|
28
|
+
judge: anthropicProvider,
|
|
29
|
+
model: 'anthropic/claude-haiku-4-5',
|
|
30
|
+
rubric: 'Severity matches the actual urgency of the issue.',
|
|
31
|
+
}),
|
|
32
|
+
],
|
|
33
|
+
passThreshold: 0.8,
|
|
34
|
+
});
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Scorers
|
|
38
|
+
|
|
39
|
+
- **`exactMatchScorer()`** — strict JSON-equality against `case.expected`.
|
|
40
|
+
- **`containsTextScorer(text?)`** — substring check on stringified output.
|
|
41
|
+
- **`regexMatchScorer(pattern, options?)`** — regex match against the stringified output.
|
|
42
|
+
- **`jsonShapeMatchScorer({ field: 'string' | 'number' | 'boolean' | 'array' | 'object' | nested })`** — schema-shape check with detailed mismatch reasons.
|
|
43
|
+
- **`llmAsJudgeScorer({ judge, model, rubric, passThreshold? })`** — LLM-as-judge with structural JSON parsing and graceful fallback on malformed responses.
|
|
44
|
+
|
|
13
45
|
## Documentation
|
|
14
46
|
|
|
15
|
-
- [Evals reference](https://
|
|
47
|
+
- [Evals reference](https://harness.fabric.pro/docs/building/evals)
|
|
16
48
|
|
|
17
49
|
## License
|
|
18
50
|
|
package/dist/index.d.ts
CHANGED
|
@@ -53,4 +53,64 @@ export interface EvalSuiteResult<TOutput = unknown> {
|
|
|
53
53
|
export declare function runEvalSuite<TInput, TOutput, TExpected>(suite: EvalSuite<TInput, TOutput, TExpected>): Promise<EvalSuiteResult<TOutput>>;
|
|
54
54
|
export declare function exactMatchScorer(name?: string): EvalScorer<unknown, unknown, unknown>;
|
|
55
55
|
export declare function containsTextScorer(expectedText?: string, name?: string): EvalScorer<unknown, unknown, unknown>;
|
|
56
|
+
/**
|
|
57
|
+
* Score against a regex applied to the stringified output.
|
|
58
|
+
*
|
|
59
|
+
* ```ts
|
|
60
|
+
* regexMatchScorer(/^[A-Z]{3}-\d{4}$/, { name: 'ticket-format' })
|
|
61
|
+
* ```
|
|
62
|
+
*/
|
|
63
|
+
export declare function regexMatchScorer(pattern: RegExp | string, options?: {
|
|
64
|
+
name?: string;
|
|
65
|
+
flags?: string;
|
|
66
|
+
}): EvalScorer<unknown, unknown, unknown>;
|
|
67
|
+
/**
|
|
68
|
+
* Score against a JSON shape skeleton: required keys present, types match.
|
|
69
|
+
* Extra keys are ignored.
|
|
70
|
+
*
|
|
71
|
+
* ```ts
|
|
72
|
+
* jsonShapeMatchScorer({ severity: 'string', summary: 'string', labels: 'array' })
|
|
73
|
+
* ```
|
|
74
|
+
*/
|
|
75
|
+
export type JsonShape = {
|
|
76
|
+
[key: string]: 'string' | 'number' | 'boolean' | 'array' | 'object' | JsonShape;
|
|
77
|
+
};
|
|
78
|
+
export declare function jsonShapeMatchScorer(shape: JsonShape, options?: {
|
|
79
|
+
name?: string;
|
|
80
|
+
}): EvalScorer<unknown, unknown, unknown>;
|
|
81
|
+
/**
|
|
82
|
+
* Use a model provider to judge whether the output meets the rubric. Returns
|
|
83
|
+
* a 0-1 score parsed from the judge's response.
|
|
84
|
+
*
|
|
85
|
+
* ```ts
|
|
86
|
+
* llmAsJudgeScorer({
|
|
87
|
+
* judge: anthropicProvider,
|
|
88
|
+
* model: 'anthropic/claude-haiku-4-5',
|
|
89
|
+
* rubric: 'Reply is friendly, concise, and answers the question.',
|
|
90
|
+
* })
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
export interface LlmJudgeProvider {
|
|
94
|
+
generate(request: {
|
|
95
|
+
messages: Array<{
|
|
96
|
+
role: 'system' | 'user' | 'assistant';
|
|
97
|
+
content: string;
|
|
98
|
+
}>;
|
|
99
|
+
model: string;
|
|
100
|
+
}): Promise<{
|
|
101
|
+
message?: {
|
|
102
|
+
content?: string;
|
|
103
|
+
};
|
|
104
|
+
content?: string;
|
|
105
|
+
}>;
|
|
106
|
+
}
|
|
107
|
+
export interface LlmJudgeOptions {
|
|
108
|
+
judge: LlmJudgeProvider;
|
|
109
|
+
model: string;
|
|
110
|
+
rubric: string;
|
|
111
|
+
name?: string;
|
|
112
|
+
/** Score below which the case is considered failing. Defaults to 0.7. */
|
|
113
|
+
passThreshold?: number;
|
|
114
|
+
}
|
|
115
|
+
export declare function llmAsJudgeScorer(options: LlmJudgeOptions): EvalScorer<unknown, unknown, unknown>;
|
|
56
116
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,WAAW,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO;IAC7D,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,MAAM,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO;IACnE,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;AAC9J,MAAM,MAAM,UAAU,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO,IAAI,CAAC,KAAK,EAAE;IACzF,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAClC,MAAM,EAAE,OAAO,CAAC;CACjB,KAAK,OAAO,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;AAErC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;IACrC,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAC/C,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;IAClD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,OAAO,GAAG,OAAO;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CA+B9I;AAED,wBAAgB,gBAAgB,CAAC,IAAI,SAAgB,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAK5F;AAED,wBAAgB,kBAAkB,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,SAAkB,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAOvH"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,WAAW,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO;IAC7D,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,MAAM,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO;IACnE,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;AAC9J,MAAM,MAAM,UAAU,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO,IAAI,CAAC,KAAK,EAAE;IACzF,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAClC,MAAM,EAAE,OAAO,CAAC;CACjB,KAAK,OAAO,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;AAErC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,EAAE,SAAS,GAAG,OAAO;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;IACrC,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAC/C,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;IAClD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,OAAO,GAAG,OAAO;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CA+B9I;AAED,wBAAgB,gBAAgB,CAAC,IAAI,SAAgB,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAK5F;AAED,wBAAgB,kBAAkB,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,SAAkB,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAOvH;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAQjJ;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;CAAE,CAAC;AAE5G,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAO7H;AAyBD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,OAAO,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAClL;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,gBAAgB,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,eAAe,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAgChG"}
|
package/dist/index.js
CHANGED
|
@@ -46,6 +46,101 @@ export function containsTextScorer(expectedText, name = 'contains_text') {
|
|
|
46
46
|
return { name, score: passed ? 1 : 0, passed, reason: passed ? `Output contained ${needle}.` : `Output did not contain ${needle ?? '<missing expected text>'}.` };
|
|
47
47
|
};
|
|
48
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* Score against a regex applied to the stringified output.
|
|
51
|
+
*
|
|
52
|
+
* ```ts
|
|
53
|
+
* regexMatchScorer(/^[A-Z]{3}-\d{4}$/, { name: 'ticket-format' })
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
export function regexMatchScorer(pattern, options = {}) {
|
|
57
|
+
const regex = typeof pattern === 'string' ? new RegExp(pattern, options.flags) : pattern;
|
|
58
|
+
const name = options.name ?? `regex_match:${regex.source}`;
|
|
59
|
+
return ({ output }) => {
|
|
60
|
+
const haystack = typeof output === 'string' ? output : JSON.stringify(output);
|
|
61
|
+
const passed = regex.test(haystack);
|
|
62
|
+
return { name, score: passed ? 1 : 0, passed, reason: passed ? `Output matched ${regex}.` : `Output did not match ${regex}.` };
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
export function jsonShapeMatchScorer(shape, options = {}) {
|
|
66
|
+
const name = options.name ?? 'json_shape';
|
|
67
|
+
return ({ output }) => {
|
|
68
|
+
const issues = checkShape(output, shape);
|
|
69
|
+
const passed = issues.length === 0;
|
|
70
|
+
return { name, score: passed ? 1 : 0, passed, reason: passed ? 'Output matched shape.' : `Shape mismatch: ${issues.join('; ')}` };
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
function checkShape(value, shape, path = '') {
|
|
74
|
+
const issues = [];
|
|
75
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
76
|
+
issues.push(`${path || '<root>'}: expected object`);
|
|
77
|
+
return issues;
|
|
78
|
+
}
|
|
79
|
+
for (const [key, expected] of Object.entries(shape)) {
|
|
80
|
+
const subpath = path ? `${path}.${key}` : key;
|
|
81
|
+
const actual = value[key];
|
|
82
|
+
if (actual === undefined) {
|
|
83
|
+
issues.push(`${subpath}: missing`);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if (typeof expected === 'string') {
|
|
87
|
+
const actualType = Array.isArray(actual) ? 'array' : typeof actual;
|
|
88
|
+
if (actualType !== expected)
|
|
89
|
+
issues.push(`${subpath}: expected ${expected}, got ${actualType}`);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
issues.push(...checkShape(actual, expected, subpath));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return issues;
|
|
96
|
+
}
|
|
97
|
+
export function llmAsJudgeScorer(options) {
|
|
98
|
+
const name = options.name ?? 'llm_as_judge';
|
|
99
|
+
const passThreshold = options.passThreshold ?? 0.7;
|
|
100
|
+
return async ({ case: evalCase, output }) => {
|
|
101
|
+
const expectedText = evalCase.expected !== undefined ? `\nExpected (reference):\n${typeof evalCase.expected === 'string' ? evalCase.expected : JSON.stringify(evalCase.expected)}` : '';
|
|
102
|
+
const outputText = typeof output === 'string' ? output : JSON.stringify(output);
|
|
103
|
+
const inputText = typeof evalCase.input === 'string' ? evalCase.input : JSON.stringify(evalCase.input);
|
|
104
|
+
const prompt = `You are an impartial evaluator. Rate the output against the rubric on a 0.0-1.0 scale.
|
|
105
|
+
|
|
106
|
+
Rubric: ${options.rubric}
|
|
107
|
+
|
|
108
|
+
Input:
|
|
109
|
+
${inputText}
|
|
110
|
+
${expectedText}
|
|
111
|
+
|
|
112
|
+
Output to score:
|
|
113
|
+
${outputText}
|
|
114
|
+
|
|
115
|
+
Respond with a single JSON object: {"score": <number 0-1>, "reason": "<short explanation>"}.
|
|
116
|
+
Do not include any other text.`;
|
|
117
|
+
const response = await options.judge.generate({
|
|
118
|
+
model: options.model,
|
|
119
|
+
messages: [
|
|
120
|
+
{ role: 'system', content: 'You are an impartial evaluator. Output JSON only.' },
|
|
121
|
+
{ role: 'user', content: prompt },
|
|
122
|
+
],
|
|
123
|
+
});
|
|
124
|
+
const text = response.message?.content ?? response.content ?? '';
|
|
125
|
+
const parsed = parseJudgeResponse(text);
|
|
126
|
+
const passed = parsed.score >= passThreshold;
|
|
127
|
+
return { name, score: parsed.score, passed, ...(parsed.reason !== undefined ? { reason: parsed.reason } : {}) };
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
function parseJudgeResponse(text) {
|
|
131
|
+
try {
|
|
132
|
+
const match = text.match(/\{[\s\S]*\}/);
|
|
133
|
+
if (!match)
|
|
134
|
+
return { score: 0, reason: `Judge did not return JSON: ${text.slice(0, 120)}` };
|
|
135
|
+
const obj = JSON.parse(match[0]);
|
|
136
|
+
const score = typeof obj.score === 'number' ? Math.max(0, Math.min(1, obj.score)) : 0;
|
|
137
|
+
const reason = typeof obj.reason === 'string' ? obj.reason : undefined;
|
|
138
|
+
return reason !== undefined ? { score, reason } : { score };
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
return { score: 0, reason: `Judge JSON parse failed: ${error instanceof Error ? error.message : String(error)}` };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
49
144
|
function average(values) {
|
|
50
145
|
if (values.length === 0)
|
|
51
146
|
return 0;
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA4DA,MAAM,CAAC,KAAK,UAAU,YAAY,CAA6B,KAA4C;IACzG,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,KAAK,GAA8B,EAAE,CAAC;IAC5C,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC7E,MAAM,MAAM,GAAG,EAAE,CAAC;YAClB,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACjH,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI,YAAY,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;YAC/G,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QAChG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QAClK,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjG,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACtG,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,SAAS;QACT,WAAW,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC9C,UAAU,EAAE,SAAS,GAAG,OAAO;QAC/B,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,QAAQ;QACR,YAAY;QACZ,KAAK;QACL,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAI,GAAG,aAAa;IACnD,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5E,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,sCAAsC,EAAE,CAAC;IAC7I,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,YAAqB,EAAE,IAAI,GAAG,eAAe;IAC9E,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,YAAY,IAAI,CAAC,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvG,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,oBAAoB,MAAM,GAAG,CAAC,CAAC,CAAC,0BAA0B,MAAM,IAAI,yBAAyB,GAAG,EAAE,CAAC;IACpK,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA4DA,MAAM,CAAC,KAAK,UAAU,YAAY,CAA6B,KAA4C;IACzG,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,KAAK,GAA8B,EAAE,CAAC;IAC5C,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC7E,MAAM,MAAM,GAAG,EAAE,CAAC;YAClB,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACjH,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI,YAAY,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;YAC/G,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QAChG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QAClK,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjG,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACtG,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,SAAS;QACT,WAAW,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAC9C,UAAU,EAAE,SAAS,GAAG,OAAO;QAC/B,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,QAAQ;QACR,YAAY;QACZ,KAAK;QACL,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAI,GAAG,aAAa;IACnD,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5E,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,sCAAsC,EAAE,CAAC;IAC7I,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,YAAqB,EAAE,IAAI,GAAG,eAAe;IAC9E,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,YAAY,IAAI,CAAC,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvG,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,oBAAoB,MAAM,GAAG,CAAC,CAAC,CAAC,0BAA0B,MAAM,IAAI,yBAAyB,GAAG,EAAE,CAAC;IACpK,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAwB,EAAE,UAA6C,EAAE;IACxG,MAAM,KAAK,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACzF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,eAAe,KAAK,CAAC,MAAM,EAAE,CAAC;IAC3D,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QACpB,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,KAAK,GAAG,CAAC,CAAC,CAAC,wBAAwB,KAAK,GAAG,EAAE,CAAC;IACjI,CAAC,CAAC;AACJ,CAAC;AAYD,MAAM,UAAU,oBAAoB,CAAC,KAAgB,EAAE,UAA6B,EAAE;IACpF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,YAAY,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;QACnC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,mBAAmB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;IACpI,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,KAAgB,EAAE,IAAI,GAAG,EAAE;IAC7D,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,mBAAmB,CAAC,CAAC;QACpD,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9C,MAAM,MAAM,GAAI,KAAiC,CAAC,GAAG,CAAC,CAAC;QACvD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,WAAW,CAAC,CAAC;YACnC,SAAS;QACX,CAAC;QACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC;YACnE,IAAI,UAAU,KAAK,QAAQ;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,cAAc,QAAQ,SAAS,UAAU,EAAE,CAAC,CAAC;QAClG,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AA2BD,MAAM,UAAU,gBAAgB,CAAC,OAAwB;IACvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,cAAc,CAAC;IAC5C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC;IACnD,OAAO,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;QAC1C,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,4BAA4B,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxL,MAAM,UAAU,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAChF,MAAM,SAAS,GAAG,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvG,MAAM,MAAM,GAAG;;UAET,OAAO,CAAC,MAAM;;;EAGtB,SAAS;EACT,YAAY;;;EAGZ,UAAU;;;+BAGmB,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC5C,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,mDAAmD,EAAE;gBAChF,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;aAClC;SACF,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,OAAO,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;QACjE,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;QAC7C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,8BAA8B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC;QAC5F,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAA0C,CAAC;QAC1E,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QACvE,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;IAC9D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;IACpH,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fabric-harness/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Evaluation framework for Fabric Harness agents.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Fabric",
|
|
@@ -37,17 +37,17 @@
|
|
|
37
37
|
"README.md",
|
|
38
38
|
"LICENSE"
|
|
39
39
|
],
|
|
40
|
-
"scripts": {
|
|
41
|
-
"build": "tsc -p tsconfig.json",
|
|
42
|
-
"check": "tsc -b tsconfig.json",
|
|
43
|
-
"test": "vitest run"
|
|
44
|
-
},
|
|
45
40
|
"dependencies": {
|
|
46
|
-
"@fabric-harness/sdk": "
|
|
41
|
+
"@fabric-harness/sdk": "0.6.0"
|
|
47
42
|
},
|
|
48
43
|
"devDependencies": {
|
|
49
44
|
"@types/node": "^22.15.3",
|
|
50
45
|
"typescript": "^5.8.3",
|
|
51
46
|
"vitest": "^4.1.5"
|
|
47
|
+
},
|
|
48
|
+
"scripts": {
|
|
49
|
+
"build": "tsc -p tsconfig.json",
|
|
50
|
+
"check": "tsc -b tsconfig.json",
|
|
51
|
+
"test": "vitest run"
|
|
52
52
|
}
|
|
53
|
-
}
|
|
53
|
+
}
|