@axlsdk/eval 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@ Evaluation framework for Axl agentic workflows. Define datasets, scoring functio
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
npm install
|
|
8
|
+
npm install @axlsdk/eval
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
## API
|
|
@@ -15,7 +15,7 @@ npm install axl-eval
|
|
|
15
15
|
Define evaluation datasets:
|
|
16
16
|
|
|
17
17
|
```typescript
|
|
18
|
-
import { dataset } from '
|
|
18
|
+
import { dataset } from '@axlsdk/eval';
|
|
19
19
|
import { z } from 'zod';
|
|
20
20
|
|
|
21
21
|
// Inline items with schema and annotations
|
|
@@ -43,7 +43,7 @@ const ds = dataset({
|
|
|
43
43
|
Define deterministic scoring functions. The `score` callback receives `(output, input, annotations?)`:
|
|
44
44
|
|
|
45
45
|
```typescript
|
|
46
|
-
import { scorer } from '
|
|
46
|
+
import { scorer } from '@axlsdk/eval';
|
|
47
47
|
|
|
48
48
|
const exactMatch = scorer({
|
|
49
49
|
name: 'exact-match',
|
|
@@ -67,7 +67,7 @@ const partialMatch = scorer({
|
|
|
67
67
|
Use an LLM as a judge. The scorer automatically constructs a prompt from the input, output, and annotations, then validates the response against your Zod schema:
|
|
68
68
|
|
|
69
69
|
```typescript
|
|
70
|
-
import { llmScorer } from '
|
|
70
|
+
import { llmScorer } from '@axlsdk/eval';
|
|
71
71
|
import { z } from 'zod';
|
|
72
72
|
|
|
73
73
|
const qualityJudge = llmScorer({
|
|
@@ -88,7 +88,7 @@ const qualityJudge = llmScorer({
|
|
|
88
88
|
Run an evaluation:
|
|
89
89
|
|
|
90
90
|
```typescript
|
|
91
|
-
import { runEval } from '
|
|
91
|
+
import { runEval } from '@axlsdk/eval';
|
|
92
92
|
|
|
93
93
|
const results = await runEval(
|
|
94
94
|
{
|
|
@@ -111,7 +111,7 @@ console.log(results.summary);
|
|
|
111
111
|
Compare two evaluation runs:
|
|
112
112
|
|
|
113
113
|
```typescript
|
|
114
|
-
import { evalCompare } from '
|
|
114
|
+
import { evalCompare } from '@axlsdk/eval';
|
|
115
115
|
|
|
116
116
|
const comparison = evalCompare(baselineResults, candidateResults);
|
|
117
117
|
|
|
@@ -127,7 +127,7 @@ console.log(comparison.improvements);
|
|
|
127
127
|
Register an eval for CLI discovery:
|
|
128
128
|
|
|
129
129
|
```typescript
|
|
130
|
-
import { defineEval } from '
|
|
130
|
+
import { defineEval } from '@axlsdk/eval';
|
|
131
131
|
|
|
132
132
|
export default defineEval({
|
|
133
133
|
name: 'math-eval',
|
|
@@ -151,7 +151,7 @@ npx axl-eval compare ./results/baseline.json ./results/candidate.json
|
|
|
151
151
|
## Integration with AxlRuntime
|
|
152
152
|
|
|
153
153
|
```typescript
|
|
154
|
-
import { AxlRuntime } from 'axl';
|
|
154
|
+
import { AxlRuntime } from '@axlsdk/axl';
|
|
155
155
|
|
|
156
156
|
const runtime = new AxlRuntime();
|
|
157
157
|
runtime.register(myWorkflow);
|