eval-harness 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1046 -0
- package/README.zh-CN.md +597 -0
- package/bin/eval-harness.js +65 -0
- package/config/eval.example.json +70 -0
- package/docs/agent-runner-compatibility.md +47 -0
- package/docs/runners.md +82 -0
- package/package.json +43 -0
- package/prompts/judge.md +27 -0
- package/pyproject.toml +48 -0
- package/schemas/batch-comparison.schema.json +64 -0
- package/schemas/batch-summary.schema.json +85 -0
- package/schemas/evaluation.schema.json +81 -0
- package/schemas/experiment.schema.json +63 -0
- package/schemas/history-index.schema.json +37 -0
- package/schemas/history-record.schema.json +56 -0
- package/schemas/model-report.schema.json +38 -0
- package/schemas/next-action.schema.json +62 -0
- package/schemas/skill-improvement-input.schema.json +65 -0
- package/src/eval/__init__.py +3 -0
- package/src/eval/__main__.py +5 -0
- package/src/eval/cli.py +11150 -0
- package/src/eval/runners/__init__.py +0 -0
- package/src/eval/runners/base.py +197 -0
- package/src/eval/runners/claude.py +516 -0
- package/src/eval/runners/codex.py +412 -0
- package/src/eval/runners/mimocode.py +676 -0
- package/src/eval/runners/opencode.py +554 -0
- package/src/eval/schema.py +151 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Eval Harness History Record",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"required": [
|
|
6
|
+
"schema",
|
|
7
|
+
"batch_id",
|
|
8
|
+
"created_at",
|
|
9
|
+
"dry_run",
|
|
10
|
+
"count",
|
|
11
|
+
"parallel",
|
|
12
|
+
"decision_counts",
|
|
13
|
+
"recommendation_area_counts"
|
|
14
|
+
],
|
|
15
|
+
"properties": {
|
|
16
|
+
"schema": {
|
|
17
|
+
"const": "eval-harness.history-record.v1"
|
|
18
|
+
},
|
|
19
|
+
"score_dimensions": {
|
|
20
|
+
"type": "array"
|
|
21
|
+
},
|
|
22
|
+
"batch_id": {
|
|
23
|
+
"type": [
|
|
24
|
+
"string",
|
|
25
|
+
"null"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
28
|
+
"created_at": {
|
|
29
|
+
"type": [
|
|
30
|
+
"string",
|
|
31
|
+
"null"
|
|
32
|
+
]
|
|
33
|
+
},
|
|
34
|
+
"dry_run": {
|
|
35
|
+
"type": "boolean"
|
|
36
|
+
},
|
|
37
|
+
"count": {
|
|
38
|
+
"type": [
|
|
39
|
+
"integer",
|
|
40
|
+
"null"
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
"parallel": {
|
|
44
|
+
"type": [
|
|
45
|
+
"integer",
|
|
46
|
+
"null"
|
|
47
|
+
]
|
|
48
|
+
},
|
|
49
|
+
"decision_counts": {
|
|
50
|
+
"type": "object"
|
|
51
|
+
},
|
|
52
|
+
"recommendation_area_counts": {
|
|
53
|
+
"type": "object"
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Eval Harness Model Report",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"required": [
|
|
6
|
+
"schema",
|
|
7
|
+
"batch_id",
|
|
8
|
+
"created_at",
|
|
9
|
+
"model_count",
|
|
10
|
+
"models"
|
|
11
|
+
],
|
|
12
|
+
"properties": {
|
|
13
|
+
"schema": {
|
|
14
|
+
"const": "eval-harness.model-report.v1"
|
|
15
|
+
},
|
|
16
|
+
"score_dimensions": {
|
|
17
|
+
"type": "array"
|
|
18
|
+
},
|
|
19
|
+
"batch_id": {
|
|
20
|
+
"type": [
|
|
21
|
+
"string",
|
|
22
|
+
"null"
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
"created_at": {
|
|
26
|
+
"type": [
|
|
27
|
+
"string",
|
|
28
|
+
"null"
|
|
29
|
+
]
|
|
30
|
+
},
|
|
31
|
+
"model_count": {
|
|
32
|
+
"type": "integer"
|
|
33
|
+
},
|
|
34
|
+
"models": {
|
|
35
|
+
"type": "object"
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Eval Harness Next Action",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"required": [
|
|
6
|
+
"schema",
|
|
7
|
+
"generated_at",
|
|
8
|
+
"run_id",
|
|
9
|
+
"score",
|
|
10
|
+
"decision",
|
|
11
|
+
"priority",
|
|
12
|
+
"area",
|
|
13
|
+
"change_type",
|
|
14
|
+
"target_files",
|
|
15
|
+
"validation_plan",
|
|
16
|
+
"guardrails"
|
|
17
|
+
],
|
|
18
|
+
"properties": {
|
|
19
|
+
"schema": {
|
|
20
|
+
"const": "eval-harness.next-action.v1"
|
|
21
|
+
},
|
|
22
|
+
"generated_at": {
|
|
23
|
+
"type": "string"
|
|
24
|
+
},
|
|
25
|
+
"run_id": {
|
|
26
|
+
"type": [
|
|
27
|
+
"string",
|
|
28
|
+
"null"
|
|
29
|
+
]
|
|
30
|
+
},
|
|
31
|
+
"score": {
|
|
32
|
+
"type": [
|
|
33
|
+
"integer",
|
|
34
|
+
"null"
|
|
35
|
+
]
|
|
36
|
+
},
|
|
37
|
+
"decision": {
|
|
38
|
+
"type": [
|
|
39
|
+
"string",
|
|
40
|
+
"null"
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
"priority": {
|
|
44
|
+
"type": "string"
|
|
45
|
+
},
|
|
46
|
+
"area": {
|
|
47
|
+
"type": "string"
|
|
48
|
+
},
|
|
49
|
+
"change_type": {
|
|
50
|
+
"type": "string"
|
|
51
|
+
},
|
|
52
|
+
"target_files": {
|
|
53
|
+
"type": "array"
|
|
54
|
+
},
|
|
55
|
+
"validation_plan": {
|
|
56
|
+
"type": "array"
|
|
57
|
+
},
|
|
58
|
+
"guardrails": {
|
|
59
|
+
"type": "array"
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Eval Harness Skill Improvement Input",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"required": [
|
|
6
|
+
"schema",
|
|
7
|
+
"generated_at",
|
|
8
|
+
"run_id",
|
|
9
|
+
"score",
|
|
10
|
+
"decision",
|
|
11
|
+
"primary_failure_category",
|
|
12
|
+
"failure_taxonomy",
|
|
13
|
+
"next_action",
|
|
14
|
+
"target_candidates",
|
|
15
|
+
"evidence",
|
|
16
|
+
"recommended_change_set"
|
|
17
|
+
],
|
|
18
|
+
"properties": {
|
|
19
|
+
"schema": {
|
|
20
|
+
"const": "eval-harness.skill-improvement-input.v1"
|
|
21
|
+
},
|
|
22
|
+
"generated_at": {
|
|
23
|
+
"type": "string"
|
|
24
|
+
},
|
|
25
|
+
"run_id": {
|
|
26
|
+
"type": [
|
|
27
|
+
"string",
|
|
28
|
+
"null"
|
|
29
|
+
]
|
|
30
|
+
},
|
|
31
|
+
"score": {
|
|
32
|
+
"type": [
|
|
33
|
+
"integer",
|
|
34
|
+
"null"
|
|
35
|
+
]
|
|
36
|
+
},
|
|
37
|
+
"decision": {
|
|
38
|
+
"type": [
|
|
39
|
+
"string",
|
|
40
|
+
"null"
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
"primary_failure_category": {
|
|
44
|
+
"type": [
|
|
45
|
+
"string",
|
|
46
|
+
"null"
|
|
47
|
+
]
|
|
48
|
+
},
|
|
49
|
+
"failure_taxonomy": {
|
|
50
|
+
"type": "array"
|
|
51
|
+
},
|
|
52
|
+
"next_action": {
|
|
53
|
+
"type": "object"
|
|
54
|
+
},
|
|
55
|
+
"target_candidates": {
|
|
56
|
+
"type": "array"
|
|
57
|
+
},
|
|
58
|
+
"evidence": {
|
|
59
|
+
"type": "object"
|
|
60
|
+
},
|
|
61
|
+
"recommended_change_set": {
|
|
62
|
+
"type": "object"
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|