@tryinget/pi-evalset-lab 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/LICENSE +78 -0
- package/README.md +141 -0
- package/examples/.gitkeep +0 -0
- package/examples/evalset-compare-sample-embedded.html +142 -0
- package/examples/evalset-compare-sample.png +0 -0
- package/examples/fixed-task-set-v2.json +127 -0
- package/examples/fixed-task-set-v3.json +126 -0
- package/examples/fixed-task-set.json +22 -0
- package/examples/system-baseline.txt +1 -0
- package/examples/system-candidate.txt +6 -0
- package/extensions/evalset.ts +1148 -0
- package/package.json +85 -0
- package/policy/security-policy.json +10 -0
- package/policy/stack-lane.json +10 -0
- package/prompts/implementation-planning.md +21 -0
- package/prompts/security-review.md +21 -0
- package/scripts/export-evalset-report-html.mjs +364 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "maintainer-clarity-v2",
|
|
3
|
+
"systemPrompt": "Answer in concise, plain English. Prefer direct wording over jargon.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{
|
|
6
|
+
"id": "fixed-task-set-definition",
|
|
7
|
+
"input": "In plain English, what does a fixed task set mean in eval workflows?",
|
|
8
|
+
"expectContains": ["same", "tasks"]
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"id": "fixed-task-set-benefit",
|
|
12
|
+
"input": "Why does using a fixed task set improve comparison quality across runs?",
|
|
13
|
+
"expectContains": ["compare", "runs"]
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"id": "reproducibility-two-factors",
|
|
17
|
+
"input": "Name two things you should lock or record for reproducible eval runs.",
|
|
18
|
+
"expectContains": ["model", "dataset"]
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "pass-rate-calculation",
|
|
22
|
+
"input": "If 18 of 24 scored cases pass, what pass rate should be reported?",
|
|
23
|
+
"expectContains": ["75"]
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"id": "scored-case-definition",
|
|
27
|
+
"input": "What makes a case \"scored\" in this evalset format?",
|
|
28
|
+
"expectContains": ["checks", "scored"]
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"id": "variant-hash-purpose",
|
|
32
|
+
"input": "What does variantHash help you verify?",
|
|
33
|
+
"expectContains": ["variant", "hash"]
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"id": "dataset-hash-purpose",
|
|
37
|
+
"input": "Why is datasetHash useful when comparing two reports?",
|
|
38
|
+
"expectContains": ["dataset", "same"]
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "delta-interpretation-speed-cost",
|
|
42
|
+
"input": "Candidate delta avg latency is -800ms and delta total cost is +0.0003. Summarize the tradeoff.",
|
|
43
|
+
"expectContains": ["faster", "cost"]
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"id": "command-non-interactive-pattern",
|
|
47
|
+
"input": "Show the correct non-interactive pattern to run compare with a local extension file.",
|
|
48
|
+
"expectContains": ["pi -e", "-p", "/evalset compare"]
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"id": "slash-command-shell",
|
|
52
|
+
"input": "Can you run /evalset directly in bash without pi? Answer yes or no and one reason.",
|
|
53
|
+
"expectContains": ["no", "pi"]
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"id": "report-default-location",
|
|
57
|
+
"input": "If --out is omitted, where are evalset reports written by default?",
|
|
58
|
+
"expectContains": [".evalset/reports"]
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"id": "model-prerequisite",
|
|
62
|
+
"input": "What should you do before /evalset run if no active model is selected?",
|
|
63
|
+
"expectContains": ["/model"]
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"id": "max-cases-behavior",
|
|
67
|
+
"input": "What does --max-cases 5 do during run/compare?",
|
|
68
|
+
"expectContains": ["first", "5"]
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "temperature-range",
|
|
72
|
+
"input": "What is the accepted numeric range for --temperature in this extension?",
|
|
73
|
+
"expectContains": ["0", "2"]
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"id": "system-merge-behavior",
|
|
77
|
+
"input": "If dataset.systemPrompt exists and --system-file is provided, are prompts merged or replaced?",
|
|
78
|
+
"expectContains": ["merge"]
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"id": "mutually-exclusive-system-options",
|
|
82
|
+
"input": "Can --system-file and --system-text be used together?",
|
|
83
|
+
"expectContains": ["no"]
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"id": "run-identity-fields",
|
|
87
|
+
"input": "Name any three run identity fields in a run report.",
|
|
88
|
+
"expectContains": ["runid", "datasethash", "caseshash"]
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"id": "compare-identity-fields",
|
|
92
|
+
"input": "Name the two run-id fields that link baseline and candidate inside compare.run.",
|
|
93
|
+
"expectContains": ["baselinerunid", "candidaterunid"]
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"id": "delta-passrate-zero",
|
|
97
|
+
"input": "If delta pass rate is 0, what does that imply?",
|
|
98
|
+
"expectContains": ["same", "pass rate"]
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"id": "keyword-check-limitation",
|
|
102
|
+
"input": "Why can simple keyword checks be misleading for quality?",
|
|
103
|
+
"expectContains": ["wording", "false"]
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"id": "improve-weak-evalset",
|
|
107
|
+
"input": "Give two concrete ways to improve a weak 3-case evalset.",
|
|
108
|
+
"expectContains": ["more", "cases", "criteria"]
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"id": "no-overclaim-rollout",
|
|
112
|
+
"input": "Should this be pitched as a huge replacement right away? Answer yes or no and one short reason.",
|
|
113
|
+
"expectContains": ["no"],
|
|
114
|
+
"expectNotContains": ["huge replacement"]
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"id": "stakeholder-brief",
|
|
118
|
+
"input": "Write a one-line stakeholder brief that includes pilot scope and measurement.",
|
|
119
|
+
"expectContains": ["pilot", "measure"]
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
"id": "tie-communication",
|
|
123
|
+
"input": "Baseline and candidate both scored 33.3% pass rate. How should that be communicated?",
|
|
124
|
+
"expectContains": ["same", "pass rate"]
|
|
125
|
+
}
|
|
126
|
+
]
|
|
127
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "maintainer-clarity-v3",
|
|
3
|
+
"systemPrompt": "Answer in concise, plain English. Prefer direct wording over jargon.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{
|
|
6
|
+
"id": "fixed-task-set-definition",
|
|
7
|
+
"input": "In plain English, what does a fixed task set mean in eval workflows?",
|
|
8
|
+
"expectContains": ["same", "tasks"]
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"id": "fixed-task-set-benefit",
|
|
12
|
+
"input": "Why does a fixed task set improve comparison quality across runs?",
|
|
13
|
+
"expectContains": ["compar"]
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"id": "reproducibility-two-factors",
|
|
17
|
+
"input": "Name two things you should lock or record for reproducible eval runs.",
|
|
18
|
+
"expectContains": ["model", "dataset"]
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "pass-rate-calculation",
|
|
22
|
+
"input": "If 18 of 24 scored cases pass, what pass rate should be reported?",
|
|
23
|
+
"expectContains": ["75"]
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"id": "scored-case-definition",
|
|
27
|
+
"input": "What makes a case scored in this evalset format?",
|
|
28
|
+
"expectRegex": "([Ee]xpect|[Cc]heck|[Cc]riteria|[Rr]ule)"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"id": "variant-hash-purpose",
|
|
32
|
+
"input": "What does variantHash help you verify?",
|
|
33
|
+
"expectContains": ["variant", "hash"]
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"id": "dataset-hash-purpose",
|
|
37
|
+
"input": "Why is datasetHash useful when comparing two reports?",
|
|
38
|
+
"expectContains": ["dataset", "hash"]
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "delta-interpretation-speed-cost",
|
|
42
|
+
"input": "Candidate delta avg latency is -800ms and delta total cost is +0.0003. Summarize the tradeoff.",
|
|
43
|
+
"expectContains": ["faster"]
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"id": "command-non-interactive-pattern",
|
|
47
|
+
"input": "Show the correct non-interactive pattern to run compare from shell.",
|
|
48
|
+
"expectContains": ["pi", "-p", "evalset"]
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"id": "slash-command-shell",
|
|
52
|
+
"input": "Can you run /evalset directly in bash without pi? Answer yes or no and one reason.",
|
|
53
|
+
"expectContains": ["no", "slash"]
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"id": "report-default-location",
|
|
57
|
+
"input": "If --out is omitted, where are evalset reports written by default?",
|
|
58
|
+
"expectContains": ["evalset", "report"]
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"id": "model-prerequisite",
|
|
62
|
+
"input": "What should you do before /evalset run if no active model is selected?",
|
|
63
|
+
"expectContains": ["/model"]
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"id": "max-cases-behavior",
|
|
67
|
+
"input": "What does --max-cases 5 do during run/compare?",
|
|
68
|
+
"expectRegex": "([Ff]irst|[Ll]imit|5.*case|case.*5)"
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "temperature-range",
|
|
72
|
+
"input": "What is the accepted numeric range for --temperature in this extension?",
|
|
73
|
+
"expectContains": ["0", "2"]
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"id": "system-merge-behavior",
|
|
77
|
+
"input": "If dataset.systemPrompt exists and --system-file is provided, are prompts merged or replaced?",
|
|
78
|
+
"expectContains": ["merge"]
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"id": "mutually-exclusive-system-options",
|
|
82
|
+
"input": "Can --system-file and --system-text be used together?",
|
|
83
|
+
"expectRegex": "([Nn]o|[Ee]ither|[Oo]ne|not together|[Bb]oth)"
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"id": "run-identity-fields",
|
|
87
|
+
"input": "Name any three run identity fields in a run report.",
|
|
88
|
+
"expectContains": ["run", "dataset", "case"]
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"id": "compare-identity-fields",
|
|
92
|
+
"input": "Name the two run-id fields that link baseline and candidate inside compare.run.",
|
|
93
|
+
"expectContains": ["baseline", "candidate", "run"]
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"id": "delta-passrate-zero",
|
|
97
|
+
"input": "If delta pass rate is 0, what does that imply?",
|
|
98
|
+
"expectRegex": "([Nn]o change|same pass rate|[Uu]nchanged)"
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"id": "keyword-check-limitation",
|
|
102
|
+
"input": "Why can simple keyword checks be misleading for quality?",
|
|
103
|
+
"expectRegex": "([Kk]eyword).*(mislead|false|[Ee]rror|[Bb]rittle)|([Mm]islead|false|[Ee]rror|[Bb]rittle).*[Kk]eyword"
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"id": "improve-weak-evalset",
|
|
107
|
+
"input": "Give two concrete ways to improve a weak 3-case evalset.",
|
|
108
|
+
"expectContains": ["more", "cases"]
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"id": "no-overclaim-rollout",
|
|
112
|
+
"input": "Should this be pitched as a huge replacement right away? Answer yes or no and one short reason.",
|
|
113
|
+
"expectContains": ["no", "phase"]
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"id": "stakeholder-brief",
|
|
117
|
+
"input": "Write a one-line stakeholder brief that includes pilot scope and measurement.",
|
|
118
|
+
"expectContains": ["pilot", "measure"]
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"id": "tie-communication",
|
|
122
|
+
"input": "Baseline and candidate both scored 33.3% pass rate. How should that be communicated?",
|
|
123
|
+
"expectRegex": "([Ss]ame|[Nn]o difference|[Uu]nchanged)"
|
|
124
|
+
}
|
|
125
|
+
]
|
|
126
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "maintainer-clarity-smoke",
|
|
3
|
+
"systemPrompt": "Answer concisely and explicitly. No fluff.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{
|
|
6
|
+
"id": "fixed-task-set-definition",
|
|
7
|
+
"input": "In one sentence, define what a fixed task set means for eval workflows.",
|
|
8
|
+
"expectContains": ["same tasks"]
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"id": "extension-gaps",
|
|
12
|
+
"input": "Name two extension-level gaps that can affect reproducible eval workflows in pi.",
|
|
13
|
+
"expectContains": ["trace", "reproducibility"]
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"id": "do-not-overclaim",
|
|
17
|
+
"input": "Should this be pitched as a huge replacement right away? Answer yes or no and one short reason.",
|
|
18
|
+
"expectContains": ["no"],
|
|
19
|
+
"expectNotContains": ["huge replacement"]
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
You are a concise technical assistant. Keep answers short and avoid jargon.
|