@niwa-labs/flow-eval 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Niwa Labs contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,27 @@
1
- # Temporary Holding Version
1
+ # @niwa-labs/flow-eval
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Workflow helpers for evaluation: blind candidate sets, rubric grading, evaluation suites and transcript chain checks.
4
+
5
+ ```ts
6
+ import { blindSet, gradeCase } from "@niwa-labs/flow-eval";
7
+ ```
8
+
9
+ Built on `@niwa-labs/flow` alone; an evaluation of work in a repository imports `@niwa-labs/flow-code`
10
+ itself.
11
+
12
+ - `blindSet` labels candidates A, B, C in an order drawn with the script's recorded `random()`
13
+ (the names are put in a fixed order first, so the same candidates in another order get the same
14
+ labels on a replay), and reveals the name behind a label.
15
+ - `gradeCase` scores a subject against a rubric (0 to 1 per item) by graders of another vendor than
16
+ the candidate's (a seat's `vendor`, else its harness; a `candidateVendor` of null says the subject
17
+ is not the work of a model). A grader answers with a score for every item or not at all. The rule
18
+ counts every independent grader, so the verdict is `INCONCLUSIVE` when graders that gave no answer
19
+ could still change it, and when no grader is independent.
20
+ - `evalSuite` runs every case on every variant, grades each output and recommends the best variant.
21
+ `vendorOf` is required: it names the vendor of each variant, or null. A cell that has no score
22
+ (its run failed, listed in `failures`, or no independent grader answered) is null in the table, and
23
+ a `perCase` or `recommend` that would rest on it is null, as it is on a tie.
24
+ - `chainFromTranscript` reads a session with `history.read` and says which expected files, skills or
25
+ tools it opened, in the order it called them: a text matches exactly the target, a path or the
26
+ name of a tool call, a read counts only with a result that did not fail, and a message that names
27
+ a file opens nothing. An unreadable transcript throws.
package/package.json CHANGED
@@ -1,6 +1,16 @@
1
1
  {
2
2
  "name": "@niwa-labs/flow-eval",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.0.0",
4
+ "license": "MIT",
5
+ "type": "module",
6
+ "exports": {
7
+ ".": "./src/index.ts"
8
+ },
9
+ "dependencies": {
10
+ "@niwa-labs/flow": "0.0.0"
11
+ },
12
+ "devDependencies": {
13
+ "@niwa-labs/testing": "0.0.0",
14
+ "zod": "^4.6.7"
15
+ }
16
+ }
package/src/index.ts ADDED
@@ -0,0 +1,245 @@
1
+ /**
2
+ * Workflow helpers for evaluation: blind candidate sets, rubric grading by graders independent of
3
+ * the candidate, suites of cases over variants, and checks of what a session opened. They run on
4
+ * the globals of a script, like the helpers of `@niwa-labs/flow`, and each opens a scope named by
5
+ * its key. An evaluation of work in a repository imports `@niwa-labs/flow-code` itself.
6
+ */
7
+
8
+ import { askSeat, blindLabel, checkKey, checkKeys, map, script, seatKeys, vendorOf } from "@niwa-labs/flow";
9
+ import type { Seat, Verdict } from "@niwa-labs/flow";
10
+
11
+ /** One line of a rubric: its id, and what it asks of the subject. */
12
+ export type RubricItem = { id: string; text: string };
13
+
14
+ /**
15
+ * Candidates under labels a judge cannot read anything into: A, B, C in an order drawn with the
16
+ * script's `random()`, which the run records. The names are put in a fixed order before the draws
17
+ * shuffle them, so a replay that collects the same candidates in another order labels them the
18
+ * same. `reveal` names the candidate behind a label.
19
+ */
20
+ export async function blindSet<C>(key: string, candidates: Record<string, C>): Promise<{ labeled: Record<string, C>; reveal: (label: string) => string }> {
21
+ checkKey(key, "the key of a blind set");
22
+ const names = Object.keys(candidates).sort();
23
+ const g = script();
24
+ const order = await g.scope(key, undefined, async () => {
25
+ const shuffled = [...names];
26
+ // Fisher-Yates over recorded draws.
27
+ for (let at = shuffled.length - 1; at > 0; at -= 1) {
28
+ const pick = Math.floor((await g.random()) * (at + 1));
29
+ [shuffled[at], shuffled[pick]] = [shuffled[pick] as string, shuffled[at] as string];
30
+ }
31
+ return shuffled;
32
+ });
33
+ const labels = order.map((name, index) => [blindLabel(index), name] as const);
34
+ const behind = new Map<string, string>(labels);
35
+ const labeled: Record<string, C> = Object.fromEntries(labels.map(([label, name]) => [label, candidates[name] as C]));
36
+ return {
37
+ labeled,
38
+ reveal: (label) => {
39
+ const name = behind.get(label);
40
+ if (name === undefined) throw new Error(`blindSet "${key}": "${label}" is not a label of the set`);
41
+ return name;
42
+ },
43
+ };
44
+ }
45
+
46
+ /**
47
+ * Whether a rubric item passes on the scores the graders gave it: a score of 0.5 or more is a pass,
48
+ * and the item passes by the rule (more than half of the graders, or all of them).
49
+ */
50
+ export function itemPasses(scores: readonly number[], rule: "majority" | "unanimous"): boolean {
51
+ const passing = scores.filter((score) => score >= 0.5).length;
52
+ return scores.length > 0 && (rule === "unanimous" ? passing === scores.length : passing * 2 > scores.length);
53
+ }
54
+
55
+ /** The ids of a rubric, each a key and none given twice: checked before anything runs. */
56
+ function checkRubric(rubric: readonly RubricItem[]): string[] {
57
+ return checkKeys(
58
+ rubric.map((item) => item.id),
59
+ "the id of a rubric item",
60
+ );
61
+ }
62
+
63
+ /**
64
+ * Grade a subject against a rubric, each item from 0 to 1, by graders of another vendor than the
65
+ * candidate's: a grader of the candidate's vendor does not grade. A grader answers with a score for
66
+ * every item or not at all. The rule (majority by default) is applied to all the independent graders:
67
+ * the verdict is a pass or a fail when the graders that did not answer could not change it, and
68
+ * `INCONCLUSIVE` otherwise, with no independent grader, or with no answer. A `candidateVendor` of
69
+ * null says the subject is not the work of a model, so every grader is independent of it.
70
+ */
71
+ export async function gradeCase<S>(
72
+ key: string,
73
+ opts: { subject: S; rubric: readonly RubricItem[]; graders: readonly Seat[]; candidateVendor: string | null; rule?: "majority" | "unanimous" },
74
+ ): Promise<{ scores: Record<string, number>; legs: { grader: Seat; scores: Record<string, number> }[]; verdict: Verdict }> {
75
+ checkKey(key, "the key of a grade");
76
+ const ids = checkRubric(opts.rubric);
77
+ const graders = opts.candidateVendor === null ? [...opts.graders] : opts.graders.filter((seat) => vendorOf(seat) !== opts.candidateVendor);
78
+ if (graders.length === 0) return { scores: {}, legs: [], verdict: "INCONCLUSIVE" };
79
+ const keys = seatKeys(graders, "the key of a grader");
80
+ const { z } = script();
81
+ const sheet = z.object({
82
+ scores: z
83
+ .array(z.object({ id: z.string(), score: z.number().min(0).max(1) }))
84
+ .refine((given) => given.length === ids.length && ids.every((id) => given.some((one) => one.id === id)), "a score for every item of the rubric, once each, by its id"),
85
+ });
86
+ const subject = typeof opts.subject === "string" ? opts.subject : JSON.stringify(opts.subject, null, 2);
87
+ const prompt = [
88
+ "Grade the subject below against each item of the rubric, from 0 (not at all) to 1 (fully). Answer with a score for every item, once each, as a list of { id, score }.",
89
+ "",
90
+ "Rubric:",
91
+ ...opts.rubric.map((item) => `- ${item.id}: ${item.text}`),
92
+ "",
93
+ "Subject:",
94
+ subject,
95
+ ].join("\n");
96
+ const answers = await map(key, graders.map((grader, index) => ({ grader, key: keys[index] as string })), { key: (one) => one.key }, (one) => askSeat(one.grader, prompt, sheet));
97
+ const legs = answers.flatMap((answer, index) =>
98
+ answer.ok
99
+ ? [{ grader: graders[index] as Seat, scores: Object.fromEntries(ids.map((id) => [id, (answer.value.scores.find((one) => one.id === id) as { score: number }).score])) as Record<string, number> }]
100
+ : [],
101
+ );
102
+ if (legs.length === 0) return { scores: {}, legs, verdict: "INCONCLUSIVE" };
103
+ const rule = opts.rule ?? "majority";
104
+ const scores = Object.fromEntries(ids.map((id) => [id, legs.reduce((sum, leg) => sum + (leg.scores[id] as number), 0) / legs.length])) as Record<string, number>;
105
+ // The graders that gave no answer still count toward the threshold: the verdict stands only if
106
+ // it is the same whether each of them would have passed every item or failed every one.
107
+ const missing = graders.length - legs.length;
108
+ const passesWith = (fill: number): boolean => ids.every((id) => itemPasses([...legs.map((leg) => leg.scores[id] as number), ...Array<number>(missing).fill(fill)], rule));
109
+ const best = passesWith(1);
110
+ const worst = passesWith(0);
111
+ return { scores, legs, verdict: best && worst ? "PASS" : !best && !worst ? "FAIL" : "INCONCLUSIVE" };
112
+ }
113
+
114
+ /** Scores closer than this are the same score: sums of the same numbers in another order differ by more than none. */
115
+ const SAME_SCORE = 1e-9;
116
+
117
+ /** The mean of the scores of a grade, or null with none. */
118
+ const meanOf = (scores: readonly number[]): number | null => (scores.length === 0 ? null : scores.reduce((sum, score) => sum + score, 0) / scores.length);
119
+
120
+ /** The single best of named numbers, or null on a tie at the top (scores the same within rounding) or with none. */
121
+ export function bestOf(values: Record<string, number>): string | null {
122
+ const entries = Object.entries(values);
123
+ if (entries.length === 0) return null;
124
+ const top = Math.max(...entries.map(([, value]) => value));
125
+ const leaders = entries.filter(([, value]) => top - value <= SAME_SCORE);
126
+ return leaders.length === 1 ? (leaders[0] as [string, number])[0] : null;
127
+ }
128
+
129
+ /** The names of the numbers when every one is known, or null when any is missing: a comparison with a gap compares nothing. */
130
+ function wholeOf(values: readonly [string, number | null][]): Record<string, number> | null {
131
+ const known = values.flatMap(([name, value]) => (value === null ? [] : [[name, value] as const]));
132
+ return known.length === values.length ? Object.fromEntries(known) : null;
133
+ }
134
+
135
+ /**
136
+ * Run every case on every variant and grade each output against the rubric. `dir` is the cell's
137
+ * own path, `<variant>/<case>`, for a run that keeps what it makes apart. A variant whose output is
138
+ * graded is the candidate, so `vendorOf` names its vendor (graders of that vendor do not grade it),
139
+ * or null for a variant that is not the work of a model. `table[variant][case]` is the mean score,
140
+ * or null when the cell has none: its run failed (it is then in `failures`) or no independent grader
141
+ * answered. `perCase` is the best variant of each case and `recommend` the best variant overall;
142
+ * either is null on a tie, and when any score it rests on is missing.
143
+ */
144
+ export async function evalSuite<V, O>(
145
+ key: string,
146
+ opts: {
147
+ cases: readonly { id: string; prompt: string }[];
148
+ variants: Record<string, V>;
149
+ run: (variant: V, c: { id: string; prompt: string }, dir: string) => Promise<O>;
150
+ rubric: readonly RubricItem[];
151
+ graders: readonly Seat[];
152
+ vendorOf: (variant: string) => string | null;
153
+ concurrency?: number;
154
+ },
155
+ ): Promise<{
156
+ table: Record<string, Record<string, number | null>>;
157
+ perCase: Record<string, string | null>;
158
+ recommend: string | null;
159
+ failures: { variant: string; case: string; error: string }[];
160
+ }> {
161
+ checkKey(key, "the key of a suite");
162
+ const variants = checkKeys(Object.keys(opts.variants), "the name of a variant");
163
+ checkKeys(
164
+ opts.cases.map((one) => one.id),
165
+ "the id of a case",
166
+ );
167
+ checkRubric(opts.rubric);
168
+ // The length of the variant's name tells where it ends, so no two pairs share a key.
169
+ const cells = variants.flatMap((variant) => opts.cases.map((one) => ({ variant, case: one, key: `v${String(variant.length)}-${variant}-${one.id}` })));
170
+ const graded = await map(key, cells, { key: (cell) => cell.key, ...(opts.concurrency === undefined ? {} : { concurrency: opts.concurrency }) }, async (cell) => {
171
+ const output = await opts.run(opts.variants[cell.variant] as V, cell.case, `${cell.variant}/${cell.case.id}`);
172
+ const grade = await gradeCase("grade", {
173
+ subject: { case: cell.case.prompt, output },
174
+ rubric: opts.rubric,
175
+ graders: opts.graders,
176
+ candidateVendor: opts.vendorOf(cell.variant),
177
+ });
178
+ return grade.legs.length === 0 ? null : meanOf(Object.values(grade.scores));
179
+ });
180
+ const scoreAt = (index: number): number | null => {
181
+ const result = graded[index] as (typeof graded)[number];
182
+ return result.ok ? result.value : null;
183
+ };
184
+ const table = Object.fromEntries(
185
+ variants.map((variant) => [variant, Object.fromEntries(opts.cases.map((one) => [one.id, scoreAt(cells.findIndex((cell) => cell.variant === variant && cell.case.id === one.id))]))]),
186
+ ) as Record<string, Record<string, number | null>>;
187
+ const failures = graded.flatMap((result, index) => (result.ok ? [] : [{ variant: (cells[index] as (typeof cells)[number]).variant, case: (cells[index] as (typeof cells)[number]).case.id, error: result.error.message }]));
188
+ const at = (variant: string, id: string): number | null => (table[variant] as Record<string, number | null>)[id] as number | null;
189
+ const perCase = Object.fromEntries(
190
+ opts.cases.map((one) => {
191
+ const whole = wholeOf(variants.map((variant) => [variant, at(variant, one.id)]));
192
+ return [one.id, whole === null ? null : bestOf(whole)];
193
+ }),
194
+ );
195
+ const totals = wholeOf(
196
+ variants.map((variant): [string, number | null] => {
197
+ const scores = wholeOf(opts.cases.map((one) => [one.id, at(variant, one.id)]));
198
+ return [variant, scores === null ? null : meanOf(Object.values(scores))];
199
+ }),
200
+ );
201
+ return { table, perCase, recommend: totals === null ? null : bestOf(totals), failures };
202
+ }
203
+
204
+ /** A session as `history.read` names it: the harness it runs in and the id that harness gave it. */
205
+ export type SessionRef = { harness: string; native: string };
206
+
207
+ /** The fields of a record of a transcript that a check of what was opened reads. */
208
+ type TranscriptRecord = { kind?: string; callId?: string; name?: string; category?: string; target?: string; paths?: string[]; ok?: boolean };
209
+
210
+ /**
211
+ * Whether a session opened what it was expected to: each expected text (a file, a skill, a tool) is
212
+ * matched exactly against the target, the paths and the name of the tool calls in the transcript,
213
+ * read whole with `history.read`; what the person or the model wrote in a message opens nothing. A
214
+ * call that reads a file counts only with a result that did not fail. `opened` lists the ones found
215
+ * in the order the session first called them, `missing` the others. A transcript that could not be
216
+ * read throws: nothing read is not nothing opened.
217
+ */
218
+ export async function chainFromTranscript(key: string, session: SessionRef, expect: { opened: readonly string[] }): Promise<{ opened: string[]; missing: string[] }> {
219
+ checkKey(key, "the key of a transcript check");
220
+ const g = script();
221
+ const answer = (await g.scope(key, undefined, () => g.call("history.read", { session, mode: "full" }))) as {
222
+ records: TranscriptRecord[];
223
+ coverage: { status: string; reason?: string };
224
+ };
225
+ if (answer.coverage.status === "failed" || answer.coverage.status === "unavailable") {
226
+ throw new Error(`the session ${session.harness}:${session.native} could not be read (${answer.coverage.status}): ${answer.coverage.reason ?? "no reason given"}`);
227
+ }
228
+ const failed = new Set<string>();
229
+ const answered = new Set<string>();
230
+ for (const record of answer.records) {
231
+ if (record.kind !== "tool-result" || record.callId === undefined) continue;
232
+ answered.add(record.callId);
233
+ if (record.ok === false) failed.add(record.callId);
234
+ }
235
+ const firstAt = new Map<string, number>();
236
+ answer.records.forEach((record, index) => {
237
+ if (record.kind !== "tool-call") return;
238
+ const succeeded = record.category !== "read" || (record.callId !== undefined && answered.has(record.callId) && !failed.has(record.callId));
239
+ if (!succeeded) return;
240
+ const names = [record.name, record.target, ...(record.paths ?? [])];
241
+ for (const wanted of expect.opened) if (!firstAt.has(wanted) && names.includes(wanted)) firstAt.set(wanted, index);
242
+ });
243
+ const opened = [...firstAt].sort(([, a], [, b]) => a - b).map(([wanted]) => wanted);
244
+ return { opened, missing: expect.opened.filter((wanted) => !firstAt.has(wanted)) };
245
+ }
@@ -0,0 +1,397 @@
1
+ // The evaluation helpers in a real script: a blind set hides the names behind labels it can reveal;
2
+ // a grade is made only by graders independent of the candidate; a suite picks the best variant;
3
+ // a transcript check finds what a session opened, and refuses to answer for an unreadable one.
4
+
5
+ import { afterEach, expect, test } from "bun:test";
6
+ import { realpathSync } from "node:fs";
7
+ import { mkdir, symlink, writeFile } from "node:fs/promises";
8
+ import { join } from "node:path";
9
+
10
+ import { z } from "zod";
11
+
12
+ import { createNiwa, historyRead, plugin, requests, stubAction, tempHome, timers, workflow } from "@niwa-labs/testing";
13
+
14
+ import { bestOf, blindSet, itemPasses } from "../src/index.ts";
15
+
16
+ type Reply = (call: { prompt: string }) => unknown;
17
+
18
+ /** The answer of a judge: a score for each item, by its id. */
19
+ const sheet = (scores: Record<string, number>) => ({ scores: Object.entries(scores).map(([id, score]) => ({ id, score })) });
20
+
21
+ const cleanups: Array<() => Promise<void>> = [];
22
+ afterEach(async () => {
23
+ for (const cleanup of cleanups.splice(0)) await cleanup();
24
+ }, 30_000);
25
+
26
+ const readCall = (callId: string, target: string) => ({ kind: "tool-call", callId, name: "Read", category: "read", target });
27
+ const readResult = (callId: string, ok?: boolean) => ({ kind: "tool-result", callId, ...(ok === undefined ? {} : { ok }) });
28
+
29
+ const PACKAGES = join(import.meta.dir, "..", "..");
30
+
31
+ /** Run a script that imports the evaluation helpers, with stub agents and a stub `history.read`, and answer its result. */
32
+ async function run(source: string, agents: Record<string, Reply>, transcript: unknown = { records: [], coverage: { status: "complete" } }): Promise<{ result: unknown; prompts: Record<string, string[]> }> {
33
+ const temp = await tempHome();
34
+ const home = realpathSync(temp.path);
35
+ const prompts: Record<string, string[]> = {};
36
+ const actions = Object.entries(agents).map(([harness, reply]) => {
37
+ const seen: string[] = [];
38
+ prompts[harness] = seen;
39
+ return [
40
+ harness,
41
+ stubAction(`session.start.${harness}`, {
42
+ home,
43
+ input: z.looseObject({ prompt: z.string() }),
44
+ output: z.any(),
45
+ recovery: "repeat",
46
+ result: (input) => {
47
+ seen.push(input.prompt);
48
+ return reply(input);
49
+ },
50
+ }),
51
+ ] as const;
52
+ });
53
+ // The stub takes what the real action takes: a script that names a session another way fails here as it would there.
54
+ const history = stubAction("history.read", { home, input: historyRead.input, output: z.any(), recovery: "repeat", effect: "read", result: () => transcript });
55
+ const host = createNiwa({ home });
56
+ host.use(
57
+ requests(),
58
+ timers([]),
59
+ plugin("stubs", (api) => {
60
+ for (const [, one] of actions) api.action(one);
61
+ api.action(history);
62
+ }),
63
+ workflow({ agents: Object.fromEntries(actions) }),
64
+ );
65
+ await host.serve();
66
+ cleanups.push(async () => {
67
+ await host.close().catch(() => {});
68
+ await temp.cleanup();
69
+ });
70
+ const src = join(home, "src");
71
+ await mkdir(join(src, "node_modules", "@niwa-labs"), { recursive: true });
72
+ for (const name of ["flow", "flow-eval"]) await symlink(realpathSync(join(PACKAGES, name)), join(src, "node_modules", "@niwa-labs", name), "dir");
73
+ await writeFile(join(src, "eval.ts"), source);
74
+ const ended = (await host.call("workflow.start" as never, { file: join(src, "eval.ts"), wait: true, timeoutMs: 30_000 } as never)) as { state: string; outcome?: { status: string; result?: unknown } };
75
+ if (ended.outcome?.status !== "done") throw new Error(`the run did not finish: ${JSON.stringify(ended).slice(0, 1500)}`);
76
+ return { result: ended.outcome.result, prompts };
77
+ }
78
+
79
+ test("a blind set labels every candidate and reveals the name behind each label", async () => {
80
+ const { result } = await run(
81
+ `
82
+ import { blindSet } from "@niwa-labs/flow-eval";
83
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
84
+ const set = await blindSet("blind", { alpha: "a", beta: "b", gamma: "g" });
85
+ return Object.keys(set.labeled).map((label) => ({ label, value: set.labeled[label], name: set.reveal(label) }));
86
+ });`,
87
+ {},
88
+ );
89
+ const rows = result as Array<{ label: string; value: string; name: string }>;
90
+ expect(rows.map((row) => row.label)).toEqual(["A", "B", "C"]);
91
+ expect(rows.map((row) => row.name).sort()).toEqual(["alpha", "beta", "gamma"]);
92
+ for (const row of rows) expect(row.value).toBe(row.name[0]);
93
+ }, 30_000);
94
+
95
+ test("a grade is made only by graders of another vendor, and passes an item by the rule", async () => {
96
+ const { result, prompts } = await run(
97
+ `
98
+ import { gradeCase } from "@niwa-labs/flow-eval";
99
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
100
+ const rubric = [{ id: "correct", text: "the answer is right" }, { id: "short", text: "the answer is short" }];
101
+ const graded = await gradeCase("grade", {
102
+ subject: "42",
103
+ rubric,
104
+ graders: [{ harness: "own", vendor: "acme" }, { harness: "g1", vendor: "v1" }, { harness: "g2", vendor: "v2" }, { harness: "g3", vendor: "v3" }],
105
+ candidateVendor: "acme",
106
+ });
107
+ const alone = await gradeCase("alone", { subject: "42", rubric, graders: [{ harness: "own", vendor: "acme" }], candidateVendor: "acme" });
108
+ return { verdict: graded.verdict, scores: graded.scores, legs: graded.legs.length, alone: alone.verdict };
109
+ });`,
110
+ {
111
+ own: () => sheet({ correct: 1, short: 1 }),
112
+ g1: () => sheet({ correct: 1, short: 0 }),
113
+ g2: () => sheet({ correct: 1, short: 1 }),
114
+ g3: () => sheet({ correct: 0.5, short: 0 }),
115
+ },
116
+ );
117
+ expect(result).toEqual({ verdict: "FAIL", scores: { correct: 2.5 / 3, short: 1 / 3 }, legs: 3, alone: "INCONCLUSIVE" });
118
+ expect(prompts.own).toHaveLength(0);
119
+ }, 30_000);
120
+
121
+ test("a suite runs every case on every variant and recommends the best variant", async () => {
122
+ const { result } = await run(
123
+ `
124
+ import { evalSuite } from "@niwa-labs/flow-eval";
125
+ export default workflow({ output: z.json() }, async (): Promise<any> =>
126
+ evalSuite("suite", {
127
+ cases: [{ id: "c1", prompt: "one" }, { id: "c2", prompt: "two" }],
128
+ variants: { terse: "terse", wordy: "wordy" },
129
+ run: async (variant, c, dir) => variant + ":" + c.prompt + "@" + dir,
130
+ rubric: [{ id: "good", text: "it is good" }],
131
+ graders: ["grader"],
132
+ vendorOf: () => "acme",
133
+ }));`,
134
+ { grader: (call) => sheet({ good: call.prompt.includes("terse:two") ? 0.1 : call.prompt.includes("terse") ? 1 : 0.6 }) },
135
+ );
136
+ expect(result).toEqual({ table: { terse: { c1: 1, c2: 0.1 }, wordy: { c1: 0.6, c2: 0.6 } }, perCase: { c1: "terse", c2: "wordy" }, recommend: "wordy", failures: [] });
137
+ }, 30_000);
138
+
139
+ test("a transcript check finds what the session opened, in the order it met them", async () => {
140
+ const records = [
141
+ { kind: "user", text: "fix it" },
142
+ readCall("c1", "src/b.ts"),
143
+ readResult("c1"),
144
+ readCall("c2", "src/a.ts"),
145
+ readResult("c2"),
146
+ ];
147
+ const { result } = await run(
148
+ `
149
+ import { chainFromTranscript } from "@niwa-labs/flow-eval";
150
+ export default workflow({ output: z.json() }, async (): Promise<any> => chainFromTranscript("chain", { harness: "claude", native: "s-1" }, { opened: ["src/a.ts", "src/b.ts", "docs/plan.md"] }));`,
151
+ {},
152
+ { records, coverage: { status: "complete" } },
153
+ );
154
+ expect(result).toEqual({ opened: ["src/b.ts", "src/a.ts"], missing: ["docs/plan.md"] });
155
+ }, 30_000);
156
+
157
+ test("an item passes on more than half of the graders, or all of them; a tie at the top has no best", () => {
158
+ expect(itemPasses([1, 0.6, 0], "majority")).toBe(true);
159
+ expect(itemPasses([1, 0], "majority")).toBe(false);
160
+ expect(itemPasses([1, 0.6, 0], "unanimous")).toBe(false);
161
+ expect(itemPasses([], "majority")).toBe(false);
162
+ expect(bestOf({ a: 1, b: 2 })).toBe("b");
163
+ expect(bestOf({ a: 2, b: 2 })).toBeNull();
164
+ });
165
+
166
+ test("a transcript that could not be read is an error, not a session that opened nothing", async () => {
167
+ const { result } = await run(
168
+ `
169
+ import { chainFromTranscript } from "@niwa-labs/flow-eval";
170
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
171
+ try {
172
+ return await chainFromTranscript("chain", { harness: "claude", native: "s-1" }, { opened: ["src/a.ts"] });
173
+ } catch (error) {
174
+ return (error as Error).message;
175
+ }
176
+ });`,
177
+ {},
178
+ { records: [], coverage: { status: "unavailable", reason: "no reading side" } },
179
+ );
180
+ expect(result).toBe("the session claude:s-1 could not be read (unavailable): no reading side");
181
+ }, 30_000);
182
+
183
+ /** Run a script that grades with three independent judges by the unanimous or the majority rule, and answer the verdict. */
184
+ async function verdictOf(rule: string, replies: Record<string, Reply>): Promise<{ verdict: string; scores: Record<string, number>; legs: number }> {
185
+ const { result } = await run(
186
+ `
187
+ import { gradeCase } from "@niwa-labs/flow-eval";
188
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
189
+ const graded = await gradeCase("grade", {
190
+ subject: "42",
191
+ rubric: [{ id: "correct", text: "right" }, { id: "short", text: "short" }],
192
+ graders: [{ harness: "g1", vendor: "v1" }, { harness: "g2", vendor: "v2" }, { harness: "g3", vendor: "v3" }],
193
+ candidateVendor: "acme",
194
+ rule: "${rule}",
195
+ });
196
+ return { verdict: graded.verdict, scores: graded.scores, legs: graded.legs.length };
197
+ });`,
198
+ replies,
199
+ );
200
+ return result as { verdict: string; scores: Record<string, number>; legs: number };
201
+ }
202
+
203
+ /** A judge whose answer does not fit the schema: its call fails and the grade goes on without it. */
204
+ const down: Reply = () => ({ nothing: true });
205
+
206
+ test("graders that gave no complete answer keep their place in the threshold", async () => {
207
+ // One judge scores both items, one skips an item and one fails: a single vote cannot pass the item by consensus.
208
+ const partial = await verdictOf("unanimous", { g1: () => sheet({ correct: 1, short: 1 }), g2: () => sheet({ correct: 1 }), g3: down });
209
+ expect(partial.verdict).toBe("INCONCLUSIVE");
210
+ expect(partial.legs).toBe(1);
211
+ // By majority one vote of three does not settle it either way.
212
+ expect((await verdictOf("majority", { g1: () => sheet({ correct: 1, short: 1 }), g2: down, g3: down })).verdict).toBe("INCONCLUSIVE");
213
+ // Two passes of three carry the majority whatever the third would say; two fails of three sink it.
214
+ expect((await verdictOf("majority", { g1: () => sheet({ correct: 1, short: 1 }), g2: () => sheet({ correct: 1, short: 1 }), g3: down })).verdict).toBe("PASS");
215
+ expect((await verdictOf("majority", { g1: () => sheet({ correct: 1, short: 0 }), g2: () => sheet({ correct: 1, short: 0 }), g3: down })).verdict).toBe("FAIL");
216
+ // A failing vote is a fail by unanimity whoever is missing.
217
+ expect((await verdictOf("unanimous", { g1: () => sheet({ correct: 1, short: 0 }), g2: down, g3: down })).verdict).toBe("FAIL");
218
+ // Every answer in: the verdict is the rule's.
219
+ expect((await verdictOf("unanimous", { g1: () => sheet({ correct: 1, short: 1 }), g2: () => sheet({ correct: 1, short: 1 }), g3: () => sheet({ correct: 1, short: 1 }) })).verdict).toBe("PASS");
220
+ }, 60_000);
221
+
222
+ const SUITE = (extra: string, cases = '[{ id: "c1", prompt: "one" }]', graders = '["grader"]', variants = '{ solo: "solo" }') => `
223
+ import { evalSuite } from "@niwa-labs/flow-eval";
224
+ export default workflow({ output: z.json() }, async (): Promise<any> =>
225
+ evalSuite("suite", {
226
+ cases: ${cases},
227
+ variants: ${variants},
228
+ run: async (variant, c) => variant + ":" + c.prompt,
229
+ rubric: [{ id: "good", text: "it is good" }],
230
+ graders: ${graders},
231
+ vendorOf: () => "acme",
232
+ ${extra}
233
+ }));`;
234
+
235
+ test("a suite with no independent grader scores nothing and recommends no variant", async () => {
236
+ const none = await run(SUITE("", undefined, "[]"), {});
237
+ expect(none.result).toEqual({ table: { solo: { c1: null } }, perCase: { c1: null }, recommend: null, failures: [] });
238
+ // The only grader is of the candidate's vendor.
239
+ const own = await run(SUITE("", undefined, '[{ harness: "grader", vendor: "acme" }]'), { grader: () => sheet({ good: 1 }) });
240
+ expect(own.result).toEqual({ table: { solo: { c1: null } }, perCase: { c1: null }, recommend: null, failures: [] });
241
+ expect(own.prompts.grader).toHaveLength(0);
242
+ // A variant that is not the work of a model is graded by any grader.
243
+ const free = await run(SUITE("").replace("vendorOf: () => \"acme\"", "vendorOf: () => null"), { grader: () => sheet({ good: 1 }) });
244
+ expect(free.result).toEqual({ table: { solo: { c1: 1 } }, perCase: { c1: "solo" }, recommend: "solo", failures: [] });
245
+ }, 60_000);
246
+
247
+ test("a suite with no case recommends no variant", async () => {
248
+ const { result } = await run(SUITE("", "[]"), { grader: () => sheet({ good: 1 }) });
249
+ expect(result).toEqual({ table: { solo: {} }, perCase: {}, recommend: null, failures: [] });
250
+ }, 30_000);
251
+
252
+ test("a cell whose run fails has no score and is listed, and no comparison rests on it", async () => {
253
+ const { result } = await run(
254
+ `
255
+ import { evalSuite } from "@niwa-labs/flow-eval";
256
+ export default workflow({ output: z.json() }, async (): Promise<any> =>
257
+ evalSuite("suite", {
258
+ cases: [{ id: "c1", prompt: "one" }],
259
+ variants: { fine: 1, broken: 2 },
260
+ run: async (variant) => { if (variant === 2) throw new Error("it crashed"); return "ok"; },
261
+ rubric: [{ id: "good", text: "it is good" }],
262
+ graders: ["grader"],
263
+ vendorOf: () => "acme",
264
+ }));`,
265
+ { grader: () => sheet({ good: 1 }) },
266
+ );
267
+ expect(result).toEqual({
268
+ table: { fine: { c1: 1 }, broken: { c1: null } },
269
+ perCase: { c1: null },
270
+ recommend: null,
271
+ failures: [{ variant: "broken", case: "c1", error: "it crashed" }],
272
+ });
273
+ }, 30_000);
274
+
275
+ test("variants and cases whose names join to the same text are separate cells", async () => {
276
+ const { result } = await run(
277
+ SUITE("", '[{ id: "c", prompt: "one" }, { id: "b-c", prompt: "two" }]', undefined, '{ "a-b": "x", a: "y" }'),
278
+ { grader: () => sheet({ good: 1 }) },
279
+ );
280
+ expect((result as { table: unknown }).table).toEqual({ "a-b": { c: 1, "b-c": 1 }, a: { c: 1, "b-c": 1 } });
281
+ }, 30_000);
282
+
283
+ test("a rubric given a duplicate id fails the suite before any run", async () => {
284
+ const { result } = await run(
285
+ `
286
+ import { evalSuite } from "@niwa-labs/flow-eval";
287
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
288
+ let runs = 0;
289
+ try {
290
+ await evalSuite("suite", {
291
+ cases: [{ id: "c1", prompt: "one" }],
292
+ variants: { solo: "solo" },
293
+ run: async () => { runs += 1; return "out"; },
294
+ rubric: [{ id: "good", text: "a" }, { id: "good", text: "b" }],
295
+ graders: ["grader"],
296
+ vendorOf: () => "acme",
297
+ });
298
+ } catch (error) {
299
+ return { runs, message: (error as Error).message };
300
+ }
301
+ return { runs, message: "no error" };
302
+ });`,
303
+ { grader: () => sheet({ good: 1 }) },
304
+ );
305
+ expect((result as { runs: number }).runs).toBe(0);
306
+ expect((result as { message: string }).message).toContain('"good" is given twice');
307
+ }, 30_000);
308
+
309
+ test("a case or an item named __proto__ keeps its score", async () => {
310
+ const { result } = await run(
311
+ `
312
+ import { evalSuite } from "@niwa-labs/flow-eval";
313
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
314
+ const suite = await evalSuite("suite", {
315
+ cases: [{ id: "__proto__", prompt: "one" }],
316
+ variants: { solo: "solo" },
317
+ run: async () => "out",
318
+ rubric: [{ id: "__proto__", text: "it is good" }],
319
+ graders: ["grader"],
320
+ vendorOf: () => "acme",
321
+ });
322
+ return { own: Object.hasOwn(suite.table.solo, "__proto__"), score: suite.table.solo["__proto__"] };
323
+ });`,
324
+ { grader: () => sheet({ ["__proto__"]: 0.75 }) },
325
+ );
326
+ expect(result).toEqual({ own: true, score: 0.75 });
327
+ }, 30_000);
328
+
329
+ test("scores that are the same up to the order of the sums are a tie", () => {
330
+ const sum = (values: number[]) => values.reduce((a, b) => a + b, 0) / values.length;
331
+ expect(sum([0.1, 0.2, 0.3])).not.toBe(sum([0.3, 0.2, 0.1]));
332
+ expect(bestOf({ a: sum([0.1, 0.2, 0.3]), b: sum([0.3, 0.2, 0.1]) })).toBeNull();
333
+ expect(bestOf({ a: 0.2, b: 0.2000001 })).toBe("b");
334
+ });
335
+
336
+ test("a blind set labels the same candidates the same whatever order they are given in", async () => {
337
+ const globals = globalThis as unknown as Record<string, unknown>;
338
+ const names = ["z", "a", "m", "k"];
339
+ const none = () => Promise.resolve(undefined);
340
+ const saved = Object.fromEntries(["agent", "call", "scope", "parallel", "stream", "wait", "sleep", "now", "random", "z", "ask", "read"].map((name) => [name, globals[name]]));
341
+ try {
342
+ for (const name of ["ask", "call", "parallel", "stream", "wait", "sleep", "now"]) globals[name] = none;
343
+ for (const name of ["z", "read"]) globals[name] = {};
344
+ globals.scope = (_name: string, _opts: unknown, fn: () => Promise<unknown>) => fn();
345
+ const labelling = async (order: string[]) => {
346
+ const draws = [0.9, 0.1, 0.5];
347
+ globals.random = () => Promise.resolve(draws.shift());
348
+ const set = await blindSet("blind", Object.fromEntries(order.map((name) => [name, name.toUpperCase()])));
349
+ expect(Object.values(set.labeled).sort()).toEqual(["A", "K", "M", "Z"].sort());
350
+ return Object.keys(set.labeled).map((label) => [label, set.reveal(label)]);
351
+ };
352
+ expect(await labelling([...names].reverse())).toEqual(await labelling(names));
353
+ } finally {
354
+ for (const [name, value] of Object.entries(saved)) {
355
+ if (value === undefined) delete globals[name];
356
+ else globals[name] = value;
357
+ }
358
+ }
359
+ });
360
+
361
+ test("a transcript check counts a call that opened the file, not a message that names it", async () => {
362
+ const records = [
363
+ { kind: "user", text: "open src/a.ts and src/b.ts" },
364
+ { kind: "assistant", text: "I will read src/a.ts" },
365
+ readCall("c1", "src/b.ts"),
366
+ readResult("c1"),
367
+ readCall("c2", "src/a.tsx"),
368
+ readResult("c2"),
369
+ ];
370
+ const { result } = await run(
371
+ `
372
+ import { chainFromTranscript } from "@niwa-labs/flow-eval";
373
+ export default workflow({ output: z.json() }, async (): Promise<any> => chainFromTranscript("chain", { harness: "claude", native: "s-1" }, { opened: ["src/a.ts", "src/b.ts"] }));`,
374
+ {},
375
+ { records, coverage: { status: "complete" } },
376
+ );
377
+ expect(result).toEqual({ opened: ["src/b.ts"], missing: ["src/a.ts"] });
378
+ }, 30_000);
379
+
380
+ test("a read that failed or has no result did not open the file; any other call opens what it names", async () => {
381
+ const records = [
382
+ readCall("c1", "src/failed.ts"),
383
+ readResult("c1", false),
384
+ readCall("c2", "src/pending.ts"),
385
+ { kind: "tool-call", callId: "c3", name: "Edit", category: "edit", target: "src/one.ts", paths: ["src/one.ts", "src/two.ts"] },
386
+ { kind: "tool-call", callId: "c4", name: "Skill", category: "other", target: "review" },
387
+ ];
388
+ const { result } = await run(
389
+ `
390
+ import { chainFromTranscript } from "@niwa-labs/flow-eval";
391
+ export default workflow({ output: z.json() }, async (): Promise<any> =>
392
+ chainFromTranscript("chain", { harness: "claude", native: "s-1" }, { opened: ["src/failed.ts", "src/pending.ts", "src/two.ts", "review", "Skill"] }));`,
393
+ {},
394
+ { records, coverage: { status: "complete" } },
395
+ );
396
+ expect(result).toEqual({ opened: ["src/two.ts", "review", "Skill"], missing: ["src/failed.ts", "src/pending.ts"] });
397
+ }, 30_000);
package/tsconfig.json ADDED
@@ -0,0 +1,6 @@
1
+ {
2
+ "extends": "../../tsconfig.base.json",
3
+ "include": [
4
+ "src"
5
+ ]
6
+ }