@agentpro/scroll 0.0.0-stage → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +55 -0
- package/LICENSE +21 -0
- package/README.md +293 -2
- package/SPEC.md +1079 -0
- package/bin/scroll.js +8 -0
- package/docs/prompt-caching.md +128 -0
- package/index.js +33 -0
- package/lib/audit.js +358 -0
- package/lib/auth.js +849 -0
- package/lib/chat-sdk/admin-controller.js +289 -0
- package/lib/chat-sdk/debouncer.js +94 -0
- package/lib/chat-sdk/group-grants.js +150 -0
- package/lib/chat-sdk/index.js +222 -0
- package/lib/chat-sdk/long-task-runner.js +413 -0
- package/lib/chat-session.js +1788 -0
- package/lib/chat-telemetry.js +557 -0
- package/lib/codex-app-server.js +851 -0
- package/lib/digest.js +112 -0
- package/lib/effects.js +157 -0
- package/lib/eval.js +408 -0
- package/lib/grounding.js +356 -0
- package/lib/installers.js +176 -0
- package/lib/knowledge.js +335 -0
- package/lib/ledger.js +38 -0
- package/lib/loop.js +197 -0
- package/lib/mcp-client.js +926 -0
- package/lib/memory-cli.js +66 -0
- package/lib/memory-tracker.js +1472 -0
- package/lib/memory.js +1279 -0
- package/lib/people-cli.js +81 -0
- package/lib/people.js +534 -0
- package/lib/permissions.js +272 -0
- package/lib/providers.js +1572 -0
- package/lib/runtime.js +928 -0
- package/lib/sandbox.js +744 -0
- package/lib/scheduler.js +402 -0
- package/lib/scroll.js +1225 -0
- package/lib/skill-cli.js +120 -0
- package/lib/skill-eval.js +156 -0
- package/lib/skills.js +860 -0
- package/lib/style.js +126 -0
- package/lib/tool-retriever.js +489 -0
- package/lib/tools.js +1833 -0
- package/lib/user.js +711 -0
- package/lib/vault.js +151 -0
- package/package.json +88 -4
- package/templates/agent/IDENTITY.md +56 -0
- package/templates/agent/SOUL.md +23 -0
- package/templates/agent/TOOLS.md +25 -0
- package/templates/agent/evals/case-02-citation-discipline.md +29 -0
- package/templates/agent/evals/case-03-boundary.md +27 -0
- package/templates/agent/evals/case-example.md +43 -0
- package/templates/agent/hard-rules.md +11 -0
- package/templates/agent/memory/lessons.md +5 -0
- package/templates/agent/memory/log/.gitkeep +0 -0
- package/templates/external-safety-contract.md +5 -0
- package/templates/long-task-planner.md +17 -0
- package/templates/work/LOOP.md +37 -0
- package/templates/work/WORK.md +50 -0
- package/types/chat.d.ts +61 -0
- package/types/index.d.ts +112 -0
package/lib/eval.js
ADDED
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
// SCROLL — eval engine (real). Runs an agent's gold cases N times, grades them
|
|
2
|
+
// deterministically (machine checks) + optional LLM-judge, measures consistency,
|
|
3
|
+
// and writes a hash-bound audit record. No DB, no engine — files + providers only.
|
|
4
|
+
//
|
|
5
|
+
// runEvals(name, { cwd, n, provider, model, judge }) -> report
|
|
6
|
+
//
|
|
7
|
+
// A gold case lives at agents/<name>/evals/<case>.md :
|
|
8
|
+
// ---
|
|
9
|
+
// id: case-01
|
|
10
|
+
// runs: 5 # consistency repeats (overridable by --n)
|
|
11
|
+
// pass_consistency: 0.8 # fraction of runs that must pass ALL machine checks
|
|
12
|
+
// ---
|
|
13
|
+
// ## Input ← the task/prompt given to the agent
|
|
14
|
+
// ...
|
|
15
|
+
// ```checks ← MACHINE truth (deterministic). YAML list of assertions.
|
|
16
|
+
// - { op: icontains, value: "solar" }
|
|
17
|
+
// - { op: cites, value: "S1" }
|
|
18
|
+
// - { op: not_cites, value: "S4" }
|
|
19
|
+
// ```
|
|
20
|
+
// ## Rubric ← optional, scored 1–5 by an LLM judge when --judge is set
|
|
21
|
+
// - Faithfulness: ...
|
|
22
|
+
import fs from 'node:fs';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
import crypto from 'node:crypto';
|
|
25
|
+
import { loadAgent, parseFrontmatter, parseYaml } from './scroll.js';
|
|
26
|
+
import { pickProvider } from './providers.js';
|
|
27
|
+
|
|
28
|
+
const sha = (s) => crypto.createHash('sha256').update(String(s)).digest('hex').slice(0, 16);
|
|
29
|
+
|
|
30
|
+
// ── deterministic check ops (the machine truth a node/agent must satisfy) ──
|
|
31
|
+
const norm = (s) => String(s == null ? '' : s);
|
|
32
|
+
const CHECK_OPS = {
|
|
33
|
+
contains: (t, v) => t.includes(v),
|
|
34
|
+
not_contains: (t, v) => !t.includes(v),
|
|
35
|
+
icontains: (t, v) => t.toLowerCase().includes(String(v).toLowerCase()),
|
|
36
|
+
not_icontains: (t, v) => !t.toLowerCase().includes(String(v).toLowerCase()),
|
|
37
|
+
regex: (t, v, f) => new RegExp(v, f || '').test(t),
|
|
38
|
+
not_regex: (t, v, f) => !new RegExp(v, f || '').test(t),
|
|
39
|
+
cites: (t, v) => new RegExp(`\\b${escapeRe(v)}\\b`).test(t),
|
|
40
|
+
not_cites: (t, v) => !new RegExp(`\\b${escapeRe(v)}\\b`).test(t),
|
|
41
|
+
min_words: (t, v) => (t.trim().split(/\s+/).filter(Boolean).length) >= Number(v),
|
|
42
|
+
max_words: (t, v) => (t.trim().split(/\s+/).filter(Boolean).length) <= Number(v),
|
|
43
|
+
equals: (t, v) => t.trim() === String(v).trim(),
|
|
44
|
+
};
|
|
45
|
+
function escapeRe(s) { return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); }
|
|
46
|
+
|
|
47
|
+
export function runChecks(text, checks) {
|
|
48
|
+
const t = norm(text);
|
|
49
|
+
const nonEmpty = t.length > 0;
|
|
50
|
+
const results = (checks || []).map((c) => {
|
|
51
|
+
const fn = CHECK_OPS[c.op];
|
|
52
|
+
if (!fn) return { ...c, ok: false, err: `unknown op "${c.op}"` };
|
|
53
|
+
// An empty output MUST NOT trivially pass purely negative assertions (e.g. `not_contains`, `not_regex`)
|
|
54
|
+
if (!nonEmpty && (c.op === 'not_contains' || c.op === 'not_regex')) {
|
|
55
|
+
return { op: c.op, value: c.value, ok: false, err: 'empty_output_fails_negative_check' };
|
|
56
|
+
}
|
|
57
|
+
let ok = false, err = null;
|
|
58
|
+
try { ok = !!fn(t, c.value, c.flags); } catch (e) { err = e.message; }
|
|
59
|
+
return { op: c.op, value: c.value, ok, err };
|
|
60
|
+
});
|
|
61
|
+
const passed = results.filter((r) => r.ok).length;
|
|
62
|
+
return { passed, total: results.length, allPass: results.length > 0 && passed === results.length, results };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// ── §30 fixture provenance ──
|
|
66
|
+
// §8 grades the end state rather than the execution path. It says nothing about where the
|
|
67
|
+
// PRECONDITION came from — and that silence lets a green suite prove nothing.
|
|
68
|
+
//
|
|
69
|
+
// If a case's precondition was hand-built, the case says nothing about the path that was supposed
|
|
70
|
+
// to build it. The classic shape: a scheduler's EXECUTE path is exercised against a schedule the
|
|
71
|
+
// engineer wrote by hand, so the ARM path is never run once. Every night the job fires, the suite
|
|
72
|
+
// stays green, and arming has been broken since the day it shipped.
|
|
73
|
+
export const PROVENANCE = ['product-path', 'recorded', 'synthetic'];
|
|
74
|
+
|
|
75
|
+
export function parseFixture(f = {}) {
|
|
76
|
+
const fx = f.fixture || {};
|
|
77
|
+
const provenance = typeof fx.provenance === 'string' ? fx.provenance.trim() : null;
|
|
78
|
+
const issues = [];
|
|
79
|
+
if (!provenance) {
|
|
80
|
+
issues.push('fixture.provenance not declared (product-path | recorded | synthetic)');
|
|
81
|
+
} else if (!PROVENANCE.includes(provenance)) {
|
|
82
|
+
issues.push(`fixture.provenance "${provenance}" is not one of ${PROVENANCE.join(' | ')}`);
|
|
83
|
+
} else if (provenance === 'product-path' && !fx.setup) {
|
|
84
|
+
issues.push('provenance=product-path requires `setup:` (the path that creates the precondition)');
|
|
85
|
+
} else if (provenance === 'synthetic' && !fx.justification) {
|
|
86
|
+
issues.push('provenance=synthetic requires `justification:` — and never counts as evidence for the path that would otherwise create this precondition');
|
|
87
|
+
}
|
|
88
|
+
return { provenance, setup: fx.setup || null, justification: fx.justification || null, issues };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// ── parse a gold case .md → { id, runs, threshold, input, checks, rubric, fixture } ──
|
|
92
|
+
export function parseGoldCase(raw, file) {
|
|
93
|
+
const { frontmatter: fm, body } = parseFrontmatter(raw);
|
|
94
|
+
const f = fm || {};
|
|
95
|
+
const sect = (name) => {
|
|
96
|
+
// NB: no 'm' flag — with 'm', `$` matches end-of-LINE and truncates the section to its first
|
|
97
|
+
// line. We anchor the heading with (?:^|\n) and let `$` mean end-of-string instead.
|
|
98
|
+
const re = new RegExp(`(?:^|\\n)##\\s+${name}[^\\n]*\\n([\\s\\S]*?)(?=\\n##\\s|$)`, 'i');
|
|
99
|
+
const m = body.match(re); return m ? m[1].trim() : '';
|
|
100
|
+
};
|
|
101
|
+
// ```checks fenced YAML list (the machine truth)
|
|
102
|
+
let checks = [];
|
|
103
|
+
const cm = body.match(/```checks\n([\s\S]*?)```/);
|
|
104
|
+
if (cm) {
|
|
105
|
+
const parsed = parseYaml(cm[1]);
|
|
106
|
+
if (Array.isArray(parsed)) checks = parsed.filter((x) => x && x.op);
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
id: f.id || (file ? path.basename(file, '.md') : 'case'),
|
|
110
|
+
runs: Number(f.runs || 3),
|
|
111
|
+
threshold: Number(f.pass_consistency != null ? f.pass_consistency : 0.66),
|
|
112
|
+
judgeMin: Number(f.judge_min != null ? f.judge_min : 4.0),
|
|
113
|
+
input: sect('Input') || body.split('```checks')[0].trim(),
|
|
114
|
+
checks,
|
|
115
|
+
rubric: sect('Rubric'),
|
|
116
|
+
fixture: parseFixture(f), // §30
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// ── LLM-judge (qualitative, 1–5 per rubric criterion) ──
|
|
121
|
+
// Language follows IDENTITY.language, same contract as lib/digest.js (§4). Default: English.
|
|
122
|
+
// (Before: this prompt was hard-coded in one natural language, so every consumer of the
|
|
123
|
+
// framework got a judge speaking it regardless of their agent's declared language.)
|
|
124
|
+
export const JUDGE_SENTINEL = 'SCROLL_JUDGE_JSON';
|
|
125
|
+
const JUDGE_L = {
|
|
126
|
+
en: {
|
|
127
|
+
head: (s) => `You are a strict grader (${s}). Score the OUTPUT against each criterion in the RUBRIC on a 1–5 scale.`,
|
|
128
|
+
only: 'Return JSON ONLY, no prose: {"scores": {"<criterion>": <1-5>}, "average": <number>}',
|
|
129
|
+
rubric: '## RUBRIC', output: '## OUTPUT', none: '(none)', empty: '(empty)',
|
|
130
|
+
},
|
|
131
|
+
vi: {
|
|
132
|
+
head: (s) => `Bạn là giám khảo nghiêm khắc (${s}). Chấm OUTPUT theo từng tiêu chí trong RUBRIC, thang 1–5.`,
|
|
133
|
+
only: 'CHỈ trả JSON, không giải thích: {"scores": {"<tiêu chí>": <1-5>}, "average": <số>}',
|
|
134
|
+
rubric: '## RUBRIC', output: '## OUTPUT', none: '(không có)', empty: '(rỗng)',
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
function judgePrompt(output, rubric, language = 'en') {
|
|
138
|
+
const t = JUDGE_L[language] || JUDGE_L.en;
|
|
139
|
+
return [
|
|
140
|
+
t.head(JUDGE_SENTINEL),
|
|
141
|
+
t.only,
|
|
142
|
+
'', t.rubric, rubric || t.none, '', t.output, output || t.empty,
|
|
143
|
+
].join('\n');
|
|
144
|
+
}
|
|
145
|
+
async function judgeOutput(output, rubric, prov, model, language = 'en') {
|
|
146
|
+
let res; try { res = await prov.generate(judgePrompt(output, rubric, language), { model }); }
|
|
147
|
+
catch (e) { return { ok: false, error: 'judge call failed: ' + String(e.message || e) }; }
|
|
148
|
+
const m = String(res.text || '').match(/\{[\s\S]*\}/);
|
|
149
|
+
if (!m) return { ok: false, error: 'judge returned no JSON', raw: String(res.text || '').slice(0, 160) };
|
|
150
|
+
let j; try { j = JSON.parse(m[0]); } catch (e) { return { ok: false, error: 'judge JSON parse: ' + e.message }; }
|
|
151
|
+
const scores = j.scores || {};
|
|
152
|
+
const vals = Object.values(scores).map(Number).filter((n) => !isNaN(n));
|
|
153
|
+
const average = typeof j.average === 'number' ? j.average : (vals.length ? vals.reduce((a, b) => a + b, 0) / vals.length : 0);
|
|
154
|
+
return { ok: true, average: +Number(average).toFixed(2), scores };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Stable system prefix from the agent folder (SOUL + identity + tools + hard-rules).
|
|
158
|
+
function assembleSystem(agent) {
|
|
159
|
+
return [agent.soul || '', agent.identityBody || '', agent.tools || '', agent.hardRules || '']
|
|
160
|
+
.filter(Boolean).join('\n\n').trim();
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// ── run all gold cases for an agent ──
|
|
164
|
+
export async function runEvals(name, { cwd = process.cwd(), n, provider = 'auto', model, judge = false, judgeProvider, judgeModel, onEvent , language = 'en'} = {}) {
|
|
165
|
+
const agent = loadAgent(name, cwd);
|
|
166
|
+
if (!agent.evals.length) return { name, error: 'no gold cases under evals/ (need ≥1; ≥3 for L1)', cases: [], ok: false };
|
|
167
|
+
const prov = pickProvider(provider, { model });
|
|
168
|
+
if (!prov.ready) return { name, error: `provider "${prov.name}" not ready: ${prov.missing}`, cases: [], ok: false };
|
|
169
|
+
const system = assembleSystem(agent);
|
|
170
|
+
const emit = (ev, d) => onEvent && onEvent(ev, d);
|
|
171
|
+
// optional LLM-judge provider (defaults to the same provider). Skipped silently if not ready.
|
|
172
|
+
let judgeProv = null;
|
|
173
|
+
if (judge) { try { judgeProv = pickProvider(judgeProvider || provider, { model: judgeModel || model }); if (!judgeProv.ready) judgeProv = null; } catch { judgeProv = null; } }
|
|
174
|
+
|
|
175
|
+
const cases = [];
|
|
176
|
+
for (const file of agent.evals) {
|
|
177
|
+
const raw = fs.readFileSync(path.join(agent.dir, 'evals', file), 'utf8');
|
|
178
|
+
const gc = parseGoldCase(raw, file);
|
|
179
|
+
const runs = Math.max(1, n ? Number(n) : gc.runs);
|
|
180
|
+
// §29 — no assertions means nobody looked. Report it as `ungraded`, never as a quiet pass.
|
|
181
|
+
if (!gc.checks.length) { cases.push({ id: gc.id, skipped: true, verdict: 'ungraded', fixture: gc.fixture, reason: 'no ```checks block (machine assertions) — case is not gradeable' }); continue; }
|
|
182
|
+
|
|
183
|
+
const prompt = system ? `${system}\n\n${gc.input}` : gc.input;
|
|
184
|
+
const perRun = [];
|
|
185
|
+
let passCount = 0, lastText = '';
|
|
186
|
+
for (let i = 0; i < runs; i++) {
|
|
187
|
+
emit('run_start', { case: gc.id, i: i + 1, of: runs });
|
|
188
|
+
let text = '', genErr = null;
|
|
189
|
+
try { const out = await prov.generate(prompt, { cwd, model, system, cachePrefix: system }); text = out.text || ''; }
|
|
190
|
+
catch (e) { genErr = String(e.message || e); }
|
|
191
|
+
lastText = text;
|
|
192
|
+
const chk = genErr ? { passed: 0, total: gc.checks.length, allPass: false, results: [] } : runChecks(text, gc.checks);
|
|
193
|
+
if (chk.allPass) passCount++;
|
|
194
|
+
perRun.push({ i: i + 1, hashIn: sha(prompt), hashOut: sha(text), passed: chk.passed, total: chk.total, allPass: chk.allPass, err: genErr, fails: chk.results.filter((r) => !r.ok).map((r) => `${r.op}:${r.value}`) });
|
|
195
|
+
emit('run_done', { case: gc.id, i: i + 1, allPass: chk.allPass });
|
|
196
|
+
}
|
|
197
|
+
const consistency = passCount / runs;
|
|
198
|
+
const machinePass = consistency >= gc.threshold;
|
|
199
|
+
// LLM-judge (one call per case, token-disciplined) — only if requested + rubric present + judge ready
|
|
200
|
+
// §29: a judge that could not RUN did not say "bad" — it said nothing. That is `ungraded`,
|
|
201
|
+
// not `fail`. (Previously a judge call error was scored as a failure of the output.)
|
|
202
|
+
let judgeRes = null, judgePass = true, judgeUngraded = false;
|
|
203
|
+
if (judgeProv && gc.rubric) {
|
|
204
|
+
judgeRes = await judgeOutput(lastText, gc.rubric, judgeProv, judgeModel || model, language);
|
|
205
|
+
if (judgeRes.ok) {
|
|
206
|
+
judgePass = judgeRes.average >= gc.judgeMin;
|
|
207
|
+
} else {
|
|
208
|
+
judgeUngraded = true; judgePass = false;
|
|
209
|
+
}
|
|
210
|
+
judgeRes.min = gc.judgeMin; judgeRes.pass = judgeRes.ok ? judgePass : null;
|
|
211
|
+
}
|
|
212
|
+
// §29 — verdict is TERNARY: 'pass' | 'fail' | 'ungraded'.
|
|
213
|
+
// A deterministic machine-check failure is ALWAYS 'fail' even when the LLM judge could not run!
|
|
214
|
+
const verdict = !machinePass ? 'fail' : (judgeUngraded ? 'ungraded' : (judgePass ? 'pass' : 'fail'));
|
|
215
|
+
cases.push({ id: gc.id, runs, passCount, consistency: +consistency.toFixed(3), threshold: gc.threshold, machinePass, judge: judgeRes, verdict, checks: gc.checks.length, fixture: gc.fixture, perRun });
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// §29 — `ungraded` is excluded from `ok` and counted on its own. It is never folded into `pass`
|
|
219
|
+
// (that is the "green because nobody looked" failure) nor into `fail` (nothing was measured).
|
|
220
|
+
const graded = cases.filter((c) => !c.skipped && c.verdict !== 'ungraded');
|
|
221
|
+
const ungraded = cases.filter((c) => c.skipped || c.verdict === 'ungraded');
|
|
222
|
+
const ok = graded.length > 0 && graded.every((c) => c.verdict === 'pass');
|
|
223
|
+
const report = { name, provider: prov.name, model: prov.model, ts: new Date().toISOString(), cases, ok,
|
|
224
|
+
gradable: graded.length, passed: graded.filter((c) => c.verdict === 'pass').length,
|
|
225
|
+
ungraded: ungraded.length };
|
|
226
|
+
|
|
227
|
+
// hash-bound audit record (SCROLL §20.6 spirit — tamper-evident, file-native)
|
|
228
|
+
try {
|
|
229
|
+
const dir = path.join(cwd, '.scroll', 'eval'); fs.mkdirSync(dir, { recursive: true });
|
|
230
|
+
report.recordHash = sha(JSON.stringify(report.cases) + report.ts);
|
|
231
|
+
fs.writeFileSync(path.join(dir, `${name}.json`), JSON.stringify(report, null, 2) + '\n');
|
|
232
|
+
} catch { /* audit is best-effort */ }
|
|
233
|
+
return report;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// ── SCROLL 2.5 — Multi-turn Chat Eval Harness with `pass^k` reliability (G15) ──
|
|
237
|
+
// Executes multi-turn scenarios against `ChatSessionRuntime` across `trials` independent sessions
|
|
238
|
+
// and computes both single-trial pass rate and empirical `pass^k` (probability ALL k trials pass).
|
|
239
|
+
export async function runChatEvals(name, {
|
|
240
|
+
cwd = process.cwd(),
|
|
241
|
+
cases = null,
|
|
242
|
+
trials = 3,
|
|
243
|
+
provider = 'mock',
|
|
244
|
+
providerFactory = null,
|
|
245
|
+
toolsFactory = null,
|
|
246
|
+
model = undefined,
|
|
247
|
+
onEvent = null,
|
|
248
|
+
} = {}) {
|
|
249
|
+
const { createChatSessionRuntime } = await import('./chat-session.js');
|
|
250
|
+
const os = await import('node:os');
|
|
251
|
+
const agent = loadAgent(name, cwd);
|
|
252
|
+
const k = Math.max(1, Number(trials || 3));
|
|
253
|
+
const emit = (ev, d) => onEvent && onEvent(ev, d);
|
|
254
|
+
|
|
255
|
+
let scenarios = Array.isArray(cases) ? cases : [];
|
|
256
|
+
if (!scenarios.length) {
|
|
257
|
+
const chatDir = path.join(agent.dir, 'evals', 'chat');
|
|
258
|
+
if (fs.existsSync(chatDir)) {
|
|
259
|
+
for (const f of fs.readdirSync(chatDir).sort()) {
|
|
260
|
+
if (!f.endsWith('.json')) continue;
|
|
261
|
+
try {
|
|
262
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(chatDir, f), 'utf8'));
|
|
263
|
+
if (Array.isArray(parsed)) scenarios.push(...parsed);
|
|
264
|
+
else if (parsed && Array.isArray(parsed.turns)) scenarios.push({ id: parsed.id || path.basename(f, '.json'), ...parsed });
|
|
265
|
+
} catch { /* ignore malformed file */ }
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
if (!scenarios.length) {
|
|
270
|
+
return { name, mode: 'chat', trials: k, error: 'no chat eval scenarios found (pass `cases` or create agents/<name>/evals/chat/*.json)', cases: [], ok: false };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const results = [];
|
|
274
|
+
for (const sc of scenarios) {
|
|
275
|
+
const id = sc.id || 'chat-case';
|
|
276
|
+
const turns = Array.isArray(sc.turns) ? sc.turns : [];
|
|
277
|
+
const caseTrials = Math.max(1, Number(sc.trials || k));
|
|
278
|
+
const perTrial = [];
|
|
279
|
+
let passedTrials = 0;
|
|
280
|
+
|
|
281
|
+
for (let tIdx = 0; tIdx < caseTrials; tIdx++) {
|
|
282
|
+
emit('chat_trial_start', { case: id, trial: tIdx + 1, of: caseTrials });
|
|
283
|
+
const tmpState = fs.mkdtempSync(path.join(os.tmpdir(), 'scroll-chat-eval-'));
|
|
284
|
+
const prov = typeof providerFactory === 'function'
|
|
285
|
+
? await providerFactory({ scenario: sc, trial: tIdx })
|
|
286
|
+
: (typeof provider === 'object' && provider && typeof provider.generate === 'function'
|
|
287
|
+
? provider
|
|
288
|
+
: pickProvider(provider, { model, scriptedTurns: sc.scriptedTurns || null }));
|
|
289
|
+
const toolReg = typeof toolsFactory === 'function'
|
|
290
|
+
? await toolsFactory({ scenario: sc, trial: tIdx, stateDir: tmpState })
|
|
291
|
+
: null;
|
|
292
|
+
|
|
293
|
+
const runtime = createChatSessionRuntime({
|
|
294
|
+
agentName: name,
|
|
295
|
+
cwd,
|
|
296
|
+
stateDir: tmpState,
|
|
297
|
+
provider: prov,
|
|
298
|
+
tools: toolReg,
|
|
299
|
+
model,
|
|
300
|
+
acquireLock: false,
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
let trialOk = true;
|
|
304
|
+
const turnReports = [];
|
|
305
|
+
try {
|
|
306
|
+
for (let uIdx = 0; uIdx < turns.length; uIdx++) {
|
|
307
|
+
const turn = turns[uIdx];
|
|
308
|
+
const convId = turn.conversationId || `eval-${id}-t${tIdx}`;
|
|
309
|
+
let res = null;
|
|
310
|
+
let err = null;
|
|
311
|
+
try {
|
|
312
|
+
res = await runtime.sendTurn({
|
|
313
|
+
conversationId: convId,
|
|
314
|
+
text: turn.text || '',
|
|
315
|
+
sender: turn.sender || null,
|
|
316
|
+
participants: turn.participants || null,
|
|
317
|
+
isGroup: Boolean(turn.isGroup),
|
|
318
|
+
channel: turn.channel || 'eval',
|
|
319
|
+
approved: Boolean(turn.approved),
|
|
320
|
+
toolsAllow: turn.toolsAllow || null,
|
|
321
|
+
delegatedTools: turn.delegatedTools || null,
|
|
322
|
+
});
|
|
323
|
+
} catch (e) {
|
|
324
|
+
err = String(e?.message || e);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const failures = [];
|
|
328
|
+
if (err && !turn.expectError) failures.push(`unexpected_error:${err}`);
|
|
329
|
+
if (turn.expectError && !err) failures.push(`expected_error_missing:${turn.expectError}`);
|
|
330
|
+
if (res) {
|
|
331
|
+
if (Array.isArray(turn.checks) && turn.checks.length) {
|
|
332
|
+
const chk = runChecks(res.text, turn.checks);
|
|
333
|
+
for (const r of chk.results) {
|
|
334
|
+
if (!r.ok) failures.push(`${r.op}:${r.value}`);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
const resolveTraceTool = (tr) => {
|
|
338
|
+
if (tr.name !== 'tool_invoke') return tr.name;
|
|
339
|
+
const target = tr.arguments?.name || tr.arguments?.tool;
|
|
340
|
+
return target ? String(target) : tr.name;
|
|
341
|
+
};
|
|
342
|
+
const usedTools = (res.toolTraces || []).map(resolveTraceTool);
|
|
343
|
+
for (const reqTool of turn.expectTools || []) {
|
|
344
|
+
if (!usedTools.includes(reqTool)) failures.push(`missing_tool:${reqTool}`);
|
|
345
|
+
}
|
|
346
|
+
for (const badTool of turn.forbidTools || []) {
|
|
347
|
+
const executedOk = (res.toolTraces || []).some((tr) => resolveTraceTool(tr) === badTool && tr.result?.ok);
|
|
348
|
+
if (executedOk) failures.push(`forbidden_tool_succeeded:${badTool}`);
|
|
349
|
+
}
|
|
350
|
+
if (turn.expectGroundingOk === true && res.grounding?.ok === false) {
|
|
351
|
+
failures.push('grounding_failed');
|
|
352
|
+
}
|
|
353
|
+
if (turn.expectGroundingOk === false && res.grounding?.ok !== false) {
|
|
354
|
+
failures.push('expected_grounding_failure');
|
|
355
|
+
}
|
|
356
|
+
if (turn.expectStreamHeld === true && !res.streamHeld) {
|
|
357
|
+
failures.push('expected_stream_held');
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
const turnOk = failures.length === 0;
|
|
361
|
+
if (!turnOk) trialOk = false;
|
|
362
|
+
turnReports.push({ turn: uIdx + 1, ok: turnOk, failures, err, toolCount: res?.toolTraces?.length || 0 });
|
|
363
|
+
}
|
|
364
|
+
} finally {
|
|
365
|
+
await runtime.close().catch(() => {});
|
|
366
|
+
try { fs.rmSync(tmpState, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
if (trialOk) passedTrials++;
|
|
370
|
+
perTrial.push({ trial: tIdx + 1, ok: trialOk, turns: turnReports });
|
|
371
|
+
emit('chat_trial_done', { case: id, trial: tIdx + 1, ok: trialOk });
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
const passRate = passedTrials / caseTrials;
|
|
375
|
+
const passPowK = +Math.pow(passRate, caseTrials).toFixed(4);
|
|
376
|
+
const minPassPowK = sc.minPassPowK != null ? Number(sc.minPassPowK) : 1.0;
|
|
377
|
+
const verdict = passPowK >= minPassPowK ? 'pass' : 'fail';
|
|
378
|
+
results.push({
|
|
379
|
+
id,
|
|
380
|
+
trials: caseTrials,
|
|
381
|
+
passedTrials,
|
|
382
|
+
passRate: +passRate.toFixed(4),
|
|
383
|
+
passPowK,
|
|
384
|
+
minPassPowK,
|
|
385
|
+
allTrialsPassed: passedTrials === caseTrials,
|
|
386
|
+
verdict,
|
|
387
|
+
perTrial,
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const ok = results.length > 0 && results.every((r) => r.verdict === 'pass');
|
|
392
|
+
const report = {
|
|
393
|
+
name,
|
|
394
|
+
mode: 'chat',
|
|
395
|
+
trials: k,
|
|
396
|
+
ts: new Date().toISOString(),
|
|
397
|
+
cases: results,
|
|
398
|
+
ok,
|
|
399
|
+
passed: results.filter((r) => r.verdict === 'pass').length,
|
|
400
|
+
total: results.length,
|
|
401
|
+
};
|
|
402
|
+
try {
|
|
403
|
+
const dir = path.join(cwd, '.scroll', 'eval'); fs.mkdirSync(dir, { recursive: true });
|
|
404
|
+
report.recordHash = sha(JSON.stringify(report.cases) + report.ts);
|
|
405
|
+
fs.writeFileSync(path.join(dir, `${name}-chat.json`), JSON.stringify(report, null, 2) + '\n');
|
|
406
|
+
} catch { /* best-effort */ }
|
|
407
|
+
return report;
|
|
408
|
+
}
|