@agentpro/scroll 0.0.0-stage → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +55 -0
- package/LICENSE +21 -0
- package/README.md +293 -2
- package/SPEC.md +1079 -0
- package/bin/scroll.js +8 -0
- package/docs/prompt-caching.md +128 -0
- package/index.js +33 -0
- package/lib/audit.js +358 -0
- package/lib/auth.js +849 -0
- package/lib/chat-sdk/admin-controller.js +289 -0
- package/lib/chat-sdk/debouncer.js +94 -0
- package/lib/chat-sdk/group-grants.js +150 -0
- package/lib/chat-sdk/index.js +222 -0
- package/lib/chat-sdk/long-task-runner.js +413 -0
- package/lib/chat-session.js +1788 -0
- package/lib/chat-telemetry.js +557 -0
- package/lib/codex-app-server.js +851 -0
- package/lib/digest.js +112 -0
- package/lib/effects.js +157 -0
- package/lib/eval.js +408 -0
- package/lib/grounding.js +356 -0
- package/lib/installers.js +176 -0
- package/lib/knowledge.js +335 -0
- package/lib/ledger.js +38 -0
- package/lib/loop.js +197 -0
- package/lib/mcp-client.js +926 -0
- package/lib/memory-cli.js +66 -0
- package/lib/memory-tracker.js +1472 -0
- package/lib/memory.js +1279 -0
- package/lib/people-cli.js +81 -0
- package/lib/people.js +534 -0
- package/lib/permissions.js +272 -0
- package/lib/providers.js +1572 -0
- package/lib/runtime.js +928 -0
- package/lib/sandbox.js +744 -0
- package/lib/scheduler.js +402 -0
- package/lib/scroll.js +1225 -0
- package/lib/skill-cli.js +120 -0
- package/lib/skill-eval.js +156 -0
- package/lib/skills.js +860 -0
- package/lib/style.js +126 -0
- package/lib/tool-retriever.js +489 -0
- package/lib/tools.js +1833 -0
- package/lib/user.js +711 -0
- package/lib/vault.js +151 -0
- package/package.json +88 -4
- package/templates/agent/IDENTITY.md +56 -0
- package/templates/agent/SOUL.md +23 -0
- package/templates/agent/TOOLS.md +25 -0
- package/templates/agent/evals/case-02-citation-discipline.md +29 -0
- package/templates/agent/evals/case-03-boundary.md +27 -0
- package/templates/agent/evals/case-example.md +43 -0
- package/templates/agent/hard-rules.md +11 -0
- package/templates/agent/memory/lessons.md +5 -0
- package/templates/agent/memory/log/.gitkeep +0 -0
- package/templates/external-safety-contract.md +5 -0
- package/templates/long-task-planner.md +17 -0
- package/templates/work/LOOP.md +37 -0
- package/templates/work/WORK.md +50 -0
- package/types/chat.d.ts +61 -0
- package/types/index.d.ts +112 -0
package/lib/skill-cli.js
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// SCROLL — `scroll skill <list|view|index|scan|check|eval|sources>` (v1.8). `add` stays in installers.js.
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { loadAgent } from './scroll.js';
|
|
5
|
+
import { openSkills, checkSkills, writeCompatFile, detectSkillDirs, bindingsToolFor } from './skills.js';
|
|
6
|
+
|
|
7
|
+
const c = {
|
|
8
|
+
red: (s) => `\x1b[31m${s}\x1b[0m`, green: (s) => `\x1b[32m${s}\x1b[0m`, yellow: (s) => `\x1b[33m${s}\x1b[0m`,
|
|
9
|
+
dim: (s) => `\x1b[2m${s}\x1b[0m`, bold: (s) => `\x1b[1m${s}\x1b[0m`, cyan: (s) => `\x1b[36m${s}\x1b[0m`,
|
|
10
|
+
};
|
|
11
|
+
export const SKILL_HELP = `usage:
|
|
12
|
+
scroll skill add <folder> [--agent a] [--force] install a skill folder into skills/ (scanned first)
|
|
13
|
+
scroll skill add --new <name> [--agent a] scaffold a skill
|
|
14
|
+
scroll skill list [--agent a] [--all] [--json] skills the agent can use (--all: hidden, blocked, invalid too)
|
|
15
|
+
scroll skill view <name> [path] [--agent a] what skill_view returns for this skill or one of its files
|
|
16
|
+
scroll skill index [--agent a] the <available_skills> block for the cached prefix
|
|
17
|
+
scroll skill scan [--agent a] [--runtime r] write skills-compat.json (status per skill; your overrides are kept)
|
|
18
|
+
scroll skill check [--agent a] validate skills; errors for skills this repo ships
|
|
19
|
+
scroll skill eval <name> [--agent a] [--provider p1,p2] [--n N] [--judge] with vs without the skill
|
|
20
|
+
scroll skill sources [--detect] configured sources; --detect lists other tools' skill folders found here`;
|
|
21
|
+
|
|
22
|
+
// what the agent can do, from IDENTITY.capabilities (+ TOOLS.md bindings for a runtime)
|
|
23
|
+
function agentContext(cwd, agentName, runtime) {
|
|
24
|
+
if (!agentName) return {};
|
|
25
|
+
const ag = loadAgent(agentName, cwd);
|
|
26
|
+
const fm = ag.frontmatter || {};
|
|
27
|
+
return {
|
|
28
|
+
agent: agentName, identity: fm, pinned: fm.skills,
|
|
29
|
+
available: { capabilities: Array.isArray(fm.capabilities) ? fm.capabilities : [] },
|
|
30
|
+
toolFor: bindingsToolFor(ag.tools, runtime),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function skillCli(sub, a, cwd = process.cwd()) {
|
|
35
|
+
const agent = typeof a.agent === 'string' ? a.agent : null;
|
|
36
|
+
const runtime = typeof a.runtime === 'string' ? a.runtime : null;
|
|
37
|
+
const ctx = () => agentContext(cwd, agent, runtime);
|
|
38
|
+
switch (sub) {
|
|
39
|
+
case 'list': {
|
|
40
|
+
const reg = openSkills({ cwd, ...ctx() });
|
|
41
|
+
const rep = reg.report();
|
|
42
|
+
const rows = a.all ? [...rep.visible, ...rep.gated, ...rep.invalid] : rep.visible;
|
|
43
|
+
if (a.json) { console.log(JSON.stringify(a.all ? rep : rep.visible, null, 2)); return; }
|
|
44
|
+
if (!rows.length) { console.log(c.dim('no skills found (sources: agents/<a>/skills, skills/, skills.external_dirs in scroll.config.yaml)')); return; }
|
|
45
|
+
for (const r of rows) {
|
|
46
|
+
const st = r.state === 'visible' ? (r.compat === 'partial' ? c.yellow('partial') : c.green('ok')) : r.state === 'blocked' ? c.red('blocked') : r.state === 'invalid' ? c.red('invalid') : c.yellow(r.state);
|
|
47
|
+
console.log(`${st.padEnd(18)} ${c.bold(r.name || '?')} ${c.dim(`[${r.source}]`)}${r.reason ? c.dim(' — ' + r.reason) : ''}${r.state === 'invalid' ? c.dim(' — ' + r.errors.join('; ')) : ''}`);
|
|
48
|
+
}
|
|
49
|
+
console.log(c.dim(`\n${rep.visible.length} usable · ${rep.gated.filter((r) => r.state === 'hidden').length} hidden · ${rep.gated.filter((r) => r.state === 'blocked').length} blocked · ${rep.invalid.length} invalid · ${rep.shadowed.length} shadowed`));
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
case 'view': {
|
|
53
|
+
const name = a._[1]; if (!name) throw new Error('usage: scroll skill view <name> [path]');
|
|
54
|
+
const reg = openSkills({ cwd, ...ctx() });
|
|
55
|
+
const v = reg.view(name, a._[2] || null);
|
|
56
|
+
if (!v.ok) { console.log(`${c.red('✖')} ${v.error}`); process.exitCode = 1; return; }
|
|
57
|
+
console.log(v.text);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
case 'index': {
|
|
61
|
+
const reg = openSkills({ cwd, ...ctx() });
|
|
62
|
+
const ix = reg.index();
|
|
63
|
+
if (a.json) { console.log(JSON.stringify(ix, null, 2)); return; }
|
|
64
|
+
console.log(ix.text || c.dim('(empty: no usable skills)'));
|
|
65
|
+
console.log(c.dim(`\n${ix.included.length} skill(s) · ~${ix.tokens} tokens · hash ${ix.hash}${ix.omitted.length ? ` · ${ix.omitted.length} omitted (reachable with skill_list): ${ix.omitted.slice(0, 8).join(', ')}${ix.omitted.length > 8 ? ', …' : ''}` : ''}`));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
case 'scan': {
|
|
69
|
+
const reg = openSkills({ cwd, ...ctx() });
|
|
70
|
+
const file = typeof a.out === 'string' ? path.resolve(cwd, a.out) : (agent ? path.join(cwd, 'agents', agent, 'skills-compat.json') : path.join(cwd, 'skills-compat.json'));
|
|
71
|
+
const out = writeCompatFile(file, reg, { runtime });
|
|
72
|
+
const vals = Object.values(out.skills);
|
|
73
|
+
console.log(`${c.green('✔')} wrote ${path.relative(cwd, file) || file} — ${vals.filter((v) => v.state === 'visible' && v.compat !== 'partial').length} ok · ${vals.filter((v) => v.compat === 'partial').length} partial · ${vals.filter((v) => v.state === 'hidden').length} hidden · ${vals.filter((v) => v.state === 'blocked').length} blocked`);
|
|
74
|
+
console.log(c.dim(' correct the machine under "overrides": { "<skill>": { "status": "ok|hidden", "map": { "ForeignTool": "local_tool" }, "requires": [] } }'));
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
case 'check': {
|
|
78
|
+
const res = checkSkills(cwd, agent, ctx());
|
|
79
|
+
for (const w of res.warnings) console.log(`${c.yellow('⚠')} ${w}`);
|
|
80
|
+
for (const e of res.errors) console.log(`${c.red('✖')} ${e}`);
|
|
81
|
+
const k = res.counts;
|
|
82
|
+
console.log(`${res.errors.length ? c.red('✖') : c.green('✔')} skills: ${k.visible} usable · ${k.hidden} hidden · ${k.blocked} blocked · ${k.invalid} invalid · ${k.shadowed} shadowed`);
|
|
83
|
+
if (res.errors.length) process.exitCode = 1;
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
case 'eval': {
|
|
87
|
+
const name = a._[1]; if (!name) throw new Error('usage: scroll skill eval <name> [--provider p1,p2] [--n N] [--judge]');
|
|
88
|
+
const { runSkillEval } = await import('./skill-eval.js');
|
|
89
|
+
const providers = typeof a.provider === 'string' ? a.provider.split(',').map((s) => s.trim()).filter(Boolean) : ['auto'];
|
|
90
|
+
const rep = await runSkillEval(name, { cwd, agent, providers, n: a.n ? Number(a.n) : undefined, judge: !!a.judge, model: typeof a.model === 'string' ? a.model : undefined, registry: openSkills({ cwd, compatFile: false, ...ctx() }) });
|
|
91
|
+
if (a.json) { console.log(JSON.stringify(rep, null, 2)); if (!rep.ok) process.exitCode = 1; return; }
|
|
92
|
+
if (rep.error) { console.log(`${c.red('✖')} ${rep.error}`); process.exitCode = 1; return; }
|
|
93
|
+
for (const p of rep.providers) {
|
|
94
|
+
console.log(c.bold(`\n${p.provider} (${p.model})`) + (p.error ? ` ${c.red(p.error)}` : ''));
|
|
95
|
+
for (const r of p.cases) {
|
|
96
|
+
const mark = r.verdict === 'pass' ? c.green('✔') : r.verdict === 'ungraded' ? c.yellow('·') : c.red('✖');
|
|
97
|
+
const cmp = r.compare ? (r.compare.ok ? ` · blind: ${r.compare.winner}` : c.yellow(` · blind: ${r.compare.error}`)) : '';
|
|
98
|
+
console.log(`${mark} ${r.id} — with ${r.with.pass}/${r.runs} · without ${r.without.pass}/${r.runs}${cmp} · tokens ${r.with.tokens} vs ${r.without.tokens}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const verdict = rep.ok ? c.green('PASS') : c.red('FAIL');
|
|
102
|
+
console.log(`\n${verdict} — recommendation: ${c.bold(rep.recommendation)} · record .scroll/skill-eval/${name}.json`);
|
|
103
|
+
if (!rep.ok) process.exitCode = 1;
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
case 'sources': {
|
|
107
|
+
const reg = openSkills({ cwd, ...ctx() });
|
|
108
|
+
for (const s of reg.report().sources) console.log(`${s.exists ? c.green('●') : c.dim('○')} ${c.bold(s.id)} ${s.path} ${c.dim(`(${s.kind}, trust ${s.trust}${s.layout !== 'dir' ? ', layout ' + s.layout : ''})`)}`);
|
|
109
|
+
if (a.detect) {
|
|
110
|
+
const found = detectSkillDirs();
|
|
111
|
+
console.log(c.bold('\nskill folders of other tools on this machine (not read unless listed in scroll.config.yaml):'));
|
|
112
|
+
for (const k of found) console.log(` ${k.path}${k.layout ? c.dim(' layout ' + k.layout) : ''} ${c.dim('— ' + k.note)}`);
|
|
113
|
+
if (found.length) console.log(c.dim('\n add one: skills:\n external_dirs:\n - { path: ~/.agents/skills, trust: trusted }'));
|
|
114
|
+
}
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
default:
|
|
118
|
+
console.log(SKILL_HELP);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// SCROLL — skill eval (v1.8 §36.3): does a skill make the work better, on each model it will run on?
|
|
2
|
+
//
|
|
3
|
+
// Every case runs twice per model: once with the skill, once without (the baseline). Machine checks
|
|
4
|
+
// grade both; an optional judge compares the two answers blind (it never learns which one had the
|
|
5
|
+
// skill). The record says what the skill adds, not only whether the answer passed:
|
|
6
|
+
// keep the skill passes and beats the baseline somewhere
|
|
7
|
+
// retire-or-rewrite the model already does as well without it (the skill only costs tokens)
|
|
8
|
+
// fix the skill's own runs fail
|
|
9
|
+
// Verdicts stay ternary (§29): a case nobody could grade is `ungraded`, never a pass.
|
|
10
|
+
//
|
|
11
|
+
// Cases live in the skill folder, so they travel with it:
|
|
12
|
+
// evals/*.md SCROLL gold cases (§8: ## Input, ```checks, ## Rubric)
|
|
13
|
+
// evals/evals.json { "evals": [{ "id", "prompt", "expected_output", "expectations": [], "files": [], "checks": [] }] }
|
|
14
|
+
// (the layout other skill tools use; `checks` is the SCROLL machine-check extension)
|
|
15
|
+
import fs from 'node:fs';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
import crypto from 'node:crypto';
|
|
18
|
+
import { parseGoldCase, runChecks } from './eval.js';
|
|
19
|
+
import { pickProvider } from './providers.js';
|
|
20
|
+
import { openSkills } from './skills.js';
|
|
21
|
+
|
|
22
|
+
const sha = (s) => crypto.createHash('sha256').update(String(s)).digest('hex').slice(0, 16);
|
|
23
|
+
const arr = (x) => (Array.isArray(x) ? x : x == null ? [] : [x]);
|
|
24
|
+
|
|
25
|
+
export function loadSkillCases(skillDir) {
|
|
26
|
+
const ev = path.join(skillDir, 'evals');
|
|
27
|
+
const out = [];
|
|
28
|
+
if (!fs.existsSync(ev)) return out;
|
|
29
|
+
for (const f of fs.readdirSync(ev).filter((x) => x.endsWith('.md') && x !== 'rubric.md').sort()) {
|
|
30
|
+
const gc = parseGoldCase(fs.readFileSync(path.join(ev, f), 'utf8'), f);
|
|
31
|
+
out.push({ id: gc.id, input: gc.input, checks: gc.checks, rubric: gc.rubric, runs: gc.runs, threshold: gc.threshold, files: [], from: `evals/${f}` });
|
|
32
|
+
}
|
|
33
|
+
const jp = path.join(ev, 'evals.json');
|
|
34
|
+
if (fs.existsSync(jp)) {
|
|
35
|
+
let j; try { j = JSON.parse(fs.readFileSync(jp, 'utf8')); } catch (e) { throw new Error(`evals/evals.json: ${e.message}`); }
|
|
36
|
+
arr(j.evals).forEach((e, i) => {
|
|
37
|
+
if (!e || !e.prompt) return;
|
|
38
|
+
const rubric = [e.expected_output, ...arr(e.expectations)].filter(Boolean).map((x) => `- ${x}`).join('\n');
|
|
39
|
+
out.push({ id: String(e.id ?? i + 1), input: String(e.prompt), checks: arr(e.checks).filter((c) => c && c.op), rubric, runs: Number(e.runs) || 3, threshold: Number(e.pass_consistency ?? 0.66), files: arr(e.files).map(String), from: 'evals/evals.json' });
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function attachFiles(skillDir, files) {
|
|
46
|
+
const parts = [];
|
|
47
|
+
for (const f of files) {
|
|
48
|
+
const p = path.resolve(skillDir, f);
|
|
49
|
+
if (!p.startsWith(path.resolve(skillDir) + path.sep) || !fs.existsSync(p)) { parts.push(`[attached file ${f}: not found]`); continue; }
|
|
50
|
+
const buf = fs.readFileSync(p);
|
|
51
|
+
parts.push(buf.subarray(0, 8192).includes(0) ? `[attached file ${f}: binary, ${buf.length} bytes]` : `Attached file ${f}:\n${buf.toString('utf8').slice(0, 32000)}`);
|
|
52
|
+
}
|
|
53
|
+
return parts.join('\n\n');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const COMPARE_SENTINEL = 'SCROLL_COMPARE_JSON';
|
|
57
|
+
function comparePrompt(task, rubric, a, b) {
|
|
58
|
+
return [
|
|
59
|
+
`You compare two answers to the same task (${COMPARE_SENTINEL}). Judge only against the criteria.`,
|
|
60
|
+
'Return JSON only: {"winner": "A" | "B" | "tie", "why": "<one sentence>"}',
|
|
61
|
+
'', '## TASK', task, '', '## CRITERIA', rubric || '(none: judge correctness and completeness)', '', '## ANSWER A', a || '(empty)', '', '## ANSWER B', b || '(empty)',
|
|
62
|
+
].join('\n');
|
|
63
|
+
}
|
|
64
|
+
// The order of the two answers alternates case by case (from a per-model seed), so over a suite the
|
|
65
|
+
// skill's answer is "A" about half the time: replayable, and a judge that prefers a position gains nothing.
|
|
66
|
+
async function compareBlind(prov, model, task, rubric, withText, withoutText, seed, position) {
|
|
67
|
+
const swap = (parseInt(sha(seed).slice(0, 2), 16) + position) % 2 === 1;
|
|
68
|
+
const [a, b] = swap ? [withoutText, withText] : [withText, withoutText];
|
|
69
|
+
let res; try { res = await prov.generate(comparePrompt(task, rubric, a, b), { model }); } catch (e) { return { ok: false, error: String(e.message || e) }; }
|
|
70
|
+
const m = String(res.text || '').match(/\{[\s\S]*\}/);
|
|
71
|
+
if (!m) return { ok: false, error: 'comparator returned no JSON' };
|
|
72
|
+
let j; try { j = JSON.parse(m[0]); } catch (e) { return { ok: false, error: 'comparator JSON: ' + e.message }; }
|
|
73
|
+
const w = String(j.winner || '').toUpperCase();
|
|
74
|
+
if (!['A', 'B', 'TIE'].includes(w)) return { ok: false, error: `comparator winner "${j.winner}"` };
|
|
75
|
+
const winner = w === 'TIE' ? 'tie' : ((w === 'A') !== swap ? 'with' : 'without');
|
|
76
|
+
return { ok: true, winner, why: String(j.why || '').slice(0, 300) };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function runSide(prov, model, prompt, runs, checks, cwd) {
|
|
80
|
+
let pass = 0, tokens = 0, ms = 0, last = '', errs = 0;
|
|
81
|
+
for (let i = 0; i < runs; i++) {
|
|
82
|
+
let out = null;
|
|
83
|
+
try { out = await prov.generate(prompt, { cwd, model }); } catch { errs++; continue; }
|
|
84
|
+
last = out.text || '';
|
|
85
|
+
tokens += (out.usage?.uncachedInputTokens || 0) + (out.usage?.cachedInputTokens || 0) + (out.usage?.outputTokens || 0);
|
|
86
|
+
ms += out.durationMs || 0;
|
|
87
|
+
if (checks.length && runChecks(last, checks).allPass) pass++;
|
|
88
|
+
}
|
|
89
|
+
return { pass, runs, rate: runs ? +(pass / runs).toFixed(3) : 0, tokens, ms, last, errors: errs };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export async function runSkillEval(name, opts = {}) {
|
|
93
|
+
const cwd = opts.cwd || process.cwd();
|
|
94
|
+
const reg = opts.registry || openSkills({ cwd, agent: opts.agent || null, compatFile: false });
|
|
95
|
+
const skill = reg.get(name);
|
|
96
|
+
if (!skill) return { skill: name, ok: false, error: `skill "${name}" is not available (run: scroll skill list --all)` };
|
|
97
|
+
const cases = loadSkillCases(skill.dir);
|
|
98
|
+
if (!cases.length) return { skill: name, ok: false, error: 'no eval cases in evals/ (gold-case .md files or evals/evals.json)' };
|
|
99
|
+
const view = reg.view(name);
|
|
100
|
+
if (!view.ok) return { skill: name, ok: false, error: view.error };
|
|
101
|
+
const providers = arr(opts.providers && opts.providers.length ? opts.providers : ['auto']);
|
|
102
|
+
const results = [];
|
|
103
|
+
for (const pv of providers) {
|
|
104
|
+
const prov = typeof pv === 'object' ? pv : pickProvider(pv, { model: opts.model });
|
|
105
|
+
if (!prov.ready) { results.push({ provider: prov.name, model: prov.model, error: `not ready: ${prov.missing}`, cases: [], ok: false }); continue; }
|
|
106
|
+
let judge = null;
|
|
107
|
+
if (opts.judge) {
|
|
108
|
+
const jp = opts.judgeProvider;
|
|
109
|
+
judge = typeof jp === 'object' && jp ? jp : pickProvider(jp || (typeof pv === 'string' ? pv : 'auto'), { model: opts.judgeModel || opts.model });
|
|
110
|
+
if (!judge.ready) judge = null;
|
|
111
|
+
}
|
|
112
|
+
const rows = [];
|
|
113
|
+
for (const cs of cases) {
|
|
114
|
+
const runs = Math.max(1, Number(opts.n) || cs.runs || 3);
|
|
115
|
+
const files = cs.files.length ? `\n\n${attachFiles(skill.dir, cs.files)}` : '';
|
|
116
|
+
const task = `${cs.input}${files}`;
|
|
117
|
+
const withPrompt = `Follow this skill for the task below.\n\n<skill name="${skill.name}">\n${view.text}\n</skill>\n\nTask:\n${task}`;
|
|
118
|
+
const withoutPrompt = `Task:\n${task}`;
|
|
119
|
+
const w = await runSide(prov, opts.model, withPrompt, runs, cs.checks, cwd);
|
|
120
|
+
const wo = await runSide(prov, opts.model, withoutPrompt, runs, cs.checks, cwd);
|
|
121
|
+
let cmp = null;
|
|
122
|
+
if (judge) cmp = await compareBlind(judge, opts.judgeModel || opts.model, task, cs.rubric, w.last, wo.last, `${prov.name}|${prov.model}`, rows.length);
|
|
123
|
+
const machine = cs.checks.length > 0;
|
|
124
|
+
let verdict;
|
|
125
|
+
if (!machine && !(cmp && cmp.ok)) verdict = 'ungraded';
|
|
126
|
+
else if (machine && w.errors === runs) verdict = 'ungraded';
|
|
127
|
+
else {
|
|
128
|
+
const machinePass = !machine || w.rate >= cs.threshold;
|
|
129
|
+
const judgePass = !(cmp && cmp.ok) || cmp.winner !== 'without';
|
|
130
|
+
verdict = machinePass && judgePass ? 'pass' : 'fail';
|
|
131
|
+
}
|
|
132
|
+
const better = (machine && w.rate > wo.rate) || (cmp && cmp.ok && cmp.winner === 'with');
|
|
133
|
+
const noBetter = (machine ? w.rate <= wo.rate : true) && (!cmp || !cmp.ok || cmp.winner !== 'with');
|
|
134
|
+
rows.push({ id: cs.id, from: cs.from, runs, checks: cs.checks.length, verdict,
|
|
135
|
+
with: { pass: w.pass, rate: w.rate, tokens: w.tokens, ms: w.ms, errors: w.errors },
|
|
136
|
+
without: { pass: wo.pass, rate: wo.rate, tokens: wo.tokens, ms: wo.ms, errors: wo.errors },
|
|
137
|
+
compare: cmp, addsValue: !!better, baselineAsGood: verdict !== 'ungraded' && noBetter });
|
|
138
|
+
}
|
|
139
|
+
const graded = rows.filter((r) => r.verdict !== 'ungraded');
|
|
140
|
+
results.push({ provider: prov.name, model: prov.model, cases: rows, ok: graded.length > 0 && graded.every((r) => r.verdict === 'pass'),
|
|
141
|
+
gradable: graded.length, passed: graded.filter((r) => r.verdict === 'pass').length, ungraded: rows.length - graded.length,
|
|
142
|
+
tokensWith: rows.reduce((s, r) => s + r.with.tokens, 0), tokensWithout: rows.reduce((s, r) => s + r.without.tokens, 0) });
|
|
143
|
+
}
|
|
144
|
+
const graded = results.flatMap((r) => r.cases.filter((c) => c.verdict !== 'ungraded'));
|
|
145
|
+
const ok = results.length > 0 && results.every((r) => r.ok);
|
|
146
|
+
const recommendation = !graded.length ? 'ungraded' : !ok ? 'fix' : graded.every((c) => c.baselineAsGood) ? 'retire-or-rewrite' : 'keep';
|
|
147
|
+
const report = { skill: name, hash: skill.hash, source: skill.source, ts: new Date().toISOString(), cases: cases.length, providers: results, ok, recommendation };
|
|
148
|
+
if (opts.write !== false) {
|
|
149
|
+
try {
|
|
150
|
+
const dir = path.join(cwd, '.scroll', 'skill-eval'); fs.mkdirSync(dir, { recursive: true });
|
|
151
|
+
report.recordHash = sha(JSON.stringify(report.providers) + report.hash + report.ts);
|
|
152
|
+
fs.writeFileSync(path.join(dir, `${name}.json`), JSON.stringify(report, null, 2) + '\n');
|
|
153
|
+
} catch { /* the record is best-effort; the verdict above is not */ }
|
|
154
|
+
}
|
|
155
|
+
return report;
|
|
156
|
+
}
|