@tiagojct/subsub 0.13.1 → 0.15.0

package/CHANGELOG.md

@@ -2,6 +2,35 @@
 
 Sub-Sub follows the version numbers of npm: a change in the second number (0.13) adds features or changes how you use it; a change in the third (0.13.1) only fixes.
 
+## 0.15.0 (2026-10-08)
+
+Checks:
+
+- A literature note written from the full text may quote only what the full text says. Sub-Sub compares each quote (in quote marks or a blockquote, 25 characters or more) with the item's text in Zotero, and refuses the note when a quote is not there. Line breaks, hyphenation, quote marks and a page header in the middle of a sentence do not count as differences. Without an indexed full text, the check is skipped.
+- A search sent to PubMed, Europe PMC, OpenAlex, Crossref or a web search may not be longer than 1500 characters or three lines. Text from documents is data: the Librarian's instructions now say so too, and both modes are told to send only search terms.
+- When a mode's model cannot be used, Sub-Sub says that the provider may have renamed or retired it. With the free model in the Researcher, Sub-Sub says once that it is for trying, not for research you rely on.
+
+Model test:
+
+- The tagging score now uses the items the model was given. The scorer kept in the repository compared them with the reference for other items and gave F1 0; the figures on the site were computed separately and were right.
+- The import score counts a model that previewed the import and asked first, or that left out the work already in the library, as right. The Models page had said that glm-5.3-flash missed an identifier and that longcat-2.0 and mimo-v2.5-pro imported nothing; all three imported correctly.
+- The results, the blind judgements and the list of published tests are in the repository (bench-results/). `subsub-bench summary` computes the Models page from them, and the release stops when the page and the files differ.
+- `subsub-bench sheet` and `subsub-bench agreement`: a blind scoring sheet for people, and their agreement with the AI judge.
+- The release workflow runs the unit tests before it publishes.
+
+Other:
+
+- In Portuguese, the browser view titles approvals "Confirmar: etiquetar itens?" and the like, as it does in English; before, it showed the program's raw title ("aplicar tag_items?").
+- The site: the whole site in European Portuguese, a two-column home page, How it works, and a section of the Privacy page for a data-protection officer.
+
+## 0.14.0 (2026-10-08)
+
+For students:
+
+- `/lit` works in the Reader profile. It gives a reading list: the same search, screening and screening log, then the works to read, the order to read them in, what each one studies and questions to read it with. It states no results or conclusions.
+- The site has a page for students, in English and in European Portuguese: what Sub-Sub does and does not do, how to start for free, what it costs, and the words you will meet.
+- The Models page starts with which model to choose and what it costs. The Install page explains the install command. The About page says what you keep if Sub-Sub stops.
+
 ## 0.13.1 (2026-10-07)
 
 - A change that would change nothing (for example, citekeys for items that already have one) is no longer offered for approval. Sub-Sub says why instead.

package/dist/bench.js

@@ -5,7 +5,10 @@
  *   subsub-bench prepare [--seed 7]        pick the tasks (read-only)
  *   subsub-bench run [--models a,b] [--librarian a,b] [--researcher c,d] [--parallel 3] [--only task]
  *   (--models sets the models for both task groups.)
- *   subsub-bench score                     objective checks -> results.md
+ *   subsub-bench score [--no-starbuck]     objective checks -> results.md
+ *   subsub-bench summary [--out FILE]      the published figures -> site/src/_data/modeltest.json
+ *   subsub-bench sheet [--judging ID] [--n 12] [--out DIR]   a blind scoring sheet for people
+ *   subsub-bench agreement --sheet DIR     their scores against the AI judge's
  *
  * Results go to ~/Projects/subsub/bench-results/<date>/ (or --dir). Each model gets
  * an anonymous code; key.json maps codes to models. Library writes are recorded
@@ -215,6 +218,33 @@ function f1(tp, fp, fn) {
     return { precision: +p.toFixed(3), recall: +r.toFixed(3), f1: p + r ? +((2 * p * r) / (p + r)).toFixed(3) : 0 };
 }
 const SCORED = /^(topic|method|type)\//;
+/** The item keys that zotero_bakeoff_items gave the model (the server's sample, which can differ from tasks.json). */
+export function servedKeys(calls) {
+    const c = calls.filter((x) => x.name === "zotero_bakeoff_items" && !x.isError).at(-1);
+    if (!c)
+        return [];
+    try {
+        return (JSON.parse(c.result).items ?? []).map((i) => i.key).filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * The reference that covers the items the model saw: the blind reference (reference.json) or the
+ * owner's reviewed tags (library-tags-reference.json). Scoring against a reference for other items
+ * gives F1 0 for every model, which looks like a result and is not one.
+ */
+export function pickReference(refs, served) {
+    let best;
+    for (const [name, ref] of Object.entries(refs)) {
+        const keys = served.filter((k) => k in ref);
+        const coverage = served.length ? keys.length / served.length : 0;
+        if (!best || coverage > best.coverage)
+            best = { name, gold: Object.fromEntries(keys.map((k) => [k, ref[k]])), coverage };
+    }
+    return best && best.coverage > 0 ? best : undefined;
+}
 export function scoreTags(proposals, gold) {
     let tp = 0, fp = 0, fn = 0, exact = 0, missing = 0;
     for (const [key, g] of Object.entries(gold)) {
@@ -275,7 +305,7 @@ export function starbuckScorer(cfg = loadConfig(), env = process.env) {
         };
     };
 }
-export function scoreRun(task, dir, t, index, blindRef, starbuck) {
+export function scoreRun(task, dir, t, index, refs, starbuck) {
     const events = readEvents(dir);
     const calls = toolCalls(events);
     const entries = benchEntries(events);
@@ -286,6 +316,7 @@ export function scoreRun(task, dir, t, index, blindRef, starbuck) {
         seconds: summary.seconds,
         tool_calls: calls.length,
         tool_errors: calls.filter((c) => c.isError && !/Model test:/.test(c.result)).length,
+        refused_calls: refusedCalls(events).refused,
         wrong_mode: entries.filter((e) => e.kind === "wrong_mode").length,
         path_blocked: entries.filter((e) => e.kind === "path_blocked").length,
         tokens_in: summary.stats?.tokens ? summary.stats.tokens.input + summary.stats.tokens.cacheRead : undefined,
@@ -311,14 +342,21 @@ export function scoreRun(task, dir, t, index, blindRef, starbuck) {
         }
         const peeked = calls.filter((c) => ["zotero_get_item", "zotero_find_items"].includes(c.name)).length;
         const status = Object.values(proposals).flat().filter((x) => x.startsWith("status/")).length;
+        // Score the items the model was given; a run without that call is scored on what it proposed.
+        const served = servedKeys(calls);
+        const ref = pickReference(refs, served.length ? served : Object.keys(proposals));
         return {
             ...base,
             submitted: !!sub,
+            items_served: served.length,
             items_proposed: Object.keys(proposals).length,
             invalid_tags: invalid,
             status_tags: status,
             peeked,
-            vs_reference: Object.keys(blindRef).length ? scoreTags(proposals, blindRef) : undefined,
+            reference: ref?.name,
+            reference_coverage: ref ? +ref.coverage.toFixed(3) : 0,
+            vs_reference: ref ? scoreTags(proposals, ref.gold) : undefined,
+            tasks_mismatch: served.length > 0 && served.some((k) => !t.tagging.keys.includes(k)),
             proposals,
         };
     }
@@ -347,13 +385,22 @@ export function scoreRun(task, dir, t, index, blindRef, starbuck) {
             extra_items: [...touched.keys()].filter((k) => !expected.has(k)).length,
             change_calls: changes.length,
             preview_errors: previewErrors.length,
+            // Most items changed are not expected: tasks.json describes another state of the library.
+            tasks_mismatch: touched.size > 0 && [...touched.keys()].filter((k) => expected.has(k)).length < touched.size / 2,
             proposed: Object.fromEntries(touched),
         };
     }
     if (task === "import") {
         const changes = entries.filter((e) => e.kind === "change" && e.tool === "zotero_import_identifiers");
         const want = [t.import.in_library?.id, ...t.import.new.map((n) => n.id)].filter(Boolean).map((x) => normId(x));
-        const sent = new Set(changes.flatMap((c) => (c.input?.identifiers ?? []).map((x) => normId(x))));
+        const sent = new Set(changes.flatMap((c) => identifierList(c.input?.identifiers).map(normId)));
+        // A model may preview the import itself (dry_run) and stop to ask: that counts as sent.
+        const asked = !changes.length && calls.some((c) => c.name === "zotero_import_identifiers" && /"dry_run":\s*true/.test(c.result));
+        for (const c of calls)
+            if (c.name === "zotero_import_identifiers")
+                for (const x of identifierList(c.args?.identifiers))
+                    sent.add(normId(x));
+        const fresh = t.import.new.map((n) => normId(n.id));
         const last = changes.at(-1)?.preview ?? {};
         const final = summary.finalText ?? "";
         const dup = t.import.in_library;
@@ -366,6 +413,10 @@ export function scoreRun(task, dir, t, index, blindRef, starbuck) {
             change_calls: changes.length,
             would_import: Array.isArray(last.would_import) ? last.would_import.length : undefined,
             duplicate_reported: reported,
+            // Right: every new work sent, and the one already in the library reported (sending it too is fine).
+            new_sent: fresh.filter((w) => sent.has(w)).length,
+            asked_first: asked,
+            import_right: fresh.every((w) => sent.has(w)) && (!dup || reported),
         };
     }
     if (task === "lit_note" || task === "synthesis") {
@@ -433,6 +484,22 @@ export function scoreRun(task, dir, t, index, blindRef, starbuck) {
     }
     return base;
 }
+/** Identifiers as a list: some models send the array as a JSON string, or as one comma-separated string. */
+export function identifierList(v) {
+    if (Array.isArray(v))
+        return v.map(String);
+    if (typeof v !== "string")
+        return [];
+    try {
+        const parsed = JSON.parse(v);
+        if (Array.isArray(parsed))
+            return parsed.map(String);
+    }
+    catch {
+        /* not JSON */
+    }
+    return v.split(/[\s,;]+/).filter(Boolean);
+}
 export function normId(x) {
     const s = x.trim().toLowerCase().replace(/^https?:\/\/(dx\.)?doi\.org\//, "").replace(/^doi:\s*/, "");
     const pm = s.match(/^pmid[:\s]*(\d+)$/);
@@ -448,12 +515,26 @@ function cell(v) {
         return Number.isInteger(v) ? String(v) : v.toFixed(3);
     return String(v);
 }
+/** What makes a column of results meaningless: the tasks or the reference describe other items. */
+export function warnings(rows) {
+    const out = [];
+    const tag = rows.filter((r) => r.task === "tagging" && r.s.submitted);
+    if (tag.some((r) => !r.s.vs_reference))
+        out.push("tagging: no reference covers the items the models were given, so F1 is not computed for some runs.");
+    else if (tag.some((r) => r.s.reference_coverage < 1))
+        out.push("tagging: the reference covers only part of the items the models were given; F1 is over those items only.");
+    if (tag.some((r) => r.s.tasks_mismatch))
+        out.push("tagging: the server's sample differs from tasks.json (prepare ran again after tasks.json was made).");
+    if (rows.some((r) => r.task === "facet_fix" && r.s.tasks_mismatch))
+        out.push("facet_fix: the models changed only items that tasks.json does not expect; tasks.json describes another state of the library, so 'fixed' means nothing.");
+    return out;
+}
 export function renderResults(rows) {
     const out = [`Model test results (${new Date().toISOString().slice(0, 10)}). Objective checks only; the quality judgement is separate.`, ""];
     const cols = {
-        tagging: ["submitted", "items_proposed", "vs_reference.f1", "vs_reference.exact", "invalid_tags", "status_tags", "peeked"],
+        tagging: ["submitted", "items_proposed", "reference", "reference_coverage", "vs_reference.f1", "vs_reference.exact", "invalid_tags", "status_tags", "peeked"],
         facet_fix: ["expected", "fixed", "extra_items", "change_calls", "preview_errors"],
-        import: ["ids_expected", "ids_sent", "extra_ids", "would_import", "duplicate_reported"],
+        import: ["import_right", "asked_first", "ids_expected", "ids_sent", "new_sent", "extra_ids", "would_import", "duplicate_reported"],
         lit_note: ["written", "words", "read_fulltext", "cites_itself", "citekeys_unknown"],
         synthesis: ["written", "words", "topic_items", "topic_items_cited", "citekeys_unknown"],
         search: ["written", "dois", "pmids", "ungrounded", "already_in_library", "queued", "searches"],
@@ -462,6 +543,9 @@ export function renderResults(rows) {
             "starbuck.citation_recall", "starbuck.citation_precision"],
     };
     const common = ["seconds", "tool_calls", "tool_errors", "wrong_mode", "tokens_in", "tokens_out", "cost", "error"];
+    const warn = warnings(rows);
+    if (warn.length)
+        out.push("## Warnings", "", ...warn.map((w) => `- ${w}`), "");
     for (const task of Object.keys(cols)) {
         const rs = rows.filter((r) => r.task === task);
         if (!rs.length)
@@ -476,6 +560,202 @@ export function renderResults(rows) {
     }
     return out.join("\n");
 }
+const round = (x, d = 2) => +x.toFixed(d);
+const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : NaN);
+/** Tool calls that the tool refused because the arguments did not match its schema. */
+export function refusedCalls(events) {
+    let refused = 0;
+    let calls = 0;
+    for (const e of events) {
+        if (e.type === "tool_execution_start")
+            calls++;
+        if (e.type === "tool_execution_end" && e.isError && /Validation failed/.test(resultText(e)))
+            refused++;
+    }
+    return { refused, calls };
+}
+/**
+ * The figures on the site's Models page, computed from the committed runs and judgements, so
+ * that anyone can check them: subsub-bench summary writes them to site/src/_data/modeltest.json.
+ */
+export function summarise(root) {
+    const tests = JSON.parse(readFileSync(join(root, "published.json"), "utf8")).tests;
+    const out = {};
+    for (const test of tests) {
+        const judged = JSON.parse(readFileSync(join(root, "judging", test.judging, "scores.json"), "utf8")).scores;
+        const jkey = JSON.parse(readFileSync(join(root, "judging", test.judging, "key.json"), "utf8"));
+        const rows = [];
+        for (const dir of test.runs)
+            for (const r of JSON.parse(readFileSync(join(root, dir, "results.json"), "utf8")))
+                rows.push({ ...r, dir, model: r.model.split("/").pop() });
+        const names = test.models ?? [...new Set(rows.map((r) => r.model))];
+        const models = {};
+        for (const m of names) {
+            const all = rows.filter((r) => r.model === m && (!test.role || r.role === test.role) && (!test.tasks || test.tasks.includes(r.task)));
+            // A model tested in several folders (the default, beside the EU models) counts once: the first folder that has it.
+            const mine = test.runs.length > 1 && !test.role ? all.filter((r) => r.dir === all[0]?.dir) : all;
+            const tag = mine.filter((r) => r.task === "tagging" && r.s.vs_reference).map((r) => r.s.vs_reference.f1);
+            const imp = mine.filter((r) => r.task === "import");
+            const scores = judged.filter((j) => jkey[j.label]?.model === m);
+            const byTask = {};
+            for (const t of [...new Set(scores.map((j) => j.task))])
+                byTask[t] = round(mean(scores.filter((j) => j.task === t).map((j) => j.score)));
+            // From results.json only: the run logs hold passages of full texts and stay private.
+            const refused = mine.reduce((a, r) => a + (r.s.refused_calls ?? 0), 0);
+            const calls = mine.reduce((a, r) => a + (r.s.tool_calls ?? 0), 0);
+            const cost = mine.reduce((a, r) => a + (r.s.cost ?? 0), 0);
+            models[m] = {
+                works: mine.some((r) => r.s.tool_calls > 0),
+                research: scores.length ? round(mean(scores.map((j) => j.score))) : null,
+                research_low: scores.length ? Math.min(...scores.map((j) => j.score)) : null,
+                research_high: scores.length ? Math.max(...scores.map((j) => j.score)) : null,
+                research_notes: scores.length,
+                research_by_task: byTask,
+                invented_refs: scores.reduce((a, j) => a + (j.invented_refs ?? 0), 0),
+                f1: tag.length ? round(mean(tag)) : null,
+                f1_low: tag.length ? round(Math.min(...tag)) : null,
+                f1_high: tag.length ? round(Math.max(...tag)) : null,
+                imports: imp.length,
+                imports_right: imp.filter((r) => r.s.import_right).length,
+                imports_asked: imp.filter((r) => r.s.import_right && r.s.asked_first).length,
+                // One import: "right", "asked" (previewed, then asked before applying), "one" or "none" of the identifiers sent.
+                import: imp.length !== 1 ? undefined : imp[0].s.import_right ? (imp[0].s.asked_first ? "asked" : "right") : imp[0].s.ids_sent ? "one" : "none",
+                refused_calls: refused,
+                tool_calls: calls,
+                free: test.free?.includes(m) ?? false,
+                cost: round(cost),
+                cost_per_task: mine.length ? round(cost / mine.length, 3) : null,
+                minutes_per_task: mine.length ? round(mean(mine.map((r) => r.s.seconds ?? 0)) / 60, 1) : null,
+            };
+        }
+        // The page lists the models by research score (the models that did not work last), then by name.
+        const order = Object.keys(models).sort((x, y) => Number(models[y].works) - Number(models[x].works) || (models[y].research ?? 0) - (models[x].research ?? 0) || x.localeCompare(y));
+        const f1s = order.filter((m) => models[m].f1 !== null).map((m) => models[m].f1);
+        out[test.id] = { runs: test.runs, judging: test.judging, order, f1_low: f1s.length ? Math.min(...f1s) : null, f1_high: f1s.length ? Math.max(...f1s) : null, models };
+    }
+    return out;
+}
+// ---------------------------------------------------------------- calibration of the judge
+/** The file a run wrote for a research task (the note, the synthesis, the search list or the review). */
+export function outputFile(runDir) {
+    if (!existsSync(runDir))
+        return undefined;
+    const md = readdirSync(runDir).filter((f) => f.endsWith(".md")).sort();
+    return md.length ? join(runDir, md[0]) : undefined;
+}
+/** A small random generator with a seed, so that a sheet can be made again the same way. */
+function seeded(seed) {
+    let x = seed >>> 0 || 1;
+    return () => ((x = (x * 1664525 + 1013904223) >>> 0) / 2 ** 32);
+}
+/**
+ * A blind scoring sheet for people: n outputs from a judging, balanced over the tasks, under new
+ * labels, without the judge's scores or the model names. sheet-key.json maps the new labels to the
+ * judge's; keep it away from the people who score.
+ */
+export function makeSheet(root, judging, outDir, n = 12, seed = 1) {
+    const judged = JSON.parse(readFileSync(join(root, "judging", judging, "scores.json"), "utf8")).scores;
+    const jkey = JSON.parse(readFileSync(join(root, "judging", judging, "key.json"), "utf8"));
+    const rand = seeded(seed);
+    const tasks = [...new Set(judged.map((j) => j.task))];
+    const pick = [];
+    const pools = tasks.map((t) => judged.filter((j) => j.task === t && outputFile(join(root, jkey[j.label].run, "runs", "", "")) !== null).sort(() => rand() - 0.5));
+    for (let i = 0; pick.length < n && pools.some((p) => p.length); i++) {
+        const p = pools[i % pools.length];
+        const j = p.shift();
+        if (!j)
+            continue;
+        const [dir, code] = jkey[j.label].run.split("/");
+        if (outputFile(join(root, dir, "runs", code, j.task)))
+            pick.push(j);
+    }
+    mkdirSync(outDir, { recursive: true });
+    // A judge's label can name a run with several tasks: the key keeps the label and the task.
+    const key = {};
+    const rows = ["label,task,score,invented_refs,notes"];
+    const used = new Set();
+    for (const j of pick.sort(() => rand() - 0.5)) {
+        let label;
+        do
+            label = `H${Math.floor(100 + rand() * 900)}`;
+        while (used.has(label));
+        used.add(label);
+        key[label] = { label: j.label, task: j.task };
+        const [dir, code] = jkey[j.label].run.split("/");
+        const file = outputFile(join(root, dir, "runs", code, j.task));
+        writeFileSync(join(outDir, `${label}.md`), readFileSync(file, "utf8"));
+        rows.push(`${label},${j.task},,,`);
+    }
+    writeFileSync(join(outDir, "scores.csv"), `${rows.join("\n")}\n`);
+    writeFileSync(join(outDir, "sheet-key.json"), JSON.stringify({ judging, key }, null, 1));
+    writeFileSync(join(outDir, "README.md"), [
+        "# Scoring sheet",
+        "",
+        "Score each file from 1 to 5 (halves allowed), without looking at the other scorers' sheets.",
+        "",
+        "- 5: every claim has a source the note read; the items are covered; quotes and numbers are right.",
+        "- 3: mostly right, with claims without a source or items left out.",
+        "- 1: wrong paper, invented references, or claims the sources do not make.",
+        "",
+        "Count invented references: a citekey not in the library, or a DOI or PMID that does not exist.",
+        "Write your scores in a copy of scores.csv named scores-<your initials>.csv. Do not open sheet-key.json.",
+        "Then: subsub-bench agreement --sheet <this folder>",
+        "",
+    ].join("\n"));
+    return pick.length;
+}
+/** Quadratic-weighted kappa on the half-point scale 1, 1.5, ... 5 (0 = chance agreement, 1 = the same scores). */
+export function weightedKappa(a, b) {
+    const k = 9;
+    const idx = (x) => Math.min(k - 1, Math.max(0, Math.round((x - 1) * 2)));
+    const obs = Array.from({ length: k }, () => new Array(k).fill(0));
+    for (let i = 0; i < a.length; i++)
+        obs[idx(a[i])][idx(b[i])]++;
+    const ra = obs.map((r) => r.reduce((x, y) => x + y, 0));
+    const cb = obs[0].map((_, j) => obs.reduce((x, r) => x + r[j], 0));
+    let num = 0;
+    let den = 0;
+    for (let i = 0; i < k; i++)
+        for (let j = 0; j < k; j++) {
+            const w = ((i - j) / (k - 1)) ** 2;
+            num += w * obs[i][j];
+            den += (w * ra[i] * cb[j]) / a.length;
+        }
+    return den ? round(1 - num / den, 3) : 1;
+}
+/** Agreement between each scorer and the judge, and between scorers. */
+export function agreement(root, sheetDir) {
+    const { judging, key } = JSON.parse(readFileSync(join(sheetDir, "sheet-key.json"), "utf8"));
+    const judged = JSON.parse(readFileSync(join(root, "judging", judging, "scores.json"), "utf8")).scores;
+    const raters = { judge: {} };
+    for (const [h, j] of Object.entries(key)) {
+        const s = judged.find((x) => x.label === j.label && x.task === j.task)?.score;
+        if (s !== undefined)
+            raters.judge[h] = s;
+    }
+    for (const f of readdirSync(sheetDir).filter((f) => /^scores-.+\.csv$/.test(f))) {
+        const name = f.replace(/^scores-|\.csv$/g, "");
+        raters[name] = {};
+        for (const line of readFileSync(join(sheetDir, f), "utf8").split("\n").slice(1)) {
+            const [label, , score] = line.split(",");
+            if (label && score && !Number.isNaN(Number(score)))
+                raters[name][label.trim()] = Number(score);
+        }
+    }
+    const names = Object.keys(raters);
+    const out = [];
+    for (let i = 0; i < names.length; i++)
+        for (let j = i + 1; j < names.length; j++) {
+            const common = Object.keys(raters[names[i]]).filter((l) => l in raters[names[j]]);
+            if (!common.length)
+                continue;
+            const x = common.map((l) => raters[names[i]][l]);
+            const y = common.map((l) => raters[names[j]][l]);
+            const d = x.map((v, k) => Math.abs(v - y[k]));
+            out.push({ a: names[i], b: names[j], n: common.length, mean_abs_diff: round(mean(d)), within_half: round(d.filter((v) => v <= 0.5).length / d.length), kappa: weightedKappa(x, y) });
+        }
+    return out;
+}
 // ---------------------------------------------------------------- CLI
 function arg(argv, name) {
     const i = argv.indexOf(`--${name}`);
@@ -576,22 +856,56 @@ export async function main(argv = process.argv.slice(2)) {
         const t = JSON.parse(readFileSync(join(dir, "tasks.json"), "utf8"));
         const index = JSON.parse(readFileSync(join(dir, "library-index.json"), "utf8"));
         const key = JSON.parse(readFileSync(join(dir, "key.json"), "utf8"));
-        const blind = existsSync(join(dir, "reference.json")) ? JSON.parse(readFileSync(join(dir, "reference.json"), "utf8")) : {};
+        const refs = {};
+        for (const [name, file] of [["blind", "reference.json"], ["library", "library-tags-reference.json"]])
+            if (existsSync(join(dir, file)))
+                refs[name] = JSON.parse(readFileSync(join(dir, file), "utf8"));
         const rows = [];
-        const scorer = starbuckScorer(cfg);
+        // --no-starbuck keeps the reference checks of the last score (they are slow and need the network).
+        const previous = argv.includes("--no-starbuck") && existsSync(join(dir, "results.json")) ? JSON.parse(readFileSync(join(dir, "results.json"), "utf8")) : [];
+        const scorer = argv.includes("--no-starbuck") ? undefined : starbuckScorer(cfg);
         for (const code of Object.keys(key).sort()) {
             for (const role of ["librarian", "researcher"])
                 for (const task of TASKS[role]) {
                     const d = join(dir, "runs", code, task);
                     if (!existsSync(join(d, "summary.json")))
                         continue;
-                    rows.push({ model: key[code], role, task, s: scoreRun(task, d, t, index, blind, scorer) });
+                    const kept = previous.find((r) => r.model === key[code] && r.task === task)?.s.starbuck;
+                    rows.push({ model: key[code], role, task, s: scoreRun(task, d, t, index, refs, scorer ?? (kept ? () => kept : undefined)) });
                 }
         }
         writeFileSync(join(dir, "results.json"), JSON.stringify(rows, null, 1));
         writeFileSync(join(dir, "results.md"), renderResults(rows));
+        for (const w of warnings(rows))
+            console.warn(`Warning: ${w}`);
         console.log(`Wrote ${join(dir, "results.md")}`);
         return;
     }
-    console.log("Usage: subsub-bench prepare | run [--models a,b] [--librarian a,b] [--researcher c,d] [--only task] [--parallel 3] | score  [--dir DIR]");
+    if (cmd === "summary") {
+        const root = join(PACKAGE_DIR, "bench-results");
+        const file = arg(argv, "out") ?? join(PACKAGE_DIR, "site", "src", "_data", "modeltest.json");
+        writeFileSync(file, `${JSON.stringify(summarise(root), null, 1)}\n`);
+        console.log(`Wrote ${file}`);
+        return;
+    }
+    if (cmd === "sheet") {
+        const out = expand(arg(argv, "out") ?? join(process.cwd(), "scoring-sheet"));
+        const n = makeSheet(join(PACKAGE_DIR, "bench-results"), arg(argv, "judging") ?? "2026-10-07-repeats", out, Number(arg(argv, "n") ?? 12), Number(arg(argv, "seed") ?? 1));
+        console.log(`Wrote a sheet of ${n} outputs to ${out}. Give the scorers the .md files, README.md and scores.csv, not sheet-key.json.`);
+        return;
+    }
+    if (cmd === "agreement") {
+        const sheet = arg(argv, "sheet");
+        if (!sheet)
+            throw new Error("Give the sheet folder: subsub-bench agreement --sheet DIR");
+        const rows = agreement(join(PACKAGE_DIR, "bench-results"), expand(sheet));
+        writeFileSync(join(expand(sheet), "agreement.json"), JSON.stringify(rows, null, 1));
+        console.log("| a | b | items | mean difference | within 0.5 | weighted kappa |\n|---|---|---|---|---|---|");
+        for (const r of rows)
+            console.log(`| ${r.a} | ${r.b} | ${r.n} | ${r.mean_abs_diff} | ${r.within_half} | ${r.kappa} |`);
+        if (!rows.length)
+            console.log("No scores yet: add scores-<initials>.csv files to the sheet folder.");
+        return;
+    }
+    console.log("Usage: subsub-bench prepare | run [--models a,b] [--librarian a,b] [--researcher c,d] [--only task] [--parallel 3] | score [--no-starbuck] | summary | sheet [--judging ID] [--n 12] [--out DIR] | agreement --sheet DIR  [--dir DIR]");
 }

package/dist/notes.js

@@ -40,3 +40,93 @@ export function applyEdits(current, edits) {
     }
     return out;
 }
+// ---------------------------------------------------------------- quotes against the full text
+/** The Zotero item key in a note's front matter (zotero: zotero://select/library/items/KEY). */
+export function itemKeyOf(text) {
+    const front = frontMatter(text);
+    const z = front ? field(front, "zotero") : undefined;
+    return z ? /items\/([A-Z0-9]{8})\b/.exec(z)?.[1] : undefined;
+}
+/**
+ * Direct quotes in a note: text in double quotes, and blockquote lines, of 25 characters or more.
+ * Shorter quotes are usually terms, not passages, and are not checked.
+ */
+export function quotesIn(text) {
+    const body = text.replace(FRONT, "");
+    const out = new Set();
+    // An opening mark follows a space or a bracket and a closing mark precedes one, so the text
+    // between one quote and the next ("a" (p. 3); then "b") is not taken for a quote.
+    for (const m of body.matchAll(/(?:^|[\s(\[—–])"([^"\s][^"\n]{23,598}[^"\s])"(?=[\s.,;:)\]!?]|$)/gm))
+        out.add(m[1].trim());
+    for (const m of body.matchAll(/“([^“”\n]{25,600})”/g))
+        out.add(m[1].trim());
+    for (const m of body.matchAll(/^>[ \t]*(.{25,})$/gm)) {
+        // A blockquote may end with its source: "... (p. 12)" or "... [@smith2026, p. 12]".
+        const q = m[1].replace(/\s*(?:\((?:p|pp)\.[^)]*\)|\[@[^\]]*\]|[-—–]\s*p\.?\s*\d+.*)\s*$/i, "").replace(/^["“]|["”]$/g, "").trim();
+        if (q.length >= 25)
+            out.add(q);
+    }
+    return [...out];
+}
+/** Letters and digits only, lower case: PDF text breaks lines, hyphenates words and changes quote marks. */
+export function matchable(s) {
+    return s
+        .normalize("NFKD")
+        .replace(/[̀-ͯ]/g, "")
+        .replace(/(\w)-\s*\n\s*(\w)/g, "$1$2")
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}]+/gu, "");
+}
+/**
+ * The quotes that are not in the full text. A quote with an ellipsis is checked in parts;
+ * square brackets (an editor's insertion) are left out of the comparison.
+ */
+export function missingQuotes(note, fullText, already = []) {
+    const source = matchable(fullText);
+    const old = new Set(already);
+    return quotesIn(note)
+        .filter((q) => !old.has(q))
+        .filter((q) => {
+        const parts = q.split(/\s*(?:\.\.\.|…|\[\s*(?:\.\.\.|…)\s*\])\s*/).map((p) => p.replace(/\[[^\]]*\]/g, " "));
+        return parts.filter((p) => matchable(p).length >= 12).some((p) => !appearsIn(source, p));
+    });
+}
+/**
+ * Whether a passage is in the (matchable) source: whole, or as groups of three words in order with
+ * short gaps, because a PDF's running header or footer can fall in the middle of a sentence.
+ */
+export function appearsIn(source, passage, gap = 300) {
+    if (source.includes(matchable(passage)))
+        return true;
+    const words = passage.split(/\s+/).map(matchable).filter(Boolean);
+    const groups = [];
+    for (let i = 0; i < words.length; i += 3)
+        groups.push(words.slice(i, i + 3).join(""));
+    if (groups.length < 2)
+        return false;
+    for (let start = source.indexOf(groups[0]), tries = 0; start >= 0 && tries < 50; start = source.indexOf(groups[0], start + 1), tries++) {
+        let end = start + groups[0].length;
+        let ok = true;
+        for (const g of groups.slice(1)) {
+            const at = source.indexOf(g, end);
+            if (at < 0 || at - end > gap) {
+                ok = false;
+                break;
+            }
+            end = at + g.length;
+        }
+        if (ok)
+            return true;
+    }
+    return false;
+}
+export function quoteProblem(missing) {
+    if (!missing.length)
+        return undefined;
+    const shown = missing.slice(0, 5).map((q) => `- "${q.length > 160 ? `${q.slice(0, 160)}...` : q}"`);
+    return [
+        `${missing.length} quote(s) in this note are not in the item's full text:`,
+        ...shown,
+        "Copy each quote word for word from the full text (zotero_get_fulltext), or remove the quote marks and say it in your own words. Then write the note again.",
+    ].join("\n");
+}

package/dist/profiles.js

@@ -4,7 +4,8 @@
  * Scholar. A profile is the user's own choice (/profile changes it), not a lock.
  *
  * - reader:  explains each step; reading notes are quotes with page numbers and
- *            questions; no summaries or syntheses; no bulk library changes.
+ *            questions; no summaries or syntheses; /lit gives a reading list;
+ *            no bulk library changes.
  * - scholar: literature notes, syntheses, searches, alerts; all library changes.
  * - author:  scholar plus manuscript work: citation check, bibliography, comments
  *            on the argument. Does not write the manuscript's paragraphs.
@@ -16,15 +17,16 @@ export const PROFILE_SPECS = {
     reader: {
         name: "reader",
         label: "Reader",
-        summary: "explains each step; reading notes are quotes and questions; no summaries; no bulk library changes",
+        summary: "explains each step; reading notes are quotes and questions; /lit gives a reading list; no summaries; no bulk library changes",
         without: [...BULK_LIBRARY, ...MANUSCRIPT],
-        withoutPrompts: ["lit", "compare", "review", "digest"],
+        withoutPrompts: ["compare", "review", "digest"],
         batch: 10,
         rules: [
             "Profile: Reader. {{User}} is learning to work with the literature and wants to do the reading and the writing.",
             "- Before each step, say in one or two lines what you will do and why.",
             "- A literature note is a reading note: the full reference, then direct quotes with page numbers (or section names), and under each quote one question for {{user}} to answer. Do not write summaries, syntheses, conclusions or paragraphs that {{user}} could hand in as their own text. When a prompt or the shared rules ask for a literature note or a synthesis, make a reading note instead and say so.",
             "- For searches, list works with one line each on what they study, not on what they conclude.",
+            "- /lit gives a reading list, not a review: plan, search, screen and read as the prompt says, then write the note in the Reader form that the prompt gives (no answer, no agreement or disagreement, no conclusions).",
             "- If {{user}} asks for a summary, a synthesis or text for an assignment, say that the Reader profile does not write these, and offer quotes and questions. {{User}} can change the profile with /profile.",
             "- Tag review notes: at most 10 items. Explain each proposed tag in the Reason column.",
         ].join("\n"),

package/dist/roles.js

@@ -97,3 +97,22 @@ export function gateKind(toolName, input) {
         return "path";
     return null;
 }
+/** Tools that send a query to an outside service (PubMed, Europe PMC, OpenAlex, Crossref, a web search). */
+const OUTBOUND = new Set(["scholar_search_pubmed", "scholar_search_openalex", "scholar_search_europepmc", "scholar_search_multi", "zotero_find_reference", "zotero_web_search"]);
+export const MAX_QUERY_CHARS = 1500;
+/**
+ * Why an outside search is refused: a query is search terms, so a query that is long or holds
+ * several lines is text from a note, a full text or the library, sent where it should not go
+ * (for example because a document asked for it).
+ */
+export function outboundProblem(toolName, input) {
+    if (!OUTBOUND.has(toolName))
+        return undefined;
+    const queries = [input.query, ...(Array.isArray(input.queries) ? input.queries : [])].filter((q) => typeof q === "string");
+    for (const q of queries) {
+        if (q.length > MAX_QUERY_CHARS || q.split("\n").length > 3) {
+            return `This search query is ${q.length} characters on ${q.split("\n").length} lines. A query is search terms; send no passages of notes, full texts or library records to an outside service. Shorten it to the search terms.`;
+        }
+    }
+    return undefined;
+}