@tiagojct/subsub 0.19.0 → 0.20.0

package/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 Sub-Sub follows the version numbers of npm: a change in the second number (0.13) adds features or changes how you use it; a change in the third (0.13.1) only fixes.
 
+## 0.20.0 (2026-10-23)
+
+Changes from a technical evaluation of 0.19.0 and Starbuck 0.2.0 (LLMSSH 2026).
+
+- Searching the library: when you ask what you have on a topic, Sub-Sub first searches with your own words, then with close synonyms and other word forms, and says which words it searched. In the evaluation it missed items whose titles use a synonym or repeat your words. In a test on the 20 evaluation tasks, recall rose from 0.87 to 0.96 and precision fell from 0.98 to 0.91: the wider search also lists some items that are only near the topic. Check the list.
+- Drafts: Sub-Sub now works from quotes. It reads the full text of each item, chooses a quote for each sentence, and writes the sentence so that it says no more than the quote, with one source for each sentence. A table "Quotes for the check" follows the draft with the quote, word for word, and the page or section for each sentence. A sentence without a quote is cut. In the evaluation, 37% to 53% of the factual sentences in drafts were only partly supported or unsupported by the works they cited. In a test on 18 drafts by two language-model raters, the share fell to 12% to 14% with the new rule. A second test on nine drafts of three other tasks, with the same raters, gave 14% and 16% against 35% and 37% for the old rule; one of the three tasks showed no change. The drafts have more and shorter sentences. Read the table as a way to check, not as a guarantee.
+- Reference checks: a Check with "not found by search" is reported as "not confirmed", with the note that books, theses, reports, archives and works in Portuguese are often missing from Crossref and PubMed. Claim verdicts are reported as sentences to check; Sub-Sub does not change the text or the citation because of a verdict.
+- Guide pages (English and Portuguese): the same two notes on reference checks.
+- Website: a Changes page (English; the Portuguese menu links to it) shows this file.
+- Bench: `retrievalScore` scores a retrieval answer by precision and recall against gold items, naming items by citekey or by title.
+
 ## 0.19.0 (2026-10-16)
 
 A new look for the browser view, the website, the tutorial and the workshop, after the engraved plates of 19th-century natural history books.

package/dist/bench.js

@@ -212,6 +212,37 @@ export function prf(pred, gold) {
     const fn = gold.size - tp;
     return { tp, fp, fn };
 }
+function plainTitle(t) {
+    return t.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+}
+/**
+ * The library items that an answer names. An item is named by its citekey, or by the first 40 characters of its title
+ * (at least 15). A duplicate copy (citekey ending in -dup) counts as its original.
+ */
+export function itemsNamed(text, library) {
+    const plain = plainTitle(text);
+    const found = new Set();
+    for (const item of library) {
+        const key = item.citekey.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+        const title = plainTitle(item.title).slice(0, 40).trim();
+        if (new RegExp(`(?<![A-Za-z0-9_])${key}(?![A-Za-z0-9_])`).test(text) || (title.length >= 15 && plain.includes(title))) {
+            found.add(item.citekey.replace(/-dup$/, ""));
+        }
+    }
+    return found;
+}
+/**
+ * Precision and recall of the items in a retrieval answer against the gold items. Items in `acceptable` are on the topic but
+ * outside the gold list: they count as neither hits nor errors. "(copy)" and "-dup" mark a duplicate and count as the original.
+ */
+export function retrievalScore(text, gold, acceptable, library) {
+    const original = (k) => k.replace(/\s*\(copy\)\s*$/, "").replace(/-dup$/, "");
+    const goldSet = new Set(gold.map(original));
+    const skip = new Set(acceptable.map(original));
+    const said = new Set([...itemsNamed(text, library)].filter((k) => goldSet.has(k) || !skip.has(k)));
+    const { tp, fp, fn } = prf(said, goldSet);
+    return { ...f1(tp, fp, fn), tp, fp, fn };
+}
 function f1(tp, fp, fn) {
     const p = tp + fp ? tp / (tp + fp) : 0;
     const r = tp + fn ? tp / (tp + fn) : 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiagojct/subsub",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "Sub-Sub: a research assistant that works from your own Zotero library, in the browser or the terminal, built on pi",
   "type": "module",
   "license": "MIT",

package/prompts/compare.md

@@ -8,7 +8,7 @@ Make a short slug: lower case, hyphens, at most five words. Use today's date (YY
 
 1. Find the sources.
    - If the user gave citekeys, DOIs or PMIDs, use exactly those.
-   - Otherwise: library first (zotero_find_items), then scholar_search_multi with three phrasings. Choose 3 to 10 sources and say why.
+   - Otherwise: library first (zotero_find_items, with the user's own words and then close synonyms), then scholar_search_multi with three phrasings. Choose 3 to 10 sources and say why.
 2. Read each source before you describe it: zotero_get_fulltext for library items, scholar_read_oa_fulltext (methods and results) for open-access works, scholar_get_work for the abstract of the others. Note whether you read the full text or only the abstract.
 3. Build the matrix, one row per source:
    | # | Source | Claim on the question | Evidence type | Population and setting | Caveats | Confidence |

package/prompts/lit.md

@@ -18,7 +18,7 @@ Make a short slug from the topic: lower case, hyphens, at most five words. Use t
 
 ## 2. Gather
 
-1. Library first: zotero_find_items for the topic (and the topic tags). Note what the library already has.
+1. Library first: zotero_find_items for the topic, with the user's own words first and then with close synonyms and word forms, one call for each (and the topic tags). Note what the library already has.
 2. Outside: scholar_search_multi with three or four phrasings of the question (synonyms, the test or drug name, the condition name, MeSH terms). Give include_terms (the key concepts) and exclude_terms (populations or designs outside the question) so that the server pre-screens the results. Use year limits when the question needs recent work.
 3. Screen. For every work in the merged list, decide: read, maybe, or exclude, with a reason of a few words. Prefer systematic reviews, guidelines and primary studies with the right design. Write the screening log to `Research/.plans/<slug>-screening.md`: a table (#, work, identifier, pre-screen, decision, reason) and the counts for a flow diagram: found by the searches, unique works, pre-screen excluded, screened, read, used.
 4. Read before you summarise:

package/prompts/verify.md

@@ -7,12 +7,12 @@ Check the references of $1 with Starbuck.
 1. If the verify_ tools are not available, tell the user that reference checks need Starbuck: add "addons": ["starbuck"] to the Sub-Sub settings file and start Sub-Sub again. Then stop.
 2. Call verify_check_manuscript with path $1.
 3. Report the summary: how many references failed, need checking, have notes and passed.
-4. For each Fail and each Check, give the citation key, Starbuck's reason and its suggested action. Keep Starbuck's result; do not make it stronger or weaker.
+4. For each Fail and each Check, give the citation key, Starbuck's reason and its suggested action. Keep Starbuck's result; do not make it stronger or weaker. When the reason is that no record was found, write "not confirmed" and say that books, theses, reports, archives and works in Portuguese are often missing from Crossref and PubMed.
 5. List the cited keys without a bibliography entry.
 6. Give the path of the HTML report.
 7. If the second argument is "claims", or the user asks whether the sources support the text:
    1. Call verify_prepare_claims with path $1. Get all pages (offset, limit).
    2. Judge each claim with status ready from its passages only: supported, partly_supported, not_supported or cannot_assess. Copy the passage you rely on character for character into quote.
    3. Call verify_record_claims once, with the verdicts of all claims.
-   4. Report the claims that are not supported or only partly supported, with the line and the reason. Say for each whether the evidence was the full text or the abstract only.
+   4. Report the claims that are not supported or only partly supported as sentences to check, not as errors, with the line and the reason. Say for each whether the evidence was the full text or the abstract only.
 8. Do not change the manuscript. Offer fixes and wait for the user's answer.

package/roles/researcher.md

@@ -30,7 +30,7 @@ You read the Zotero library (zotero_ tools), search PubMed, Europe PMC and OpenA
 
 ## Questions about the library
 
-- "What do I have on X": zotero_find_items (query, tags, fulltext=true when needed). Answer in the chat: citekey, year, one line per item.
+- "What do I have on X": search with zotero_find_items (query, tags, fulltext=true when needed). First search with {{user}}'s own words, as written. Then search again with close synonyms and other word forms (singular and plural, noun and adjective, spelling variants, an older or foreign name), one call for each. Join the results and list each item once. Answer in the chat: citekey, year, one line per item, and the words you searched.
 
 ## Outside search and gaps
 
@@ -103,10 +103,12 @@ If zotero_get_vocabulary has no topic/ tags, propose topics before tagging: read
 2. Propose fixes for missing keys. Do not edit the manuscript text without approval.
 3. scholar_export_bibliography to the file named in the YAML header, or references.json next to the manuscript.
 
-Draft text for {{user}}'s manuscripts or theses only when {{user}} asks. Cite library items as [@citekey], use only claims from sources you read, and mark drafted text clearly so that {{user}} can check it. Never draft text for an assignment (coursework that {{user}} hands in for a grade), even when asked: say so, and offer /read, quotes and questions instead.
+Draft text for {{user}}'s manuscripts or theses only when {{user}} asks. Cite library items as [@citekey], use only claims from sources you read, and mark drafted text clearly so that {{user}} can check it. Work from quotes. Read the full text (zotero_get_fulltext) of each item you will cite. First choose the quotes: for each sentence of the draft, one passage of one source, word for word, with its page or section. Then write each sentence so that it says no more than its quote says: one source for each sentence, no sentence that joins claims from two sources, no figure, date, name or cause that is not in the quote, and no stronger wording than the quote has. End every factual sentence (a fact, date, number, name, cause or comparison) with [@citekey]. If you have only the abstract or the metadata of an item, say so, and use it for no more than it says. After the draft, give a table "Quotes for the check" with one row for each factual sentence: its number, the citekey, the quote, and the page or section. A sentence that you cannot support with a quote is cut. Never draft text for an assignment (coursework that {{user}} hands in for a grade), even when asked: say so, and offer /read, quotes and questions instead.
 
 ## Reference checks (Starbuck, when the verify_ tools are available)
 
 - verify_check_manuscript checks that each cited work exists, matches its citation and was not retracted. Use it on {{user}}'s manuscripts and on your own reports before you deliver them.
 - Report Starbuck's results as Starbuck states them. Do not turn a Check into a Fail, or a Pass into "verified". Give the path of the report.
+- When Starbuck found no record for a reference (a Check with "not found by search"), say "not confirmed", not "wrong". The work may exist: books, theses, reports, archives and works in Portuguese are often not in Crossref or PubMed. Suggest a library catalogue or the publisher's page.
+- Report claim verdicts as sentences for {{user}} to check. Do not change the text or the citation because of a verdict. For a verdict of not supported, write "not supported by the passages I saw" and give the passage.
 - Claim support (level 4) only when {{user}} asks: verify_prepare_claims, then judge each claim from its passages only, then verify_record_claims with all verdicts. Quote the passage you rely on character for character. With an abstract only, answer cannot_assess for details an abstract would not report.

package/src/bench.ts

@@ -262,6 +262,46 @@ export function prf(pred: Set<string>, gold: Set<string>) {
 	return { tp, fp, fn };
 }
 
+/** A library item as the retrieval score needs it: the citekey and the title. */
+export interface LibraryItem {
+	citekey: string;
+	title: string;
+}
+
+function plainTitle(t: string): string {
+	return t.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+}
+
+/**
+ * The library items that an answer names. An item is named by its citekey, or by the first 40 characters of its title
+ * (at least 15). A duplicate copy (citekey ending in -dup) counts as its original.
+ */
+export function itemsNamed(text: string, library: LibraryItem[]): Set<string> {
+	const plain = plainTitle(text);
+	const found = new Set<string>();
+	for (const item of library) {
+		const key = item.citekey.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+		const title = plainTitle(item.title).slice(0, 40).trim();
+		if (new RegExp(`(?<![A-Za-z0-9_])${key}(?![A-Za-z0-9_])`).test(text) || (title.length >= 15 && plain.includes(title))) {
+			found.add(item.citekey.replace(/-dup$/, ""));
+		}
+	}
+	return found;
+}
+
+/**
+ * Precision and recall of the items in a retrieval answer against the gold items. Items in `acceptable` are on the topic but
+ * outside the gold list: they count as neither hits nor errors. "(copy)" and "-dup" mark a duplicate and count as the original.
+ */
+export function retrievalScore(text: string, gold: string[], acceptable: string[], library: LibraryItem[]) {
+	const original = (k: string) => k.replace(/\s*\(copy\)\s*$/, "").replace(/-dup$/, "");
+	const goldSet = new Set(gold.map(original));
+	const skip = new Set(acceptable.map(original));
+	const said = new Set([...itemsNamed(text, library)].filter((k) => goldSet.has(k) || !skip.has(k)));
+	const { tp, fp, fn } = prf(said, goldSet);
+	return { ...f1(tp, fp, fn), tp, fp, fn };
+}
+
 function f1(tp: number, fp: number, fn: number) {
 	const p = tp + fp ? tp / (tp + fp) : 0;
 	const r = tp + fn ? tp / (tp + fn) : 0;