@tiagojct/subsub 0.14.0 → 0.15.0

package/src/notes.ts

@@ -44,3 +44,92 @@ export function applyEdits(current: string, edits: Array<{ oldText?: unknown; ne
 	}
 	return out;
 }
+
+// ---------------------------------------------------------------- quotes against the full text
+
+/** The Zotero item key in a note's front matter (zotero: zotero://select/library/items/KEY). */
+export function itemKeyOf(text: string): string | undefined {
+	const front = frontMatter(text);
+	const z = front ? field(front, "zotero") : undefined;
+	return z ? /items\/([A-Z0-9]{8})\b/.exec(z)?.[1] : undefined;
+}
+
+/**
+ * Direct quotes in a note: text in double quotes, and blockquote lines, of 25 characters or more.
+ * Shorter quotes are usually terms, not passages, and are not checked.
+ */
+export function quotesIn(text: string): string[] {
+	const body = text.replace(FRONT, "");
+	const out = new Set<string>();
+	// An opening mark follows a space or a bracket and a closing mark precedes one, so the text
+	// between one quote and the next ("a" (p. 3); then "b") is not taken for a quote.
+	for (const m of body.matchAll(/(?:^|[\s(\[—–])"([^"\s][^"\n]{23,598}[^"\s])"(?=[\s.,;:)\]!?]|$)/gm)) out.add(m[1].trim());
+	for (const m of body.matchAll(/“([^“”\n]{25,600})”/g)) out.add(m[1].trim());
+	for (const m of body.matchAll(/^>[ \t]*(.{25,})$/gm)) {
+		// A blockquote may end with its source: "... (p. 12)" or "... [@smith2026, p. 12]".
+		const q = m[1].replace(/\s*(?:\((?:p|pp)\.[^)]*\)|\[@[^\]]*\]|[-—–]\s*p\.?\s*\d+.*)\s*$/i, "").replace(/^["“]|["”]$/g, "").trim();
+		if (q.length >= 25) out.add(q);
+	}
+	return [...out];
+}
+
+/** Letters and digits only, lower case: PDF text breaks lines, hyphenates words and changes quote marks. */
+export function matchable(s: string): string {
+	return s
+		.normalize("NFKD")
+		.replace(/[̀-ͯ]/g, "")
+		.replace(/(\w)-\s*\n\s*(\w)/g, "$1$2")
+		.toLowerCase()
+		.replace(/[^\p{L}\p{N}]+/gu, "");
+}
+
+/**
+ * The quotes that are not in the full text. A quote with an ellipsis is checked in parts;
+ * square brackets (an editor's insertion) are left out of the comparison.
+ */
+export function missingQuotes(note: string, fullText: string, already: string[] = []): string[] {
+	const source = matchable(fullText);
+	const old = new Set(already);
+	return quotesIn(note)
+		.filter((q) => !old.has(q))
+		.filter((q) => {
+			const parts = q.split(/\s*(?:\.\.\.|…|\[\s*(?:\.\.\.|…)\s*\])\s*/).map((p) => p.replace(/\[[^\]]*\]/g, " "));
+			return parts.filter((p) => matchable(p).length >= 12).some((p) => !appearsIn(source, p));
+		});
+}
+
+/**
+ * Whether a passage is in the (matchable) source: whole, or as groups of three words in order with
+ * short gaps, because a PDF's running header or footer can fall in the middle of a sentence.
+ */
+export function appearsIn(source: string, passage: string, gap = 300): boolean {
+	if (source.includes(matchable(passage))) return true;
+	const words = passage.split(/\s+/).map(matchable).filter(Boolean);
+	const groups: string[] = [];
+	for (let i = 0; i < words.length; i += 3) groups.push(words.slice(i, i + 3).join(""));
+	if (groups.length < 2) return false;
+	for (let start = source.indexOf(groups[0]), tries = 0; start >= 0 && tries < 50; start = source.indexOf(groups[0], start + 1), tries++) {
+		let end = start + groups[0].length;
+		let ok = true;
+		for (const g of groups.slice(1)) {
+			const at = source.indexOf(g, end);
+			if (at < 0 || at - end > gap) {
+				ok = false;
+				break;
+			}
+			end = at + g.length;
+		}
+		if (ok) return true;
+	}
+	return false;
+}
+
+export function quoteProblem(missing: string[]): string | undefined {
+	if (!missing.length) return undefined;
+	const shown = missing.slice(0, 5).map((q) => `- "${q.length > 160 ? `${q.slice(0, 160)}...` : q}"`);
+	return [
+		`${missing.length} quote(s) in this note are not in the item's full text:`,
+		...shown,
+		"Copy each quote word for word from the full text (zotero_get_fulltext), or remove the quote marks and say it in your own words. Then write the note again.",
+	].join("\n");
+}

package/src/roles.ts

@@ -116,3 +116,23 @@ export function gateKind(toolName: string, input: Record<string, unknown>): Gate
 	if (toolName === "write" || toolName === "edit") return "path";
 	return null;
 }
+
+/** Tools that send a query to an outside service (PubMed, Europe PMC, OpenAlex, Crossref, a web search). */
+const OUTBOUND = new Set(["scholar_search_pubmed", "scholar_search_openalex", "scholar_search_europepmc", "scholar_search_multi", "zotero_find_reference", "zotero_web_search"]);
+export const MAX_QUERY_CHARS = 1500;
+
+/**
+ * Why an outside search is refused: a query is search terms, so a query that is long or holds
+ * several lines is text from a note, a full text or the library, sent where it should not go
+ * (for example because a document asked for it).
+ */
+export function outboundProblem(toolName: string, input: Record<string, unknown>): string | undefined {
+	if (!OUTBOUND.has(toolName)) return undefined;
+	const queries = [input.query, ...(Array.isArray(input.queries) ? input.queries : [])].filter((q): q is string => typeof q === "string");
+	for (const q of queries) {
+		if (q.length > MAX_QUERY_CHARS || q.split("\n").length > 3) {
+			return `This search query is ${q.length} characters on ${q.split("\n").length} lines. A query is search terms; send no passages of notes, full texts or library records to an outside service. Shorten it to the search terms.`;
+		}
+	}
+	return undefined;
+}

package/src/subsub.ts

@@ -24,6 +24,7 @@ import { Bridge, type ServerSpec } from "./bridge.ts";
 import {
 	expand,
 	findUv,
+	FREE_MODELS,
 	isProfile,
 	loadConfig,
 	parseEnvFile,
@@ -39,14 +40,14 @@ import {
 } from "./config.ts";
 import { profileList, profileSpec, promptBlocked } from "./profiles.ts";
 import { coerceArgs, loosenArrays, resolvePathArgs } from "./args.ts";
-import { applyEdits, checkLiteratureNote } from "./notes.ts";
+import { applyEdits, checkLiteratureNote, field, frontMatter, itemKeyOf, missingQuotes, quoteProblem, quotesIn } from "./notes.ts";
 import { buildPolicy, isSecret, judgePath, normalizeToolPath, realish, within } from "./paths.ts";
 import { describeArgs, formatPreview, isEmptyPreview } from "./preview.ts";
 import { perMinuteWait, providerTrouble, TROUBLE_TEXT, TROUBLE_TEXT_PT } from "./keys.ts";
 import { headerLines, type LibraryState, paintPreview, QUOTES, type Scheme, schemeFromAnsi, themeName } from "./look.ts";
 import { systemAddition } from "./prompt.ts";
 import { logUsage } from "./usage.ts";
-import { gateKind, SERVER_PREFIXES, toolsFor, unavailableReason } from "./roles.ts";
+import { gateKind, SERVER_PREFIXES, toolsFor, unavailableReason, outboundProblem } from "./roles.ts";
 
 const MODE_ENTRY = "subsub-mode";
 
@@ -265,6 +266,7 @@ export async function createSubsub(pi: ExtensionAPI, deps: SubsubDeps = {}): Pro
 		emitWeb();
 	}
 
+	let freeNoted = false;
 	/** Set the mode's tools and, unless `switchModel` is false, the mode's model. */
 	async function applyMode(ctx: ExtensionContext, next: Mode, remember: boolean, switchModel = true): Promise<void> {
 		mode = next;
@@ -274,7 +276,25 @@ export async function createSubsub(pi: ExtensionAPI, deps: SubsubDeps = {}): Pro
 			const i = spec.indexOf("/");
 			const model = i > 0 ? ctx.modelRegistry.find(spec.slice(0, i), spec.slice(i + 1)) : undefined;
 			const ok = model ? await pi.setModel(model) : false;
-			if (!ok) ctx.ui.notify(`Sub-Sub: cannot use ${spec} (unknown model or no key); keeping the current model.`, "warning");
+			if (!ok)
+				ctx.ui.notify(
+					L(
+						`Sub-Sub: cannot use ${spec}: no key for that provider, or the provider renamed or retired the model. Keeping the current model. Choose another with the model button (subsub doctor lists the settings).`,
+						`Sub-Sub: não é possível usar ${spec}: não há chave para esse fornecedor, ou o fornecedor mudou o nome do modelo ou retirou-o. Mantém-se o modelo atual. Escolha outro com o botão do modelo (o subsub doctor mostra as definições).`,
+					),
+					"warning",
+				);
+			// The free model is for trying Sub-Sub: its research notes were thin in the model test.
+			else if (mode === "researcher" && spec === FREE_MODELS.researcher && !freeNoted) {
+				freeNoted = true;
+				ctx.ui.notify(
+					L(
+						"Sub-Sub: the free model is good for trying Sub-Sub and for library tasks. In our test its literature notes and reviews were thin; for research you rely on, choose a paid model (see the Models page).",
+						"Sub-Sub: o modelo gratuito serve para experimentar o Sub-Sub e para tarefas de biblioteca. No nosso teste, as suas notas de literatura e revisões foram pobres; para investigação em que precise de confiar, escolha um modelo pago (veja a página Modelos).",
+					),
+					"info",
+				);
+			}
 		}
 		if (remember) pi.appendEntry(MODE_ENTRY, { mode });
 		status(ctx);
@@ -419,6 +439,11 @@ export async function createSubsub(pi: ExtensionAPI, deps: SubsubDeps = {}): Pro
 				return { block: true, reason: why };
 			}
 		}
+		const outbound = outboundProblem(name, input);
+		if (outbound) {
+			if (benchOut) record({ kind: "outbound_blocked", tool: name });
+			return { block: true, reason: outbound };
+		}
 		// grep, find and ls search the working folder when no path is given.
 		if (["read", "grep", "find", "ls"].includes(name) && isSecret(normalizeToolPath(typeof input.path === "string" ? input.path : ".", ctx.cwd), name === "grep", cfg.envFile)) {
 			return { block: true, reason: "That file holds keys or passwords. Sub-Sub does not read it." };
@@ -429,7 +454,7 @@ export async function createSubsub(pi: ExtensionAPI, deps: SubsubDeps = {}): Pro
 		if (benchOut) return benchGate(name, input, kind, ctx.cwd, ctx.signal);
 		if (kind === "path") {
 			const target = normalizeToolPath(String(input.path ?? ""), ctx.cwd);
-			const noteProblem = literatureNoteProblem(name, target, input);
+			const noteProblem = await literatureNoteProblem(name, target, input, ctx.signal);
 			if (noteProblem) return { block: true, reason: noteProblem };
 			const verdict = judgePath(target, buildPolicy({ vault: cfg.vault, cwd: ctx.cwd, serverDir: cfg.serverDir, packageDir: PACKAGE_DIR }));
 			if (verdict.ok) return undefined;
@@ -487,15 +512,49 @@ export async function createSubsub(pi: ExtensionAPI, deps: SubsubDeps = {}): Pro
 		return { block: true, reason: "The library kept changing while the preview was open. Nothing was applied; try again." };
 	});
 
-	/** A literature note (front matter with a citekey) must state its evidence; see notes.ts. */
-	function literatureNoteProblem(name: string, target: string, input: Record<string, unknown>): string | undefined {
+	/**
+	 * A literature note (front matter with a citekey) must state its evidence, and a note written
+	 * from the full text may quote only what the full text says; see notes.ts.
+	 */
+	async function literatureNoteProblem(name: string, target: string, input: Record<string, unknown>, signal?: AbortSignal): Promise<string | undefined> {
 		if (!target.toLowerCase().endsWith(".md")) return undefined;
-		if (name === "write") return checkLiteratureNote(String(input.content ?? ""));
-		if (name === "edit" && Array.isArray(input.edits) && existsSync(target)) {
-			const current = readFileSync(target, "utf8");
-			return checkLiteratureNote(applyEdits(current, input.edits as Array<{ oldText?: unknown; newText?: unknown }>));
+		let text: string | undefined;
+		let before = "";
+		if (name === "write") text = String(input.content ?? "");
+		else if (name === "edit" && Array.isArray(input.edits) && existsSync(target)) {
+			before = readFileSync(target, "utf8");
+			text = applyEdits(before, input.edits as Array<{ oldText?: unknown; newText?: unknown }>);
+		}
+		if (text === undefined) return undefined;
+		const problem = checkLiteratureNote(text);
+		if (problem) return problem;
+		const front = frontMatter(text);
+		const key = itemKeyOf(text);
+		if (!front || !key || field(front, "evidence")?.toLowerCase() !== "full text" || !quotesIn(text).length) return undefined;
+		const full = await fullTextOf(key, signal);
+		// Without the full text (no PDF indexed, Zotero closed) there is nothing to check against.
+		return full ? quoteProblem(missingQuotes(text, full, quotesIn(before))) : undefined;
+	}
+
+	/** The whole indexed full text of an item, read in parts, kept for the session. */
+	const fullTexts = new Map<string, string | null>();
+	async function fullTextOf(key: string, signal?: AbortSignal): Promise<string | undefined> {
+		if (fullTexts.has(key)) return fullTexts.get(key) ?? undefined;
+		let text = "";
+		let offset: number | null = 0;
+		try {
+			for (let part = 0; offset !== null && part < 20; part++) {
+				const res = await current().call("zotero_get_fulltext", { key, offset, max_chars: 60000 }, signal);
+				if (res.isError) break;
+				const data = (res.data ?? {}) as { text?: string; next_offset?: number | null };
+				text += data.text ?? "";
+				offset = data.next_offset ?? null;
+			}
+		} catch {
+			/* no full text: the check is skipped */
 		}
-		return undefined;
+		fullTexts.set(key, text || null);
+		return text || undefined;
 	}
 
 	/**
@@ -528,7 +587,7 @@ export async function createSubsub(pi: ExtensionAPI, deps: SubsubDeps = {}): Pro
 	async function benchGate(name: string, input: Record<string, unknown>, kind: string, cwd: string, signal?: AbortSignal) {
 		if (kind === "path") {
 			const target = normalizeToolPath(String(input.path ?? ""), cwd);
-			const noteProblem = literatureNoteProblem(name, target, input);
+			const noteProblem = await literatureNoteProblem(name, target, input, signal);
 			if (noteProblem) {
 				record({ kind: "note_blocked", tool: name, path: target, reason: noteProblem });
 				return { block: true, reason: noteProblem };

package/web/app.js

@@ -844,10 +844,11 @@ async function refreshSession() {
 function dialogTitle(d) {
 	const raw = String(d.title ?? "").replace(/^Sub-Sub:\s*/, "");
 	let m;
-	if ((m = raw.match(/^apply (\S+)\?$/))) return t("apply", { x: toolLabel(m[1]) });
-	if ((m = raw.match(/^run (\S+)\?$/))) return t("run", { x: toolLabel(m[1]) });
-	if (raw === "undo?") return t("undo");
-	if ((m = raw.match(/^write the Starbuck report to (.+)\?$/))) return t("starbuckReport", { x: m[1] });
+	// The program sends the title in the user's language; both forms get the same friendly title.
+	if ((m = raw.match(/^(?:apply|aplicar) (\S+)\?$/))) return t("apply", { x: toolLabel(m[1]) });
+	if ((m = raw.match(/^(?:run|executar) (\S+)\?$/))) return t("run", { x: toolLabel(m[1]) });
+	if (raw === "undo?" || raw === "anular?") return t("undo");
+	if ((m = raw.match(/^(?:write the Starbuck report to|escrever o relatório do Starbuck em) (.+)\?$/))) return t("starbuckReport", { x: m[1] });
 	if ((m = raw.match(/^(write|edit) (.+)\?$/))) return t("write", { x: m[2] });
 	return raw;
 }

package/web/i18n.js

@@ -143,7 +143,7 @@ export const STRINGS = {
 		ok: "OK",
 		cancel: "Cancelar",
 		close: "Fechar",
-		apply: "Aplicar {x}?",
+		apply: "Confirmar: {x}?",
 		run: "Executar {x}?",
 		undo: "Desfazer esta alteração?",
 		write: "Permitir escrever em {x}?",