@333eco/corpus 2.4.0 → 2.4.1

package/README.md

@@ -83,6 +83,18 @@ verbatim. **A zero result was recording the matcher's limits while being read as
 ⭐ On a zero result the response now names **which terms appear in no document at all**
 (`absent_terms`), so a dead end says something instead of nothing.
 
+⭐ **Diacritics are optional.** `metta` finds what `mettā` finds, and `Tonle Sap` reaches the
+text that writes `Tonlé Sap` — before 2.4.1 a plain-ASCII query found 6 documents where the
+marked one found 20. ⛔ **Fold to match, serve verbatim:** the fold touches only what is
+compared, and every excerpt carries the corpus's own spelling, because a Pāli or Sanskrit
+spelling is a tradition marker rather than a typo. Only Latin combining diacritics are folded;
+Khmer and Burmese vowel signs are marks too, and are left exactly as written.
+
+⭐ **Every result says it is an excerpt.** The text ends with a line naming how many of the
+matches it shows and the `read_documents` call that reads those documents in full. It sits in
+the result rather than only in the server's instructions, because a result reaches the model in
+every client, at the moment it decides whether to keep reading.
+
 ### Reading a whole shelf
 
 ⭐ **Search answers from excerpts, and a model answers from what it opened.** A frontier
@@ -103,6 +115,13 @@ corpus already fits one — but a way to read the shelf.
   split** — half a document verifies nothing — so one larger than the page arrives alone and
   says `over_budget`. Follow `next_cursor`; a cursor from a different corpus version is refused
   rather than serving page 2 of another corpus.
+- **`list_documents` says how to read what it listed:** `read_with` carries the same filters as
+  `read_documents` arguments.
+
+⚠️ **The server can point; it cannot make a model read.** Its instructions say *for a broad
+question, read the relevant shelf in full* — deliberately conditional, because a whole shelf read
+to answer a one-fact question is slower, costlier and often worse. Clients differ in whether they
+show server instructions at all, which is why the pointers also live in the results.
 
 ⭐ **Every document ends with an `[END OF DOCUMENT — <slug> · <n> words]` line**, by every
 route. The server never cuts a document, but a client may, and a cut document reads as a

package/dist/corpus.json

@@ -1,5 +1,5 @@
 {
-  "package_version": "2.4.0",
+  "package_version": "2.4.1",
   "generated_from": [
     "H3/publications",
     "TH/publications",

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@333eco/corpus",
-    "version": "2.4.0",
+    "version": "2.4.1",
     "description": "An MCP server for an open-licensed corpus, served with verifiable provenance — every document carries its sha256, DOI and OpenTimestamps proof so a consuming agent can check its own citation.",
     "license": "CC0-1.0",
     "author": "Thon Ly",

package/src/base-tools.mjs

@@ -27,9 +27,9 @@ export const BASE_TOOLS = [
         title: "Search the corpus",
         annotations: { ...READ_ONLY, title: "Search the corpus" },
         description:
-            "Full-text search across the open-licensed corpus. Returns matching documents with a provenance envelope " +
-            "and a short excerpt around each match — not the full text; call get_document for that. Every result can " +
-            "be independently verified via its sha256, DOI and OpenTimestamps proof.",
+            "Full-text search across the open-licensed corpus; diacritics are optional. Returns matching documents with " +
+            "a provenance envelope and a short excerpt around each match — not the full text; read_documents or " +
+            "get_document return that. Every result can be verified via its sha256, DOI and OpenTimestamps proof.",
         inputSchema: {
             type: "object",
             properties: {

package/src/read-tools.mjs

@@ -65,9 +65,27 @@ export const READ_TOOLS = [
 
 export const READ_TOOL_NAMES = READ_TOOLS.map((t) => t.name);
 
+// ⚠️ CONDITIONAL ON PURPOSE (2.4.1). "Prefer read_documents" unqualified would send a model to
+// read a 260,000-token shelf to answer a one-fact question — slower, costlier and often worse.
+// ⛔ It names shelves by their facets (list_documents' own vocabulary), never by what the
+// documents say: describing the API, not the subject matter.
 export const READ_INSTRUCTIONS =
-    " Search returns excerpts; to answer broadly, read whole documents with read_documents. Every document ends " +
-    "with an [END OF DOCUMENT] line — if it is missing, the text was cut short.";
+    " For a broad question, read the relevant shelf in full: choose it with list_documents, read it with " +
+    "read_documents. Search returns excerpts, suited to finding one passage. Every document ends with an " +
+    "[END OF DOCUMENT] line; if it is missing, the text was cut short.";
+
+/**
+ * list_documents' pointer to reading what it just listed: the same filters, as arguments
+ * read_documents accepts. Only the facets actually given, so the pointer reads back exactly.
+ */
+export const readWith = (args, count) =>
+    count
+        ? {
+              tool: "read_documents",
+              arguments: Object.fromEntries(["genre", "category", "voice", "licence"].filter((k) => args?.[k]).map((k) => [k, args[k]])),
+              note: "Reads every document listed here in full, paged by size."
+          }
+        : null;
 
 // ⚠️ The cursor carries the INDEX VERSION as well as the offset. Pages are offsets
 // into a list, and a list read across a redeploy is two lists: without the version

package/src/search.mjs

@@ -1,8 +1,8 @@
 // How `search_corpus` decides a document matches.
 //
-// ⭐⭐ SHARED, AND IT HAS TO BE. The two surfaces keep their own `callTool` on
-// purpose — twenty lines each, diffable by eye. Matching is not that: it is
-// ranking, tokenising and window-finding, and if the stdio server and the worker
+// ⭐⭐ SHARED, AND IT HAS TO BE — matching and, since 2.4.1, the whole search_corpus
+// tool (see searchCorpus below). Matching is ranking, tokenising and window-finding, and
+// if the stdio server and the worker
 // ever disagreed about which documents answer a query, the corpus would have two
 // opinions about itself. That is the same failure `sync.mjs` exists to prevent,
 // arriving through the search path instead of the data path.
@@ -33,15 +33,49 @@
 // any document containing "b" and "heart" separately — precision traded away for
 // nothing, in the corpus where those marks matter most.
 
+import { structuredWithText } from "./results.mjs";
+
 const TOKEN = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu;
 
+// ⭐⭐ DIACRITICS ARE OPTIONAL IN A QUERY (2.4.1). A model types `metta`; the corpus writes
+// `mettā`. Without a fold that search found 6 documents against 20, and `Tonle Sap` found none
+// — a reader typing plain ASCII, which is most of them, silently missed most of the Pāli
+// material. ⛔ FOLD TO MATCH, SERVE VERBATIM: the spelling is a tradition marker (Pāli for a
+// Theravāda citation, Sanskrit for a Hindu one), so the fold touches only what is COMPARED —
+// every excerpt is cut from the original text.
+// ⚠️ LENGTH-PRESERVING, character by character, so a position in the folded text is the same
+// position in the original and the excerpt window lands where the match is. Only Latin
+// combining diacritics (U+0300–U+036F) are removed, and only where the base letter survives
+// as one character: Khmer and Burmese vowel signs are marks too, and stripping them would
+// make Khmer text match what it does not say.
+const LATIN_MARKS = /[\u0300-\u036f]/g;
+const foldChar = (c) => {
+    const f = c.normalize("NFD").replace(LATIN_MARKS, "");
+    return f.length === c.length ? f : c;
+};
+export const fold = (s) => String(s ?? "").replace(/[^\x00-\x7f]/gu, foldChar);
+
+// The folded, lowercased body, computed once per document text rather than on every search.
+// Keyed by the string itself: the corpus holds one string per document for the life of the
+// process, so the cache is bounded by the corpus.
+const HAY = new Map();
+const hayOf = (body) => {
+    let h = HAY.get(body);
+    if (h === undefined) {
+        h = fold(body.toLowerCase());
+        HAY.set(body, h);
+    }
+    return h;
+};
+const foldQuery = (query) => fold(String(query ?? "").normalize("NFC").toLowerCase());
+
 // Dropped only when something survives the dropping. These are words present in
 // essentially every document, so they never narrow the result set — but they do
 // wreck the tightest-window calculation, because an "and" is always close by.
 const STOP = new Set(["a", "an", "and", "as", "at", "be", "by", "for", "from", "in", "is", "it", "of", "on", "or", "the", "to", "with"]);
 
 export const terms = (query) => {
-    const all = String(query ?? "").toLowerCase().match(TOKEN) ?? [];
+    const all = foldQuery(query).match(TOKEN) ?? [];
     const kept = all.filter((t) => !STOP.has(t));
     return kept.length ? kept : all;
 };
@@ -138,8 +172,8 @@ const tightestWindow = (lists) => {
 // term match's excerpt may NOT contain the literal query, and a caller reading
 // an excerpt deserves to know which question it answered.
 export const match = (body, query, span = 320) => {
-    const hay = body.toLowerCase();
-    const phrase = String(query ?? "").toLowerCase().trim();
+    const hay = hayOf(body);
+    const phrase = foldQuery(query).trim();
 
     if (phrase) {
         const i = hay.indexOf(phrase);
@@ -175,6 +209,57 @@ export const rank = (hits, limit) =>
 export const absentTerms = (documents, query) => {
     const ts = [...new Set(terms(query))];
     if (ts.length === 0) return [];
-    const hays = documents.map((d) => d.text.toLowerCase());
+    const hays = documents.map((d) => hayOf(d.text));
     return ts.filter((t) => !hays.some((h) => h.includes(t)));
 };
+
+/* ---------------------------------------------------------- search_corpus ---
+   ⭐ THE WHOLE TOOL, SHARED (2.4.1). It lived twice, once in each surface's callTool, and
+   its guidance line was about to be written twice too — the shape in which the envelope
+   drifted. One function, imported by both. */
+
+// A search returns EXCERPTS, and a model that reads only excerpts answers from them. So the
+// text a model reads says so at the moment it decides what to do next — in the result, where
+// every client delivers it, rather than only in server instructions a client may drop.
+const readingLine = (results, matches) => {
+    const slugs = results.map((r) => r.slug);
+    return (
+        `[Showing ${results.length} of ${matches} matching document${matches === 1 ? "" : "s"}, as excerpts. ` +
+        `To answer from the full texts, call read_documents with slugs ${JSON.stringify(slugs)}` +
+        (matches > results.length ? `; raise limit to see more of the ${matches} matches` : "") +
+        ".]"
+    );
+};
+
+export const searchCorpus = ({ documents, envelope }, args) => {
+    const q = String(args?.query ?? "");
+    if (!q) throw new Error("query is required");
+    const pool = documents.filter((d) => !args?.genre || d.genre === args.genre);
+    const hits = pool.map((d) => ({ d, m: match(d.text, q) })).filter((h) => h.m !== null);
+    const results = rank(hits, Number(args?.limit ?? 10)).map((h) => ({
+        ...envelope(h.d),
+        genre: h.d.genre,
+        excerpt: h.m.excerpt,
+        // ⭐ HOW it matched, not just that it did: a "terms" excerpt need not
+        // contain the literal query, and a caller reading it should know
+        // which question the excerpt is answering.
+        match: h.m.mode
+    }));
+    // ⚠️ `matches` is the TOTAL found, not the number returned — it used to be
+    // capped at `limit`, which made "10 matches" and "at least 10 matches"
+    // indistinguishable. `returned` carries the capped count.
+    const absent = hits.length ? [] : absentTerms(pool, q);
+    const readable = results.length
+        ? results.map((h) => `${h.slug} — ${h.title}${h.match === "terms" ? "  [all terms, not the phrase]" : ""}\n  ${h.excerpt}`).join("\n\n") +
+          "\n\n" + readingLine(results, hits.length)
+        : absent.length
+          ? `no document matches "${q}". No document contains: ${absent.join(", ")}.`
+          : `no document matches "${q}" — every term appears somewhere, but no single document holds them all. Try fewer terms.`;
+    return structuredWithText(readable, {
+        query: q,
+        matches: hits.length,
+        returned: results.length,
+        ...(absent.length ? { absent_terms: absent } : {}),
+        results: results
+    });
+};

package/src/server.mjs

@@ -40,9 +40,9 @@ import { documentText } from "./resources.mjs";
 import { envelope } from "./envelope.mjs";
 import { PROMPTS, getPrompt } from "./prompts.mjs";
 import { BASE_TOOLS, withFacets } from "./base-tools.mjs";
-import { READ_TOOLS, READ_TOOL_NAMES, READ_INSTRUCTIONS, filterDocuments, readDocuments, completenessOf, manifestProblems } from "./read-tools.mjs";
+import { READ_TOOLS, READ_TOOL_NAMES, READ_INSTRUCTIONS, filterDocuments, readDocuments, readWith, completenessOf, manifestProblems } from "./read-tools.mjs";
 import { PROGRAM_TOOLS, PROGRAM_TOOL_NAMES, PROGRAM_INSTRUCTIONS, callProgramTool } from "./program-tools.mjs";
-import { match, rank, absentTerms } from "./search.mjs";
+import { searchCorpus } from "./search.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const INDEX = resolve(HERE, "..", "dist", "corpus.json");
@@ -121,35 +121,9 @@ const text = (value) => ({ content: [{ type: "text", text: typeof value === "str
 
 const callTool = (name, args) => {
     if (name === "search_corpus") {
-        const q = String(args?.query ?? "");
-        if (!q) throw new Error("query is required");
-        const pool = corpus.documents.filter((d) => !args?.genre || d.genre === args.genre);
-        const hits = pool.map((d) => ({ d, m: match(d.text, q) })).filter((h) => h.m !== null);
-        const results = rank(hits, Number(args?.limit ?? 10)).map((h) => ({
-            ...envelope(h.d),
-            genre: h.d.genre,
-            excerpt: h.m.excerpt,
-            // ⭐ HOW it matched, not just that it did: a "terms" excerpt need not
-            // contain the literal query, and a caller reading it should know
-            // which question the excerpt is answering.
-            match: h.m.mode
-        }));
-        // ⚠️ `matches` is the TOTAL found, not the number returned — it used to be
-        // capped at `limit`, which made "10 matches" and "at least 10 matches"
-        // indistinguishable. `returned` carries the capped count.
-        const absent = hits.length ? [] : absentTerms(pool, q);
-        const readable = results.length
-            ? results.map((h) => `${h.slug} — ${h.title}${h.match === "terms" ? "  [all terms, not the phrase]" : ""}\n  ${h.excerpt}`).join("\n\n")
-            : absent.length
-              ? `no document matches "${q}". No document contains: ${absent.join(", ")}.`
-              : `no document matches "${q}" — every term appears somewhere, but no single document holds them all. Try fewer terms.`;
-        return structuredWithText(readable, {
-            query: q,
-            matches: hits.length,
-            returned: results.length,
-            ...(absent.length ? { absent_terms: absent } : {}),
-            results: results
-        });
+        // Shared with the other surface (search.mjs): the answer, the excerpts and the
+        // reading line cannot differ between them.
+        return searchCorpus({ documents: corpus.documents, envelope }, args);
     }
 
     if (name === "get_document") {
@@ -233,6 +207,8 @@ const callTool = (name, args) => {
             ...(corpus.voices ? { voices: corpus.voices } : {}),
             // A87: the manifest's counts and reasons beside the listing they account for.
             ...(corpus.manifest ? { completeness: completenessOf(corpus) } : {}),
+            // 2.4.1: what to call to read this listing in full — the same filters, as arguments.
+            ...(readWith(args, list.length) ? { read_with: readWith(args, list.length) } : {}),
             documents: list
         });
     }