@333eco/corpus 2.0.0 → 2.1.1

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@333eco/corpus",
-    "version": "2.0.0",
-    "description": "An MCP server for an open-licensed corpus, served with verifiable provenance \u2014 every document carries its sha256, DOI and OpenTimestamps proof so a consuming agent can check its own citation.",
+    "version": "2.1.1",
+    "description": "An MCP server for an open-licensed corpus, served with verifiable provenance — every document carries its sha256, DOI and OpenTimestamps proof so a consuming agent can check its own citation.",
     "license": "CC0-1.0",
     "author": "Thon Ly",
     "homepage": "https://thonly.org/research",

package/src/report.mjs

@@ -0,0 +1,104 @@
+// `--report-gap` and `--report-bug` — the only place this package makes an
+// outbound request, and it is reached only by typing one of the flags.
+//
+// ⭐⭐ WHY THIS IS A COMMAND AND NOT TELEMETRY, because the distinction is the
+// whole design. The useful signal from a corpus server is *what someone looked
+// for and did not find*. The remote endpoint at corpus.333.eco collects that
+// from its own callers as a property of being the server they called. This
+// package runs on YOUR machine, so the same collection there would be an
+// outbound report about your private reading — and the guard against that is
+// not a consent flag or an opt-out. It is that THE SERVER PATH CANNOT REACH
+// THIS FILE: `server.mjs` loads it with a dynamic import inside the argv branch,
+// so during normal operation the module is never even read off disk.
+//
+// ⚠️ THE CHECKABLE CLAIM CHANGED SHAPE WHEN THIS FILE WAS ADDED, AND THAT IS
+// WORTH STATING PLAINLY. Before it, "this package makes no network call" was
+// verifiable by `grep -r fetch src/` returning nothing at all — the strongest
+// kind of evidence, because it needs no reasoning. Now the honest claim is
+// narrower: there is exactly ONE fetch in the package, it lives in this file,
+// and this file is imported from exactly one place — a branch that requires an
+// explicit flag. Still inspectable in under a minute, but it is a chain of two
+// facts rather than one absence. ⛔ If a second import of this module ever
+// appears, that chain is broken and the claim must be rewritten rather than
+// repeated.
+//
+// ⛔ IT REPORTS TO THE WORKER, NOT TO THE NOTIFICATION BEACON. Sending to
+// thonly.org/api/track would mean shipping every user of this package a working
+// recipe for writing into the founder's admin notification channel, behind an
+// Origin header a CLI can trivially assert. The worker's /gap endpoint writes to
+// Analytics Engine and pushes nothing at anyone.
+
+const ENDPOINT = "https://corpus.333.eco/report";
+
+// ⚠️ A BUG REPORT AND A GAP REPORT CARRY THE SAME PAYLOAD, DELIBERATELY. It is
+// tempting to attach a node version and a platform to a bug — genuinely useful
+// to whoever fixes it — but that would give this command two different promises
+// about what it sends, and the promise is the valuable part. Anything about your
+// environment that matters, put in the text; then you have said it on purpose.
+const KINDS = {
+    gap: {
+        prompt: "what you looked for and did not find",
+        thanks: "Thank you — recorded. It joins the gaps the hosted endpoint already sees."
+    },
+    bug: {
+        prompt: "what went wrong, and what you expected instead",
+        thanks: "Thank you — recorded. Issues are also welcome at github.com/333eco/corpus.333.eco/issues."
+    }
+};
+const MAX = 200;
+
+// ⭐ WHAT IS SENT IS THE WHOLE OF WHAT IS SENT. The text you typed, and the
+// version of the corpus you have. No machine id, no username, no hostname, no
+// path, no timestamp of your own — and the receiving end deliberately does not
+// record the country it could resolve, because a voluntary note about a missing
+// document has no use for where the sender was standing.
+export const report = async (kind, text, version) => {
+    const spec = KINDS[kind];
+    if (!spec) throw new Error(`unknown report kind: ${kind}`);
+    const body = String(text ?? "").trim().slice(0, MAX);
+    if (!body) {
+        console.error(`usage: corpus-mcp --report-${kind} "${spec.prompt}"`);
+        return 1;
+    }
+
+    const payload = { kind, text: body, version: version ?? null };
+
+    // Printed BEFORE the request, not after, so the disclosure is not
+    // contingent on the request succeeding.
+    console.log("Sending this, and nothing else:\n");
+    console.log("  " + JSON.stringify(payload));
+    console.log("\n  to " + ENDPOINT + "\n");
+
+    try {
+        const res = await fetch(ENDPOINT, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(payload)
+        });
+        // ⚠️⚠️ THE STATUS CODE IS NOT THE CONTRACT, AND TRUSTING IT REPORTED A
+        // FALSE SUCCESS ON THE FIRST RUN. A server predating /gap treats any
+        // POST without a JSON-RPC `id` as a NOTIFICATION and answers 202 with an
+        // empty body — so `res.ok` was true, and this printed "recorded" while
+        // nothing had been. The success signal must therefore be something only
+        // the real handler can produce: an explicit `ok` in the body. That also
+        // makes version skew safe in both directions, since an old server can
+        // never accidentally satisfy it.
+        const ack = await res.json().catch(() => null);
+        if (res.ok && ack?.ok === true) {
+            console.log(spec.thanks);
+            return 0;
+        }
+        console.error(
+            res.ok
+                ? "The endpoint accepted the request but did not confirm it recorded anything —\n" +
+                  "it is probably running a version without /report. Nothing was recorded."
+                : `The endpoint answered ${res.status}. Nothing was recorded.`
+        );
+        return 1;
+    } catch (e) {
+        // A failure here is worth nobody's day. Say so and exit cleanly.
+        console.error(`Could not reach ${ENDPOINT}: ${e.message}`);
+        console.error("Nothing was sent. This is entirely optional — carry on.");
+        return 1;
+    }
+};

package/src/search.mjs

@@ -0,0 +1,176 @@
+// How `search_corpus` decides a document matches.
+//
+// ⭐⭐ SHARED, AND IT HAS TO BE. The two surfaces keep their own `callTool` on
+// purpose — twenty lines each, diffable by eye. Matching is not that: it is
+// ranking, tokenising and window-finding, and if the stdio server and the worker
+// ever disagreed about which documents answer a query, the corpus would have two
+// opinions about itself. That is the same failure `sync.mjs` exists to prevent,
+// arriving through the search path instead of the data path.
+//
+// ⚠️ WHY THIS REPLACED A BARE `indexOf`, and the evidence was a real caller.
+// The first external client to reach the hosted endpoint searched
+// `"gratitude alignment human wellbeing kindness"` and got ZERO results — while
+// `gratitude` alone returns 50, `alignment` 50, `kindness` 50, `wellbeing` 16.
+// The corpus was not missing the material; the matcher required that exact
+// five-word string to appear verbatim, which of course it never does. A literal
+// substring search silently punishes anyone who types a sentence, and it makes
+// the zero-result signal MEAN THE WRONG THING: those queries were recording the
+// matcher's limits while being read as gaps in the corpus.
+//
+// ⭐ PHRASE FIRST, THEN TERMS — the order is what keeps precision. This corpus is
+// full of hyphenated marks (`B-Heart`, `Re-Tip`, `B-Sey`), and an exact phrase is
+// always the strongest evidence a document is about the thing asked for. So an
+// exact hit wins and is ranked above every term match. Only when the phrase is
+// absent do we ask the weaker question: does this document contain ALL of these
+// words, anywhere?
+//
+// ⚠️ The tokeniser KEEPS intra-word hyphens and apostrophes, so `B-Heart` is one
+// token and not two. Splitting them would have made a search for `B-Heart` match
+// any document containing "b" and "heart" separately — precision traded away for
+// nothing, in the corpus where those marks matter most.
+
+const TOKEN = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu;
+
+// Dropped only when something survives the dropping. These are words present in
+// essentially every document, so they never narrow the result set — but they do
+// wreck the tightest-window calculation, because an "and" is always close by.
+const STOP = new Set(["a", "an", "and", "as", "at", "be", "by", "for", "from", "in", "is", "it", "of", "on", "or", "the", "to", "with"]);
+
+export const terms = (query) => {
+    const all = String(query ?? "").toLowerCase().match(TOKEN) ?? [];
+    const kept = all.filter((t) => !STOP.has(t));
+    return kept.length ? kept : all;
+};
+
+// ⚠️ THE EDGES SNAP TO WHITESPACE, because a fixed-width slice cuts words in
+// half — `…ratitude is immen…` — and a half-word at an excerpt boundary is not a
+// cosmetic problem here. These excerpts are read by agents deciding whether a
+// document is worth fetching, and a truncated token is a token that can be
+// matched, quoted or reasoned about as if it were a word.
+//
+// ⚠️ THE SNAP IS BUDGETED, AND THE BUDGET IS THE WHOLE SAFETY OF IT. A corpus
+// document can contain a 200-character unbroken run — a base64 blob, a sha256, a
+// long URL — and an unbounded search for whitespace would eat the entire excerpt
+// looking for a space that is not there. Past LOOK characters we accept the hard
+// cut: a slightly ugly excerpt beats an empty one.
+const LOOK = 40;
+
+const clip = (body, centre, span) => {
+    if (body.length <= span) return body.trim();
+
+    let start = Math.max(0, Math.min(Math.round(centre - span / 2), body.length - span));
+    let end = start + span;
+
+    // ⭐ EACH EDGE TRIES BOTH DIRECTIONS, and the second direction is what fixes
+    // the long-token case. Snapping INWARD is preferred (it never grows the
+    // excerpt), but this corpus is full of runs that exceed LOOK with no space
+    // in them at all — `project_future_kindness_operating_noun`, bare URLs,
+    // markdown link targets, sha256 hex. Inward alone gave up on exactly those
+    // and left the half-word it was meant to prevent. Snapping OUTWARD instead
+    // costs at most LOOK extra characters and always lands on a real boundary.
+    if (start > 0) {
+        const ahead = body.slice(start, start + LOOK).search(/\s/);
+        if (ahead !== -1) {
+            start += ahead + 1;
+        } else {
+            const back = /\s\S*$/.exec(body.slice(Math.max(0, start - LOOK), start));
+            if (back) start = Math.max(0, start - LOOK) + back.index + 1;
+        }
+    }
+    if (end < body.length) {
+        const from = Math.max(start + 1, end - LOOK);
+        const back = /\s\S*$/.exec(body.slice(from, end));
+        if (back) {
+            end = from + back.index;
+        } else {
+            const ahead = body.slice(end, end + LOOK).search(/\s/);
+            end = ahead !== -1 ? end + ahead : Math.min(body.length, end + LOOK);
+        }
+    }
+    if (end <= start) end = Math.min(body.length, start + span); // pathological input
+
+    return (start > 0 ? "…" : "") + body.slice(start, end).trim() + (end < body.length ? "…" : "");
+};
+
+// Positions of a term, capped: a common word can appear thousands of times and
+// the window sweep only needs enough of them to find a tight one.
+const positions = (hay, term, cap = 200) => {
+    const out = [];
+    let i = hay.indexOf(term);
+    while (i !== -1 && out.length < cap) {
+        out.push(i);
+        i = hay.indexOf(term, i + term.length);
+    }
+    return out;
+};
+
+// Smallest span containing at least one occurrence of every term. Classic sweep
+// over the merged, sorted occurrence list — linear in the number of positions.
+const tightestWindow = (lists) => {
+    const merged = [];
+    lists.forEach((ps, t) => ps.forEach((p) => merged.push([p, t])));
+    merged.sort((a, b) => a[0] - b[0]);
+    const need = lists.length;
+    const seen = new Map();
+    let best = null;
+    let left = 0;
+    for (let right = 0; right < merged.length; right++) {
+        seen.set(merged[right][1], (seen.get(merged[right][1]) ?? 0) + 1);
+        while (seen.size === need) {
+            const width = merged[right][0] - merged[left][0];
+            if (!best || width < best.width) best = { width, from: merged[left][0], to: merged[right][0] };
+            const t = merged[left][1];
+            const n = seen.get(t) - 1;
+            if (n === 0) seen.delete(t);
+            else seen.set(t, n);
+            left++;
+        }
+    }
+    return best;
+};
+
+// null when the document does not match; otherwise the excerpt, how it matched,
+// and a score for ranking. ⭐ `mode` is returned rather than hidden because a
+// term match's excerpt may NOT contain the literal query, and a caller reading
+// an excerpt deserves to know which question it answered.
+export const match = (body, query, span = 320) => {
+    const hay = body.toLowerCase();
+    const phrase = String(query ?? "").toLowerCase().trim();
+
+    if (phrase) {
+        const i = hay.indexOf(phrase);
+        // An exact hit always wins: score is above anything a window can score.
+        if (i !== -1) return { mode: "phrase", excerpt: clip(body, i + phrase.length / 2, span), score: 1e9 };
+    }
+
+    const ts = [...new Set(terms(query))];
+    // A single term has already been tried as a phrase; there is no weaker
+    // question left to ask, so a miss is a miss.
+    if (ts.length < 2) return null;
+
+    const lists = ts.map((t) => positions(hay, t));
+    if (lists.some((l) => l.length === 0)) return null; // AND, not OR
+
+    const win = tightestWindow(lists);
+    if (!win) return null;
+    // Tighter is better; every term match ranks below every phrase match.
+    return { mode: "terms", excerpt: clip(body, Math.floor((win.from + win.to) / 2), span), score: 1e6 / (1 + win.width) };
+};
+
+// Rank and cut. Phrase matches first, then tighter term windows, corpus order
+// breaking ties (Array.prototype.sort is stable).
+export const rank = (hits, limit) =>
+    hits.slice().sort((a, b) => b.m.score - a.m.score).slice(0, Math.max(0, limit));
+
+// ⭐⭐ WHICH TERMS APPEAR IN NO DOCUMENT AT ALL. Under AND-matching a zero result
+// means at least one term is missing, and saying WHICH turns a dead end into a
+// finding: `unicorn` absent is a fact about the corpus, while `wellbeing` absent
+// would be a fact about the query. Computed only when a search returns nothing,
+// so the cost never lands on a successful call — and it is exactly the sentence
+// a caller would otherwise have to write to us by hand.
+export const absentTerms = (documents, query) => {
+    const ts = [...new Set(terms(query))];
+    if (ts.length === 0) return [];
+    const hays = documents.map((d) => d.text.toLowerCase());
+    return ts.filter((t) => !hays.some((h) => h.includes(t)));
+};

package/src/server.mjs

@@ -40,12 +40,38 @@ import { provenanceHeader } from "./resources.mjs";
 import { PROMPTS, getPrompt } from "./prompts.mjs";
 import { BASE_TOOLS } from "./base-tools.mjs";
 import { PROGRAM_TOOLS, PROGRAM_TOOL_NAMES, PROGRAM_INSTRUCTIONS, callProgramTool } from "./program-tools.mjs";
+import { match, rank, absentTerms } from "./search.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const INDEX = resolve(HERE, "..", "dist", "corpus.json");
 
 const log = (...a) => console.error("[corpus-mcp]", ...a);
 
+// ⭐⭐ THE ONE NON-SERVING PATH, AND IT IS GUARDED BY BEING A SEPARATE MODULE.
+// `--report-gap "<text>"` sends a voluntary note about something this corpus does
+// not contain. It is a COMMAND, never telemetry: nothing here runs unless the
+// flag is typed, and the reporter is loaded with a DYNAMIC IMPORT so the serving
+// path does not so much as read the file off disk. ⛔ Never import report-gap.mjs
+// at the top of this file — a static import would put a fetch in the module graph
+// of every session, which is exactly the property this arrangement preserves.
+// ⚠️ Handled before the index check on purpose: reporting a gap must not require
+// a built corpus, since "there is no corpus here" is itself a reportable gap.
+// ⭐ Both flags share ONE module and ONE dynamic import, so adding the second
+// kind did not add a second way into the network. `--report-gap` says what the
+// corpus is missing; `--report-bug` says what this server got wrong.
+const REPORTS = { "--report-gap": "gap", "--report-bug": "bug" };
+const flag = process.argv.find((a) => a in REPORTS);
+if (flag) {
+    const { report } = await import("./report.mjs");
+    let version = null;
+    try {
+        version = JSON.parse(readFileSync(resolve(HERE, "..", "package.json"), "utf8")).version;
+    } catch {
+        // Version is a convenience for whoever reads the report, never required.
+    }
+    process.exit(await report(REPORTS[flag], process.argv[process.argv.indexOf(flag) + 1], version));
+}
+
 if (!existsSync(INDEX)) {
     log("dist/corpus.json is missing. Build it: node scripts/build-index.mjs --from <corpus repos>");
     process.exit(1);
@@ -137,28 +163,38 @@ const KNOWN_TOOLS = new Set([...BASE_TOOLS, ...PROGRAM_TOOLS].map((t) => t.name)
 
 const text = (value) => ({ content: [{ type: "text", text: typeof value === "string" ? value : JSON.stringify(value, null, 2) }] });
 
-const excerpt = (body, query, span = 320) => {
-    const i = body.toLowerCase().indexOf(query.toLowerCase());
-    if (i === -1) return null;
-    const from = Math.max(0, i - span / 2);
-    return (from > 0 ? "…" : "") + body.slice(from, from + span).trim() + (from + span < body.length ? "…" : "");
-};
 
 const callTool = (name, args) => {
     if (name === "search_corpus") {
         const q = String(args?.query ?? "");
         if (!q) throw new Error("query is required");
-        const limit = Number(args?.limit ?? 10);
-        const hits = corpus.documents
-            .filter((d) => !args?.genre || d.genre === args.genre)
-            .map((d) => ({ d, ex: excerpt(d.text, q) }))
-            .filter((h) => h.ex !== null)
-            .slice(0, limit)
-            .map((h) => ({ ...envelope(h.d), genre: h.d.genre, excerpt: h.ex }));
-        const readable = hits.length
-            ? hits.map((h) => `${h.slug} — ${h.title}\n  ${h.excerpt}`).join("\n\n")
-            : `no document matches "${q}".`;
-        return structuredWithText(readable, { query: q, matches: hits.length, results: hits });
+        const pool = corpus.documents.filter((d) => !args?.genre || d.genre === args.genre);
+        const hits = pool.map((d) => ({ d, m: match(d.text, q) })).filter((h) => h.m !== null);
+        const results = rank(hits, Number(args?.limit ?? 10)).map((h) => ({
+            ...envelope(h.d),
+            genre: h.d.genre,
+            excerpt: h.m.excerpt,
+            // ⭐ HOW it matched, not just that it did: a "terms" excerpt need not
+            // contain the literal query, and a caller reading it should know
+            // which question the excerpt is answering.
+            match: h.m.mode
+        }));
+        // ⚠️ `matches` is the TOTAL found, not the number returned — it used to be
+        // capped at `limit`, which made "10 matches" and "at least 10 matches"
+        // indistinguishable. `returned` carries the capped count.
+        const absent = hits.length ? [] : absentTerms(pool, q);
+        const readable = results.length
+            ? results.map((h) => `${h.slug} — ${h.title}${h.match === "terms" ? "  [all terms, not the phrase]" : ""}\n  ${h.excerpt}`).join("\n\n")
+            : absent.length
+              ? `no document matches "${q}". No document contains: ${absent.join(", ")}.`
+              : `no document matches "${q}" — every term appears somewhere, but no single document holds them all. Try fewer terms.`;
+        return structuredWithText(readable, {
+            query: q,
+            matches: hits.length,
+            returned: results.length,
+            ...(absent.length ? { absent_terms: absent } : {}),
+            results: results
+        });
     }
 
     if (name === "get_document") {