@333eco/corpus 1.2.5 → 2.1.0

package/src/resources.mjs

@@ -0,0 +1,215 @@
+// MCP resources over the corpus — shared by both surfaces.
+//
+// ⭐⭐ THE ONE DECISION THAT SHAPES THIS FILE: A RESOURCE CARRIES ITS PROVENANCE
+// IN THE TEXT, NOT BESIDE IT. A tool response wraps a document in an envelope and
+// a caller reads the envelope. A resource is different in kind — clients hand its
+// contents straight to a model as context, and a `mimeType` field does not travel
+// with a quotation. Serving bare text here would hand out corpus material stripped
+// of the one property this server exists to provide.
+//
+// ⚠️ This is not a new judgement; it is the letters' rule applied a second time.
+// The letters mark voice INLINE — "[VERBATIM — Thon Ly]" / "[SCAFFOLD …]" — rather
+// than in a metadata field, because *with a field an agent must LOOK to know; with
+// a marker it must STRIP not to*. The same asymmetry decides this: a header the
+// model must delete is safe, a field it must consult is not.
+//
+// ⚠️⚠️ AND THE HEADER MUST SAY WHAT THE HASH DOES NOT COVER. `provenance.sha256`
+// is computed over the whole source file, front matter included; `text` is the
+// body. Prepending a header makes the served bytes hash to nothing at all — so
+// the header states plainly that it is not part of the hashed artifact and points
+// at the file that is. A "verifiable" resource that quietly cannot be verified
+// would be worse than one that never claimed it.
+
+const CORPUS_SCHEME = "corpus://";
+const PAGE = 50;
+
+export const RESOURCE_TEMPLATES = [
+    {
+        uriTemplate: "corpus://{slug}",
+        name: "corpus-document",
+        title: "Corpus document by slug",
+        description:
+            "Any document in the corpus, addressed by its slug — e.g. corpus://co-presence-gated-redemption. " +
+            "Returns the canonical text prefixed with a provenance header carrying its licence, sha256, DOI and " +
+            "a runnable verification command. Slugs come from resources/list or the list_documents tool.",
+        mimeType: "text/markdown"
+    }
+];
+
+const mime = (d) => (d.metadata_convention === "html" ? "text/plain" : "text/markdown");
+
+// The catalogue entry. Deliberately terse: a client renders this in a picker, and
+// the licence is the one thing a person choosing a document needs to see.
+export const resourceEntry = (d) => ({
+    uri: CORPUS_SCHEME + d.slug,
+    name: d.slug,
+    title: d.title,
+    description:
+        `${d.genre} · ${d.licence.id}` +
+        (d.date ? ` · ${d.date}` : "") +
+        (d.provenance.doi ? ` · doi:${d.provenance.doi}` : "") +
+        (d.subtitle ? ` — ${d.subtitle}` : ""),
+    mimeType: mime(d),
+    // ⚠️ `lastModified` carries the document's own date and NOTHING MORE PRECISE.
+    // ISO 8601 permits a date alone, and inventing "T00:00:00Z" would assert a
+    // time this corpus does not record — a small lie in a provenance server.
+    annotations: {
+        audience: ["user", "assistant"],
+        // Anchored papers rank above unanchored scaffold: a consumer choosing
+        // among 140 documents should meet the ones with proofs first.
+        priority: d.provenance.doi ? 0.9 : d.provenance.opentimestamps ? 0.6 : 0.4,
+        ...(d.date ? { lastModified: d.date } : {})
+    }
+});
+
+export const listResources = (documents, cursor) => {
+    // Cursor-based paging, because a corpus grows and a client should not have to
+    // take 140 entries to find one. The cursor is an offset encoded as a string —
+    // opaque to the client, which is all the protocol asks of it.
+    // ⚠️ btoa/atob, NOT Buffer. The worker runs without nodejs_compat, so Buffer is
+    // undefined there and only there — a break that passes every local test and
+    // fails once, in production, on the surface nobody runs by hand.
+    let start = 0;
+    if (cursor) {
+        try {
+            start = Number(atob(String(cursor)));
+        } catch {
+            throw new Error(`invalid cursor: ${cursor}`);
+        }
+    }
+    if (!Number.isInteger(start) || start < 0 || start > documents.length) {
+        throw new Error(`invalid cursor: ${cursor}`);
+    }
+    const page = documents.slice(start, start + PAGE);
+    const next = start + PAGE < documents.length ? btoa(String(start + PAGE)) : undefined;
+    return { resources: page.map(resourceEntry), ...(next ? { nextCursor: next } : {}) };
+};
+
+// ⚠️ Attribution is stated as an instruction, not a licence id. CC-BY on seven of
+// these documents means a real obligation, and "CC-BY-4.0" alone leaves an agent
+// to know what that entails.
+const licenceLine = (d) =>
+    d.licence.attribution_required
+        ? `${d.licence.id} — ATTRIBUTION REQUIRED. Attribute to: ${d.authors ?? "⚠️ UNKNOWN — the index carries no author for this document; do not quote it until that is fixed"}. ${d.licence.url}`
+        : `${d.licence.id} — no attribution required, though citation is welcome. ${d.licence.url}`;
+
+export const provenanceHeader = (d) => {
+    const p = d.provenance;
+    const L = [];
+    L.push("[PROVENANCE — corpus.333.eco. This header is NOT part of the document; the document begins below.]");
+    L.push(`title:   ${d.title}`);
+    L.push(`slug:    ${d.slug}   (${d.genre}${d.date ? `, ${d.date}` : ""})`);
+    L.push(`licence: ${licenceLine(d)}`);
+    if (p.source_url) L.push(`source:  ${p.source_url}`);
+    if (p.canonical_url) L.push(`canonical: ${p.canonical_url}`);
+    L.push(`sha256:  ${p.sha256}`);
+    // The single most misreadable field, so it gets a full sentence.
+    L.push("         ⚠️ covers the COMPLETE SOURCE FILE at `source`, metadata block included —");
+    L.push("            NOT the text below, and NOT this header. Hashing what you were served");
+    L.push("            will not reproduce it. Fetch `source` to check.");
+    if (p.doi) L.push(`doi:     https://doi.org/${p.doi}   (this exact version)`);
+    if (p.concept_doi) L.push(`concept: https://doi.org/${p.concept_doi}   (follows the document across versions)`);
+    if (p.opentimestamps) L.push(`proof:   ots verify ${d.path}.ots   — in the source repository, anchored in Bitcoin`);
+    if (p.source_url) L.push(`verify:  curl -sL ${p.source_url} | shasum -a 256   # compare with sha256 above`);
+
+    if (p.deposited_matches_current === false) {
+        L.push("status:  ⚠️ REVISED SINCE ITS DEPOSIT. The DOI above resolves to the deposited");
+        L.push("            version; the text below is newer. They differ legitimately.");
+    }
+    if (d.status === "living" && p.concept_doi) {
+        L.push("living:  This document is revised on purpose. CITE the concept DOI, which always");
+        L.push("            resolves to the newest version; VERIFY against the version DOI and");
+        L.push("            sha256 above, which pin these exact bytes.");
+    }
+    // ⚠️ The letters' inline voice markers are meaningless without the sentence
+    // that explains them, and that sentence lives in `editorial` — a field a
+    // resource read would otherwise drop, reintroducing the exact bug the tool
+    // responses were once shipped with.
+    if (d.editorial?.annotation) {
+        L.push(`editorial: ${d.editorial.annotation.replace(/\s+/g, " ")}`);
+    }
+    L.push("[END PROVENANCE — everything below this line is the document, verbatim.]");
+    return L.join("\n");
+};
+
+export const readResource = (uri, bySlug) => {
+    const raw = String(uri ?? "");
+    if (!raw.startsWith(CORPUS_SCHEME)) {
+        throw new Error(`unsupported resource uri "${raw}". Corpus documents are addressed as corpus://<slug>.`);
+    }
+    const slug = raw.slice(CORPUS_SCHEME.length);
+    const d = bySlug.get(slug);
+    if (!d) throw new Error(`no document with slug "${slug}". Call resources/list to see what is available.`);
+    return {
+        contents: [
+            {
+                uri: raw,
+                mimeType: mime(d),
+                // Header, blank line, then the document exactly as the tools return it.
+                text: provenanceHeader(d) + "\n\n" + d.text
+            }
+        ]
+    };
+};
+
+/* ------------------------------------------------------------- completions ---
+   ⭐ A URI TEMPLATE WITHOUT COMPLETION IS A TEMPLATE YOU MUST ALREADY KNOW THE
+   ANSWER TO USE. `corpus://{slug}` is only usable by someone who already has the
+   slug; this is what turns it into something a person can discover by typing.
+
+   ⚠️ Prefix matches rank above substring matches, because a slug is a name and a
+   person typing one is almost always typing its beginning. Within each group the
+   order is the corpus's own (alphabetical by slug) — stable, so the same keystroke
+   never reorders the list under the user's cursor. */
+
+const COMPLETION_MAX = 100; // the spec's ceiling
+
+const slugCompletion = (value, documents) => {
+    const q = String(value ?? "").toLowerCase();
+    const slugs = documents.map((d) => d.slug);
+    const starts = slugs.filter((s) => s.startsWith(q));
+    const contains = q ? slugs.filter((s) => !s.startsWith(q) && s.includes(q)) : [];
+    const all = [...starts, ...contains];
+    return {
+        completion: {
+            values: all.slice(0, COMPLETION_MAX),
+            total: all.length,
+            hasMore: all.length > COMPLETION_MAX
+        }
+    };
+};
+
+export const completeArgument = (ref, argument, documents) => {
+    // ⭐ Prompts and completions compose: `verify_a_quote` takes a slug, and the
+    // same slug list that completes corpus://{slug} completes it here. A prompt
+    // argument nobody can autocomplete is a prompt you must already know how to
+    // fill in — the same defect the resource template had before completions.
+    if (ref?.type === "ref/prompt") {
+        if (ref.name === "verify_a_quote" && argument?.name === "slug") {
+            return slugCompletion(argument?.value, documents);
+        }
+        const e = new Error(
+            `no completions for prompt "${ref?.name}" argument "${argument?.name}". ` +
+                "The completable prompt argument is verify_a_quote(slug)."
+        );
+        e.code = -32602;
+        throw e;
+    }
+    if (ref?.type !== "ref/resource") {
+        const e = new Error(`unsupported completion reference type "${ref?.type}"`);
+        e.code = -32602;
+        throw e;
+    }
+    if (ref.uri !== "corpus://{slug}") {
+        const e = new Error(`no completions for "${ref.uri}". The completable template is corpus://{slug}.`);
+        e.code = -32602;
+        throw e;
+    }
+    if (argument?.name !== "slug") {
+        const e = new Error(`corpus://{slug} has one argument, "slug"; got "${argument?.name}"`);
+        e.code = -32602;
+        throw e;
+    }
+
+    return slugCompletion(argument?.value, documents);
+};

package/src/results.mjs

@@ -0,0 +1,40 @@
+// How a tool result is shaped.
+//
+// ⭐⭐ JSON IS THE PAYLOAD; THE TEXT BLOCK IS THE HUMAN FORM. Until 2026-09-02 every
+// tool returned pretty-printed JSON inside a text block, which meant a consuming
+// program had to parse a string to reach data the server already had as an object.
+// `structuredContent` is where that object belongs.
+//
+// ⚠️ THE SPEC'S BACK-COMPAT ADVICE — also serialise the JSON into `content` — is a
+// SHOULD, and following it literally DOUBLES every response (measured: 1.76×–2.00×
+// across the six tools). So this file does not duplicate. It SPLITS BY ROLE:
+//
+//     content            — what a reader reads: document text, excerpts, a listing
+//     structuredContent  — what a program validates: the typed envelope and metadata
+//
+// Nothing appears in both. For the document tools that makes `content` the text
+// WITH ITS PROVENANCE HEADER — identical to what resources/read returns, so the two
+// ways into the same document finally agree — and `structuredContent` the envelope
+// without the body. Measured cost: 1.00× on the large tools, 0.76×–0.78× on the
+// metadata tools, which are smaller than what they replaced.
+//
+// ⛔ NO `outputSchema` YET, and that is a decision rather than an omission. The spec
+// makes a declared schema binding — "servers MUST provide structured results that
+// conform" — and the envelope changed twice on the day this was written. A schema
+// is a promise kept on every future change; publishing one over a shape still in
+// motion buys validation now and breaks validating clients later.
+
+// A payload that is data all the way down: the object, plus a COMPACT serialisation
+// for clients that read only `content`. Compact, not pretty — indentation is the
+// one part of the old shape that was pure cost.
+export const structured = (value) => ({
+    content: [{ type: "text", text: JSON.stringify(value) }],
+    structuredContent: value
+});
+
+// A payload with a human form: `text` is read, `value` is validated, and the two
+// carry different things rather than the same thing twice.
+export const structuredWithText = (text, value) => ({
+    content: [{ type: "text", text }],
+    structuredContent: value
+});

package/src/search.mjs

@@ -0,0 +1,176 @@
+// How `search_corpus` decides a document matches.
+//
+// ⭐⭐ SHARED, AND IT HAS TO BE. The two surfaces keep their own `callTool` on
+// purpose — twenty lines each, diffable by eye. Matching is not that: it is
+// ranking, tokenising and window-finding, and if the stdio server and the worker
+// ever disagreed about which documents answer a query, the corpus would have two
+// opinions about itself. That is the same failure `sync.mjs` exists to prevent,
+// arriving through the search path instead of the data path.
+//
+// ⚠️ WHY THIS REPLACED A BARE `indexOf`, and the evidence was a real caller.
+// The first external client to reach the hosted endpoint searched
+// `"gratitude alignment human wellbeing kindness"` and got ZERO results — while
+// `gratitude` alone returns 50, `alignment` 50, `kindness` 50, `wellbeing` 16.
+// The corpus was not missing the material; the matcher required that exact
+// five-word string to appear verbatim, which of course it never does. A literal
+// substring search silently punishes anyone who types a sentence, and it makes
+// the zero-result signal MEAN THE WRONG THING: those queries were recording the
+// matcher's limits while being read as gaps in the corpus.
+//
+// ⭐ PHRASE FIRST, THEN TERMS — the order is what keeps precision. This corpus is
+// full of hyphenated marks (`B-Heart`, `Re-Tip`, `B-Sey`), and an exact phrase is
+// always the strongest evidence a document is about the thing asked for. So an
+// exact hit wins and is ranked above every term match. Only when the phrase is
+// absent do we ask the weaker question: does this document contain ALL of these
+// words, anywhere?
+//
+// ⚠️ The tokeniser KEEPS intra-word hyphens and apostrophes, so `B-Heart` is one
+// token and not two. Splitting them would have made a search for `B-Heart` match
+// any document containing "b" and "heart" separately — precision traded away for
+// nothing, in the corpus where those marks matter most.
+
+const TOKEN = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu;
+
+// Dropped only when something survives the dropping. These are words present in
+// essentially every document, so they never narrow the result set — but they do
+// wreck the tightest-window calculation, because an "and" is always close by.
+const STOP = new Set(["a", "an", "and", "as", "at", "be", "by", "for", "from", "in", "is", "it", "of", "on", "or", "the", "to", "with"]);
+
+export const terms = (query) => {
+    const all = String(query ?? "").toLowerCase().match(TOKEN) ?? [];
+    const kept = all.filter((t) => !STOP.has(t));
+    return kept.length ? kept : all;
+};
+
+// ⚠️ THE EDGES SNAP TO WHITESPACE, because a fixed-width slice cuts words in
+// half — `…ratitude is immen…` — and a half-word at an excerpt boundary is not a
+// cosmetic problem here. These excerpts are read by agents deciding whether a
+// document is worth fetching, and a truncated token is a token that can be
+// matched, quoted or reasoned about as if it were a word.
+//
+// ⚠️ THE SNAP IS BUDGETED, AND THE BUDGET IS THE WHOLE SAFETY OF IT. A corpus
+// document can contain a 200-character unbroken run — a base64 blob, a sha256, a
+// long URL — and an unbounded search for whitespace would eat the entire excerpt
+// looking for a space that is not there. Past LOOK characters we accept the hard
+// cut: a slightly ugly excerpt beats an empty one.
+const LOOK = 40;
+
+const clip = (body, centre, span) => {
+    if (body.length <= span) return body.trim();
+
+    let start = Math.max(0, Math.min(Math.round(centre - span / 2), body.length - span));
+    let end = start + span;
+
+    // ⭐ EACH EDGE TRIES BOTH DIRECTIONS, and the second direction is what fixes
+    // the long-token case. Snapping INWARD is preferred (it never grows the
+    // excerpt), but this corpus is full of runs that exceed LOOK with no space
+    // in them at all — `project_future_kindness_operating_noun`, bare URLs,
+    // markdown link targets, sha256 hex. Inward alone gave up on exactly those
+    // and left the half-word it was meant to prevent. Snapping OUTWARD instead
+    // costs at most LOOK extra characters and always lands on a real boundary.
+    if (start > 0) {
+        const ahead = body.slice(start, start + LOOK).search(/\s/);
+        if (ahead !== -1) {
+            start += ahead + 1;
+        } else {
+            const back = /\s\S*$/.exec(body.slice(Math.max(0, start - LOOK), start));
+            if (back) start = Math.max(0, start - LOOK) + back.index + 1;
+        }
+    }
+    if (end < body.length) {
+        const from = Math.max(start + 1, end - LOOK);
+        const back = /\s\S*$/.exec(body.slice(from, end));
+        if (back) {
+            end = from + back.index;
+        } else {
+            const ahead = body.slice(end, end + LOOK).search(/\s/);
+            end = ahead !== -1 ? end + ahead : Math.min(body.length, end + LOOK);
+        }
+    }
+    if (end <= start) end = Math.min(body.length, start + span); // pathological input
+
+    return (start > 0 ? "…" : "") + body.slice(start, end).trim() + (end < body.length ? "…" : "");
+};
+
+// Positions of a term, capped: a common word can appear thousands of times and
+// the window sweep only needs enough of them to find a tight one.
+const positions = (hay, term, cap = 200) => {
+    const out = [];
+    let i = hay.indexOf(term);
+    while (i !== -1 && out.length < cap) {
+        out.push(i);
+        i = hay.indexOf(term, i + term.length);
+    }
+    return out;
+};
+
+// Smallest span containing at least one occurrence of every term. Classic sweep
+// over the merged, sorted occurrence list — linear in the number of positions.
+const tightestWindow = (lists) => {
+    const merged = [];
+    lists.forEach((ps, t) => ps.forEach((p) => merged.push([p, t])));
+    merged.sort((a, b) => a[0] - b[0]);
+    const need = lists.length;
+    const seen = new Map();
+    let best = null;
+    let left = 0;
+    for (let right = 0; right < merged.length; right++) {
+        seen.set(merged[right][1], (seen.get(merged[right][1]) ?? 0) + 1);
+        while (seen.size === need) {
+            const width = merged[right][0] - merged[left][0];
+            if (!best || width < best.width) best = { width, from: merged[left][0], to: merged[right][0] };
+            const t = merged[left][1];
+            const n = seen.get(t) - 1;
+            if (n === 0) seen.delete(t);
+            else seen.set(t, n);
+            left++;
+        }
+    }
+    return best;
+};
+
+// null when the document does not match; otherwise the excerpt, how it matched,
+// and a score for ranking. ⭐ `mode` is returned rather than hidden because a
+// term match's excerpt may NOT contain the literal query, and a caller reading
+// an excerpt deserves to know which question it answered.
+export const match = (body, query, span = 320) => {
+    const hay = body.toLowerCase();
+    const phrase = String(query ?? "").toLowerCase().trim();
+
+    if (phrase) {
+        const i = hay.indexOf(phrase);
+        // An exact hit always wins: score is above anything a window can score.
+        if (i !== -1) return { mode: "phrase", excerpt: clip(body, i + phrase.length / 2, span), score: 1e9 };
+    }
+
+    const ts = [...new Set(terms(query))];
+    // A single term has already been tried as a phrase; there is no weaker
+    // question left to ask, so a miss is a miss.
+    if (ts.length < 2) return null;
+
+    const lists = ts.map((t) => positions(hay, t));
+    if (lists.some((l) => l.length === 0)) return null; // AND, not OR
+
+    const win = tightestWindow(lists);
+    if (!win) return null;
+    // Tighter is better; every term match ranks below every phrase match.
+    return { mode: "terms", excerpt: clip(body, Math.floor((win.from + win.to) / 2), span), score: 1e6 / (1 + win.width) };
+};
+
+// Rank and cut. Phrase matches first, then tighter term windows, corpus order
+// breaking ties (Array.prototype.sort is stable).
+export const rank = (hits, limit) =>
+    hits.slice().sort((a, b) => b.m.score - a.m.score).slice(0, Math.max(0, limit));
+
+// ⭐⭐ WHICH TERMS APPEAR IN NO DOCUMENT AT ALL. Under AND-matching a zero result
+// means at least one term is missing, and saying WHICH turns a dead end into a
+// finding: `unicorn` absent is a fact about the corpus, while `wellbeing` absent
+// would be a fact about the query. Computed only when a search returns nothing,
+// so the cost never lands on a successful call — and it is exactly the sentence
+// a caller would otherwise have to write to us by hand.
+export const absentTerms = (documents, query) => {
+    const ts = [...new Set(terms(query))];
+    if (ts.length === 0) return [];
+    const hays = documents.map((d) => d.text.toLowerCase());
+    return ts.filter((t) => !hays.some((h) => h.includes(t)));
+};