@333eco/corpus 1.2.5 → 2.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@333eco/corpus",
-    "version": "1.2.5",
+    "version": "2.1.0",
     "description": "An MCP server for an open-licensed corpus, served with verifiable provenance — every document carries its sha256, DOI and OpenTimestamps proof so a consuming agent can check its own citation.",
     "license": "CC0-1.0",
     "author": "Thon Ly",
@@ -40,7 +40,8 @@
     "scripts": {
         "start": "node src/server.mjs",
         "build": "node scripts/build-index.mjs --from ../../../TH/publications ../../../H3/publications ../../../missaquarius.org/missaquarius.org",
-        "check": "node scripts/build-index.mjs --check --from ../../../TH/publications ../../../H3/publications ../../../missaquarius.org/missaquarius.org"
+        "check": "node scripts/check-parity.mjs && node scripts/build-index.mjs --check --from ../../../TH/publications ../../../H3/publications ../../../missaquarius.org/missaquarius.org",
+        "check:parity": "node scripts/check-parity.mjs"
     },
     "publishConfig": {
         "access": "public"

package/src/base-tools.mjs

@@ -0,0 +1,72 @@
+// The three base tools, shared by both surfaces.
+//
+// ⭐ Definitions are shared; DISPATCH is not. `callTool` takes different arguments
+// on each surface (the worker is handed the corpus per request, the stdio server
+// closes over it) and those functions are genuinely different code. What must
+// never differ is what the two servers ADVERTISE — a client picks a tool by name
+// and schema, so a drifted definition is a client calling something that isn't
+// there. The envelope stays duplicated on purpose; that one is twenty lines and
+// diffable by eye.
+
+// ⭐⭐ EVERY TOOL HERE IS READ-ONLY, AND SAYING SO IS NOT DECORATION. A client that
+// knows a call cannot mutate anything can stop putting a confirmation dialog in
+// front of a corpus lookup. This server has no write path at all: it opens exactly
+// one file, dist/corpus.json, and never writes.
+// ⚠️ `openWorldHint: false` is the honest value — the corpus is a closed, fixed
+// set for the life of a build, not an open-ended external system.
+export const READ_ONLY = {
+    readOnlyHint: true,
+    destructiveHint: false,
+    idempotentHint: true,
+    openWorldHint: false
+};
+
+export const BASE_TOOLS = [
+    {
+        name: "search_corpus",
+        title: "Search the corpus",
+        annotations: { ...READ_ONLY, title: "Search the corpus" },
+        description:
+            "Full-text search across the open-licensed corpus. Returns matching documents with a provenance envelope " +
+            "and a short excerpt around each match — not the full text; call get_document for that. Every result can " +
+            "be independently verified via its sha256, DOI and OpenTimestamps proof.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                query: { type: "string", description: "Text to search for. Case-insensitive." },
+                genre: { type: "string", description: "Optional: restrict to a genre, e.g. essays, defensive-publications, positions, white-papers." },
+                limit: { type: "number", description: "Maximum documents to return. Default 10." }
+            },
+            required: ["query"]
+        }
+    },
+    {
+        name: "get_document",
+        title: "Read one document in full",
+        annotations: { ...READ_ONLY, title: "Read one document in full" },
+        description:
+            "Return one document in full, with its provenance envelope. The text is the canonical source — never a " +
+            "summary — so its sha256 can be checked against the envelope and against the anchored proof.",
+        inputSchema: {
+            type: "object",
+            properties: { slug: { type: "string", description: "Document slug, as returned by search_corpus or list_documents." } },
+            required: ["slug"]
+        }
+    },
+    {
+        name: "list_documents",
+        title: "List the corpus",
+        annotations: { ...READ_ONLY, title: "List the corpus" },
+        description:
+            "List the corpus: slugs, titles, genres, licences and provenance summaries, without full text. Use to " +
+            "orient before searching, or to enumerate what is available under a given licence.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                genre: { type: "string", description: "Optional: restrict to a genre — essays, defensive-publications, positions, white-papers, letters, program." },
+                category: { type: "string", description: "Optional: restrict to a topic category — institutional (the four-body architecture and the institution itself), mechanism, alignment, essays, letters, program, capabilities. The response lists every category with its count." },
+                licence: { type: "string", description: "Optional: restrict to a licence id, e.g. CC0-1.0 or CC-BY-4.0." }
+            }
+        }
+    }
+];

package/src/program-tools.mjs

@@ -0,0 +1,181 @@
+// The research-program tools, shared by both surfaces.
+//
+// ⭐ WHY THIS IS A MODULE AND THE ENVELOPE IS NOT. The envelope is deliberately
+// duplicated between the stdio server and the worker "so the two can be diffed by
+// eye". That argument works for twenty lines of object literal. It does not work
+// for a tool surface: these three tools are ~120 lines of filtering, fallback and
+// wording, and a hand-maintained second copy of that is how corpus.333.eco ends up
+// advertising tools the npm package does not have, or worse, answering the same
+// question differently. The repo already refuses to let the two surfaces disagree
+// about what a DOCUMENT says; this is the same refusal applied to what a TOOL says.
+//
+// ⚠️ The envelope is INJECTED rather than imported, so each surface keeps its own
+// — that duplication is intentional and is not undone here.
+//
+// ⚠️ Only bundles JavaScript. The corpus itself stays a static asset in the worker
+// (1.61 MB gzipped, over the script cap); this module is a few kilobytes of logic.
+
+import { READ_ONLY } from "./base-tools.mjs";
+import { structured } from "./results.mjs";
+
+export const PROGRAM_TOOLS = [
+    {
+        name: "list_predictions",
+        title: "List the pre-registered predictions",
+        annotations: { ...READ_ONLY, title: "List the pre-registered predictions" },
+        description:
+            "List the pre-registered predictions of the Which Way Value Moves research program, with each one's " +
+            "registered falsifier and current status. Filterable by state, chapter, level and stating paper. Use to " +
+            "find what would falsify a claim, or what has actually been run — the program registers what could show " +
+            "it wrong, so an unrun prediction is a disclosure, not a gap.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                state: { type: "string", description: "Optional facet: unrun, running, run, contradicted, retired, other. The verbatim status is always returned beside it." },
+                chapter: { type: "string", description: "Optional, matched as a substring: e.g. \"Core-level\", \"Scale\"." },
+                level: { type: "string", description: "Optional: core (failing one ends the program) or chapter." },
+                paper: { type: "string", description: "Optional: slug of the stating paper, e.g. co-presence-gated-redemption." }
+            }
+        }
+    },
+    {
+        name: "get_prediction",
+        title: "Read one prediction, with its paper's proof",
+        annotations: { ...READ_ONLY, title: "Read one prediction, with its paper's proof" },
+        description:
+            "Return one pre-registered prediction by identifier (e.g. P-L2) with its registered wording, its " +
+            "falsifier, its status, and the PROVENANCE ENVELOPE OF THE PAPER THAT REGISTERED IT — hash, DOI and " +
+            "OpenTimestamps command — so the registration itself can be verified rather than trusted.",
+        inputSchema: {
+            type: "object",
+            properties: { id: { type: "string", description: "Prediction identifier, e.g. P-L2, P-K1, P-CS5." } },
+            required: ["id"]
+        }
+    },
+    {
+        name: "get_program",
+        title: "The research program in one call",
+        annotations: { ...READ_ONLY, title: "The research program in one call" },
+        description:
+            "The shape of the research program in one call: its hard core, its four chapters, its stopping rule, its " +
+            "revision rule and its own count reconciliation — all verbatim, drawn from the two documents that state " +
+            "it, with both provenance envelopes attached. Use to orient before querying predictions.",
+        inputSchema: { type: "object", properties: {} }
+    }
+];
+
+export const PROGRAM_TOOL_NAMES = PROGRAM_TOOLS.map((t) => t.name);
+
+export const PROGRAM_INSTRUCTIONS =
+    " This corpus also carries a research program with a public register of falsifiable predictions: " +
+    "list_predictions, get_prediction and get_program expose what would show the program wrong and what has " +
+    "actually been run. A prediction's authority is the paper that registered it, so those tools return that " +
+    "paper's provenance envelope rather than the register's.";
+
+// A prediction as returned: the record, plus its stating paper resolved to a real
+// envelope. ⚠️ `stating_paper` gains nothing where the register itself is the
+// origin, or where the row names no paper — never silently filled in.
+const view = (p, bySlug, envelope) => {
+    const paper = p.stating_paper.slug ? bySlug.get(p.stating_paper.slug) : null;
+    return {
+        ...p,
+        stating_paper: {
+            ...p.stating_paper,
+            ...(paper
+                ? {
+                      title: paper.title,
+                      // The whole point: verify the registration against the paper
+                      // that made it, not against the index that lists it.
+                      provenance: envelope(paper).provenance,
+                      full_text: `call get_document with slug "${paper.slug}"`
+                  }
+                : {})
+        }
+    };
+};
+
+export const callProgramTool = ({ program, bySlug, envelope }, name, args) => {
+    if (!program) throw new Error("this index carries no research program block");
+
+    if (name === "list_predictions") {
+        const eq = (a, b) => !b || String(b).toLowerCase() === String(a ?? "").toLowerCase();
+        const hits = program.predictions.filter(
+            (p) =>
+                eq(p.state, args?.state) &&
+                eq(p.level, args?.level) &&
+                (!args?.chapter || p.chapter.toLowerCase().includes(String(args.chapter).toLowerCase())) &&
+                eq(p.stating_paper.slug, args?.paper)
+        );
+        return structured({
+            question: program.question,
+            matched: hits.length,
+            of: program.prediction_count,
+            // ⚠️ RETURNED ON EVERY LIST, UNFILTERED, because a set of predictions
+            // means nothing without the honest denominator beside it: this program
+            // has 2 run, both desk censuses, and 0 field tests. A caller shown only
+            // its own filter could read a long list as a body of evidence.
+            by_state: program.predictions.reduce((a, x) => ((a[x.state] = (a[x.state] ?? 0) + 1), a), {}),
+            predictions: hits.map((p) => view(p, bySlug, envelope))
+        });
+    }
+
+    if (name === "get_prediction") {
+        const id = String(args?.id ?? "").trim();
+        const hit = program.predictions.find((x) => x.id && x.id.toLowerCase() === id.toLowerCase());
+        if (hit) return structured(view(hit, bySlug, envelope));
+
+        // ⭐ A WITHHELD PREDICTION IS ANSWERED, NOT DENIED. The register publishes
+        // the identifier and the reason precisely so the total is honest; a lookup
+        // that said "no such prediction" would undo that on the server's side.
+        const w = program.withheld.find((x) => x.id.toLowerCase() === id.toLowerCase());
+        if (w) {
+            return structured({
+                id: w.id,
+                withheld: true,
+                reason: w.reason,
+                note: "Recorded in the register so the total is honest; the wording is not published."
+            });
+        }
+        const out = program.instrumented_outside_core.find((x) =>
+            x.ids.split(/,\s*/).map((s) => s.trim().toLowerCase()).includes(id.toLowerCase())
+        );
+        if (out) {
+            return structured({
+                id,
+                outside_core: true,
+                paper: out.paper,
+                subject: out.subject,
+                note: "Named by a paper but testing something other than the direction core — not evidence for or against the program."
+            });
+        }
+        throw new Error(`no prediction "${id}". Call list_predictions to see the register, or get_program for its shape.`);
+    }
+
+    if (name === "get_program") {
+        return structured({
+            question: program.question,
+            documents: program.documents.map((slug) => {
+                const d = bySlug.get(slug);
+                return d ? { ...envelope(d), full_text: `call get_document with slug "${slug}"` } : { slug };
+            }),
+            hard_core: program.hard_core,
+            four_chapters: program.four_chapters,
+            chapters: program.chapters,
+            honest_limits: program.honest_limits,
+            stopping_rule: program.stopping_rule,
+            revision_rule: program.revision_rule,
+            verification: program.verification,
+            counts: program.counts,
+            // The register's arithmetic, re-derived from its own tables at build
+            // time rather than copied from its Summary.
+            reconciliation: program.reconciliation,
+            withheld: program.withheld,
+            instrumented_outside_core: program.instrumented_outside_core,
+            // ⚠️ Surfaced, never buried: an attribution the builder could not
+            // confirm against the paper it names.
+            unverified_attributions: program.unverified_attributions
+        });
+    }
+
+    throw new Error(`unknown program tool: ${name}`);
+};

package/src/prompts.mjs

@@ -0,0 +1,134 @@
+// Prompts — worked examples of how to use this server, shared by both surfaces.
+//
+// ⛔⛔ THE GUARD, AND IT IS THE WHOLE REASON THESE ARE SAFE TO SHIP: A PROMPT HERE
+// MAY DESCRIBE THE API. IT MAY NEVER DESCRIBE THE SUBJECT MATTER. The moment one
+// of these says something about gratitude, the gift, the four bodies or what the
+// research program has shown, this server has begun editorialising on its own
+// corpus — and a retrieval server with opinions about its documents is exactly the
+// thing the provenance envelope exists to make unnecessary. Read every line below
+// as an answer to "how do I check this?" and never to "what should I conclude?"
+//
+// ⚠️ A SECOND, NARROWER REFUSAL, recorded because it was the tempting version:
+// there is no `verify_before_citing` prompt telling a model to behave well. That
+// would be a RULE where the server already has a PROPERTY — every document, by
+// every route, arrives with a provenance header the reader must actively strip.
+// A rule that fires only when a human picks it from a menu is the weak form of a
+// guarantee that already fires always. These prompts demonstrate the API; they do
+// not ask anyone to be careful.
+
+export const PROMPTS = [
+    {
+        name: "orient",
+        title: "What is in this corpus?",
+        description:
+            "A worked example of the discovery tools: what genres and topic categories exist, how many documents " +
+            "carry proofs, and where the research program's own statement of itself lives. Calls tools; asserts nothing.",
+        arguments: []
+    },
+    {
+        name: "verify_a_quote",
+        title: "Check a quotation against its source",
+        description:
+            "A worked example of the verification path: fetch a document, then check the served text against the " +
+            "hash, the DOI and the OpenTimestamps proof its envelope names. Shows what to run, not what to believe.",
+        arguments: [
+            { name: "slug", description: "Document slug, e.g. co-presence-gated-redemption. Completable.", required: true },
+            { name: "quote", description: "Optional: the passage you intend to cite.", required: false }
+        ]
+    },
+    {
+        name: "what_would_falsify",
+        title: "Find what would show a claim wrong",
+        description:
+            "A worked example of the program tools: search the prediction register for the registered falsifier of a " +
+            "claim, then resolve it to the paper that registered it and verify THAT paper. Demonstrates the one " +
+            "non-obvious move in this API — a prediction's authority is its stating paper, not the register.",
+        arguments: [{ name: "claim", description: "The claim or topic you want to test, in your own words.", required: true }]
+    }
+];
+
+export const PROMPT_NAMES = PROMPTS.map((p) => p.name);
+
+const message = (text) => ({ role: "user", content: { type: "text", text } });
+
+export const getPrompt = (name, args) => {
+    if (name === "orient") {
+        return {
+            description: "Discover the shape of the corpus using its own tools.",
+            messages: [
+                message(
+                    "Show me what this corpus contains, using the server's tools rather than your prior knowledge.\n\n" +
+                        "1. Call list_documents with no arguments. Report the total, the licence split, and the `categories` map — " +
+                        "those are the corpus's own shelves, and `institutional` is where the institution describes itself.\n" +
+                        "2. Call list_documents with category \"institutional\" and list what is there.\n" +
+                        "3. Call get_program to see the research program's own statement of its hard core, its chapters and its " +
+                        "count reconciliation.\n\n" +
+                        "Quote the documents' own words where you summarise them, and name the slug you took each claim from. " +
+                        "Do not fill gaps from memory: if the corpus does not cover something, say that it does not."
+                )
+            ]
+        };
+    }
+
+    if (name === "verify_a_quote") {
+        const slug = String(args?.slug ?? "").trim();
+        if (!slug) {
+            const e = new Error("verify_a_quote requires a `slug` argument");
+            e.code = -32602;
+            throw e;
+        }
+        const quote = String(args?.quote ?? "").trim();
+        return {
+            description: `Verify a passage of ${slug} against its anchored proof.`,
+            messages: [
+                message(
+                    `Check a quotation from \`${slug}\` against its source, using this server's provenance envelope.\n\n` +
+                        `1. Call get_document with slug "${slug}". The document arrives in content[0].text behind a ` +
+                        "[PROVENANCE] header; the typed envelope is in structuredContent.\n" +
+                        (quote
+                            ? `2. Find this passage in the text and report whether it appears verbatim:\n\n   "${quote}"\n\n`
+                            : "2. Choose the passage you intend to cite and report it verbatim.\n\n") +
+                        "3. Run the envelope's own `verify.sha256` command and compare the result with `provenance.sha256`. " +
+                        "Report whether they match, and note that the hash covers the complete source file — metadata block " +
+                        "included — and not the body alone.\n" +
+                        "4. If `provenance.doi` is present, give the DOI to cite. If the document's status is `living`, cite " +
+                        "the concept DOI and verify against the version DOI, as the envelope's `citation` block says.\n" +
+                        "5. If `licence.attribution_required` is true, name whom to attribute.\n\n" +
+                        "Report what the commands actually returned. Do not vouch for the text on the server's behalf."
+                )
+            ]
+        };
+    }
+
+    if (name === "what_would_falsify") {
+        const claim = String(args?.claim ?? "").trim();
+        if (!claim) {
+            const e = new Error("what_would_falsify requires a `claim` argument");
+            e.code = -32602;
+            throw e;
+        }
+        return {
+            description: "Find the registered falsifier for a claim, and verify where it was registered.",
+            messages: [
+                message(
+                    `Find out what would show this claim wrong, according to the corpus's own prediction register:\n\n` +
+                        `   "${claim}"\n\n` +
+                        "1. Call list_predictions and look for predictions bearing on it. `by_state` is returned unfiltered — " +
+                        "report it, because a long list of predictions is not a body of evidence.\n" +
+                        "2. For the closest match, call get_prediction with its id. Report the registered wording, the " +
+                        "registered falsifier, and the status verbatim.\n" +
+                        "3. ⭐ The result's `stating_paper` carries the envelope of the PAPER that registered the prediction, " +
+                        "not of the register. That is deliberate: the register says to verify the stating paper against its " +
+                        "stored proof rather than to trust the register. Follow that — verify the stating paper.\n" +
+                        "4. Say plainly whether the prediction has been run. An unrun prediction is a disclosure, not a result, " +
+                        "and must not be reported as evidence either way.\n\n" +
+                        "If no registered prediction bears on the claim, say so rather than constructing one."
+                )
+            ]
+        };
+    }
+
+    const e = new Error(`no prompt "${name}". Call prompts/list to see what is available.`);
+    e.code = -32602;
+    throw e;
+};

package/src/report.mjs

@@ -0,0 +1,104 @@
+// `--report-gap` and `--report-bug` — the only place this package makes an
+// outbound request, and it is reached only by typing one of the flags.
+//
+// ⭐⭐ WHY THIS IS A COMMAND AND NOT TELEMETRY, because the distinction is the
+// whole design. The useful signal from a corpus server is *what someone looked
+// for and did not find*. The remote endpoint at corpus.333.eco collects that
+// from its own callers as a property of being the server they called. This
+// package runs on YOUR machine, so the same collection there would be an
+// outbound report about your private reading — and the guard against that is
+// not a consent flag or an opt-out. It is that THE SERVER PATH CANNOT REACH
+// THIS FILE: `server.mjs` loads it with a dynamic import inside the argv branch,
+// so during normal operation the module is never even read off disk.
+//
+// ⚠️ THE CHECKABLE CLAIM CHANGED SHAPE WHEN THIS FILE WAS ADDED, AND THAT IS
+// WORTH STATING PLAINLY. Before it, "this package makes no network call" was
+// verifiable by `grep -r fetch src/` returning nothing at all — the strongest
+// kind of evidence, because it needs no reasoning. Now the honest claim is
+// narrower: there is exactly ONE fetch in the package, it lives in this file,
+// and this file is imported from exactly one place — a branch that requires an
+// explicit flag. Still inspectable in under a minute, but it is a chain of two
+// facts rather than one absence. ⛔ If a second import of this module ever
+// appears, that chain is broken and the claim must be rewritten rather than
+// repeated.
+//
+// ⛔ IT REPORTS TO THE WORKER, NOT TO THE NOTIFICATION BEACON. Sending to
+// thonly.org/api/track would mean shipping every user of this package a working
+// recipe for writing into the founder's admin notification channel, behind an
+// Origin header a CLI can trivially assert. The worker's /gap endpoint writes to
+// Analytics Engine and pushes nothing at anyone.
+
+const ENDPOINT = "https://corpus.333.eco/report";
+
+// ⚠️ A BUG REPORT AND A GAP REPORT CARRY THE SAME PAYLOAD, DELIBERATELY. It is
+// tempting to attach a node version and a platform to a bug — genuinely useful
+// to whoever fixes it — but that would give this command two different promises
+// about what it sends, and the promise is the valuable part. Anything about your
+// environment that matters, put in the text; then you have said it on purpose.
+const KINDS = {
+    gap: {
+        prompt: "what you looked for and did not find",
+        thanks: "Thank you — recorded. It joins the gaps the hosted endpoint already sees."
+    },
+    bug: {
+        prompt: "what went wrong, and what you expected instead",
+        thanks: "Thank you — recorded. Issues are also welcome at github.com/333eco/corpus.333.eco/issues."
+    }
+};
+const MAX = 200;
+
+// ⭐ WHAT IS SENT IS THE WHOLE OF WHAT IS SENT. The text you typed, and the
+// version of the corpus you have. No machine id, no username, no hostname, no
+// path, no timestamp of your own — and the receiving end deliberately does not
+// record the country it could resolve, because a voluntary note about a missing
+// document has no use for where the sender was standing.
+export const report = async (kind, text, version) => {
+    const spec = KINDS[kind];
+    if (!spec) throw new Error(`unknown report kind: ${kind}`);
+    const body = String(text ?? "").trim().slice(0, MAX);
+    if (!body) {
+        console.error(`usage: corpus-mcp --report-${kind} "${spec.prompt}"`);
+        return 1;
+    }
+
+    const payload = { kind, text: body, version: version ?? null };
+
+    // Printed BEFORE the request, not after, so the disclosure is not
+    // contingent on the request succeeding.
+    console.log("Sending this, and nothing else:\n");
+    console.log("  " + JSON.stringify(payload));
+    console.log("\n  to " + ENDPOINT + "\n");
+
+    try {
+        const res = await fetch(ENDPOINT, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(payload)
+        });
+        // ⚠️⚠️ THE STATUS CODE IS NOT THE CONTRACT, AND TRUSTING IT REPORTED A
+        // FALSE SUCCESS ON THE FIRST RUN. A server predating /gap treats any
+        // POST without a JSON-RPC `id` as a NOTIFICATION and answers 202 with an
+        // empty body — so `res.ok` was true, and this printed "recorded" while
+        // nothing had been. The success signal must therefore be something only
+        // the real handler can produce: an explicit `ok` in the body. That also
+        // makes version skew safe in both directions, since an old server can
+        // never accidentally satisfy it.
+        const ack = await res.json().catch(() => null);
+        if (res.ok && ack?.ok === true) {
+            console.log(spec.thanks);
+            return 0;
+        }
+        console.error(
+            res.ok
+                ? "The endpoint accepted the request but did not confirm it recorded anything —\n" +
+                  "it is probably running a version without /report. Nothing was recorded."
+                : `The endpoint answered ${res.status}. Nothing was recorded.`
+        );
+        return 1;
+    } catch (e) {
+        // A failure here is worth nobody's day. Say so and exit cleanly.
+        console.error(`Could not reach ${ENDPOINT}: ${e.message}`);
+        console.error("Nothing was sent. This is entirely optional — carry on.");
+        return 1;
+    }
+};