@333eco/corpus 2.3.5 → 2.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@333eco/corpus",
-    "version": "2.3.5",
+    "version": "2.4.1",
     "description": "An MCP server for an open-licensed corpus, served with verifiable provenance — every document carries its sha256, DOI and OpenTimestamps proof so a consuming agent can check its own citation.",
     "license": "CC0-1.0",
     "author": "Thon Ly",

package/src/base-tools.mjs

@@ -5,8 +5,8 @@
 // closes over it) and those functions are genuinely different code. What must
 // never differ is what the two servers ADVERTISE — a client picks a tool by name
 // and schema, so a drifted definition is a client calling something that isn't
-// there. The envelope stays duplicated on purpose; that one is twenty lines and
-// diffable by eye.
+// there. ⚠️ The envelope was once duplicated on purpose, "diffable by eye"; it drifted
+// unseen, and now lives in envelope.mjs, imported by both.
 
 // ⭐⭐ EVERY TOOL HERE IS READ-ONLY, AND SAYING SO IS NOT DECORATION. A client that
 // knows a call cannot mutate anything can stop putting a confirmation dialog in
@@ -27,9 +27,9 @@ export const BASE_TOOLS = [
         title: "Search the corpus",
         annotations: { ...READ_ONLY, title: "Search the corpus" },
         description:
-            "Full-text search across the open-licensed corpus. Returns matching documents with a provenance envelope " +
-            "and a short excerpt around each match — not the full text; call get_document for that. Every result can " +
-            "be independently verified via its sha256, DOI and OpenTimestamps proof.",
+            "Full-text search across the open-licensed corpus; diacritics are optional. Returns matching documents with " +
+            "a provenance envelope and a short excerpt around each match — not the full text; read_documents or " +
+            "get_document return that. Every result can be verified via its sha256, DOI and OpenTimestamps proof.",
         inputSchema: {
             type: "object",
             properties: {
@@ -51,7 +51,11 @@ export const BASE_TOOLS = [
             type: "object",
             properties: { slug: { type: "string", description: "Document slug, as returned by search_corpus or list_documents." } },
             required: ["slug"]
-        }
+        },
+        // ⚠️ The largest document is ~140 KB — over the 25,000-token default at which
+        // Claude Code moves a tool result into a file. A document is the unit that
+        // verifies, so it is kept whole in the conversation.
+        _meta: { "anthropic/maxResultSizeChars": 250_000 }
     },
     {
         name: "list_documents",

package/src/envelope.mjs

@@ -0,0 +1,64 @@
+// The provenance envelope — attached to every document a tool returns, on both surfaces.
+//
+// `verify` is the instruction rather than a promise: it tells the agent exactly what
+// to run, so the claim is checkable without trusting this sentence either.
+//
+// ⚠️⚠️ IT WAS WRITTEN TWICE, ON PURPOSE, AND THE PURPOSE FAILED. Each server kept its own
+// copy "so the two can be diffed by eye; if they ever disagree, that is a bug in one of
+// them and the diff is where it shows." They did disagree — the OpenTimestamps
+// instruction and the revised-since-deposit note had different wording on the two
+// surfaces — and no eye caught it. The first check that COMPARED THE ANSWERS did, on its
+// first run (2026-09-13). A rule that depends on someone looking is replaced by one
+// function both surfaces import: the difference can no longer be written.
+
+export const envelope = (d) => ({
+    slug: d.slug,
+    title: d.title,
+    licence: {
+        id: d.licence.id,
+        url: d.licence.url,
+        attribution_required: d.licence.attribution_required,
+        // Only present where it is actually owed, so an agent can act on the
+        // field's presence rather than parsing the licence id.
+        ...(d.licence.attribution_required && d.authors ? { attribute_to: d.authors } : {})
+    },
+    provenance: {
+        ...d.provenance,
+        // ⚠️ A LIVING DOCUMENT IS VERIFIED AND CITED WITH DIFFERENT DOIs, and
+        // saying only "cite accordingly" leaves the reader to guess which.
+        // The VERSION doi is the only one that can be true of the bytes here
+        // — it pins them — so it is what a hash check resolves against. The
+        // CONCEPT doi follows the document, so it is what a citation should
+        // name: a living register is revised on purpose, and a citation
+        // pinned to one revision goes stale by design rather than by accident.
+        ...(d.status === "living" && d.provenance.concept_doi
+            ? {
+                  citation: {
+                      cite: `https://doi.org/${d.provenance.concept_doi}`,
+                      verify_against: d.provenance.doi ? `https://doi.org/${d.provenance.doi}` : null,
+                      why: "This document is living — it is revised on purpose. Cite the concept DOI, which always resolves to the newest version; verify the text you were served against the version DOI and sha256 above, which pin these exact bytes."
+                  }
+              }
+            : {}),
+        verify: {
+            // Concrete, because an instruction that says "the source file"
+            // without saying which one is not an instruction. All three source
+            // repositories are public, so this is genuinely runnable.
+            sha256: d.provenance.source_url
+                ? `curl -sL ${d.provenance.source_url} | shasum -a 256   # compare to provenance.sha256`
+                : "shasum -a 256 <the source file>   # compare to provenance.sha256",
+            ...(d.provenance.doi ? { doi: `https://doi.org/${d.provenance.doi}` } : {}),
+            ...(d.provenance.opentimestamps
+                ? { opentimestamps: `ots verify ${d.path}.ots  # in the source repository; the proof is anchored in the Bitcoin blockchain` }
+                : {}),
+            ...(d.provenance.deposited_matches_current === false
+                ? {
+                      note:
+                          "This document has been REVISED since its Zenodo deposit, so provenance.sha256 " +
+                          "and provenance.deposited_sha256 differ legitimately. The DOI resolves to the " +
+                          "deposited version; the text served here is newer. Cite accordingly."
+                  }
+                : {})
+        }
+    }
+});

package/src/read-tools.mjs

@@ -0,0 +1,228 @@
+// Reading more than one document at a time, and knowing that nothing is missing —
+// shared by both surfaces.
+//
+// ⭐ WHY THIS EXISTS (A140). A model connected to this server gave answers that were
+// "quite accurate but incomplete": search returns excerpts, ten by default, and a
+// model answers from the few documents it happens to open. The window was not the
+// limit — a shelf of the corpus fits — the reading pattern was. So a caller can now
+// ask for a whole slice and see its size first.
+//
+// ⭐ Imported by both surfaces, like the program tools and the envelope: the filter
+// here must be the SAME filter list_documents applies, or
+// "read what I just listed" returns a different set. One function makes that true
+// by construction instead of by vigilance.
+
+import { READ_ONLY } from "./base-tools.mjs";
+import { documentText } from "./resources.mjs";
+
+// ~22k tokens of English prose: under the 25,000-token default at which Claude Code
+// moves a tool result out of the conversation and into a file. Other clients cap
+// differently; the caller can ask for more.
+export const DEFAULT_PAGE_BYTES = 90_000;
+export const MAX_PAGE_BYTES = 400_000;
+
+const byteLength = (s) => new TextEncoder().encode(s).length;
+
+/** The list_documents filter. Every facet is optional; absent means "any". */
+export const filterDocuments = (documents, args) =>
+    documents.filter(
+        (d) =>
+            (!args?.genre || d.genre === args.genre) &&
+            (!args?.licence || d.licence.id === args.licence) &&
+            (!args?.voice || d.voice === args.voice) &&
+            (!args?.category || d.category === args.category)
+    );
+
+export const READ_TOOLS = [
+    {
+        name: "read_documents",
+        title: "Read a set of documents in full",
+        annotations: { ...READ_ONLY, title: "Read a set of documents in full" },
+        // ⚠️ KEPT SHORT ON PURPOSE: every client that loads tool definitions up front pays
+        // for this text in every session, whether or not the corpus is used. The filter
+        // facets are described once, on list_documents.
+        description:
+            "Whole documents, never summarised: those matching the list_documents filters, or the given slugs. " +
+            "One content block each. Paged: follow next_cursor.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                genre: { type: "string" },
+                category: { type: "string" },
+                voice: { type: "string" },
+                licence: { type: "string" },
+                slugs: { type: "array", items: { type: "string" } },
+                max_bytes: { type: "number", description: `Page size. Default ${DEFAULT_PAGE_BYTES}, max ${MAX_PAGE_BYTES}.` },
+                cursor: { type: "string" }
+            }
+        },
+        // ⚠️ Claude Code moves a tool result over its limit (25,000 tokens by default)
+        // into a file. A page is sized by the CALLER, so the tool declares the
+        // largest page it can return and the client keeps it in the conversation.
+        _meta: { "anthropic/maxResultSizeChars": MAX_PAGE_BYTES + 100_000 }
+    }
+];
+
+export const READ_TOOL_NAMES = READ_TOOLS.map((t) => t.name);
+
+// ⚠️ CONDITIONAL ON PURPOSE (2.4.1). "Prefer read_documents" unqualified would send a model to
+// read a 260,000-token shelf to answer a one-fact question — slower, costlier and often worse.
+// ⛔ It names shelves by their facets (list_documents' own vocabulary), never by what the
+// documents say: describing the API, not the subject matter.
+export const READ_INSTRUCTIONS =
+    " For a broad question, read the relevant shelf in full: choose it with list_documents, read it with " +
+    "read_documents. Search returns excerpts, suited to finding one passage. Every document ends with an " +
+    "[END OF DOCUMENT] line; if it is missing, the text was cut short.";
+
+/**
+ * list_documents' pointer to reading what it just listed: the same filters, as arguments
+ * read_documents accepts. Only the facets actually given, so the pointer reads back exactly.
+ */
+export const readWith = (args, count) =>
+    count
+        ? {
+              tool: "read_documents",
+              arguments: Object.fromEntries(["genre", "category", "voice", "licence"].filter((k) => args?.[k]).map((k) => [k, args[k]])),
+              note: "Reads every document listed here in full, paged by size."
+          }
+        : null;
+
+// ⚠️ The cursor carries the INDEX VERSION as well as the offset. Pages are offsets
+// into a list, and a list read across a redeploy is two lists: without the version
+// a caller could be handed page 2 of a different corpus and never know.
+const encodeCursor = (version, offset) => btoa(`${version}:${offset}`);
+const decodeCursor = (cursor, version) => {
+    let raw;
+    try {
+        raw = atob(String(cursor));
+    } catch {
+        throw new Error(`invalid cursor: ${cursor}`);
+    }
+    const at = raw.lastIndexOf(":");
+    const v = raw.slice(0, at);
+    const offset = Number(raw.slice(at + 1));
+    if (at < 0 || !Number.isInteger(offset) || offset < 0) throw new Error(`invalid cursor: ${cursor}`);
+    if (v !== version) {
+        throw new Error(`this cursor belongs to corpus ${v}, and the server now serves ${version}. Start again without a cursor.`);
+    }
+    return offset;
+};
+
+/**
+ * read_documents. `ctx` is what differs between the surfaces:
+ * { documents, bySlug, envelope, version }.
+ */
+export const readDocuments = ({ documents, bySlug, envelope, version }, args) => {
+    const hasFilter = ["genre", "category", "voice", "licence"].some((k) => args?.[k]);
+    let matched;
+    if (args?.slugs !== undefined) {
+        if (hasFilter) throw new Error("give slugs or filters, not both.");
+        if (!Array.isArray(args.slugs) || !args.slugs.length) throw new Error("slugs must be a non-empty array of document slugs.");
+        const slugs = [...new Set(args.slugs.map(String))];
+        const unknown = slugs.filter((s) => !bySlug.has(s));
+        if (unknown.length) throw new Error(`no document with slug ${unknown.map((s) => `"${s}"`).join(", ")}. Call list_documents to see what is available.`);
+        matched = slugs.map((s) => bySlug.get(s));
+    } else {
+        matched = filterDocuments(documents, args);
+    }
+
+    const budget = args?.max_bytes === undefined ? DEFAULT_PAGE_BYTES : Number(args.max_bytes);
+    if (!Number.isFinite(budget) || budget <= 0) throw new Error("max_bytes must be a positive number.");
+    if (budget > MAX_PAGE_BYTES) throw new Error(`max_bytes may be at most ${MAX_PAGE_BYTES}; follow next_cursor for the rest.`);
+
+    const start = args?.cursor ? decodeCursor(args.cursor, version) : 0;
+    if (start > matched.length) throw new Error(`invalid cursor: ${args.cursor}`);
+
+    // Whole documents only. ⛔ A document is never split across pages: its hash, its
+    // header and its end marker describe the whole, and half of one verifies nothing.
+    // So a document larger than the budget arrives alone, over budget, and says so.
+    const page = [];
+    let used = 0;
+    for (let i = start; i < matched.length; i++) {
+        const text = documentText(matched[i]);
+        const size = byteLength(text);
+        if (page.length && used + size > budget) break;
+        page.push({ d: matched[i], text, size });
+        used += size;
+    }
+    const nextOffset = start + page.length;
+    const next = nextOffset < matched.length ? encodeCursor(version, nextOffset) : undefined;
+
+    if (!matched.length) {
+        return {
+            content: [{ type: "text", text: "no documents match these filters. Call list_documents to see the facets and their counts." }],
+            structuredContent: { matched: 0, returned: 0, offset: 0, page_bytes: 0, max_bytes: budget, documents: [] }
+        };
+    }
+
+    const content = page.map((p) => ({ type: "text", text: p.text }));
+    // The one block that is not a document: what a reader of `content` alone needs in
+    // order to keep going, or to know it has everything.
+    content.push({
+        type: "text",
+        text: next
+            ? `[PAGE — documents ${start + 1}–${nextOffset} of ${matched.length}. ${matched.length - nextOffset} more: call read_documents again with the same arguments and cursor "${next}".]`
+            : `[PAGE — documents ${start + 1}–${nextOffset} of ${matched.length}. That is every matching document.]`
+    });
+
+    return {
+        content,
+        structuredContent: {
+            matched: matched.length,
+            returned: page.length,
+            offset: start,
+            page_bytes: used,
+            max_bytes: budget,
+            ...(page.length === 1 && used > budget ? { over_budget: "one document larger than max_bytes, returned whole rather than split" } : {}),
+            ...(next ? { next_cursor: next } : {}),
+            documents: page.map((p) => ({
+                ...envelope(p.d),
+                genre: p.d.genre,
+                category: p.d.category,
+                voice: p.d.voice,
+                bytes: p.d.bytes,
+                words: p.d.words,
+                text_in: "content, one block per document in this order, each prefixed by its provenance header"
+            }))
+        }
+    };
+};
+
+/**
+ * list_documents' completeness block: the manifest's counts and the reasons, so the
+ * listing a caller already has is checkable against what the build considered.
+ */
+export const completenessOf = (corpus) =>
+    corpus.manifest
+        ? {
+              candidates: corpus.manifest.candidates,
+              served: corpus.manifest.served,
+              excluded: corpus.manifest.excluded_documents,
+              held: corpus.manifest.held_documents,
+              note: "An unfiltered listing should name every served document; the full manifest is at https://corpus.333.eco/manifest.json."
+          }
+        : null;
+
+/**
+ * ⛔ What both surfaces check before serving anything: do the documents loaded match
+ * the manifest the build wrote? Returns the problems; empty means serve.
+ * ⚠️ An index built before the manifest existed has none, and that is not refused —
+ * it is the previous release, not a damaged one.
+ */
+export const manifestProblems = (corpus) => {
+    const m = corpus.manifest;
+    if (!m) return [];
+    const problems = [];
+    if (m.candidates !== m.served + m.excluded + m.held) {
+        problems.push(`manifest does not reconcile: ${m.candidates} candidates vs ${m.served} + ${m.excluded} + ${m.held}`);
+    }
+    if (corpus.documents.length !== m.served) {
+        problems.push(`${corpus.documents.length} documents loaded, manifest names ${m.served}`);
+    }
+    const loaded = new Map(corpus.documents.map((d) => [d.slug, d.provenance?.sha256]));
+    for (const s of m.served_documents ?? []) {
+        if (!loaded.has(s.slug)) problems.push(`manifest names "${s.slug}", which is not loaded`);
+        else if (loaded.get(s.slug) !== s.sha256) problems.push(`"${s.slug}" does not carry the sha256 the manifest records`);
+    }
+    return problems;
+};

package/src/resources.mjs

@@ -128,10 +128,23 @@ export const provenanceHeader = (d) => {
     if (d.editorial?.annotation) {
         L.push(`editorial: ${d.editorial.annotation.replace(/\s+/g, " ")}`);
     }
-    L.push("[END PROVENANCE — everything below this line is the document, verbatim.]");
+    L.push("[END PROVENANCE — the document follows verbatim and ends at its [END OF DOCUMENT] line.]");
     return L.join("\n");
 };
 
+// ⭐⭐ THE ELISION MARKER (A87, the peyyāla). The server never cuts a document — but a
+// client may, and a cut document reads as a complete one: its last sentence is a real
+// sentence. So the end is stated IN THE TEXT, where a cut removes it, rather than in
+// a field that survives the cut and asserts a completeness the text no longer has.
+// ⚠️ Like the header, it is NOT part of the hashed artifact; the hash covers the source.
+export const endMarker = (d) =>
+    `[END OF DOCUMENT — ${d.slug}${typeof d.words === "number" ? ` · ${d.words} words` : ""}. ` +
+    "If this line is missing, the text above was cut short after it left the server.]";
+
+// ONE function for every door into a document — get_document, resources/read and
+// read_documents — so three ways in cannot become three different texts.
+export const documentText = (d) => provenanceHeader(d) + "\n\n" + d.text + "\n\n" + endMarker(d);
+
 export const readResource = (uri, bySlug) => {
     const raw = String(uri ?? "");
     if (!raw.startsWith(CORPUS_SCHEME)) {
@@ -145,8 +158,8 @@ export const readResource = (uri, bySlug) => {
             {
                 uri: raw,
                 mimeType: mime(d),
-                // Header, blank line, then the document exactly as the tools return it.
-                text: provenanceHeader(d) + "\n\n" + d.text
+                // Header, the document, the end marker — exactly as the tools return it.
+                text: documentText(d)
             }
         ]
     };

package/src/search.mjs

@@ -1,8 +1,8 @@
 // How `search_corpus` decides a document matches.
 //
-// ⭐⭐ SHARED, AND IT HAS TO BE. The two surfaces keep their own `callTool` on
-// purpose — twenty lines each, diffable by eye. Matching is not that: it is
-// ranking, tokenising and window-finding, and if the stdio server and the worker
+// ⭐⭐ SHARED, AND IT HAS TO BE — matching and, since 2.4.1, the whole search_corpus
+// tool (see searchCorpus below). Matching is ranking, tokenising and window-finding, and
+// if the stdio server and the worker
 // ever disagreed about which documents answer a query, the corpus would have two
 // opinions about itself. That is the same failure `sync.mjs` exists to prevent,
 // arriving through the search path instead of the data path.
@@ -33,15 +33,49 @@
 // any document containing "b" and "heart" separately — precision traded away for
 // nothing, in the corpus where those marks matter most.
 
+import { structuredWithText } from "./results.mjs";
+
 const TOKEN = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu;
 
+// ⭐⭐ DIACRITICS ARE OPTIONAL IN A QUERY (2.4.1). A model types `metta`; the corpus writes
+// `mettā`. Without a fold that search found 6 documents against 20, and `Tonle Sap` found none
+// — a reader typing plain ASCII, which is most of them, silently missed most of the Pāli
+// material. ⛔ FOLD TO MATCH, SERVE VERBATIM: the spelling is a tradition marker (Pāli for a
+// Theravāda citation, Sanskrit for a Hindu one), so the fold touches only what is COMPARED —
+// every excerpt is cut from the original text.
+// ⚠️ LENGTH-PRESERVING, character by character, so a position in the folded text is the same
+// position in the original and the excerpt window lands where the match is. Only Latin
+// combining diacritics (U+0300–U+036F) are removed, and only where the base letter survives
+// as one character: Khmer and Burmese vowel signs are marks too, and stripping them would
+// make Khmer text match what it does not say.
+const LATIN_MARKS = /[\u0300-\u036f]/g;
+const foldChar = (c) => {
+    const f = c.normalize("NFD").replace(LATIN_MARKS, "");
+    return f.length === c.length ? f : c;
+};
+export const fold = (s) => String(s ?? "").replace(/[^\x00-\x7f]/gu, foldChar);
+
+// The folded, lowercased body, computed once per document text rather than on every search.
+// Keyed by the string itself: the corpus holds one string per document for the life of the
+// process, so the cache is bounded by the corpus.
+const HAY = new Map();
+const hayOf = (body) => {
+    let h = HAY.get(body);
+    if (h === undefined) {
+        h = fold(body.toLowerCase());
+        HAY.set(body, h);
+    }
+    return h;
+};
+const foldQuery = (query) => fold(String(query ?? "").normalize("NFC").toLowerCase());
+
 // Dropped only when something survives the dropping. These are words present in
 // essentially every document, so they never narrow the result set — but they do
 // wreck the tightest-window calculation, because an "and" is always close by.
 const STOP = new Set(["a", "an", "and", "as", "at", "be", "by", "for", "from", "in", "is", "it", "of", "on", "or", "the", "to", "with"]);
 
 export const terms = (query) => {
-    const all = String(query ?? "").toLowerCase().match(TOKEN) ?? [];
+    const all = foldQuery(query).match(TOKEN) ?? [];
     const kept = all.filter((t) => !STOP.has(t));
     return kept.length ? kept : all;
 };
@@ -138,8 +172,8 @@ const tightestWindow = (lists) => {
 // term match's excerpt may NOT contain the literal query, and a caller reading
 // an excerpt deserves to know which question it answered.
 export const match = (body, query, span = 320) => {
-    const hay = body.toLowerCase();
-    const phrase = String(query ?? "").toLowerCase().trim();
+    const hay = hayOf(body);
+    const phrase = foldQuery(query).trim();
 
     if (phrase) {
         const i = hay.indexOf(phrase);
@@ -175,6 +209,57 @@ export const rank = (hits, limit) =>
 export const absentTerms = (documents, query) => {
     const ts = [...new Set(terms(query))];
     if (ts.length === 0) return [];
-    const hays = documents.map((d) => d.text.toLowerCase());
+    const hays = documents.map((d) => hayOf(d.text));
     return ts.filter((t) => !hays.some((h) => h.includes(t)));
 };
+
+/* ---------------------------------------------------------- search_corpus ---
+   ⭐ THE WHOLE TOOL, SHARED (2.4.1). It lived twice, once in each surface's callTool, and
+   its guidance line was about to be written twice too — the shape in which the envelope
+   drifted. One function, imported by both. */
+
+// A search returns EXCERPTS, and a model that reads only excerpts answers from them. So the
+// text a model reads says so at the moment it decides what to do next — in the result, where
+// every client delivers it, rather than only in server instructions a client may drop.
+const readingLine = (results, matches) => {
+    const slugs = results.map((r) => r.slug);
+    return (
+        `[Showing ${results.length} of ${matches} matching document${matches === 1 ? "" : "s"}, as excerpts. ` +
+        `To answer from the full texts, call read_documents with slugs ${JSON.stringify(slugs)}` +
+        (matches > results.length ? `; raise limit to see more of the ${matches} matches` : "") +
+        ".]"
+    );
+};
+
+export const searchCorpus = ({ documents, envelope }, args) => {
+    const q = String(args?.query ?? "");
+    if (!q) throw new Error("query is required");
+    const pool = documents.filter((d) => !args?.genre || d.genre === args.genre);
+    const hits = pool.map((d) => ({ d, m: match(d.text, q) })).filter((h) => h.m !== null);
+    const results = rank(hits, Number(args?.limit ?? 10)).map((h) => ({
+        ...envelope(h.d),
+        genre: h.d.genre,
+        excerpt: h.m.excerpt,
+        // ⭐ HOW it matched, not just that it did: a "terms" excerpt need not
+        // contain the literal query, and a caller reading it should know
+        // which question the excerpt is answering.
+        match: h.m.mode
+    }));
+    // ⚠️ `matches` is the TOTAL found, not the number returned — it used to be
+    // capped at `limit`, which made "10 matches" and "at least 10 matches"
+    // indistinguishable. `returned` carries the capped count.
+    const absent = hits.length ? [] : absentTerms(pool, q);
+    const readable = results.length
+        ? results.map((h) => `${h.slug} — ${h.title}${h.match === "terms" ? "  [all terms, not the phrase]" : ""}\n  ${h.excerpt}`).join("\n\n") +
+          "\n\n" + readingLine(results, hits.length)
+        : absent.length
+          ? `no document matches "${q}". No document contains: ${absent.join(", ")}.`
+          : `no document matches "${q}" — every term appears somewhere, but no single document holds them all. Try fewer terms.`;
+    return structuredWithText(readable, {
+        query: q,
+        matches: hits.length,
+        returned: results.length,
+        ...(absent.length ? { absent_terms: absent } : {}),
+        results: results
+    });
+};