@333eco/corpus 2.3.4 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@333eco/corpus",
3
- "version": "2.3.4",
3
+ "version": "2.4.0",
4
4
  "description": "An MCP server for an open-licensed corpus, served with verifiable provenance — every document carries its sha256, DOI and OpenTimestamps proof so a consuming agent can check its own citation.",
5
5
  "license": "CC0-1.0",
6
6
  "author": "Thon Ly",
@@ -5,8 +5,8 @@
5
5
  // closes over it) and those functions are genuinely different code. What must
6
6
  // never differ is what the two servers ADVERTISE — a client picks a tool by name
7
7
  // and schema, so a drifted definition is a client calling something that isn't
8
- // there. The envelope stays duplicated on purpose; that one is twenty lines and
9
- // diffable by eye.
8
+ // there. ⚠️ The envelope was once duplicated on purpose, "diffable by eye"; it drifted
9
+ // unseen, and now lives in envelope.mjs, imported by both.
10
10
 
11
11
  // ⭐⭐ EVERY TOOL HERE IS READ-ONLY, AND SAYING SO IS NOT DECORATION. A client that
12
12
  // knows a call cannot mutate anything can stop putting a confirmation dialog in
@@ -51,7 +51,11 @@ export const BASE_TOOLS = [
51
51
  type: "object",
52
52
  properties: { slug: { type: "string", description: "Document slug, as returned by search_corpus or list_documents." } },
53
53
  required: ["slug"]
54
- }
54
+ },
55
+ // ⚠️ The largest document is ~140 KB — over the 25,000-token default at which
56
+ // Claude Code moves a tool result into a file. A document is the unit that
57
+ // verifies, so it is kept whole in the conversation.
58
+ _meta: { "anthropic/maxResultSizeChars": 250_000 }
55
59
  },
56
60
  {
57
61
  name: "list_documents",
@@ -0,0 +1,64 @@
1
+ // The provenance envelope — attached to every document a tool returns, on both surfaces.
2
+ //
3
+ // `verify` is the instruction rather than a promise: it tells the agent exactly what
4
+ // to run, so the claim is checkable without trusting this sentence either.
5
+ //
6
+ // ⚠️⚠️ IT WAS WRITTEN TWICE, ON PURPOSE, AND THE PURPOSE FAILED. Each server kept its own
7
+ // copy "so the two can be diffed by eye; if they ever disagree, that is a bug in one of
8
+ // them and the diff is where it shows." They did disagree — the OpenTimestamps
9
+ // instruction and the revised-since-deposit note had different wording on the two
10
+ // surfaces — and no eye caught it. The first check that COMPARED THE ANSWERS did, on its
11
+ // first run (2026-09-13). A rule that depends on someone looking is replaced by one
12
+ // function both surfaces import: the difference can no longer be written.
13
+
14
+ export const envelope = (d) => ({
15
+ slug: d.slug,
16
+ title: d.title,
17
+ licence: {
18
+ id: d.licence.id,
19
+ url: d.licence.url,
20
+ attribution_required: d.licence.attribution_required,
21
+ // Only present where it is actually owed, so an agent can act on the
22
+ // field's presence rather than parsing the licence id.
23
+ ...(d.licence.attribution_required && d.authors ? { attribute_to: d.authors } : {})
24
+ },
25
+ provenance: {
26
+ ...d.provenance,
27
+ // ⚠️ A LIVING DOCUMENT IS VERIFIED AND CITED WITH DIFFERENT DOIs, and
28
+ // saying only "cite accordingly" leaves the reader to guess which.
29
+ // The VERSION doi is the only one that can be true of the bytes here
30
+ // — it pins them — so it is what a hash check resolves against. The
31
+ // CONCEPT doi follows the document, so it is what a citation should
32
+ // name: a living register is revised on purpose, and a citation
33
+ // pinned to one revision goes stale by design rather than by accident.
34
+ ...(d.status === "living" && d.provenance.concept_doi
35
+ ? {
36
+ citation: {
37
+ cite: `https://doi.org/${d.provenance.concept_doi}`,
38
+ verify_against: d.provenance.doi ? `https://doi.org/${d.provenance.doi}` : null,
39
+ why: "This document is living — it is revised on purpose. Cite the concept DOI, which always resolves to the newest version; verify the text you were served against the version DOI and sha256 above, which pin these exact bytes."
40
+ }
41
+ }
42
+ : {}),
43
+ verify: {
44
+ // Concrete, because an instruction that says "the source file"
45
+ // without saying which one is not an instruction. All three source
46
+ // repositories are public, so this is genuinely runnable.
47
+ sha256: d.provenance.source_url
48
+ ? `curl -sL ${d.provenance.source_url} | shasum -a 256 # compare to provenance.sha256`
49
+ : "shasum -a 256 <the source file> # compare to provenance.sha256",
50
+ ...(d.provenance.doi ? { doi: `https://doi.org/${d.provenance.doi}` } : {}),
51
+ ...(d.provenance.opentimestamps
52
+ ? { opentimestamps: `ots verify ${d.path}.ots # in the source repository; the proof is anchored in the Bitcoin blockchain` }
53
+ : {}),
54
+ ...(d.provenance.deposited_matches_current === false
55
+ ? {
56
+ note:
57
+ "This document has been REVISED since its Zenodo deposit, so provenance.sha256 " +
58
+ "and provenance.deposited_sha256 differ legitimately. The DOI resolves to the " +
59
+ "deposited version; the text served here is newer. Cite accordingly."
60
+ }
61
+ : {})
62
+ }
63
+ }
64
+ });
@@ -0,0 +1,210 @@
1
+ // Reading more than one document at a time, and knowing that nothing is missing —
2
+ // shared by both surfaces.
3
+ //
4
+ // ⭐ WHY THIS EXISTS (A140). A model connected to this server gave answers that were
5
+ // "quite accurate but incomplete": search returns excerpts, ten by default, and a
6
+ // model answers from the few documents it happens to open. The window was not the
7
+ // limit — a shelf of the corpus fits — the reading pattern was. So a caller can now
8
+ // ask for a whole slice and see its size first.
9
+ //
10
+ // ⭐ Imported by both surfaces, like the program tools and the envelope: the filter
11
+ // here must be the SAME filter list_documents applies, or
12
+ // "read what I just listed" returns a different set. One function makes that true
13
+ // by construction instead of by vigilance.
14
+
15
+ import { READ_ONLY } from "./base-tools.mjs";
16
+ import { documentText } from "./resources.mjs";
17
+
18
+ // ~22k tokens of English prose: under the 25,000-token default at which Claude Code
19
+ // moves a tool result out of the conversation and into a file. Other clients cap
20
+ // differently; the caller can ask for more.
21
+ export const DEFAULT_PAGE_BYTES = 90_000;
22
+ export const MAX_PAGE_BYTES = 400_000;
23
+
24
+ const byteLength = (s) => new TextEncoder().encode(s).length;
25
+
26
+ /** The list_documents filter. Every facet is optional; absent means "any". */
27
+ export const filterDocuments = (documents, args) =>
28
+ documents.filter(
29
+ (d) =>
30
+ (!args?.genre || d.genre === args.genre) &&
31
+ (!args?.licence || d.licence.id === args.licence) &&
32
+ (!args?.voice || d.voice === args.voice) &&
33
+ (!args?.category || d.category === args.category)
34
+ );
35
+
36
+ export const READ_TOOLS = [
37
+ {
38
+ name: "read_documents",
39
+ title: "Read a set of documents in full",
40
+ annotations: { ...READ_ONLY, title: "Read a set of documents in full" },
41
+ // ⚠️ KEPT SHORT ON PURPOSE: every client that loads tool definitions up front pays
42
+ // for this text in every session, whether or not the corpus is used. The filter
43
+ // facets are described once, on list_documents.
44
+ description:
45
+ "Whole documents, never summarised: those matching the list_documents filters, or the given slugs. " +
46
+ "One content block each. Paged: follow next_cursor.",
47
+ inputSchema: {
48
+ type: "object",
49
+ properties: {
50
+ genre: { type: "string" },
51
+ category: { type: "string" },
52
+ voice: { type: "string" },
53
+ licence: { type: "string" },
54
+ slugs: { type: "array", items: { type: "string" } },
55
+ max_bytes: { type: "number", description: `Page size. Default ${DEFAULT_PAGE_BYTES}, max ${MAX_PAGE_BYTES}.` },
56
+ cursor: { type: "string" }
57
+ }
58
+ },
59
+ // ⚠️ Claude Code moves a tool result over its limit (25,000 tokens by default)
60
+ // into a file. A page is sized by the CALLER, so the tool declares the
61
+ // largest page it can return and the client keeps it in the conversation.
62
+ _meta: { "anthropic/maxResultSizeChars": MAX_PAGE_BYTES + 100_000 }
63
+ }
64
+ ];
65
+
66
+ export const READ_TOOL_NAMES = READ_TOOLS.map((t) => t.name);
67
+
68
+ export const READ_INSTRUCTIONS =
69
+ " Search returns excerpts; to answer broadly, read whole documents with read_documents. Every document ends " +
70
+ "with an [END OF DOCUMENT] line — if it is missing, the text was cut short.";
71
+
72
+ // ⚠️ The cursor carries the INDEX VERSION as well as the offset. Pages are offsets
73
+ // into a list, and a list read across a redeploy is two lists: without the version
74
+ // a caller could be handed page 2 of a different corpus and never know.
75
+ const encodeCursor = (version, offset) => btoa(`${version}:${offset}`);
76
+ const decodeCursor = (cursor, version) => {
77
+ let raw;
78
+ try {
79
+ raw = atob(String(cursor));
80
+ } catch {
81
+ throw new Error(`invalid cursor: ${cursor}`);
82
+ }
83
+ const at = raw.lastIndexOf(":");
84
+ const v = raw.slice(0, at);
85
+ const offset = Number(raw.slice(at + 1));
86
+ if (at < 0 || !Number.isInteger(offset) || offset < 0) throw new Error(`invalid cursor: ${cursor}`);
87
+ if (v !== version) {
88
+ throw new Error(`this cursor belongs to corpus ${v}, and the server now serves ${version}. Start again without a cursor.`);
89
+ }
90
+ return offset;
91
+ };
92
+
93
+ /**
94
+ * read_documents. `ctx` is what differs between the surfaces:
95
+ * { documents, bySlug, envelope, version }.
96
+ */
97
+ export const readDocuments = ({ documents, bySlug, envelope, version }, args) => {
98
+ const hasFilter = ["genre", "category", "voice", "licence"].some((k) => args?.[k]);
99
+ let matched;
100
+ if (args?.slugs !== undefined) {
101
+ if (hasFilter) throw new Error("give slugs or filters, not both.");
102
+ if (!Array.isArray(args.slugs) || !args.slugs.length) throw new Error("slugs must be a non-empty array of document slugs.");
103
+ const slugs = [...new Set(args.slugs.map(String))];
104
+ const unknown = slugs.filter((s) => !bySlug.has(s));
105
+ if (unknown.length) throw new Error(`no document with slug ${unknown.map((s) => `"${s}"`).join(", ")}. Call list_documents to see what is available.`);
106
+ matched = slugs.map((s) => bySlug.get(s));
107
+ } else {
108
+ matched = filterDocuments(documents, args);
109
+ }
110
+
111
+ const budget = args?.max_bytes === undefined ? DEFAULT_PAGE_BYTES : Number(args.max_bytes);
112
+ if (!Number.isFinite(budget) || budget <= 0) throw new Error("max_bytes must be a positive number.");
113
+ if (budget > MAX_PAGE_BYTES) throw new Error(`max_bytes may be at most ${MAX_PAGE_BYTES}; follow next_cursor for the rest.`);
114
+
115
+ const start = args?.cursor ? decodeCursor(args.cursor, version) : 0;
116
+ if (start > matched.length) throw new Error(`invalid cursor: ${args.cursor}`);
117
+
118
+ // Whole documents only. ⛔ A document is never split across pages: its hash, its
119
+ // header and its end marker describe the whole, and half of one verifies nothing.
120
+ // So a document larger than the budget arrives alone, over budget, and says so.
121
+ const page = [];
122
+ let used = 0;
123
+ for (let i = start; i < matched.length; i++) {
124
+ const text = documentText(matched[i]);
125
+ const size = byteLength(text);
126
+ if (page.length && used + size > budget) break;
127
+ page.push({ d: matched[i], text, size });
128
+ used += size;
129
+ }
130
+ const nextOffset = start + page.length;
131
+ const next = nextOffset < matched.length ? encodeCursor(version, nextOffset) : undefined;
132
+
133
+ if (!matched.length) {
134
+ return {
135
+ content: [{ type: "text", text: "no documents match these filters. Call list_documents to see the facets and their counts." }],
136
+ structuredContent: { matched: 0, returned: 0, offset: 0, page_bytes: 0, max_bytes: budget, documents: [] }
137
+ };
138
+ }
139
+
140
+ const content = page.map((p) => ({ type: "text", text: p.text }));
141
+ // The one block that is not a document: what a reader of `content` alone needs in
142
+ // order to keep going, or to know it has everything.
143
+ content.push({
144
+ type: "text",
145
+ text: next
146
+ ? `[PAGE — documents ${start + 1}–${nextOffset} of ${matched.length}. ${matched.length - nextOffset} more: call read_documents again with the same arguments and cursor "${next}".]`
147
+ : `[PAGE — documents ${start + 1}–${nextOffset} of ${matched.length}. That is every matching document.]`
148
+ });
149
+
150
+ return {
151
+ content,
152
+ structuredContent: {
153
+ matched: matched.length,
154
+ returned: page.length,
155
+ offset: start,
156
+ page_bytes: used,
157
+ max_bytes: budget,
158
+ ...(page.length === 1 && used > budget ? { over_budget: "one document larger than max_bytes, returned whole rather than split" } : {}),
159
+ ...(next ? { next_cursor: next } : {}),
160
+ documents: page.map((p) => ({
161
+ ...envelope(p.d),
162
+ genre: p.d.genre,
163
+ category: p.d.category,
164
+ voice: p.d.voice,
165
+ bytes: p.d.bytes,
166
+ words: p.d.words,
167
+ text_in: "content, one block per document in this order, each prefixed by its provenance header"
168
+ }))
169
+ }
170
+ };
171
+ };
172
+
173
+ /**
174
+ * list_documents' completeness block: the manifest's counts and the reasons, so the
175
+ * listing a caller already has is checkable against what the build considered.
176
+ */
177
+ export const completenessOf = (corpus) =>
178
+ corpus.manifest
179
+ ? {
180
+ candidates: corpus.manifest.candidates,
181
+ served: corpus.manifest.served,
182
+ excluded: corpus.manifest.excluded_documents,
183
+ held: corpus.manifest.held_documents,
184
+ note: "An unfiltered listing should name every served document; the full manifest is at https://corpus.333.eco/manifest.json."
185
+ }
186
+ : null;
187
+
188
+ /**
189
+ * ⛔ What both surfaces check before serving anything: do the documents loaded match
190
+ * the manifest the build wrote? Returns the problems; empty means serve.
191
+ * ⚠️ An index built before the manifest existed has none, and that is not refused —
192
+ * it is the previous release, not a damaged one.
193
+ */
194
+ export const manifestProblems = (corpus) => {
195
+ const m = corpus.manifest;
196
+ if (!m) return [];
197
+ const problems = [];
198
+ if (m.candidates !== m.served + m.excluded + m.held) {
199
+ problems.push(`manifest does not reconcile: ${m.candidates} candidates vs ${m.served} + ${m.excluded} + ${m.held}`);
200
+ }
201
+ if (corpus.documents.length !== m.served) {
202
+ problems.push(`${corpus.documents.length} documents loaded, manifest names ${m.served}`);
203
+ }
204
+ const loaded = new Map(corpus.documents.map((d) => [d.slug, d.provenance?.sha256]));
205
+ for (const s of m.served_documents ?? []) {
206
+ if (!loaded.has(s.slug)) problems.push(`manifest names "${s.slug}", which is not loaded`);
207
+ else if (loaded.get(s.slug) !== s.sha256) problems.push(`"${s.slug}" does not carry the sha256 the manifest records`);
208
+ }
209
+ return problems;
210
+ };
package/src/resources.mjs CHANGED
@@ -128,10 +128,23 @@ export const provenanceHeader = (d) => {
128
128
  if (d.editorial?.annotation) {
129
129
  L.push(`editorial: ${d.editorial.annotation.replace(/\s+/g, " ")}`);
130
130
  }
131
- L.push("[END PROVENANCE — everything below this line is the document, verbatim.]");
131
+ L.push("[END PROVENANCE — the document follows verbatim and ends at its [END OF DOCUMENT] line.]");
132
132
  return L.join("\n");
133
133
  };
134
134
 
135
+ // ⭐⭐ THE ELISION MARKER (A87, the peyyāla). The server never cuts a document — but a
136
+ // client may, and a cut document reads as a complete one: its last sentence is a real
137
+ // sentence. So the end is stated IN THE TEXT, where a cut removes it, rather than in
138
+ // a field that survives the cut and asserts a completeness the text no longer has.
139
+ // ⚠️ Like the header, it is NOT part of the hashed artifact; the hash covers the source.
140
+ export const endMarker = (d) =>
141
+ `[END OF DOCUMENT — ${d.slug}${typeof d.words === "number" ? ` · ${d.words} words` : ""}. ` +
142
+ "If this line is missing, the text above was cut short after it left the server.]";
143
+
144
+ // ONE function for every door into a document — get_document, resources/read and
145
+ // read_documents — so three ways in cannot become three different texts.
146
+ export const documentText = (d) => provenanceHeader(d) + "\n\n" + d.text + "\n\n" + endMarker(d);
147
+
135
148
  export const readResource = (uri, bySlug) => {
136
149
  const raw = String(uri ?? "");
137
150
  if (!raw.startsWith(CORPUS_SCHEME)) {
@@ -145,8 +158,8 @@ export const readResource = (uri, bySlug) => {
145
158
  {
146
159
  uri: raw,
147
160
  mimeType: mime(d),
148
- // Header, blank line, then the document exactly as the tools return it.
149
- text: provenanceHeader(d) + "\n\n" + d.text
161
+ // Header, the document, the end marker — exactly as the tools return it.
162
+ text: documentText(d)
150
163
  }
151
164
  ]
152
165
  };
package/src/server.mjs CHANGED
@@ -36,9 +36,11 @@ import { fileURLToPath } from "node:url";
36
36
  import { createInterface } from "node:readline";
37
37
  import { RESOURCE_TEMPLATES, listResources, readResource, completeArgument } from "./resources.mjs";
38
38
  import { structured, structuredWithText } from "./results.mjs";
39
- import { provenanceHeader } from "./resources.mjs";
39
+ import { documentText } from "./resources.mjs";
40
+ import { envelope } from "./envelope.mjs";
40
41
  import { PROMPTS, getPrompt } from "./prompts.mjs";
41
42
  import { BASE_TOOLS, withFacets } from "./base-tools.mjs";
43
+ import { READ_TOOLS, READ_TOOL_NAMES, READ_INSTRUCTIONS, filterDocuments, readDocuments, completenessOf, manifestProblems } from "./read-tools.mjs";
42
44
  import { PROGRAM_TOOLS, PROGRAM_TOOL_NAMES, PROGRAM_INSTRUCTIONS, callProgramTool } from "./program-tools.mjs";
43
45
  import { match, rank, absentTerms } from "./search.mjs";
44
46
 
@@ -84,6 +86,15 @@ const bySlug = new Map(corpus.documents.map((d) => [d.slug, d]));
84
86
  // not advertised at all. ⭐ An unadvertised tool is better than a tool that
85
87
  // exists and always errors — a client can reason about the first.
86
88
  const program = corpus.program ?? null;
89
+ // ⛔ A CORPUS THAT DOES NOT MATCH ITS OWN MANIFEST IS NOT SERVED. Every envelope would
90
+ // still verify, which is exactly why a missing document could otherwise go unseen.
91
+ {
92
+ const problems = manifestProblems(corpus);
93
+ if (problems.length) {
94
+ log("dist/corpus.json does not match its manifest — refusing to serve:\n " + problems.join("\n "));
95
+ process.exit(1);
96
+ }
97
+ }
87
98
  log(`${corpus.document_count} documents loaded —`, JSON.stringify(corpus.licences));
88
99
  if (program) log(`research program: ${program.prediction_count} predictions, ${program.reconciliation.reconciles ? "register arithmetic reconciles" : "⚠️ REGISTER ARITHMETIC DOES NOT RECONCILE"}`);
89
100
 
@@ -98,68 +109,12 @@ const send = (msg) => process.stdout.write(JSON.stringify(msg) + "\n");
98
109
  const result = (id, value) => send({ jsonrpc: "2.0", id, result: value });
99
110
  const failure = (id, code, message) => send({ jsonrpc: "2.0", id, error: { code, message } });
100
111
 
101
- /* ------------------------------------------------------------- the envelope ---
102
- Attached to every document a tool returns. `verify` is the instruction rather
103
- than a promise: it tells the agent exactly what to run, so the claim is
104
- checkable without trusting this sentence either. */
105
-
106
- const envelope = (d) => ({
107
- slug: d.slug,
108
- title: d.title,
109
- licence: {
110
- id: d.licence.id,
111
- url: d.licence.url,
112
- attribution_required: d.licence.attribution_required,
113
- // Only present where it is actually owed, so an agent can act on the
114
- // field's presence rather than parsing the licence id.
115
- ...(d.licence.attribution_required && d.authors ? { attribute_to: d.authors } : {})
116
- },
117
- provenance: {
118
- ...d.provenance,
119
- // ⚠️ A LIVING DOCUMENT IS VERIFIED AND CITED WITH DIFFERENT DOIs, and
120
- // saying only "cite accordingly" leaves the reader to guess which.
121
- // The VERSION doi is the only one that can be true of the bytes here
122
- // — it pins them — so it is what a hash check resolves against. The
123
- // CONCEPT doi follows the document, so it is what a citation should
124
- // name: a living register is revised on purpose, and a citation
125
- // pinned to one revision goes stale by design rather than by accident.
126
- ...(d.status === "living" && d.provenance.concept_doi
127
- ? {
128
- citation: {
129
- cite: `https://doi.org/${d.provenance.concept_doi}`,
130
- verify_against: d.provenance.doi ? `https://doi.org/${d.provenance.doi}` : null,
131
- why: "This document is living — it is revised on purpose. Cite the concept DOI, which always resolves to the newest version; verify the text you were served against the version DOI and sha256 above, which pin these exact bytes."
132
- }
133
- }
134
- : {}),
135
- verify: {
136
- // Concrete, because an instruction that says "the source file"
137
- // without saying which one is not an instruction. All three source
138
- // repositories are public, so this is genuinely runnable.
139
- sha256: d.provenance.source_url
140
- ? `curl -sL ${d.provenance.source_url} | shasum -a 256 # compare to provenance.sha256`
141
- : "shasum -a 256 <the source file> # compare to provenance.sha256",
142
- ...(d.provenance.doi ? { doi: `https://doi.org/${d.provenance.doi}` } : {}),
143
- ...(d.provenance.opentimestamps
144
- ? { opentimestamps: `ots verify ${d.path}.ots # in the source repository; the proof is anchored in the Bitcoin blockchain` }
145
- : {}),
146
- ...(d.provenance.deposited_matches_current === false
147
- ? {
148
- note:
149
- "This document has been REVISED since its Zenodo deposit, so provenance.sha256 " +
150
- "and provenance.deposited_sha256 differ legitimately. The DOI resolves to the " +
151
- "deposited version; the text served here is newer. Cite accordingly."
152
- }
153
- : {})
154
- }
155
- }
156
- });
157
112
 
158
113
  /* -------------------------------------------------------------------- tools --- */
159
114
 
160
- const TOOLS = BASE_TOOLS;
115
+ const TOOLS = [...BASE_TOOLS, ...READ_TOOLS];
161
116
 
162
- const KNOWN_TOOLS = new Set([...BASE_TOOLS, ...PROGRAM_TOOLS].map((t) => t.name));
117
+ const KNOWN_TOOLS = new Set([...TOOLS, ...PROGRAM_TOOLS].map((t) => t.name));
163
118
 
164
119
  const text = (value) => ({ content: [{ type: "text", text: typeof value === "string" ? value : JSON.stringify(value, null, 2) }] });
165
120
 
@@ -204,9 +159,11 @@ const callTool = (name, args) => {
204
159
  // bytes resources/read returns, so the two doors into a document agree —
205
160
  // and the envelope goes to `structuredContent` WITHOUT the body. Split by
206
161
  // role; nothing is sent twice.
207
- return structuredWithText(provenanceHeader(d) + "\n\n" + d.text, {
162
+ return structuredWithText(documentText(d), {
208
163
  ...envelope(d),
209
164
  genre: d.genre,
165
+ bytes: d.bytes,
166
+ words: d.words,
210
167
  // ⚠️ category and voice were in list_documents and NOT here, so a
211
168
  // caller who fetched ONE document could not see the facets it had
212
169
  // just filtered on — the discovery call and the read call disagreed
@@ -229,16 +186,18 @@ const callTool = (name, args) => {
229
186
  ...(d.segments ? { segments: d.segments } : {}),
230
187
  // ⚠️ Deliberately absent: the body is in `content`. Carrying it here
231
188
  // too is the duplication this shape exists to avoid.
232
- text_in: "content[0].text, prefixed by the provenance header"
189
+ text_in: "content[0].text, prefixed by the provenance header and ending at its [END OF DOCUMENT] line"
233
190
  });
234
191
  }
235
192
 
193
+ if (name === "read_documents") {
194
+ return readDocuments({ documents: corpus.documents, bySlug, envelope, version: String(corpus.package_version ?? "") }, args);
195
+ }
196
+
236
197
  if (name === "list_documents") {
237
- const list = corpus.documents
238
- .filter((d) => (!args?.genre || d.genre === args.genre) &&
239
- (!args?.licence || d.licence.id === args.licence) &&
240
- (!args?.voice || d.voice === args.voice) &&
241
- (!args?.category || d.category === args.category))
198
+ // ⭐ The SAME filter read_documents applies (read-tools.mjs), so "read what I
199
+ // just listed" cannot return a different set.
200
+ const list = filterDocuments(corpus.documents, args)
242
201
  .map((d) => ({
243
202
  slug: d.slug,
244
203
  title: d.title,
@@ -255,10 +214,16 @@ const callTool = (name, args) => {
255
214
  date: d.date,
256
215
  licence: d.licence.id,
257
216
  doi: d.provenance.doi,
258
- opentimestamps: d.provenance.opentimestamps
217
+ opentimestamps: d.provenance.opentimestamps,
218
+ // A140: size before reading. Bytes and words, never tokens.
219
+ bytes: d.bytes,
220
+ words: d.words
259
221
  }));
260
222
  return structured({
261
223
  count: list.length,
224
+ // For THIS filtered set — what reading it with read_documents would cost.
225
+ total_bytes: list.reduce((n, d) => n + (d.bytes ?? 0), 0),
226
+ total_words: list.reduce((n, d) => n + (d.words ?? 0), 0),
262
227
  licences: corpus.licences,
263
228
  // The shelves, so a caller can narrow without guessing the vocabulary.
264
229
  categories: corpus.documents.reduce((a, d) => ((a[d.category ?? "uncategorised"] = (a[d.category ?? "uncategorised"] ?? 0) + 1), a), {}),
@@ -266,6 +231,8 @@ const callTool = (name, args) => {
266
231
  // documents means nothing without the honest denominator beside it,
267
232
  // the same reason list_predictions returns by_state unfiltered.
268
233
  ...(corpus.voices ? { voices: corpus.voices } : {}),
234
+ // A87: the manifest's counts and reasons beside the listing they account for.
235
+ ...(corpus.manifest ? { completeness: completenessOf(corpus) } : {}),
269
236
  documents: list
270
237
  });
271
238
  }
@@ -292,6 +259,7 @@ const handlers = {
292
259
  "carry a DOI and an OpenTimestamps proof anchored in Bitcoin, so you can check any passage you intend to " +
293
260
  "cite rather than trusting this server. Documents under CC-BY carry attribute_to in their licence block; " +
294
261
  "honour it. Text is returned verbatim and is never summarised, because a summary cannot be hash-verified." +
262
+ READ_INSTRUCTIONS +
295
263
  (program ? PROGRAM_INSTRUCTIONS : "")
296
264
  }),
297
265
  // ⭐ withFacets fills list_documents' facet enumerations FROM THE CORPUS.