@333eco/corpus 2.3.4 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -17
- package/dist/corpus.json +1618 -293
- package/package.json +1 -1
- package/src/base-tools.mjs +7 -3
- package/src/envelope.mjs +64 -0
- package/src/read-tools.mjs +210 -0
- package/src/resources.mjs +16 -3
- package/src/server.mjs +35 -67
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@333eco/corpus",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "An MCP server for an open-licensed corpus, served with verifiable provenance — every document carries its sha256, DOI and OpenTimestamps proof so a consuming agent can check its own citation.",
|
|
5
5
|
"license": "CC0-1.0",
|
|
6
6
|
"author": "Thon Ly",
|
package/src/base-tools.mjs
CHANGED
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
// closes over it) and those functions are genuinely different code. What must
|
|
6
6
|
// never differ is what the two servers ADVERTISE — a client picks a tool by name
|
|
7
7
|
// and schema, so a drifted definition is a client calling something that isn't
|
|
8
|
-
// there. The envelope
|
|
9
|
-
//
|
|
8
|
+
// there. ⚠️ The envelope was once duplicated on purpose, "diffable by eye"; it drifted
|
|
9
|
+
// unseen, and now lives in envelope.mjs, imported by both.
|
|
10
10
|
|
|
11
11
|
// ⭐⭐ EVERY TOOL HERE IS READ-ONLY, AND SAYING SO IS NOT DECORATION. A client that
|
|
12
12
|
// knows a call cannot mutate anything can stop putting a confirmation dialog in
|
|
@@ -51,7 +51,11 @@ export const BASE_TOOLS = [
|
|
|
51
51
|
type: "object",
|
|
52
52
|
properties: { slug: { type: "string", description: "Document slug, as returned by search_corpus or list_documents." } },
|
|
53
53
|
required: ["slug"]
|
|
54
|
-
}
|
|
54
|
+
},
|
|
55
|
+
// ⚠️ The largest document is ~140 KB — over the 25,000-token default at which
|
|
56
|
+
// Claude Code moves a tool result into a file. A document is the unit that
|
|
57
|
+
// verifies, so it is kept whole in the conversation.
|
|
58
|
+
_meta: { "anthropic/maxResultSizeChars": 250_000 }
|
|
55
59
|
},
|
|
56
60
|
{
|
|
57
61
|
name: "list_documents",
|
package/src/envelope.mjs
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// The provenance envelope — attached to every document a tool returns, on both surfaces.
|
|
2
|
+
//
|
|
3
|
+
// `verify` is the instruction rather than a promise: it tells the agent exactly what
|
|
4
|
+
// to run, so the claim is checkable without trusting this sentence either.
|
|
5
|
+
//
|
|
6
|
+
// ⚠️⚠️ IT WAS WRITTEN TWICE, ON PURPOSE, AND THE PURPOSE FAILED. Each server kept its own
|
|
7
|
+
// copy "so the two can be diffed by eye; if they ever disagree, that is a bug in one of
|
|
8
|
+
// them and the diff is where it shows." They did disagree — the OpenTimestamps
|
|
9
|
+
// instruction and the revised-since-deposit note had different wording on the two
|
|
10
|
+
// surfaces — and no eye caught it. The first check that COMPARED THE ANSWERS did, on its
|
|
11
|
+
// first run (2026-09-13). A rule that depends on someone looking is replaced by one
|
|
12
|
+
// function both surfaces import: the difference can no longer be written.
|
|
13
|
+
|
|
14
|
+
export const envelope = (d) => ({
|
|
15
|
+
slug: d.slug,
|
|
16
|
+
title: d.title,
|
|
17
|
+
licence: {
|
|
18
|
+
id: d.licence.id,
|
|
19
|
+
url: d.licence.url,
|
|
20
|
+
attribution_required: d.licence.attribution_required,
|
|
21
|
+
// Only present where it is actually owed, so an agent can act on the
|
|
22
|
+
// field's presence rather than parsing the licence id.
|
|
23
|
+
...(d.licence.attribution_required && d.authors ? { attribute_to: d.authors } : {})
|
|
24
|
+
},
|
|
25
|
+
provenance: {
|
|
26
|
+
...d.provenance,
|
|
27
|
+
// ⚠️ A LIVING DOCUMENT IS VERIFIED AND CITED WITH DIFFERENT DOIs, and
|
|
28
|
+
// saying only "cite accordingly" leaves the reader to guess which.
|
|
29
|
+
// The VERSION doi is the only one that can be true of the bytes here
|
|
30
|
+
// — it pins them — so it is what a hash check resolves against. The
|
|
31
|
+
// CONCEPT doi follows the document, so it is what a citation should
|
|
32
|
+
// name: a living register is revised on purpose, and a citation
|
|
33
|
+
// pinned to one revision goes stale by design rather than by accident.
|
|
34
|
+
...(d.status === "living" && d.provenance.concept_doi
|
|
35
|
+
? {
|
|
36
|
+
citation: {
|
|
37
|
+
cite: `https://doi.org/${d.provenance.concept_doi}`,
|
|
38
|
+
verify_against: d.provenance.doi ? `https://doi.org/${d.provenance.doi}` : null,
|
|
39
|
+
why: "This document is living — it is revised on purpose. Cite the concept DOI, which always resolves to the newest version; verify the text you were served against the version DOI and sha256 above, which pin these exact bytes."
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
: {}),
|
|
43
|
+
verify: {
|
|
44
|
+
// Concrete, because an instruction that says "the source file"
|
|
45
|
+
// without saying which one is not an instruction. All three source
|
|
46
|
+
// repositories are public, so this is genuinely runnable.
|
|
47
|
+
sha256: d.provenance.source_url
|
|
48
|
+
? `curl -sL ${d.provenance.source_url} | shasum -a 256 # compare to provenance.sha256`
|
|
49
|
+
: "shasum -a 256 <the source file> # compare to provenance.sha256",
|
|
50
|
+
...(d.provenance.doi ? { doi: `https://doi.org/${d.provenance.doi}` } : {}),
|
|
51
|
+
...(d.provenance.opentimestamps
|
|
52
|
+
? { opentimestamps: `ots verify ${d.path}.ots # in the source repository; the proof is anchored in the Bitcoin blockchain` }
|
|
53
|
+
: {}),
|
|
54
|
+
...(d.provenance.deposited_matches_current === false
|
|
55
|
+
? {
|
|
56
|
+
note:
|
|
57
|
+
"This document has been REVISED since its Zenodo deposit, so provenance.sha256 " +
|
|
58
|
+
"and provenance.deposited_sha256 differ legitimately. The DOI resolves to the " +
|
|
59
|
+
"deposited version; the text served here is newer. Cite accordingly."
|
|
60
|
+
}
|
|
61
|
+
: {})
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
});
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
// Reading more than one document at a time, and knowing that nothing is missing —
|
|
2
|
+
// shared by both surfaces.
|
|
3
|
+
//
|
|
4
|
+
// ⭐ WHY THIS EXISTS (A140). A model connected to this server gave answers that were
|
|
5
|
+
// "quite accurate but incomplete": search returns excerpts, ten by default, and a
|
|
6
|
+
// model answers from the few documents it happens to open. The window was not the
|
|
7
|
+
// limit — a shelf of the corpus fits — the reading pattern was. So a caller can now
|
|
8
|
+
// ask for a whole slice and see its size first.
|
|
9
|
+
//
|
|
10
|
+
// ⭐ Imported by both surfaces, like the program tools and the envelope: the filter
|
|
11
|
+
// here must be the SAME filter list_documents applies, or
|
|
12
|
+
// "read what I just listed" returns a different set. One function makes that true
|
|
13
|
+
// by construction instead of by vigilance.
|
|
14
|
+
|
|
15
|
+
import { READ_ONLY } from "./base-tools.mjs";
|
|
16
|
+
import { documentText } from "./resources.mjs";
|
|
17
|
+
|
|
18
|
+
// ~22k tokens of English prose: under the 25,000-token default at which Claude Code
|
|
19
|
+
// moves a tool result out of the conversation and into a file. Other clients cap
|
|
20
|
+
// differently; the caller can ask for more.
|
|
21
|
+
export const DEFAULT_PAGE_BYTES = 90_000;
|
|
22
|
+
export const MAX_PAGE_BYTES = 400_000;
|
|
23
|
+
|
|
24
|
+
const byteLength = (s) => new TextEncoder().encode(s).length;
|
|
25
|
+
|
|
26
|
+
/** The list_documents filter. Every facet is optional; absent means "any". */
|
|
27
|
+
export const filterDocuments = (documents, args) =>
|
|
28
|
+
documents.filter(
|
|
29
|
+
(d) =>
|
|
30
|
+
(!args?.genre || d.genre === args.genre) &&
|
|
31
|
+
(!args?.licence || d.licence.id === args.licence) &&
|
|
32
|
+
(!args?.voice || d.voice === args.voice) &&
|
|
33
|
+
(!args?.category || d.category === args.category)
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
export const READ_TOOLS = [
|
|
37
|
+
{
|
|
38
|
+
name: "read_documents",
|
|
39
|
+
title: "Read a set of documents in full",
|
|
40
|
+
annotations: { ...READ_ONLY, title: "Read a set of documents in full" },
|
|
41
|
+
// ⚠️ KEPT SHORT ON PURPOSE: every client that loads tool definitions up front pays
|
|
42
|
+
// for this text in every session, whether or not the corpus is used. The filter
|
|
43
|
+
// facets are described once, on list_documents.
|
|
44
|
+
description:
|
|
45
|
+
"Whole documents, never summarised: those matching the list_documents filters, or the given slugs. " +
|
|
46
|
+
"One content block each. Paged: follow next_cursor.",
|
|
47
|
+
inputSchema: {
|
|
48
|
+
type: "object",
|
|
49
|
+
properties: {
|
|
50
|
+
genre: { type: "string" },
|
|
51
|
+
category: { type: "string" },
|
|
52
|
+
voice: { type: "string" },
|
|
53
|
+
licence: { type: "string" },
|
|
54
|
+
slugs: { type: "array", items: { type: "string" } },
|
|
55
|
+
max_bytes: { type: "number", description: `Page size. Default ${DEFAULT_PAGE_BYTES}, max ${MAX_PAGE_BYTES}.` },
|
|
56
|
+
cursor: { type: "string" }
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
// ⚠️ Claude Code moves a tool result over its limit (25,000 tokens by default)
|
|
60
|
+
// into a file. A page is sized by the CALLER, so the tool declares the
|
|
61
|
+
// largest page it can return and the client keeps it in the conversation.
|
|
62
|
+
_meta: { "anthropic/maxResultSizeChars": MAX_PAGE_BYTES + 100_000 }
|
|
63
|
+
}
|
|
64
|
+
];
|
|
65
|
+
|
|
66
|
+
export const READ_TOOL_NAMES = READ_TOOLS.map((t) => t.name);
|
|
67
|
+
|
|
68
|
+
export const READ_INSTRUCTIONS =
|
|
69
|
+
" Search returns excerpts; to answer broadly, read whole documents with read_documents. Every document ends " +
|
|
70
|
+
"with an [END OF DOCUMENT] line — if it is missing, the text was cut short.";
|
|
71
|
+
|
|
72
|
+
// ⚠️ The cursor carries the INDEX VERSION as well as the offset. Pages are offsets
|
|
73
|
+
// into a list, and a list read across a redeploy is two lists: without the version
|
|
74
|
+
// a caller could be handed page 2 of a different corpus and never know.
|
|
75
|
+
const encodeCursor = (version, offset) => btoa(`${version}:${offset}`);
|
|
76
|
+
const decodeCursor = (cursor, version) => {
|
|
77
|
+
let raw;
|
|
78
|
+
try {
|
|
79
|
+
raw = atob(String(cursor));
|
|
80
|
+
} catch {
|
|
81
|
+
throw new Error(`invalid cursor: ${cursor}`);
|
|
82
|
+
}
|
|
83
|
+
const at = raw.lastIndexOf(":");
|
|
84
|
+
const v = raw.slice(0, at);
|
|
85
|
+
const offset = Number(raw.slice(at + 1));
|
|
86
|
+
if (at < 0 || !Number.isInteger(offset) || offset < 0) throw new Error(`invalid cursor: ${cursor}`);
|
|
87
|
+
if (v !== version) {
|
|
88
|
+
throw new Error(`this cursor belongs to corpus ${v}, and the server now serves ${version}. Start again without a cursor.`);
|
|
89
|
+
}
|
|
90
|
+
return offset;
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* read_documents. `ctx` is what differs between the surfaces:
|
|
95
|
+
* { documents, bySlug, envelope, version }.
|
|
96
|
+
*/
|
|
97
|
+
export const readDocuments = ({ documents, bySlug, envelope, version }, args) => {
|
|
98
|
+
const hasFilter = ["genre", "category", "voice", "licence"].some((k) => args?.[k]);
|
|
99
|
+
let matched;
|
|
100
|
+
if (args?.slugs !== undefined) {
|
|
101
|
+
if (hasFilter) throw new Error("give slugs or filters, not both.");
|
|
102
|
+
if (!Array.isArray(args.slugs) || !args.slugs.length) throw new Error("slugs must be a non-empty array of document slugs.");
|
|
103
|
+
const slugs = [...new Set(args.slugs.map(String))];
|
|
104
|
+
const unknown = slugs.filter((s) => !bySlug.has(s));
|
|
105
|
+
if (unknown.length) throw new Error(`no document with slug ${unknown.map((s) => `"${s}"`).join(", ")}. Call list_documents to see what is available.`);
|
|
106
|
+
matched = slugs.map((s) => bySlug.get(s));
|
|
107
|
+
} else {
|
|
108
|
+
matched = filterDocuments(documents, args);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const budget = args?.max_bytes === undefined ? DEFAULT_PAGE_BYTES : Number(args.max_bytes);
|
|
112
|
+
if (!Number.isFinite(budget) || budget <= 0) throw new Error("max_bytes must be a positive number.");
|
|
113
|
+
if (budget > MAX_PAGE_BYTES) throw new Error(`max_bytes may be at most ${MAX_PAGE_BYTES}; follow next_cursor for the rest.`);
|
|
114
|
+
|
|
115
|
+
const start = args?.cursor ? decodeCursor(args.cursor, version) : 0;
|
|
116
|
+
if (start > matched.length) throw new Error(`invalid cursor: ${args.cursor}`);
|
|
117
|
+
|
|
118
|
+
// Whole documents only. ⛔ A document is never split across pages: its hash, its
|
|
119
|
+
// header and its end marker describe the whole, and half of one verifies nothing.
|
|
120
|
+
// So a document larger than the budget arrives alone, over budget, and says so.
|
|
121
|
+
const page = [];
|
|
122
|
+
let used = 0;
|
|
123
|
+
for (let i = start; i < matched.length; i++) {
|
|
124
|
+
const text = documentText(matched[i]);
|
|
125
|
+
const size = byteLength(text);
|
|
126
|
+
if (page.length && used + size > budget) break;
|
|
127
|
+
page.push({ d: matched[i], text, size });
|
|
128
|
+
used += size;
|
|
129
|
+
}
|
|
130
|
+
const nextOffset = start + page.length;
|
|
131
|
+
const next = nextOffset < matched.length ? encodeCursor(version, nextOffset) : undefined;
|
|
132
|
+
|
|
133
|
+
if (!matched.length) {
|
|
134
|
+
return {
|
|
135
|
+
content: [{ type: "text", text: "no documents match these filters. Call list_documents to see the facets and their counts." }],
|
|
136
|
+
structuredContent: { matched: 0, returned: 0, offset: 0, page_bytes: 0, max_bytes: budget, documents: [] }
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const content = page.map((p) => ({ type: "text", text: p.text }));
|
|
141
|
+
// The one block that is not a document: what a reader of `content` alone needs in
|
|
142
|
+
// order to keep going, or to know it has everything.
|
|
143
|
+
content.push({
|
|
144
|
+
type: "text",
|
|
145
|
+
text: next
|
|
146
|
+
? `[PAGE — documents ${start + 1}–${nextOffset} of ${matched.length}. ${matched.length - nextOffset} more: call read_documents again with the same arguments and cursor "${next}".]`
|
|
147
|
+
: `[PAGE — documents ${start + 1}–${nextOffset} of ${matched.length}. That is every matching document.]`
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
content,
|
|
152
|
+
structuredContent: {
|
|
153
|
+
matched: matched.length,
|
|
154
|
+
returned: page.length,
|
|
155
|
+
offset: start,
|
|
156
|
+
page_bytes: used,
|
|
157
|
+
max_bytes: budget,
|
|
158
|
+
...(page.length === 1 && used > budget ? { over_budget: "one document larger than max_bytes, returned whole rather than split" } : {}),
|
|
159
|
+
...(next ? { next_cursor: next } : {}),
|
|
160
|
+
documents: page.map((p) => ({
|
|
161
|
+
...envelope(p.d),
|
|
162
|
+
genre: p.d.genre,
|
|
163
|
+
category: p.d.category,
|
|
164
|
+
voice: p.d.voice,
|
|
165
|
+
bytes: p.d.bytes,
|
|
166
|
+
words: p.d.words,
|
|
167
|
+
text_in: "content, one block per document in this order, each prefixed by its provenance header"
|
|
168
|
+
}))
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* list_documents' completeness block: the manifest's counts and the reasons, so the
|
|
175
|
+
* listing a caller already has is checkable against what the build considered.
|
|
176
|
+
*/
|
|
177
|
+
export const completenessOf = (corpus) =>
|
|
178
|
+
corpus.manifest
|
|
179
|
+
? {
|
|
180
|
+
candidates: corpus.manifest.candidates,
|
|
181
|
+
served: corpus.manifest.served,
|
|
182
|
+
excluded: corpus.manifest.excluded_documents,
|
|
183
|
+
held: corpus.manifest.held_documents,
|
|
184
|
+
note: "An unfiltered listing should name every served document; the full manifest is at https://corpus.333.eco/manifest.json."
|
|
185
|
+
}
|
|
186
|
+
: null;
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* ⛔ What both surfaces check before serving anything: do the documents loaded match
|
|
190
|
+
* the manifest the build wrote? Returns the problems; empty means serve.
|
|
191
|
+
* ⚠️ An index built before the manifest existed has none, and that is not refused —
|
|
192
|
+
* it is the previous release, not a damaged one.
|
|
193
|
+
*/
|
|
194
|
+
export const manifestProblems = (corpus) => {
|
|
195
|
+
const m = corpus.manifest;
|
|
196
|
+
if (!m) return [];
|
|
197
|
+
const problems = [];
|
|
198
|
+
if (m.candidates !== m.served + m.excluded + m.held) {
|
|
199
|
+
problems.push(`manifest does not reconcile: ${m.candidates} candidates vs ${m.served} + ${m.excluded} + ${m.held}`);
|
|
200
|
+
}
|
|
201
|
+
if (corpus.documents.length !== m.served) {
|
|
202
|
+
problems.push(`${corpus.documents.length} documents loaded, manifest names ${m.served}`);
|
|
203
|
+
}
|
|
204
|
+
const loaded = new Map(corpus.documents.map((d) => [d.slug, d.provenance?.sha256]));
|
|
205
|
+
for (const s of m.served_documents ?? []) {
|
|
206
|
+
if (!loaded.has(s.slug)) problems.push(`manifest names "${s.slug}", which is not loaded`);
|
|
207
|
+
else if (loaded.get(s.slug) !== s.sha256) problems.push(`"${s.slug}" does not carry the sha256 the manifest records`);
|
|
208
|
+
}
|
|
209
|
+
return problems;
|
|
210
|
+
};
|
package/src/resources.mjs
CHANGED
|
@@ -128,10 +128,23 @@ export const provenanceHeader = (d) => {
|
|
|
128
128
|
if (d.editorial?.annotation) {
|
|
129
129
|
L.push(`editorial: ${d.editorial.annotation.replace(/\s+/g, " ")}`);
|
|
130
130
|
}
|
|
131
|
-
L.push("[END PROVENANCE —
|
|
131
|
+
L.push("[END PROVENANCE — the document follows verbatim and ends at its [END OF DOCUMENT] line.]");
|
|
132
132
|
return L.join("\n");
|
|
133
133
|
};
|
|
134
134
|
|
|
135
|
+
// ⭐⭐ THE ELISION MARKER (A87, the peyyāla). The server never cuts a document — but a
|
|
136
|
+
// client may, and a cut document reads as a complete one: its last sentence is a real
|
|
137
|
+
// sentence. So the end is stated IN THE TEXT, where a cut removes it, rather than in
|
|
138
|
+
// a field that survives the cut and asserts a completeness the text no longer has.
|
|
139
|
+
// ⚠️ Like the header, it is NOT part of the hashed artifact; the hash covers the source.
|
|
140
|
+
export const endMarker = (d) =>
|
|
141
|
+
`[END OF DOCUMENT — ${d.slug}${typeof d.words === "number" ? ` · ${d.words} words` : ""}. ` +
|
|
142
|
+
"If this line is missing, the text above was cut short after it left the server.]";
|
|
143
|
+
|
|
144
|
+
// ONE function for every door into a document — get_document, resources/read and
|
|
145
|
+
// read_documents — so three ways in cannot become three different texts.
|
|
146
|
+
export const documentText = (d) => provenanceHeader(d) + "\n\n" + d.text + "\n\n" + endMarker(d);
|
|
147
|
+
|
|
135
148
|
export const readResource = (uri, bySlug) => {
|
|
136
149
|
const raw = String(uri ?? "");
|
|
137
150
|
if (!raw.startsWith(CORPUS_SCHEME)) {
|
|
@@ -145,8 +158,8 @@ export const readResource = (uri, bySlug) => {
|
|
|
145
158
|
{
|
|
146
159
|
uri: raw,
|
|
147
160
|
mimeType: mime(d),
|
|
148
|
-
// Header,
|
|
149
|
-
text:
|
|
161
|
+
// Header, the document, the end marker — exactly as the tools return it.
|
|
162
|
+
text: documentText(d)
|
|
150
163
|
}
|
|
151
164
|
]
|
|
152
165
|
};
|
package/src/server.mjs
CHANGED
|
@@ -36,9 +36,11 @@ import { fileURLToPath } from "node:url";
|
|
|
36
36
|
import { createInterface } from "node:readline";
|
|
37
37
|
import { RESOURCE_TEMPLATES, listResources, readResource, completeArgument } from "./resources.mjs";
|
|
38
38
|
import { structured, structuredWithText } from "./results.mjs";
|
|
39
|
-
import {
|
|
39
|
+
import { documentText } from "./resources.mjs";
|
|
40
|
+
import { envelope } from "./envelope.mjs";
|
|
40
41
|
import { PROMPTS, getPrompt } from "./prompts.mjs";
|
|
41
42
|
import { BASE_TOOLS, withFacets } from "./base-tools.mjs";
|
|
43
|
+
import { READ_TOOLS, READ_TOOL_NAMES, READ_INSTRUCTIONS, filterDocuments, readDocuments, completenessOf, manifestProblems } from "./read-tools.mjs";
|
|
42
44
|
import { PROGRAM_TOOLS, PROGRAM_TOOL_NAMES, PROGRAM_INSTRUCTIONS, callProgramTool } from "./program-tools.mjs";
|
|
43
45
|
import { match, rank, absentTerms } from "./search.mjs";
|
|
44
46
|
|
|
@@ -84,6 +86,15 @@ const bySlug = new Map(corpus.documents.map((d) => [d.slug, d]));
|
|
|
84
86
|
// not advertised at all. ⭐ An unadvertised tool is better than a tool that
|
|
85
87
|
// exists and always errors — a client can reason about the first.
|
|
86
88
|
const program = corpus.program ?? null;
|
|
89
|
+
// ⛔ A CORPUS THAT DOES NOT MATCH ITS OWN MANIFEST IS NOT SERVED. Every envelope would
|
|
90
|
+
// still verify, which is exactly why a missing document could otherwise go unseen.
|
|
91
|
+
{
|
|
92
|
+
const problems = manifestProblems(corpus);
|
|
93
|
+
if (problems.length) {
|
|
94
|
+
log("dist/corpus.json does not match its manifest — refusing to serve:\n " + problems.join("\n "));
|
|
95
|
+
process.exit(1);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
87
98
|
log(`${corpus.document_count} documents loaded —`, JSON.stringify(corpus.licences));
|
|
88
99
|
if (program) log(`research program: ${program.prediction_count} predictions, ${program.reconciliation.reconciles ? "register arithmetic reconciles" : "⚠️ REGISTER ARITHMETIC DOES NOT RECONCILE"}`);
|
|
89
100
|
|
|
@@ -98,68 +109,12 @@ const send = (msg) => process.stdout.write(JSON.stringify(msg) + "\n");
|
|
|
98
109
|
const result = (id, value) => send({ jsonrpc: "2.0", id, result: value });
|
|
99
110
|
const failure = (id, code, message) => send({ jsonrpc: "2.0", id, error: { code, message } });
|
|
100
111
|
|
|
101
|
-
/* ------------------------------------------------------------- the envelope ---
|
|
102
|
-
Attached to every document a tool returns. `verify` is the instruction rather
|
|
103
|
-
than a promise: it tells the agent exactly what to run, so the claim is
|
|
104
|
-
checkable without trusting this sentence either. */
|
|
105
|
-
|
|
106
|
-
const envelope = (d) => ({
|
|
107
|
-
slug: d.slug,
|
|
108
|
-
title: d.title,
|
|
109
|
-
licence: {
|
|
110
|
-
id: d.licence.id,
|
|
111
|
-
url: d.licence.url,
|
|
112
|
-
attribution_required: d.licence.attribution_required,
|
|
113
|
-
// Only present where it is actually owed, so an agent can act on the
|
|
114
|
-
// field's presence rather than parsing the licence id.
|
|
115
|
-
...(d.licence.attribution_required && d.authors ? { attribute_to: d.authors } : {})
|
|
116
|
-
},
|
|
117
|
-
provenance: {
|
|
118
|
-
...d.provenance,
|
|
119
|
-
// ⚠️ A LIVING DOCUMENT IS VERIFIED AND CITED WITH DIFFERENT DOIs, and
|
|
120
|
-
// saying only "cite accordingly" leaves the reader to guess which.
|
|
121
|
-
// The VERSION doi is the only one that can be true of the bytes here
|
|
122
|
-
// — it pins them — so it is what a hash check resolves against. The
|
|
123
|
-
// CONCEPT doi follows the document, so it is what a citation should
|
|
124
|
-
// name: a living register is revised on purpose, and a citation
|
|
125
|
-
// pinned to one revision goes stale by design rather than by accident.
|
|
126
|
-
...(d.status === "living" && d.provenance.concept_doi
|
|
127
|
-
? {
|
|
128
|
-
citation: {
|
|
129
|
-
cite: `https://doi.org/${d.provenance.concept_doi}`,
|
|
130
|
-
verify_against: d.provenance.doi ? `https://doi.org/${d.provenance.doi}` : null,
|
|
131
|
-
why: "This document is living — it is revised on purpose. Cite the concept DOI, which always resolves to the newest version; verify the text you were served against the version DOI and sha256 above, which pin these exact bytes."
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
: {}),
|
|
135
|
-
verify: {
|
|
136
|
-
// Concrete, because an instruction that says "the source file"
|
|
137
|
-
// without saying which one is not an instruction. All three source
|
|
138
|
-
// repositories are public, so this is genuinely runnable.
|
|
139
|
-
sha256: d.provenance.source_url
|
|
140
|
-
? `curl -sL ${d.provenance.source_url} | shasum -a 256 # compare to provenance.sha256`
|
|
141
|
-
: "shasum -a 256 <the source file> # compare to provenance.sha256",
|
|
142
|
-
...(d.provenance.doi ? { doi: `https://doi.org/${d.provenance.doi}` } : {}),
|
|
143
|
-
...(d.provenance.opentimestamps
|
|
144
|
-
? { opentimestamps: `ots verify ${d.path}.ots # in the source repository; the proof is anchored in the Bitcoin blockchain` }
|
|
145
|
-
: {}),
|
|
146
|
-
...(d.provenance.deposited_matches_current === false
|
|
147
|
-
? {
|
|
148
|
-
note:
|
|
149
|
-
"This document has been REVISED since its Zenodo deposit, so provenance.sha256 " +
|
|
150
|
-
"and provenance.deposited_sha256 differ legitimately. The DOI resolves to the " +
|
|
151
|
-
"deposited version; the text served here is newer. Cite accordingly."
|
|
152
|
-
}
|
|
153
|
-
: {})
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
});
|
|
157
112
|
|
|
158
113
|
/* -------------------------------------------------------------------- tools --- */
|
|
159
114
|
|
|
160
|
-
const TOOLS = BASE_TOOLS;
|
|
115
|
+
const TOOLS = [...BASE_TOOLS, ...READ_TOOLS];
|
|
161
116
|
|
|
162
|
-
const KNOWN_TOOLS = new Set([...
|
|
117
|
+
const KNOWN_TOOLS = new Set([...TOOLS, ...PROGRAM_TOOLS].map((t) => t.name));
|
|
163
118
|
|
|
164
119
|
const text = (value) => ({ content: [{ type: "text", text: typeof value === "string" ? value : JSON.stringify(value, null, 2) }] });
|
|
165
120
|
|
|
@@ -204,9 +159,11 @@ const callTool = (name, args) => {
|
|
|
204
159
|
// bytes resources/read returns, so the two doors into a document agree —
|
|
205
160
|
// and the envelope goes to `structuredContent` WITHOUT the body. Split by
|
|
206
161
|
// role; nothing is sent twice.
|
|
207
|
-
return structuredWithText(
|
|
162
|
+
return structuredWithText(documentText(d), {
|
|
208
163
|
...envelope(d),
|
|
209
164
|
genre: d.genre,
|
|
165
|
+
bytes: d.bytes,
|
|
166
|
+
words: d.words,
|
|
210
167
|
// ⚠️ category and voice were in list_documents and NOT here, so a
|
|
211
168
|
// caller who fetched ONE document could not see the facets it had
|
|
212
169
|
// just filtered on — the discovery call and the read call disagreed
|
|
@@ -229,16 +186,18 @@ const callTool = (name, args) => {
|
|
|
229
186
|
...(d.segments ? { segments: d.segments } : {}),
|
|
230
187
|
// ⚠️ Deliberately absent: the body is in `content`. Carrying it here
|
|
231
188
|
// too is the duplication this shape exists to avoid.
|
|
232
|
-
text_in: "content[0].text, prefixed by the provenance header"
|
|
189
|
+
text_in: "content[0].text, prefixed by the provenance header and ending at its [END OF DOCUMENT] line"
|
|
233
190
|
});
|
|
234
191
|
}
|
|
235
192
|
|
|
193
|
+
if (name === "read_documents") {
|
|
194
|
+
return readDocuments({ documents: corpus.documents, bySlug, envelope, version: String(corpus.package_version ?? "") }, args);
|
|
195
|
+
}
|
|
196
|
+
|
|
236
197
|
if (name === "list_documents") {
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
(!args?.voice || d.voice === args.voice) &&
|
|
241
|
-
(!args?.category || d.category === args.category))
|
|
198
|
+
// ⭐ The SAME filter read_documents applies (read-tools.mjs), so "read what I
|
|
199
|
+
// just listed" cannot return a different set.
|
|
200
|
+
const list = filterDocuments(corpus.documents, args)
|
|
242
201
|
.map((d) => ({
|
|
243
202
|
slug: d.slug,
|
|
244
203
|
title: d.title,
|
|
@@ -255,10 +214,16 @@ const callTool = (name, args) => {
|
|
|
255
214
|
date: d.date,
|
|
256
215
|
licence: d.licence.id,
|
|
257
216
|
doi: d.provenance.doi,
|
|
258
|
-
opentimestamps: d.provenance.opentimestamps
|
|
217
|
+
opentimestamps: d.provenance.opentimestamps,
|
|
218
|
+
// A140: size before reading. Bytes and words, never tokens.
|
|
219
|
+
bytes: d.bytes,
|
|
220
|
+
words: d.words
|
|
259
221
|
}));
|
|
260
222
|
return structured({
|
|
261
223
|
count: list.length,
|
|
224
|
+
// For THIS filtered set — what reading it with read_documents would cost.
|
|
225
|
+
total_bytes: list.reduce((n, d) => n + (d.bytes ?? 0), 0),
|
|
226
|
+
total_words: list.reduce((n, d) => n + (d.words ?? 0), 0),
|
|
262
227
|
licences: corpus.licences,
|
|
263
228
|
// The shelves, so a caller can narrow without guessing the vocabulary.
|
|
264
229
|
categories: corpus.documents.reduce((a, d) => ((a[d.category ?? "uncategorised"] = (a[d.category ?? "uncategorised"] ?? 0) + 1), a), {}),
|
|
@@ -266,6 +231,8 @@ const callTool = (name, args) => {
|
|
|
266
231
|
// documents means nothing without the honest denominator beside it,
|
|
267
232
|
// the same reason list_predictions returns by_state unfiltered.
|
|
268
233
|
...(corpus.voices ? { voices: corpus.voices } : {}),
|
|
234
|
+
// A87: the manifest's counts and reasons beside the listing they account for.
|
|
235
|
+
...(corpus.manifest ? { completeness: completenessOf(corpus) } : {}),
|
|
269
236
|
documents: list
|
|
270
237
|
});
|
|
271
238
|
}
|
|
@@ -292,6 +259,7 @@ const handlers = {
|
|
|
292
259
|
"carry a DOI and an OpenTimestamps proof anchored in Bitcoin, so you can check any passage you intend to " +
|
|
293
260
|
"cite rather than trusting this server. Documents under CC-BY carry attribute_to in their licence block; " +
|
|
294
261
|
"honour it. Text is returned verbatim and is never summarised, because a summary cannot be hash-verified." +
|
|
262
|
+
READ_INSTRUCTIONS +
|
|
295
263
|
(program ? PROGRAM_INSTRUCTIONS : "")
|
|
296
264
|
}),
|
|
297
265
|
// ⭐ withFacets fills list_documents' facet enumerations FROM THE CORPUS.
|