@mfdaves/okf-mcp 0.9.1 → 0.12.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +172 -17
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/policies/authoring-safety.md +11 -1
- package/package.json +17 -5
- package/server.json +2 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +391 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +776 -0
- package/src/authoring/live.js +789 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +137 -0
- package/src/authoring/receipts.js +329 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli.js +184 -151
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/registry.js +328 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +472 -0
- package/src/mcp/tools/catalog.js +244 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +213 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +195 -77
- package/src/producers.js +265 -101
- package/src/project.js +277 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,567 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Budgeted context assembly over the lexical index (plan C-1, C-3, WP3a).
|
|
4
|
+
//
|
|
5
|
+
// Candidate concepts come from the 0.10 `searchConcepts` with every filter,
|
|
6
|
+
// mode and confidence field intact (at most `conceptLimit` of them). Only
|
|
7
|
+
// their sections are then scored, on the fly, against the same analysed
|
|
8
|
+
// query (src/context/score.js: BM25 over each section's own text with the
|
|
9
|
+
// concept bundle's stemming, plus at most 20% for the concept's shared
|
|
10
|
+
// metadata). Nothing is indexed or cached per OKF index, so a write costs
|
|
11
|
+
// context assembly nothing; the one memo is `conceptSections(doc)` per
|
|
12
|
+
// document object, which a functional index update keeps for unchanged
|
|
13
|
+
// documents. Order: the concept rank from searchConcepts leads (it already
|
|
14
|
+
// weighed title, description, tags and aliases); within a concept, sections
|
|
15
|
+
// whose own text matches come before sections found through metadata only,
|
|
16
|
+
// by score; an own-text section may move ahead of at most `conceptLift`
|
|
17
|
+
// weaker concepts in proportion to its score over the best section's. Ties
|
|
18
|
+
// fall back to document order. Sections are deduplicated by anchor and by
|
|
19
|
+
// identical text hash and packed greedily in that order while they fit.
|
|
20
|
+
//
|
|
21
|
+
// The budget covers the whole response as delivered (0.11.0-rc.2): usedTokens
|
|
22
|
+
// is ceil(characters / 4) of the JSON serialisation of the entire result,
|
|
23
|
+
// every top-level field included (query, concepts, queryTerms, the counters,
|
|
24
|
+
// usedTokens itself) and every section field (scoring fields too). Sections
|
|
25
|
+
// are packed into what the rest of the response leaves; a section's `tokens`
|
|
26
|
+
// is the same estimate of the section object alone. The estimator is named in
|
|
27
|
+
// the result and the budget is never exceeded, unless it is smaller than the
|
|
28
|
+
// response's fixed fields (the query echo and field names), in which case no
|
|
29
|
+
// section and no concept is returned and usedTokens says how large the
|
|
30
|
+
// response is. When the fields around the sections do not fit, the candidate
|
|
31
|
+
// concept list is shortened from its tail first (omittedConcepts counts the
|
|
32
|
+
// concepts left out). Provenance is `compact` by default (status, freshness,
|
|
33
|
+
// verified, sourceCount); `provenance: "full"` adds staleAfter, trustTier,
|
|
34
|
+
// the verified and generated events and the sources, and `provenance: "none"`
|
|
35
|
+
// (or `includeProvenance: false`) leaves it out. When the best section alone
|
|
36
|
+
// is larger than what is left it is returned on its own with its text cut at
|
|
37
|
+
// a sentence boundary, and `truncated` says so; when not even its fields
|
|
38
|
+
// without text fit, no section is returned.
|
|
39
|
+
//
|
|
40
|
+
// The result is structured JSON; catalog text appears only inside section
|
|
41
|
+
// fields (B-6, C-3). Provenance per section is the concept's v0.2 signals.
|
|
42
|
+
|
|
43
|
+
const { normalizeV02Signals } = require("../v02");
|
|
44
|
+
const { analyzeQuery, describeSearchIndex, searchLanguages } = require("../search/lexical");
|
|
45
|
+
const { FRESHNESS_PRIOR, searchConcepts } = require("../search/query");
|
|
46
|
+
const { scoreSections } = require("./score");
|
|
47
|
+
const { conceptSections } = require("./sections");
|
|
48
|
+
|
|
49
|
+
const CONTEXT_TOKEN_ESTIMATOR = "chars/4";
|
|
50
|
+
const CHARS_PER_TOKEN = 4;
|
|
51
|
+
const DEFAULT_CONTEXT_BUDGET_TOKENS = 1200;
|
|
52
|
+
const MAX_CONTEXT_BUDGET_TOKENS = 100000;
|
|
53
|
+
// The budget is the primary constraint; the section cap only stops a swarm
|
|
54
|
+
// of tiny sections (fixture sections run 100-450 characters, so 1,200
|
|
55
|
+
// tokens hold 10-15 of them).
|
|
56
|
+
const DEFAULT_CONTEXT_LIMIT = 20;
|
|
57
|
+
const MAX_CONTEXT_LIMIT = 250;
|
|
58
|
+
// Concepts whose sections are ranked; the candidate step's page size.
|
|
59
|
+
const DEFAULT_CONCEPT_LIMIT = 10;
|
|
60
|
+
const MAX_CONCEPT_LIMIT = 50;
|
|
61
|
+
// Multiplier on the preamble's score (title, description, aliases and the
|
|
62
|
+
// text before the first heading). Chosen on the dev split of both fixtures
|
|
63
|
+
// among 1.0, 0.7 and 0.5 for the best labelled-anchor MRR without a lower
|
|
64
|
+
// anchor hit rate (test/fixtures/retrieval/reports/context-0.11-2026-10-24).
|
|
65
|
+
const DEFAULT_PREAMBLE_WEIGHT = 0.5;
|
|
66
|
+
// Sections with metadata evidence only score at most this share of the
|
|
67
|
+
// weakest section with own-text evidence.
|
|
68
|
+
const METADATA_ONLY_CEILING = 0.5;
|
|
69
|
+
// A truncation keeps the last sentence end in the allowed span only when it
|
|
70
|
+
// leaves at least this share of the span; otherwise it cuts at a word.
|
|
71
|
+
const MIN_SENTENCE_SHARE = 0.5;
|
|
72
|
+
const SENTENCE_END = /[.!?](?=\s|$)|\n/g;
|
|
73
|
+
// Options of assembleContext itself; everything else goes to searchConcepts.
|
|
74
|
+
const CONTEXT_OPTIONS = [
|
|
75
|
+
"budgetTokens",
|
|
76
|
+
"includeProvenance",
|
|
77
|
+
"provenance",
|
|
78
|
+
"conceptLimit",
|
|
79
|
+
"conceptLift",
|
|
80
|
+
"preambleWeight",
|
|
81
|
+
"abstainOnLowConfidence",
|
|
82
|
+
];
|
|
83
|
+
const SOURCE_FIELDS = ["id", "resource", "author"];
|
|
84
|
+
const PROVENANCE_MODES = Object.freeze(["compact", "full", "none"]);
|
|
85
|
+
const DEFAULT_PROVENANCE = "compact";
|
|
86
|
+
// How many weaker concepts a strong own-text section may move ahead of (one
|
|
87
|
+
// scoring as high as the best own-text section moves the full distance).
|
|
88
|
+
// Chosen on the dev split among 0, 2 and 4 by the highest anchor hit summed
|
|
89
|
+
// over both fixtures, then the highest anchor MRR (the sweep is in
|
|
90
|
+
// test/fixtures/retrieval/reports/context-0.11-2026-10-24).
|
|
91
|
+
const DEFAULT_CONCEPT_LIFT = 4;
|
|
92
|
+
const MAX_CONCEPT_LIFT = 10;
|
|
93
|
+
// A section's content fields (the scoring fields are tokens, score, relevance,
|
|
94
|
+
// matchedTerms and coverage). Since 0.11.0-rc.2 every field counts against
|
|
95
|
+
// the budget; the list stays exported for library callers.
|
|
96
|
+
const CONTENT_FIELDS = Object.freeze([
|
|
97
|
+
"uri",
|
|
98
|
+
"anchor",
|
|
99
|
+
"heading",
|
|
100
|
+
"title",
|
|
101
|
+
"text",
|
|
102
|
+
"truncated",
|
|
103
|
+
"provenance",
|
|
104
|
+
]);
|
|
105
|
+
|
|
106
|
+
function estimateContextTokens(text) {
|
|
107
|
+
return Math.ceil(String(text || "").length / CHARS_PER_TOKEN);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Characters of a value as delivered (compact JSON).
|
|
111
|
+
function serializedLength(value) {
|
|
112
|
+
return JSON.stringify(value).length;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// A section's `tokens` is part of what it costs: settle it on the estimate of
|
|
116
|
+
// the section object including its own digits. Returns the section's length.
|
|
117
|
+
function settleTokens(section) {
|
|
118
|
+
let tokens = 0;
|
|
119
|
+
for (;;) {
|
|
120
|
+
section.tokens = tokens;
|
|
121
|
+
const length = serializedLength(section);
|
|
122
|
+
const next = Math.ceil(length / CHARS_PER_TOKEN);
|
|
123
|
+
if (next <= tokens) return length;
|
|
124
|
+
tokens = next;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// The whole response's estimate with usedTokens set to it (a fixed point:
|
|
129
|
+
// usedTokens is part of the response it measures).
|
|
130
|
+
function settleUsedTokens(response) {
|
|
131
|
+
let used = 0;
|
|
132
|
+
for (;;) {
|
|
133
|
+
response.usedTokens = used;
|
|
134
|
+
const next = Math.ceil(serializedLength(response) / CHARS_PER_TOKEN);
|
|
135
|
+
if (next <= used) return used;
|
|
136
|
+
used = next;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function integerOption(value, name, fallback, maximum) {
|
|
141
|
+
if (value === undefined || value === null || value === "") {
|
|
142
|
+
return fallback;
|
|
143
|
+
}
|
|
144
|
+
const number = Number(value);
|
|
145
|
+
if (!Number.isInteger(number) || number < 1 || number > maximum) {
|
|
146
|
+
throw new TypeError(`${name} must be an integer from 1 through ${maximum}.`);
|
|
147
|
+
}
|
|
148
|
+
return number;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function weightOption(value) {
|
|
152
|
+
if (value === undefined || value === null || value === "") {
|
|
153
|
+
return DEFAULT_PREAMBLE_WEIGHT;
|
|
154
|
+
}
|
|
155
|
+
const number = Number(value);
|
|
156
|
+
if (!Number.isFinite(number) || number <= 0 || number > 1) {
|
|
157
|
+
throw new TypeError("preambleWeight must be a number greater than 0 and at most 1.");
|
|
158
|
+
}
|
|
159
|
+
return number;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function provenanceOption(config) {
|
|
163
|
+
if (config.includeProvenance === false) {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
const mode =
|
|
167
|
+
config.provenance === undefined || config.provenance === null
|
|
168
|
+
? DEFAULT_PROVENANCE
|
|
169
|
+
: config.provenance;
|
|
170
|
+
if (!PROVENANCE_MODES.includes(mode)) {
|
|
171
|
+
throw new TypeError(`provenance must be one of ${PROVENANCE_MODES.join(", ")}.`);
|
|
172
|
+
}
|
|
173
|
+
return mode === "none" ? null : mode;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function liftOption(value) {
|
|
177
|
+
if (value === undefined || value === null || value === "") {
|
|
178
|
+
return DEFAULT_CONCEPT_LIFT;
|
|
179
|
+
}
|
|
180
|
+
const number = Number(value);
|
|
181
|
+
if (!Number.isFinite(number) || number < 0 || number > MAX_CONCEPT_LIFT) {
|
|
182
|
+
throw new TypeError(`conceptLift must be a number from 0 through ${MAX_CONCEPT_LIFT}.`);
|
|
183
|
+
}
|
|
184
|
+
return number;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function signalsFor(doc, asOf) {
|
|
188
|
+
return asOf ? normalizeV02Signals(doc, { asOf }) : doc.signals || {};
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// R-6 factors, the same as concept search applies to candidates.
|
|
192
|
+
function priorFactor(signals) {
|
|
193
|
+
let factor = 1;
|
|
194
|
+
if (signals.status === "deprecated") {
|
|
195
|
+
factor *= FRESHNESS_PRIOR.deprecated;
|
|
196
|
+
} else if (signals.status === "draft") {
|
|
197
|
+
factor *= FRESHNESS_PRIOR.draft;
|
|
198
|
+
}
|
|
199
|
+
if (signals.freshness === "stale") {
|
|
200
|
+
factor *= FRESHNESS_PRIOR.stale;
|
|
201
|
+
}
|
|
202
|
+
return factor;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function compactSource(source) {
|
|
206
|
+
const out = {};
|
|
207
|
+
SOURCE_FIELDS.forEach((field) => {
|
|
208
|
+
if (source[field] !== undefined) {
|
|
209
|
+
out[field] = source[field];
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
return out;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const validSources = (signals) =>
|
|
216
|
+
(signals.sources || []).filter(
|
|
217
|
+
(source) => source && typeof source === "object" && source.valid !== false,
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
function provenanceOf(signals, mode) {
|
|
221
|
+
if (mode === "compact") {
|
|
222
|
+
return {
|
|
223
|
+
status: signals.status ?? null,
|
|
224
|
+
freshness: signals.freshness ?? null,
|
|
225
|
+
verified: (signals.verifiedEvents || []).length > 0,
|
|
226
|
+
sourceCount: validSources(signals).length,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
return {
|
|
230
|
+
status: signals.status ?? null,
|
|
231
|
+
staleAfter: signals.staleAfter ?? null,
|
|
232
|
+
freshness: signals.freshness ?? null,
|
|
233
|
+
trustTier: signals.trustTier ?? null,
|
|
234
|
+
verified: (signals.verifiedEvents || []).map((event) => ({
|
|
235
|
+
by: event.by ?? null,
|
|
236
|
+
at: event.at ?? null,
|
|
237
|
+
})),
|
|
238
|
+
generated: signals.generated
|
|
239
|
+
? { by: signals.generated.by ?? null, at: signals.generated.at ?? null }
|
|
240
|
+
: null,
|
|
241
|
+
sources: validSources(signals).map(compactSource),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// The stemming languages of a bundle: its detected set under `auto`, else
|
|
246
|
+
// the configured union (the same plan concept search indexes with).
|
|
247
|
+
function bundleLanguages(index, options) {
|
|
248
|
+
const plan = describeSearchIndex(index, options);
|
|
249
|
+
return (bundle) => plan.bundles?.[bundle]?.languages || plan.languages;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Sections of the candidate concepts that match at least one query term,
|
|
253
|
+
// ordered by score, candidate rank and document position, one per anchor
|
|
254
|
+
// and per distinct text.
|
|
255
|
+
function rankSections(index, options, analysis, candidates, context) {
|
|
256
|
+
const entries = [];
|
|
257
|
+
candidates.forEach((candidate, conceptRank) => {
|
|
258
|
+
const doc = index.byUri.get(candidate.uri);
|
|
259
|
+
if (!doc) {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
conceptSections(doc).forEach((section, position) => {
|
|
263
|
+
entries.push({ doc, concept: doc, section, position, conceptRank });
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
const scored = scoreSections(entries, analysis, {
|
|
267
|
+
languagesFor: bundleLanguages(index, options),
|
|
268
|
+
preambleWeight: context.preambleWeight,
|
|
269
|
+
});
|
|
270
|
+
const prior = (hit) =>
|
|
271
|
+
options.freshnessPrior ? priorFactor(context.signalsOf(hit.entry.doc)) : 1;
|
|
272
|
+
const ranked = scored.map((hit) => ({
|
|
273
|
+
entry: hit.entry,
|
|
274
|
+
tier: hit.tier,
|
|
275
|
+
score: hit.tier ? hit.score * prior(hit) : 0,
|
|
276
|
+
coverage: hit.coverage,
|
|
277
|
+
positions: hit.positions,
|
|
278
|
+
conceptRank: hit.entry.conceptRank,
|
|
279
|
+
}));
|
|
280
|
+
// A metadata-only section scores below the weakest own-text section of its
|
|
281
|
+
// concept (of any candidate when its concept has none).
|
|
282
|
+
const weakest = new Map();
|
|
283
|
+
ranked.forEach((hit) => {
|
|
284
|
+
if (hit.tier) {
|
|
285
|
+
const uri = hit.entry.doc.uri;
|
|
286
|
+
weakest.set(uri, Math.min(weakest.get(uri) ?? Number.POSITIVE_INFINITY, hit.score));
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
const ownScores = ranked.filter((hit) => hit.tier).map((hit) => hit.score);
|
|
290
|
+
const fallback = ownScores.length ? Math.min(...ownScores) : 1;
|
|
291
|
+
const best = ownScores.length ? Math.max(...ownScores) : 0;
|
|
292
|
+
ranked.forEach((hit, position) => {
|
|
293
|
+
if (!hit.tier) {
|
|
294
|
+
const floor = weakest.get(hit.entry.doc.uri) ?? fallback;
|
|
295
|
+
hit.score =
|
|
296
|
+
METADATA_ONLY_CEILING * floor * scored[position].metadataShare * prior(scored[position]);
|
|
297
|
+
}
|
|
298
|
+
// Concept rank leads; an own-text section moves ahead of at most
|
|
299
|
+
// `conceptLift` weaker concepts, in proportion to its score over the best.
|
|
300
|
+
hit.key =
|
|
301
|
+
hit.conceptRank - (hit.tier && best > 0 ? context.conceptLift * (hit.score / best) : 0);
|
|
302
|
+
});
|
|
303
|
+
ranked.sort(
|
|
304
|
+
(left, right) =>
|
|
305
|
+
left.key - right.key ||
|
|
306
|
+
right.tier - left.tier ||
|
|
307
|
+
right.score - left.score ||
|
|
308
|
+
left.entry.position - right.entry.position,
|
|
309
|
+
);
|
|
310
|
+
const top = best;
|
|
311
|
+
const anchors = new Set();
|
|
312
|
+
const hashes = new Set();
|
|
313
|
+
return ranked.filter((candidate) => {
|
|
314
|
+
const { anchor, hash } = candidate.entry.section;
|
|
315
|
+
if (anchors.has(anchor) || hashes.has(hash)) {
|
|
316
|
+
return false;
|
|
317
|
+
}
|
|
318
|
+
anchors.add(anchor);
|
|
319
|
+
hashes.add(hash);
|
|
320
|
+
candidate.relevance = top > 0 ? candidate.score / top : 0;
|
|
321
|
+
return true;
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// The longest prefix within `maxChars` that ends a sentence (keeping at least
|
|
326
|
+
// half of the span), else the last word boundary, else a hard cut.
|
|
327
|
+
function truncateText(text, maxChars) {
|
|
328
|
+
if (text.length <= maxChars) {
|
|
329
|
+
return text;
|
|
330
|
+
}
|
|
331
|
+
let cut = -1;
|
|
332
|
+
SENTENCE_END.lastIndex = 0;
|
|
333
|
+
for (let match = SENTENCE_END.exec(text); match; match = SENTENCE_END.exec(text)) {
|
|
334
|
+
const end = match[0] === "\n" ? match.index : match.index + 1;
|
|
335
|
+
if (end > maxChars) {
|
|
336
|
+
break;
|
|
337
|
+
}
|
|
338
|
+
cut = end;
|
|
339
|
+
}
|
|
340
|
+
if (cut < maxChars * MIN_SENTENCE_SHARE) {
|
|
341
|
+
const space = text.lastIndexOf(" ", maxChars);
|
|
342
|
+
cut = space > 0 ? space : maxChars;
|
|
343
|
+
}
|
|
344
|
+
return text.slice(0, cut).trimEnd();
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// The best section cut to fit `limitChars` once serialised (its tokens
|
|
348
|
+
// settled), or null when its fields without any text already exceed it. JSON
|
|
349
|
+
// escaping can make the text cost more than its length, so the allowance
|
|
350
|
+
// shrinks until it fits.
|
|
351
|
+
function truncatedSection(build, text, limitChars) {
|
|
352
|
+
const empty = build("", true);
|
|
353
|
+
let allowance = limitChars - settleTokens(empty);
|
|
354
|
+
while (allowance > 0) {
|
|
355
|
+
const section = build(truncateText(text, allowance), true);
|
|
356
|
+
const excess = settleTokens(section) - limitChars;
|
|
357
|
+
if (excess <= 0) {
|
|
358
|
+
return section;
|
|
359
|
+
}
|
|
360
|
+
allowance -= excess;
|
|
361
|
+
}
|
|
362
|
+
return null;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// Greedy by rank into `availableChars` (what the rest of the response leaves
|
|
366
|
+
// of the budget): a section is taken when its serialised length, plus the
|
|
367
|
+
// comma that separates it from the previous one, fits what is left and the
|
|
368
|
+
// limit allows it, otherwise skipped. A best section larger than everything
|
|
369
|
+
// available is returned alone, cut.
|
|
370
|
+
function packSections(ranked, availableChars, limit, build) {
|
|
371
|
+
if (ranked.length) {
|
|
372
|
+
const best = ranked[0];
|
|
373
|
+
const whole = build(best, best.entry.section.text, false);
|
|
374
|
+
if (settleTokens(whole) > availableChars) {
|
|
375
|
+
const cut = truncatedSection(
|
|
376
|
+
(text, isCut) => build(best, text, isCut),
|
|
377
|
+
best.entry.section.text,
|
|
378
|
+
availableChars,
|
|
379
|
+
);
|
|
380
|
+
if (!cut) {
|
|
381
|
+
return { packed: [], omitted: ranked.length, truncated: false };
|
|
382
|
+
}
|
|
383
|
+
return { packed: [cut], omitted: ranked.length - 1, truncated: true };
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
const packed = [];
|
|
387
|
+
let used = 0;
|
|
388
|
+
let omitted = 0;
|
|
389
|
+
ranked.forEach((candidate) => {
|
|
390
|
+
const section = build(candidate, candidate.entry.section.text, false);
|
|
391
|
+
const cost = settleTokens(section) + (packed.length ? 1 : 0);
|
|
392
|
+
if (packed.length >= limit || used + cost > availableChars) {
|
|
393
|
+
omitted += 1;
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
packed.push(section);
|
|
397
|
+
used += cost;
|
|
398
|
+
});
|
|
399
|
+
return { packed, omitted, truncated: false };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
// The response around the sections, sized for the worst case of the fields
|
|
403
|
+
// packing can still change (usedTokens at the full budget, every section
|
|
404
|
+
// omitted, truncated false), so what it leaves is a safe allowance. The
|
|
405
|
+
// candidate list is shortened from its tail while even that does not fit.
|
|
406
|
+
function responseFrame(fields, candidates, budgetTokens, rankedCount) {
|
|
407
|
+
const limitChars = budgetTokens * CHARS_PER_TOKEN;
|
|
408
|
+
const frame = (concepts) =>
|
|
409
|
+
Object.assign({}, fields, {
|
|
410
|
+
usedTokens: budgetTokens,
|
|
411
|
+
concepts,
|
|
412
|
+
sections: [],
|
|
413
|
+
totalSections: rankedCount,
|
|
414
|
+
omittedSections: rankedCount,
|
|
415
|
+
truncated: false,
|
|
416
|
+
omittedConcepts: candidates.length - concepts.length,
|
|
417
|
+
});
|
|
418
|
+
let concepts = candidates;
|
|
419
|
+
let length = serializedLength(frame(concepts));
|
|
420
|
+
while (length > limitChars && concepts.length) {
|
|
421
|
+
concepts = concepts.slice(0, -1);
|
|
422
|
+
length = serializedLength(frame(concepts));
|
|
423
|
+
}
|
|
424
|
+
return { concepts, availableChars: Math.max(0, limitChars - length) };
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
function sectionResult(candidate, text, truncated, analysis, provenanceMode, signalsOf) {
|
|
428
|
+
const { entry } = candidate;
|
|
429
|
+
const result = {
|
|
430
|
+
uri: entry.concept.uri,
|
|
431
|
+
anchor: entry.section.anchor,
|
|
432
|
+
heading: entry.section.heading,
|
|
433
|
+
title: entry.concept.title,
|
|
434
|
+
text,
|
|
435
|
+
tokens: 0,
|
|
436
|
+
truncated,
|
|
437
|
+
score: candidate.score,
|
|
438
|
+
relevance: candidate.relevance,
|
|
439
|
+
matchedTerms: candidate.positions.map((position) => analysis.tokens[position].raw),
|
|
440
|
+
coverage: candidate.coverage,
|
|
441
|
+
};
|
|
442
|
+
if (provenanceMode) {
|
|
443
|
+
result.provenance = provenanceOf(signalsOf(entry.concept), provenanceMode);
|
|
444
|
+
}
|
|
445
|
+
return result;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function assembleContext(index, options) {
|
|
449
|
+
const config = options || {};
|
|
450
|
+
const query = String(config.query || "").trim();
|
|
451
|
+
if (!query) {
|
|
452
|
+
throw new TypeError("Context assembly requires a query.");
|
|
453
|
+
}
|
|
454
|
+
const budgetTokens = integerOption(
|
|
455
|
+
config.budgetTokens,
|
|
456
|
+
"budgetTokens",
|
|
457
|
+
DEFAULT_CONTEXT_BUDGET_TOKENS,
|
|
458
|
+
MAX_CONTEXT_BUDGET_TOKENS,
|
|
459
|
+
);
|
|
460
|
+
const limit = integerOption(config.limit, "limit", DEFAULT_CONTEXT_LIMIT, MAX_CONTEXT_LIMIT);
|
|
461
|
+
const conceptLimit = integerOption(
|
|
462
|
+
config.conceptLimit,
|
|
463
|
+
"conceptLimit",
|
|
464
|
+
DEFAULT_CONCEPT_LIMIT,
|
|
465
|
+
MAX_CONCEPT_LIMIT,
|
|
466
|
+
);
|
|
467
|
+
const preambleWeight = weightOption(config.preambleWeight);
|
|
468
|
+
const conceptLift = liftOption(config.conceptLift);
|
|
469
|
+
const provenanceMode = provenanceOption(config);
|
|
470
|
+
if (
|
|
471
|
+
config.abstainOnLowConfidence !== undefined &&
|
|
472
|
+
typeof config.abstainOnLowConfidence !== "boolean"
|
|
473
|
+
) {
|
|
474
|
+
throw new TypeError("abstainOnLowConfidence must be a boolean.");
|
|
475
|
+
}
|
|
476
|
+
const searchOptions = Object.assign({}, config, {
|
|
477
|
+
query,
|
|
478
|
+
limit: conceptLimit,
|
|
479
|
+
offset: 0,
|
|
480
|
+
detail: "compact",
|
|
481
|
+
});
|
|
482
|
+
CONTEXT_OPTIONS.forEach((name) => {
|
|
483
|
+
delete searchOptions[name];
|
|
484
|
+
});
|
|
485
|
+
const response = searchConcepts(index, searchOptions);
|
|
486
|
+
const candidates = response.results.map((entry, position) => ({
|
|
487
|
+
uri: entry.uri,
|
|
488
|
+
rank: position + 1,
|
|
489
|
+
coverage: entry.coverage,
|
|
490
|
+
relevance: entry.relevance,
|
|
491
|
+
}));
|
|
492
|
+
const analysis = analyzeQuery(query, { languages: searchLanguages(index, searchOptions) });
|
|
493
|
+
// Signals once per concept per call: with asOf they are re-derived.
|
|
494
|
+
const signalsCache = new Map();
|
|
495
|
+
const signalsOf = (doc) => {
|
|
496
|
+
if (!signalsCache.has(doc.uri)) {
|
|
497
|
+
signalsCache.set(doc.uri, signalsFor(doc, config.asOf));
|
|
498
|
+
}
|
|
499
|
+
return signalsCache.get(doc.uri);
|
|
500
|
+
};
|
|
501
|
+
// WP3c-2: with abstainOnLowConfidence, a low-confidence candidate step yields no sections;
|
|
502
|
+
// the response still carries the confidence block and the candidate concepts.
|
|
503
|
+
const abstained = config.abstainOnLowConfidence === true && Boolean(response.lowConfidence);
|
|
504
|
+
const ranked =
|
|
505
|
+
analysis.tokens.length && candidates.length && !abstained
|
|
506
|
+
? rankSections(index, searchOptions, analysis, candidates, {
|
|
507
|
+
preambleWeight,
|
|
508
|
+
conceptLift,
|
|
509
|
+
signalsOf,
|
|
510
|
+
})
|
|
511
|
+
: [];
|
|
512
|
+
const fields = {
|
|
513
|
+
query,
|
|
514
|
+
budgetTokens,
|
|
515
|
+
estimator: CONTEXT_TOKEN_ESTIMATOR,
|
|
516
|
+
usedTokens: 0,
|
|
517
|
+
mode: response.mode,
|
|
518
|
+
lowConfidence: Boolean(response.lowConfidence),
|
|
519
|
+
queryTerms: response.queryTerms || [],
|
|
520
|
+
droppedTerms: response.droppedTerms || [],
|
|
521
|
+
confidence: response.confidence || null,
|
|
522
|
+
};
|
|
523
|
+
if (config.abstainOnLowConfidence !== undefined) fields.abstained = abstained;
|
|
524
|
+
const frame = responseFrame(fields, candidates, budgetTokens, ranked.length);
|
|
525
|
+
const packing = packSections(ranked, frame.availableChars, limit, (candidate, text, truncated) =>
|
|
526
|
+
sectionResult(candidate, text, truncated, analysis, provenanceMode, signalsOf),
|
|
527
|
+
);
|
|
528
|
+
const result = Object.assign(fields, {
|
|
529
|
+
concepts: frame.concepts,
|
|
530
|
+
sections: packing.packed,
|
|
531
|
+
totalSections: ranked.length,
|
|
532
|
+
omittedSections: packing.omitted,
|
|
533
|
+
truncated: packing.truncated,
|
|
534
|
+
omittedConcepts: candidates.length - frame.concepts.length,
|
|
535
|
+
});
|
|
536
|
+
// The frame was sized for the worst case, so this holds; the loop is a
|
|
537
|
+
// guard that keeps the promise if a field ever grows past that estimate.
|
|
538
|
+
while (settleUsedTokens(result) > budgetTokens && result.sections.length) {
|
|
539
|
+
result.sections.pop();
|
|
540
|
+
result.omittedSections += 1;
|
|
541
|
+
if (!result.sections.length) result.truncated = false;
|
|
542
|
+
}
|
|
543
|
+
return result;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// Defaults and ceilings of the sizing options, for the tool schema and CLI.
|
|
547
|
+
const CONTEXT_LIMITS = Object.freeze({
|
|
548
|
+
budgetTokens: Object.freeze({
|
|
549
|
+
default: DEFAULT_CONTEXT_BUDGET_TOKENS,
|
|
550
|
+
maximum: MAX_CONTEXT_BUDGET_TOKENS,
|
|
551
|
+
}),
|
|
552
|
+
limit: Object.freeze({ default: DEFAULT_CONTEXT_LIMIT, maximum: MAX_CONTEXT_LIMIT }),
|
|
553
|
+
conceptLimit: Object.freeze({ default: DEFAULT_CONCEPT_LIMIT, maximum: MAX_CONCEPT_LIMIT }),
|
|
554
|
+
});
|
|
555
|
+
|
|
556
|
+
module.exports = {
|
|
557
|
+
CONTEXT_CONTENT_FIELDS: CONTENT_FIELDS,
|
|
558
|
+
CONTEXT_LIMITS,
|
|
559
|
+
PROVENANCE_MODES,
|
|
560
|
+
CONTEXT_TOKEN_ESTIMATOR,
|
|
561
|
+
DEFAULT_CONTEXT_BUDGET_TOKENS,
|
|
562
|
+
DEFAULT_CONCEPT_LIFT,
|
|
563
|
+
DEFAULT_PREAMBLE_WEIGHT,
|
|
564
|
+
DEFAULT_PROVENANCE,
|
|
565
|
+
assembleContext,
|
|
566
|
+
estimateContextTokens,
|
|
567
|
+
};
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// The `context` command of the CLI (plan WP3a, section 12), kept out of
|
|
4
|
+
// src/cli.js so its line ratchet only shrinks: the token budget, section cap
|
|
5
|
+
// and provenance flags, plus the dispatch. `parseCommandOption` is the one
|
|
6
|
+
// flag hook src/cli.js calls for both commands: it tries the `search` flag
|
|
7
|
+
// parser (whose filters `context` shares) and then the context flags, and
|
|
8
|
+
// returns how many extra arguments it used, or -1 when the flag is neither.
|
|
9
|
+
// Context flags are kept on the parsed search options as a non-enumerable
|
|
10
|
+
// `context` record, so nothing else that spreads the search options sees
|
|
11
|
+
// them; `contextOptions(args)` reads them back. `fail` builds the CLI usage
|
|
12
|
+
// error.
|
|
13
|
+
|
|
14
|
+
const { parseSearchOption } = require("../search/cli-options");
|
|
15
|
+
const { PROVENANCE_MODES, assembleContext } = require("./assemble");
|
|
16
|
+
|
|
17
|
+
const OPTIONS = Object.freeze({ "--budget": "budgetTokens", "--limit": "limit" });
|
|
18
|
+
const POSITIVE_INTEGER = /^[1-9]\d*$/;
|
|
19
|
+
|
|
20
|
+
// The context record of a parsed search-options object, created on demand.
|
|
21
|
+
function contextRecord(search) {
|
|
22
|
+
if (!Object.hasOwn(search, "context")) {
|
|
23
|
+
Object.defineProperty(search, "context", { value: {}, enumerable: false });
|
|
24
|
+
}
|
|
25
|
+
return search.context;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function parseContextOption(arg, argv, index, context, fail) {
|
|
29
|
+
if (arg === "--provenance") {
|
|
30
|
+
const mode = argv[index + 1];
|
|
31
|
+
if (!PROVENANCE_MODES.includes(mode)) {
|
|
32
|
+
throw fail(`--provenance requires one of ${PROVENANCE_MODES.join(", ")}.`);
|
|
33
|
+
}
|
|
34
|
+
context.provenance = mode;
|
|
35
|
+
return 1;
|
|
36
|
+
}
|
|
37
|
+
if (!Object.hasOwn(OPTIONS, arg)) {
|
|
38
|
+
return -1;
|
|
39
|
+
}
|
|
40
|
+
const value = argv[index + 1];
|
|
41
|
+
if (!value || !POSITIVE_INTEGER.test(value)) {
|
|
42
|
+
throw fail(`${arg} requires a positive integer.`);
|
|
43
|
+
}
|
|
44
|
+
context[OPTIONS[arg]] = Number(value);
|
|
45
|
+
return 1;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function parseCommandOption(arg, argv, index, search, fail) {
|
|
49
|
+
const consumed = parseSearchOption(arg, argv, index, search, fail);
|
|
50
|
+
if (consumed >= 0) {
|
|
51
|
+
return consumed;
|
|
52
|
+
}
|
|
53
|
+
if (arg !== "--provenance" && !Object.hasOwn(OPTIONS, arg)) {
|
|
54
|
+
return -1;
|
|
55
|
+
}
|
|
56
|
+
return parseContextOption(arg, argv, index, contextRecord(search), fail);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// The context flags parsed into `args` (empty when none were given).
|
|
60
|
+
function contextOptions(args) {
|
|
61
|
+
return Object.assign({}, args?.search?.context);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Positional words after the command form the query; the parsed search
|
|
65
|
+
// filters and context flags become the options of assembleContext.
|
|
66
|
+
function contextRequest(args, fail) {
|
|
67
|
+
const query = (args.positional || []).slice(1).join(" ").trim();
|
|
68
|
+
if (!query) {
|
|
69
|
+
throw fail("context requires a query.");
|
|
70
|
+
}
|
|
71
|
+
const request = Object.assign({}, args.search, contextOptions(args), { query });
|
|
72
|
+
if (!Object.keys(request.frontmatter || {}).length) {
|
|
73
|
+
delete request.frontmatter;
|
|
74
|
+
}
|
|
75
|
+
return request;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function runContext(index, args, fail) {
|
|
79
|
+
return assembleContext(index, contextRequest(args, fail));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
module.exports = {
|
|
83
|
+
contextOptions,
|
|
84
|
+
contextRequest,
|
|
85
|
+
parseCommandOption,
|
|
86
|
+
parseContextOption,
|
|
87
|
+
runContext,
|
|
88
|
+
};
|