@mfdaves/okf-mcp 0.9.1 → 0.13.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +100 -0
  2. package/README.md +452 -26
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/distribution/reference-bundle.md +10 -3
  5. package/okf/okf-mcp/index.md +2 -1
  6. package/okf/okf-mcp/interfaces/http-authoring-api.md +11 -2
  7. package/okf/okf-mcp/interfaces/mcp-http.md +64 -0
  8. package/okf/okf-mcp/interfaces/mcp-tools.md +25 -6
  9. package/okf/okf-mcp/overview/okf-mcp.md +16 -6
  10. package/okf/okf-mcp/policies/authoring-safety.md +28 -5
  11. package/okf/okf-mcp/runtime/indexer.md +14 -1
  12. package/okf/okf-mcp/runtime/mcp-server.md +16 -8
  13. package/okf/okf-mcp/specs/producer-api.md +16 -2
  14. package/package.json +17 -5
  15. package/server.json +38 -2
  16. package/src/assets.js +165 -73
  17. package/src/authoring/candidates.js +434 -0
  18. package/src/authoring/git-preflight.js +225 -0
  19. package/src/authoring/git-publish.js +783 -0
  20. package/src/authoring/live.js +757 -0
  21. package/src/authoring/paths.js +112 -0
  22. package/src/authoring/policy-hook.js +221 -0
  23. package/src/authoring/proposal-policy.js +159 -0
  24. package/src/authoring/receipts.js +373 -0
  25. package/src/authoring/render.js +38 -0
  26. package/src/authoring/shared.js +75 -0
  27. package/src/authoring/staging.js +216 -0
  28. package/src/authoring.js +386 -287
  29. package/src/cache/files.js +147 -0
  30. package/src/cache/lock.js +201 -0
  31. package/src/cache/paths.js +197 -0
  32. package/src/cache/store.js +703 -0
  33. package/src/catalog-text.js +19 -0
  34. package/src/cli/commands.js +448 -0
  35. package/src/cli/http-options.js +193 -0
  36. package/src/cli/options.js +554 -0
  37. package/src/cli/usage.js +96 -0
  38. package/src/cli.js +18 -932
  39. package/src/computation.js +99 -61
  40. package/src/context/assemble.js +567 -0
  41. package/src/context/cli-options.js +88 -0
  42. package/src/context/score.js +234 -0
  43. package/src/context/sections.js +274 -0
  44. package/src/embed/interface.js +160 -0
  45. package/src/embed/openai.js +715 -0
  46. package/src/embed/prepare.js +182 -0
  47. package/src/embed/units.js +96 -0
  48. package/src/error-details.js +53 -0
  49. package/src/git-source.js +112 -52
  50. package/src/graph.js +95 -37
  51. package/src/http-server.js +36 -17
  52. package/src/index.js +15 -0
  53. package/src/indexer/assemble.js +566 -0
  54. package/src/indexer/build.js +290 -0
  55. package/src/indexer/cache.js +81 -0
  56. package/src/indexer/edges.js +430 -0
  57. package/src/indexer/index.js +44 -0
  58. package/src/indexer/paths.js +178 -0
  59. package/src/indexer/resolve.js +282 -0
  60. package/src/indexer/update.js +720 -0
  61. package/src/indexer.js +3 -1068
  62. package/src/live-authoring.js +4 -2329
  63. package/src/markdown.js +21 -16
  64. package/src/mcp/auth.js +409 -0
  65. package/src/mcp/http.js +317 -0
  66. package/src/mcp/principal.js +59 -0
  67. package/src/mcp/registry.js +356 -0
  68. package/src/mcp/resources.js +45 -0
  69. package/src/mcp/server.js +567 -0
  70. package/src/mcp/tools/catalog.js +247 -0
  71. package/src/mcp/tools/computation.js +83 -0
  72. package/src/mcp/tools/context.js +133 -0
  73. package/src/mcp/tools/graph.js +167 -0
  74. package/src/mcp/tools/live-authoring.js +316 -0
  75. package/src/mcp/tools/migration.js +86 -0
  76. package/src/mcp/tools/producers.js +79 -0
  77. package/src/mcp/tools/proposals.js +225 -0
  78. package/src/mcp/tools/provenance.js +87 -0
  79. package/src/mcp/tools/remote.js +79 -0
  80. package/src/mcp/tools/search.js +242 -0
  81. package/src/mcp-server.js +3 -1611
  82. package/src/migration.js +431 -253
  83. package/src/parser.js +245 -130
  84. package/src/plugins.js +71 -48
  85. package/src/policy/write-policy.js +713 -0
  86. package/src/producer-publisher.js +202 -78
  87. package/src/producers.js +311 -103
  88. package/src/project.js +346 -93
  89. package/src/remote.js +96 -61
  90. package/src/search/cli-options.js +107 -0
  91. package/src/search/confidence.js +61 -0
  92. package/src/search/dense.js +304 -0
  93. package/src/search/embedding-cli.js +57 -0
  94. package/src/search/embedding-config.js +217 -0
  95. package/src/search/fusion.js +249 -0
  96. package/src/search/hybrid.js +526 -0
  97. package/src/search/languages.js +200 -0
  98. package/src/search/lexical.js +695 -0
  99. package/src/search/query.js +620 -0
  100. package/src/search/stem/among.js +43 -0
  101. package/src/search/stem/base-stemmer.js +311 -0
  102. package/src/search/stem/en-regions.js +456 -0
  103. package/src/search/stem/en-suffixes.js +664 -0
  104. package/src/search/stem/en-tables.js +176 -0
  105. package/src/search/stem/en.js +238 -0
  106. package/src/search/stem/it-regions.js +459 -0
  107. package/src/search/stem/it-suffixes.js +471 -0
  108. package/src/search/stem/it-tables.js +257 -0
  109. package/src/search/stem/it.js +155 -0
  110. package/src/search-index.js +2 -81
  111. package/src/search.js +2 -252
  112. package/src/store.js +134 -81
  113. package/src/tool-packs.js +346 -0
  114. package/src/v02.js +301 -105
  115. package/src/validation.js +24 -20
@@ -0,0 +1,567 @@
1
+ "use strict";
2
+
3
+ // Budgeted context assembly over the lexical index (plan C-1, C-3, WP3a).
4
+ //
5
+ // Candidate concepts come from the 0.10 `searchConcepts` with every filter,
6
+ // mode and confidence field intact (at most `conceptLimit` of them). Only
7
+ // their sections are then scored, on the fly, against the same analysed
8
+ // query (src/context/score.js: BM25 over each section's own text with the
9
+ // concept bundle's stemming, plus at most 20% for the concept's shared
10
+ // metadata). Nothing is indexed or cached per OKF index, so a write costs
11
+ // context assembly nothing; the one memo is `conceptSections(doc)` per
12
+ // document object, which a functional index update keeps for unchanged
13
+ // documents. Order: the concept rank from searchConcepts leads (it already
14
+ // weighed title, description, tags and aliases); within a concept, sections
15
+ // whose own text matches come before sections found through metadata only,
16
+ // by score; an own-text section may move ahead of at most `conceptLift`
17
+ // weaker concepts in proportion to its score over the best section's. Ties
18
+ // fall back to document order. Sections are deduplicated by anchor and by
19
+ // identical text hash and packed greedily in that order while they fit.
20
+ //
21
+ // The budget covers the whole response as delivered (0.11.0-rc.2): usedTokens
22
+ // is ceil(characters / 4) of the JSON serialisation of the entire result,
23
+ // every top-level field included (query, concepts, queryTerms, the counters,
24
+ // usedTokens itself) and every section field (scoring fields too). Sections
25
+ // are packed into what the rest of the response leaves; a section's `tokens`
26
+ // is the same estimate of the section object alone. The estimator is named in
27
+ // the result and the budget is never exceeded, unless it is smaller than the
28
+ // response's fixed fields (the query echo and field names), in which case no
29
+ // section and no concept is returned and usedTokens says how large the
30
+ // response is. When the fields around the sections do not fit, the candidate
31
+ // concept list is shortened from its tail first (omittedConcepts counts the
32
+ // concepts left out). Provenance is `compact` by default (status, freshness,
33
+ // verified, sourceCount); `provenance: "full"` adds staleAfter, trustTier,
34
+ // the verified and generated events and the sources, and `provenance: "none"`
35
+ // (or `includeProvenance: false`) leaves it out. When the best section alone
36
+ // is larger than what is left it is returned on its own with its text cut at
37
+ // a sentence boundary, and `truncated` says so; when not even its fields
38
+ // without text fit, no section is returned.
39
+ //
40
+ // The result is structured JSON; catalog text appears only inside section
41
+ // fields (B-6, C-3). Provenance per section is the concept's v0.2 signals.
42
+
43
+ const { normalizeV02Signals } = require("../v02");
44
+ const { analyzeQuery, describeSearchIndex, searchLanguages } = require("../search/lexical");
45
+ const { FRESHNESS_PRIOR, searchConcepts } = require("../search/query");
46
+ const { scoreSections } = require("./score");
47
+ const { conceptSections } = require("./sections");
48
+
49
+ const CONTEXT_TOKEN_ESTIMATOR = "chars/4";
50
+ const CHARS_PER_TOKEN = 4;
51
+ const DEFAULT_CONTEXT_BUDGET_TOKENS = 1200;
52
+ const MAX_CONTEXT_BUDGET_TOKENS = 100000;
53
+ // The budget is the primary constraint; the section cap only stops a swarm
54
+ // of tiny sections (fixture sections run 100-450 characters, so 1,200
55
+ // tokens hold 10-15 of them).
56
+ const DEFAULT_CONTEXT_LIMIT = 20;
57
+ const MAX_CONTEXT_LIMIT = 250;
58
+ // Concepts whose sections are ranked; the candidate step's page size.
59
+ const DEFAULT_CONCEPT_LIMIT = 10;
60
+ const MAX_CONCEPT_LIMIT = 50;
61
+ // Multiplier on the preamble's score (title, description, aliases and the
62
+ // text before the first heading). Chosen on the dev split of both fixtures
63
+ // among 1.0, 0.7 and 0.5 for the best labelled-anchor MRR without a lower
64
+ // anchor hit rate (test/fixtures/retrieval/reports/context-0.11-2026-10-24).
65
+ const DEFAULT_PREAMBLE_WEIGHT = 0.5;
66
+ // Sections with metadata evidence only score at most this share of the
67
+ // weakest section with own-text evidence.
68
+ const METADATA_ONLY_CEILING = 0.5;
69
+ // A truncation keeps the last sentence end in the allowed span only when it
70
+ // leaves at least this share of the span; otherwise it cuts at a word.
71
+ const MIN_SENTENCE_SHARE = 0.5;
72
+ const SENTENCE_END = /[.!?](?=\s|$)|\n/g;
73
+ // Options of assembleContext itself; everything else goes to searchConcepts.
74
+ const CONTEXT_OPTIONS = [
75
+ "budgetTokens",
76
+ "includeProvenance",
77
+ "provenance",
78
+ "conceptLimit",
79
+ "conceptLift",
80
+ "preambleWeight",
81
+ "abstainOnLowConfidence",
82
+ ];
83
+ const SOURCE_FIELDS = ["id", "resource", "author"];
84
+ const PROVENANCE_MODES = Object.freeze(["compact", "full", "none"]);
85
+ const DEFAULT_PROVENANCE = "compact";
86
+ // How many weaker concepts a strong own-text section may move ahead of (one
87
+ // scoring as high as the best own-text section moves the full distance).
88
+ // Chosen on the dev split among 0, 2 and 4 by the highest anchor hit summed
89
+ // over both fixtures, then the highest anchor MRR (the sweep is in
90
+ // test/fixtures/retrieval/reports/context-0.11-2026-10-24).
91
+ const DEFAULT_CONCEPT_LIFT = 4;
92
+ const MAX_CONCEPT_LIFT = 10;
93
+ // A section's content fields (the scoring fields are tokens, score, relevance,
94
+ // matchedTerms and coverage). Since 0.11.0-rc.2 every field counts against
95
+ // the budget; the list stays exported for library callers.
96
+ const CONTENT_FIELDS = Object.freeze([
97
+ "uri",
98
+ "anchor",
99
+ "heading",
100
+ "title",
101
+ "text",
102
+ "truncated",
103
+ "provenance",
104
+ ]);
105
+
106
+ function estimateContextTokens(text) {
107
+ return Math.ceil(String(text || "").length / CHARS_PER_TOKEN);
108
+ }
109
+
110
+ // Characters of a value as delivered (compact JSON).
111
+ function serializedLength(value) {
112
+ return JSON.stringify(value).length;
113
+ }
114
+
115
+ // A section's `tokens` is part of what it costs: settle it on the estimate of
116
+ // the section object including its own digits. Returns the section's length.
117
+ function settleTokens(section) {
118
+ let tokens = 0;
119
+ for (;;) {
120
+ section.tokens = tokens;
121
+ const length = serializedLength(section);
122
+ const next = Math.ceil(length / CHARS_PER_TOKEN);
123
+ if (next <= tokens) return length;
124
+ tokens = next;
125
+ }
126
+ }
127
+
128
+ // The whole response's estimate with usedTokens set to it (a fixed point:
129
+ // usedTokens is part of the response it measures).
130
+ function settleUsedTokens(response) {
131
+ let used = 0;
132
+ for (;;) {
133
+ response.usedTokens = used;
134
+ const next = Math.ceil(serializedLength(response) / CHARS_PER_TOKEN);
135
+ if (next <= used) return used;
136
+ used = next;
137
+ }
138
+ }
139
+
140
+ function integerOption(value, name, fallback, maximum) {
141
+ if (value === undefined || value === null || value === "") {
142
+ return fallback;
143
+ }
144
+ const number = Number(value);
145
+ if (!Number.isInteger(number) || number < 1 || number > maximum) {
146
+ throw new TypeError(`${name} must be an integer from 1 through ${maximum}.`);
147
+ }
148
+ return number;
149
+ }
150
+
151
+ function weightOption(value) {
152
+ if (value === undefined || value === null || value === "") {
153
+ return DEFAULT_PREAMBLE_WEIGHT;
154
+ }
155
+ const number = Number(value);
156
+ if (!Number.isFinite(number) || number <= 0 || number > 1) {
157
+ throw new TypeError("preambleWeight must be a number greater than 0 and at most 1.");
158
+ }
159
+ return number;
160
+ }
161
+
162
+ function provenanceOption(config) {
163
+ if (config.includeProvenance === false) {
164
+ return null;
165
+ }
166
+ const mode =
167
+ config.provenance === undefined || config.provenance === null
168
+ ? DEFAULT_PROVENANCE
169
+ : config.provenance;
170
+ if (!PROVENANCE_MODES.includes(mode)) {
171
+ throw new TypeError(`provenance must be one of ${PROVENANCE_MODES.join(", ")}.`);
172
+ }
173
+ return mode === "none" ? null : mode;
174
+ }
175
+
176
+ function liftOption(value) {
177
+ if (value === undefined || value === null || value === "") {
178
+ return DEFAULT_CONCEPT_LIFT;
179
+ }
180
+ const number = Number(value);
181
+ if (!Number.isFinite(number) || number < 0 || number > MAX_CONCEPT_LIFT) {
182
+ throw new TypeError(`conceptLift must be a number from 0 through ${MAX_CONCEPT_LIFT}.`);
183
+ }
184
+ return number;
185
+ }
186
+
187
+ function signalsFor(doc, asOf) {
188
+ return asOf ? normalizeV02Signals(doc, { asOf }) : doc.signals || {};
189
+ }
190
+
191
+ // R-6 factors, the same as concept search applies to candidates.
192
+ function priorFactor(signals) {
193
+ let factor = 1;
194
+ if (signals.status === "deprecated") {
195
+ factor *= FRESHNESS_PRIOR.deprecated;
196
+ } else if (signals.status === "draft") {
197
+ factor *= FRESHNESS_PRIOR.draft;
198
+ }
199
+ if (signals.freshness === "stale") {
200
+ factor *= FRESHNESS_PRIOR.stale;
201
+ }
202
+ return factor;
203
+ }
204
+
205
+ function compactSource(source) {
206
+ const out = {};
207
+ SOURCE_FIELDS.forEach((field) => {
208
+ if (source[field] !== undefined) {
209
+ out[field] = source[field];
210
+ }
211
+ });
212
+ return out;
213
+ }
214
+
215
+ const validSources = (signals) =>
216
+ (signals.sources || []).filter(
217
+ (source) => source && typeof source === "object" && source.valid !== false,
218
+ );
219
+
220
+ function provenanceOf(signals, mode) {
221
+ if (mode === "compact") {
222
+ return {
223
+ status: signals.status ?? null,
224
+ freshness: signals.freshness ?? null,
225
+ verified: (signals.verifiedEvents || []).length > 0,
226
+ sourceCount: validSources(signals).length,
227
+ };
228
+ }
229
+ return {
230
+ status: signals.status ?? null,
231
+ staleAfter: signals.staleAfter ?? null,
232
+ freshness: signals.freshness ?? null,
233
+ trustTier: signals.trustTier ?? null,
234
+ verified: (signals.verifiedEvents || []).map((event) => ({
235
+ by: event.by ?? null,
236
+ at: event.at ?? null,
237
+ })),
238
+ generated: signals.generated
239
+ ? { by: signals.generated.by ?? null, at: signals.generated.at ?? null }
240
+ : null,
241
+ sources: validSources(signals).map(compactSource),
242
+ };
243
+ }
244
+
245
+ // The stemming languages of a bundle: its detected set under `auto`, else
246
+ // the configured union (the same plan concept search indexes with).
247
+ function bundleLanguages(index, options) {
248
+ const plan = describeSearchIndex(index, options);
249
+ return (bundle) => plan.bundles?.[bundle]?.languages || plan.languages;
250
+ }
251
+
252
+ // Sections of the candidate concepts that match at least one query term,
253
+ // ordered by score, candidate rank and document position, one per anchor
254
+ // and per distinct text.
255
+ function rankSections(index, options, analysis, candidates, context) {
256
+ const entries = [];
257
+ candidates.forEach((candidate, conceptRank) => {
258
+ const doc = index.byUri.get(candidate.uri);
259
+ if (!doc) {
260
+ return;
261
+ }
262
+ conceptSections(doc).forEach((section, position) => {
263
+ entries.push({ doc, concept: doc, section, position, conceptRank });
264
+ });
265
+ });
266
+ const scored = scoreSections(entries, analysis, {
267
+ languagesFor: bundleLanguages(index, options),
268
+ preambleWeight: context.preambleWeight,
269
+ });
270
+ const prior = (hit) =>
271
+ options.freshnessPrior ? priorFactor(context.signalsOf(hit.entry.doc)) : 1;
272
+ const ranked = scored.map((hit) => ({
273
+ entry: hit.entry,
274
+ tier: hit.tier,
275
+ score: hit.tier ? hit.score * prior(hit) : 0,
276
+ coverage: hit.coverage,
277
+ positions: hit.positions,
278
+ conceptRank: hit.entry.conceptRank,
279
+ }));
280
+ // A metadata-only section scores below the weakest own-text section of its
281
+ // concept (of any candidate when its concept has none).
282
+ const weakest = new Map();
283
+ ranked.forEach((hit) => {
284
+ if (hit.tier) {
285
+ const uri = hit.entry.doc.uri;
286
+ weakest.set(uri, Math.min(weakest.get(uri) ?? Number.POSITIVE_INFINITY, hit.score));
287
+ }
288
+ });
289
+ const ownScores = ranked.filter((hit) => hit.tier).map((hit) => hit.score);
290
+ const fallback = ownScores.length ? Math.min(...ownScores) : 1;
291
+ const best = ownScores.length ? Math.max(...ownScores) : 0;
292
+ ranked.forEach((hit, position) => {
293
+ if (!hit.tier) {
294
+ const floor = weakest.get(hit.entry.doc.uri) ?? fallback;
295
+ hit.score =
296
+ METADATA_ONLY_CEILING * floor * scored[position].metadataShare * prior(scored[position]);
297
+ }
298
+ // Concept rank leads; an own-text section moves ahead of at most
299
+ // `conceptLift` weaker concepts, in proportion to its score over the best.
300
+ hit.key =
301
+ hit.conceptRank - (hit.tier && best > 0 ? context.conceptLift * (hit.score / best) : 0);
302
+ });
303
+ ranked.sort(
304
+ (left, right) =>
305
+ left.key - right.key ||
306
+ right.tier - left.tier ||
307
+ right.score - left.score ||
308
+ left.entry.position - right.entry.position,
309
+ );
310
+ const top = best;
311
+ const anchors = new Set();
312
+ const hashes = new Set();
313
+ return ranked.filter((candidate) => {
314
+ const { anchor, hash } = candidate.entry.section;
315
+ if (anchors.has(anchor) || hashes.has(hash)) {
316
+ return false;
317
+ }
318
+ anchors.add(anchor);
319
+ hashes.add(hash);
320
+ candidate.relevance = top > 0 ? candidate.score / top : 0;
321
+ return true;
322
+ });
323
+ }
324
+
325
+ // The longest prefix within `maxChars` that ends a sentence (keeping at least
326
+ // half of the span), else the last word boundary, else a hard cut.
327
+ function truncateText(text, maxChars) {
328
+ if (text.length <= maxChars) {
329
+ return text;
330
+ }
331
+ let cut = -1;
332
+ SENTENCE_END.lastIndex = 0;
333
+ for (let match = SENTENCE_END.exec(text); match; match = SENTENCE_END.exec(text)) {
334
+ const end = match[0] === "\n" ? match.index : match.index + 1;
335
+ if (end > maxChars) {
336
+ break;
337
+ }
338
+ cut = end;
339
+ }
340
+ if (cut < maxChars * MIN_SENTENCE_SHARE) {
341
+ const space = text.lastIndexOf(" ", maxChars);
342
+ cut = space > 0 ? space : maxChars;
343
+ }
344
+ return text.slice(0, cut).trimEnd();
345
+ }
346
+
347
+ // The best section cut to fit `limitChars` once serialised (its tokens
348
+ // settled), or null when its fields without any text already exceed it. JSON
349
+ // escaping can make the text cost more than its length, so the allowance
350
+ // shrinks until it fits.
351
+ function truncatedSection(build, text, limitChars) {
352
+ const empty = build("", true);
353
+ let allowance = limitChars - settleTokens(empty);
354
+ while (allowance > 0) {
355
+ const section = build(truncateText(text, allowance), true);
356
+ const excess = settleTokens(section) - limitChars;
357
+ if (excess <= 0) {
358
+ return section;
359
+ }
360
+ allowance -= excess;
361
+ }
362
+ return null;
363
+ }
364
+
365
+ // Greedy by rank into `availableChars` (what the rest of the response leaves
366
+ // of the budget): a section is taken when its serialised length, plus the
367
+ // comma that separates it from the previous one, fits what is left and the
368
+ // limit allows it, otherwise skipped. A best section larger than everything
369
+ // available is returned alone, cut.
370
+ function packSections(ranked, availableChars, limit, build) {
371
+ if (ranked.length) {
372
+ const best = ranked[0];
373
+ const whole = build(best, best.entry.section.text, false);
374
+ if (settleTokens(whole) > availableChars) {
375
+ const cut = truncatedSection(
376
+ (text, isCut) => build(best, text, isCut),
377
+ best.entry.section.text,
378
+ availableChars,
379
+ );
380
+ if (!cut) {
381
+ return { packed: [], omitted: ranked.length, truncated: false };
382
+ }
383
+ return { packed: [cut], omitted: ranked.length - 1, truncated: true };
384
+ }
385
+ }
386
+ const packed = [];
387
+ let used = 0;
388
+ let omitted = 0;
389
+ ranked.forEach((candidate) => {
390
+ const section = build(candidate, candidate.entry.section.text, false);
391
+ const cost = settleTokens(section) + (packed.length ? 1 : 0);
392
+ if (packed.length >= limit || used + cost > availableChars) {
393
+ omitted += 1;
394
+ return;
395
+ }
396
+ packed.push(section);
397
+ used += cost;
398
+ });
399
+ return { packed, omitted, truncated: false };
400
+ }
401
+
402
+ // The response around the sections, sized for the worst case of the fields
403
+ // packing can still change (usedTokens at the full budget, every section
404
+ // omitted, truncated false), so what it leaves is a safe allowance. The
405
+ // candidate list is shortened from its tail while even that does not fit.
406
+ function responseFrame(fields, candidates, budgetTokens, rankedCount) {
407
+ const limitChars = budgetTokens * CHARS_PER_TOKEN;
408
+ const frame = (concepts) =>
409
+ Object.assign({}, fields, {
410
+ usedTokens: budgetTokens,
411
+ concepts,
412
+ sections: [],
413
+ totalSections: rankedCount,
414
+ omittedSections: rankedCount,
415
+ truncated: false,
416
+ omittedConcepts: candidates.length - concepts.length,
417
+ });
418
+ let concepts = candidates;
419
+ let length = serializedLength(frame(concepts));
420
+ while (length > limitChars && concepts.length) {
421
+ concepts = concepts.slice(0, -1);
422
+ length = serializedLength(frame(concepts));
423
+ }
424
+ return { concepts, availableChars: Math.max(0, limitChars - length) };
425
+ }
426
+
427
+ function sectionResult(candidate, text, truncated, analysis, provenanceMode, signalsOf) {
428
+ const { entry } = candidate;
429
+ const result = {
430
+ uri: entry.concept.uri,
431
+ anchor: entry.section.anchor,
432
+ heading: entry.section.heading,
433
+ title: entry.concept.title,
434
+ text,
435
+ tokens: 0,
436
+ truncated,
437
+ score: candidate.score,
438
+ relevance: candidate.relevance,
439
+ matchedTerms: candidate.positions.map((position) => analysis.tokens[position].raw),
440
+ coverage: candidate.coverage,
441
+ };
442
+ if (provenanceMode) {
443
+ result.provenance = provenanceOf(signalsOf(entry.concept), provenanceMode);
444
+ }
445
+ return result;
446
+ }
447
+
448
+ function assembleContext(index, options) {
449
+ const config = options || {};
450
+ const query = String(config.query || "").trim();
451
+ if (!query) {
452
+ throw new TypeError("Context assembly requires a query.");
453
+ }
454
+ const budgetTokens = integerOption(
455
+ config.budgetTokens,
456
+ "budgetTokens",
457
+ DEFAULT_CONTEXT_BUDGET_TOKENS,
458
+ MAX_CONTEXT_BUDGET_TOKENS,
459
+ );
460
+ const limit = integerOption(config.limit, "limit", DEFAULT_CONTEXT_LIMIT, MAX_CONTEXT_LIMIT);
461
+ const conceptLimit = integerOption(
462
+ config.conceptLimit,
463
+ "conceptLimit",
464
+ DEFAULT_CONCEPT_LIMIT,
465
+ MAX_CONCEPT_LIMIT,
466
+ );
467
+ const preambleWeight = weightOption(config.preambleWeight);
468
+ const conceptLift = liftOption(config.conceptLift);
469
+ const provenanceMode = provenanceOption(config);
470
+ if (
471
+ config.abstainOnLowConfidence !== undefined &&
472
+ typeof config.abstainOnLowConfidence !== "boolean"
473
+ ) {
474
+ throw new TypeError("abstainOnLowConfidence must be a boolean.");
475
+ }
476
+ const searchOptions = Object.assign({}, config, {
477
+ query,
478
+ limit: conceptLimit,
479
+ offset: 0,
480
+ detail: "compact",
481
+ });
482
+ CONTEXT_OPTIONS.forEach((name) => {
483
+ delete searchOptions[name];
484
+ });
485
+ const response = searchConcepts(index, searchOptions);
486
+ const candidates = response.results.map((entry, position) => ({
487
+ uri: entry.uri,
488
+ rank: position + 1,
489
+ coverage: entry.coverage,
490
+ relevance: entry.relevance,
491
+ }));
492
+ const analysis = analyzeQuery(query, { languages: searchLanguages(index, searchOptions) });
493
+ // Signals once per concept per call: with asOf they are re-derived.
494
+ const signalsCache = new Map();
495
+ const signalsOf = (doc) => {
496
+ if (!signalsCache.has(doc.uri)) {
497
+ signalsCache.set(doc.uri, signalsFor(doc, config.asOf));
498
+ }
499
+ return signalsCache.get(doc.uri);
500
+ };
501
+ // WP3c-2: with abstainOnLowConfidence, a low-confidence candidate step yields no sections;
502
+ // the response still carries the confidence block and the candidate concepts.
503
+ const abstained = config.abstainOnLowConfidence === true && Boolean(response.lowConfidence);
504
+ const ranked =
505
+ analysis.tokens.length && candidates.length && !abstained
506
+ ? rankSections(index, searchOptions, analysis, candidates, {
507
+ preambleWeight,
508
+ conceptLift,
509
+ signalsOf,
510
+ })
511
+ : [];
512
+ const fields = {
513
+ query,
514
+ budgetTokens,
515
+ estimator: CONTEXT_TOKEN_ESTIMATOR,
516
+ usedTokens: 0,
517
+ mode: response.mode,
518
+ lowConfidence: Boolean(response.lowConfidence),
519
+ queryTerms: response.queryTerms || [],
520
+ droppedTerms: response.droppedTerms || [],
521
+ confidence: response.confidence || null,
522
+ };
523
+ if (config.abstainOnLowConfidence !== undefined) fields.abstained = abstained;
524
+ const frame = responseFrame(fields, candidates, budgetTokens, ranked.length);
525
+ const packing = packSections(ranked, frame.availableChars, limit, (candidate, text, truncated) =>
526
+ sectionResult(candidate, text, truncated, analysis, provenanceMode, signalsOf),
527
+ );
528
+ const result = Object.assign(fields, {
529
+ concepts: frame.concepts,
530
+ sections: packing.packed,
531
+ totalSections: ranked.length,
532
+ omittedSections: packing.omitted,
533
+ truncated: packing.truncated,
534
+ omittedConcepts: candidates.length - frame.concepts.length,
535
+ });
536
+ // The frame was sized for the worst case, so this holds; the loop is a
537
+ // guard that keeps the promise if a field ever grows past that estimate.
538
+ while (settleUsedTokens(result) > budgetTokens && result.sections.length) {
539
+ result.sections.pop();
540
+ result.omittedSections += 1;
541
+ if (!result.sections.length) result.truncated = false;
542
+ }
543
+ return result;
544
+ }
545
+
546
+ // Defaults and ceilings of the sizing options, for the tool schema and CLI.
547
+ const CONTEXT_LIMITS = Object.freeze({
548
+ budgetTokens: Object.freeze({
549
+ default: DEFAULT_CONTEXT_BUDGET_TOKENS,
550
+ maximum: MAX_CONTEXT_BUDGET_TOKENS,
551
+ }),
552
+ limit: Object.freeze({ default: DEFAULT_CONTEXT_LIMIT, maximum: MAX_CONTEXT_LIMIT }),
553
+ conceptLimit: Object.freeze({ default: DEFAULT_CONCEPT_LIMIT, maximum: MAX_CONCEPT_LIMIT }),
554
+ });
555
+
556
+ module.exports = {
557
+ CONTEXT_CONTENT_FIELDS: CONTENT_FIELDS,
558
+ CONTEXT_LIMITS,
559
+ PROVENANCE_MODES,
560
+ CONTEXT_TOKEN_ESTIMATOR,
561
+ DEFAULT_CONTEXT_BUDGET_TOKENS,
562
+ DEFAULT_CONCEPT_LIFT,
563
+ DEFAULT_PREAMBLE_WEIGHT,
564
+ DEFAULT_PROVENANCE,
565
+ assembleContext,
566
+ estimateContextTokens,
567
+ };
@@ -0,0 +1,88 @@
1
+ "use strict";
2
+
3
+ // The `context` command of the CLI (plan WP3a, section 12), kept out of the
4
+ // main option parser (src/cli/options.js): the token budget, section cap
5
+ // and provenance flags, plus the dispatch. `parseCommandOption` is the one
6
+ // flag hook the parser calls for both commands: it tries the `search` flag
7
+ // parser (whose filters `context` shares) and then the context flags, and
8
+ // returns how many extra arguments it used, or -1 when the flag is neither.
9
+ // Context flags are kept on the parsed search options as a non-enumerable
10
+ // `context` record, so nothing else that spreads the search options sees
11
+ // them; `contextOptions(args)` reads them back. `fail` builds the CLI usage
12
+ // error.
13
+
14
+ const { parseSearchOption } = require("../search/cli-options");
15
+ const { PROVENANCE_MODES, assembleContext } = require("./assemble");
16
+
17
+ const OPTIONS = Object.freeze({ "--budget": "budgetTokens", "--limit": "limit" });
18
+ const POSITIVE_INTEGER = /^[1-9]\d*$/;
19
+
20
+ // The context record of a parsed search-options object, created on demand.
21
+ function contextRecord(search) {
22
+ if (!Object.hasOwn(search, "context")) {
23
+ Object.defineProperty(search, "context", { value: {}, enumerable: false });
24
+ }
25
+ return search.context;
26
+ }
27
+
28
+ function parseContextOption(arg, argv, index, context, fail) {
29
+ if (arg === "--provenance") {
30
+ const mode = argv[index + 1];
31
+ if (!PROVENANCE_MODES.includes(mode)) {
32
+ throw fail(`--provenance requires one of ${PROVENANCE_MODES.join(", ")}.`);
33
+ }
34
+ context.provenance = mode;
35
+ return 1;
36
+ }
37
+ if (!Object.hasOwn(OPTIONS, arg)) {
38
+ return -1;
39
+ }
40
+ const value = argv[index + 1];
41
+ if (!value || !POSITIVE_INTEGER.test(value)) {
42
+ throw fail(`${arg} requires a positive integer.`);
43
+ }
44
+ context[OPTIONS[arg]] = Number(value);
45
+ return 1;
46
+ }
47
+
48
+ function parseCommandOption(arg, argv, index, search, fail) {
49
+ const consumed = parseSearchOption(arg, argv, index, search, fail);
50
+ if (consumed >= 0) {
51
+ return consumed;
52
+ }
53
+ if (arg !== "--provenance" && !Object.hasOwn(OPTIONS, arg)) {
54
+ return -1;
55
+ }
56
+ return parseContextOption(arg, argv, index, contextRecord(search), fail);
57
+ }
58
+
59
+ // The context flags parsed into `args` (empty when none were given).
60
+ function contextOptions(args) {
61
+ return Object.assign({}, args?.search?.context);
62
+ }
63
+
64
+ // Positional words after the command form the query; the parsed search
65
+ // filters and context flags become the options of assembleContext.
66
+ function contextRequest(args, fail) {
67
+ const query = (args.positional || []).slice(1).join(" ").trim();
68
+ if (!query) {
69
+ throw fail("context requires a query.");
70
+ }
71
+ const request = Object.assign({}, args.search, contextOptions(args), { query });
72
+ if (!Object.keys(request.frontmatter || {}).length) {
73
+ delete request.frontmatter;
74
+ }
75
+ return request;
76
+ }
77
+
78
+ function runContext(index, args, fail) {
79
+ return assembleContext(index, contextRequest(args, fail));
80
+ }
81
+
82
+ module.exports = {
83
+ contextOptions,
84
+ contextRequest,
85
+ parseCommandOption,
86
+ parseContextOption,
87
+ runContext,
88
+ };