@mfdaves/okf-mcp 0.9.0 → 0.12.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +176 -19
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/policies/authoring-safety.md +11 -1
  5. package/package.json +17 -5
  6. package/server.json +2 -2
  7. package/src/assets.js +165 -73
  8. package/src/authoring/candidates.js +391 -0
  9. package/src/authoring/git-preflight.js +225 -0
  10. package/src/authoring/git-publish.js +776 -0
  11. package/src/authoring/live.js +789 -0
  12. package/src/authoring/paths.js +112 -0
  13. package/src/authoring/policy-hook.js +221 -0
  14. package/src/authoring/proposal-policy.js +137 -0
  15. package/src/authoring/receipts.js +329 -0
  16. package/src/authoring/render.js +38 -0
  17. package/src/authoring/shared.js +75 -0
  18. package/src/authoring/staging.js +216 -0
  19. package/src/authoring.js +386 -287
  20. package/src/cache/files.js +147 -0
  21. package/src/cache/lock.js +201 -0
  22. package/src/cache/paths.js +197 -0
  23. package/src/cache/store.js +703 -0
  24. package/src/catalog-text.js +19 -0
  25. package/src/cli.js +184 -151
  26. package/src/computation.js +99 -61
  27. package/src/context/assemble.js +567 -0
  28. package/src/context/cli-options.js +88 -0
  29. package/src/context/score.js +234 -0
  30. package/src/context/sections.js +274 -0
  31. package/src/embed/interface.js +160 -0
  32. package/src/embed/openai.js +715 -0
  33. package/src/embed/prepare.js +182 -0
  34. package/src/embed/units.js +96 -0
  35. package/src/git-source.js +112 -52
  36. package/src/graph.js +95 -37
  37. package/src/http-server.js +36 -17
  38. package/src/index.js +15 -0
  39. package/src/indexer/assemble.js +566 -0
  40. package/src/indexer/build.js +290 -0
  41. package/src/indexer/cache.js +81 -0
  42. package/src/indexer/edges.js +430 -0
  43. package/src/indexer/index.js +44 -0
  44. package/src/indexer/paths.js +178 -0
  45. package/src/indexer/resolve.js +282 -0
  46. package/src/indexer/update.js +720 -0
  47. package/src/indexer.js +3 -1068
  48. package/src/live-authoring.js +4 -2329
  49. package/src/markdown.js +21 -16
  50. package/src/mcp/registry.js +328 -0
  51. package/src/mcp/resources.js +45 -0
  52. package/src/mcp/server.js +472 -0
  53. package/src/mcp/tools/catalog.js +244 -0
  54. package/src/mcp/tools/computation.js +83 -0
  55. package/src/mcp/tools/context.js +133 -0
  56. package/src/mcp/tools/graph.js +167 -0
  57. package/src/mcp/tools/live-authoring.js +316 -0
  58. package/src/mcp/tools/migration.js +86 -0
  59. package/src/mcp/tools/producers.js +79 -0
  60. package/src/mcp/tools/proposals.js +213 -0
  61. package/src/mcp/tools/provenance.js +87 -0
  62. package/src/mcp/tools/remote.js +79 -0
  63. package/src/mcp/tools/search.js +242 -0
  64. package/src/mcp-server.js +3 -1611
  65. package/src/migration.js +431 -253
  66. package/src/parser.js +245 -130
  67. package/src/plugins.js +71 -48
  68. package/src/policy/write-policy.js +713 -0
  69. package/src/producer-publisher.js +195 -77
  70. package/src/producers.js +265 -101
  71. package/src/project.js +277 -93
  72. package/src/remote.js +96 -61
  73. package/src/search/cli-options.js +107 -0
  74. package/src/search/confidence.js +61 -0
  75. package/src/search/dense.js +304 -0
  76. package/src/search/embedding-cli.js +57 -0
  77. package/src/search/embedding-config.js +217 -0
  78. package/src/search/fusion.js +249 -0
  79. package/src/search/hybrid.js +526 -0
  80. package/src/search/languages.js +200 -0
  81. package/src/search/lexical.js +695 -0
  82. package/src/search/query.js +620 -0
  83. package/src/search/stem/among.js +43 -0
  84. package/src/search/stem/base-stemmer.js +311 -0
  85. package/src/search/stem/en-regions.js +456 -0
  86. package/src/search/stem/en-suffixes.js +664 -0
  87. package/src/search/stem/en-tables.js +176 -0
  88. package/src/search/stem/en.js +238 -0
  89. package/src/search/stem/it-regions.js +459 -0
  90. package/src/search/stem/it-suffixes.js +471 -0
  91. package/src/search/stem/it-tables.js +257 -0
  92. package/src/search/stem/it.js +155 -0
  93. package/src/search-index.js +2 -81
  94. package/src/search.js +2 -252
  95. package/src/store.js +134 -81
  96. package/src/v02.js +301 -105
  97. package/src/validation.js +24 -20
@@ -0,0 +1,695 @@
1
+ "use strict";
2
+
3
+ // Lexical index over an OKF index (plan R-1, R-2, R-3, R-5): one MiniSearch
4
+ // instance per OKF index and language setting, built lazily and cached in a
5
+ // WeakMap. Languages are detected per bundle (the union of languages present
6
+ // in at least 5% of that bundle's documents) unless configured; each document
7
+ // is stemmed with its own bundle's set and queries with the union of all sets
8
+ // (or the filtered bundle's set). Text fields keep the raw token (diacritics
9
+ // folded) plus one stem per language; the `exact` field (path, identifiers,
10
+ // aliases, raw title tokens) is literal: never stemmed, folded or
11
+ // stop-worded. Query time folds, drops stop words and single letters, and
12
+ // stems every plain term; quoted text and identifier-looking tokens are
13
+ // literal terms that only count when the whole compound is in the exact
14
+ // field or their pieces form an adjacent run in one field's raw tokens,
15
+ // never through a stem. One BM25 tree scores the query; coverage is counted
16
+ // over logical terms after that verification.
17
+
18
+ const MiniSearch = require("minisearch");
19
+ const {
20
+ detectLanguages,
21
+ foldDiacritics,
22
+ normalizeLanguages,
23
+ stemForms,
24
+ stopWordsFor,
25
+ } = require("./languages");
26
+
27
+ const MAX_QUERY_CHARACTERS = 512;
28
+ const MAX_QUERY_TERMS = 16;
29
+ const EXACT_FIELD = "exact";
30
+ const TEXT_FIELDS = Object.freeze([
31
+ "title",
32
+ "type",
33
+ "tags",
34
+ "aliases",
35
+ "description",
36
+ "path",
37
+ "body",
38
+ ]);
39
+ const SEARCH_FIELDS = Object.freeze(TEXT_FIELDS.concat([EXACT_FIELD]));
40
+ const SEARCH_BOOSTS = Object.freeze({
41
+ title: 8,
42
+ type: 5,
43
+ tags: 4,
44
+ aliases: 4,
45
+ description: 3,
46
+ path: 2,
47
+ body: 1,
48
+ exact: 6,
49
+ });
50
+ // Body characters sampled per document for language detection.
51
+ const DETECTION_SAMPLE_CHARACTERS = 4000;
52
+ // Distinct tokens whose index forms are memoised per language set.
53
+ const TERM_FORMS_CACHE_LIMIT = 50000;
54
+ // Documents whose raw token streams are kept for literal-term verification.
55
+ const STREAM_CACHE_LIMIT = 2000;
56
+ // OKF index -> { detection, variants: Map<language key, variant> }
57
+ const SEARCH_INDEXES = new WeakMap();
58
+ // MiniSearch instance -> Map<document id, { text, exact }>
59
+ const STREAM_CACHES = new WeakMap();
60
+ // Separates fields in the cached text: a control character is never part of
61
+ // a token and never a separator, so an adjacent run cannot cross it.
62
+ const FIELD_BREAK = "\n\u0001\n";
63
+ const SEPARATOR_CLASS = "[\\n\\r\\p{Z}\\p{P}\\p{S}]";
64
+ // MiniSearch's default tokenizer splits on space and punctuation only; symbols
65
+ // such as Markdown backticks would stay glued to code-formatted terms
66
+ // ("`kill"), so the lexical index splits on symbols as well, everywhere.
67
+ const TOKEN_SEPARATORS = /[\n\r\p{Z}\p{P}\p{S}]+/u;
68
+ const tokenize = (text) => String(text).split(TOKEN_SEPARATORS).filter(Boolean);
69
+ const identity = (value) => value;
70
+ const identityTokenize = (text) => [text];
71
+ // Leaves are pre-processed; MiniSearch must neither tokenize nor expand them.
72
+ const LEAF_OPTIONS = Object.freeze({
73
+ tokenize: identityTokenize,
74
+ processTerm: identity,
75
+ prefix: false,
76
+ fuzzy: false,
77
+ });
78
+ // Query pieces in order: "double quoted", “typographic quoted”, or a run of
79
+ // characters that are neither spaces nor quotes, so punctuation around a
80
+ // quoted group never swallows it.
81
+ const QUERY_PIECES = /"([^"]*)"|“([^”]*)”|([^\s"“”]+)/gu;
82
+ // An unquoted chunk that looks like an identifier: digits or an internal hyphen.
83
+ const IDENTIFIER_CHUNK = /\p{N}|[\p{L}\p{N}]-[\p{L}\p{N}]/u;
84
+ // An indexed chunk kept whole beside its pieces: an internal hyphen, underscore or dot.
85
+ const IDENTIFIER_SEGMENT = /[\p{L}\p{N}][-_.][\p{L}\p{N}]/u;
86
+ const EDGE_PUNCTUATION = /^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu;
87
+ const FILE_EXTENSION = /\.[a-z0-9]+$/;
88
+
89
+ const canonicalCodes = (codes) => Array.from(new Set(codes)).sort();
90
+ const languageKey = (codes) => canonicalCodes(codes).join(",") || "none";
91
+ // Index normalisation of raw tokens: lower-cased, diacritics folded, no stems.
92
+ const rawTokens = (text) =>
93
+ tokenize(String(text || "")).map((token) => foldDiacritics(token.toLowerCase()));
94
+
95
+ // A custom id is a full okf:// URI; only its concept path is an identifier.
96
+ function customIdPath(customId) {
97
+ return customId ? String(customId).replace(/^okf:\/\/[^/]+\//, "") : "";
98
+ }
99
+
100
+ // One segment per identifier-bearing value; the exact tokenizer sees them
101
+ // separately so whole titles and aliases become tokens of their own.
102
+ function exactSegments(doc) {
103
+ return [doc.conceptId, customIdPath(doc.customId), doc.path, doc.title]
104
+ .concat(doc.aliases || [])
105
+ .filter(Boolean)
106
+ .map((value) => String(value).replace(/\s+/g, " ").trim())
107
+ .filter(Boolean);
108
+ }
109
+
110
+ function textFieldValues(doc) {
111
+ return [
112
+ String(doc.title || ""),
113
+ String(doc.type || ""),
114
+ (doc.tags || []).join(" "),
115
+ (doc.aliases || []).join(" "),
116
+ String(doc.description || ""),
117
+ String(doc.path || ""),
118
+ String(doc.body || ""),
119
+ ];
120
+ }
121
+
122
+ function searchDocument(doc) {
123
+ const [title, type, tags, aliases, description, path, body] = textFieldValues(doc);
124
+ return {
125
+ id: doc.uri,
126
+ title,
127
+ type,
128
+ tags,
129
+ aliases,
130
+ description,
131
+ path,
132
+ body,
133
+ exact: exactSegments(doc).join("\n"),
134
+ };
135
+ }
136
+
137
+ // Exact-field tokens are literal (lower-cased, diacritics kept): each segment
138
+ // as a whole, its default tokens, and every identifier chunk with and without
139
+ // a file extension, so "TRK-W-042" survives as one token beside "trk", "w"
140
+ // and "042".
141
+ function exactTokens(text) {
142
+ const out = new Set();
143
+ String(text)
144
+ .split("\n")
145
+ .forEach((line) => {
146
+ const segment = line.trim().toLowerCase();
147
+ if (!segment) {
148
+ return;
149
+ }
150
+ out.add(segment);
151
+ tokenize(segment).forEach((token) => {
152
+ if (token) {
153
+ out.add(token);
154
+ }
155
+ });
156
+ segment.split(/[\s/]+/).forEach((chunk) => {
157
+ const clean = chunk.replace(EDGE_PUNCTUATION, "");
158
+ if (clean && IDENTIFIER_SEGMENT.test(clean)) {
159
+ out.add(clean);
160
+ const bare = clean.replace(FILE_EXTENSION, "");
161
+ if (bare && bare !== clean) {
162
+ out.add(bare);
163
+ }
164
+ }
165
+ });
166
+ });
167
+ return Array.from(out);
168
+ }
169
+
170
+ function fieldTokenize(text, field) {
171
+ return field === EXACT_FIELD ? exactTokens(text) : tokenize(text);
172
+ }
173
+
174
+ // Index-time term processing. `current` names the language set of the
175
+ // document being added (set by the variant before each add): the folded token
176
+ // plus its stems for text fields, the literal token for the exact field.
177
+ // Memoised per language set because catalog vocabulary repeats heavily.
178
+ function createTermProcessor(current) {
179
+ const caches = new Map();
180
+ return (term, field) => {
181
+ if (field === EXACT_FIELD) {
182
+ return term;
183
+ }
184
+ const lower = term.toLowerCase();
185
+ if (!current.languages.length) {
186
+ return foldDiacritics(lower);
187
+ }
188
+ let cache = caches.get(current.key);
189
+ if (!cache) {
190
+ cache = new Map();
191
+ caches.set(current.key, cache);
192
+ }
193
+ let forms = cache.get(lower);
194
+ if (forms === undefined) {
195
+ const folded = foldDiacritics(lower);
196
+ const stems = stemForms(lower, current.languages);
197
+ forms = stems.length ? [folded].concat(stems) : folded;
198
+ if (cache.size >= TERM_FORMS_CACHE_LIMIT) {
199
+ cache.delete(cache.keys().next().value);
200
+ }
201
+ cache.set(lower, forms);
202
+ }
203
+ return forms;
204
+ };
205
+ }
206
+
207
+ function detectionTokens(doc) {
208
+ const text = [
209
+ doc.title,
210
+ doc.description,
211
+ String(doc.body || "").slice(0, DETECTION_SAMPLE_CHARACTERS),
212
+ ]
213
+ .map((value) => String(value || ""))
214
+ .join("\n");
215
+ return tokenize(text).map((token) => token.toLowerCase());
216
+ }
217
+
218
+ function indexState(index) {
219
+ let state = SEARCH_INDEXES.get(index);
220
+ if (!state) {
221
+ state = { detection: null, variants: new Map() };
222
+ SEARCH_INDEXES.set(index, state);
223
+ }
224
+ return state;
225
+ }
226
+
227
+ // Bundle-level detection (R-2): each bundle's set from its own documents; the
228
+ // union stems queries.
229
+ function detectIndexLanguages(index, state) {
230
+ if (!state.detection) {
231
+ const tokenLists = new Map();
232
+ index.concepts.forEach((doc) => {
233
+ const bundle = String(doc.bundle);
234
+ if (!tokenLists.has(bundle)) {
235
+ tokenLists.set(bundle, []);
236
+ }
237
+ tokenLists.get(bundle).push(detectionTokens(doc));
238
+ });
239
+ const bundles = {};
240
+ const union = new Set();
241
+ Array.from(tokenLists.keys())
242
+ .sort()
243
+ .forEach((bundle) => {
244
+ bundles[bundle] = detectLanguages(tokenLists.get(bundle));
245
+ bundles[bundle].languages.forEach((code) => {
246
+ union.add(code);
247
+ });
248
+ });
249
+ state.detection = { bundles, languages: canonicalCodes(Array.from(union)) };
250
+ }
251
+ return state.detection;
252
+ }
253
+
254
+ // Precedence: explicit option, `index.languages` (CLI --languages),
255
+ // `index.project.languages` (project config), then per-bundle detection.
256
+ function requestedLanguages(index, options) {
257
+ let value;
258
+ if (options && options.languages !== undefined) {
259
+ value = options.languages;
260
+ } else if (index.languages !== undefined) {
261
+ value = index.languages;
262
+ } else if (index.project && index.project.languages !== undefined) {
263
+ value = index.project.languages;
264
+ }
265
+ const normalized = normalizeLanguages(value);
266
+ return normalized === "auto" ? "auto" : canonicalCodes(normalized);
267
+ }
268
+
269
+ // The effective setting: the union used for queries, a per-bundle lookup for
270
+ // documents, and the cache key (canonical, so ["it","en"] and ["en","it"]
271
+ // share one index).
272
+ function languagePlan(index, options) {
273
+ const requested = requestedLanguages(index, options);
274
+ if (requested !== "auto") {
275
+ return {
276
+ requested,
277
+ languages: requested,
278
+ key: languageKey(requested),
279
+ bundles: null,
280
+ languagesFor: () => requested,
281
+ };
282
+ }
283
+ const detection = detectIndexLanguages(index, indexState(index));
284
+ return {
285
+ requested,
286
+ languages: detection.languages,
287
+ key: "auto",
288
+ bundles: detection.bundles,
289
+ languagesFor: (bundle) =>
290
+ detection.bundles[bundle] ? detection.bundles[bundle].languages : detection.languages,
291
+ };
292
+ }
293
+
294
+ // Languages a query is analysed with: the filtered bundle's own set when a
295
+ // `bundle` filter is given and detection ran, otherwise the union.
296
+ function searchLanguages(index, options) {
297
+ const plan = languagePlan(index, options);
298
+ const bundle = options?.bundle;
299
+ if (bundle && plan.bundles?.[String(bundle)]) {
300
+ return canonicalCodes(plan.bundles[String(bundle)].languages);
301
+ }
302
+ return plan.languages;
303
+ }
304
+
305
+ function buildVariant(index, plan) {
306
+ const current = { languages: [], key: "none" };
307
+ const searchIndex = new MiniSearch({
308
+ fields: SEARCH_FIELDS,
309
+ idField: "id",
310
+ tokenize: fieldTokenize,
311
+ processTerm: createTermProcessor(current),
312
+ searchOptions: {
313
+ boost: SEARCH_BOOSTS,
314
+ combineWith: "AND",
315
+ },
316
+ });
317
+ const upsert = (doc) => {
318
+ current.languages = plan.languagesFor(String(doc.bundle));
319
+ current.key = languageKey(current.languages);
320
+ const document = searchDocument(doc);
321
+ if (searchIndex.has(document.id)) {
322
+ searchIndex.replace(document);
323
+ } else {
324
+ searchIndex.add(document);
325
+ }
326
+ const streams = STREAM_CACHES.get(searchIndex);
327
+ if (streams) {
328
+ streams.delete(document.id);
329
+ }
330
+ };
331
+ index.concepts.forEach(upsert);
332
+ return { searchIndex, upsert, languages: plan.languages, bundles: plan.bundles };
333
+ }
334
+
335
+ function searchVariant(index, options) {
336
+ const state = indexState(index);
337
+ const plan = languagePlan(index, options);
338
+ let variant = state.variants.get(plan.key);
339
+ if (!variant) {
340
+ variant = buildVariant(index, plan);
341
+ state.variants.set(plan.key, variant);
342
+ }
343
+ return variant;
344
+ }
345
+
346
+ function prepareSearchIndex(index, options) {
347
+ return searchVariant(index, options).searchIndex;
348
+ }
349
+
350
+ // What a search over `index` would use, without building anything but the
351
+ // detection pass: the requested setting, the effective union, and per-bundle
352
+ // sets with document counts when detection ran.
353
+ function describeSearchIndex(index, options) {
354
+ const plan = languagePlan(index, options);
355
+ return { requested: plan.requested, languages: plan.languages, bundles: plan.bundles };
356
+ }
357
+
358
+ const isSingleLetter = (token) => token.length === 1 && !/\d/.test(token);
359
+
360
+ function plainTerm(raw, languages) {
361
+ const folded = foldDiacritics(raw);
362
+ const forms = [folded].concat(stemForms(raw, languages));
363
+ return {
364
+ kind: "plain",
365
+ raw,
366
+ literal: raw,
367
+ forms,
368
+ pieces: [],
369
+ leaves: [raw].concat(forms),
370
+ snippetTerms: [folded],
371
+ stems: forms.slice(1),
372
+ };
373
+ }
374
+
375
+ // Quoted text and identifier-looking chunks: the lower-cased literal as a
376
+ // whole (accents kept, for the exact field) plus its pieces folded the way
377
+ // the text fields index raw tokens; nothing stemmed or dropped.
378
+ function literalTerm(kind, text) {
379
+ const literal = text.toLowerCase().replace(/\s+/g, " ").trim();
380
+ const pieces = rawTokens(literal);
381
+ return {
382
+ kind,
383
+ raw: literal,
384
+ literal,
385
+ forms: [],
386
+ pieces,
387
+ leaves: [literal].concat(pieces),
388
+ snippetTerms: [literal].concat(pieces),
389
+ stems: [],
390
+ };
391
+ }
392
+
393
+ // Query analysis (R-3): quoted spans and identifier-looking chunks become
394
+ // literal terms; other chunks are tokenised, lower-cased, and lose stop words
395
+ // of the configured languages and single letters (reported as `dropped`).
396
+ // When nothing would be left, the dropped tokens are searched literally
397
+ // instead of returning nothing; the term limit counts what is left.
398
+ function analyzeQuery(value, options) {
399
+ const config = options || {};
400
+ const query = String(value || "").trim();
401
+ if (Array.from(query).length > MAX_QUERY_CHARACTERS) {
402
+ throw new TypeError(`Search query must not exceed ${MAX_QUERY_CHARACTERS} characters.`);
403
+ }
404
+ const languages = config.languages || [];
405
+ const stopWords = config.stopWords || stopWordsFor(languages);
406
+ const tokens = [];
407
+ const dropped = [];
408
+ const seen = new Set();
409
+ const push = (term) => {
410
+ const key = `${term.kind}:${term.raw}`;
411
+ if (!seen.has(key)) {
412
+ seen.add(key);
413
+ tokens.push(term);
414
+ }
415
+ };
416
+ for (const piece of query.matchAll(QUERY_PIECES)) {
417
+ const quoted = piece[1] !== undefined ? piece[1] : piece[2];
418
+ if (quoted !== undefined) {
419
+ const term = literalTerm("exact", quoted);
420
+ if (term.pieces.length) {
421
+ push(term);
422
+ }
423
+ continue;
424
+ }
425
+ const chunk = piece[3].replace(EDGE_PUNCTUATION, "");
426
+ if (!chunk) {
427
+ continue;
428
+ }
429
+ if (IDENTIFIER_CHUNK.test(chunk)) {
430
+ push(literalTerm("identifier", chunk));
431
+ continue;
432
+ }
433
+ tokenize(chunk)
434
+ .map((token) => token.toLowerCase())
435
+ .filter(Boolean)
436
+ .forEach((token) => {
437
+ if (isSingleLetter(token) || stopWords.has(token)) {
438
+ if (!dropped.includes(token)) {
439
+ dropped.push(token);
440
+ }
441
+ return;
442
+ }
443
+ push(plainTerm(token, languages));
444
+ });
445
+ }
446
+ if (!tokens.length && dropped.length) {
447
+ dropped.splice(0, dropped.length).forEach((token) => {
448
+ push(plainTerm(token, languages));
449
+ });
450
+ }
451
+ if (tokens.length > MAX_QUERY_TERMS) {
452
+ throw new TypeError(`Search query must not exceed ${MAX_QUERY_TERMS} terms.`);
453
+ }
454
+ return { query, terms: tokens.map((term) => term.raw), tokens, dropped, languages };
455
+ }
456
+
457
+ // 0.9.1 shape ({ query, terms }) plus the analysis fields.
458
+ function normalizeSearchQuery(value, options) {
459
+ return analyzeQuery(value, options);
460
+ }
461
+
462
+ // One term of the query tree. Plain terms: the literal against the exact
463
+ // field, the folded token and stems against the text fields (one node over
464
+ // every field when the literal needs no folding, so stems alone are kept
465
+ // away from the exact field). Literal terms: the whole compound against the
466
+ // exact field, or every piece somewhere in the document; the run is
467
+ // verified for adjacency afterwards.
468
+ function termNode(term) {
469
+ if (term.kind === "plain") {
470
+ const [folded, ...stems] = term.forms;
471
+ if (term.literal === folded) {
472
+ const queries = [{ combineWith: "OR", fields: SEARCH_FIELDS, queries: [folded] }];
473
+ if (stems.length) {
474
+ queries.push({ combineWith: "OR", fields: TEXT_FIELDS, queries: stems });
475
+ }
476
+ return queries.length === 1 ? queries[0] : { combineWith: "OR", queries };
477
+ }
478
+ return {
479
+ combineWith: "OR",
480
+ queries: [
481
+ { combineWith: "OR", fields: [EXACT_FIELD], queries: [term.literal] },
482
+ { combineWith: "OR", fields: TEXT_FIELDS, queries: term.forms },
483
+ ],
484
+ };
485
+ }
486
+ if (term.pieces.length === 1 && term.pieces[0] === term.literal) {
487
+ return { combineWith: "OR", fields: SEARCH_FIELDS, queries: [term.literal] };
488
+ }
489
+ return {
490
+ combineWith: "OR",
491
+ queries: [
492
+ { combineWith: "OR", fields: [EXACT_FIELD], queries: [term.literal] },
493
+ { combineWith: "AND", fields: SEARCH_FIELDS, queries: term.pieces },
494
+ ],
495
+ };
496
+ }
497
+
498
+ function queryTree(tokens, combineWith) {
499
+ return { combineWith, queries: tokens.map(termNode) };
500
+ }
501
+
502
+ // A document's text fields and exact segments in index normalisation
503
+ // (lower-cased, diacritics folded, no stems), joined by a control character
504
+ // no run can cross, plus its literal exact tokens; cached per MiniSearch
505
+ // instance for literal-term verification.
506
+ function documentText(searchIndex, id, doc) {
507
+ let cache = STREAM_CACHES.get(searchIndex);
508
+ if (!cache) {
509
+ cache = new Map();
510
+ STREAM_CACHES.set(searchIndex, cache);
511
+ }
512
+ let entry = cache.get(id);
513
+ if (!entry) {
514
+ const segments = exactSegments(doc);
515
+ entry = {
516
+ text: textFieldValues(doc)
517
+ .concat(segments)
518
+ .map((value) => foldDiacritics(value.toLowerCase()))
519
+ .join(FIELD_BREAK),
520
+ exact: new Set(exactTokens(segments.join("\n"))),
521
+ };
522
+ if (cache.size >= STREAM_CACHE_LIMIT) {
523
+ cache.delete(cache.keys().next().value);
524
+ }
525
+ cache.set(id, entry);
526
+ }
527
+ return entry;
528
+ }
529
+
530
+ const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
531
+
532
+ // Matches `pieces` as whole tokens, adjacent and in order, within one field.
533
+ function runPattern(pieces) {
534
+ const run = pieces.map(escapeRegExp).join(`${SEPARATOR_CLASS}+`);
535
+ return new RegExp(`(?:^|${SEPARATOR_CLASS})${run}(?=${SEPARATOR_CLASS}|$)`, "u");
536
+ }
537
+
538
+ // A literal term counts for a document only when the whole compound is one
539
+ // of its exact tokens or its pieces form an adjacent run in one of its
540
+ // fields; a stem never satisfies it. Documents the tree matched without the
541
+ // literal evidence are skipped before any normalisation.
542
+ function literalTermMatched(searchIndex, term, result, options) {
543
+ const matchedLeaves = result.queryTerms;
544
+ const evidence =
545
+ matchedLeaves.includes(term.literal) ||
546
+ term.pieces.every((piece) => matchedLeaves.includes(piece));
547
+ if (!evidence) {
548
+ return false;
549
+ }
550
+ const doc = options.document ? options.document(String(result.id)) : null;
551
+ if (!doc) {
552
+ return evidence;
553
+ }
554
+ const entry = documentText(searchIndex, String(result.id), doc);
555
+ if (entry.exact.has(term.literal)) {
556
+ return true;
557
+ }
558
+ if (!term.pattern) {
559
+ term.pattern = runPattern(term.pieces);
560
+ }
561
+ return term.pattern.test(entry.text);
562
+ }
563
+
564
+ // One MiniSearch run over analysed terms. `combineWith` AND requires every
565
+ // logical term, OR accepts any. Prefix and fuzzy matching stay off (R-12).
566
+ // Plain terms are attributed through their matched leaves, literal terms
567
+ // through verification; each hit reports the raw terms it matched and the
568
+ // share of distinct logical terms matched. `options.document(id)` supplies
569
+ // documents for the verification.
570
+ function runQuery(searchIndex, tokens, options) {
571
+ const config = options || {};
572
+ const combineWith = config.combineWith === "OR" ? "OR" : "AND";
573
+ // Leaf string -> positions of the plain terms it belongs to.
574
+ const leafPositions = new Map();
575
+ const literalPositions = [];
576
+ tokens.forEach((term, position) => {
577
+ if (term.kind !== "plain") {
578
+ literalPositions.push(position);
579
+ return;
580
+ }
581
+ term.leaves.forEach((leaf) => {
582
+ if (!leafPositions.has(leaf)) {
583
+ leafPositions.set(leaf, []);
584
+ }
585
+ leafPositions.get(leaf).push(position);
586
+ });
587
+ });
588
+ const results = searchIndex.search(
589
+ queryTree(tokens, combineWith),
590
+ Object.assign({ filter: config.filter }, LEAF_OPTIONS),
591
+ );
592
+ const hits = [];
593
+ results.forEach((result) => {
594
+ const seen = new Array(tokens.length).fill(false);
595
+ result.queryTerms.forEach((leaf) => {
596
+ const owners = leafPositions.get(leaf);
597
+ if (owners) {
598
+ owners.forEach((position) => {
599
+ seen[position] = true;
600
+ });
601
+ }
602
+ });
603
+ literalPositions.forEach((position) => {
604
+ if (literalTermMatched(searchIndex, tokens[position], result, config)) {
605
+ seen[position] = true;
606
+ }
607
+ });
608
+ const positions = [];
609
+ seen.forEach((flag, position) => {
610
+ if (flag) {
611
+ positions.push(position);
612
+ }
613
+ });
614
+ if (!positions.length || (combineWith === "AND" && positions.length !== tokens.length)) {
615
+ return;
616
+ }
617
+ hits.push({
618
+ id: String(result.id),
619
+ score: result.score,
620
+ positions,
621
+ coverage: positions.length / tokens.length,
622
+ });
623
+ });
624
+ return hits;
625
+ }
626
+
627
+ // Incremental maintenance for a functional index update (WP2): every cached
628
+ // variant of `index` learns the new documents (stemmed with their bundle's
629
+ // set) or forgets the URIs. Detection is not re-run; a bundle unknown at
630
+ // build time takes the union. Both return false when `index` has no lexical
631
+ // index yet, in which case the next search builds one from the documents.
632
+ function addDocuments(index, docs) {
633
+ const state = SEARCH_INDEXES.get(index);
634
+ if (!state) {
635
+ return false;
636
+ }
637
+ state.variants.forEach((variant) => {
638
+ (docs || []).forEach((doc) => {
639
+ variant.upsert(doc);
640
+ });
641
+ });
642
+ return true;
643
+ }
644
+
645
+ function removeDocuments(index, uris) {
646
+ const state = SEARCH_INDEXES.get(index);
647
+ if (!state) {
648
+ return false;
649
+ }
650
+ state.variants.forEach((variant) => {
651
+ (uris || []).forEach((uri) => {
652
+ if (variant.searchIndex.has(uri)) {
653
+ variant.searchIndex.discard(uri);
654
+ }
655
+ const streams = STREAM_CACHES.get(variant.searchIndex);
656
+ if (streams) {
657
+ streams.delete(uri);
658
+ }
659
+ });
660
+ });
661
+ return true;
662
+ }
663
+
664
+ // Moves the cached lexical index from one OKF index object to its successor,
665
+ // so an updated index keeps the structure instead of rebuilding it. The
666
+ // previous index must not be searched afterwards.
667
+ function adoptSearchIndex(previous, next) {
668
+ const state = SEARCH_INDEXES.get(previous);
669
+ if (!state || previous === next) {
670
+ return false;
671
+ }
672
+ SEARCH_INDEXES.delete(previous);
673
+ SEARCH_INDEXES.set(next, state);
674
+ return true;
675
+ }
676
+
677
+ module.exports = {
678
+ EXACT_FIELD,
679
+ MAX_QUERY_CHARACTERS,
680
+ MAX_QUERY_TERMS,
681
+ SEARCH_BOOSTS,
682
+ SEARCH_FIELDS,
683
+ TEXT_FIELDS,
684
+ addDocuments,
685
+ adoptSearchIndex,
686
+ analyzeQuery,
687
+ describeSearchIndex,
688
+ exactTokens,
689
+ normalizeSearchQuery,
690
+ prepareSearchIndex,
691
+ removeDocuments,
692
+ runQuery,
693
+ searchLanguages,
694
+ searchVariant,
695
+ };