@mfdaves/okf-mcp 0.9.1 → 0.12.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +172 -17
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/policies/authoring-safety.md +11 -1
  5. package/package.json +17 -5
  6. package/server.json +2 -2
  7. package/src/assets.js +165 -73
  8. package/src/authoring/candidates.js +391 -0
  9. package/src/authoring/git-preflight.js +225 -0
  10. package/src/authoring/git-publish.js +776 -0
  11. package/src/authoring/live.js +789 -0
  12. package/src/authoring/paths.js +112 -0
  13. package/src/authoring/policy-hook.js +221 -0
  14. package/src/authoring/proposal-policy.js +137 -0
  15. package/src/authoring/receipts.js +329 -0
  16. package/src/authoring/render.js +38 -0
  17. package/src/authoring/shared.js +75 -0
  18. package/src/authoring/staging.js +216 -0
  19. package/src/authoring.js +386 -287
  20. package/src/cache/files.js +147 -0
  21. package/src/cache/lock.js +201 -0
  22. package/src/cache/paths.js +197 -0
  23. package/src/cache/store.js +703 -0
  24. package/src/catalog-text.js +19 -0
  25. package/src/cli.js +184 -151
  26. package/src/computation.js +99 -61
  27. package/src/context/assemble.js +567 -0
  28. package/src/context/cli-options.js +88 -0
  29. package/src/context/score.js +234 -0
  30. package/src/context/sections.js +274 -0
  31. package/src/embed/interface.js +160 -0
  32. package/src/embed/openai.js +715 -0
  33. package/src/embed/prepare.js +182 -0
  34. package/src/embed/units.js +96 -0
  35. package/src/git-source.js +112 -52
  36. package/src/graph.js +95 -37
  37. package/src/http-server.js +36 -17
  38. package/src/index.js +15 -0
  39. package/src/indexer/assemble.js +566 -0
  40. package/src/indexer/build.js +290 -0
  41. package/src/indexer/cache.js +81 -0
  42. package/src/indexer/edges.js +430 -0
  43. package/src/indexer/index.js +44 -0
  44. package/src/indexer/paths.js +178 -0
  45. package/src/indexer/resolve.js +282 -0
  46. package/src/indexer/update.js +720 -0
  47. package/src/indexer.js +3 -1068
  48. package/src/live-authoring.js +4 -2329
  49. package/src/markdown.js +21 -16
  50. package/src/mcp/registry.js +328 -0
  51. package/src/mcp/resources.js +45 -0
  52. package/src/mcp/server.js +472 -0
  53. package/src/mcp/tools/catalog.js +244 -0
  54. package/src/mcp/tools/computation.js +83 -0
  55. package/src/mcp/tools/context.js +133 -0
  56. package/src/mcp/tools/graph.js +167 -0
  57. package/src/mcp/tools/live-authoring.js +316 -0
  58. package/src/mcp/tools/migration.js +86 -0
  59. package/src/mcp/tools/producers.js +79 -0
  60. package/src/mcp/tools/proposals.js +213 -0
  61. package/src/mcp/tools/provenance.js +87 -0
  62. package/src/mcp/tools/remote.js +79 -0
  63. package/src/mcp/tools/search.js +242 -0
  64. package/src/mcp-server.js +3 -1611
  65. package/src/migration.js +431 -253
  66. package/src/parser.js +245 -130
  67. package/src/plugins.js +71 -48
  68. package/src/policy/write-policy.js +713 -0
  69. package/src/producer-publisher.js +195 -77
  70. package/src/producers.js +265 -101
  71. package/src/project.js +277 -93
  72. package/src/remote.js +96 -61
  73. package/src/search/cli-options.js +107 -0
  74. package/src/search/confidence.js +61 -0
  75. package/src/search/dense.js +304 -0
  76. package/src/search/embedding-cli.js +57 -0
  77. package/src/search/embedding-config.js +217 -0
  78. package/src/search/fusion.js +249 -0
  79. package/src/search/hybrid.js +526 -0
  80. package/src/search/languages.js +200 -0
  81. package/src/search/lexical.js +695 -0
  82. package/src/search/query.js +620 -0
  83. package/src/search/stem/among.js +43 -0
  84. package/src/search/stem/base-stemmer.js +311 -0
  85. package/src/search/stem/en-regions.js +456 -0
  86. package/src/search/stem/en-suffixes.js +664 -0
  87. package/src/search/stem/en-tables.js +176 -0
  88. package/src/search/stem/en.js +238 -0
  89. package/src/search/stem/it-regions.js +459 -0
  90. package/src/search/stem/it-suffixes.js +471 -0
  91. package/src/search/stem/it-tables.js +257 -0
  92. package/src/search/stem/it.js +155 -0
  93. package/src/search-index.js +2 -81
  94. package/src/search.js +2 -252
  95. package/src/store.js +134 -81
  96. package/src/v02.js +301 -105
  97. package/src/validation.js +24 -20
@@ -0,0 +1,620 @@
1
+ "use strict";
2
+
3
+ const { conceptSummary, edgesBySource, linkSets } = require("../indexer");
4
+ const { normalizeV02Signals } = require("../v02");
5
+ const { resolveConcept } = require("../indexer");
6
+ const { foldDiacritics } = require("./languages");
7
+ const { analyzeQuery, prepareSearchIndex, runQuery, searchLanguages } = require("./lexical");
8
+ const { denseSearch } = require("./dense");
9
+ const { fuseRankings, fusionNormalization, resolveAlpha } = require("./fusion");
10
+ const { LOW_CONFIDENCE_COVERAGE, assessConfidence } = require("./confidence");
11
+
12
+ const MAX_COMPACT_TEXT = 500;
13
+ // `auto` runs the all-term query first; when it yields fewer than `autoFill`
14
+ // hits, the any-term results follow the all-term hits (R-1). It is the
15
+ // default since 0.10 (section 10). With no all-term hit the page is the
16
+ // any-term ranking (mode "any"); with some, the all-term hits keep their order
17
+ // and precedence and the any-term results are appended (mode "auto"); with
18
+ // `autoFill` or more, only the all-term hits are returned (mode "all").
19
+ const SEARCH_MODES = Object.freeze(["auto", "all", "any"]);
20
+ // 0.11.0-rc.2: a single long document that happens to contain every term must
21
+ // not hide the documents that match most of them.
22
+ const DEFAULT_AUTO_FILL = 3;
23
+ const MAX_AUTO_FILL = 250;
24
+ // R-4: a response is flagged when the any-term fallback produced it and its
25
+ // top result matched strictly fewer than half of the query terms; with the
26
+ // semantic side, also when no candidate is similar enough (./confidence.js).
27
+ // R-6: optional score multipliers, off by default, applied only when the
28
+ // signal is present on the document.
29
+ const FRESHNESS_PRIOR = Object.freeze({ deprecated: 0.5, stale: 0.7, draft: 0.85 });
30
+ // R-10: how many dense neighbours join the lexical hits before fusion. Fixed,
31
+ // independent of offset and limit, so every page of one query paginates the
32
+ // same candidate set: total and order are identical across pages.
33
+ const DENSE_DEPTH = 200;
34
+
35
+ function lower(value) {
36
+ return String(value || "").toLowerCase();
37
+ }
38
+
39
+ function compactText(value) {
40
+ const text = String(value || "");
41
+ return text.length > MAX_COMPACT_TEXT ? `${text.slice(0, MAX_COMPACT_TEXT - 1)}…` : text;
42
+ }
43
+
44
+ function compactConceptSummary(doc) {
45
+ return {
46
+ uri: doc.uri,
47
+ title: compactText(doc.title),
48
+ type: compactText(doc.type),
49
+ description: compactText(doc.description),
50
+ };
51
+ }
52
+
53
+ function asArray(value) {
54
+ if (value === undefined || value === null) {
55
+ return [];
56
+ }
57
+ return Array.isArray(value) ? value : [value];
58
+ }
59
+
60
+ function frontmatterMatches(frontmatter, filters) {
61
+ const entries = Object.entries(filters || {});
62
+ return entries.every(([key, expected]) => {
63
+ const actual = frontmatter[key];
64
+ if (Array.isArray(actual)) {
65
+ return actual.map(String).some((entry) => lower(entry) === lower(expected));
66
+ }
67
+ return lower(actual) === lower(expected);
68
+ });
69
+ }
70
+
71
+ function docText(doc) {
72
+ return [
73
+ doc.title,
74
+ doc.description,
75
+ doc.type,
76
+ doc.path,
77
+ doc.tags.join(" "),
78
+ doc.aliases.join(" "),
79
+ doc.body,
80
+ ].join("\n");
81
+ }
82
+
83
+ const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
84
+
85
+ // Earliest occurrence of a query term's literal or folded form (diacritics
86
+ // folded on both sides); failing that, of a word starting with one of its stems.
87
+ function snippetFor(doc, tokens) {
88
+ if (!tokens.length) {
89
+ return "";
90
+ }
91
+ const text = docText(doc).replace(/\s+/g, " ");
92
+ const normalized = foldDiacritics(lower(text));
93
+ let idx = -1;
94
+ let matchedLength = 0;
95
+ const consider = (candidate, length) => {
96
+ if (candidate !== -1 && (idx === -1 || candidate < idx)) {
97
+ idx = candidate;
98
+ matchedLength = length;
99
+ }
100
+ };
101
+ tokens.forEach((token) => {
102
+ token.snippetTerms.forEach((term) => {
103
+ const folded = foldDiacritics(term);
104
+ consider(normalized.indexOf(folded), folded.length);
105
+ });
106
+ });
107
+ if (idx === -1) {
108
+ tokens.forEach((token) => {
109
+ token.stems.forEach((stem) => {
110
+ const match = new RegExp(
111
+ `(?<![\\p{L}\\p{N}])${escapeRegExp(stem)}[\\p{L}\\p{N}]*`,
112
+ "u",
113
+ ).exec(normalized);
114
+ if (match) {
115
+ consider(match.index, match[0].length);
116
+ }
117
+ });
118
+ });
119
+ }
120
+ if (idx === -1) {
121
+ return "";
122
+ }
123
+ const start = Math.max(0, idx - 50);
124
+ return text.slice(start, idx + matchedLength + 80).trim();
125
+ }
126
+
127
+ function signalsForDoc(doc, asOf) {
128
+ if (!asOf) {
129
+ return doc.signals || {};
130
+ }
131
+ const signals = normalizeV02Signals(doc, { asOf });
132
+ if (signals.computation && doc.signals?.computation) {
133
+ signals.computation.structuralReady = signals.computation.ready;
134
+ signals.computation.assetsReady = doc.signals.computation.assetsReady;
135
+ signals.computation.attestationReady = Boolean(
136
+ signals.computation.structuralReady && signals.computation.assetsReady,
137
+ );
138
+ }
139
+ return signals;
140
+ }
141
+
142
+ function applyFilters(index, options) {
143
+ const config = options || {};
144
+ const typeFilters = asArray(config.types || config.type).map(lower);
145
+ const tagsAny = asArray(config.tagsAny || config.tag).map(lower);
146
+ const tagsAll = asArray(config.tagsAll).map(lower);
147
+ const statuses = asArray(config.statuses || config.status).map(lower);
148
+ const trustTiers = asArray(config.trustTiers || config.trustTier).map(lower);
149
+ const freshness = asArray(config.freshness).map(lower);
150
+ const linkedToDoc = resolveConcept(index, config.linkedTo);
151
+ const linkedFromDoc = resolveConcept(index, config.linkedFrom);
152
+ const linkedTo = linkedToDoc ? linkedToDoc.uri : config.linkedTo;
153
+ const linkedFrom = linkedFromDoc ? linkedFromDoc.uri : config.linkedFrom;
154
+ const { outbound, inbound } = linkSets(index);
155
+ return index.concepts.filter((doc) => {
156
+ const signals = signalsForDoc(doc, config.asOf);
157
+ if (config.bundle && doc.bundle !== config.bundle) {
158
+ return false;
159
+ }
160
+ if (config.pathPrefix && !doc.path.startsWith(config.pathPrefix)) {
161
+ return false;
162
+ }
163
+ if (typeFilters.length && !typeFilters.includes(lower(doc.type))) {
164
+ return false;
165
+ }
166
+ const docTags = doc.tags.map(lower);
167
+ if (tagsAny.length && !tagsAny.some((tag) => docTags.includes(tag))) {
168
+ return false;
169
+ }
170
+ if (tagsAll.length && !tagsAll.every((tag) => docTags.includes(tag))) {
171
+ return false;
172
+ }
173
+ if (config.frontmatter && !frontmatterMatches(doc.frontmatter, config.frontmatter)) {
174
+ return false;
175
+ }
176
+ if (linkedTo && !(outbound.get(doc.uri) || new Set()).has(linkedTo)) {
177
+ return false;
178
+ }
179
+ if (linkedFrom && !(inbound.get(doc.uri) || new Set()).has(linkedFrom)) {
180
+ return false;
181
+ }
182
+ if (
183
+ config.orphanOnly &&
184
+ ((outbound.get(doc.uri) || new Set()).size > 0 ||
185
+ (inbound.get(doc.uri) || new Set()).size > 0)
186
+ ) {
187
+ return false;
188
+ }
189
+ if (config.relationType) {
190
+ const hasRelation = (edgesBySource(index).get(doc.uri) || []).some(
191
+ (edge) =>
192
+ !edge.broken &&
193
+ edge.kind === "relation" &&
194
+ lower(edge.relationType) === lower(config.relationType),
195
+ );
196
+ if (!hasRelation) {
197
+ return false;
198
+ }
199
+ }
200
+ if (statuses.length && !statuses.includes(lower(signals.status))) {
201
+ return false;
202
+ }
203
+ if (trustTiers.length && !trustTiers.includes(lower(signals.trustTier))) {
204
+ return false;
205
+ }
206
+ if (freshness.length && !freshness.includes(lower(signals.freshness))) {
207
+ return false;
208
+ }
209
+ if (config.hasSources !== undefined && Boolean(signals.sources?.length) !== config.hasSources) {
210
+ return false;
211
+ }
212
+ if (config.runtime && lower(signals.computation?.runtime) !== lower(config.runtime)) {
213
+ return false;
214
+ }
215
+ if (
216
+ config.attestationReady !== undefined &&
217
+ Boolean(signals.computation?.attestationReady) !== config.attestationReady
218
+ ) {
219
+ return false;
220
+ }
221
+ if (config.generatedBy && lower(signals.generated?.by) !== lower(config.generatedBy)) {
222
+ return false;
223
+ }
224
+ if (
225
+ config.verifiedBy &&
226
+ !(signals.verifiedEvents || []).some((event) => lower(event.by) === lower(config.verifiedBy))
227
+ ) {
228
+ return false;
229
+ }
230
+ return true;
231
+ });
232
+ }
233
+
234
+ function compareDocs(left, right) {
235
+ return left.path.localeCompare(right.path) || left.uri.localeCompare(right.uri);
236
+ }
237
+
238
+ function searchMode(value) {
239
+ if (value === undefined || value === null || value === "") {
240
+ return "auto";
241
+ }
242
+ if (!SEARCH_MODES.includes(value)) {
243
+ throw new TypeError(`Search mode must be one of ${SEARCH_MODES.join(", ")}.`);
244
+ }
245
+ return value;
246
+ }
247
+
248
+ // Precedence: explicit search option, then the project's `autoFill`, then
249
+ // DEFAULT_AUTO_FILL. 0 restores the 0.10 behaviour (fall back only on zero).
250
+ function autoFillOption(value, index) {
251
+ let raw = value;
252
+ if (raw === undefined || raw === null) {
253
+ raw = index?.project?.autoFill;
254
+ }
255
+ if (raw === undefined || raw === null) {
256
+ return DEFAULT_AUTO_FILL;
257
+ }
258
+ const number = Number(raw);
259
+ if (!Number.isSafeInteger(number) || number < 0 || number > MAX_AUTO_FILL) {
260
+ throw new TypeError(`autoFill must be an integer between 0 and ${MAX_AUTO_FILL}.`);
261
+ }
262
+ return number;
263
+ }
264
+
265
+ function floorOption(value, name) {
266
+ if (value === undefined || value === null) {
267
+ return 0;
268
+ }
269
+ const number = Number(value);
270
+ if (!Number.isFinite(number) || number < 0 || number > 1) {
271
+ throw new TypeError(`${name} must be a number between 0 and 1.`);
272
+ }
273
+ return number;
274
+ }
275
+
276
+ function priorFactor(doc, asOf) {
277
+ const signals = signalsForDoc(doc, asOf);
278
+ let factor = 1;
279
+ if (signals.status === "deprecated") {
280
+ factor *= FRESHNESS_PRIOR.deprecated;
281
+ } else if (signals.status === "draft") {
282
+ factor *= FRESHNESS_PRIOR.draft;
283
+ }
284
+ if (signals.freshness === "stale") {
285
+ factor *= FRESHNESS_PRIOR.stale;
286
+ }
287
+ return factor;
288
+ }
289
+
290
+ // Runs the analysed query under `mode` over the filtered documents. Returns
291
+ // the ranked results (doc, score, matchedTerms, coverage, relevance), the mode
292
+ // that produced them, and the all-term count: the AND result size or, under
293
+ // `any`, the hits that matched every token. The optional prior multiplies
294
+ // scores before ranking; `relevance` is score over the best score shown, and
295
+ // the floors drop results below either value after that. Under `auto` the
296
+ // all-term hits lead in their own order and, when there are fewer than
297
+ // `autoFill` of them, the remaining any-term hits follow in theirs: both
298
+ // passes run over the whole filtered set, so the candidate list, and every
299
+ // page cut from it, does not depend on offset or limit.
300
+ function rankQuery(index, config, analysis, docsByUri) {
301
+ const searchIndex = prepareSearchIndex(index, config);
302
+ const run = (combineWith) =>
303
+ runQuery(searchIndex, analysis.tokens, {
304
+ combineWith,
305
+ filter: (result) => docsByUri.has(String(result.id)),
306
+ document: (id) => docsByUri.get(id),
307
+ });
308
+ const prior = config.freshnessPrior ? (doc) => priorFactor(doc, config.asOf) : () => 1;
309
+ const ranked = (hits) => {
310
+ const list = hits
311
+ .filter((hit) => docsByUri.has(hit.id))
312
+ .map((hit) => {
313
+ const doc = docsByUri.get(hit.id);
314
+ return Object.assign({ doc }, hit, { score: hit.score * prior(doc) });
315
+ });
316
+ list.sort((a, b) => b.score - a.score || compareDocs(a.doc, b.doc));
317
+ return list;
318
+ };
319
+ let results;
320
+ let mode;
321
+ let allTermsTotal;
322
+ // URIs of the all-term hits when the all-term pass ran (the literal guard's group).
323
+ let allTermUris = null;
324
+ if (config.mode === "any") {
325
+ results = ranked(run("OR"));
326
+ mode = "any";
327
+ allTermsTotal = results.filter((hit) => hit.coverage === 1).length;
328
+ } else {
329
+ results = ranked(run("AND"));
330
+ mode = "all";
331
+ allTermsTotal = results.length;
332
+ allTermUris = new Set(results.map((entry) => entry.doc.uri));
333
+ if (config.mode === "auto" && results.length < config.autoFill) {
334
+ const fill = ranked(run("OR")).filter((entry) => !allTermUris.has(entry.doc.uri));
335
+ if (!results.length) {
336
+ results = fill;
337
+ mode = "any";
338
+ allTermUris = null;
339
+ } else if (fill.length) {
340
+ results = results.concat(fill);
341
+ mode = "auto";
342
+ }
343
+ }
344
+ }
345
+ const top = results.reduce((best, entry) => Math.max(best, entry.score), 0);
346
+ results.forEach((entry) => {
347
+ entry.relevance = top > 0 ? entry.score / top : 0;
348
+ });
349
+ // Lexical evidence (matched terms, coverage, score) for candidates the
350
+ // lexical pass did not return: dense neighbours that match only some terms
351
+ // under an all-term pass. An any-term pass (alone or appended) already
352
+ // holds every match.
353
+ const evidence = (uris) => {
354
+ if (mode !== "all" || !uris.size) return [];
355
+ const subset = new Map(Array.from(uris, (uri) => [uri, docsByUri.get(uri)]));
356
+ return runQuery(searchIndex, analysis.tokens, {
357
+ combineWith: "OR",
358
+ filter: (result) => subset.has(String(result.id)),
359
+ document: (id) => subset.get(id),
360
+ })
361
+ .filter((hit) => subset.has(hit.id))
362
+ .map((hit) => {
363
+ const doc = subset.get(hit.id);
364
+ return Object.assign({ doc }, hit, { score: hit.score * prior(doc) });
365
+ })
366
+ .sort((a, b) => b.score - a.score || compareDocs(a.doc, b.doc));
367
+ };
368
+ return {
369
+ results: applyFloors(results, config),
370
+ mode,
371
+ allTermsTotal,
372
+ allTermUris,
373
+ unfloored: results,
374
+ evidence,
375
+ };
376
+ }
377
+
378
+ // The floors keep their lexical meaning under hybrid ranking too: coverage of
379
+ // the query terms and the lexical score relative to the lexical top result.
380
+ function applyFloors(results, config) {
381
+ if (!(config.minCoverage > 0 || config.minRelevance > 0)) {
382
+ return results;
383
+ }
384
+ return results.filter(
385
+ (entry) =>
386
+ entry.coverage >= config.minCoverage &&
387
+ (entry.lexicalRelevance ?? entry.relevance) >= config.minRelevance,
388
+ );
389
+ }
390
+
391
+ // Hybrid ranking (R-10): the lexical hits, before floors, fused with the dense
392
+ // ranking of the filtered candidates. An explicit mode "all" keeps its
393
+ // meaning, every term required, so dense scores only re-rank the all-term
394
+ // hits; under auto and any the dense neighbours join the candidates.
395
+ //
396
+ // Literal guard: when the all-term pass ran and the query carries literal
397
+ // terms (quoted or identifier-like), the all-term hits keep precedence over
398
+ // every other candidate; fusion orders each group. Exact lookups then never
399
+ // lose a document that contains the identifier to a semantic neighbour.
400
+ function hybridRank(index, config, ranking, filteredDocs, analysis) {
401
+ const lexical = ranking.unfloored;
402
+ const passUris = new Set(lexical.map((entry) => entry.doc.uri));
403
+ const byUri = new Map(lexical.map((entry) => [entry.doc.uri, entry]));
404
+ const candidates = config.mode === "all" ? lexical.map((entry) => entry.doc) : filteredDocs;
405
+ const prior = config.freshnessPrior ? (doc) => priorFactor(doc, config.asOf) : null;
406
+ const docs = new Map(candidates.map((doc) => [doc.uri, doc]));
407
+ const nearest = denseSearch(index, {
408
+ queryVector: config.queryVector,
409
+ cache: config.vectorCache,
410
+ units: config.embedUnits,
411
+ candidates,
412
+ limit: DENSE_DEPTH,
413
+ });
414
+ // The best raw cosine over the candidates, before any prior (abstention, WP3c-2).
415
+ const topSimilarity = nearest.length ? nearest[0].score : null;
416
+ const dense = nearest.map((entry) => {
417
+ // R-6 multiplies scores; a negative cosine is divided so it still drops.
418
+ const factor = prior ? prior(docs.get(entry.id)) : 1;
419
+ const score = entry.score >= 0 ? entry.score * factor : entry.score / factor;
420
+ return { id: entry.id, score, unit: entry.unit };
421
+ });
422
+ const units = new Map(dense.map((entry) => [entry.id, entry.unit]));
423
+ // One weight per query (WP3c-2): fixed, or chosen by the adaptive rule from
424
+ // the lexical pass, which does not depend on offset or limit.
425
+ const choice = resolveAlpha(
426
+ config.alpha,
427
+ {
428
+ literal:
429
+ ranking.allTermUris !== null &&
430
+ ranking.allTermUris.size > 0 &&
431
+ analysis.tokens.some((token) => token.kind !== "plain"),
432
+ allTermsTotal: ranking.allTermsTotal,
433
+ autoFill: config.autoFill,
434
+ topCoverage: lexical.reduce((best, entry) => Math.max(best, entry.coverage || 0), 0),
435
+ },
436
+ config.adaptiveAlpha,
437
+ );
438
+ const alpha = choice.alpha;
439
+ // Partial matches among the dense neighbours join the lexical side after
440
+ // the pass's hits (never at alpha 0, which must reproduce the lexical list).
441
+ const extra =
442
+ alpha > 0
443
+ ? ranking.evidence(new Set(dense.map((entry) => entry.id).filter((id) => !passUris.has(id))))
444
+ : [];
445
+ const top = lexical.length ? lexical[0].score : 0;
446
+ extra.forEach((entry) => {
447
+ entry.relevance = top > 0 ? entry.score / top : 0;
448
+ byUri.set(entry.doc.uri, entry);
449
+ });
450
+ const fused = fuseRankings({
451
+ lexical: lexical.concat(extra).map((entry) => ({ id: entry.doc.uri, score: entry.score })),
452
+ dense,
453
+ alpha,
454
+ normalization: config.fusionNormalization,
455
+ });
456
+ // Not at alpha 1, which must reproduce the dense ranking exactly. At alpha
457
+ // 0 the all-term hits lead whatever the terms, as in the lexical ranking
458
+ // (auto appends the any-term results after them), which alpha 0 must
459
+ // reproduce.
460
+ const guard =
461
+ alpha < 1 &&
462
+ ranking.allTermUris !== null &&
463
+ ranking.allTermUris.size > 0 &&
464
+ (alpha === 0 || analysis.tokens.some((token) => token.kind !== "plain"));
465
+ if (guard) {
466
+ const group = (entry) => (ranking.allTermUris.has(entry.id) ? 0 : 1);
467
+ // Array.prototype.sort is stable: fused order holds within each group.
468
+ fused.sort((a, b) => group(a) - group(b));
469
+ }
470
+ const best = fused.length ? Math.max(...fused.map((entry) => entry.fusedScore)) : 0;
471
+ const results = fused.map((entry) => {
472
+ const hit = byUri.get(entry.id);
473
+ const base = hit
474
+ ? Object.assign({}, hit, { lexicalRelevance: hit.relevance })
475
+ : { doc: docs.get(entry.id), positions: [], coverage: 0, lexicalRelevance: 0 };
476
+ return Object.assign(base, {
477
+ score: entry.fusedScore,
478
+ relevance: best > 0 ? entry.fusedScore / best : 0,
479
+ lexicalScore: entry.lexicalScore,
480
+ denseScore: entry.denseScore,
481
+ fusedScore: entry.fusedScore,
482
+ unit: units.get(entry.id),
483
+ });
484
+ });
485
+ return {
486
+ results: applyFloors(results, config),
487
+ alpha,
488
+ alphaReason: choice.reason,
489
+ topSimilarity,
490
+ };
491
+ }
492
+
493
+ function hybridFields(entry) {
494
+ const fields = {
495
+ lexicalScore: entry.lexicalScore,
496
+ denseScore: entry.denseScore,
497
+ fusedScore: entry.fusedScore,
498
+ };
499
+ // Section units cite the section that matched (`uri#anchor`).
500
+ if (entry.unit && entry.unit !== entry.doc.uri) {
501
+ fields.unit = entry.unit;
502
+ }
503
+ return fields;
504
+ }
505
+
506
+ function searchConcepts(index, options) {
507
+ const config = Object.assign({}, options || {}, {
508
+ mode: searchMode(options?.mode),
509
+ autoFill: autoFillOption(options?.autoFill, index),
510
+ minCoverage: floorOption(options?.minCoverage, "minCoverage"),
511
+ minRelevance: floorOption(options?.minRelevance, "minRelevance"),
512
+ });
513
+ const query = String(config.query || "").trim();
514
+ const analysis = analyzeQuery(query, {
515
+ languages: query ? searchLanguages(index, config) : [],
516
+ });
517
+ const limit = Math.max(1, Math.min(Number(config.limit || 25), 250));
518
+ const offset = Math.max(0, Number(config.offset || 0));
519
+ const filteredDocs = applyFilters(index, config);
520
+ let results;
521
+ let ranking = null;
522
+ let hybrid = false;
523
+ let fusion = null;
524
+ if (analysis.query) {
525
+ if (!analysis.tokens.length) {
526
+ // Punctuation-only text: nothing to run, reported under the requested mode.
527
+ results = [];
528
+ ranking = {
529
+ mode: config.mode === "any" ? "any" : "all",
530
+ allTermsTotal: 0,
531
+ allTermUris: null,
532
+ results,
533
+ };
534
+ } else {
535
+ const docsByUri = new Map(filteredDocs.map((doc) => [doc.uri, doc]));
536
+ ranking = rankQuery(index, config, analysis, docsByUri);
537
+ hybrid = config.hybrid === true && Boolean(config.queryVector && config.vectorCache);
538
+ fusion = hybrid ? hybridRank(index, config, ranking, filteredDocs, analysis) : null;
539
+ results = hybrid ? fusion.results : ranking.results;
540
+ }
541
+ } else {
542
+ results = filteredDocs.map((doc) => ({ doc, score: 0 }));
543
+ results.sort((a, b) => compareDocs(a.doc, b.doc));
544
+ }
545
+ const page = results.slice(offset, offset + limit).map((entry) => {
546
+ if (config.detail === "compact") {
547
+ const compact = compactConceptSummary(entry.doc);
548
+ if (ranking) {
549
+ Object.assign(compact, { coverage: entry.coverage, relevance: entry.relevance });
550
+ }
551
+ return hybrid ? Object.assign(compact, hybridFields(entry)) : compact;
552
+ }
553
+ const summary = conceptSummary(entry.doc);
554
+ if (config.asOf) {
555
+ const signals = signalsForDoc(entry.doc, config.asOf);
556
+ summary.signals = Object.assign({}, summary.signals, {
557
+ status: signals.status,
558
+ staleAfter: signals.staleAfter,
559
+ freshness: signals.freshness,
560
+ asOf: signals.asOf,
561
+ trustTier: signals.trustTier,
562
+ });
563
+ }
564
+ Object.assign(summary, {
565
+ score: entry.score,
566
+ snippet: snippetFor(entry.doc, analysis.tokens),
567
+ });
568
+ if (ranking) {
569
+ Object.assign(summary, {
570
+ matchedTerms: entry.positions.map((position) => analysis.tokens[position].raw),
571
+ coverage: entry.coverage,
572
+ relevance: entry.relevance,
573
+ });
574
+ }
575
+ return hybrid ? Object.assign(summary, hybridFields(entry)) : summary;
576
+ });
577
+ const response = { total: results.length, limit, offset };
578
+ if (ranking) {
579
+ // lowConfidence reads the displayed top result's coverage (lexical
580
+ // evidence is computed for every candidate, dense ones included) and,
581
+ // when the semantic side ran, the best similarity (WP3c-2).
582
+ const confidence = assessConfidence(
583
+ {
584
+ mode: ranking.mode,
585
+ coverage: results.length ? results[0].coverage : null,
586
+ topSimilarity: hybrid ? fusion.topSimilarity : null,
587
+ },
588
+ config.minSimilarity,
589
+ );
590
+ Object.assign(response, {
591
+ mode: ranking.mode,
592
+ allTermsTotal: ranking.allTermsTotal,
593
+ lowConfidence: confidence.lowConfidence,
594
+ queryTerms: analysis.terms,
595
+ droppedTerms: analysis.dropped,
596
+ confidence,
597
+ });
598
+ if (hybrid) {
599
+ Object.assign(response, {
600
+ hybrid: true,
601
+ alpha: fusion.alpha,
602
+ normalization: fusionNormalization(config.fusionNormalization),
603
+ // WP3c-2: the weight this query used and why (`fixed` unless adaptive).
604
+ alphaUsed: fusion.alpha,
605
+ alphaReason: fusion.alphaReason,
606
+ });
607
+ }
608
+ }
609
+ response.results = page;
610
+ return response;
611
+ }
612
+
613
+ module.exports = {
614
+ DEFAULT_AUTO_FILL,
615
+ FRESHNESS_PRIOR,
616
+ LOW_CONFIDENCE_COVERAGE,
617
+ SEARCH_MODES,
618
+ applyFilters,
619
+ searchConcepts,
620
+ };
@@ -0,0 +1,43 @@
1
+ /*
2
+ * Copyright (c) AXA Group Operations Spain S.A.
3
+ *
4
+ * Permission is hereby granted, free of charge, to any person obtaining
5
+ * a copy of this software and associated documentation files (the
6
+ * "Software"), to deal in the Software without restriction, including
7
+ * without limitation the rights to use, copy, modify, merge, publish,
8
+ * distribute, sublicense, and/or sell copies of the Software, and to
9
+ * permit persons to whom the Software is furnished to do so, subject to
10
+ * the following conditions:
11
+ *
12
+ * The above copyright notice and this permission notice shall be
13
+ * included in all copies or substantial portions of the Software.
14
+ *
15
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ */
23
+
24
+ // Provenance: derived from @nlpjs/core 4.26.1, src/among.js (MIT, header
25
+ // above). Unchanged apart from the strict-mode prologue; vendored so the
26
+ // lexical search carries no runtime dependency on @nlpjs (plan B-8, WP1).
27
+ "use strict";
28
+
29
+ // One entry of a Snowball suffix table: the suffix, the index of the entry to
30
+ // try next when this one does not apply, the result code, and an optional
31
+ // guard method with the instance it is called on.
32
+ class Among {
33
+ constructor(s, sub, result, method, instance) {
34
+ this.s_size = s.length;
35
+ this.s = s;
36
+ this.substring_i = sub;
37
+ this.result = result;
38
+ this.method = method;
39
+ this.instance = instance;
40
+ }
41
+ }
42
+
43
+ module.exports = Among;