@mfdaves/okf-mcp 0.9.1 → 0.13.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/README.md +452 -26
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/distribution/reference-bundle.md +10 -3
- package/okf/okf-mcp/index.md +2 -1
- package/okf/okf-mcp/interfaces/http-authoring-api.md +11 -2
- package/okf/okf-mcp/interfaces/mcp-http.md +64 -0
- package/okf/okf-mcp/interfaces/mcp-tools.md +25 -6
- package/okf/okf-mcp/overview/okf-mcp.md +16 -6
- package/okf/okf-mcp/policies/authoring-safety.md +28 -5
- package/okf/okf-mcp/runtime/indexer.md +14 -1
- package/okf/okf-mcp/runtime/mcp-server.md +16 -8
- package/okf/okf-mcp/specs/producer-api.md +16 -2
- package/package.json +17 -5
- package/server.json +38 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +434 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +783 -0
- package/src/authoring/live.js +757 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +159 -0
- package/src/authoring/receipts.js +373 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli/commands.js +448 -0
- package/src/cli/http-options.js +193 -0
- package/src/cli/options.js +554 -0
- package/src/cli/usage.js +96 -0
- package/src/cli.js +18 -932
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/error-details.js +53 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/auth.js +409 -0
- package/src/mcp/http.js +317 -0
- package/src/mcp/principal.js +59 -0
- package/src/mcp/registry.js +356 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +567 -0
- package/src/mcp/tools/catalog.js +247 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +225 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +202 -78
- package/src/producers.js +311 -103
- package/src/project.js +346 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/tool-packs.js +346 -0
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,620 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { conceptSummary, edgesBySource, linkSets } = require("../indexer");
|
|
4
|
+
const { normalizeV02Signals } = require("../v02");
|
|
5
|
+
const { resolveConcept } = require("../indexer");
|
|
6
|
+
const { foldDiacritics } = require("./languages");
|
|
7
|
+
const { analyzeQuery, prepareSearchIndex, runQuery, searchLanguages } = require("./lexical");
|
|
8
|
+
const { denseSearch } = require("./dense");
|
|
9
|
+
const { fuseRankings, fusionNormalization, resolveAlpha } = require("./fusion");
|
|
10
|
+
const { LOW_CONFIDENCE_COVERAGE, assessConfidence } = require("./confidence");
|
|
11
|
+
|
|
12
|
+
const MAX_COMPACT_TEXT = 500;
|
|
13
|
+
// `auto` runs the all-term query first; when it yields fewer than `autoFill`
|
|
14
|
+
// hits, the any-term results follow the all-term hits (R-1). It is the
|
|
15
|
+
// default since 0.10 (section 10). With no all-term hit the page is the
|
|
16
|
+
// any-term ranking (mode "any"); with some, the all-term hits keep their order
|
|
17
|
+
// and precedence and the any-term results are appended (mode "auto"); with
|
|
18
|
+
// `autoFill` or more, only the all-term hits are returned (mode "all").
|
|
19
|
+
const SEARCH_MODES = Object.freeze(["auto", "all", "any"]);
|
|
20
|
+
// 0.11.0-rc.2: a single long document that happens to contain every term must
|
|
21
|
+
// not hide the documents that match most of them.
|
|
22
|
+
const DEFAULT_AUTO_FILL = 3;
|
|
23
|
+
const MAX_AUTO_FILL = 250;
|
|
24
|
+
// R-4: a response is flagged when the any-term fallback produced it and its
|
|
25
|
+
// top result matched strictly fewer than half of the query terms; with the
|
|
26
|
+
// semantic side, also when no candidate is similar enough (./confidence.js).
|
|
27
|
+
// R-6: optional score multipliers, off by default, applied only when the
|
|
28
|
+
// signal is present on the document.
|
|
29
|
+
const FRESHNESS_PRIOR = Object.freeze({ deprecated: 0.5, stale: 0.7, draft: 0.85 });
|
|
30
|
+
// R-10: how many dense neighbours join the lexical hits before fusion. Fixed,
|
|
31
|
+
// independent of offset and limit, so every page of one query paginates the
|
|
32
|
+
// same candidate set: total and order are identical across pages.
|
|
33
|
+
const DENSE_DEPTH = 200;
|
|
34
|
+
|
|
35
|
+
function lower(value) {
|
|
36
|
+
return String(value || "").toLowerCase();
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function compactText(value) {
|
|
40
|
+
const text = String(value || "");
|
|
41
|
+
return text.length > MAX_COMPACT_TEXT ? `${text.slice(0, MAX_COMPACT_TEXT - 1)}…` : text;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function compactConceptSummary(doc) {
|
|
45
|
+
return {
|
|
46
|
+
uri: doc.uri,
|
|
47
|
+
title: compactText(doc.title),
|
|
48
|
+
type: compactText(doc.type),
|
|
49
|
+
description: compactText(doc.description),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function asArray(value) {
|
|
54
|
+
if (value === undefined || value === null) {
|
|
55
|
+
return [];
|
|
56
|
+
}
|
|
57
|
+
return Array.isArray(value) ? value : [value];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function frontmatterMatches(frontmatter, filters) {
|
|
61
|
+
const entries = Object.entries(filters || {});
|
|
62
|
+
return entries.every(([key, expected]) => {
|
|
63
|
+
const actual = frontmatter[key];
|
|
64
|
+
if (Array.isArray(actual)) {
|
|
65
|
+
return actual.map(String).some((entry) => lower(entry) === lower(expected));
|
|
66
|
+
}
|
|
67
|
+
return lower(actual) === lower(expected);
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function docText(doc) {
|
|
72
|
+
return [
|
|
73
|
+
doc.title,
|
|
74
|
+
doc.description,
|
|
75
|
+
doc.type,
|
|
76
|
+
doc.path,
|
|
77
|
+
doc.tags.join(" "),
|
|
78
|
+
doc.aliases.join(" "),
|
|
79
|
+
doc.body,
|
|
80
|
+
].join("\n");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
84
|
+
|
|
85
|
+
// Earliest occurrence of a query term's literal or folded form (diacritics
|
|
86
|
+
// folded on both sides); failing that, of a word starting with one of its stems.
|
|
87
|
+
function snippetFor(doc, tokens) {
|
|
88
|
+
if (!tokens.length) {
|
|
89
|
+
return "";
|
|
90
|
+
}
|
|
91
|
+
const text = docText(doc).replace(/\s+/g, " ");
|
|
92
|
+
const normalized = foldDiacritics(lower(text));
|
|
93
|
+
let idx = -1;
|
|
94
|
+
let matchedLength = 0;
|
|
95
|
+
const consider = (candidate, length) => {
|
|
96
|
+
if (candidate !== -1 && (idx === -1 || candidate < idx)) {
|
|
97
|
+
idx = candidate;
|
|
98
|
+
matchedLength = length;
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
tokens.forEach((token) => {
|
|
102
|
+
token.snippetTerms.forEach((term) => {
|
|
103
|
+
const folded = foldDiacritics(term);
|
|
104
|
+
consider(normalized.indexOf(folded), folded.length);
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
if (idx === -1) {
|
|
108
|
+
tokens.forEach((token) => {
|
|
109
|
+
token.stems.forEach((stem) => {
|
|
110
|
+
const match = new RegExp(
|
|
111
|
+
`(?<![\\p{L}\\p{N}])${escapeRegExp(stem)}[\\p{L}\\p{N}]*`,
|
|
112
|
+
"u",
|
|
113
|
+
).exec(normalized);
|
|
114
|
+
if (match) {
|
|
115
|
+
consider(match.index, match[0].length);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
if (idx === -1) {
|
|
121
|
+
return "";
|
|
122
|
+
}
|
|
123
|
+
const start = Math.max(0, idx - 50);
|
|
124
|
+
return text.slice(start, idx + matchedLength + 80).trim();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function signalsForDoc(doc, asOf) {
|
|
128
|
+
if (!asOf) {
|
|
129
|
+
return doc.signals || {};
|
|
130
|
+
}
|
|
131
|
+
const signals = normalizeV02Signals(doc, { asOf });
|
|
132
|
+
if (signals.computation && doc.signals?.computation) {
|
|
133
|
+
signals.computation.structuralReady = signals.computation.ready;
|
|
134
|
+
signals.computation.assetsReady = doc.signals.computation.assetsReady;
|
|
135
|
+
signals.computation.attestationReady = Boolean(
|
|
136
|
+
signals.computation.structuralReady && signals.computation.assetsReady,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
return signals;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function applyFilters(index, options) {
|
|
143
|
+
const config = options || {};
|
|
144
|
+
const typeFilters = asArray(config.types || config.type).map(lower);
|
|
145
|
+
const tagsAny = asArray(config.tagsAny || config.tag).map(lower);
|
|
146
|
+
const tagsAll = asArray(config.tagsAll).map(lower);
|
|
147
|
+
const statuses = asArray(config.statuses || config.status).map(lower);
|
|
148
|
+
const trustTiers = asArray(config.trustTiers || config.trustTier).map(lower);
|
|
149
|
+
const freshness = asArray(config.freshness).map(lower);
|
|
150
|
+
const linkedToDoc = resolveConcept(index, config.linkedTo);
|
|
151
|
+
const linkedFromDoc = resolveConcept(index, config.linkedFrom);
|
|
152
|
+
const linkedTo = linkedToDoc ? linkedToDoc.uri : config.linkedTo;
|
|
153
|
+
const linkedFrom = linkedFromDoc ? linkedFromDoc.uri : config.linkedFrom;
|
|
154
|
+
const { outbound, inbound } = linkSets(index);
|
|
155
|
+
return index.concepts.filter((doc) => {
|
|
156
|
+
const signals = signalsForDoc(doc, config.asOf);
|
|
157
|
+
if (config.bundle && doc.bundle !== config.bundle) {
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
if (config.pathPrefix && !doc.path.startsWith(config.pathPrefix)) {
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
if (typeFilters.length && !typeFilters.includes(lower(doc.type))) {
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
const docTags = doc.tags.map(lower);
|
|
167
|
+
if (tagsAny.length && !tagsAny.some((tag) => docTags.includes(tag))) {
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
170
|
+
if (tagsAll.length && !tagsAll.every((tag) => docTags.includes(tag))) {
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
if (config.frontmatter && !frontmatterMatches(doc.frontmatter, config.frontmatter)) {
|
|
174
|
+
return false;
|
|
175
|
+
}
|
|
176
|
+
if (linkedTo && !(outbound.get(doc.uri) || new Set()).has(linkedTo)) {
|
|
177
|
+
return false;
|
|
178
|
+
}
|
|
179
|
+
if (linkedFrom && !(inbound.get(doc.uri) || new Set()).has(linkedFrom)) {
|
|
180
|
+
return false;
|
|
181
|
+
}
|
|
182
|
+
if (
|
|
183
|
+
config.orphanOnly &&
|
|
184
|
+
((outbound.get(doc.uri) || new Set()).size > 0 ||
|
|
185
|
+
(inbound.get(doc.uri) || new Set()).size > 0)
|
|
186
|
+
) {
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
if (config.relationType) {
|
|
190
|
+
const hasRelation = (edgesBySource(index).get(doc.uri) || []).some(
|
|
191
|
+
(edge) =>
|
|
192
|
+
!edge.broken &&
|
|
193
|
+
edge.kind === "relation" &&
|
|
194
|
+
lower(edge.relationType) === lower(config.relationType),
|
|
195
|
+
);
|
|
196
|
+
if (!hasRelation) {
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
if (statuses.length && !statuses.includes(lower(signals.status))) {
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
if (trustTiers.length && !trustTiers.includes(lower(signals.trustTier))) {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
if (freshness.length && !freshness.includes(lower(signals.freshness))) {
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
if (config.hasSources !== undefined && Boolean(signals.sources?.length) !== config.hasSources) {
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
if (config.runtime && lower(signals.computation?.runtime) !== lower(config.runtime)) {
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
if (
|
|
216
|
+
config.attestationReady !== undefined &&
|
|
217
|
+
Boolean(signals.computation?.attestationReady) !== config.attestationReady
|
|
218
|
+
) {
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
if (config.generatedBy && lower(signals.generated?.by) !== lower(config.generatedBy)) {
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
if (
|
|
225
|
+
config.verifiedBy &&
|
|
226
|
+
!(signals.verifiedEvents || []).some((event) => lower(event.by) === lower(config.verifiedBy))
|
|
227
|
+
) {
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
230
|
+
return true;
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function compareDocs(left, right) {
|
|
235
|
+
return left.path.localeCompare(right.path) || left.uri.localeCompare(right.uri);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function searchMode(value) {
|
|
239
|
+
if (value === undefined || value === null || value === "") {
|
|
240
|
+
return "auto";
|
|
241
|
+
}
|
|
242
|
+
if (!SEARCH_MODES.includes(value)) {
|
|
243
|
+
throw new TypeError(`Search mode must be one of ${SEARCH_MODES.join(", ")}.`);
|
|
244
|
+
}
|
|
245
|
+
return value;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Precedence: explicit search option, then the project's `autoFill`, then
|
|
249
|
+
// DEFAULT_AUTO_FILL. 0 restores the 0.10 behaviour (fall back only on zero).
|
|
250
|
+
function autoFillOption(value, index) {
|
|
251
|
+
let raw = value;
|
|
252
|
+
if (raw === undefined || raw === null) {
|
|
253
|
+
raw = index?.project?.autoFill;
|
|
254
|
+
}
|
|
255
|
+
if (raw === undefined || raw === null) {
|
|
256
|
+
return DEFAULT_AUTO_FILL;
|
|
257
|
+
}
|
|
258
|
+
const number = Number(raw);
|
|
259
|
+
if (!Number.isSafeInteger(number) || number < 0 || number > MAX_AUTO_FILL) {
|
|
260
|
+
throw new TypeError(`autoFill must be an integer between 0 and ${MAX_AUTO_FILL}.`);
|
|
261
|
+
}
|
|
262
|
+
return number;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function floorOption(value, name) {
|
|
266
|
+
if (value === undefined || value === null) {
|
|
267
|
+
return 0;
|
|
268
|
+
}
|
|
269
|
+
const number = Number(value);
|
|
270
|
+
if (!Number.isFinite(number) || number < 0 || number > 1) {
|
|
271
|
+
throw new TypeError(`${name} must be a number between 0 and 1.`);
|
|
272
|
+
}
|
|
273
|
+
return number;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function priorFactor(doc, asOf) {
|
|
277
|
+
const signals = signalsForDoc(doc, asOf);
|
|
278
|
+
let factor = 1;
|
|
279
|
+
if (signals.status === "deprecated") {
|
|
280
|
+
factor *= FRESHNESS_PRIOR.deprecated;
|
|
281
|
+
} else if (signals.status === "draft") {
|
|
282
|
+
factor *= FRESHNESS_PRIOR.draft;
|
|
283
|
+
}
|
|
284
|
+
if (signals.freshness === "stale") {
|
|
285
|
+
factor *= FRESHNESS_PRIOR.stale;
|
|
286
|
+
}
|
|
287
|
+
return factor;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Runs the analysed query under `mode` over the filtered documents. Returns
|
|
291
|
+
// the ranked results (doc, score, matchedTerms, coverage, relevance), the mode
|
|
292
|
+
// that produced them, and the all-term count: the AND result size or, under
|
|
293
|
+
// `any`, the hits that matched every token. The optional prior multiplies
|
|
294
|
+
// scores before ranking; `relevance` is score over the best score shown, and
|
|
295
|
+
// the floors drop results below either value after that. Under `auto` the
|
|
296
|
+
// all-term hits lead in their own order and, when there are fewer than
|
|
297
|
+
// `autoFill` of them, the remaining any-term hits follow in theirs: both
|
|
298
|
+
// passes run over the whole filtered set, so the candidate list, and every
|
|
299
|
+
// page cut from it, does not depend on offset or limit.
|
|
300
|
+
function rankQuery(index, config, analysis, docsByUri) {
|
|
301
|
+
const searchIndex = prepareSearchIndex(index, config);
|
|
302
|
+
const run = (combineWith) =>
|
|
303
|
+
runQuery(searchIndex, analysis.tokens, {
|
|
304
|
+
combineWith,
|
|
305
|
+
filter: (result) => docsByUri.has(String(result.id)),
|
|
306
|
+
document: (id) => docsByUri.get(id),
|
|
307
|
+
});
|
|
308
|
+
const prior = config.freshnessPrior ? (doc) => priorFactor(doc, config.asOf) : () => 1;
|
|
309
|
+
const ranked = (hits) => {
|
|
310
|
+
const list = hits
|
|
311
|
+
.filter((hit) => docsByUri.has(hit.id))
|
|
312
|
+
.map((hit) => {
|
|
313
|
+
const doc = docsByUri.get(hit.id);
|
|
314
|
+
return Object.assign({ doc }, hit, { score: hit.score * prior(doc) });
|
|
315
|
+
});
|
|
316
|
+
list.sort((a, b) => b.score - a.score || compareDocs(a.doc, b.doc));
|
|
317
|
+
return list;
|
|
318
|
+
};
|
|
319
|
+
let results;
|
|
320
|
+
let mode;
|
|
321
|
+
let allTermsTotal;
|
|
322
|
+
// URIs of the all-term hits when the all-term pass ran (the literal guard's group).
|
|
323
|
+
let allTermUris = null;
|
|
324
|
+
if (config.mode === "any") {
|
|
325
|
+
results = ranked(run("OR"));
|
|
326
|
+
mode = "any";
|
|
327
|
+
allTermsTotal = results.filter((hit) => hit.coverage === 1).length;
|
|
328
|
+
} else {
|
|
329
|
+
results = ranked(run("AND"));
|
|
330
|
+
mode = "all";
|
|
331
|
+
allTermsTotal = results.length;
|
|
332
|
+
allTermUris = new Set(results.map((entry) => entry.doc.uri));
|
|
333
|
+
if (config.mode === "auto" && results.length < config.autoFill) {
|
|
334
|
+
const fill = ranked(run("OR")).filter((entry) => !allTermUris.has(entry.doc.uri));
|
|
335
|
+
if (!results.length) {
|
|
336
|
+
results = fill;
|
|
337
|
+
mode = "any";
|
|
338
|
+
allTermUris = null;
|
|
339
|
+
} else if (fill.length) {
|
|
340
|
+
results = results.concat(fill);
|
|
341
|
+
mode = "auto";
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
const top = results.reduce((best, entry) => Math.max(best, entry.score), 0);
|
|
346
|
+
results.forEach((entry) => {
|
|
347
|
+
entry.relevance = top > 0 ? entry.score / top : 0;
|
|
348
|
+
});
|
|
349
|
+
// Lexical evidence (matched terms, coverage, score) for candidates the
|
|
350
|
+
// lexical pass did not return: dense neighbours that match only some terms
|
|
351
|
+
// under an all-term pass. An any-term pass (alone or appended) already
|
|
352
|
+
// holds every match.
|
|
353
|
+
const evidence = (uris) => {
|
|
354
|
+
if (mode !== "all" || !uris.size) return [];
|
|
355
|
+
const subset = new Map(Array.from(uris, (uri) => [uri, docsByUri.get(uri)]));
|
|
356
|
+
return runQuery(searchIndex, analysis.tokens, {
|
|
357
|
+
combineWith: "OR",
|
|
358
|
+
filter: (result) => subset.has(String(result.id)),
|
|
359
|
+
document: (id) => subset.get(id),
|
|
360
|
+
})
|
|
361
|
+
.filter((hit) => subset.has(hit.id))
|
|
362
|
+
.map((hit) => {
|
|
363
|
+
const doc = subset.get(hit.id);
|
|
364
|
+
return Object.assign({ doc }, hit, { score: hit.score * prior(doc) });
|
|
365
|
+
})
|
|
366
|
+
.sort((a, b) => b.score - a.score || compareDocs(a.doc, b.doc));
|
|
367
|
+
};
|
|
368
|
+
return {
|
|
369
|
+
results: applyFloors(results, config),
|
|
370
|
+
mode,
|
|
371
|
+
allTermsTotal,
|
|
372
|
+
allTermUris,
|
|
373
|
+
unfloored: results,
|
|
374
|
+
evidence,
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// The floors keep their lexical meaning under hybrid ranking too: coverage of
|
|
379
|
+
// the query terms and the lexical score relative to the lexical top result.
|
|
380
|
+
function applyFloors(results, config) {
|
|
381
|
+
if (!(config.minCoverage > 0 || config.minRelevance > 0)) {
|
|
382
|
+
return results;
|
|
383
|
+
}
|
|
384
|
+
return results.filter(
|
|
385
|
+
(entry) =>
|
|
386
|
+
entry.coverage >= config.minCoverage &&
|
|
387
|
+
(entry.lexicalRelevance ?? entry.relevance) >= config.minRelevance,
|
|
388
|
+
);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// Hybrid ranking (R-10): the lexical hits, before floors, fused with the dense
|
|
392
|
+
// ranking of the filtered candidates. An explicit mode "all" keeps its
|
|
393
|
+
// meaning, every term required, so dense scores only re-rank the all-term
|
|
394
|
+
// hits; under auto and any the dense neighbours join the candidates.
|
|
395
|
+
//
|
|
396
|
+
// Literal guard: when the all-term pass ran and the query carries literal
|
|
397
|
+
// terms (quoted or identifier-like), the all-term hits keep precedence over
|
|
398
|
+
// every other candidate; fusion orders each group. Exact lookups then never
|
|
399
|
+
// lose a document that contains the identifier to a semantic neighbour.
|
|
400
|
+
function hybridRank(index, config, ranking, filteredDocs, analysis) {
|
|
401
|
+
const lexical = ranking.unfloored;
|
|
402
|
+
const passUris = new Set(lexical.map((entry) => entry.doc.uri));
|
|
403
|
+
const byUri = new Map(lexical.map((entry) => [entry.doc.uri, entry]));
|
|
404
|
+
const candidates = config.mode === "all" ? lexical.map((entry) => entry.doc) : filteredDocs;
|
|
405
|
+
const prior = config.freshnessPrior ? (doc) => priorFactor(doc, config.asOf) : null;
|
|
406
|
+
const docs = new Map(candidates.map((doc) => [doc.uri, doc]));
|
|
407
|
+
const nearest = denseSearch(index, {
|
|
408
|
+
queryVector: config.queryVector,
|
|
409
|
+
cache: config.vectorCache,
|
|
410
|
+
units: config.embedUnits,
|
|
411
|
+
candidates,
|
|
412
|
+
limit: DENSE_DEPTH,
|
|
413
|
+
});
|
|
414
|
+
// The best raw cosine over the candidates, before any prior (abstention, WP3c-2).
|
|
415
|
+
const topSimilarity = nearest.length ? nearest[0].score : null;
|
|
416
|
+
const dense = nearest.map((entry) => {
|
|
417
|
+
// R-6 multiplies scores; a negative cosine is divided so it still drops.
|
|
418
|
+
const factor = prior ? prior(docs.get(entry.id)) : 1;
|
|
419
|
+
const score = entry.score >= 0 ? entry.score * factor : entry.score / factor;
|
|
420
|
+
return { id: entry.id, score, unit: entry.unit };
|
|
421
|
+
});
|
|
422
|
+
const units = new Map(dense.map((entry) => [entry.id, entry.unit]));
|
|
423
|
+
// One weight per query (WP3c-2): fixed, or chosen by the adaptive rule from
|
|
424
|
+
// the lexical pass, which does not depend on offset or limit.
|
|
425
|
+
const choice = resolveAlpha(
|
|
426
|
+
config.alpha,
|
|
427
|
+
{
|
|
428
|
+
literal:
|
|
429
|
+
ranking.allTermUris !== null &&
|
|
430
|
+
ranking.allTermUris.size > 0 &&
|
|
431
|
+
analysis.tokens.some((token) => token.kind !== "plain"),
|
|
432
|
+
allTermsTotal: ranking.allTermsTotal,
|
|
433
|
+
autoFill: config.autoFill,
|
|
434
|
+
topCoverage: lexical.reduce((best, entry) => Math.max(best, entry.coverage || 0), 0),
|
|
435
|
+
},
|
|
436
|
+
config.adaptiveAlpha,
|
|
437
|
+
);
|
|
438
|
+
const alpha = choice.alpha;
|
|
439
|
+
// Partial matches among the dense neighbours join the lexical side after
|
|
440
|
+
// the pass's hits (never at alpha 0, which must reproduce the lexical list).
|
|
441
|
+
const extra =
|
|
442
|
+
alpha > 0
|
|
443
|
+
? ranking.evidence(new Set(dense.map((entry) => entry.id).filter((id) => !passUris.has(id))))
|
|
444
|
+
: [];
|
|
445
|
+
const top = lexical.length ? lexical[0].score : 0;
|
|
446
|
+
extra.forEach((entry) => {
|
|
447
|
+
entry.relevance = top > 0 ? entry.score / top : 0;
|
|
448
|
+
byUri.set(entry.doc.uri, entry);
|
|
449
|
+
});
|
|
450
|
+
const fused = fuseRankings({
|
|
451
|
+
lexical: lexical.concat(extra).map((entry) => ({ id: entry.doc.uri, score: entry.score })),
|
|
452
|
+
dense,
|
|
453
|
+
alpha,
|
|
454
|
+
normalization: config.fusionNormalization,
|
|
455
|
+
});
|
|
456
|
+
// Not at alpha 1, which must reproduce the dense ranking exactly. At alpha
|
|
457
|
+
// 0 the all-term hits lead whatever the terms, as in the lexical ranking
|
|
458
|
+
// (auto appends the any-term results after them), which alpha 0 must
|
|
459
|
+
// reproduce.
|
|
460
|
+
const guard =
|
|
461
|
+
alpha < 1 &&
|
|
462
|
+
ranking.allTermUris !== null &&
|
|
463
|
+
ranking.allTermUris.size > 0 &&
|
|
464
|
+
(alpha === 0 || analysis.tokens.some((token) => token.kind !== "plain"));
|
|
465
|
+
if (guard) {
|
|
466
|
+
const group = (entry) => (ranking.allTermUris.has(entry.id) ? 0 : 1);
|
|
467
|
+
// Array.prototype.sort is stable: fused order holds within each group.
|
|
468
|
+
fused.sort((a, b) => group(a) - group(b));
|
|
469
|
+
}
|
|
470
|
+
const best = fused.length ? Math.max(...fused.map((entry) => entry.fusedScore)) : 0;
|
|
471
|
+
const results = fused.map((entry) => {
|
|
472
|
+
const hit = byUri.get(entry.id);
|
|
473
|
+
const base = hit
|
|
474
|
+
? Object.assign({}, hit, { lexicalRelevance: hit.relevance })
|
|
475
|
+
: { doc: docs.get(entry.id), positions: [], coverage: 0, lexicalRelevance: 0 };
|
|
476
|
+
return Object.assign(base, {
|
|
477
|
+
score: entry.fusedScore,
|
|
478
|
+
relevance: best > 0 ? entry.fusedScore / best : 0,
|
|
479
|
+
lexicalScore: entry.lexicalScore,
|
|
480
|
+
denseScore: entry.denseScore,
|
|
481
|
+
fusedScore: entry.fusedScore,
|
|
482
|
+
unit: units.get(entry.id),
|
|
483
|
+
});
|
|
484
|
+
});
|
|
485
|
+
return {
|
|
486
|
+
results: applyFloors(results, config),
|
|
487
|
+
alpha,
|
|
488
|
+
alphaReason: choice.reason,
|
|
489
|
+
topSimilarity,
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function hybridFields(entry) {
|
|
494
|
+
const fields = {
|
|
495
|
+
lexicalScore: entry.lexicalScore,
|
|
496
|
+
denseScore: entry.denseScore,
|
|
497
|
+
fusedScore: entry.fusedScore,
|
|
498
|
+
};
|
|
499
|
+
// Section units cite the section that matched (`uri#anchor`).
|
|
500
|
+
if (entry.unit && entry.unit !== entry.doc.uri) {
|
|
501
|
+
fields.unit = entry.unit;
|
|
502
|
+
}
|
|
503
|
+
return fields;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
function searchConcepts(index, options) {
|
|
507
|
+
const config = Object.assign({}, options || {}, {
|
|
508
|
+
mode: searchMode(options?.mode),
|
|
509
|
+
autoFill: autoFillOption(options?.autoFill, index),
|
|
510
|
+
minCoverage: floorOption(options?.minCoverage, "minCoverage"),
|
|
511
|
+
minRelevance: floorOption(options?.minRelevance, "minRelevance"),
|
|
512
|
+
});
|
|
513
|
+
const query = String(config.query || "").trim();
|
|
514
|
+
const analysis = analyzeQuery(query, {
|
|
515
|
+
languages: query ? searchLanguages(index, config) : [],
|
|
516
|
+
});
|
|
517
|
+
const limit = Math.max(1, Math.min(Number(config.limit || 25), 250));
|
|
518
|
+
const offset = Math.max(0, Number(config.offset || 0));
|
|
519
|
+
const filteredDocs = applyFilters(index, config);
|
|
520
|
+
let results;
|
|
521
|
+
let ranking = null;
|
|
522
|
+
let hybrid = false;
|
|
523
|
+
let fusion = null;
|
|
524
|
+
if (analysis.query) {
|
|
525
|
+
if (!analysis.tokens.length) {
|
|
526
|
+
// Punctuation-only text: nothing to run, reported under the requested mode.
|
|
527
|
+
results = [];
|
|
528
|
+
ranking = {
|
|
529
|
+
mode: config.mode === "any" ? "any" : "all",
|
|
530
|
+
allTermsTotal: 0,
|
|
531
|
+
allTermUris: null,
|
|
532
|
+
results,
|
|
533
|
+
};
|
|
534
|
+
} else {
|
|
535
|
+
const docsByUri = new Map(filteredDocs.map((doc) => [doc.uri, doc]));
|
|
536
|
+
ranking = rankQuery(index, config, analysis, docsByUri);
|
|
537
|
+
hybrid = config.hybrid === true && Boolean(config.queryVector && config.vectorCache);
|
|
538
|
+
fusion = hybrid ? hybridRank(index, config, ranking, filteredDocs, analysis) : null;
|
|
539
|
+
results = hybrid ? fusion.results : ranking.results;
|
|
540
|
+
}
|
|
541
|
+
} else {
|
|
542
|
+
results = filteredDocs.map((doc) => ({ doc, score: 0 }));
|
|
543
|
+
results.sort((a, b) => compareDocs(a.doc, b.doc));
|
|
544
|
+
}
|
|
545
|
+
const page = results.slice(offset, offset + limit).map((entry) => {
|
|
546
|
+
if (config.detail === "compact") {
|
|
547
|
+
const compact = compactConceptSummary(entry.doc);
|
|
548
|
+
if (ranking) {
|
|
549
|
+
Object.assign(compact, { coverage: entry.coverage, relevance: entry.relevance });
|
|
550
|
+
}
|
|
551
|
+
return hybrid ? Object.assign(compact, hybridFields(entry)) : compact;
|
|
552
|
+
}
|
|
553
|
+
const summary = conceptSummary(entry.doc);
|
|
554
|
+
if (config.asOf) {
|
|
555
|
+
const signals = signalsForDoc(entry.doc, config.asOf);
|
|
556
|
+
summary.signals = Object.assign({}, summary.signals, {
|
|
557
|
+
status: signals.status,
|
|
558
|
+
staleAfter: signals.staleAfter,
|
|
559
|
+
freshness: signals.freshness,
|
|
560
|
+
asOf: signals.asOf,
|
|
561
|
+
trustTier: signals.trustTier,
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
Object.assign(summary, {
|
|
565
|
+
score: entry.score,
|
|
566
|
+
snippet: snippetFor(entry.doc, analysis.tokens),
|
|
567
|
+
});
|
|
568
|
+
if (ranking) {
|
|
569
|
+
Object.assign(summary, {
|
|
570
|
+
matchedTerms: entry.positions.map((position) => analysis.tokens[position].raw),
|
|
571
|
+
coverage: entry.coverage,
|
|
572
|
+
relevance: entry.relevance,
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
return hybrid ? Object.assign(summary, hybridFields(entry)) : summary;
|
|
576
|
+
});
|
|
577
|
+
const response = { total: results.length, limit, offset };
|
|
578
|
+
if (ranking) {
|
|
579
|
+
// lowConfidence reads the displayed top result's coverage (lexical
|
|
580
|
+
// evidence is computed for every candidate, dense ones included) and,
|
|
581
|
+
// when the semantic side ran, the best similarity (WP3c-2).
|
|
582
|
+
const confidence = assessConfidence(
|
|
583
|
+
{
|
|
584
|
+
mode: ranking.mode,
|
|
585
|
+
coverage: results.length ? results[0].coverage : null,
|
|
586
|
+
topSimilarity: hybrid ? fusion.topSimilarity : null,
|
|
587
|
+
},
|
|
588
|
+
config.minSimilarity,
|
|
589
|
+
);
|
|
590
|
+
Object.assign(response, {
|
|
591
|
+
mode: ranking.mode,
|
|
592
|
+
allTermsTotal: ranking.allTermsTotal,
|
|
593
|
+
lowConfidence: confidence.lowConfidence,
|
|
594
|
+
queryTerms: analysis.terms,
|
|
595
|
+
droppedTerms: analysis.dropped,
|
|
596
|
+
confidence,
|
|
597
|
+
});
|
|
598
|
+
if (hybrid) {
|
|
599
|
+
Object.assign(response, {
|
|
600
|
+
hybrid: true,
|
|
601
|
+
alpha: fusion.alpha,
|
|
602
|
+
normalization: fusionNormalization(config.fusionNormalization),
|
|
603
|
+
// WP3c-2: the weight this query used and why (`fixed` unless adaptive).
|
|
604
|
+
alphaUsed: fusion.alpha,
|
|
605
|
+
alphaReason: fusion.alphaReason,
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
response.results = page;
|
|
610
|
+
return response;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
module.exports = {
|
|
614
|
+
DEFAULT_AUTO_FILL,
|
|
615
|
+
FRESHNESS_PRIOR,
|
|
616
|
+
LOW_CONFIDENCE_COVERAGE,
|
|
617
|
+
SEARCH_MODES,
|
|
618
|
+
applyFilters,
|
|
619
|
+
searchConcepts,
|
|
620
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) AXA Group Operations Spain S.A.
|
|
3
|
+
*
|
|
4
|
+
* Permission is hereby granted, free of charge, to any person obtaining
|
|
5
|
+
* a copy of this software and associated documentation files (the
|
|
6
|
+
* "Software"), to deal in the Software without restriction, including
|
|
7
|
+
* without limitation the rights to use, copy, modify, merge, publish,
|
|
8
|
+
* distribute, sublicense, and/or sell copies of the Software, and to
|
|
9
|
+
* permit persons to whom the Software is furnished to do so, subject to
|
|
10
|
+
* the following conditions:
|
|
11
|
+
*
|
|
12
|
+
* The above copyright notice and this permission notice shall be
|
|
13
|
+
* included in all copies or substantial portions of the Software.
|
|
14
|
+
*
|
|
15
|
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
16
|
+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
17
|
+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
18
|
+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
19
|
+
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
20
|
+
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
21
|
+
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
// Provenance: derived from @nlpjs/core 4.26.1, src/among.js (MIT, header
|
|
25
|
+
// above). Unchanged apart from the strict-mode prologue; vendored so the
|
|
26
|
+
// lexical search carries no runtime dependency on @nlpjs (plan B-8, WP1).
|
|
27
|
+
"use strict";
|
|
28
|
+
|
|
29
|
+
// One entry of a Snowball suffix table: the suffix, the index of the entry to
|
|
30
|
+
// try next when this one does not apply, the result code, and an optional
|
|
31
|
+
// guard method with the instance it is called on.
|
|
32
|
+
class Among {
|
|
33
|
+
constructor(s, sub, result, method, instance) {
|
|
34
|
+
this.s_size = s.length;
|
|
35
|
+
this.s = s;
|
|
36
|
+
this.substring_i = sub;
|
|
37
|
+
this.result = result;
|
|
38
|
+
this.method = method;
|
|
39
|
+
this.instance = instance;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
module.exports = Among;
|