@mfdaves/okf-mcp 0.9.0 → 0.12.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +176 -19
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/policies/authoring-safety.md +11 -1
- package/package.json +17 -5
- package/server.json +2 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +391 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +776 -0
- package/src/authoring/live.js +789 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +137 -0
- package/src/authoring/receipts.js +329 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli.js +184 -151
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/registry.js +328 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +472 -0
- package/src/mcp/tools/catalog.js +244 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +213 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +195 -77
- package/src/producers.js +265 -101
- package/src/project.js +277 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,695 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Lexical index over an OKF index (plan R-1, R-2, R-3, R-5): one MiniSearch
|
|
4
|
+
// instance per OKF index and language setting, built lazily and cached in a
|
|
5
|
+
// WeakMap. Languages are detected per bundle (the union of languages present
|
|
6
|
+
// in at least 5% of that bundle's documents) unless configured; each document
|
|
7
|
+
// is stemmed with its own bundle's set and queries with the union of all sets
|
|
8
|
+
// (or the filtered bundle's set). Text fields keep the raw token (diacritics
|
|
9
|
+
// folded) plus one stem per language; the `exact` field (path, identifiers,
|
|
10
|
+
// aliases, raw title tokens) is literal: never stemmed, folded or
|
|
11
|
+
// stop-worded. Query time folds, drops stop words and single letters, and
|
|
12
|
+
// stems every plain term; quoted text and identifier-looking tokens are
|
|
13
|
+
// literal terms that only count when the whole compound is in the exact
|
|
14
|
+
// field or their pieces form an adjacent run in one field's raw tokens,
|
|
15
|
+
// never through a stem. One BM25 tree scores the query; coverage is counted
|
|
16
|
+
// over logical terms after that verification.
|
|
17
|
+
|
|
18
|
+
const MiniSearch = require("minisearch");
|
|
19
|
+
const {
|
|
20
|
+
detectLanguages,
|
|
21
|
+
foldDiacritics,
|
|
22
|
+
normalizeLanguages,
|
|
23
|
+
stemForms,
|
|
24
|
+
stopWordsFor,
|
|
25
|
+
} = require("./languages");
|
|
26
|
+
|
|
27
|
+
const MAX_QUERY_CHARACTERS = 512;
|
|
28
|
+
const MAX_QUERY_TERMS = 16;
|
|
29
|
+
const EXACT_FIELD = "exact";
|
|
30
|
+
const TEXT_FIELDS = Object.freeze([
|
|
31
|
+
"title",
|
|
32
|
+
"type",
|
|
33
|
+
"tags",
|
|
34
|
+
"aliases",
|
|
35
|
+
"description",
|
|
36
|
+
"path",
|
|
37
|
+
"body",
|
|
38
|
+
]);
|
|
39
|
+
const SEARCH_FIELDS = Object.freeze(TEXT_FIELDS.concat([EXACT_FIELD]));
|
|
40
|
+
const SEARCH_BOOSTS = Object.freeze({
|
|
41
|
+
title: 8,
|
|
42
|
+
type: 5,
|
|
43
|
+
tags: 4,
|
|
44
|
+
aliases: 4,
|
|
45
|
+
description: 3,
|
|
46
|
+
path: 2,
|
|
47
|
+
body: 1,
|
|
48
|
+
exact: 6,
|
|
49
|
+
});
|
|
50
|
+
// Body characters sampled per document for language detection.
|
|
51
|
+
const DETECTION_SAMPLE_CHARACTERS = 4000;
|
|
52
|
+
// Distinct tokens whose index forms are memoised per language set.
|
|
53
|
+
const TERM_FORMS_CACHE_LIMIT = 50000;
|
|
54
|
+
// Documents whose raw token streams are kept for literal-term verification.
|
|
55
|
+
const STREAM_CACHE_LIMIT = 2000;
|
|
56
|
+
// OKF index -> { detection, variants: Map<language key, variant> }
|
|
57
|
+
const SEARCH_INDEXES = new WeakMap();
|
|
58
|
+
// MiniSearch instance -> Map<document id, { text, exact }>
|
|
59
|
+
const STREAM_CACHES = new WeakMap();
|
|
60
|
+
// Separates fields in the cached text: a control character is never part of
|
|
61
|
+
// a token and never a separator, so an adjacent run cannot cross it.
|
|
62
|
+
const FIELD_BREAK = "\n\u0001\n";
|
|
63
|
+
const SEPARATOR_CLASS = "[\\n\\r\\p{Z}\\p{P}\\p{S}]";
|
|
64
|
+
// MiniSearch's default tokenizer splits on space and punctuation only; symbols
|
|
65
|
+
// such as Markdown backticks would stay glued to code-formatted terms
|
|
66
|
+
// ("`kill"), so the lexical index splits on symbols as well, everywhere.
|
|
67
|
+
const TOKEN_SEPARATORS = /[\n\r\p{Z}\p{P}\p{S}]+/u;
|
|
68
|
+
const tokenize = (text) => String(text).split(TOKEN_SEPARATORS).filter(Boolean);
|
|
69
|
+
const identity = (value) => value;
|
|
70
|
+
const identityTokenize = (text) => [text];
|
|
71
|
+
// Leaves are pre-processed; MiniSearch must neither tokenize nor expand them.
|
|
72
|
+
const LEAF_OPTIONS = Object.freeze({
|
|
73
|
+
tokenize: identityTokenize,
|
|
74
|
+
processTerm: identity,
|
|
75
|
+
prefix: false,
|
|
76
|
+
fuzzy: false,
|
|
77
|
+
});
|
|
78
|
+
// Query pieces in order: "double quoted", “typographic quoted”, or a run of
|
|
79
|
+
// characters that are neither spaces nor quotes, so punctuation around a
|
|
80
|
+
// quoted group never swallows it.
|
|
81
|
+
const QUERY_PIECES = /"([^"]*)"|“([^”]*)”|([^\s"“”]+)/gu;
|
|
82
|
+
// An unquoted chunk that looks like an identifier: digits or an internal hyphen.
|
|
83
|
+
const IDENTIFIER_CHUNK = /\p{N}|[\p{L}\p{N}]-[\p{L}\p{N}]/u;
|
|
84
|
+
// An indexed chunk kept whole beside its pieces: an internal hyphen, underscore or dot.
|
|
85
|
+
const IDENTIFIER_SEGMENT = /[\p{L}\p{N}][-_.][\p{L}\p{N}]/u;
|
|
86
|
+
const EDGE_PUNCTUATION = /^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu;
|
|
87
|
+
const FILE_EXTENSION = /\.[a-z0-9]+$/;
|
|
88
|
+
|
|
89
|
+
const canonicalCodes = (codes) => Array.from(new Set(codes)).sort();
|
|
90
|
+
const languageKey = (codes) => canonicalCodes(codes).join(",") || "none";
|
|
91
|
+
// Index normalisation of raw tokens: lower-cased, diacritics folded, no stems.
|
|
92
|
+
const rawTokens = (text) =>
|
|
93
|
+
tokenize(String(text || "")).map((token) => foldDiacritics(token.toLowerCase()));
|
|
94
|
+
|
|
95
|
+
// A custom id is a full okf:// URI; only its concept path is an identifier.
|
|
96
|
+
function customIdPath(customId) {
|
|
97
|
+
return customId ? String(customId).replace(/^okf:\/\/[^/]+\//, "") : "";
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// One segment per identifier-bearing value; the exact tokenizer sees them
|
|
101
|
+
// separately so whole titles and aliases become tokens of their own.
|
|
102
|
+
function exactSegments(doc) {
|
|
103
|
+
return [doc.conceptId, customIdPath(doc.customId), doc.path, doc.title]
|
|
104
|
+
.concat(doc.aliases || [])
|
|
105
|
+
.filter(Boolean)
|
|
106
|
+
.map((value) => String(value).replace(/\s+/g, " ").trim())
|
|
107
|
+
.filter(Boolean);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function textFieldValues(doc) {
|
|
111
|
+
return [
|
|
112
|
+
String(doc.title || ""),
|
|
113
|
+
String(doc.type || ""),
|
|
114
|
+
(doc.tags || []).join(" "),
|
|
115
|
+
(doc.aliases || []).join(" "),
|
|
116
|
+
String(doc.description || ""),
|
|
117
|
+
String(doc.path || ""),
|
|
118
|
+
String(doc.body || ""),
|
|
119
|
+
];
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function searchDocument(doc) {
|
|
123
|
+
const [title, type, tags, aliases, description, path, body] = textFieldValues(doc);
|
|
124
|
+
return {
|
|
125
|
+
id: doc.uri,
|
|
126
|
+
title,
|
|
127
|
+
type,
|
|
128
|
+
tags,
|
|
129
|
+
aliases,
|
|
130
|
+
description,
|
|
131
|
+
path,
|
|
132
|
+
body,
|
|
133
|
+
exact: exactSegments(doc).join("\n"),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Exact-field tokens are literal (lower-cased, diacritics kept): each segment
|
|
138
|
+
// as a whole, its default tokens, and every identifier chunk with and without
|
|
139
|
+
// a file extension, so "TRK-W-042" survives as one token beside "trk", "w"
|
|
140
|
+
// and "042".
|
|
141
|
+
function exactTokens(text) {
|
|
142
|
+
const out = new Set();
|
|
143
|
+
String(text)
|
|
144
|
+
.split("\n")
|
|
145
|
+
.forEach((line) => {
|
|
146
|
+
const segment = line.trim().toLowerCase();
|
|
147
|
+
if (!segment) {
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
out.add(segment);
|
|
151
|
+
tokenize(segment).forEach((token) => {
|
|
152
|
+
if (token) {
|
|
153
|
+
out.add(token);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
segment.split(/[\s/]+/).forEach((chunk) => {
|
|
157
|
+
const clean = chunk.replace(EDGE_PUNCTUATION, "");
|
|
158
|
+
if (clean && IDENTIFIER_SEGMENT.test(clean)) {
|
|
159
|
+
out.add(clean);
|
|
160
|
+
const bare = clean.replace(FILE_EXTENSION, "");
|
|
161
|
+
if (bare && bare !== clean) {
|
|
162
|
+
out.add(bare);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
return Array.from(out);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function fieldTokenize(text, field) {
|
|
171
|
+
return field === EXACT_FIELD ? exactTokens(text) : tokenize(text);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Index-time term processing. `current` names the language set of the
|
|
175
|
+
// document being added (set by the variant before each add): the folded token
|
|
176
|
+
// plus its stems for text fields, the literal token for the exact field.
|
|
177
|
+
// Memoised per language set because catalog vocabulary repeats heavily.
|
|
178
|
+
function createTermProcessor(current) {
|
|
179
|
+
const caches = new Map();
|
|
180
|
+
return (term, field) => {
|
|
181
|
+
if (field === EXACT_FIELD) {
|
|
182
|
+
return term;
|
|
183
|
+
}
|
|
184
|
+
const lower = term.toLowerCase();
|
|
185
|
+
if (!current.languages.length) {
|
|
186
|
+
return foldDiacritics(lower);
|
|
187
|
+
}
|
|
188
|
+
let cache = caches.get(current.key);
|
|
189
|
+
if (!cache) {
|
|
190
|
+
cache = new Map();
|
|
191
|
+
caches.set(current.key, cache);
|
|
192
|
+
}
|
|
193
|
+
let forms = cache.get(lower);
|
|
194
|
+
if (forms === undefined) {
|
|
195
|
+
const folded = foldDiacritics(lower);
|
|
196
|
+
const stems = stemForms(lower, current.languages);
|
|
197
|
+
forms = stems.length ? [folded].concat(stems) : folded;
|
|
198
|
+
if (cache.size >= TERM_FORMS_CACHE_LIMIT) {
|
|
199
|
+
cache.delete(cache.keys().next().value);
|
|
200
|
+
}
|
|
201
|
+
cache.set(lower, forms);
|
|
202
|
+
}
|
|
203
|
+
return forms;
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function detectionTokens(doc) {
|
|
208
|
+
const text = [
|
|
209
|
+
doc.title,
|
|
210
|
+
doc.description,
|
|
211
|
+
String(doc.body || "").slice(0, DETECTION_SAMPLE_CHARACTERS),
|
|
212
|
+
]
|
|
213
|
+
.map((value) => String(value || ""))
|
|
214
|
+
.join("\n");
|
|
215
|
+
return tokenize(text).map((token) => token.toLowerCase());
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function indexState(index) {
|
|
219
|
+
let state = SEARCH_INDEXES.get(index);
|
|
220
|
+
if (!state) {
|
|
221
|
+
state = { detection: null, variants: new Map() };
|
|
222
|
+
SEARCH_INDEXES.set(index, state);
|
|
223
|
+
}
|
|
224
|
+
return state;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Bundle-level detection (R-2): each bundle's set from its own documents; the
|
|
228
|
+
// union stems queries.
|
|
229
|
+
function detectIndexLanguages(index, state) {
|
|
230
|
+
if (!state.detection) {
|
|
231
|
+
const tokenLists = new Map();
|
|
232
|
+
index.concepts.forEach((doc) => {
|
|
233
|
+
const bundle = String(doc.bundle);
|
|
234
|
+
if (!tokenLists.has(bundle)) {
|
|
235
|
+
tokenLists.set(bundle, []);
|
|
236
|
+
}
|
|
237
|
+
tokenLists.get(bundle).push(detectionTokens(doc));
|
|
238
|
+
});
|
|
239
|
+
const bundles = {};
|
|
240
|
+
const union = new Set();
|
|
241
|
+
Array.from(tokenLists.keys())
|
|
242
|
+
.sort()
|
|
243
|
+
.forEach((bundle) => {
|
|
244
|
+
bundles[bundle] = detectLanguages(tokenLists.get(bundle));
|
|
245
|
+
bundles[bundle].languages.forEach((code) => {
|
|
246
|
+
union.add(code);
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
state.detection = { bundles, languages: canonicalCodes(Array.from(union)) };
|
|
250
|
+
}
|
|
251
|
+
return state.detection;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// Precedence: explicit option, `index.languages` (CLI --languages),
|
|
255
|
+
// `index.project.languages` (project config), then per-bundle detection.
|
|
256
|
+
function requestedLanguages(index, options) {
|
|
257
|
+
let value;
|
|
258
|
+
if (options && options.languages !== undefined) {
|
|
259
|
+
value = options.languages;
|
|
260
|
+
} else if (index.languages !== undefined) {
|
|
261
|
+
value = index.languages;
|
|
262
|
+
} else if (index.project && index.project.languages !== undefined) {
|
|
263
|
+
value = index.project.languages;
|
|
264
|
+
}
|
|
265
|
+
const normalized = normalizeLanguages(value);
|
|
266
|
+
return normalized === "auto" ? "auto" : canonicalCodes(normalized);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// The effective setting: the union used for queries, a per-bundle lookup for
|
|
270
|
+
// documents, and the cache key (canonical, so ["it","en"] and ["en","it"]
|
|
271
|
+
// share one index).
|
|
272
|
+
function languagePlan(index, options) {
|
|
273
|
+
const requested = requestedLanguages(index, options);
|
|
274
|
+
if (requested !== "auto") {
|
|
275
|
+
return {
|
|
276
|
+
requested,
|
|
277
|
+
languages: requested,
|
|
278
|
+
key: languageKey(requested),
|
|
279
|
+
bundles: null,
|
|
280
|
+
languagesFor: () => requested,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
const detection = detectIndexLanguages(index, indexState(index));
|
|
284
|
+
return {
|
|
285
|
+
requested,
|
|
286
|
+
languages: detection.languages,
|
|
287
|
+
key: "auto",
|
|
288
|
+
bundles: detection.bundles,
|
|
289
|
+
languagesFor: (bundle) =>
|
|
290
|
+
detection.bundles[bundle] ? detection.bundles[bundle].languages : detection.languages,
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// Languages a query is analysed with: the filtered bundle's own set when a
|
|
295
|
+
// `bundle` filter is given and detection ran, otherwise the union.
|
|
296
|
+
function searchLanguages(index, options) {
|
|
297
|
+
const plan = languagePlan(index, options);
|
|
298
|
+
const bundle = options?.bundle;
|
|
299
|
+
if (bundle && plan.bundles?.[String(bundle)]) {
|
|
300
|
+
return canonicalCodes(plan.bundles[String(bundle)].languages);
|
|
301
|
+
}
|
|
302
|
+
return plan.languages;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function buildVariant(index, plan) {
|
|
306
|
+
const current = { languages: [], key: "none" };
|
|
307
|
+
const searchIndex = new MiniSearch({
|
|
308
|
+
fields: SEARCH_FIELDS,
|
|
309
|
+
idField: "id",
|
|
310
|
+
tokenize: fieldTokenize,
|
|
311
|
+
processTerm: createTermProcessor(current),
|
|
312
|
+
searchOptions: {
|
|
313
|
+
boost: SEARCH_BOOSTS,
|
|
314
|
+
combineWith: "AND",
|
|
315
|
+
},
|
|
316
|
+
});
|
|
317
|
+
const upsert = (doc) => {
|
|
318
|
+
current.languages = plan.languagesFor(String(doc.bundle));
|
|
319
|
+
current.key = languageKey(current.languages);
|
|
320
|
+
const document = searchDocument(doc);
|
|
321
|
+
if (searchIndex.has(document.id)) {
|
|
322
|
+
searchIndex.replace(document);
|
|
323
|
+
} else {
|
|
324
|
+
searchIndex.add(document);
|
|
325
|
+
}
|
|
326
|
+
const streams = STREAM_CACHES.get(searchIndex);
|
|
327
|
+
if (streams) {
|
|
328
|
+
streams.delete(document.id);
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
index.concepts.forEach(upsert);
|
|
332
|
+
return { searchIndex, upsert, languages: plan.languages, bundles: plan.bundles };
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function searchVariant(index, options) {
|
|
336
|
+
const state = indexState(index);
|
|
337
|
+
const plan = languagePlan(index, options);
|
|
338
|
+
let variant = state.variants.get(plan.key);
|
|
339
|
+
if (!variant) {
|
|
340
|
+
variant = buildVariant(index, plan);
|
|
341
|
+
state.variants.set(plan.key, variant);
|
|
342
|
+
}
|
|
343
|
+
return variant;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function prepareSearchIndex(index, options) {
|
|
347
|
+
return searchVariant(index, options).searchIndex;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// What a search over `index` would use, without building anything but the
|
|
351
|
+
// detection pass: the requested setting, the effective union, and per-bundle
|
|
352
|
+
// sets with document counts when detection ran.
|
|
353
|
+
function describeSearchIndex(index, options) {
|
|
354
|
+
const plan = languagePlan(index, options);
|
|
355
|
+
return { requested: plan.requested, languages: plan.languages, bundles: plan.bundles };
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const isSingleLetter = (token) => token.length === 1 && !/\d/.test(token);
|
|
359
|
+
|
|
360
|
+
function plainTerm(raw, languages) {
|
|
361
|
+
const folded = foldDiacritics(raw);
|
|
362
|
+
const forms = [folded].concat(stemForms(raw, languages));
|
|
363
|
+
return {
|
|
364
|
+
kind: "plain",
|
|
365
|
+
raw,
|
|
366
|
+
literal: raw,
|
|
367
|
+
forms,
|
|
368
|
+
pieces: [],
|
|
369
|
+
leaves: [raw].concat(forms),
|
|
370
|
+
snippetTerms: [folded],
|
|
371
|
+
stems: forms.slice(1),
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Quoted text and identifier-looking chunks: the lower-cased literal as a
|
|
376
|
+
// whole (accents kept, for the exact field) plus its pieces folded the way
|
|
377
|
+
// the text fields index raw tokens; nothing stemmed or dropped.
|
|
378
|
+
function literalTerm(kind, text) {
|
|
379
|
+
const literal = text.toLowerCase().replace(/\s+/g, " ").trim();
|
|
380
|
+
const pieces = rawTokens(literal);
|
|
381
|
+
return {
|
|
382
|
+
kind,
|
|
383
|
+
raw: literal,
|
|
384
|
+
literal,
|
|
385
|
+
forms: [],
|
|
386
|
+
pieces,
|
|
387
|
+
leaves: [literal].concat(pieces),
|
|
388
|
+
snippetTerms: [literal].concat(pieces),
|
|
389
|
+
stems: [],
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Query analysis (R-3): quoted spans and identifier-looking chunks become
|
|
394
|
+
// literal terms; other chunks are tokenised, lower-cased, and lose stop words
|
|
395
|
+
// of the configured languages and single letters (reported as `dropped`).
|
|
396
|
+
// When nothing would be left, the dropped tokens are searched literally
|
|
397
|
+
// instead of returning nothing; the term limit counts what is left.
|
|
398
|
+
function analyzeQuery(value, options) {
|
|
399
|
+
const config = options || {};
|
|
400
|
+
const query = String(value || "").trim();
|
|
401
|
+
if (Array.from(query).length > MAX_QUERY_CHARACTERS) {
|
|
402
|
+
throw new TypeError(`Search query must not exceed ${MAX_QUERY_CHARACTERS} characters.`);
|
|
403
|
+
}
|
|
404
|
+
const languages = config.languages || [];
|
|
405
|
+
const stopWords = config.stopWords || stopWordsFor(languages);
|
|
406
|
+
const tokens = [];
|
|
407
|
+
const dropped = [];
|
|
408
|
+
const seen = new Set();
|
|
409
|
+
const push = (term) => {
|
|
410
|
+
const key = `${term.kind}:${term.raw}`;
|
|
411
|
+
if (!seen.has(key)) {
|
|
412
|
+
seen.add(key);
|
|
413
|
+
tokens.push(term);
|
|
414
|
+
}
|
|
415
|
+
};
|
|
416
|
+
for (const piece of query.matchAll(QUERY_PIECES)) {
|
|
417
|
+
const quoted = piece[1] !== undefined ? piece[1] : piece[2];
|
|
418
|
+
if (quoted !== undefined) {
|
|
419
|
+
const term = literalTerm("exact", quoted);
|
|
420
|
+
if (term.pieces.length) {
|
|
421
|
+
push(term);
|
|
422
|
+
}
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
const chunk = piece[3].replace(EDGE_PUNCTUATION, "");
|
|
426
|
+
if (!chunk) {
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
if (IDENTIFIER_CHUNK.test(chunk)) {
|
|
430
|
+
push(literalTerm("identifier", chunk));
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
tokenize(chunk)
|
|
434
|
+
.map((token) => token.toLowerCase())
|
|
435
|
+
.filter(Boolean)
|
|
436
|
+
.forEach((token) => {
|
|
437
|
+
if (isSingleLetter(token) || stopWords.has(token)) {
|
|
438
|
+
if (!dropped.includes(token)) {
|
|
439
|
+
dropped.push(token);
|
|
440
|
+
}
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
push(plainTerm(token, languages));
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
if (!tokens.length && dropped.length) {
|
|
447
|
+
dropped.splice(0, dropped.length).forEach((token) => {
|
|
448
|
+
push(plainTerm(token, languages));
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
if (tokens.length > MAX_QUERY_TERMS) {
|
|
452
|
+
throw new TypeError(`Search query must not exceed ${MAX_QUERY_TERMS} terms.`);
|
|
453
|
+
}
|
|
454
|
+
return { query, terms: tokens.map((term) => term.raw), tokens, dropped, languages };
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// 0.9.1 shape ({ query, terms }) plus the analysis fields.
|
|
458
|
+
function normalizeSearchQuery(value, options) {
|
|
459
|
+
return analyzeQuery(value, options);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// One term of the query tree. Plain terms: the literal against the exact
|
|
463
|
+
// field, the folded token and stems against the text fields (one node over
|
|
464
|
+
// every field when the literal needs no folding, so stems alone are kept
|
|
465
|
+
// away from the exact field). Literal terms: the whole compound against the
|
|
466
|
+
// exact field, or every piece somewhere in the document; the run is
|
|
467
|
+
// verified for adjacency afterwards.
|
|
468
|
+
function termNode(term) {
|
|
469
|
+
if (term.kind === "plain") {
|
|
470
|
+
const [folded, ...stems] = term.forms;
|
|
471
|
+
if (term.literal === folded) {
|
|
472
|
+
const queries = [{ combineWith: "OR", fields: SEARCH_FIELDS, queries: [folded] }];
|
|
473
|
+
if (stems.length) {
|
|
474
|
+
queries.push({ combineWith: "OR", fields: TEXT_FIELDS, queries: stems });
|
|
475
|
+
}
|
|
476
|
+
return queries.length === 1 ? queries[0] : { combineWith: "OR", queries };
|
|
477
|
+
}
|
|
478
|
+
return {
|
|
479
|
+
combineWith: "OR",
|
|
480
|
+
queries: [
|
|
481
|
+
{ combineWith: "OR", fields: [EXACT_FIELD], queries: [term.literal] },
|
|
482
|
+
{ combineWith: "OR", fields: TEXT_FIELDS, queries: term.forms },
|
|
483
|
+
],
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
if (term.pieces.length === 1 && term.pieces[0] === term.literal) {
|
|
487
|
+
return { combineWith: "OR", fields: SEARCH_FIELDS, queries: [term.literal] };
|
|
488
|
+
}
|
|
489
|
+
return {
|
|
490
|
+
combineWith: "OR",
|
|
491
|
+
queries: [
|
|
492
|
+
{ combineWith: "OR", fields: [EXACT_FIELD], queries: [term.literal] },
|
|
493
|
+
{ combineWith: "AND", fields: SEARCH_FIELDS, queries: term.pieces },
|
|
494
|
+
],
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
function queryTree(tokens, combineWith) {
|
|
499
|
+
return { combineWith, queries: tokens.map(termNode) };
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// A document's text fields and exact segments in index normalisation
|
|
503
|
+
// (lower-cased, diacritics folded, no stems), joined by a control character
|
|
504
|
+
// no run can cross, plus its literal exact tokens; cached per MiniSearch
|
|
505
|
+
// instance for literal-term verification.
|
|
506
|
+
function documentText(searchIndex, id, doc) {
|
|
507
|
+
let cache = STREAM_CACHES.get(searchIndex);
|
|
508
|
+
if (!cache) {
|
|
509
|
+
cache = new Map();
|
|
510
|
+
STREAM_CACHES.set(searchIndex, cache);
|
|
511
|
+
}
|
|
512
|
+
let entry = cache.get(id);
|
|
513
|
+
if (!entry) {
|
|
514
|
+
const segments = exactSegments(doc);
|
|
515
|
+
entry = {
|
|
516
|
+
text: textFieldValues(doc)
|
|
517
|
+
.concat(segments)
|
|
518
|
+
.map((value) => foldDiacritics(value.toLowerCase()))
|
|
519
|
+
.join(FIELD_BREAK),
|
|
520
|
+
exact: new Set(exactTokens(segments.join("\n"))),
|
|
521
|
+
};
|
|
522
|
+
if (cache.size >= STREAM_CACHE_LIMIT) {
|
|
523
|
+
cache.delete(cache.keys().next().value);
|
|
524
|
+
}
|
|
525
|
+
cache.set(id, entry);
|
|
526
|
+
}
|
|
527
|
+
return entry;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
531
|
+
|
|
532
|
+
// Matches `pieces` as whole tokens, adjacent and in order, within one field.
|
|
533
|
+
function runPattern(pieces) {
|
|
534
|
+
const run = pieces.map(escapeRegExp).join(`${SEPARATOR_CLASS}+`);
|
|
535
|
+
return new RegExp(`(?:^|${SEPARATOR_CLASS})${run}(?=${SEPARATOR_CLASS}|$)`, "u");
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// A literal term counts for a document only when the whole compound is one
|
|
539
|
+
// of its exact tokens or its pieces form an adjacent run in one of its
|
|
540
|
+
// fields; a stem never satisfies it. Documents the tree matched without the
|
|
541
|
+
// literal evidence are skipped before any normalisation.
|
|
542
|
+
function literalTermMatched(searchIndex, term, result, options) {
|
|
543
|
+
const matchedLeaves = result.queryTerms;
|
|
544
|
+
const evidence =
|
|
545
|
+
matchedLeaves.includes(term.literal) ||
|
|
546
|
+
term.pieces.every((piece) => matchedLeaves.includes(piece));
|
|
547
|
+
if (!evidence) {
|
|
548
|
+
return false;
|
|
549
|
+
}
|
|
550
|
+
const doc = options.document ? options.document(String(result.id)) : null;
|
|
551
|
+
if (!doc) {
|
|
552
|
+
return evidence;
|
|
553
|
+
}
|
|
554
|
+
const entry = documentText(searchIndex, String(result.id), doc);
|
|
555
|
+
if (entry.exact.has(term.literal)) {
|
|
556
|
+
return true;
|
|
557
|
+
}
|
|
558
|
+
if (!term.pattern) {
|
|
559
|
+
term.pattern = runPattern(term.pieces);
|
|
560
|
+
}
|
|
561
|
+
return term.pattern.test(entry.text);
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// One MiniSearch run over analysed terms. `combineWith` AND requires every
|
|
565
|
+
// logical term, OR accepts any. Prefix and fuzzy matching stay off (R-12).
|
|
566
|
+
// Plain terms are attributed through their matched leaves, literal terms
|
|
567
|
+
// through verification; each hit reports the raw terms it matched and the
|
|
568
|
+
// share of distinct logical terms matched. `options.document(id)` supplies
|
|
569
|
+
// documents for the verification.
|
|
570
|
+
function runQuery(searchIndex, tokens, options) {
|
|
571
|
+
const config = options || {};
|
|
572
|
+
const combineWith = config.combineWith === "OR" ? "OR" : "AND";
|
|
573
|
+
// Leaf string -> positions of the plain terms it belongs to.
|
|
574
|
+
const leafPositions = new Map();
|
|
575
|
+
const literalPositions = [];
|
|
576
|
+
tokens.forEach((term, position) => {
|
|
577
|
+
if (term.kind !== "plain") {
|
|
578
|
+
literalPositions.push(position);
|
|
579
|
+
return;
|
|
580
|
+
}
|
|
581
|
+
term.leaves.forEach((leaf) => {
|
|
582
|
+
if (!leafPositions.has(leaf)) {
|
|
583
|
+
leafPositions.set(leaf, []);
|
|
584
|
+
}
|
|
585
|
+
leafPositions.get(leaf).push(position);
|
|
586
|
+
});
|
|
587
|
+
});
|
|
588
|
+
const results = searchIndex.search(
|
|
589
|
+
queryTree(tokens, combineWith),
|
|
590
|
+
Object.assign({ filter: config.filter }, LEAF_OPTIONS),
|
|
591
|
+
);
|
|
592
|
+
const hits = [];
|
|
593
|
+
results.forEach((result) => {
|
|
594
|
+
const seen = new Array(tokens.length).fill(false);
|
|
595
|
+
result.queryTerms.forEach((leaf) => {
|
|
596
|
+
const owners = leafPositions.get(leaf);
|
|
597
|
+
if (owners) {
|
|
598
|
+
owners.forEach((position) => {
|
|
599
|
+
seen[position] = true;
|
|
600
|
+
});
|
|
601
|
+
}
|
|
602
|
+
});
|
|
603
|
+
literalPositions.forEach((position) => {
|
|
604
|
+
if (literalTermMatched(searchIndex, tokens[position], result, config)) {
|
|
605
|
+
seen[position] = true;
|
|
606
|
+
}
|
|
607
|
+
});
|
|
608
|
+
const positions = [];
|
|
609
|
+
seen.forEach((flag, position) => {
|
|
610
|
+
if (flag) {
|
|
611
|
+
positions.push(position);
|
|
612
|
+
}
|
|
613
|
+
});
|
|
614
|
+
if (!positions.length || (combineWith === "AND" && positions.length !== tokens.length)) {
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
hits.push({
|
|
618
|
+
id: String(result.id),
|
|
619
|
+
score: result.score,
|
|
620
|
+
positions,
|
|
621
|
+
coverage: positions.length / tokens.length,
|
|
622
|
+
});
|
|
623
|
+
});
|
|
624
|
+
return hits;
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
// Incremental maintenance for a functional index update (WP2): every cached
|
|
628
|
+
// variant of `index` learns the new documents (stemmed with their bundle's
|
|
629
|
+
// set) or forgets the URIs. Detection is not re-run; a bundle unknown at
|
|
630
|
+
// build time takes the union. Both return false when `index` has no lexical
|
|
631
|
+
// index yet, in which case the next search builds one from the documents.
|
|
632
|
+
function addDocuments(index, docs) {
|
|
633
|
+
const state = SEARCH_INDEXES.get(index);
|
|
634
|
+
if (!state) {
|
|
635
|
+
return false;
|
|
636
|
+
}
|
|
637
|
+
state.variants.forEach((variant) => {
|
|
638
|
+
(docs || []).forEach((doc) => {
|
|
639
|
+
variant.upsert(doc);
|
|
640
|
+
});
|
|
641
|
+
});
|
|
642
|
+
return true;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function removeDocuments(index, uris) {
|
|
646
|
+
const state = SEARCH_INDEXES.get(index);
|
|
647
|
+
if (!state) {
|
|
648
|
+
return false;
|
|
649
|
+
}
|
|
650
|
+
state.variants.forEach((variant) => {
|
|
651
|
+
(uris || []).forEach((uri) => {
|
|
652
|
+
if (variant.searchIndex.has(uri)) {
|
|
653
|
+
variant.searchIndex.discard(uri);
|
|
654
|
+
}
|
|
655
|
+
const streams = STREAM_CACHES.get(variant.searchIndex);
|
|
656
|
+
if (streams) {
|
|
657
|
+
streams.delete(uri);
|
|
658
|
+
}
|
|
659
|
+
});
|
|
660
|
+
});
|
|
661
|
+
return true;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// Moves the cached lexical index from one OKF index object to its successor,
|
|
665
|
+
// so an updated index keeps the structure instead of rebuilding it. The
|
|
666
|
+
// previous index must not be searched afterwards.
|
|
667
|
+
function adoptSearchIndex(previous, next) {
|
|
668
|
+
const state = SEARCH_INDEXES.get(previous);
|
|
669
|
+
if (!state || previous === next) {
|
|
670
|
+
return false;
|
|
671
|
+
}
|
|
672
|
+
SEARCH_INDEXES.delete(previous);
|
|
673
|
+
SEARCH_INDEXES.set(next, state);
|
|
674
|
+
return true;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
module.exports = {
|
|
678
|
+
EXACT_FIELD,
|
|
679
|
+
MAX_QUERY_CHARACTERS,
|
|
680
|
+
MAX_QUERY_TERMS,
|
|
681
|
+
SEARCH_BOOSTS,
|
|
682
|
+
SEARCH_FIELDS,
|
|
683
|
+
TEXT_FIELDS,
|
|
684
|
+
addDocuments,
|
|
685
|
+
adoptSearchIndex,
|
|
686
|
+
analyzeQuery,
|
|
687
|
+
describeSearchIndex,
|
|
688
|
+
exactTokens,
|
|
689
|
+
normalizeSearchQuery,
|
|
690
|
+
prepareSearchIndex,
|
|
691
|
+
removeDocuments,
|
|
692
|
+
runQuery,
|
|
693
|
+
searchLanguages,
|
|
694
|
+
searchVariant,
|
|
695
|
+
};
|