@mfdaves/okf-mcp 0.9.1 → 0.13.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/README.md +452 -26
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/distribution/reference-bundle.md +10 -3
- package/okf/okf-mcp/index.md +2 -1
- package/okf/okf-mcp/interfaces/http-authoring-api.md +11 -2
- package/okf/okf-mcp/interfaces/mcp-http.md +64 -0
- package/okf/okf-mcp/interfaces/mcp-tools.md +25 -6
- package/okf/okf-mcp/overview/okf-mcp.md +16 -6
- package/okf/okf-mcp/policies/authoring-safety.md +28 -5
- package/okf/okf-mcp/runtime/indexer.md +14 -1
- package/okf/okf-mcp/runtime/mcp-server.md +16 -8
- package/okf/okf-mcp/specs/producer-api.md +16 -2
- package/package.json +17 -5
- package/server.json +38 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +434 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +783 -0
- package/src/authoring/live.js +757 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +159 -0
- package/src/authoring/receipts.js +373 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli/commands.js +448 -0
- package/src/cli/http-options.js +193 -0
- package/src/cli/options.js +554 -0
- package/src/cli/usage.js +96 -0
- package/src/cli.js +18 -932
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/error-details.js +53 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/auth.js +409 -0
- package/src/mcp/http.js +317 -0
- package/src/mcp/principal.js +59 -0
- package/src/mcp/registry.js +356 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +567 -0
- package/src/mcp/tools/catalog.js +247 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +225 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +202 -78
- package/src/producers.js +311 -103
- package/src/project.js +346 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/tool-packs.js +346 -0
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Language registry for lexical search (plan R-2, R-3): per code a stemmer, a
|
|
4
|
+
// short stop-word list, and the detector that scores a document's tokens
|
|
5
|
+
// against the stop lists. Supported at 1.0: English and Italian; adding a
|
|
6
|
+
// language is a new entry here plus a vendored stemmer under ./stem.
|
|
7
|
+
|
|
8
|
+
const StemmerEn = require("./stem/en");
|
|
9
|
+
const StemmerIt = require("./stem/it");
|
|
10
|
+
|
|
11
|
+
// Share of documents a detected language must reach to enter the bundle's
|
|
12
|
+
// language set under `languages: auto` (R-2).
|
|
13
|
+
const DETECTION_SHARE = 0.05;
|
|
14
|
+
|
|
15
|
+
// Function words only, so that dropping them from a query never removes a
|
|
16
|
+
// content term: articles, prepositions, pronouns, auxiliaries, question
|
|
17
|
+
// words. Deliberately absent: particles that double as content words in
|
|
18
|
+
// catalogs ("up", "down", "out", "over", "only", "new").
|
|
19
|
+
const STOP_WORDS = Object.freeze({
|
|
20
|
+
en: Object.freeze(
|
|
21
|
+
(
|
|
22
|
+
"a about above after again against all am an and any are as at be because been before " +
|
|
23
|
+
"being below between both but by can could did do does doing during each few for from " +
|
|
24
|
+
"further had has have having he her here hers herself him himself his how i if in into is " +
|
|
25
|
+
"it its itself just me more most my myself no nor not of on once or other our ours " +
|
|
26
|
+
"ourselves per she should so some such than that the their theirs them themselves then there " +
|
|
27
|
+
"these they this those through to too under until very was we were what when where which " +
|
|
28
|
+
"while who whom why will with would you your yours yourself yourselves"
|
|
29
|
+
).split(" "),
|
|
30
|
+
),
|
|
31
|
+
it: Object.freeze(
|
|
32
|
+
(
|
|
33
|
+
"a ad agli ai al all alla alle allo anche avere abbiamo avete c che chi ci ciò cio come con " +
|
|
34
|
+
"cosa cui da dagli dai dal dall dalla dalle dallo degli dei del dell della delle dello di " +
|
|
35
|
+
"dove e è ed era essere fa fra gli ha hai hanno ho i il in io l la le lei li lo loro lui " +
|
|
36
|
+
"ma me mi mia mie miei mio ne negli nei nel nell nella nelle nello noi non nostra nostre " +
|
|
37
|
+
"nostri nostro o od per perché perche più piu può puo qual quale quali quando quanto " +
|
|
38
|
+
"quella quelle quelli quello questa queste questi questo se sei si sia siamo siete sono " +
|
|
39
|
+
"sta stanno su sua sue sugli sui sul sull sulla sulle sullo suo suoi ti tra tu tua tue tuo " +
|
|
40
|
+
"tuoi un una uno vi voi vorrei voglio vostra vostre vostri vostro"
|
|
41
|
+
).split(" "),
|
|
42
|
+
),
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const LANGUAGES = Object.freeze({
|
|
46
|
+
en: Object.freeze({
|
|
47
|
+
code: "en",
|
|
48
|
+
name: "English",
|
|
49
|
+
stemmer: new StemmerEn(),
|
|
50
|
+
stopWords: new Set(STOP_WORDS.en),
|
|
51
|
+
}),
|
|
52
|
+
it: Object.freeze({
|
|
53
|
+
code: "it",
|
|
54
|
+
name: "Italian",
|
|
55
|
+
stemmer: new StemmerIt(),
|
|
56
|
+
stopWords: new Set(STOP_WORDS.it),
|
|
57
|
+
}),
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
const SUPPORTED_LANGUAGES = Object.freeze(Object.keys(LANGUAGES).sort());
|
|
61
|
+
|
|
62
|
+
// Stop words shared by several languages carry no signal for detection.
|
|
63
|
+
const AMBIGUOUS_STOP_WORDS = new Set(
|
|
64
|
+
SUPPORTED_LANGUAGES.flatMap((code) =>
|
|
65
|
+
STOP_WORDS[code].filter((word) =>
|
|
66
|
+
SUPPORTED_LANGUAGES.some((other) => other !== code && LANGUAGES[other].stopWords.has(word)),
|
|
67
|
+
),
|
|
68
|
+
),
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
const NON_ASCII = /[\u0080-]/;
|
|
72
|
+
const COMBINING_MARKS = /[̀-ͯ]/g;
|
|
73
|
+
|
|
74
|
+
// Strip diacritics ("città" -> "citta"); ASCII input is returned unchanged.
|
|
75
|
+
function foldDiacritics(term) {
|
|
76
|
+
const text = String(term);
|
|
77
|
+
return NON_ASCII.test(text) ? text.normalize("NFD").replace(COMBINING_MARKS, "") : text;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function languageFor(code) {
|
|
81
|
+
const language = LANGUAGES[String(code).toLowerCase()];
|
|
82
|
+
if (!language) {
|
|
83
|
+
throw new TypeError(
|
|
84
|
+
`Unsupported search language: ${code}. Supported: ${SUPPORTED_LANGUAGES.join(", ")}.`,
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
return language;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Accepts "auto", "none", a code list ("it,en" or ["it", "en"]) or an empty
|
|
91
|
+
// value (auto). Returns "auto" or a de-duplicated array of supported codes.
|
|
92
|
+
function normalizeLanguages(value) {
|
|
93
|
+
if (value === undefined || value === null || value === "" || value === "auto") {
|
|
94
|
+
return "auto";
|
|
95
|
+
}
|
|
96
|
+
if (value === "none" || value === false) {
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
const list = Array.isArray(value) ? value : String(value).split(",");
|
|
100
|
+
const codes = [];
|
|
101
|
+
list.forEach((entry) => {
|
|
102
|
+
const code = String(entry).trim().toLowerCase();
|
|
103
|
+
if (!code || code === "none") {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (code === "auto") {
|
|
107
|
+
throw new TypeError('Search languages cannot mix "auto" with language codes.');
|
|
108
|
+
}
|
|
109
|
+
languageFor(code);
|
|
110
|
+
if (!codes.includes(code)) {
|
|
111
|
+
codes.push(code);
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
return codes;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function stopWordsFor(codes) {
|
|
118
|
+
const words = new Set();
|
|
119
|
+
(codes || []).forEach((code) => {
|
|
120
|
+
languageFor(code).stopWords.forEach((word) => {
|
|
121
|
+
words.add(word);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
return words;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function isStopWord(term, codes) {
|
|
128
|
+
return (codes || []).some((code) => languageFor(code).stopWords.has(term));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Folded stems of a lower-cased term under each code, de-duplicated and
|
|
132
|
+
// without the term itself; empty when no language changes the term.
|
|
133
|
+
function stemForms(term, codes) {
|
|
134
|
+
const folded = foldDiacritics(term);
|
|
135
|
+
const forms = [];
|
|
136
|
+
(codes || []).forEach((code) => {
|
|
137
|
+
const stem = foldDiacritics(languageFor(code).stemmer.stemWord(term));
|
|
138
|
+
if (stem && stem !== folded && !forms.includes(stem)) {
|
|
139
|
+
forms.push(stem);
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
return forms;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Scores lower-cased tokens against every stop list, ignoring words shared by
|
|
146
|
+
// two lists. Returns the code with strictly the most hits, or null when no
|
|
147
|
+
// language scores or two tie.
|
|
148
|
+
function detectLanguage(tokens) {
|
|
149
|
+
const scores = SUPPORTED_LANGUAGES.map((code) => ({ code, hits: 0 }));
|
|
150
|
+
(tokens || []).forEach((token) => {
|
|
151
|
+
if (AMBIGUOUS_STOP_WORDS.has(token)) {
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
scores.forEach((entry) => {
|
|
155
|
+
if (LANGUAGES[entry.code].stopWords.has(token)) {
|
|
156
|
+
entry.hits += 1;
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
scores.sort((left, right) => right.hits - left.hits || left.code.localeCompare(right.code));
|
|
161
|
+
if (!scores[0].hits || (scores[1] && scores[1].hits === scores[0].hits)) {
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
return scores[0].code;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Bundle-level detection (R-2): detect each document, keep the languages that
|
|
168
|
+
// cover at least DETECTION_SHARE of all documents, dominant language first.
|
|
169
|
+
function detectLanguages(tokenLists) {
|
|
170
|
+
const counts = Object.fromEntries(SUPPORTED_LANGUAGES.map((code) => [code, 0]));
|
|
171
|
+
let documents = 0;
|
|
172
|
+
let detected = 0;
|
|
173
|
+
tokenLists.forEach((tokens) => {
|
|
174
|
+
documents += 1;
|
|
175
|
+
const code = detectLanguage(tokens);
|
|
176
|
+
if (code) {
|
|
177
|
+
counts[code] += 1;
|
|
178
|
+
detected += 1;
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
const languages = SUPPORTED_LANGUAGES.filter(
|
|
182
|
+
(code) => documents > 0 && counts[code] / documents >= DETECTION_SHARE,
|
|
183
|
+
).sort((left, right) => counts[right] - counts[left] || left.localeCompare(right));
|
|
184
|
+
return { languages, counts, documents, detected };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
module.exports = {
|
|
188
|
+
DETECTION_SHARE,
|
|
189
|
+
LANGUAGES,
|
|
190
|
+
STOP_WORDS,
|
|
191
|
+
SUPPORTED_LANGUAGES,
|
|
192
|
+
detectLanguage,
|
|
193
|
+
detectLanguages,
|
|
194
|
+
foldDiacritics,
|
|
195
|
+
isStopWord,
|
|
196
|
+
languageFor,
|
|
197
|
+
normalizeLanguages,
|
|
198
|
+
stemForms,
|
|
199
|
+
stopWordsFor,
|
|
200
|
+
};
|