@mfdaves/okf-mcp 0.9.1 → 0.13.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +100 -0
  2. package/README.md +452 -26
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/distribution/reference-bundle.md +10 -3
  5. package/okf/okf-mcp/index.md +2 -1
  6. package/okf/okf-mcp/interfaces/http-authoring-api.md +11 -2
  7. package/okf/okf-mcp/interfaces/mcp-http.md +64 -0
  8. package/okf/okf-mcp/interfaces/mcp-tools.md +25 -6
  9. package/okf/okf-mcp/overview/okf-mcp.md +16 -6
  10. package/okf/okf-mcp/policies/authoring-safety.md +28 -5
  11. package/okf/okf-mcp/runtime/indexer.md +14 -1
  12. package/okf/okf-mcp/runtime/mcp-server.md +16 -8
  13. package/okf/okf-mcp/specs/producer-api.md +16 -2
  14. package/package.json +17 -5
  15. package/server.json +38 -2
  16. package/src/assets.js +165 -73
  17. package/src/authoring/candidates.js +434 -0
  18. package/src/authoring/git-preflight.js +225 -0
  19. package/src/authoring/git-publish.js +783 -0
  20. package/src/authoring/live.js +757 -0
  21. package/src/authoring/paths.js +112 -0
  22. package/src/authoring/policy-hook.js +221 -0
  23. package/src/authoring/proposal-policy.js +159 -0
  24. package/src/authoring/receipts.js +373 -0
  25. package/src/authoring/render.js +38 -0
  26. package/src/authoring/shared.js +75 -0
  27. package/src/authoring/staging.js +216 -0
  28. package/src/authoring.js +386 -287
  29. package/src/cache/files.js +147 -0
  30. package/src/cache/lock.js +201 -0
  31. package/src/cache/paths.js +197 -0
  32. package/src/cache/store.js +703 -0
  33. package/src/catalog-text.js +19 -0
  34. package/src/cli/commands.js +448 -0
  35. package/src/cli/http-options.js +193 -0
  36. package/src/cli/options.js +554 -0
  37. package/src/cli/usage.js +96 -0
  38. package/src/cli.js +18 -932
  39. package/src/computation.js +99 -61
  40. package/src/context/assemble.js +567 -0
  41. package/src/context/cli-options.js +88 -0
  42. package/src/context/score.js +234 -0
  43. package/src/context/sections.js +274 -0
  44. package/src/embed/interface.js +160 -0
  45. package/src/embed/openai.js +715 -0
  46. package/src/embed/prepare.js +182 -0
  47. package/src/embed/units.js +96 -0
  48. package/src/error-details.js +53 -0
  49. package/src/git-source.js +112 -52
  50. package/src/graph.js +95 -37
  51. package/src/http-server.js +36 -17
  52. package/src/index.js +15 -0
  53. package/src/indexer/assemble.js +566 -0
  54. package/src/indexer/build.js +290 -0
  55. package/src/indexer/cache.js +81 -0
  56. package/src/indexer/edges.js +430 -0
  57. package/src/indexer/index.js +44 -0
  58. package/src/indexer/paths.js +178 -0
  59. package/src/indexer/resolve.js +282 -0
  60. package/src/indexer/update.js +720 -0
  61. package/src/indexer.js +3 -1068
  62. package/src/live-authoring.js +4 -2329
  63. package/src/markdown.js +21 -16
  64. package/src/mcp/auth.js +409 -0
  65. package/src/mcp/http.js +317 -0
  66. package/src/mcp/principal.js +59 -0
  67. package/src/mcp/registry.js +356 -0
  68. package/src/mcp/resources.js +45 -0
  69. package/src/mcp/server.js +567 -0
  70. package/src/mcp/tools/catalog.js +247 -0
  71. package/src/mcp/tools/computation.js +83 -0
  72. package/src/mcp/tools/context.js +133 -0
  73. package/src/mcp/tools/graph.js +167 -0
  74. package/src/mcp/tools/live-authoring.js +316 -0
  75. package/src/mcp/tools/migration.js +86 -0
  76. package/src/mcp/tools/producers.js +79 -0
  77. package/src/mcp/tools/proposals.js +225 -0
  78. package/src/mcp/tools/provenance.js +87 -0
  79. package/src/mcp/tools/remote.js +79 -0
  80. package/src/mcp/tools/search.js +242 -0
  81. package/src/mcp-server.js +3 -1611
  82. package/src/migration.js +431 -253
  83. package/src/parser.js +245 -130
  84. package/src/plugins.js +71 -48
  85. package/src/policy/write-policy.js +713 -0
  86. package/src/producer-publisher.js +202 -78
  87. package/src/producers.js +311 -103
  88. package/src/project.js +346 -93
  89. package/src/remote.js +96 -61
  90. package/src/search/cli-options.js +107 -0
  91. package/src/search/confidence.js +61 -0
  92. package/src/search/dense.js +304 -0
  93. package/src/search/embedding-cli.js +57 -0
  94. package/src/search/embedding-config.js +217 -0
  95. package/src/search/fusion.js +249 -0
  96. package/src/search/hybrid.js +526 -0
  97. package/src/search/languages.js +200 -0
  98. package/src/search/lexical.js +695 -0
  99. package/src/search/query.js +620 -0
  100. package/src/search/stem/among.js +43 -0
  101. package/src/search/stem/base-stemmer.js +311 -0
  102. package/src/search/stem/en-regions.js +456 -0
  103. package/src/search/stem/en-suffixes.js +664 -0
  104. package/src/search/stem/en-tables.js +176 -0
  105. package/src/search/stem/en.js +238 -0
  106. package/src/search/stem/it-regions.js +459 -0
  107. package/src/search/stem/it-suffixes.js +471 -0
  108. package/src/search/stem/it-tables.js +257 -0
  109. package/src/search/stem/it.js +155 -0
  110. package/src/search-index.js +2 -81
  111. package/src/search.js +2 -252
  112. package/src/store.js +134 -81
  113. package/src/tool-packs.js +346 -0
  114. package/src/v02.js +301 -105
  115. package/src/validation.js +24 -20
@@ -0,0 +1,200 @@
1
+ "use strict";
2
+
3
+ // Language registry for lexical search (plan R-2, R-3): per code a stemmer, a
4
+ // short stop-word list, and the detector that scores a document's tokens
5
+ // against the stop lists. Supported at 1.0: English and Italian; adding a
6
+ // language is a new entry here plus a vendored stemmer under ./stem.
7
+
8
+ const StemmerEn = require("./stem/en");
9
+ const StemmerIt = require("./stem/it");
10
+
11
+ // Share of documents a detected language must reach to enter the bundle's
12
+ // language set under `languages: auto` (R-2).
13
+ const DETECTION_SHARE = 0.05;
14
+
15
+ // Function words only, so that dropping them from a query never removes a
16
+ // content term: articles, prepositions, pronouns, auxiliaries, question
17
+ // words. Deliberately absent: particles that double as content words in
18
+ // catalogs ("up", "down", "out", "over", "only", "new").
19
+ const STOP_WORDS = Object.freeze({
20
+ en: Object.freeze(
21
+ (
22
+ "a about above after again against all am an and any are as at be because been before " +
23
+ "being below between both but by can could did do does doing during each few for from " +
24
+ "further had has have having he her here hers herself him himself his how i if in into is " +
25
+ "it its itself just me more most my myself no nor not of on once or other our ours " +
26
+ "ourselves per she should so some such than that the their theirs them themselves then there " +
27
+ "these they this those through to too under until very was we were what when where which " +
28
+ "while who whom why will with would you your yours yourself yourselves"
29
+ ).split(" "),
30
+ ),
31
+ it: Object.freeze(
32
+ (
33
+ "a ad agli ai al all alla alle allo anche avere abbiamo avete c che chi ci ciò cio come con " +
34
+ "cosa cui da dagli dai dal dall dalla dalle dallo degli dei del dell della delle dello di " +
35
+ "dove e è ed era essere fa fra gli ha hai hanno ho i il in io l la le lei li lo loro lui " +
36
+ "ma me mi mia mie miei mio ne negli nei nel nell nella nelle nello noi non nostra nostre " +
37
+ "nostri nostro o od per perché perche più piu può puo qual quale quali quando quanto " +
38
+ "quella quelle quelli quello questa queste questi questo se sei si sia siamo siete sono " +
39
+ "sta stanno su sua sue sugli sui sul sull sulla sulle sullo suo suoi ti tra tu tua tue tuo " +
40
+ "tuoi un una uno vi voi vorrei voglio vostra vostre vostri vostro"
41
+ ).split(" "),
42
+ ),
43
+ });
44
+
45
+ const LANGUAGES = Object.freeze({
46
+ en: Object.freeze({
47
+ code: "en",
48
+ name: "English",
49
+ stemmer: new StemmerEn(),
50
+ stopWords: new Set(STOP_WORDS.en),
51
+ }),
52
+ it: Object.freeze({
53
+ code: "it",
54
+ name: "Italian",
55
+ stemmer: new StemmerIt(),
56
+ stopWords: new Set(STOP_WORDS.it),
57
+ }),
58
+ });
59
+
60
+ const SUPPORTED_LANGUAGES = Object.freeze(Object.keys(LANGUAGES).sort());
61
+
62
+ // Stop words shared by several languages carry no signal for detection.
63
+ const AMBIGUOUS_STOP_WORDS = new Set(
64
+ SUPPORTED_LANGUAGES.flatMap((code) =>
65
+ STOP_WORDS[code].filter((word) =>
66
+ SUPPORTED_LANGUAGES.some((other) => other !== code && LANGUAGES[other].stopWords.has(word)),
67
+ ),
68
+ ),
69
+ );
70
+
71
+ const NON_ASCII = /[\u0080-￿]/;
72
+ const COMBINING_MARKS = /[̀-ͯ]/g;
73
+
74
+ // Strip diacritics ("città" -> "citta"); ASCII input is returned unchanged.
75
+ function foldDiacritics(term) {
76
+ const text = String(term);
77
+ return NON_ASCII.test(text) ? text.normalize("NFD").replace(COMBINING_MARKS, "") : text;
78
+ }
79
+
80
+ function languageFor(code) {
81
+ const language = LANGUAGES[String(code).toLowerCase()];
82
+ if (!language) {
83
+ throw new TypeError(
84
+ `Unsupported search language: ${code}. Supported: ${SUPPORTED_LANGUAGES.join(", ")}.`,
85
+ );
86
+ }
87
+ return language;
88
+ }
89
+
90
+ // Accepts "auto", "none", a code list ("it,en" or ["it", "en"]) or an empty
91
+ // value (auto). Returns "auto" or a de-duplicated array of supported codes.
92
+ function normalizeLanguages(value) {
93
+ if (value === undefined || value === null || value === "" || value === "auto") {
94
+ return "auto";
95
+ }
96
+ if (value === "none" || value === false) {
97
+ return [];
98
+ }
99
+ const list = Array.isArray(value) ? value : String(value).split(",");
100
+ const codes = [];
101
+ list.forEach((entry) => {
102
+ const code = String(entry).trim().toLowerCase();
103
+ if (!code || code === "none") {
104
+ return;
105
+ }
106
+ if (code === "auto") {
107
+ throw new TypeError('Search languages cannot mix "auto" with language codes.');
108
+ }
109
+ languageFor(code);
110
+ if (!codes.includes(code)) {
111
+ codes.push(code);
112
+ }
113
+ });
114
+ return codes;
115
+ }
116
+
117
+ function stopWordsFor(codes) {
118
+ const words = new Set();
119
+ (codes || []).forEach((code) => {
120
+ languageFor(code).stopWords.forEach((word) => {
121
+ words.add(word);
122
+ });
123
+ });
124
+ return words;
125
+ }
126
+
127
+ function isStopWord(term, codes) {
128
+ return (codes || []).some((code) => languageFor(code).stopWords.has(term));
129
+ }
130
+
131
+ // Folded stems of a lower-cased term under each code, de-duplicated and
132
+ // without the term itself; empty when no language changes the term.
133
+ function stemForms(term, codes) {
134
+ const folded = foldDiacritics(term);
135
+ const forms = [];
136
+ (codes || []).forEach((code) => {
137
+ const stem = foldDiacritics(languageFor(code).stemmer.stemWord(term));
138
+ if (stem && stem !== folded && !forms.includes(stem)) {
139
+ forms.push(stem);
140
+ }
141
+ });
142
+ return forms;
143
+ }
144
+
145
+ // Scores lower-cased tokens against every stop list, ignoring words shared by
146
+ // two lists. Returns the code with strictly the most hits, or null when no
147
+ // language scores or two tie.
148
+ function detectLanguage(tokens) {
149
+ const scores = SUPPORTED_LANGUAGES.map((code) => ({ code, hits: 0 }));
150
+ (tokens || []).forEach((token) => {
151
+ if (AMBIGUOUS_STOP_WORDS.has(token)) {
152
+ return;
153
+ }
154
+ scores.forEach((entry) => {
155
+ if (LANGUAGES[entry.code].stopWords.has(token)) {
156
+ entry.hits += 1;
157
+ }
158
+ });
159
+ });
160
+ scores.sort((left, right) => right.hits - left.hits || left.code.localeCompare(right.code));
161
+ if (!scores[0].hits || (scores[1] && scores[1].hits === scores[0].hits)) {
162
+ return null;
163
+ }
164
+ return scores[0].code;
165
+ }
166
+
167
+ // Bundle-level detection (R-2): detect each document, keep the languages that
168
+ // cover at least DETECTION_SHARE of all documents, dominant language first.
169
+ function detectLanguages(tokenLists) {
170
+ const counts = Object.fromEntries(SUPPORTED_LANGUAGES.map((code) => [code, 0]));
171
+ let documents = 0;
172
+ let detected = 0;
173
+ tokenLists.forEach((tokens) => {
174
+ documents += 1;
175
+ const code = detectLanguage(tokens);
176
+ if (code) {
177
+ counts[code] += 1;
178
+ detected += 1;
179
+ }
180
+ });
181
+ const languages = SUPPORTED_LANGUAGES.filter(
182
+ (code) => documents > 0 && counts[code] / documents >= DETECTION_SHARE,
183
+ ).sort((left, right) => counts[right] - counts[left] || left.localeCompare(right));
184
+ return { languages, counts, documents, detected };
185
+ }
186
+
187
+ module.exports = {
188
+ DETECTION_SHARE,
189
+ LANGUAGES,
190
+ STOP_WORDS,
191
+ SUPPORTED_LANGUAGES,
192
+ detectLanguage,
193
+ detectLanguages,
194
+ foldDiacritics,
195
+ isStopWord,
196
+ languageFor,
197
+ normalizeLanguages,
198
+ stemForms,
199
+ stopWordsFor,
200
+ };