@mfdaves/okf-mcp 0.9.1 → 0.12.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +172 -17
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/policies/authoring-safety.md +11 -1
- package/package.json +17 -5
- package/server.json +2 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +391 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +776 -0
- package/src/authoring/live.js +789 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +137 -0
- package/src/authoring/receipts.js +329 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli.js +184 -151
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/registry.js +328 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +472 -0
- package/src/mcp/tools/catalog.js +244 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +213 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +195 -77
- package/src/producers.js +265 -101
- package/src/project.js +277 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const fs = require("node:fs");
|
|
4
|
+
const {
|
|
5
|
+
normalizeSlashes,
|
|
6
|
+
parseMarkdownFile,
|
|
7
|
+
parseMarkdownText,
|
|
8
|
+
safeRelativePath,
|
|
9
|
+
} = require("../parser");
|
|
10
|
+
const { DEFAULT_RELATION_TYPES, loadProjectConfig } = require("../project");
|
|
11
|
+
const { fetchRemoteBundles } = require("../remote");
|
|
12
|
+
const { validateIndex } = require("../validation");
|
|
13
|
+
const { applyProject, attachValidation, completeIndex, indexRecord } = require("./assemble");
|
|
14
|
+
const {
|
|
15
|
+
bundleAllowsPath,
|
|
16
|
+
bundleExcludesPath,
|
|
17
|
+
documentAbsolutePath,
|
|
18
|
+
documentOverrideKey,
|
|
19
|
+
fileStat,
|
|
20
|
+
parseBundleArg,
|
|
21
|
+
sanitizeBundleId,
|
|
22
|
+
uniqueBundleIds,
|
|
23
|
+
walkMarkdown,
|
|
24
|
+
} = require("./paths");
|
|
25
|
+
|
|
26
|
+
// The UTC day the parser stamps on documents when no asOf is configured
|
|
27
|
+
// (`signals.asOf`, and `freshness` against `stale_after`). An index built
|
|
28
|
+
// without an asOf floats with the calendar: it records the day it used, and
|
|
29
|
+
// updateIndex rebuilds it when asked for a later day.
|
|
30
|
+
function dayKey(value) {
|
|
31
|
+
const date = value instanceof Date ? value : value === undefined ? new Date() : new Date(value);
|
|
32
|
+
return date.toISOString().slice(0, 10);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function buildConfig(options) {
|
|
36
|
+
const config = options || {};
|
|
37
|
+
const floatingDate = config.asOf === undefined || config.asOf === null || config.asOf === "";
|
|
38
|
+
return {
|
|
39
|
+
options: config,
|
|
40
|
+
documentOverrides:
|
|
41
|
+
config.documentOverrides instanceof Map ? config.documentOverrides : new Map(),
|
|
42
|
+
documentDeletions:
|
|
43
|
+
config.documentDeletions instanceof Set ? config.documentDeletions : new Set(),
|
|
44
|
+
strictLinks: Boolean(config.strictLinks),
|
|
45
|
+
floatingDate,
|
|
46
|
+
asOf: floatingDate ? dayKey() : config.asOf,
|
|
47
|
+
maxAssetBytes: config.maxAssetBytes,
|
|
48
|
+
edgeConfig: {
|
|
49
|
+
allowedRelationTypes: new Set((config.relationTypes || DEFAULT_RELATION_TYPES).map(String)),
|
|
50
|
+
allowCustomRelationTypes: Boolean(config.allowCustomRelationTypes),
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Parses one document into an index entry. A document given as text is
|
|
56
|
+
// parsed from it; otherwise the file is read, after recording its size and
|
|
57
|
+
// mtime so a later refresh can tell whether it changed. A parse failure keeps
|
|
58
|
+
// the entry, as a parse_error, in the bundle's load order.
|
|
59
|
+
function parseEntry(bundle, relativePath, absolutePath, text, config) {
|
|
60
|
+
// `source` records where the entry's bytes came from: "disk" entries carry
|
|
61
|
+
// the file's stat, "text" entries carry the text itself so a later full
|
|
62
|
+
// rebuild can fold them back in as overrides.
|
|
63
|
+
const entry = {
|
|
64
|
+
relativePath,
|
|
65
|
+
absolutePath,
|
|
66
|
+
doc: null,
|
|
67
|
+
parseError: null,
|
|
68
|
+
stat: null,
|
|
69
|
+
source: text === undefined ? "disk" : "text",
|
|
70
|
+
text,
|
|
71
|
+
derived: null,
|
|
72
|
+
};
|
|
73
|
+
try {
|
|
74
|
+
if (text === undefined) {
|
|
75
|
+
entry.stat = fileStat(absolutePath);
|
|
76
|
+
entry.doc = parseMarkdownFile(bundle, absolutePath, { asOf: config.asOf });
|
|
77
|
+
} else {
|
|
78
|
+
entry.doc = parseMarkdownText(bundle, relativePath, text, absolutePath, {
|
|
79
|
+
asOf: config.asOf,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
} catch (error) {
|
|
83
|
+
entry.parseError = {
|
|
84
|
+
code: "parse_error",
|
|
85
|
+
bundle: bundle.id,
|
|
86
|
+
path: relativePath,
|
|
87
|
+
message: error.message,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
return entry;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function missingRootError(bundle) {
|
|
94
|
+
return {
|
|
95
|
+
code: "missing_bundle_root",
|
|
96
|
+
bundle: bundle.id,
|
|
97
|
+
path: bundle.root,
|
|
98
|
+
message: "Bundle root does not exist.",
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function loadLocalBundle(bundle, config) {
|
|
103
|
+
const bundleRecord = { bundle, rootMissing: null, entries: [], remoteWarnings: [] };
|
|
104
|
+
if (!fs.existsSync(bundle.root)) {
|
|
105
|
+
bundleRecord.rootMissing = missingRootError(bundle);
|
|
106
|
+
return bundleRecord;
|
|
107
|
+
}
|
|
108
|
+
const indexedPaths = new Set();
|
|
109
|
+
walkMarkdown(bundle.root).forEach((filePath) => {
|
|
110
|
+
const relativePath = safeRelativePath(bundle.root, filePath) || normalizeSlashes(filePath);
|
|
111
|
+
if (!bundleAllowsPath(bundle, relativePath)) {
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const key = documentOverrideKey(bundle.id, relativePath);
|
|
115
|
+
if (config.documentDeletions.has(key)) {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
indexedPaths.add(relativePath);
|
|
119
|
+
bundleRecord.entries.push(
|
|
120
|
+
parseEntry(
|
|
121
|
+
bundle,
|
|
122
|
+
relativePath,
|
|
123
|
+
filePath,
|
|
124
|
+
config.documentOverrides.has(key) ? String(config.documentOverrides.get(key)) : undefined,
|
|
125
|
+
config,
|
|
126
|
+
),
|
|
127
|
+
);
|
|
128
|
+
});
|
|
129
|
+
const prefix = `${bundle.id}\u0000`;
|
|
130
|
+
config.documentOverrides.forEach((text, key) => {
|
|
131
|
+
if (!String(key).startsWith(prefix)) {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
const relativePath = normalizeSlashes(String(key).slice(prefix.length));
|
|
135
|
+
if (
|
|
136
|
+
config.documentDeletions.has(key) ||
|
|
137
|
+
indexedPaths.has(relativePath) ||
|
|
138
|
+
!bundleAllowsPath(bundle, relativePath)
|
|
139
|
+
) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
bundleRecord.entries.push(
|
|
143
|
+
parseEntry(
|
|
144
|
+
bundle,
|
|
145
|
+
relativePath,
|
|
146
|
+
documentAbsolutePath(bundle, relativePath),
|
|
147
|
+
String(text),
|
|
148
|
+
config,
|
|
149
|
+
),
|
|
150
|
+
);
|
|
151
|
+
});
|
|
152
|
+
return bundleRecord;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function loadRemoteBundle(bundle, config) {
|
|
156
|
+
const bundleRecord = { bundle, rootMissing: null, entries: [], remoteWarnings: [] };
|
|
157
|
+
(bundle.documents || []).forEach((remoteDoc) => {
|
|
158
|
+
const relativePath = normalizeSlashes(remoteDoc.path);
|
|
159
|
+
if (!bundleAllowsPath(bundle, relativePath)) {
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const entry = parseEntry(
|
|
163
|
+
bundle,
|
|
164
|
+
relativePath,
|
|
165
|
+
remoteDoc.source || relativePath,
|
|
166
|
+
String(remoteDoc.text || ""),
|
|
167
|
+
config,
|
|
168
|
+
);
|
|
169
|
+
entry.source = "remote";
|
|
170
|
+
bundleRecord.entries.push(entry);
|
|
171
|
+
});
|
|
172
|
+
const unresolved = bundle.remoteSource?.unresolvedReferences || [];
|
|
173
|
+
bundleRecord.remoteWarnings = unresolved.map((entry) =>
|
|
174
|
+
Object.assign(
|
|
175
|
+
{
|
|
176
|
+
severity: "warning",
|
|
177
|
+
layer: "project",
|
|
178
|
+
bundle: bundle.id,
|
|
179
|
+
message: `Referenced remote asset is unavailable: ${entry.resolvedPath || entry.target || "<unknown>"}`,
|
|
180
|
+
},
|
|
181
|
+
entry,
|
|
182
|
+
),
|
|
183
|
+
);
|
|
184
|
+
return bundleRecord;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function buildIndex(bundleArgs, options) {
|
|
188
|
+
const config = buildConfig(options);
|
|
189
|
+
const requestedBundles = uniqueBundleIds((bundleArgs || []).map(parseBundleArg));
|
|
190
|
+
const seenBundleIds = new Set();
|
|
191
|
+
const bundles = [];
|
|
192
|
+
const duplicateBundleErrors = [];
|
|
193
|
+
requestedBundles.forEach((bundle) => {
|
|
194
|
+
if (seenBundleIds.has(bundle.id)) {
|
|
195
|
+
duplicateBundleErrors.push({
|
|
196
|
+
code: "duplicate_bundle_id",
|
|
197
|
+
bundle: bundle.id,
|
|
198
|
+
message: "Duplicate bundle id; the later bundle was ignored.",
|
|
199
|
+
});
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
seenBundleIds.add(bundle.id);
|
|
203
|
+
bundles.push(bundle);
|
|
204
|
+
});
|
|
205
|
+
const record = {
|
|
206
|
+
bundleArgs: (bundleArgs || []).slice(),
|
|
207
|
+
options: config.options,
|
|
208
|
+
config,
|
|
209
|
+
project: null,
|
|
210
|
+
duplicateBundleErrors,
|
|
211
|
+
bundles: bundles.map((bundle) =>
|
|
212
|
+
bundle.remote ? loadRemoteBundle(bundle, config) : loadLocalBundle(bundle, config),
|
|
213
|
+
),
|
|
214
|
+
registration: null,
|
|
215
|
+
assets: null,
|
|
216
|
+
// Keys of documents removed from the index while their files remain.
|
|
217
|
+
overlayDeletions: new Set(),
|
|
218
|
+
stats: { mode: "build" },
|
|
219
|
+
};
|
|
220
|
+
const owned = new Set();
|
|
221
|
+
record.bundles.forEach((bundleRecord) => {
|
|
222
|
+
bundleRecord.entries.forEach((entry) => {
|
|
223
|
+
owned.add(entry);
|
|
224
|
+
});
|
|
225
|
+
});
|
|
226
|
+
return completeIndex(record, { owned });
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function buildProjectIndex(projectPath) {
|
|
230
|
+
const project = loadProjectConfig(projectPath);
|
|
231
|
+
const index = buildIndex(project.bundles, {
|
|
232
|
+
relationTypes: project.relationTypes,
|
|
233
|
+
strictLinks: project.strictLinks,
|
|
234
|
+
});
|
|
235
|
+
return attachProject(index, project);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
async function loadProjectBundles(projectPath, options) {
|
|
239
|
+
const project = loadProjectConfig(projectPath);
|
|
240
|
+
const remoteBundles = await fetchRemoteBundles(project.remoteBundles, options || {});
|
|
241
|
+
return {
|
|
242
|
+
project,
|
|
243
|
+
bundles: project.bundles.concat(remoteBundles),
|
|
244
|
+
remoteBundles,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function buildProjectIndexAsync(projectPath, options) {
|
|
249
|
+
const loaded = await loadProjectBundles(projectPath, options);
|
|
250
|
+
const index = buildIndex(loaded.bundles, {
|
|
251
|
+
relationTypes: loaded.project.relationTypes,
|
|
252
|
+
strictLinks:
|
|
253
|
+
options && options.strictLinks !== undefined
|
|
254
|
+
? options.strictLinks
|
|
255
|
+
: loaded.project.strictLinks,
|
|
256
|
+
});
|
|
257
|
+
return attachProject(index, loaded.project);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// Attaches project configuration to a built index in place, as before, and
|
|
261
|
+
// remembers the project on the index's record so an incremental update of
|
|
262
|
+
// this index carries it forward.
|
|
263
|
+
function attachProject(index, project) {
|
|
264
|
+
if (!project) {
|
|
265
|
+
return index;
|
|
266
|
+
}
|
|
267
|
+
const record = indexRecord(index);
|
|
268
|
+
if (record) {
|
|
269
|
+
record.project = project;
|
|
270
|
+
}
|
|
271
|
+
applyProject(index, project);
|
|
272
|
+
return attachValidation(index);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
module.exports = {
|
|
276
|
+
attachProject,
|
|
277
|
+
attachValidation,
|
|
278
|
+
buildIndex,
|
|
279
|
+
buildProjectIndex,
|
|
280
|
+
buildProjectIndexAsync,
|
|
281
|
+
bundleAllowsPath,
|
|
282
|
+
bundleExcludesPath,
|
|
283
|
+
dayKey,
|
|
284
|
+
loadProjectBundles,
|
|
285
|
+
missingRootError,
|
|
286
|
+
parseBundleArg,
|
|
287
|
+
parseEntry,
|
|
288
|
+
sanitizeBundleId,
|
|
289
|
+
validateIndex,
|
|
290
|
+
};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { refreshIndex, updateIndex } = require("./update");
|
|
4
|
+
|
|
5
|
+
// Whether two lists name the same additional bundle snapshots: the same
|
|
6
|
+
// objects in the same order. A replacement snapshot under an existing id is
|
|
7
|
+
// another object, so it invalidates an index built over the old one.
|
|
8
|
+
function sameBundles(left, right) {
|
|
9
|
+
const before = left || [];
|
|
10
|
+
const after = right || [];
|
|
11
|
+
return (
|
|
12
|
+
before.length === after.length && before.every((bundle, position) => bundle === after[position])
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// One index a host keeps between the batches it writes: brought up to date
|
|
17
|
+
// from the files before each use, moved along with each operation's result,
|
|
18
|
+
// and tied to the additional bundle snapshots it was built over. `now` is the
|
|
19
|
+
// clock date-dependent signals are judged against (see updateIndex).
|
|
20
|
+
class IndexCache {
|
|
21
|
+
constructor(options) {
|
|
22
|
+
const config = options || {};
|
|
23
|
+
this.now = config.now || (() => new Date());
|
|
24
|
+
this.entry = null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Options for updateIndex and refreshIndex on this cache's index.
|
|
28
|
+
options() {
|
|
29
|
+
return { asOf: this.now() };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
get index() {
|
|
33
|
+
return this.entry ? this.entry.index : null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Whether the cached index came from the host (adopt) rather than a build
|
|
37
|
+
// of the cache's own.
|
|
38
|
+
get hostOwned() {
|
|
39
|
+
return Boolean(this.entry?.hostOwned);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
adopt(index, bundles) {
|
|
43
|
+
this.entry = { bundles: (bundles || []).slice(), index, hostOwned: true };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// The index for these bundles, refreshed from the files, or the result of
|
|
47
|
+
// `build` when no cached index fits.
|
|
48
|
+
current(bundles, build) {
|
|
49
|
+
const entry = this.entry;
|
|
50
|
+
if (entry && sameBundles(entry.bundles, bundles)) {
|
|
51
|
+
const refreshed = refreshIndex(entry.index, this.options());
|
|
52
|
+
this.replace(entry.index, refreshed);
|
|
53
|
+
return refreshed;
|
|
54
|
+
}
|
|
55
|
+
const index = build();
|
|
56
|
+
this.entry = { bundles: (bundles || []).slice(), index, hostOwned: false };
|
|
57
|
+
return index;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Applies changes to an index and, when it is the cached one, keeps the
|
|
61
|
+
// cache on the result.
|
|
62
|
+
update(index, changes) {
|
|
63
|
+
const next = updateIndex(index, changes, this.options());
|
|
64
|
+
this.replace(index, next);
|
|
65
|
+
return next;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
replace(index, next) {
|
|
69
|
+
if (this.entry && this.entry.index === index && next !== index) {
|
|
70
|
+
this.entry = Object.assign({}, this.entry, { index: next });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// For when the files are no longer known to match any index the cache
|
|
75
|
+
// holds (an incomplete rollback, a diverged worktree).
|
|
76
|
+
forget() {
|
|
77
|
+
this.entry = null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = { IndexCache, sameBundles };
|