@mfdaves/okf-mcp 0.9.0 → 0.12.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +176 -19
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/policies/authoring-safety.md +11 -1
  5. package/package.json +17 -5
  6. package/server.json +2 -2
  7. package/src/assets.js +165 -73
  8. package/src/authoring/candidates.js +391 -0
  9. package/src/authoring/git-preflight.js +225 -0
  10. package/src/authoring/git-publish.js +776 -0
  11. package/src/authoring/live.js +789 -0
  12. package/src/authoring/paths.js +112 -0
  13. package/src/authoring/policy-hook.js +221 -0
  14. package/src/authoring/proposal-policy.js +137 -0
  15. package/src/authoring/receipts.js +329 -0
  16. package/src/authoring/render.js +38 -0
  17. package/src/authoring/shared.js +75 -0
  18. package/src/authoring/staging.js +216 -0
  19. package/src/authoring.js +386 -287
  20. package/src/cache/files.js +147 -0
  21. package/src/cache/lock.js +201 -0
  22. package/src/cache/paths.js +197 -0
  23. package/src/cache/store.js +703 -0
  24. package/src/catalog-text.js +19 -0
  25. package/src/cli.js +184 -151
  26. package/src/computation.js +99 -61
  27. package/src/context/assemble.js +567 -0
  28. package/src/context/cli-options.js +88 -0
  29. package/src/context/score.js +234 -0
  30. package/src/context/sections.js +274 -0
  31. package/src/embed/interface.js +160 -0
  32. package/src/embed/openai.js +715 -0
  33. package/src/embed/prepare.js +182 -0
  34. package/src/embed/units.js +96 -0
  35. package/src/git-source.js +112 -52
  36. package/src/graph.js +95 -37
  37. package/src/http-server.js +36 -17
  38. package/src/index.js +15 -0
  39. package/src/indexer/assemble.js +566 -0
  40. package/src/indexer/build.js +290 -0
  41. package/src/indexer/cache.js +81 -0
  42. package/src/indexer/edges.js +430 -0
  43. package/src/indexer/index.js +44 -0
  44. package/src/indexer/paths.js +178 -0
  45. package/src/indexer/resolve.js +282 -0
  46. package/src/indexer/update.js +720 -0
  47. package/src/indexer.js +3 -1068
  48. package/src/live-authoring.js +4 -2329
  49. package/src/markdown.js +21 -16
  50. package/src/mcp/registry.js +328 -0
  51. package/src/mcp/resources.js +45 -0
  52. package/src/mcp/server.js +472 -0
  53. package/src/mcp/tools/catalog.js +244 -0
  54. package/src/mcp/tools/computation.js +83 -0
  55. package/src/mcp/tools/context.js +133 -0
  56. package/src/mcp/tools/graph.js +167 -0
  57. package/src/mcp/tools/live-authoring.js +316 -0
  58. package/src/mcp/tools/migration.js +86 -0
  59. package/src/mcp/tools/producers.js +79 -0
  60. package/src/mcp/tools/proposals.js +213 -0
  61. package/src/mcp/tools/provenance.js +87 -0
  62. package/src/mcp/tools/remote.js +79 -0
  63. package/src/mcp/tools/search.js +242 -0
  64. package/src/mcp-server.js +3 -1611
  65. package/src/migration.js +431 -253
  66. package/src/parser.js +245 -130
  67. package/src/plugins.js +71 -48
  68. package/src/policy/write-policy.js +713 -0
  69. package/src/producer-publisher.js +195 -77
  70. package/src/producers.js +265 -101
  71. package/src/project.js +277 -93
  72. package/src/remote.js +96 -61
  73. package/src/search/cli-options.js +107 -0
  74. package/src/search/confidence.js +61 -0
  75. package/src/search/dense.js +304 -0
  76. package/src/search/embedding-cli.js +57 -0
  77. package/src/search/embedding-config.js +217 -0
  78. package/src/search/fusion.js +249 -0
  79. package/src/search/hybrid.js +526 -0
  80. package/src/search/languages.js +200 -0
  81. package/src/search/lexical.js +695 -0
  82. package/src/search/query.js +620 -0
  83. package/src/search/stem/among.js +43 -0
  84. package/src/search/stem/base-stemmer.js +311 -0
  85. package/src/search/stem/en-regions.js +456 -0
  86. package/src/search/stem/en-suffixes.js +664 -0
  87. package/src/search/stem/en-tables.js +176 -0
  88. package/src/search/stem/en.js +238 -0
  89. package/src/search/stem/it-regions.js +459 -0
  90. package/src/search/stem/it-suffixes.js +471 -0
  91. package/src/search/stem/it-tables.js +257 -0
  92. package/src/search/stem/it.js +155 -0
  93. package/src/search-index.js +2 -81
  94. package/src/search.js +2 -252
  95. package/src/store.js +134 -81
  96. package/src/v02.js +301 -105
  97. package/src/validation.js +24 -20
package/src/indexer.js CHANGED
@@ -1,1070 +1,5 @@
1
1
  "use strict";
2
2
 
3
- const fs = require("fs");
4
- const path = require("path");
5
- const { normalizeSlashes, parseMarkdownFile, parseMarkdownText, safeRelativePath } = require("./parser");
6
- const {
7
- buildAssetRegistry,
8
- collectAssetReferences,
9
- isExternalReference,
10
- isLocalAssetReference,
11
- mimeIsText,
12
- } = require("./assets");
13
- const { DEFAULT_RELATION_TYPES, loadProjectConfig } = require("./project");
14
- const { fetchRemoteBundles } = require("./remote");
15
- const { validateIndex } = require("./validation");
16
-
17
- function sanitizeBundleId(value, fallback) {
18
- const raw = String(value || fallback || "bundle").trim();
19
- return raw.replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "bundle";
20
- }
21
-
22
- function parseBundleArg(arg, index) {
23
- if (arg && typeof arg === "object") {
24
- return {
25
- id: sanitizeBundleId(arg.id, `bundle-${index + 1}`),
26
- root: arg.remote ? "" : path.resolve(arg.root),
27
- remote: Boolean(arg.remote),
28
- remoteSource: arg.remoteSource || null,
29
- documents: Array.isArray(arg.documents) ? arg.documents : [],
30
- assets: Array.isArray(arg.assets) ? arg.assets : [],
31
- include: Array.isArray(arg.include) ? arg.include : [],
32
- exclude: Array.isArray(arg.exclude) ? arg.exclude : [],
33
- };
34
- }
35
- const text = String(arg || "");
36
- const eq = text.indexOf("=");
37
- if (eq > 0) {
38
- return {
39
- id: sanitizeBundleId(text.slice(0, eq), `bundle-${index + 1}`),
40
- root: path.resolve(text.slice(eq + 1)),
41
- };
42
- }
43
- const root = path.resolve(text);
44
- return {
45
- id: sanitizeBundleId(path.basename(root), `bundle-${index + 1}`),
46
- root,
47
- };
48
- }
49
-
50
- function documentOverrideKey(bundleId, relativePath) {
51
- return `${bundleId}\u0000${normalizeSlashes(relativePath)}`;
52
- }
53
-
54
- function uniqueBundleIds(bundles) {
55
- return bundles.map((bundle) => {
56
- const base = sanitizeBundleId(bundle.id);
57
- return {
58
- id: base,
59
- root: bundle.remote ? "" : path.resolve(bundle.root),
60
- remote: Boolean(bundle.remote),
61
- remoteSource: bundle.remoteSource || null,
62
- documents: Array.isArray(bundle.documents) ? bundle.documents : [],
63
- assets: Array.isArray(bundle.assets) ? bundle.assets : [],
64
- include: bundle.include || [],
65
- exclude: bundle.exclude || [],
66
- };
67
- });
68
- }
69
-
70
- function escapeRegex(value) {
71
- return String(value).replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
72
- }
73
-
74
- function patternToRegex(pattern) {
75
- const text = normalizeSlashes(pattern);
76
- let source = "";
77
- for (let index = 0; index < text.length; index += 1) {
78
- const char = text[index];
79
- if (char === "*") {
80
- if (text[index + 1] === "*") {
81
- if (text[index + 2] === "/") {
82
- source += "(?:.*/)?";
83
- index += 2;
84
- } else {
85
- source += ".*";
86
- index += 1;
87
- }
88
- } else {
89
- source += "[^/]*";
90
- }
91
- continue;
92
- }
93
- source += escapeRegex(char);
94
- }
95
- return new RegExp(`^${source}$`);
96
- }
97
-
98
- function matchesPattern(relativePath, pattern) {
99
- const normalizedPath = normalizeSlashes(relativePath);
100
- const normalizedPattern = normalizeSlashes(pattern);
101
- if (!normalizedPattern) {
102
- return false;
103
- }
104
- if (normalizedPath === normalizedPattern) {
105
- return true;
106
- }
107
- if (normalizedPattern.endsWith("/")) {
108
- return normalizedPath.startsWith(normalizedPattern);
109
- }
110
- if (normalizedPattern.includes("*")) {
111
- return patternToRegex(normalizedPattern).test(normalizedPath);
112
- }
113
- return normalizedPath.startsWith(`${normalizedPattern}/`);
114
- }
115
-
116
- function bundleAllowsPath(bundle, relativePath) {
117
- const include = Array.isArray(bundle.include) ? bundle.include : [];
118
- const exclude = Array.isArray(bundle.exclude) ? bundle.exclude : [];
119
- if (include.length && !include.some((pattern) => matchesPattern(relativePath, pattern))) {
120
- return false;
121
- }
122
- if (exclude.some((pattern) => matchesPattern(relativePath, pattern))) {
123
- return false;
124
- }
125
- return true;
126
- }
127
-
128
- function bundleExcludesPath(bundle, relativePath) {
129
- const exclude = Array.isArray(bundle && bundle.exclude) ? bundle.exclude : [];
130
- return exclude.some((pattern) => matchesPattern(relativePath, pattern));
131
- }
132
-
133
- function walkMarkdown(root) {
134
- const out = [];
135
- if (!fs.existsSync(root)) {
136
- return out;
137
- }
138
- const entries = fs.readdirSync(root, { withFileTypes: true });
139
- entries.forEach((entry) => {
140
- const full = path.join(root, entry.name);
141
- if (entry.isDirectory()) {
142
- out.push.apply(out, walkMarkdown(full));
143
- return;
144
- }
145
- if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
146
- out.push(full);
147
- }
148
- });
149
- return out.sort();
150
- }
151
-
152
- function documentAtPath(documentsByPath, pathUri) {
153
- if (!documentsByPath || typeof documentsByPath.get !== "function") {
154
- return null;
155
- }
156
- const document = documentsByPath.get(pathUri);
157
- return document && document.pathUri === pathUri ? document : null;
158
- }
159
-
160
- function registerPortableConceptId(map, ambiguous, key, document) {
161
- const normalized = normalizeSlashes(String(key || "")).replace(/^\/+/, "").split("#")[0];
162
- if (!normalized || ambiguous.has(normalized)) {
163
- return;
164
- }
165
- const owner = map.get(normalized);
166
- if (owner && owner !== document) {
167
- map.delete(normalized);
168
- ambiguous.add(normalized);
169
- return;
170
- }
171
- map.set(normalized, document);
172
- }
173
-
174
- function portableConceptKeys(value) {
175
- const normalized = normalizeSlashes(String(value || ""))
176
- .replace(/^\/+/, "")
177
- .split("#")[0];
178
- if (!normalized) return [];
179
- return normalized.toLowerCase().endsWith(".md")
180
- ? [normalized, normalized.slice(0, -3)]
181
- : [normalized, `${normalized}.md`];
182
- }
183
-
184
- function portableConceptMatches(index, value) {
185
- for (const key of portableConceptKeys(value)) {
186
- const matches = (index.documents || []).filter((doc) => doc.conceptId === key || doc.path === key);
187
- if (matches.length) {
188
- return matches.sort((left, right) => left.uri.localeCompare(right.uri));
189
- }
190
- }
191
- return [];
192
- }
193
-
194
- function resolveConcept(index, locator) {
195
- const raw = String(locator || "").trim();
196
- if (!raw || !index) {
197
- return null;
198
- }
199
- const fragmentless = normalizeSlashes(raw).split("#")[0];
200
- const byUri = index.byUri && (index.byUri.get(raw) || index.byUri.get(fragmentless));
201
- if (byUri) {
202
- return byUri;
203
- }
204
- if ((index.ambiguousAliases || new Set()).has(raw)
205
- || (index.ambiguousAliases || new Set()).has(fragmentless)) {
206
- return null;
207
- }
208
- const uriMatch = fragmentless.match(/^okf:\/\/([^/]+)\/(.+)$/);
209
- if (fragmentless.startsWith("okf://")) {
210
- if (!uriMatch || (index.bundles || []).some((bundle) => bundle.id === uriMatch[1])) {
211
- return null;
212
- }
213
- }
214
- for (const key of portableConceptKeys(uriMatch ? `${uriMatch[1]}/${uriMatch[2]}` : fragmentless)) {
215
- if ((index.ambiguousConceptIds || new Set()).has(key)) return null;
216
- const owner = index.byConceptId && index.byConceptId.get(key);
217
- if (owner) return owner;
218
- }
219
- return null;
220
- }
221
-
222
- function normalizeConceptTitle(value) {
223
- return String(value || "").normalize("NFKC").trim().replace(/\s+/gu, " ").toLowerCase();
224
- }
225
-
226
- function conceptTitleSlug(value) {
227
- return normalizeConceptTitle(value)
228
- .replace(/[^a-z0-9]+/g, "-")
229
- .replace(/^-+|-+$/g, "");
230
- }
231
-
232
- function conceptIsGeneratedFile(doc) {
233
- const frontmatter = doc && doc.frontmatter || {};
234
- return Boolean(doc && (doc.generatedFile === true || doc.isGenerated === true)
235
- || frontmatter.generated_file === true || frontmatter.generatedFile === true);
236
- }
237
-
238
- function conceptMatchSummary(doc, matchReasons) {
239
- const generated = doc && doc.signals && doc.signals.generated;
240
- return {
241
- uri: doc.uri,
242
- bundle: doc.bundle,
243
- path: doc.path,
244
- conceptId: doc.conceptId || doc.path.replace(/\.md$/i, ""),
245
- type: doc.type || doc.frontmatter && doc.frontmatter.type,
246
- title: doc.title,
247
- generatedBy: generated && generated.by || null,
248
- ...(conceptIsGeneratedFile(doc) ? { generatedFile: true } : {}),
249
- ...(matchReasons && matchReasons.length ? { matchReasons } : {}),
250
- };
251
- }
252
-
253
- function findConceptMatches(index, options) {
254
- const config = options || {};
255
- const title = config.title === undefined ? null : normalizeConceptTitle(config.title);
256
- const prefix = normalizeSlashes(String(config.pathPrefix || "")).replace(/^\/+|\/+$/g, "");
257
- const documents = index && Array.isArray(index.concepts)
258
- ? index.concepts
259
- : index && Array.isArray(index.documents) ? index.documents : [];
260
- return documents
261
- .filter((doc) => (
262
- doc && doc.valid !== false && !doc.reserved
263
- && typeof doc.path === "string"
264
- && (!config.bundle || doc.bundle === config.bundle)
265
- && (!config.type || doc.type === config.type)
266
- && (title === null || normalizeConceptTitle(doc.title) === title)
267
- && (!prefix || doc.path === prefix || doc.path.startsWith(`${prefix}/`))
268
- ))
269
- .sort((left, right) => left.path.localeCompare(right.path) || left.uri.localeCompare(right.uri));
270
- }
271
-
272
- function recoveryReceipt(locator, entries, limit, allowRetry, forcedAmbiguity) {
273
- const maximum = Math.max(1, Math.min(Number(limit || 5), 20));
274
- const candidates = entries.slice(0, maximum)
275
- .map((entry) => conceptMatchSummary(entry.doc, entry.reasons));
276
- const actionable = entries.length === 1 && entries[0].doc.valid && !entries[0].doc.reserved;
277
- return {
278
- status: forcedAmbiguity || entries.length > 1 ? "ambiguous"
279
- : actionable ? "replacement_found" : "not_found",
280
- requestedLocator: locator,
281
- candidates,
282
- omitted: Math.max(0, entries.length - candidates.length),
283
- ...(allowRetry && !forcedAmbiguity && actionable
284
- ? { retryWith: { uri: entries[0].doc.uri } } : {}),
285
- };
286
- }
287
-
288
- function recoverConceptLocator(index, locator, options) {
289
- const config = options || {};
290
- const raw = normalizeSlashes(String(locator || "").trim()).split("#")[0];
291
- const uriMatch = raw.match(/^okf:\/\/([^/]+)\/(.+)$/);
292
- if (/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(raw) && !uriMatch) {
293
- return { status: "not_found", requestedLocator: raw, candidates: [], omitted: 0 };
294
- }
295
- const knownBundle = uriMatch && (index.bundles || []).some((entry) => entry.id === uriMatch[1]);
296
- const bundle = knownBundle ? uriMatch[1] : config.bundle || "";
297
- const portablePath = (uriMatch && knownBundle
298
- ? uriMatch[2]
299
- : raw.replace(/^okf:\/\//, ""))
300
- .replace(/^\/+/, "");
301
- const aliasOwners = (index.documents || []).filter((doc) => (
302
- (index.ambiguousAliases || new Set()).has(raw) && (doc.uriAliases || []).includes(raw)
303
- )).sort((left, right) => left.uri.localeCompare(right.uri));
304
- const exact = aliasOwners.length ? aliasOwners : knownBundle ? [] : portableConceptMatches(index, portablePath);
305
- if (exact.length) {
306
- const portableAmbiguous = portableConceptKeys(portablePath)
307
- .some((key) => (index.ambiguousConceptIds || new Set()).has(key));
308
- return recoveryReceipt(raw, exact.map((doc) => ({
309
- doc, reasons: [aliasOwners.length ? "uri_alias" : "portable_id"],
310
- })), config.limit, !aliasOwners.length && !portableAmbiguous,
311
- aliasOwners.length > 0 || portableAmbiguous);
312
- }
313
- const requestedPath = portablePath.replace(/\.md$/i, "");
314
- const requestedBase = path.posix.basename(requestedPath).toLowerCase();
315
- const requestedSlug = conceptTitleSlug(requestedBase);
316
- const ranked = [];
317
- if (!requestedSlug) return recoveryReceipt(raw, [], config.limit, false);
318
- findConceptMatches(index, { bundle }).forEach((doc) => {
319
- const base = path.posix.basename(doc.conceptId || doc.path.replace(/\.md$/i, "")).toLowerCase();
320
- const reasons = [];
321
- if (base === requestedBase) reasons.push("path_basename");
322
- if (conceptTitleSlug(doc.title) === requestedSlug) reasons.push("title_slug");
323
- if ((doc.aliases || []).some((alias) => conceptTitleSlug(alias) === requestedSlug)) reasons.push("alias");
324
- if (!reasons.length) return;
325
- ranked.push({
326
- doc,
327
- reasons,
328
- score: (reasons.includes("path_basename") ? 4 : 0)
329
- + (reasons.includes("title_slug") ? 2 : 0)
330
- + (reasons.includes("alias") ? 1 : 0),
331
- });
332
- });
333
- ranked.sort((left, right) => right.score - left.score
334
- || left.doc.path.localeCompare(right.doc.path)
335
- || left.doc.uri.localeCompare(right.doc.uri));
336
- return recoveryReceipt(raw, ranked, config.limit, true);
337
- }
338
-
339
- function resolveIndexedLink(bundleId, resolved, href, documentsByPath) {
340
- if (!documentsByPath || typeof documentsByPath.get !== "function") {
341
- return resolved;
342
- }
343
- const directTarget = documentAtPath(documentsByPath, resolved.uri);
344
- if (directTarget) {
345
- return Object.assign({}, resolved, { uri: directTarget.uri });
346
- }
347
- const clean = String(href || "").split("#")[0];
348
- if (!clean.endsWith("/") && path.posix.extname(resolved.path)) {
349
- return resolved;
350
- }
351
- const indexPath = normalizeSlashes(path.posix.join(resolved.path, "index.md"));
352
- const indexTarget = documentAtPath(documentsByPath, `okf://${bundleId}/${indexPath}`);
353
- if (!indexTarget) {
354
- return resolved;
355
- }
356
- return Object.assign({}, resolved, {
357
- path: indexPath,
358
- uri: indexTarget.uri,
359
- });
360
- }
361
-
362
- function resolveLinkPath(bundle, fromPath, href, documentsByPath) {
363
- const clean = String(href || "").split("#")[0];
364
- if (!clean || clean.startsWith("http://") || clean.startsWith("https://") || clean.startsWith("mailto:")) {
365
- return null;
366
- }
367
- if (bundle.remote) {
368
- const normalizedPath = clean.startsWith("/")
369
- ? path.posix.normalize(clean.slice(1))
370
- : path.posix.normalize(path.posix.join(path.posix.dirname(fromPath), clean));
371
- const resolvedPath = normalizedPath === "." ? "" : normalizedPath.replace(/\/+$/g, "");
372
- if (resolvedPath.startsWith("../") || resolvedPath === "..") {
373
- return {
374
- outsideRoot: true,
375
- path: normalizeSlashes(clean),
376
- uri: null,
377
- };
378
- }
379
- return resolveIndexedLink(bundle.id, {
380
- outsideRoot: false,
381
- path: resolvedPath,
382
- uri: `okf://${bundle.id}/${resolvedPath}`,
383
- }, clean, documentsByPath);
384
- }
385
- const base = clean.startsWith("/")
386
- ? path.join(bundle.root, clean.slice(1))
387
- : path.resolve(path.dirname(path.join(bundle.root, fromPath)), clean);
388
- const relativePath = safeRelativePath(bundle.root, base);
389
- if (relativePath === null) {
390
- return {
391
- outsideRoot: true,
392
- path: normalizeSlashes(clean),
393
- uri: null,
394
- };
395
- }
396
- return resolveIndexedLink(bundle.id, {
397
- outsideRoot: false,
398
- path: relativePath,
399
- uri: `okf://${bundle.id}/${relativePath}`,
400
- }, clean, documentsByPath);
401
- }
402
-
403
- function relationFrom(value, doc) {
404
- if (typeof value === "string") {
405
- const parts = value.trim().split(/\s+/);
406
- if (parts.length === 1) {
407
- return { type: "related_to", target: parts[0], source: doc.uri };
408
- }
409
- return { type: parts[0], target: parts.slice(1).join(" "), source: doc.uri };
410
- }
411
- if (value && typeof value === "object") {
412
- return {
413
- type: value.type || value.kind || "related_to",
414
- target: value.target || value.to || "",
415
- source: doc.uri,
416
- label: value.label || value.text || "",
417
- description: value.description || "",
418
- };
419
- }
420
- return { type: "related_to", target: "", source: doc.uri };
421
- }
422
-
423
- function assetUri(bundle, relativePath) {
424
- return `okf-asset://${bundle}/${normalizeSlashes(relativePath).split("/").map(encodeURIComponent).join("/")}`;
425
- }
426
-
427
- function relativeReferencePath(fromPath, reference) {
428
- const raw = normalizeSlashes(String(reference || "").trim()).split("#")[0];
429
- if (!raw || isExternalReference(raw)) {
430
- return null;
431
- }
432
- const relative = raw.startsWith("/")
433
- ? path.posix.normalize(raw.slice(1))
434
- : path.posix.normalize(path.posix.join(path.posix.dirname(fromPath), raw));
435
- if (!relative || relative === "." || relative === ".." || relative.startsWith("../") || path.posix.isAbsolute(relative)) {
436
- return null;
437
- }
438
- return normalizeSlashes(relative);
439
- }
440
-
441
- function semanticReferences(doc) {
442
- const frontmatter = doc.frontmatter || {};
443
- const references = [];
444
- function add(kind, field, value, metadata) {
445
- if (typeof value === "string" && value.trim()) {
446
- references.push({ kind, field, value: value.trim(), metadata: metadata || null });
447
- }
448
- }
449
- add("resource", "resource", frontmatter.resource);
450
- const sources = doc.signals && Array.isArray(doc.signals.sources)
451
- ? doc.signals.sources
452
- : Array.isArray(frontmatter.sources)
453
- ? frontmatter.sources
454
- : [];
455
- sources.forEach((source, index) => {
456
- if (source && typeof source === "object" && !Array.isArray(source)) {
457
- add("source", `sources[${index}].resource`, source.resource, source);
458
- }
459
- });
460
- add("computation", "computation", frontmatter.computation);
461
- if (frontmatter.executor && typeof frontmatter.executor === "object" && !Array.isArray(frontmatter.executor)) {
462
- add("executor", "executor.resource", frontmatter.executor.resource);
463
- }
464
- if (frontmatter.attester && typeof frontmatter.attester === "object" && !Array.isArray(frontmatter.attester)) {
465
- add("attester", "attester.resource", frontmatter.attester.resource);
466
- }
467
- return references;
468
- }
469
-
470
- function buildIndex(bundleArgs, options) {
471
- const config = options || {};
472
- const documentOverrides = config.documentOverrides instanceof Map
473
- ? config.documentOverrides
474
- : new Map();
475
- const documentDeletions = config.documentDeletions instanceof Set
476
- ? config.documentDeletions
477
- : new Set();
478
- const strictLinks = Boolean(config.strictLinks);
479
- const requestedBundles = uniqueBundleIds((bundleArgs || []).map(parseBundleArg));
480
- const allowedRelationTypes = new Set((config.relationTypes || DEFAULT_RELATION_TYPES).map(String));
481
- const allowCustomRelationTypes = Boolean(config.allowCustomRelationTypes);
482
- const documents = [];
483
- const concepts = [];
484
- const reserved = [];
485
- const warnings = [];
486
- const errors = [];
487
- const externalReferences = new Map();
488
- const seenBundleIds = new Set();
489
- const bundles = [];
490
-
491
- requestedBundles.forEach((bundle) => {
492
- if (seenBundleIds.has(bundle.id)) {
493
- errors.push({ code: "duplicate_bundle_id", bundle: bundle.id, message: "Duplicate bundle id; the later bundle was ignored." });
494
- return;
495
- }
496
- seenBundleIds.add(bundle.id);
497
- bundles.push(bundle);
498
- });
499
-
500
- bundles.forEach((bundle) => {
501
- if (bundle.remote) {
502
- (bundle.documents || []).forEach((remoteDoc) => {
503
- const relativePath = normalizeSlashes(remoteDoc.path);
504
- if (!bundleAllowsPath(bundle, relativePath)) {
505
- return;
506
- }
507
- try {
508
- const doc = parseMarkdownText(
509
- bundle,
510
- relativePath,
511
- String(remoteDoc.text || ""),
512
- remoteDoc.source || relativePath,
513
- { asOf: config.asOf },
514
- );
515
- documents.push(doc);
516
- if (doc.reserved) {
517
- reserved.push(doc);
518
- } else if (doc.valid) {
519
- concepts.push(doc);
520
- }
521
- warnings.push.apply(warnings, doc.warnings.map((warning) => Object.assign({ bundle: doc.bundle }, warning)));
522
- } catch (error) {
523
- errors.push({
524
- code: "parse_error",
525
- bundle: bundle.id,
526
- path: relativePath,
527
- message: error.message,
528
- });
529
- }
530
- });
531
- return;
532
- }
533
- if (!fs.existsSync(bundle.root)) {
534
- errors.push({ code: "missing_bundle_root", bundle: bundle.id, path: bundle.root, message: "Bundle root does not exist." });
535
- return;
536
- }
537
- const indexedPaths = new Set();
538
- const indexLocalDocument = (relativePath, text, source) => {
539
- indexedPaths.add(relativePath);
540
- try {
541
- const doc = text === undefined
542
- ? parseMarkdownFile(bundle, source, { asOf: config.asOf })
543
- : parseMarkdownText(bundle, relativePath, text, source, { asOf: config.asOf });
544
- documents.push(doc);
545
- if (doc.reserved) {
546
- reserved.push(doc);
547
- } else if (doc.valid) {
548
- concepts.push(doc);
549
- }
550
- warnings.push.apply(warnings, doc.warnings.map((warning) => Object.assign({ bundle: doc.bundle }, warning)));
551
- } catch (error) {
552
- errors.push({
553
- code: "parse_error",
554
- bundle: bundle.id,
555
- path: relativePath,
556
- message: error.message,
557
- });
558
- }
559
- };
560
- walkMarkdown(bundle.root).forEach((filePath) => {
561
- const relativePath = safeRelativePath(bundle.root, filePath) || normalizeSlashes(filePath);
562
- if (!bundleAllowsPath(bundle, relativePath)) {
563
- return;
564
- }
565
- const key = documentOverrideKey(bundle.id, relativePath);
566
- if (documentDeletions.has(key)) {
567
- return;
568
- }
569
- indexLocalDocument(
570
- relativePath,
571
- documentOverrides.has(key) ? String(documentOverrides.get(key)) : undefined,
572
- filePath,
573
- );
574
- });
575
- const prefix = `${bundle.id}\u0000`;
576
- documentOverrides.forEach((text, key) => {
577
- if (!String(key).startsWith(prefix)) {
578
- return;
579
- }
580
- const relativePath = normalizeSlashes(String(key).slice(prefix.length));
581
- if (documentDeletions.has(key) || indexedPaths.has(relativePath) || !bundleAllowsPath(bundle, relativePath)) {
582
- return;
583
- }
584
- indexLocalDocument(
585
- relativePath,
586
- String(text),
587
- path.join(bundle.root, ...relativePath.split("/")),
588
- );
589
- });
590
- });
591
-
592
- const byUri = new Map();
593
- const byPathUri = new Map();
594
- const byCanonicalUri = new Map();
595
- const byConceptId = new Map();
596
- const ambiguousConceptIds = new Set();
597
- const ambiguousAliases = new Set();
598
- documents.forEach((doc) => {
599
- if (!byPathUri.has(doc.pathUri)) {
600
- byPathUri.set(doc.pathUri, doc);
601
- }
602
- if (byCanonicalUri.has(doc.uri)) {
603
- errors.push({ code: "duplicate_concept_id", uri: doc.uri, message: "Duplicate canonical OKF Concept ID." });
604
- return;
605
- }
606
- byCanonicalUri.set(doc.uri, doc);
607
- byUri.set(doc.uri, doc);
608
- registerPortableConceptId(byConceptId, ambiguousConceptIds, doc.conceptId, doc);
609
- registerPortableConceptId(byConceptId, ambiguousConceptIds, doc.path, doc);
610
- });
611
-
612
- const localDocuments = documents.filter((doc) => {
613
- const bundle = bundles.find((entry) => entry.id === doc.bundle);
614
- return bundle && !bundle.remote;
615
- });
616
- const localAssetRegistry = buildAssetRegistry(localDocuments, bundles, {
617
- maxAssetBytes: config.maxAssetBytes,
618
- allowResolvedPath: (bundle, relativePath) => !bundleExcludesPath(bundle, relativePath),
619
- skipResolvedPath: (_bundle, relativePath) => relativePath.toLowerCase().endsWith(".md"),
620
- });
621
- warnings.push.apply(warnings, localAssetRegistry.diagnostics);
622
- const assets = localAssetRegistry.assets.filter((asset) => (
623
- !documentAtPath(byPathUri, `okf://${asset.bundle}/${asset.path}`)
624
- ));
625
- bundles.filter((bundle) => bundle.remote).forEach((bundle) => {
626
- const unresolved = bundle.remoteSource && bundle.remoteSource.unresolvedReferences || [];
627
- warnings.push.apply(warnings, unresolved.map((entry) => Object.assign({
628
- severity: "warning",
629
- layer: "project",
630
- bundle: bundle.id,
631
- message: `Referenced remote asset is unavailable: ${entry.resolvedPath || entry.target || "<unknown>"}`,
632
- }, entry)));
633
- (bundle.assets || []).forEach((asset) => {
634
- assets.push(Object.assign({}, asset, {
635
- bundle: bundle.id,
636
- uri: asset.uri || assetUri(bundle.id, asset.path),
637
- remote: true,
638
- }));
639
- });
640
- });
641
-
642
- assets.forEach((asset) => {
643
- asset.uri = asset.uri || assetUri(asset.bundle, asset.path);
644
- });
645
- const byAssetKey = new Map(assets.map((asset) => [`${asset.bundle}\u0000${asset.path}`, asset]));
646
- const byAssetUri = new Map(assets.map((asset) => [asset.uri, asset]));
647
- const aliasOwners = new Map();
648
- documents.forEach((doc) => {
649
- (doc.uriAliases || [doc.pathUri]).forEach((alias) => {
650
- const canonicalOwner = byCanonicalUri.get(alias);
651
- if (canonicalOwner && canonicalOwner !== doc) {
652
- errors.push({
653
- code: "uri_alias_conflicts_canonical",
654
- uri: alias,
655
- bundle: doc.bundle,
656
- path: doc.path,
657
- message: "Compatibility URI alias conflicts with another canonical Concept ID; the canonical target wins.",
658
- });
659
- return;
660
- }
661
- const aliasOwner = aliasOwners.get(alias);
662
- if (aliasOwner && aliasOwner !== doc) {
663
- ambiguousAliases.add(alias);
664
- byUri.delete(alias);
665
- errors.push({
666
- code: "ambiguous_uri_alias",
667
- uri: alias,
668
- bundle: doc.bundle,
669
- path: doc.path,
670
- message: "Compatibility URI alias resolves to more than one document.",
671
- });
672
- return;
673
- }
674
- if (!ambiguousAliases.has(alias)) {
675
- aliasOwners.set(alias, doc);
676
- byUri.set(alias, doc);
677
- }
678
- });
679
- });
680
-
681
- const edges = [];
682
- documents.forEach((doc) => {
683
- const bundle = bundles.find((entry) => entry.id === doc.bundle);
684
- doc.links.forEach((link) => {
685
- const resolved = resolveLinkPath(bundle, doc.path, link.href, byPathUri);
686
- if (!resolved) {
687
- return;
688
- }
689
- if (resolved.outsideRoot) {
690
- warnings.push({
691
- code: "link_outside_root",
692
- bundle: doc.bundle,
693
- path: doc.path,
694
- href: link.href,
695
- message: "Markdown link resolves outside bundle root.",
696
- });
697
- return;
698
- }
699
- const target = documentAtPath(byPathUri, `okf://${doc.bundle}/${resolved.path}`);
700
- const edge = {
701
- source: doc.uri,
702
- target: target ? target.uri : resolved.uri,
703
- kind: "markdown_link",
704
- text: link.text || "",
705
- href: link.href,
706
- broken: !target,
707
- };
708
- edges.push(edge);
709
- if (!target) {
710
- warnings.push({
711
- code: "broken_link",
712
- bundle: doc.bundle,
713
- path: doc.path,
714
- href: link.href,
715
- target: resolved.path,
716
- message: "Markdown link target does not exist in bundle.",
717
- });
718
- }
719
- });
720
- doc.relations.map((relation) => relationFrom(relation, doc)).forEach((relation) => {
721
- const type = String(relation.type || "related_to");
722
- const targetUri = String(relation.target || "").trim();
723
- if (!targetUri) {
724
- errors.push({ code: "missing_relation_target", bundle: doc.bundle, path: doc.path, relationType: type, message: "Relation has no target." });
725
- return;
726
- }
727
- const safeCustomType = /^[A-Za-z0-9_.-]+$/.test(type);
728
- if (!allowedRelationTypes.has(type) && !(allowCustomRelationTypes && safeCustomType)) {
729
- errors.push({ code: "invalid_relation_type", bundle: doc.bundle, path: doc.path, relationType: type, message: `Unsupported relation type: ${type}` });
730
- } else if (allowCustomRelationTypes && safeCustomType) {
731
- allowedRelationTypes.add(type);
732
- }
733
- let targetDocument = byUri.get(targetUri) || null;
734
- let internalTarget = targetUri.startsWith("okf://");
735
- if (!targetDocument && !/^[A-Za-z][A-Za-z0-9+.-]*:/u.test(targetUri)) {
736
- internalTarget = true;
737
- const resolved = resolveLinkPath(bundle, doc.path, targetUri, byPathUri);
738
- if (resolved && !resolved.outsideRoot) {
739
- targetDocument = documentAtPath(byPathUri, `okf://${doc.bundle}/${resolved.path}`);
740
- if (!targetDocument && !path.posix.extname(resolved.path)) {
741
- targetDocument = documentAtPath(byPathUri, `okf://${doc.bundle}/${resolved.path}.md`);
742
- }
743
- }
744
- if (!targetDocument) {
745
- const normalizedTarget = normalizeSlashes(targetUri).replace(/^\/+/, "").replace(/\.md$/i, "");
746
- const portableTarget = byConceptId.get(normalizedTarget);
747
- if (portableTarget && portableTarget.bundle === doc.bundle) {
748
- targetDocument = portableTarget;
749
- }
750
- }
751
- }
752
- const target = targetDocument && !targetDocument.reserved && targetDocument.valid
753
- ? targetDocument
754
- : null;
755
- const isExternal = !internalTarget;
756
- if (isExternal) {
757
- externalReferences.set(targetUri, { uri: targetUri, kind: "external" });
758
- }
759
- if (internalTarget && !target) {
760
- errors.push({ code: "broken_relation", bundle: doc.bundle, path: doc.path, target: targetUri, relationType: type, message: "Relation target does not resolve to a valid concept." });
761
- }
762
- edges.push({
763
- source: doc.uri,
764
- target: target ? target.uri : targetUri,
765
- kind: "relation",
766
- relationType: type,
767
- text: relation.label || type,
768
- description: relation.description || "",
769
- broken: internalTarget && !target,
770
- external: isExternal,
771
- });
772
- });
773
- semanticReferences(doc).forEach((reference) => {
774
- const rawTarget = reference.value;
775
- let target = null;
776
- let external = false;
777
- let broken = false;
778
- let resolvedAs = "opaque";
779
- if (rawTarget.startsWith("okf://")) {
780
- const targetDoc = byUri.get(rawTarget);
781
- target = targetDoc ? targetDoc.uri : rawTarget;
782
- const validTarget = Boolean(targetDoc && !targetDoc.reserved && targetDoc.valid);
783
- broken = !validTarget;
784
- resolvedAs = validTarget ? "concept" : targetDoc ? "invalid_concept" : "unresolved";
785
- if (!validTarget) {
786
- warnings.push({
787
- code: "broken_semantic_reference",
788
- severity: "warning",
789
- layer: "project",
790
- bundle: doc.bundle,
791
- path: doc.path,
792
- field: reference.field,
793
- target: rawTarget,
794
- edgeKind: reference.kind,
795
- message: targetDoc
796
- ? "Standard OKF semantic reference targets an invalid or reserved document."
797
- : "Standard OKF semantic reference targets an unknown internal concept.",
798
- });
799
- }
800
- } else if (isExternalReference(rawTarget)) {
801
- target = rawTarget;
802
- external = true;
803
- resolvedAs = "external";
804
- } else {
805
- const targetPath = relativeReferencePath(doc.path, rawTarget);
806
- const targetDoc = targetPath
807
- ? documentAtPath(byPathUri, `okf://${doc.bundle}/${targetPath}`)
808
- || byUri.get(`okf://${doc.bundle}/${targetPath.replace(/\.md$/i, "")}`)
809
- : null;
810
- const targetAsset = targetPath ? byAssetKey.get(`${doc.bundle}\u0000${targetPath}`) : null;
811
- if (targetDoc && !targetDoc.reserved && targetDoc.valid) {
812
- target = targetDoc.uri;
813
- resolvedAs = "concept";
814
- } else if (targetDoc) {
815
- target = targetDoc.uri;
816
- broken = true;
817
- resolvedAs = "invalid_concept";
818
- } else if (targetAsset) {
819
- target = targetAsset.uri;
820
- resolvedAs = "asset";
821
- } else {
822
- target = rawTarget;
823
- const opaqueSource = reference.kind === "source" && !isLocalAssetReference(rawTarget, { allowBare: false });
824
- external = opaqueSource;
825
- broken = !opaqueSource;
826
- resolvedAs = opaqueSource ? "opaque" : "unresolved";
827
- }
828
- }
829
- if (broken && !rawTarget.startsWith("okf://")) {
830
- warnings.push({
831
- code: "broken_semantic_reference",
832
- severity: "warning",
833
- layer: "project",
834
- bundle: doc.bundle,
835
- path: doc.path,
836
- field: reference.field,
837
- target: rawTarget,
838
- edgeKind: reference.kind,
839
- message: "Standard OKF semantic reference does not resolve to a valid local concept or indexed asset.",
840
- });
841
- }
842
- if (external) {
843
- externalReferences.set(target, { uri: target, kind: "external" });
844
- }
845
- edges.push({
846
- source: doc.uri,
847
- target,
848
- kind: reference.kind,
849
- field: reference.field,
850
- reference: rawTarget,
851
- sourceEntry: reference.kind === "source" ? reference.metadata : undefined,
852
- resolvedAs,
853
- broken,
854
- external,
855
- });
856
- });
857
- });
858
-
859
- concepts.forEach((doc) => {
860
- const contract = doc.signals && doc.signals.computation;
861
- if (!contract) {
862
- return;
863
- }
864
- const contractEdges = edges.filter((edge) => (
865
- edge.source === doc.uri
866
- && ["computation", "executor", "attester"].includes(edge.kind)
867
- ));
868
- const requiredKinds = contract.computation && contract.computation.mode === "file"
869
- ? ["computation", "executor", "attester"]
870
- : ["executor", "attester"];
871
- const assetsReady = requiredKinds.every((kind) => {
872
- const edge = contractEdges.find((entry) => entry.kind === kind);
873
- if (!edge || edge.broken) {
874
- return false;
875
- }
876
- if (kind === "computation") {
877
- if (edge.resolvedAs !== "asset") {
878
- return false;
879
- }
880
- const asset = byAssetUri.get(edge.target);
881
- return Boolean(asset && (!mimeIsText(asset.mimeType) || asset.kind === "text"));
882
- }
883
- if (edge.resolvedAs === "concept") {
884
- return true;
885
- }
886
- if (edge.resolvedAs !== "asset") {
887
- return false;
888
- }
889
- const asset = byAssetUri.get(edge.target);
890
- return Boolean(asset && (!mimeIsText(asset.mimeType) || asset.kind === "text"));
891
- });
892
- contract.structuralReady = Boolean(contract.ready);
893
- contract.assetsReady = assetsReady;
894
- contract.ready = Boolean(contract.structuralReady && assetsReady);
895
- contract.attestationReady = contract.ready;
896
- });
897
-
898
- const index = {
899
- bundles,
900
- documents,
901
- concepts,
902
- reserved,
903
- relationTypes: Array.from(allowedRelationTypes),
904
- externalReferences: Array.from(externalReferences.values()),
905
- assets,
906
- warnings,
907
- errors,
908
- edges,
909
- strictLinks,
910
- byUri,
911
- byPathUri,
912
- byCanonicalUri,
913
- byConceptId,
914
- byAssetKey,
915
- byAssetUri,
916
- ambiguousAliases,
917
- ambiguousConceptIds,
918
- };
919
- bundles.forEach((bundle) => {
920
- const rootIndex = documents.find((doc) => doc.bundle === bundle.id && doc.path === "index.md");
921
- const okfVersion = rootIndex
922
- && rootIndex.frontmatter
923
- && Object.prototype.hasOwnProperty.call(rootIndex.frontmatter, "okf_version")
924
- ? rootIndex.frontmatter.okf_version
925
- : null;
926
- bundle.okfVersion = okfVersion === null ? null : String(okfVersion);
927
- bundle.versionStatus = bundle.okfVersion === "0.2"
928
- ? "understood"
929
- : bundle.okfVersion === "0.1"
930
- ? "legacy"
931
- : bundle.okfVersion
932
- ? "unsupported"
933
- : "undeclared";
934
- bundle.documentCount = documents.filter((doc) => doc.bundle === bundle.id).length;
935
- bundle.assetCount = assets.filter((asset) => asset.bundle === bundle.id).length;
936
- });
937
- return attachValidation(index);
938
- }
939
-
940
- function buildProjectIndex(projectPath) {
941
- const project = loadProjectConfig(projectPath);
942
- const index = buildIndex(project.bundles, {
943
- relationTypes: project.relationTypes,
944
- strictLinks: project.strictLinks,
945
- });
946
- return attachProject(index, project);
947
- }
948
-
949
- async function loadProjectBundles(projectPath, options) {
950
- const project = loadProjectConfig(projectPath);
951
- const remoteBundles = await fetchRemoteBundles(project.remoteBundles, options || {});
952
- return {
953
- project,
954
- bundles: project.bundles.concat(remoteBundles),
955
- remoteBundles,
956
- };
957
- }
958
-
959
- async function buildProjectIndexAsync(projectPath, options) {
960
- const loaded = await loadProjectBundles(projectPath, options);
961
- const index = buildIndex(loaded.bundles, {
962
- relationTypes: loaded.project.relationTypes,
963
- strictLinks: options && options.strictLinks !== undefined
964
- ? options.strictLinks
965
- : loaded.project.strictLinks,
966
- });
967
- return attachProject(index, loaded.project);
968
- }
969
-
970
- function attachProject(index, project) {
971
- if (!project) {
972
- return index;
973
- }
974
- index.errors.unshift.apply(index.errors, project.errors || []);
975
- index.project = {
976
- name: project.project,
977
- path: project.path,
978
- root: project.root,
979
- plugins: project.plugins,
980
- producers: project.producers,
981
- remoteBundles: project.remoteBundles,
982
- strictLinks: project.strictLinks,
983
- };
984
- index.strictLinks = Boolean(index.strictLinks || project.strictLinks);
985
- return attachValidation(index);
986
- }
987
-
988
- function attachValidation(index) {
989
- const validation = validateIndex(index);
990
- index.conformant = validation.conformant;
991
- index.validForProject = validation.validForProject;
992
- index.valid = validation.valid;
993
- index.diagnostics = validation.diagnostics;
994
- return index;
995
- }
996
-
997
- function conceptSignals(doc, detailed) {
998
- const signals = doc.signals || {};
999
- const computation = signals.computation;
1000
- const output = {
1001
- status: signals.status,
1002
- staleAfter: signals.staleAfter,
1003
- freshness: signals.freshness,
1004
- asOf: signals.asOf,
1005
- trustTier: signals.trustTier,
1006
- sourceCount: Array.isArray(signals.sources) ? signals.sources.length : 0,
1007
- hasSources: Array.isArray(signals.sources) && signals.sources.length > 0,
1008
- sourcesOrigin: signals.sourcesOrigin,
1009
- generated: signals.generated,
1010
- generatedOrigin: signals.generatedOrigin,
1011
- computation: computation ? {
1012
- runtime: computation.runtime,
1013
- mode: computation.computation && computation.computation.mode,
1014
- attestationReady: Boolean(computation.attestationReady),
1015
- structuralReady: Boolean(computation.structuralReady),
1016
- assetsReady: Boolean(computation.assetsReady),
1017
- } : null,
1018
- };
1019
- if (detailed) {
1020
- output.resource = signals.resource;
1021
- output.sources = signals.sources || [];
1022
- output.usageWindow = signals.usageWindow;
1023
- output.verifiedEvents = signals.verifiedEvents || [];
1024
- output.diagnostics = signals.diagnostics || [];
1025
- if (computation) {
1026
- output.computation = Object.assign({}, computation);
1027
- }
1028
- }
1029
- return output;
1030
- }
1031
-
1032
- function conceptSummary(doc) {
1033
- return {
1034
- uri: doc.uri,
1035
- bundle: doc.bundle,
1036
- path: doc.path,
1037
- pathUri: doc.pathUri,
1038
- conceptId: doc.conceptId,
1039
- uriAliases: doc.uriAliases || [],
1040
- type: doc.type,
1041
- title: doc.title,
1042
- description: doc.description,
1043
- tags: doc.tags,
1044
- aliases: doc.aliases,
1045
- signals: conceptSignals(doc, false),
1046
- };
1047
- }
1048
-
1049
- module.exports = {
1050
- buildIndex,
1051
- buildProjectIndex,
1052
- buildProjectIndexAsync,
1053
- attachProject,
1054
- conceptSummary,
1055
- conceptSignals,
1056
- conceptIsGeneratedFile,
1057
- conceptMatchSummary,
1058
- bundleAllowsPath,
1059
- bundleExcludesPath,
1060
- findConceptMatches,
1061
- loadProjectBundles,
1062
- normalizeConceptTitle,
1063
- parseBundleArg,
1064
- recoverConceptLocator,
1065
- resolveLinkPath,
1066
- resolveConcept,
1067
- semanticReferences,
1068
- sanitizeBundleId,
1069
- validateIndex,
1070
- };
3
+ // Compatibility shim: the indexer moved to src/indexer/ in 0.10. Deep
4
+ // requires of this path keep working; new code should require "./indexer".
5
+ module.exports = require("./indexer/index");