@mfdaves/okf-mcp 0.3.3-rc.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/.agents/skills/okf-v02-migration/SKILL.md +37 -0
  2. package/.agents/skills/okf-v02-migration/agents/openai.yaml +6 -0
  3. package/.agents/skills/okf-v02-migration/references/field-mapping.md +26 -0
  4. package/README.md +192 -78
  5. package/okf/bundles/okf-mcp/distribution/reference-bundle.md +2 -2
  6. package/okf/bundles/okf-mcp/index.md +7 -0
  7. package/okf/bundles/okf-mcp/interfaces/http-authoring-api.md +2 -2
  8. package/okf/bundles/okf-mcp/interfaces/mcp-tools.md +8 -4
  9. package/okf/bundles/okf-mcp/overview/okf-mcp.md +2 -2
  10. package/okf/bundles/okf-mcp/policies/authoring-safety.md +5 -3
  11. package/okf/bundles/okf-mcp/runtime/file-concept-store.md +7 -5
  12. package/okf/bundles/okf-mcp/runtime/indexer.md +6 -4
  13. package/okf/bundles/okf-mcp/runtime/mcp-server.md +6 -6
  14. package/okf/bundles/okf-mcp/specs/attested-computation.md +24 -0
  15. package/okf/bundles/okf-mcp/specs/concept-format.md +5 -3
  16. package/okf/bundles/okf-mcp/specs/git-sources.md +23 -0
  17. package/okf/bundles/okf-mcp/workflows/v02-migration.md +24 -0
  18. package/package.json +9 -3
  19. package/server.json +5 -5
  20. package/src/assets.js +469 -0
  21. package/src/authoring.js +492 -20
  22. package/src/cli.js +467 -37
  23. package/src/computation.js +366 -0
  24. package/src/git-source.js +551 -0
  25. package/src/graph.js +71 -18
  26. package/src/http-server.js +8 -1
  27. package/src/index.js +5 -0
  28. package/src/indexer.js +428 -24
  29. package/src/markdown.js +129 -0
  30. package/src/mcp-server.js +447 -313
  31. package/src/migration.js +687 -0
  32. package/src/parser.js +135 -54
  33. package/src/plugins.js +30 -50
  34. package/src/project.js +74 -17
  35. package/src/remote.js +232 -17
  36. package/src/search.js +69 -6
  37. package/src/store.js +421 -25
  38. package/src/v02.js +545 -0
  39. package/src/validation.js +13 -2
  40. package/src/mcp-protocol.js +0 -283
package/src/remote.js CHANGED
@@ -1,11 +1,20 @@
1
1
  "use strict";
2
2
 
3
+ const crypto = require("crypto");
3
4
  const path = require("path");
5
+ const {
6
+ collectAssetReferences,
7
+ classifyAssetContent,
8
+ mimeIsText,
9
+ mimeTypeForPath,
10
+ } = require("./assets");
11
+ const { parseMarkdownText } = require("./parser");
4
12
 
5
13
  const DEFAULT_REMOTE_LIMITS = {
6
14
  maxFiles: 500,
7
15
  maxFileBytes: 1024 * 1024,
8
16
  maxTotalBytes: 5 * 1024 * 1024,
17
+ maxInventoryFiles: 5000,
9
18
  };
10
19
 
11
20
  function normalizeSlashes(value) {
@@ -28,7 +37,12 @@ function parseGitHubBundleUrl(url) {
28
37
  } catch (error) {
29
38
  throw new Error(`Invalid GitHub bundle URL: ${url || "<missing>"}`);
30
39
  }
31
- if (parsed.protocol !== "https:" || parsed.hostname !== "github.com") {
40
+ if (parsed.protocol !== "https:"
41
+ || parsed.hostname !== "github.com"
42
+ || parsed.username
43
+ || parsed.password
44
+ || parsed.search
45
+ || parsed.hash) {
32
46
  throw new Error("Remote OKF v1 supports only https://github.com URLs.");
33
47
  }
34
48
  const parts = parsed.pathname.split("/").filter(Boolean);
@@ -59,10 +73,10 @@ async function fetchJson(fetchImpl, url) {
59
73
  return response.json();
60
74
  }
61
75
 
62
- async function fetchText(fetchImpl, url) {
76
+ async function fetchBytes(fetchImpl, url) {
63
77
  const response = await fetchImpl(url, {
64
78
  headers: {
65
- Accept: "text/plain",
79
+ Accept: "application/octet-stream",
66
80
  "User-Agent": "okf-mcp",
67
81
  },
68
82
  });
@@ -70,7 +84,66 @@ async function fetchText(fetchImpl, url) {
70
84
  const status = response ? `${response.status} ${response.statusText || ""}`.trim() : "no response";
71
85
  throw new Error(`GitHub raw file request failed: ${status}`);
72
86
  }
73
- return response.text();
87
+ if (typeof response.arrayBuffer === "function") {
88
+ return Buffer.from(await response.arrayBuffer());
89
+ }
90
+ return Buffer.from(await response.text(), "utf8");
91
+ }
92
+
93
+ function gitBlobSha(content) {
94
+ const header = Buffer.from(`blob ${content.length}\0`, "utf8");
95
+ return crypto.createHash("sha1").update(header).update(content).digest("hex");
96
+ }
97
+
98
+ function patternToRegex(pattern) {
99
+ const text = normalizeSlashes(pattern);
100
+ let source = "";
101
+ for (let index = 0; index < text.length; index += 1) {
102
+ const character = text[index];
103
+ if (character === "*") {
104
+ if (text[index + 1] === "*") {
105
+ source += ".*";
106
+ index += 1;
107
+ } else {
108
+ source += "[^/]*";
109
+ }
110
+ } else {
111
+ source += character.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
112
+ }
113
+ }
114
+ return new RegExp(`^${source}$`);
115
+ }
116
+
117
+ function matchesPattern(relativePath, pattern) {
118
+ const normalizedPath = normalizeSlashes(relativePath);
119
+ const normalizedPattern = normalizeSlashes(pattern);
120
+ return normalizedPath === normalizedPattern
121
+ || (normalizedPattern.endsWith("/") && normalizedPath.startsWith(normalizedPattern))
122
+ || (normalizedPattern.includes("*") && patternToRegex(normalizedPattern).test(normalizedPath))
123
+ || (!normalizedPattern.includes("*") && normalizedPath.startsWith(`${normalizedPattern}/`));
124
+ }
125
+
126
+ function allowsRemotePath(config, relativePath) {
127
+ const include = Array.isArray(config.include) ? config.include : [];
128
+ const exclude = Array.isArray(config.exclude) ? config.exclude : [];
129
+ return (!include.length || include.some((pattern) => matchesPattern(relativePath, pattern)))
130
+ && !exclude.some((pattern) => matchesPattern(relativePath, pattern));
131
+ }
132
+
133
+ function excludesRemotePath(config, relativePath) {
134
+ const exclude = Array.isArray(config.exclude) ? config.exclude : [];
135
+ return exclude.some((pattern) => matchesPattern(relativePath, pattern));
136
+ }
137
+
138
+ function remoteReferencePath(fromPath, reference) {
139
+ const raw = normalizeSlashes(String(reference || "").trim()).split("#")[0];
140
+ const relative = raw.startsWith("/")
141
+ ? path.posix.normalize(raw.slice(1))
142
+ : path.posix.normalize(path.posix.join(path.posix.dirname(fromPath), raw));
143
+ if (!relative || relative === "." || relative === ".." || relative.startsWith("../") || path.posix.isAbsolute(relative)) {
144
+ return null;
145
+ }
146
+ return relative;
74
147
  }
75
148
 
76
149
  function relativeRemotePath(rootPath, filePath) {
@@ -97,10 +170,26 @@ async function fetchGitHubBundle(config, options) {
97
170
  }
98
171
  const limits = Object.assign({}, DEFAULT_REMOTE_LIMITS, (options && options.limits) || {});
99
172
  const documents = [];
173
+ const assets = [];
174
+ const inventory = new Map();
175
+ const unresolvedReferences = [];
100
176
  let totalBytes = 0;
101
177
 
178
+ let commitSha = /^[0-9a-f]{40}$/i.test(source.ref) ? source.ref : null;
179
+ if (!commitSha) {
180
+ const commit = await fetchJson(
181
+ fetchImpl,
182
+ `https://api.github.com/repos/${source.owner}/${source.repo}/commits/${encodeURIComponent(source.ref)}`,
183
+ );
184
+ if (!commit || !/^[0-9a-f]{40}$/i.test(String(commit.sha || ""))) {
185
+ throw new Error(`GitHub ref did not resolve to an immutable commit SHA: ${source.ref}`);
186
+ }
187
+ commitSha = String(commit.sha);
188
+ }
189
+ const pinnedRef = commitSha;
190
+
102
191
  async function walk(apiPath) {
103
- const apiUrl = `https://api.github.com/repos/${source.owner}/${source.repo}/contents/${encodePath(apiPath)}?ref=${encodeURIComponent(source.ref)}`;
192
+ const apiUrl = `https://api.github.com/repos/${source.owner}/${source.repo}/contents/${encodePath(apiPath)}?ref=${encodeURIComponent(pinnedRef)}`;
104
193
  const entries = await fetchJson(fetchImpl, apiUrl);
105
194
  const list = Array.isArray(entries) ? entries : [entries];
106
195
  for (const entry of list) {
@@ -108,20 +197,29 @@ async function fetchGitHubBundle(config, options) {
108
197
  await walk(entry.path);
109
198
  continue;
110
199
  }
111
- if (entry.type !== "file" || !String(entry.name || "").toLowerCase().endsWith(".md")) {
200
+ if (entry.type !== "file") {
112
201
  continue;
113
202
  }
114
- if (documents.length >= limits.maxFiles) {
115
- throw new Error(`Remote bundle exceeds file limit of ${limits.maxFiles}.`);
203
+ if (inventory.size >= limits.maxInventoryFiles) {
204
+ throw new Error(`Remote bundle exceeds inventory limit of ${limits.maxInventoryFiles} files.`);
116
205
  }
206
+ const relativePath = relativeRemotePath(source.path, entry.path);
207
+ inventory.set(relativePath, entry);
208
+ }
209
+ }
210
+
211
+ async function fetchInventoryEntry(entry, relativePath, kind) {
212
+ if (documents.length + assets.length >= limits.maxFiles) {
213
+ throw new Error(`Remote bundle exceeds file limit of ${limits.maxFiles}.`);
214
+ }
117
215
  if (Number(entry.size || 0) > limits.maxFileBytes) {
118
216
  throw new Error(`Remote file exceeds byte limit: ${entry.path}`);
119
217
  }
120
218
  if (!entry.download_url) {
121
219
  throw new Error(`Remote file has no download URL: ${entry.path}`);
122
220
  }
123
- const text = await fetchText(fetchImpl, entry.download_url);
124
- const bytes = Buffer.byteLength(text, "utf8");
221
+ const content = await fetchBytes(fetchImpl, entry.download_url);
222
+ const bytes = content.length;
125
223
  if (bytes > limits.maxFileBytes) {
126
224
  throw new Error(`Remote file exceeds byte limit after download: ${entry.path}`);
127
225
  }
@@ -129,20 +227,132 @@ async function fetchGitHubBundle(config, options) {
129
227
  if (totalBytes > limits.maxTotalBytes) {
130
228
  throw new Error(`Remote bundle exceeds total byte limit of ${limits.maxTotalBytes}.`);
131
229
  }
132
- documents.push({
133
- path: relativeRemotePath(source.path, entry.path),
134
- text,
135
- source: `github://${source.owner}/${source.repo}/${source.ref}/${entry.path}`,
136
- });
137
- }
230
+ if (/^[0-9a-f]{40}$/i.test(String(entry.sha || ""))
231
+ && gitBlobSha(content).toLowerCase() !== String(entry.sha).toLowerCase()) {
232
+ throw new Error(`Remote file content does not match its Git blob SHA: ${entry.path}`);
233
+ }
234
+ if (kind === "document") {
235
+ let text;
236
+ try {
237
+ text = new TextDecoder("utf-8", { fatal: true }).decode(content);
238
+ } catch {
239
+ throw new Error(`Remote Markdown document is not valid UTF-8: ${entry.path}`);
240
+ }
241
+ const document = {
242
+ path: relativePath,
243
+ text,
244
+ source: `github://${source.owner}/${source.repo}/${pinnedRef}/${entry.path}`,
245
+ blobSha: entry.sha || null,
246
+ };
247
+ documents.push(document);
248
+ return document;
249
+ }
250
+ return { content, bytes };
138
251
  }
139
252
 
140
253
  await walk(source.path);
254
+ for (const [relativePath, entry] of inventory) {
255
+ if (!relativePath.toLowerCase().endsWith(".md") || !allowsRemotePath(config, relativePath)) {
256
+ continue;
257
+ }
258
+ await fetchInventoryEntry(entry, relativePath, "document");
259
+ }
260
+
261
+ const referenceGroups = new Map();
262
+ const bundleId = sanitizeRemoteId(config.id, path.posix.basename(source.path));
263
+ documents.forEach((document) => {
264
+ let parsed;
265
+ try {
266
+ parsed = parseMarkdownText(
267
+ { id: bundleId, root: "", remote: true },
268
+ document.path,
269
+ document.text,
270
+ document.source,
271
+ );
272
+ } catch {
273
+ return;
274
+ }
275
+ collectAssetReferences(parsed).forEach((reference) => {
276
+ const resolvedPath = remoteReferencePath(document.path, reference.value);
277
+ if (!resolvedPath) {
278
+ unresolvedReferences.push({
279
+ path: document.path,
280
+ field: reference.field,
281
+ target: reference.value,
282
+ code: "asset_outside_root",
283
+ });
284
+ return;
285
+ }
286
+ if (resolvedPath.toLowerCase().endsWith(".md") && inventory.has(resolvedPath)) {
287
+ return;
288
+ }
289
+ if (excludesRemotePath(config, resolvedPath)) {
290
+ unresolvedReferences.push({
291
+ path: document.path,
292
+ field: reference.field,
293
+ target: reference.value,
294
+ resolvedPath,
295
+ code: "asset_excluded",
296
+ });
297
+ return;
298
+ }
299
+ if (!referenceGroups.has(resolvedPath)) {
300
+ referenceGroups.set(resolvedPath, []);
301
+ }
302
+ referenceGroups.get(resolvedPath).push(reference);
303
+ });
304
+ });
305
+
306
+ for (const [relativePath, references] of referenceGroups) {
307
+ const entry = inventory.get(relativePath);
308
+ if (!entry) {
309
+ unresolvedReferences.push.apply(unresolvedReferences, references.map((reference) => ({
310
+ path: reference.fromPath,
311
+ field: reference.field,
312
+ target: reference.value,
313
+ resolvedPath: relativePath,
314
+ code: "asset_missing",
315
+ })));
316
+ continue;
317
+ }
318
+ const fetched = await fetchInventoryEntry(entry, relativePath, "asset");
319
+ const mimeType = mimeTypeForPath(relativePath);
320
+ const classification = classifyAssetContent(fetched.content, mimeType);
321
+ if (mimeIsText(mimeType) && !classification.validUtf8) {
322
+ unresolvedReferences.push({
323
+ path: references[0].fromPath,
324
+ field: references[0].field,
325
+ target: references[0].value,
326
+ resolvedPath: relativePath,
327
+ code: "asset_invalid_utf8",
328
+ });
329
+ }
330
+ assets.push({
331
+ path: relativePath,
332
+ size: fetched.bytes,
333
+ sha256: `sha256:${crypto.createHash("sha256").update(fetched.content).digest("hex")}`,
334
+ mimeType,
335
+ kind: classification.kind,
336
+ encoding: classification.encoding,
337
+ text: classification.text,
338
+ ...(classification.kind === "binary" ? { base64: fetched.content.toString("base64") } : {}),
339
+ roles: Array.from(new Set(references.map((reference) => reference.role))).sort(),
340
+ referencedBy: references.map((reference) => ({
341
+ uri: reference.referencedByUri,
342
+ bundle: bundleId,
343
+ path: reference.fromPath,
344
+ field: reference.field,
345
+ role: reference.role,
346
+ })),
347
+ blobSha: entry.sha || null,
348
+ });
349
+ }
141
350
  return {
142
- id: sanitizeRemoteId(config.id, path.posix.basename(source.path)),
351
+ id: bundleId,
143
352
  remote: true,
144
353
  root: "",
145
354
  documents,
355
+ assets,
146
356
  include: Array.isArray(config.include) ? config.include : [],
147
357
  exclude: Array.isArray(config.exclude) ? config.exclude : [],
148
358
  remoteSource: {
@@ -153,7 +363,12 @@ async function fetchGitHubBundle(config, options) {
153
363
  ref: source.ref,
154
364
  path: source.path,
155
365
  fileCount: documents.length,
366
+ documentCount: documents.length,
367
+ assetCount: assets.length,
156
368
  totalBytes,
369
+ commitSha,
370
+ revision: pinnedRef,
371
+ unresolvedReferences,
157
372
  },
158
373
  };
159
374
  }
package/src/search.js CHANGED
@@ -1,6 +1,8 @@
1
1
  "use strict";
2
2
 
3
3
  const { conceptSummary } = require("./indexer");
4
+ const { normalizeV02Signals } = require("./v02");
5
+ const { resolveConcept } = require("./indexer");
4
6
 
5
7
  function lower(value) {
6
8
  return String(value || "").toLowerCase();
@@ -66,13 +68,36 @@ function linkedSets(index) {
66
68
  return { outbound, inbound };
67
69
  }
68
70
 
71
+ function signalsForDoc(doc, asOf) {
72
+ if (!asOf) {
73
+ return doc.signals || {};
74
+ }
75
+ const signals = normalizeV02Signals(doc, { asOf });
76
+ if (signals.computation && doc.signals && doc.signals.computation) {
77
+ signals.computation.structuralReady = signals.computation.ready;
78
+ signals.computation.assetsReady = doc.signals.computation.assetsReady;
79
+ signals.computation.attestationReady = Boolean(
80
+ signals.computation.structuralReady && signals.computation.assetsReady,
81
+ );
82
+ }
83
+ return signals;
84
+ }
85
+
69
86
  function applyFilters(index, options) {
70
87
  const config = options || {};
71
88
  const typeFilters = asArray(config.types || config.type).map(lower);
72
89
  const tagsAny = asArray(config.tagsAny || config.tag).map(lower);
73
90
  const tagsAll = asArray(config.tagsAll).map(lower);
91
+ const statuses = asArray(config.statuses || config.status).map(lower);
92
+ const trustTiers = asArray(config.trustTiers || config.trustTier).map(lower);
93
+ const freshness = asArray(config.freshness).map(lower);
94
+ const linkedToDoc = resolveConcept(index, config.linkedTo);
95
+ const linkedFromDoc = resolveConcept(index, config.linkedFrom);
96
+ const linkedTo = linkedToDoc ? linkedToDoc.uri : config.linkedTo;
97
+ const linkedFrom = linkedFromDoc ? linkedFromDoc.uri : config.linkedFrom;
74
98
  const { outbound, inbound } = linkedSets(index);
75
99
  return index.concepts.filter((doc) => {
100
+ const signals = signalsForDoc(doc, config.asOf);
76
101
  if (config.bundle && doc.bundle !== config.bundle) {
77
102
  return false;
78
103
  }
@@ -92,10 +117,10 @@ function applyFilters(index, options) {
92
117
  if (config.frontmatter && !frontmatterMatches(doc.frontmatter, config.frontmatter)) {
93
118
  return false;
94
119
  }
95
- if (config.linkedTo && !(outbound.get(doc.uri) || new Set()).has(config.linkedTo)) {
120
+ if (linkedTo && !(outbound.get(doc.uri) || new Set()).has(linkedTo)) {
96
121
  return false;
97
122
  }
98
- if (config.linkedFrom && !(inbound.get(doc.uri) || new Set()).has(config.linkedFrom)) {
123
+ if (linkedFrom && !(inbound.get(doc.uri) || new Set()).has(linkedFrom)) {
99
124
  return false;
100
125
  }
101
126
  if (config.orphanOnly && ((outbound.get(doc.uri) || new Set()).size > 0 || (inbound.get(doc.uri) || new Set()).size > 0)) {
@@ -112,6 +137,31 @@ function applyFilters(index, options) {
112
137
  return false;
113
138
  }
114
139
  }
140
+ if (statuses.length && !statuses.includes(lower(signals.status))) {
141
+ return false;
142
+ }
143
+ if (trustTiers.length && !trustTiers.includes(lower(signals.trustTier))) {
144
+ return false;
145
+ }
146
+ if (freshness.length && !freshness.includes(lower(signals.freshness))) {
147
+ return false;
148
+ }
149
+ if (config.hasSources !== undefined && Boolean(signals.sources && signals.sources.length) !== config.hasSources) {
150
+ return false;
151
+ }
152
+ if (config.runtime && lower(signals.computation && signals.computation.runtime) !== lower(config.runtime)) {
153
+ return false;
154
+ }
155
+ if (config.attestationReady !== undefined
156
+ && Boolean(signals.computation && signals.computation.attestationReady) !== config.attestationReady) {
157
+ return false;
158
+ }
159
+ if (config.generatedBy && lower(signals.generated && signals.generated.by) !== lower(config.generatedBy)) {
160
+ return false;
161
+ }
162
+ if (config.verifiedBy && !(signals.verifiedEvents || []).some((event) => lower(event.by) === lower(config.verifiedBy))) {
163
+ return false;
164
+ }
115
165
  return true;
116
166
  });
117
167
  }
@@ -161,10 +211,23 @@ function searchConcepts(index, options) {
161
211
  } else {
162
212
  results.sort((a, b) => a.doc.path.localeCompare(b.doc.path));
163
213
  }
164
- const page = results.slice(offset, offset + limit).map((entry) => Object.assign(conceptSummary(entry.doc), {
165
- score: entry.score,
166
- snippet: snippetFor(entry.doc, query),
167
- }));
214
+ const page = results.slice(offset, offset + limit).map((entry) => {
215
+ const summary = conceptSummary(entry.doc);
216
+ if (config.asOf) {
217
+ const signals = signalsForDoc(entry.doc, config.asOf);
218
+ summary.signals = Object.assign({}, summary.signals, {
219
+ status: signals.status,
220
+ staleAfter: signals.staleAfter,
221
+ freshness: signals.freshness,
222
+ asOf: signals.asOf,
223
+ trustTier: signals.trustTier,
224
+ });
225
+ }
226
+ return Object.assign(summary, {
227
+ score: entry.score,
228
+ snippet: snippetFor(entry.doc, query),
229
+ });
230
+ });
168
231
  return {
169
232
  total: results.length,
170
233
  limit,