@kryptosai/mcp-observatory 0.14.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/adapters/http.js +6 -2
- package/dist/src/adapters/http.js.map +1 -1
- package/dist/src/adapters/local-process.js +25 -3
- package/dist/src/adapters/local-process.js.map +1 -1
- package/dist/src/checks/conformance.js +5 -4
- package/dist/src/checks/conformance.js.map +1 -1
- package/dist/src/checks/list-check.js +2 -1
- package/dist/src/checks/list-check.js.map +1 -1
- package/dist/src/checks/resources.js +3 -2
- package/dist/src/checks/resources.js.map +1 -1
- package/dist/src/checks/tools-invoke.js +3 -2
- package/dist/src/checks/tools-invoke.js.map +1 -1
- package/dist/src/cli.js +36 -901
- package/dist/src/cli.js.map +1 -1
- package/dist/src/commands/diff.d.ts +2 -0
- package/dist/src/commands/diff.js +27 -0
- package/dist/src/commands/diff.js.map +1 -0
- package/dist/src/commands/helpers.d.ts +25 -0
- package/dist/src/commands/helpers.js +131 -0
- package/dist/src/commands/helpers.js.map +1 -0
- package/dist/src/commands/legacy.d.ts +2 -0
- package/dist/src/commands/legacy.js +77 -0
- package/dist/src/commands/legacy.js.map +1 -0
- package/dist/src/commands/record-replay.d.ts +2 -0
- package/dist/src/commands/record-replay.js +181 -0
- package/dist/src/commands/record-replay.js.map +1 -0
- package/dist/src/commands/scan.d.ts +2 -0
- package/dist/src/commands/scan.js +155 -0
- package/dist/src/commands/scan.js.map +1 -0
- package/dist/src/commands/score.d.ts +2 -0
- package/dist/src/commands/score.js +88 -0
- package/dist/src/commands/score.js.map +1 -0
- package/dist/src/commands/serve.d.ts +2 -0
- package/dist/src/commands/serve.js +10 -0
- package/dist/src/commands/serve.js.map +1 -0
- package/dist/src/commands/suggest.d.ts +2 -0
- package/dist/src/commands/suggest.js +126 -0
- package/dist/src/commands/suggest.js.map +1 -0
- package/dist/src/commands/telemetry.d.ts +2 -0
- package/dist/src/commands/telemetry.js +65 -0
- package/dist/src/commands/telemetry.js.map +1 -0
- package/dist/src/commands/test.d.ts +2 -0
- package/dist/src/commands/test.js +37 -0
- package/dist/src/commands/test.js.map +1 -0
- package/dist/src/commands/watch.d.ts +5 -0
- package/dist/src/commands/watch.js +46 -0
- package/dist/src/commands/watch.js.map +1 -0
- package/dist/src/environment.js +12 -4
- package/dist/src/environment.js.map +1 -1
- package/dist/src/runner.js +2 -1
- package/dist/src/runner.js.map +1 -1
- package/dist/src/server.d.ts +2 -0
- package/dist/src/server.js +30 -14
- package/dist/src/server.js.map +1 -1
- package/dist/src/utils/errors.d.ts +4 -0
- package/dist/src/utils/errors.js +7 -0
- package/dist/src/utils/errors.js.map +1 -0
- package/package.json +1 -1
package/dist/src/cli.js
CHANGED
|
@@ -1,135 +1,21 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import path from "node:path";
|
|
2
|
+
import { writeFile } from "node:fs/promises";
|
|
4
3
|
import { Command } from "commander";
|
|
5
|
-
import { scanForTargets } from "./discovery.js";
|
|
6
|
-
import { detectEnvironment } from "./environment.js";
|
|
7
|
-
import os from "node:os";
|
|
8
|
-
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
9
|
-
import { defaultCassettesDirectory, loadCassette, saveCassette } from "./cassette.js";
|
|
10
|
-
import { runPromptsCheck } from "./checks/prompts.js";
|
|
11
|
-
import { runResourcesCheck } from "./checks/resources.js";
|
|
12
|
-
import { runToolsCheck } from "./checks/tools.js";
|
|
13
|
-
import { runToolsInvokeCheck } from "./checks/tools-invoke.js";
|
|
14
|
-
import { generateBadgeSvg } from "./badge.js";
|
|
15
|
-
import { diffArtifacts, readArtifact, renderHtml, renderMarkdown, renderTerminal, runTarget, writeRunArtifact } from "./index.js";
|
|
16
|
-
import { renderJUnit } from "./reporters/junit.js";
|
|
17
|
-
import { renderSarif } from "./reporters/sarif.js";
|
|
18
|
-
import { runTargetRecording } from "./runner.js";
|
|
19
|
-
import { defaultRunsDirectory } from "./storage.js";
|
|
20
|
-
import { ReplayTransport } from "./transport/replay-transport.js";
|
|
21
|
-
import { SCHEMA_VERSION } from "./types.js";
|
|
22
|
-
import { buildRunId } from "./utils/ids.js";
|
|
23
|
-
import { validateTargetConfig } from "./validate.js";
|
|
24
|
-
import { compareResponses } from "./verify.js";
|
|
25
4
|
import { isCI } from "./ci.js";
|
|
26
|
-
import {
|
|
5
|
+
import { ANSI, LOGO, c, getBinName, useColor } from "./commands/helpers.js";
|
|
6
|
+
import { registerDiffCommands } from "./commands/diff.js";
|
|
7
|
+
import { registerLegacyCommands } from "./commands/legacy.js";
|
|
8
|
+
import { registerRecordReplayCommands } from "./commands/record-replay.js";
|
|
9
|
+
import { registerScanCommands } from "./commands/scan.js";
|
|
10
|
+
import { registerScoreCommands } from "./commands/score.js";
|
|
11
|
+
import { registerServeCommands } from "./commands/serve.js";
|
|
12
|
+
import { registerSuggestCommands } from "./commands/suggest.js";
|
|
13
|
+
import { registerTelemetryCommands } from "./commands/telemetry.js";
|
|
14
|
+
import { registerTestCommands } from "./commands/test.js";
|
|
15
|
+
import { registerWatchCommands } from "./commands/watch.js";
|
|
16
|
+
import { runTarget } from "./index.js";
|
|
17
|
+
import { loadTelemetryConfig, showFirstRunNotice, recordEvent, buildEvent, isTelemetryEnabled } from "./telemetry.js";
|
|
27
18
|
import { TOOL_VERSION } from "./version.js";
|
|
28
|
-
// ── ASCII Logo ──────────────────────────────────────────────────────────────
|
|
29
|
-
const LOGO = `
|
|
30
|
-
███╗ ███╗ ██████╗██████╗
|
|
31
|
-
████╗ ████║██╔════╝██╔══██╗
|
|
32
|
-
██╔████╔██║██║ ██████╔╝
|
|
33
|
-
██║╚██╔╝██║██║ ██╔═══╝
|
|
34
|
-
██║ ╚═╝ ██║╚██████╗██║
|
|
35
|
-
╚═╝ ╚═╝ ╚═════╝╚═╝
|
|
36
|
-
O B S E R V A T O R Y
|
|
37
|
-
`;
|
|
38
|
-
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
39
|
-
async function readTargetConfig(filePath) {
|
|
40
|
-
const content = await readFile(filePath, "utf8");
|
|
41
|
-
return validateTargetConfig(JSON.parse(content));
|
|
42
|
-
}
|
|
43
|
-
function targetFromCommand(args) {
|
|
44
|
-
if (args.length === 0) {
|
|
45
|
-
throw new Error("No command provided. Usage: mcp-observatory test <command> [args...]");
|
|
46
|
-
}
|
|
47
|
-
const command = args[0];
|
|
48
|
-
const restArgs = args.slice(1);
|
|
49
|
-
// Build a meaningful targetId: for wrapper commands, use the package/script name
|
|
50
|
-
let targetId = command;
|
|
51
|
-
const wrappers = new Set(["npx", "node", "docker", "uvx", "bunx", "pnpx"]);
|
|
52
|
-
if (wrappers.has(command)) {
|
|
53
|
-
const pkg = restArgs.find(a => !a.startsWith("-") && a !== "run");
|
|
54
|
-
if (pkg)
|
|
55
|
-
targetId = pkg;
|
|
56
|
-
}
|
|
57
|
-
return {
|
|
58
|
-
targetId,
|
|
59
|
-
adapter: "local-process",
|
|
60
|
-
command,
|
|
61
|
-
args: restArgs,
|
|
62
|
-
timeoutMs: 15_000,
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
// Extract args after -- before Commander sees them
|
|
66
|
-
const _rawArgv = [...process.argv];
|
|
67
|
-
const _dashDashIdx = _rawArgv.indexOf("--");
|
|
68
|
-
const _passthroughArgs = _dashDashIdx !== -1 ? _rawArgv.splice(_dashDashIdx).slice(1) : [];
|
|
69
|
-
if (_dashDashIdx !== -1) {
|
|
70
|
-
process.argv = _rawArgv;
|
|
71
|
-
}
|
|
72
|
-
function getPassthroughArgs() {
|
|
73
|
-
return _passthroughArgs;
|
|
74
|
-
}
|
|
75
|
-
async function resolveTarget(options) {
|
|
76
|
-
if (options.target) {
|
|
77
|
-
return readTargetConfig(options.target);
|
|
78
|
-
}
|
|
79
|
-
const passthrough = getPassthroughArgs();
|
|
80
|
-
if (passthrough.length > 0) {
|
|
81
|
-
return targetFromCommand(passthrough);
|
|
82
|
-
}
|
|
83
|
-
throw new Error("Provide --target <config.json> or use: mcp-observatory test <command>");
|
|
84
|
-
}
|
|
85
|
-
function useColor() {
|
|
86
|
-
return !process.env["NO_COLOR"] && !process.argv.includes("--no-color");
|
|
87
|
-
}
|
|
88
|
-
const ANSI = {
|
|
89
|
-
red: "\x1b[31m",
|
|
90
|
-
green: "\x1b[32m",
|
|
91
|
-
yellow: "\x1b[33m",
|
|
92
|
-
blue: "\x1b[34m",
|
|
93
|
-
cyan: "\x1b[36m",
|
|
94
|
-
dim: "\x1b[2m",
|
|
95
|
-
bold: "\x1b[1m",
|
|
96
|
-
reset: "\x1b[0m",
|
|
97
|
-
};
|
|
98
|
-
function c(code, text) {
|
|
99
|
-
return useColor() ? `${code}${text}${ANSI.reset}` : text;
|
|
100
|
-
}
|
|
101
|
-
function formatOutput(artifact, format) {
|
|
102
|
-
if (format === "json")
|
|
103
|
-
return JSON.stringify(artifact, null, 2);
|
|
104
|
-
if (format === "markdown")
|
|
105
|
-
return renderMarkdown(artifact);
|
|
106
|
-
if (format === "html")
|
|
107
|
-
return renderHtml(artifact);
|
|
108
|
-
if (format === "junit" && artifact.artifactType === "run")
|
|
109
|
-
return renderJUnit(artifact);
|
|
110
|
-
if (format === "sarif" && artifact.artifactType === "run")
|
|
111
|
-
return renderSarif(artifact);
|
|
112
|
-
return renderTerminal(artifact);
|
|
113
|
-
}
|
|
114
|
-
async function writeOutput(content, format, outputPath) {
|
|
115
|
-
if (outputPath !== undefined) {
|
|
116
|
-
await mkdir(path.dirname(outputPath), { recursive: true });
|
|
117
|
-
await writeFile(outputPath, content + "\n", "utf8");
|
|
118
|
-
process.stdout.write(`Wrote ${format} report to ${outputPath}\n`);
|
|
119
|
-
}
|
|
120
|
-
else {
|
|
121
|
-
process.stdout.write(`${content}\n`);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
// ── Invocation detection ─────────────────────────────────────────────────────
|
|
125
|
-
/** Returns the command the user actually typed, so tips are copy-pasteable. */
|
|
126
|
-
function getBinName() {
|
|
127
|
-
const script = process.argv[1] ?? "";
|
|
128
|
-
if (script.includes(".npm/_npx") || script.includes("npx")) {
|
|
129
|
-
return "npx @kryptosai/mcp-observatory";
|
|
130
|
-
}
|
|
131
|
-
return "mcp-observatory";
|
|
132
|
-
}
|
|
133
19
|
const MENU_GROUPS = [
|
|
134
20
|
{
|
|
135
21
|
heading: "",
|
|
@@ -301,600 +187,17 @@ async function main() {
|
|
|
301
187
|
const lines = examples.map(([cmd, desc]) => ` ${c(ANSI.dim, "$")} ${bin}${cmd}${pad(cmd)}${desc}`);
|
|
302
188
|
return ["", "Examples:", "", ...lines, ""].join("\n");
|
|
303
189
|
})());
|
|
304
|
-
//
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
// `scan deep` — scan + invoke tools
|
|
316
|
-
scanCmd
|
|
317
|
-
.command("deep")
|
|
318
|
-
.description("Scan and also invoke safe tools to verify they execute.")
|
|
319
|
-
.option("--config <path>", "Path to a specific MCP config file.")
|
|
320
|
-
.option("--security", "Run security analysis on tool schemas.")
|
|
321
|
-
.action(async (options) => {
|
|
322
|
-
// Inherit parent config option if set
|
|
323
|
-
const parentConfig = scanCmd.opts().config;
|
|
324
|
-
const parentSecurity = scanCmd.opts().security;
|
|
325
|
-
await runScan(bin, options.config ?? parentConfig, true, options.security ?? parentSecurity ?? true);
|
|
326
|
-
});
|
|
327
|
-
// ── test ──────────────────────────────────────────────────────────────
|
|
328
|
-
program
|
|
329
|
-
.command("test")
|
|
330
|
-
.passThroughOptions()
|
|
331
|
-
.description("Test a specific server by command.")
|
|
332
|
-
.argument("<command...>", "Server command and arguments to run.")
|
|
333
|
-
.option("--security", "Run security analysis on tool schemas.")
|
|
334
|
-
.option("--no-color", "Disable colored output.")
|
|
335
|
-
.action(async (commandArgs, options) => {
|
|
336
|
-
const target = targetFromCommand(commandArgs);
|
|
337
|
-
process.stdout.write(` ${c(ANSI.dim, "⟳")} Checking ${c(ANSI.bold, target.targetId)}...`);
|
|
338
|
-
const artifact = await runTarget(target, { securityCheck: options.security });
|
|
339
|
-
const outPath = await writeRunArtifact(artifact, defaultRunsDirectory(process.cwd()));
|
|
340
|
-
const toolsEvidence = artifact.checks.find(ch => ch.id === "tools");
|
|
341
|
-
const promptsEvidence = artifact.checks.find(ch => ch.id === "prompts");
|
|
342
|
-
const resourcesEvidence = artifact.checks.find(ch => ch.id === "resources");
|
|
343
|
-
const toolCount = toolsEvidence?.evidence[0]?.itemCount ?? 0;
|
|
344
|
-
const promptCount = promptsEvidence?.evidence[0]?.itemCount ?? 0;
|
|
345
|
-
const resourceCount = resourcesEvidence?.evidence[0]?.itemCount ?? 0;
|
|
346
|
-
const gateIcon = artifact.gate === "pass" ? c(ANSI.green, "✓") : c(ANSI.red, "✗");
|
|
347
|
-
process.stdout.write(`\r ${gateIcon} ${c(ANSI.bold, target.targetId)}${" ".repeat(Math.max(1, 40 - target.targetId.length))}`);
|
|
348
|
-
process.stdout.write(`${c(ANSI.dim, `${toolCount} tools, ${promptCount} prompts, ${resourceCount} resources`)}\n`);
|
|
349
|
-
for (const check of artifact.checks) {
|
|
350
|
-
if (check.status === "fail" || check.status === "partial") {
|
|
351
|
-
process.stdout.write(` ${c(ANSI.dim, "→")} ${check.id}: ${check.message}\n`);
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
process.stdout.write(`\n ${c(ANSI.dim, `Artifact: ${outPath}`)}\n\n`);
|
|
355
|
-
if (artifact.gate === "fail") {
|
|
356
|
-
process.exitCode = 1;
|
|
357
|
-
}
|
|
358
|
-
});
|
|
359
|
-
// ── diff ──────────────────────────────────────────────────────────────
|
|
360
|
-
program
|
|
361
|
-
.command("diff")
|
|
362
|
-
.description("Compare two runs and show regressions and schema drift.")
|
|
363
|
-
.argument("<base>", "Base run artifact JSON file.")
|
|
364
|
-
.argument("<head>", "Head run artifact JSON file.")
|
|
365
|
-
.option("--format <format>", "terminal, json, markdown, html, junit, or sarif", "terminal")
|
|
366
|
-
.option("--output <file>", "Write to file instead of stdout.")
|
|
367
|
-
.option("--no-color", "Disable colored output.")
|
|
368
|
-
.option("--fail-on-regression", "Exit with code 1 when regressions are present.", false)
|
|
369
|
-
.action(async (base, head, options) => {
|
|
370
|
-
const baseArtifact = await readArtifact(base);
|
|
371
|
-
const headArtifact = await readArtifact(head);
|
|
372
|
-
if (baseArtifact.artifactType !== "run" || headArtifact.artifactType !== "run") {
|
|
373
|
-
throw new Error("The diff command only accepts run artifacts.");
|
|
374
|
-
}
|
|
375
|
-
const artifact = diffArtifacts(baseArtifact, headArtifact);
|
|
376
|
-
const output = formatOutput(artifact, options.format);
|
|
377
|
-
await writeOutput(output, options.format, options.output);
|
|
378
|
-
if (options.failOnRegression && artifact.gate === "fail") {
|
|
379
|
-
process.exitCode = 1;
|
|
380
|
-
}
|
|
381
|
-
});
|
|
382
|
-
// ── watch ─────────────────────────────────────────────────────────────
|
|
383
|
-
program
|
|
384
|
-
.command("watch")
|
|
385
|
-
.description("Watch a server for changes, alert on regressions.")
|
|
386
|
-
.argument("<config>", "Path to a target config JSON file.")
|
|
387
|
-
.option("--interval <seconds>", "Check interval in seconds.", "30")
|
|
388
|
-
.option("--no-color", "Disable colored output.")
|
|
389
|
-
.action(async (configPath, options) => {
|
|
390
|
-
const target = await readTargetConfig(configPath);
|
|
391
|
-
const outDir = defaultRunsDirectory(process.cwd());
|
|
392
|
-
await runWatchMode(target, outDir, parseInt(options.interval, 10) || 30);
|
|
393
|
-
});
|
|
394
|
-
// ── serve ─────────────────────────────────────────────────────────────
|
|
395
|
-
program
|
|
396
|
-
.command("serve")
|
|
397
|
-
.description("Start as an MCP server for AI agents.")
|
|
398
|
-
.action(async () => {
|
|
399
|
-
const { startServer } = await import("./server.js");
|
|
400
|
-
await startServer();
|
|
401
|
-
});
|
|
402
|
-
// ── suggest ────────────────────────────────────────────────────────────
|
|
403
|
-
program
|
|
404
|
-
.command("suggest")
|
|
405
|
-
.description("Detect your stack and recommend MCP servers.")
|
|
406
|
-
.option("--cwd <path>", "Directory to scan for project signals.", process.cwd())
|
|
407
|
-
.option("--no-color", "Disable colored output.")
|
|
408
|
-
.action(async (options) => {
|
|
409
|
-
process.stdout.write(`${c(ANSI.dim, "⟳")} Scanning environment...\n\n`);
|
|
410
|
-
// 1. Current MCP servers
|
|
411
|
-
const targets = await scanForTargets();
|
|
412
|
-
if (targets.length > 0) {
|
|
413
|
-
process.stdout.write(c(ANSI.bold, " Configured MCP Servers\n"));
|
|
414
|
-
for (const t of targets) {
|
|
415
|
-
const detail = t.config.adapter === "http"
|
|
416
|
-
? t.config.url
|
|
417
|
-
: `${t.config.command} ${t.config.args.join(" ")}`;
|
|
418
|
-
process.stdout.write(` ${c(ANSI.cyan, "●")} ${c(ANSI.bold, t.config.targetId)} ${c(ANSI.dim, detail)} ${c(ANSI.dim, `← ${t.source}`)}\n`);
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
else {
|
|
422
|
-
process.stdout.write(` ${c(ANSI.yellow, "No MCP servers configured.")}\n`);
|
|
423
|
-
}
|
|
424
|
-
process.stdout.write("\n");
|
|
425
|
-
// 2. Environment detection
|
|
426
|
-
const env = await detectEnvironment(options.cwd);
|
|
427
|
-
const hasSignals = env.languages.length > 0 || env.frameworks.length > 0 || env.databases.length > 0;
|
|
428
|
-
if (hasSignals) {
|
|
429
|
-
process.stdout.write(c(ANSI.bold, " Detected Stack\n"));
|
|
430
|
-
if (env.languages.length > 0)
|
|
431
|
-
process.stdout.write(` ${c(ANSI.dim, "Languages:")} ${env.languages.join(", ")}\n`);
|
|
432
|
-
if (env.frameworks.length > 0)
|
|
433
|
-
process.stdout.write(` ${c(ANSI.dim, "Frameworks:")} ${env.frameworks.join(", ")}\n`);
|
|
434
|
-
if (env.databases.length > 0)
|
|
435
|
-
process.stdout.write(` ${c(ANSI.dim, "Databases:")} ${env.databases.join(", ")}\n`);
|
|
436
|
-
if (env.cloud.length > 0)
|
|
437
|
-
process.stdout.write(` ${c(ANSI.dim, "Cloud:")} ${env.cloud.join(", ")}\n`);
|
|
438
|
-
if (env.cicd.length > 0)
|
|
439
|
-
process.stdout.write(` ${c(ANSI.dim, "CI/CD:")} ${env.cicd.join(", ")}\n`);
|
|
440
|
-
if (env.services.length > 0)
|
|
441
|
-
process.stdout.write(` ${c(ANSI.dim, "Services:")} ${env.services.join(", ")}\n`);
|
|
442
|
-
}
|
|
443
|
-
else {
|
|
444
|
-
process.stdout.write(` ${c(ANSI.dim, "No recognizable project signals in")} ${options.cwd}\n`);
|
|
445
|
-
}
|
|
446
|
-
process.stdout.write("\n");
|
|
447
|
-
// 3. MCP Registry — filtered by detected stack
|
|
448
|
-
try {
|
|
449
|
-
const response = await fetch("https://registry.modelcontextprotocol.io/v0/servers", {
|
|
450
|
-
signal: AbortSignal.timeout(10_000),
|
|
451
|
-
headers: { "Accept": "application/json" },
|
|
452
|
-
});
|
|
453
|
-
if (response.ok) {
|
|
454
|
-
const data = await response.json();
|
|
455
|
-
const raw = Array.isArray(data) ? data : (typeof data === "object" && data !== null
|
|
456
|
-
? (data["servers"] ?? data["results"] ?? data["items"])
|
|
457
|
-
: null);
|
|
458
|
-
if (Array.isArray(raw)) {
|
|
459
|
-
const allEntries = raw;
|
|
460
|
-
// Build keyword set from detected environment
|
|
461
|
-
const keywords = new Set([
|
|
462
|
-
...env.languages, ...env.frameworks, ...env.databases,
|
|
463
|
-
...env.services, ...env.cloud, ...env.cicd,
|
|
464
|
-
].map(s => s.toLowerCase()));
|
|
465
|
-
// Also add common aliases
|
|
466
|
-
const aliases = {
|
|
467
|
-
typescript: ["ts"], javascript: ["js", "node"], python: ["py"],
|
|
468
|
-
postgresql: ["postgres"], mongodb: ["mongo"], github: ["gh"],
|
|
469
|
-
};
|
|
470
|
-
for (const kw of [...keywords]) {
|
|
471
|
-
for (const [full, abbrs] of Object.entries(aliases)) {
|
|
472
|
-
if (kw === full)
|
|
473
|
-
for (const a of abbrs)
|
|
474
|
-
keywords.add(a);
|
|
475
|
-
if (abbrs.includes(kw))
|
|
476
|
-
keywords.add(full);
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
|
-
// Score each entry against detected stack
|
|
480
|
-
const scored = allEntries.map(entry => {
|
|
481
|
-
const srv = (typeof entry["server"] === "object" && entry["server"] !== null ? entry["server"] : entry);
|
|
482
|
-
const name = typeof srv["name"] === "string" ? srv["name"] : (typeof entry["name"] === "string" ? entry["name"] : "unknown");
|
|
483
|
-
const desc = typeof srv["description"] === "string" ? srv["description"] : (typeof entry["description"] === "string" ? entry["description"] : "");
|
|
484
|
-
const text = `${name} ${desc}`.toLowerCase();
|
|
485
|
-
const matches = [...keywords].filter(k => text.includes(k)).length;
|
|
486
|
-
return { name, desc, matches };
|
|
487
|
-
});
|
|
488
|
-
const recommended = scored.filter(s => s.matches > 0).sort((a, b) => b.matches - a.matches);
|
|
489
|
-
const others = scored.filter(s => s.matches === 0);
|
|
490
|
-
if (recommended.length > 0) {
|
|
491
|
-
process.stdout.write(c(ANSI.bold, " Recommended for Your Stack\n"));
|
|
492
|
-
for (const r of recommended.slice(0, 10)) {
|
|
493
|
-
process.stdout.write(` ${c(ANSI.green, "★")} ${c(ANSI.bold, r.name)}${r.desc ? ` ${c(ANSI.dim, "—")} ${r.desc}` : ""}\n`);
|
|
494
|
-
}
|
|
495
|
-
if (recommended.length > 10) {
|
|
496
|
-
process.stdout.write(` ${c(ANSI.dim, `... and ${recommended.length - 10} more matches`)}\n`);
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
else {
|
|
500
|
-
process.stdout.write(c(ANSI.bold, " MCP Registry\n"));
|
|
501
|
-
for (const s of scored.slice(0, 10)) {
|
|
502
|
-
process.stdout.write(` ${c(ANSI.dim, "●")} ${c(ANSI.bold, s.name)}${s.desc ? ` ${c(ANSI.dim, "—")} ${s.desc}` : ""}\n`);
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
if (others.length > 0) {
|
|
506
|
-
process.stdout.write(`\n ${c(ANSI.dim, `${others.length} more servers at registry.modelcontextprotocol.io`)}\n`);
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
else {
|
|
510
|
-
process.stdout.write(` ${c(ANSI.dim, "Registry returned unexpected format.")}\n`);
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
else {
|
|
514
|
-
process.stdout.write(` ${c(ANSI.dim, `Registry returned HTTP ${response.status}`)}\n`);
|
|
515
|
-
}
|
|
516
|
-
}
|
|
517
|
-
catch (error) {
|
|
518
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
519
|
-
process.stdout.write(` ${c(ANSI.yellow, "Could not reach registry:")} ${msg}\n`);
|
|
520
|
-
}
|
|
521
|
-
process.stdout.write("\n");
|
|
522
|
-
});
|
|
523
|
-
// ── record ─────────────────────────────────────────────────────────────
|
|
524
|
-
program
|
|
525
|
-
.command("record")
|
|
526
|
-
.passThroughOptions()
|
|
527
|
-
.description("Record a server session to a cassette file for replay.")
|
|
528
|
-
.argument("[command...]", "Server command and arguments to run.")
|
|
529
|
-
.option("--target <config>", "Path to a target config JSON file.")
|
|
530
|
-
.option("--no-color", "Disable colored output.")
|
|
531
|
-
.action(async (commandArgs, options) => {
|
|
532
|
-
const target = options.target
|
|
533
|
-
? await readTargetConfig(options.target)
|
|
534
|
-
: targetFromCommand(commandArgs.length > 0 ? commandArgs : getPassthroughArgs());
|
|
535
|
-
process.stdout.write(`${c(ANSI.dim, "⟳")} Recording session with ${c(ANSI.bold, target.targetId)}...\n`);
|
|
536
|
-
const { artifact, cassetteEntries } = await runTargetRecording(target, { invokeTools: true });
|
|
537
|
-
if (!cassetteEntries || cassetteEntries.length === 0) {
|
|
538
|
-
process.stdout.write(`${c(ANSI.yellow, "⚠")} No traffic recorded.\n`);
|
|
539
|
-
process.exitCode = 1;
|
|
540
|
-
return;
|
|
541
|
-
}
|
|
542
|
-
const cassette = {
|
|
543
|
-
version: 1,
|
|
544
|
-
targetId: target.targetId,
|
|
545
|
-
recordedAt: new Date().toISOString(),
|
|
546
|
-
transport: target.adapter === "http" ? "http" : "stdio",
|
|
547
|
-
entries: cassetteEntries,
|
|
548
|
-
};
|
|
549
|
-
const cassettePath = await saveCassette(cassette, defaultCassettesDirectory(process.cwd()));
|
|
550
|
-
const summary = renderTerminal(artifact);
|
|
551
|
-
process.stdout.write(`\n${summary}\n`);
|
|
552
|
-
process.stdout.write(`\n${c(ANSI.green, "✓")} Cassette saved: ${cassettePath}\n`);
|
|
553
|
-
process.stdout.write(` ${c(ANSI.dim, `${cassetteEntries.length} entries recorded`)}\n`);
|
|
554
|
-
process.stdout.write(`\n Replay offline: ${c(ANSI.cyan, `${bin} replay ${cassettePath}`)}\n`);
|
|
555
|
-
process.stdout.write(` Verify live: ${c(ANSI.cyan, `${bin} verify ${cassettePath} ${target.adapter === "http" ? `--target <config>` : commandArgs.join(" ")}`)}\n\n`);
|
|
556
|
-
if (artifact.gate === "fail") {
|
|
557
|
-
process.exitCode = 1;
|
|
558
|
-
}
|
|
559
|
-
});
|
|
560
|
-
// ── replay ─────────────────────────────────────────────────────────────
|
|
561
|
-
program
|
|
562
|
-
.command("replay")
|
|
563
|
-
.description("Replay a cassette file offline — no live server needed.")
|
|
564
|
-
.argument("<cassette>", "Path to a cassette JSON file.")
|
|
565
|
-
.option("--no-color", "Disable colored output.")
|
|
566
|
-
.action(async (cassettePath) => {
|
|
567
|
-
const cassette = await loadCassette(cassettePath);
|
|
568
|
-
process.stdout.write(`${c(ANSI.dim, "⟳")} Replaying cassette for ${c(ANSI.bold, cassette.targetId)} (${cassette.entries.length} entries)...\n`);
|
|
569
|
-
// Create a target config for the replay
|
|
570
|
-
const replayTarget = {
|
|
571
|
-
targetId: cassette.targetId,
|
|
572
|
-
adapter: "local-process",
|
|
573
|
-
command: "replay",
|
|
574
|
-
args: [],
|
|
575
|
-
};
|
|
576
|
-
// Build a ReplayTransport and run checks against it
|
|
577
|
-
const transport = new ReplayTransport(cassette.entries);
|
|
578
|
-
const client = new Client({ name: "mcp-observatory", version: TOOL_VERSION }, { capabilities: {} });
|
|
579
|
-
await client.connect(transport);
|
|
580
|
-
const serverCapabilities = client.getServerCapabilities();
|
|
581
|
-
const checkContext = {
|
|
582
|
-
client,
|
|
583
|
-
serverCapabilities,
|
|
584
|
-
target: replayTarget,
|
|
585
|
-
timeoutMs: 10_000,
|
|
586
|
-
stderrLines: [],
|
|
587
|
-
};
|
|
588
|
-
const toolsCheck = await runToolsCheck(checkContext);
|
|
589
|
-
const promptsCheck = await runPromptsCheck(checkContext);
|
|
590
|
-
const resourcesCheck = await runResourcesCheck(checkContext);
|
|
591
|
-
const invokeCheck = await runToolsInvokeCheck(checkContext);
|
|
592
|
-
await client.close();
|
|
593
|
-
const checks = [
|
|
594
|
-
toolsCheck.result,
|
|
595
|
-
promptsCheck.result,
|
|
596
|
-
resourcesCheck.result,
|
|
597
|
-
invokeCheck.result,
|
|
598
|
-
];
|
|
599
|
-
const failCount = checks.filter((ch) => ch.status === "fail").length;
|
|
600
|
-
const gate = failCount > 0 ? "fail" : "pass";
|
|
601
|
-
const artifact = {
|
|
602
|
-
artifactType: "run",
|
|
603
|
-
schemaVersion: SCHEMA_VERSION,
|
|
604
|
-
gate,
|
|
605
|
-
runId: buildRunId(),
|
|
606
|
-
createdAt: new Date().toISOString(),
|
|
607
|
-
toolVersion: TOOL_VERSION,
|
|
608
|
-
target: {
|
|
609
|
-
targetId: cassette.targetId,
|
|
610
|
-
adapter: "local-process",
|
|
611
|
-
command: "replay",
|
|
612
|
-
args: [],
|
|
613
|
-
metadata: { source: "cassette", cassettePath },
|
|
614
|
-
},
|
|
615
|
-
environment: {
|
|
616
|
-
platform: `${os.platform()} ${os.release()}`,
|
|
617
|
-
nodeVersion: process.version,
|
|
618
|
-
},
|
|
619
|
-
summary: {
|
|
620
|
-
total: checks.length,
|
|
621
|
-
pass: checks.filter((ch) => ch.status === "pass").length,
|
|
622
|
-
fail: failCount,
|
|
623
|
-
partial: checks.filter((ch) => ch.status === "partial").length,
|
|
624
|
-
unsupported: checks.filter((ch) => ch.status === "unsupported").length,
|
|
625
|
-
flaky: checks.filter((ch) => ch.status === "flaky").length,
|
|
626
|
-
skipped: checks.filter((ch) => ch.status === "skipped").length,
|
|
627
|
-
gate,
|
|
628
|
-
},
|
|
629
|
-
checks,
|
|
630
|
-
};
|
|
631
|
-
process.stdout.write(`\n${renderTerminal(artifact)}\n`);
|
|
632
|
-
process.stdout.write(`\n${c(ANSI.dim, `Replayed from: ${cassettePath}`)}\n\n`);
|
|
633
|
-
if (artifact.gate === "fail") {
|
|
634
|
-
process.exitCode = 1;
|
|
635
|
-
}
|
|
636
|
-
});
|
|
637
|
-
// ── verify ─────────────────────────────────────────────────────────────
|
|
638
|
-
program
|
|
639
|
-
.command("verify")
|
|
640
|
-
.passThroughOptions()
|
|
641
|
-
.description("Verify a live server still matches a recorded cassette.")
|
|
642
|
-
.argument("<cassette>", "Path to a cassette JSON file.")
|
|
643
|
-
.argument("[command...]", "Server command and arguments to run.")
|
|
644
|
-
.option("--target <config>", "Path to a target config JSON file.")
|
|
645
|
-
.option("--no-color", "Disable colored output.")
|
|
646
|
-
.action(async (cassettePath, commandArgs, options) => {
|
|
647
|
-
const cassette = await loadCassette(cassettePath);
|
|
648
|
-
const target = options.target
|
|
649
|
-
? await readTargetConfig(options.target)
|
|
650
|
-
: targetFromCommand(commandArgs.length > 0 ? commandArgs : getPassthroughArgs());
|
|
651
|
-
process.stdout.write(`${c(ANSI.dim, "⟳")} Verifying ${c(ANSI.bold, target.targetId)} against cassette...\n`);
|
|
652
|
-
const { cassetteEntries } = await runTargetRecording(target, { invokeTools: true });
|
|
653
|
-
if (!cassetteEntries) {
|
|
654
|
-
process.stdout.write(`${c(ANSI.red, "✗")} Failed to record live session for comparison.\n`);
|
|
655
|
-
process.exitCode = 1;
|
|
656
|
-
return;
|
|
657
|
-
}
|
|
658
|
-
const verifyResult = compareResponses(cassette, cassetteEntries);
|
|
659
|
-
process.stdout.write("\n");
|
|
660
|
-
for (const entry of verifyResult.entries) {
|
|
661
|
-
if (entry.status === "pass") {
|
|
662
|
-
process.stdout.write(` ${c(ANSI.green, "✓")} ${entry.method}\n`);
|
|
663
|
-
}
|
|
664
|
-
else if (entry.status === "fail") {
|
|
665
|
-
process.stdout.write(` ${c(ANSI.red, "✗")} ${entry.method}\n`);
|
|
666
|
-
if (entry.diff) {
|
|
667
|
-
for (const line of entry.diff.split("\n")) {
|
|
668
|
-
process.stdout.write(` ${c(ANSI.dim, line)}\n`);
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
else {
|
|
673
|
-
process.stdout.write(` ${c(ANSI.yellow, "?")} ${entry.method} ${c(ANSI.dim, "(missing — server did not respond)")}\n`);
|
|
674
|
-
}
|
|
675
|
-
}
|
|
676
|
-
process.stdout.write("\n");
|
|
677
|
-
if (verifyResult.failed === 0 && verifyResult.missing === 0) {
|
|
678
|
-
process.stdout.write(c(ANSI.green, ` ✓ All ${verifyResult.passed} responses match cassette\n`));
|
|
679
|
-
}
|
|
680
|
-
else {
|
|
681
|
-
process.stdout.write(c(ANSI.red, ` ✗ ${verifyResult.failed} changed, ${verifyResult.missing} missing out of ${verifyResult.totalEntries} responses\n`));
|
|
682
|
-
process.exitCode = 1;
|
|
683
|
-
}
|
|
684
|
-
process.stdout.write("\n");
|
|
685
|
-
});
|
|
686
|
-
// ── Hidden legacy commands ────────────────────────────────────────────
|
|
687
|
-
program
|
|
688
|
-
.command("run", { hidden: true })
|
|
689
|
-
.description("Check one server and save a run artifact.")
|
|
690
|
-
.option("--target <config>", "Path to a target config JSON file.")
|
|
691
|
-
.option("--out-dir <directory>", "Directory for persisted run artifacts.", defaultRunsDirectory(process.cwd()))
|
|
692
|
-
.option("--watch", "Re-run checks on an interval.", false)
|
|
693
|
-
.option("--interval <seconds>", "Interval in seconds for watch mode.", "30")
|
|
694
|
-
.option("--invoke-tools", "Actually call safe tools to verify they execute.", false)
|
|
695
|
-
.option("--no-color", "Disable colored output.")
|
|
696
|
-
.action(async (options) => {
|
|
697
|
-
const target = await resolveTarget(options);
|
|
698
|
-
if (options.watch) {
|
|
699
|
-
await runWatchMode(target, options.outDir, parseInt(options.interval, 10) || 30);
|
|
700
|
-
return;
|
|
701
|
-
}
|
|
702
|
-
const artifact = await runTarget(target, { invokeTools: options.invokeTools });
|
|
703
|
-
const outPath = await writeRunArtifact(artifact, options.outDir);
|
|
704
|
-
const summary = renderTerminal(artifact);
|
|
705
|
-
process.stdout.write(`${summary}\nArtifact: ${outPath}\n`);
|
|
706
|
-
if (artifact.gate === "fail") {
|
|
707
|
-
process.exitCode = 1;
|
|
708
|
-
}
|
|
709
|
-
});
|
|
710
|
-
program
|
|
711
|
-
.command("check", { hidden: true })
|
|
712
|
-
.description("Run a single capability check.")
|
|
713
|
-
.argument("<capability>", "tools, prompts, resources, or tools-invoke.")
|
|
714
|
-
.option("--target <config>", "Path to a target config JSON file.")
|
|
715
|
-
.option("--no-color", "Disable colored output.")
|
|
716
|
-
.action(async (capability, options) => {
|
|
717
|
-
const validCapabilities = ["tools", "prompts", "resources", "tools-invoke"];
|
|
718
|
-
if (!validCapabilities.includes(capability)) {
|
|
719
|
-
throw new Error(`Invalid capability '${capability}'. Must be one of: ${validCapabilities.join(", ")}`);
|
|
720
|
-
}
|
|
721
|
-
const target = await resolveTarget(options);
|
|
722
|
-
const invokeTools = capability === "tools-invoke";
|
|
723
|
-
const artifact = await runTarget(target, { invokeTools });
|
|
724
|
-
const check = artifact.checks.find((ch) => ch.id === capability);
|
|
725
|
-
if (!check) {
|
|
726
|
-
throw new Error(`Check '${capability}' was not found in the run results.`);
|
|
727
|
-
}
|
|
728
|
-
const statusStr = colorStatus(check.status);
|
|
729
|
-
process.stdout.write(`${c(ANSI.bold, capability)}: ${statusStr}\n`);
|
|
730
|
-
process.stdout.write(`${check.message}\n`);
|
|
731
|
-
if (check.evidence.length > 0) {
|
|
732
|
-
for (const ev of check.evidence) {
|
|
733
|
-
if (ev.identifiers && ev.identifiers.length > 0) {
|
|
734
|
-
process.stdout.write(`Items: ${ev.identifiers.join(", ")}\n`);
|
|
735
|
-
}
|
|
736
|
-
if (ev.diagnostics && ev.diagnostics.length > 0) {
|
|
737
|
-
process.stdout.write(`Diagnostics: ${ev.diagnostics.join("; ")}\n`);
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
});
|
|
742
|
-
program
|
|
743
|
-
.command("report", { hidden: true })
|
|
744
|
-
.description("Render a run artifact.")
|
|
745
|
-
.requiredOption("--run <artifact>", "Run artifact JSON.")
|
|
746
|
-
.option("--format <format>", "terminal, markdown, json, or html", "terminal")
|
|
747
|
-
.option("--output <file>", "Write to file instead of stdout.")
|
|
748
|
-
.option("--no-color", "Disable colored output.")
|
|
749
|
-
.action(async (options) => {
|
|
750
|
-
const artifact = await readArtifact(options.run);
|
|
751
|
-
if (artifact.artifactType !== "run") {
|
|
752
|
-
throw new Error("The report command only accepts run artifacts.");
|
|
753
|
-
}
|
|
754
|
-
const output = formatOutput(artifact, options.format);
|
|
755
|
-
await writeOutput(output, options.format, options.output);
|
|
756
|
-
});
|
|
757
|
-
// ── score ────────────────────────────────────────────────────────────
|
|
758
|
-
program
|
|
759
|
-
.command("score")
|
|
760
|
-
.passThroughOptions()
|
|
761
|
-
.description("Score an MCP server's health (0-100).")
|
|
762
|
-
.argument("<command...>", "Server command and arguments to run.")
|
|
763
|
-
.option("--format <format>", "terminal, json, junit, markdown, html, or sarif", "terminal")
|
|
764
|
-
.option("--output <file>", "Write to file instead of stdout.")
|
|
765
|
-
.option("--no-color", "Disable colored output.")
|
|
766
|
-
.action(async (commandArgs, options) => {
|
|
767
|
-
const target = targetFromCommand(commandArgs);
|
|
768
|
-
process.stdout.write(`${c(ANSI.dim, "⟳")} Scoring ${c(ANSI.bold, target.targetId)}...\n\n`);
|
|
769
|
-
const artifact = await runTarget(target, { invokeTools: true, securityCheck: true });
|
|
770
|
-
await writeRunArtifact(artifact, defaultRunsDirectory(process.cwd()));
|
|
771
|
-
if (options.format !== "terminal") {
|
|
772
|
-
const output = formatOutput(artifact, options.format);
|
|
773
|
-
await writeOutput(output, options.format, options.output);
|
|
774
|
-
return;
|
|
775
|
-
}
|
|
776
|
-
const score = artifact.healthScore;
|
|
777
|
-
if (!score) {
|
|
778
|
-
process.stdout.write(" Could not compute health score.\n\n");
|
|
779
|
-
return;
|
|
780
|
-
}
|
|
781
|
-
const gradeColor = score.grade === "A" || score.grade === "B" ? ANSI.green
|
|
782
|
-
: score.grade === "C" ? ANSI.yellow
|
|
783
|
-
: ANSI.red;
|
|
784
|
-
process.stdout.write(c(ANSI.bold, ` MCP Health Score: ${c(gradeColor, `${score.overall}/100`)} (${c(gradeColor, score.grade)})\n\n`));
|
|
785
|
-
for (const dim of score.dimensions) {
|
|
786
|
-
const filled = Math.round(dim.score / 5);
|
|
787
|
-
const empty = 20 - filled;
|
|
788
|
-
const bar = "█".repeat(filled) + "░".repeat(empty);
|
|
789
|
-
const dimColor = dim.score >= 80 ? ANSI.green : dim.score >= 60 ? ANSI.yellow : ANSI.red;
|
|
790
|
-
const weightPct = Math.round(dim.weight * 100);
|
|
791
|
-
process.stdout.write(` ${dim.name.padEnd(22)} ${c(dimColor, bar)} ${String(dim.score).padStart(3)} ${c(ANSI.dim, `(weight: ${weightPct}%)`)}\n`);
|
|
792
|
-
}
|
|
793
|
-
process.stdout.write("\n");
|
|
794
|
-
// Show details for dimensions that aren't perfect
|
|
795
|
-
for (const dim of score.dimensions) {
|
|
796
|
-
if (dim.score < 100 && dim.details.length > 0) {
|
|
797
|
-
process.stdout.write(` ${c(ANSI.dim, dim.name + ":")}\n`);
|
|
798
|
-
for (const detail of dim.details) {
|
|
799
|
-
process.stdout.write(` ${c(ANSI.dim, "→")} ${detail}\n`);
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
process.stdout.write("\n");
|
|
804
|
-
if (artifact.gate === "fail") {
|
|
805
|
-
process.exitCode = 1;
|
|
806
|
-
}
|
|
807
|
-
});
|
|
808
|
-
// ── badge ───────────────────────────────────────────────────────────
|
|
809
|
-
program
|
|
810
|
-
.command("badge")
|
|
811
|
-
.passThroughOptions()
|
|
812
|
-
.description("Generate an SVG health score badge for your README.")
|
|
813
|
-
.argument("<command...>", "Server command and arguments to run.")
|
|
814
|
-
.option("--output <file>", "Write SVG to file (default: stdout).")
|
|
815
|
-
.option("--label <text>", "Badge label text.", "MCP Health")
|
|
816
|
-
.action(async (commandArgs, options) => {
|
|
817
|
-
const target = targetFromCommand(commandArgs);
|
|
818
|
-
process.stderr.write(`${c(ANSI.dim, "⟳")} Scoring ${c(ANSI.bold, target.targetId)}...\n`);
|
|
819
|
-
const artifact = await runTarget(target, { invokeTools: true, securityCheck: true });
|
|
820
|
-
const score = artifact.healthScore;
|
|
821
|
-
if (!score) {
|
|
822
|
-
process.stderr.write(" Could not compute health score.\n");
|
|
823
|
-
process.exitCode = 1;
|
|
824
|
-
return;
|
|
825
|
-
}
|
|
826
|
-
const svg = generateBadgeSvg({ score: score.overall, grade: score.grade, label: options.label });
|
|
827
|
-
if (options.output) {
|
|
828
|
-
await mkdir(path.dirname(options.output), { recursive: true });
|
|
829
|
-
await writeFile(options.output, svg, "utf8");
|
|
830
|
-
process.stderr.write(` Badge written to ${options.output}\n`);
|
|
831
|
-
}
|
|
832
|
-
else {
|
|
833
|
-
process.stdout.write(svg);
|
|
834
|
-
}
|
|
835
|
-
});
|
|
836
|
-
// ── telemetry ──────────────────────────────────────────────────────────
|
|
837
|
-
program
|
|
838
|
-
.command("telemetry")
|
|
839
|
-
.description("Manage anonymous usage telemetry.")
|
|
840
|
-
.argument("[action]", "enable, disable, stats, or status (default: status)")
|
|
841
|
-
.action(async (action) => {
|
|
842
|
-
const config = await loadTelemetryConfig();
|
|
843
|
-
const envDisabled = process.env["DO_NOT_TRACK"] === "1" ||
|
|
844
|
-
process.env["MCP_OBSERVATORY_TELEMETRY_DISABLED"] === "1";
|
|
845
|
-
if (action === "enable") {
|
|
846
|
-
config.telemetryEnabled = true;
|
|
847
|
-
await saveTelemetryConfig(config);
|
|
848
|
-
process.stdout.write(" Telemetry enabled.\n\n");
|
|
849
|
-
}
|
|
850
|
-
else if (action === "disable") {
|
|
851
|
-
config.telemetryEnabled = false;
|
|
852
|
-
await saveTelemetryConfig(config);
|
|
853
|
-
process.stdout.write(" Telemetry disabled.\n\n");
|
|
854
|
-
}
|
|
855
|
-
else if (action === "stats") {
|
|
856
|
-
const endpoint = process.env["MCP_OBSERVATORY_TELEMETRY_URL"] ?? "https://mcp-observatory-telemetry.kryptosai.workers.dev";
|
|
857
|
-
const token = process.env["MCP_OBSERVATORY_STATS_TOKEN"] ?? config.statsToken;
|
|
858
|
-
if (!token) {
|
|
859
|
-
process.stderr.write(" No stats token configured.\n");
|
|
860
|
-
process.stderr.write(" Set MCP_OBSERVATORY_STATS_TOKEN or add \"statsToken\" to ~/.mcp-observatory/config.json\n\n");
|
|
861
|
-
return;
|
|
862
|
-
}
|
|
863
|
-
const authHeaders = { Authorization: `Bearer ${token}` };
|
|
864
|
-
try {
|
|
865
|
-
const [all, others] = await Promise.all([
|
|
866
|
-
fetch(`${endpoint}/v1/stats`, { headers: authHeaders }).then(r => r.json()),
|
|
867
|
-
fetch(`${endpoint}/v1/stats?exclude=${config.sessionId}`, { headers: authHeaders }).then(r => r.json()),
|
|
868
|
-
]);
|
|
869
|
-
if (all.error) {
|
|
870
|
-
process.stderr.write(` Error: ${all.error}\n\n`);
|
|
871
|
-
return;
|
|
872
|
-
}
|
|
873
|
-
const totalAll = all.total ?? 0;
|
|
874
|
-
const totalOthers = others.total ?? 0;
|
|
875
|
-
const you = totalAll - totalOthers;
|
|
876
|
-
const sessionsAll = all.uniqueSessions ?? 0;
|
|
877
|
-
const sessionsOthers = others.uniqueSessions ?? 0;
|
|
878
|
-
process.stdout.write(` Total events: ${totalAll}\n`);
|
|
879
|
-
process.stdout.write(` Your events: ${you}\n`);
|
|
880
|
-
process.stdout.write(` Other events: ${totalOthers}\n`);
|
|
881
|
-
process.stdout.write(` Unique sessions: ${sessionsAll} (${sessionsOthers} excluding you)\n`);
|
|
882
|
-
process.stdout.write(` Last 24h: ${all.last24h ?? 0} (${others.last24h ?? 0} excluding you)\n\n`);
|
|
883
|
-
}
|
|
884
|
-
catch {
|
|
885
|
-
process.stderr.write(" Failed to fetch telemetry stats.\n\n");
|
|
886
|
-
}
|
|
887
|
-
}
|
|
888
|
-
else {
|
|
889
|
-
const effective = isTelemetryEnabled();
|
|
890
|
-
process.stdout.write(` Telemetry: ${effective ? "enabled" : "disabled"}\n`);
|
|
891
|
-
process.stdout.write(` Config: telemetryEnabled=${String(config.telemetryEnabled)}\n`);
|
|
892
|
-
if (envDisabled) {
|
|
893
|
-
process.stdout.write(` Override: disabled via environment variable\n`);
|
|
894
|
-
}
|
|
895
|
-
process.stdout.write(` Session: ${config.sessionId}\n\n`);
|
|
896
|
-
}
|
|
897
|
-
});
|
|
190
|
+
// Register all command modules
|
|
191
|
+
registerScanCommands(program, bin);
|
|
192
|
+
registerTestCommands(program);
|
|
193
|
+
registerDiffCommands(program);
|
|
194
|
+
registerRecordReplayCommands(program, bin);
|
|
195
|
+
registerWatchCommands(program);
|
|
196
|
+
registerServeCommands(program);
|
|
197
|
+
registerSuggestCommands(program);
|
|
198
|
+
registerScoreCommands(program);
|
|
199
|
+
registerLegacyCommands(program);
|
|
200
|
+
registerTelemetryCommands(program);
|
|
898
201
|
// ── smithery ─────────────────────────────────────────────────────────
|
|
899
202
|
const smitheryCmd = program
|
|
900
203
|
.command("smithery")
|
|
@@ -907,14 +210,14 @@ async function main() {
|
|
|
907
210
|
.option("--api-key <key>", "Smithery API key.")
|
|
908
211
|
.option("--base-url <url>", "Override Smithery API base URL.")
|
|
909
212
|
.action(async (qualifiedName, options) => {
|
|
910
|
-
const
|
|
213
|
+
const smithery = await import("./integrations/smithery.js");
|
|
911
214
|
const smitheryConfig = { apiKey: options.apiKey, baseUrl: options.baseUrl };
|
|
912
215
|
process.stdout.write(` Resolving ${qualifiedName} from Smithery...\n`);
|
|
913
|
-
const target = await resolveSmitheryTarget(qualifiedName, smitheryConfig);
|
|
216
|
+
const target = await smithery.resolveSmitheryTarget(qualifiedName, smitheryConfig);
|
|
914
217
|
process.stdout.write(` Running checks against ${target.targetId}...\n`);
|
|
915
218
|
const artifact = await runTarget(target, { securityCheck: options.security });
|
|
916
|
-
const submission =
|
|
917
|
-
const md =
|
|
219
|
+
const submission = smithery.generateSubmission(qualifiedName, artifact);
|
|
220
|
+
const md = smithery.renderSubmissionMarkdown(submission);
|
|
918
221
|
process.stdout.write(`\n${md}\n`);
|
|
919
222
|
});
|
|
920
223
|
smitheryCmd
|
|
@@ -926,14 +229,14 @@ async function main() {
|
|
|
926
229
|
.option("--api-key <key>", "Smithery API key.")
|
|
927
230
|
.option("--base-url <url>", "Override Smithery API base URL.")
|
|
928
231
|
.action(async (qualifiedName, options) => {
|
|
929
|
-
const
|
|
232
|
+
const smithery = await import("./integrations/smithery.js");
|
|
930
233
|
const smitheryConfig = { apiKey: options.apiKey, baseUrl: options.baseUrl };
|
|
931
234
|
process.stdout.write(` Resolving ${qualifiedName} from Smithery...\n`);
|
|
932
|
-
const target = await resolveSmitheryTarget(qualifiedName, smitheryConfig);
|
|
235
|
+
const target = await smithery.resolveSmitheryTarget(qualifiedName, smitheryConfig);
|
|
933
236
|
process.stdout.write(` Running checks against ${target.targetId}...\n`);
|
|
934
237
|
const artifact = await runTarget(target, { securityCheck: options.security });
|
|
935
|
-
const submission =
|
|
936
|
-
const md =
|
|
238
|
+
const submission = smithery.generateSubmission(qualifiedName, artifact);
|
|
239
|
+
const md = smithery.renderSubmissionMarkdown(submission);
|
|
937
240
|
if (options.output) {
|
|
938
241
|
await writeFile(options.output, md, "utf8");
|
|
939
242
|
process.stdout.write(` Report written to ${options.output}\n`);
|
|
@@ -950,12 +253,12 @@ async function main() {
|
|
|
950
253
|
.option("--api-key <key>", "Smithery API key.")
|
|
951
254
|
.option("--base-url <url>", "Override Smithery API base URL.")
|
|
952
255
|
.action(async (options) => {
|
|
953
|
-
const
|
|
256
|
+
const smithery = await import("./integrations/smithery.js");
|
|
954
257
|
const smitheryConfig = { apiKey: options.apiKey, baseUrl: options.baseUrl };
|
|
955
258
|
const top = parseInt(options.top, 10) || 10;
|
|
956
259
|
process.stdout.write(` Scanning top ${top} servers from Smithery registry...\n`);
|
|
957
|
-
const results = await batchScanServers((target) => runTarget(target, {}), smitheryConfig, { top });
|
|
958
|
-
const md = renderBatchReportMarkdown(results);
|
|
260
|
+
const results = await smithery.batchScanServers((target) => runTarget(target, {}), smitheryConfig, { top });
|
|
261
|
+
const md = smithery.renderBatchReportMarkdown(results);
|
|
959
262
|
if (options.output) {
|
|
960
263
|
await writeFile(options.output, md, "utf8");
|
|
961
264
|
process.stdout.write(` Batch report written to ${options.output}\n`);
|
|
@@ -976,174 +279,6 @@ async function main() {
|
|
|
976
279
|
recordEvent(buildEvent("command_run", commandName, "cli"));
|
|
977
280
|
await program.parseAsync(process.argv);
|
|
978
281
|
}
|
|
979
|
-
// ── Scan implementation ─────────────────────────────────────────────────────
|
|
980
|
-
async function runScan(bin, configPath, invokeTools, securityCheck) {
|
|
981
|
-
process.stdout.write(useColor() ? c(ANSI.cyan, LOGO) + ` ${c(ANSI.dim, `v${TOOL_VERSION}`)}\n\n` : LOGO + ` v${TOOL_VERSION}\n\n`);
|
|
982
|
-
if (configPath) {
|
|
983
|
-
try {
|
|
984
|
-
await access(configPath);
|
|
985
|
-
}
|
|
986
|
-
catch {
|
|
987
|
-
process.stdout.write(c(ANSI.red, ` ✗ Config file not found: ${configPath}\n\n`));
|
|
988
|
-
process.exitCode = 1;
|
|
989
|
-
return;
|
|
990
|
-
}
|
|
991
|
-
}
|
|
992
|
-
const targets = await scanForTargets(configPath);
|
|
993
|
-
if (targets.length === 0) {
|
|
994
|
-
process.stdout.write(c(ANSI.yellow, " No MCP servers found.\n\n"));
|
|
995
|
-
process.stdout.write(c(ANSI.dim, " Looked in ~/.claude.json, Claude Desktop config, .mcp.json\n\n"));
|
|
996
|
-
process.stdout.write(" Test a specific server:\n");
|
|
997
|
-
process.stdout.write(` ${c(ANSI.dim, "$")} ${c(ANSI.cyan, `${bin} test npx -y @modelcontextprotocol/server-filesystem .`)}\n\n`);
|
|
998
|
-
return;
|
|
999
|
-
}
|
|
1000
|
-
process.stdout.write(c(ANSI.bold, ` Found ${targets.length} MCP server${targets.length === 1 ? "" : "s"}:\n`));
|
|
1001
|
-
for (const t of targets) {
|
|
1002
|
-
process.stdout.write(` ${c(ANSI.cyan, "●")} ${c(ANSI.bold, t.config.targetId)} ${c(ANSI.dim, `← ${t.source}`)}\n`);
|
|
1003
|
-
}
|
|
1004
|
-
process.stdout.write("\n");
|
|
1005
|
-
const results = [];
|
|
1006
|
-
let passCount = 0;
|
|
1007
|
-
let failCount = 0;
|
|
1008
|
-
let totalTools = 0;
|
|
1009
|
-
let totalPrompts = 0;
|
|
1010
|
-
let totalResources = 0;
|
|
1011
|
-
for (const t of targets) {
|
|
1012
|
-
process.stdout.write(` ${c(ANSI.dim, "⟳")} Checking ${c(ANSI.bold, t.config.targetId)}...`);
|
|
1013
|
-
try {
|
|
1014
|
-
const artifact = await runTarget(t.config, { invokeTools, securityCheck });
|
|
1015
|
-
const toolsCheck = artifact.checks.find((ch) => ch.id === "tools");
|
|
1016
|
-
const promptsCheck = artifact.checks.find((ch) => ch.id === "prompts");
|
|
1017
|
-
const resourcesCheck = artifact.checks.find((ch) => ch.id === "resources");
|
|
1018
|
-
const toolCount = toolsCheck?.evidence[0]?.itemCount ?? 0;
|
|
1019
|
-
const promptCount = promptsCheck?.evidence[0]?.itemCount ?? 0;
|
|
1020
|
-
const resourceCount = resourcesCheck?.evidence[0]?.itemCount ?? 0;
|
|
1021
|
-
totalTools += toolCount;
|
|
1022
|
-
totalPrompts += promptCount;
|
|
1023
|
-
totalResources += resourceCount;
|
|
1024
|
-
const diagnostics = [];
|
|
1025
|
-
for (const check of artifact.checks) {
|
|
1026
|
-
if (check.status === "fail" || check.status === "partial") {
|
|
1027
|
-
diagnostics.push(`${check.id}: ${check.message}`);
|
|
1028
|
-
}
|
|
1029
|
-
}
|
|
1030
|
-
const gateIcon = artifact.gate === "pass" ? c(ANSI.green, " ✓") : c(ANSI.red, " ✗");
|
|
1031
|
-
process.stdout.write(`\r ${gateIcon} ${c(ANSI.bold, t.config.targetId)}${" ".repeat(Math.max(1, 40 - t.config.targetId.length))}`);
|
|
1032
|
-
process.stdout.write(`${c(ANSI.dim, `${toolCount} tools, ${promptCount} prompts, ${resourceCount} resources`)}\n`);
|
|
1033
|
-
if (artifact.fatalError) {
|
|
1034
|
-
process.stdout.write(` ${c(ANSI.red, "→")} ${artifact.fatalError.split("\n")[0]}\n`);
|
|
1035
|
-
}
|
|
1036
|
-
else if (artifact.gate === "fail" && diagnostics.length > 0) {
|
|
1037
|
-
process.stdout.write(` ${c(ANSI.dim, "→")} ${diagnostics[0]}\n`);
|
|
1038
|
-
}
|
|
1039
|
-
results.push({ targetId: t.config.targetId, gate: artifact.gate, toolCount, promptCount, resourceCount, diagnostics });
|
|
1040
|
-
if (artifact.gate === "pass")
|
|
1041
|
-
passCount++;
|
|
1042
|
-
else
|
|
1043
|
-
failCount++;
|
|
1044
|
-
}
|
|
1045
|
-
catch (error) {
|
|
1046
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
1047
|
-
let friendlyMsg = msg;
|
|
1048
|
-
if (msg.includes("ENOENT") || msg.includes("not found")) {
|
|
1049
|
-
const cmd = t.config.adapter === "http" ? t.config.url : t.config.command;
|
|
1050
|
-
friendlyMsg = `Could not start server — "${cmd}" not found. Is it installed?`;
|
|
1051
|
-
}
|
|
1052
|
-
else if (msg.includes("ECONNREFUSED")) {
|
|
1053
|
-
friendlyMsg = `Server is not running or refused the connection.`;
|
|
1054
|
-
}
|
|
1055
|
-
else if (msg.includes("timed out") || msg.includes("timeout")) {
|
|
1056
|
-
friendlyMsg = `Server took too long to respond.`;
|
|
1057
|
-
}
|
|
1058
|
-
process.stdout.write(`\r ${c(ANSI.red, "✗")} ${c(ANSI.bold, t.config.targetId)}\n`);
|
|
1059
|
-
process.stdout.write(` ${c(ANSI.red, friendlyMsg)}\n`);
|
|
1060
|
-
results.push({ targetId: t.config.targetId, gate: "fail", toolCount: 0, promptCount: 0, resourceCount: 0, error: friendlyMsg, diagnostics: [] });
|
|
1061
|
-
failCount++;
|
|
1062
|
-
}
|
|
1063
|
-
}
|
|
1064
|
-
// ── Summary ──────────────────────────────────────────────────────────
|
|
1065
|
-
process.stdout.write("\n");
|
|
1066
|
-
if (failCount === 0) {
|
|
1067
|
-
process.stdout.write(c(ANSI.green, ` ✓ All ${passCount} server${passCount === 1 ? "" : "s"} healthy`));
|
|
1068
|
-
process.stdout.write(c(ANSI.dim, ` — ${totalTools} tools, ${totalPrompts} prompts, ${totalResources} resources\n`));
|
|
1069
|
-
}
|
|
1070
|
-
else {
|
|
1071
|
-
process.stdout.write(c(ANSI.red, ` ✗ ${failCount} of ${passCount + failCount} server${passCount + failCount === 1 ? "" : "s"} failing`));
|
|
1072
|
-
if (totalTools > 0 || totalPrompts > 0 || totalResources > 0) {
|
|
1073
|
-
process.stdout.write(c(ANSI.dim, ` — ${totalTools} tools, ${totalPrompts} prompts, ${totalResources} resources found\n`));
|
|
1074
|
-
}
|
|
1075
|
-
else {
|
|
1076
|
-
process.stdout.write("\n");
|
|
1077
|
-
}
|
|
1078
|
-
}
|
|
1079
|
-
// Show diagnostics for failures or notable partials
|
|
1080
|
-
const issues = results.filter((r) => r.diagnostics.length > 0 && !r.error);
|
|
1081
|
-
if (issues.length > 0) {
|
|
1082
|
-
process.stdout.write("\n");
|
|
1083
|
-
for (const r of issues) {
|
|
1084
|
-
process.stdout.write(` ${c(ANSI.yellow, r.targetId)}:\n`);
|
|
1085
|
-
for (const d of r.diagnostics.slice(0, 3)) {
|
|
1086
|
-
process.stdout.write(` ${c(ANSI.dim, "→")} ${d}\n`);
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
}
|
|
1090
|
-
// ── Next step ────────────────────────────────────────────────────────
|
|
1091
|
-
process.stdout.write("\n");
|
|
1092
|
-
if (!invokeTools && totalTools > 0) {
|
|
1093
|
-
process.stdout.write(c(ANSI.dim, ` Next: ${c(ANSI.cyan, `${bin} scan deep`)} to also test that tools run\n`));
|
|
1094
|
-
}
|
|
1095
|
-
else {
|
|
1096
|
-
process.stdout.write(c(ANSI.dim, ` Run ${c(ANSI.cyan, `${bin} --help`)} for more commands\n`));
|
|
1097
|
-
}
|
|
1098
|
-
process.stdout.write("\n");
|
|
1099
|
-
if (failCount > 0) {
|
|
1100
|
-
process.exitCode = 1;
|
|
1101
|
-
}
|
|
1102
|
-
}
|
|
1103
|
-
// ── Utilities ───────────────────────────────────────────────────────────────
|
|
1104
|
-
function colorStatus(status) {
|
|
1105
|
-
switch (status) {
|
|
1106
|
-
case "pass":
|
|
1107
|
-
return c(ANSI.green, status);
|
|
1108
|
-
case "fail":
|
|
1109
|
-
return c(ANSI.red, status);
|
|
1110
|
-
case "partial":
|
|
1111
|
-
case "flaky":
|
|
1112
|
-
return c(ANSI.yellow, status);
|
|
1113
|
-
case "unsupported":
|
|
1114
|
-
case "skipped":
|
|
1115
|
-
return c(ANSI.dim, status);
|
|
1116
|
-
default:
|
|
1117
|
-
return status;
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
async function runWatchMode(target, outDir, intervalSeconds) {
|
|
1121
|
-
const { diffArtifacts: diff } = await import("./diff.js");
|
|
1122
|
-
process.stdout.write(`Watch mode: checking every ${intervalSeconds}s. Press Ctrl+C to stop.\n\n`);
|
|
1123
|
-
let previousArtifact = await runTarget(target);
|
|
1124
|
-
await writeRunArtifact(previousArtifact, outDir);
|
|
1125
|
-
process.stdout.write(`${renderTerminal(previousArtifact)}\n\n`);
|
|
1126
|
-
const loop = async () => {
|
|
1127
|
-
await new Promise((resolve) => setTimeout(resolve, intervalSeconds * 1000));
|
|
1128
|
-
const currentArtifact = await runTarget(target);
|
|
1129
|
-
const diffResult = diff(previousArtifact, currentArtifact);
|
|
1130
|
-
if (diffResult.summary.regressions > 0 || diffResult.summary.recoveries > 0 || diffResult.summary.added > 0 || diffResult.summary.removed > 0) {
|
|
1131
|
-
const outPath = await writeRunArtifact(currentArtifact, outDir);
|
|
1132
|
-
process.stdout.write(`\n--- Change detected at ${currentArtifact.createdAt} ---\n`);
|
|
1133
|
-
process.stdout.write(`${renderTerminal(diffResult)}\n`);
|
|
1134
|
-
process.stdout.write(`Artifact: ${outPath}\n\n`);
|
|
1135
|
-
}
|
|
1136
|
-
previousArtifact = currentArtifact;
|
|
1137
|
-
void loop();
|
|
1138
|
-
};
|
|
1139
|
-
void loop();
|
|
1140
|
-
await new Promise((resolve) => {
|
|
1141
|
-
process.on("SIGINT", () => {
|
|
1142
|
-
process.stdout.write("\nWatch mode stopped.\n");
|
|
1143
|
-
resolve();
|
|
1144
|
-
});
|
|
1145
|
-
});
|
|
1146
|
-
}
|
|
1147
282
|
void main().catch((error) => {
|
|
1148
283
|
const message = error instanceof Error ? error.message : String(error);
|
|
1149
284
|
let friendly = message;
|