mcp-scorecard 0.1.0

package/dist/resolvers/npm-resolver.js

@@ -0,0 +1,100 @@
+/**
+ * npm resolver — given a package name (optionally @version), runs `npm pack`
+ * into a fresh temp dir, unpacks the tarball, locates the bin from
+ * package.json, and returns a ResolvedTarget ready for probing.
+ *
+ * Security:
+ *   - works in /tmp/scorecard-work/probe-<random>/, never in the user's HOME.
+ *   - uses --no-fund --no-audit to keep output clean and avoid any phone-home.
+ */
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve as pathResolve } from 'node:path';
+const SCRATCH_ROOT = '/tmp/scorecard-work';
+function ensureScratchRoot() {
+    if (!existsSync(SCRATCH_ROOT)) {
+        try {
+            mkdirSync(SCRATCH_ROOT, { recursive: true });
+            return SCRATCH_ROOT;
+        }
+        catch {
+            // fall back to OS tmpdir if /tmp/scorecard-work is unwritable
+        }
+    }
+    if (existsSync(SCRATCH_ROOT))
+        return SCRATCH_ROOT;
+    return tmpdir();
+}
+function pickBin(pkg) {
+    const bin = pkg.bin;
+    if (typeof bin === 'string')
+        return bin;
+    if (bin && typeof bin === 'object') {
+        const entries = Object.values(bin);
+        return entries[0];
+    }
+    // some MCPs export `main` and rely on `node dist/index.js`
+    if (typeof pkg.main === 'string')
+        return pkg.main;
+    return undefined;
+}
+export async function resolveNpmPackage(spec) {
+    const scratch = ensureScratchRoot();
+    const dest = mkdtempSync(join(scratch, 'probe-'));
+    const pack = spawnSync('npm', ['pack', spec, '--pack-destination', dest, '--no-fund', '--no-audit'], {
+        encoding: 'utf8',
+        stdio: ['ignore', 'pipe', 'pipe']
+    });
+    if (pack.status !== 0) {
+        throw new Error(`npm pack failed for ${spec}: ${pack.stderr || pack.stdout}`);
+    }
+    const tarball = readdirSync(dest).find((f) => f.endsWith('.tgz'));
+    if (!tarball)
+        throw new Error(`npm pack produced no .tgz under ${dest}`);
+    const tarPath = join(dest, tarball);
+    const extractDir = join(dest, 'pkg');
+    mkdirSync(extractDir, { recursive: true });
+    const tar = spawnSync('tar', ['xzf', tarPath, '-C', extractDir, '--strip-components=1'], {
+        encoding: 'utf8'
+    });
+    if (tar.status !== 0) {
+        throw new Error(`tar extract failed for ${tarPath}: ${tar.stderr}`);
+    }
+    const pkgJsonPath = join(extractDir, 'package.json');
+    if (!existsSync(pkgJsonPath)) {
+        throw new Error(`Extracted package has no package.json at ${pkgJsonPath}`);
+    }
+    const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf8'));
+    // Install production deps inside the extracted tarball so the launched
+    // binary can `import` its declared dependencies. We skip devDeps,
+    // scripts, fund/audit chatter, and any postinstall side-effects.
+    const hasDeps = pkg.dependencies && Object.keys(pkg.dependencies).length > 0;
+    if (hasDeps) {
+        const inst = spawnSync('npm', ['install', '--omit=dev', '--no-fund', '--no-audit', '--ignore-scripts', '--no-package-lock'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], cwd: extractDir });
+        if (inst.status !== 0) {
+            throw new Error(`npm install (prod deps) failed in ${extractDir}: ${inst.stderr || inst.stdout}`);
+        }
+    }
+    const binRel = pickBin(pkg);
+    if (!binRel) {
+        throw new Error(`Package ${pkg.name} has no bin or main — cannot launch.`);
+    }
+    const binAbs = pathResolve(extractDir, binRel);
+    if (!existsSync(binAbs)) {
+        throw new Error(`Resolved bin does not exist after extract: ${binAbs}`);
+    }
+    // Ensure it's a file, not a dir.
+    if (!statSync(binAbs).isFile()) {
+        throw new Error(`Resolved bin is not a file: ${binAbs}`);
+    }
+    return {
+        displayName: pkg.name,
+        version: pkg.version,
+        command: 'node',
+        args: [binAbs],
+        packageDir: extractDir,
+        packageJson: pkg
+    };
+}
+//# sourceMappingURL=npm-resolver.js.map

package/dist/resolvers/npm-resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"npm-resolver.js","sourceRoot":"","sources":["../../src/resolvers/npm-resolver.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAClG,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAC;AAGzD,MAAM,YAAY,GAAG,qBAAqB,CAAC;AAE3C,SAAS,iBAAiB;IACxB,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,OAAO,YAAY,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;QAChE,CAAC;IACH,CAAC;IACD,IAAI,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,YAAY,CAAC;IAClD,OAAO,MAAM,EAAE,CAAC;AAClB,CAAC;AAED,SAAS,OAAO,CAAC,GAA4B;IAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC;IACpB,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,GAA6B,CAAC,CAAC;QAC7D,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IACD,2DAA2D;IAC3D,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC,IAAc,CAAC;IAC5D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY;IAClD,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAElD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE;QACnG,QAAQ,EAAE,MAAM;QAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;IAEzE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACrC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,sBAAsB,CAAC,EAAE;QACvF,QAAQ,EAAE,MAAM;KACjB,CAAC,CAAC;IACH,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4CAA4C,WAAW,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;IAE1D,uEAAuE;IACvE,kEAAkE;IAClE,iEAAiE;IACjE,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC7E,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,SAAS,CACpB,KAAK,EACL,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,CAAC,EAC7F,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CACzE,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CACb,qCAAqC,UAAU,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CACjF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,sCAAsC,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC/C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,iCAAiC;IACjC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,WAAW,EAAE,GAAG,CAAC,IAAc;QAC/B,OAAO,EAAE,GAAG,CAAC,OAA6B;QAC1C,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,UAAU,EAAE,UAAU;QACtB,WAAW,EAAE,GAAG;KACjB,CAAC;AACJ,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Shared types: the snapshot the probe captures and the shape of a check.
+ * Kept deliberately minimal so the audit pipeline stays portable.
+ */
+export interface ToolSnapshot {
+    name: string;
+    description?: string;
+    inputSchema?: unknown;
+    annotations?: Record<string, unknown>;
+}
+export interface ResourceSnapshot {
+    uri: string;
+    name?: string;
+    description?: string;
+    mimeType?: string;
+}
+export interface PromptSnapshot {
+    name: string;
+    description?: string;
+}
+/**
+ * What the probe captures from one MCP target. We carry just enough to
+ * answer every check without round-tripping back to the target server.
+ */
+export interface ProbeSnapshot {
+    serverInfo: {
+        name?: string;
+        version?: string;
+    };
+    tools: ToolSnapshot[];
+    resources: ResourceSnapshot[];
+    prompts: PromptSnapshot[];
+    /** Counted only — never the actual payload (privacy). */
+    agentManifest?: {
+        has_recommended_first_calls: boolean;
+        recommended_first_calls_count: number;
+        has_standard_tools: boolean;
+        standard_tools_count: number;
+        raw_keys: string[];
+    };
+}
+/**
+ * Where the audit target's source files live on disk. Some checks (smoke
+ * test detection) read from this; the probe only needs `command + args`.
+ */
+export interface ResolvedTarget {
+    /** Display label used in output. */
+    displayName: string;
+    /** What version, if known. */
+    version?: string;
+    /** Command to spawn for probe (usually 'node'). */
+    command: string;
+    /** Args to pass to command. */
+    args: string[];
+    /** Directory holding the package source — used for file-presence checks. */
+    packageDir: string;
+    /** package.json contents, if any. */
+    packageJson?: Record<string, unknown>;
+}
+export type CheckId = 'schema_validity' | 'tool_naming' | 'privacy_modes' | 'mutation_gating' | 'agent_manifest' | 'smoke_test' | 'resources' | 'tool_descriptions' | 'annotations' | 'manifest_discoverability';
+export interface CheckResult {
+    id: CheckId;
+    label: string;
+    score: number;
+    status: 'pass' | 'warn' | 'fail';
+    /** One-line headline for the scorecard table. */
+    summary: string;
+    /** Optional bullets shown in the Details section. */
+    details: string[];
+    /** Optional suggestions for Suggested fixes section. */
+    fixes: string[];
+}
+export interface AuditReport {
+    target: {
+        displayName: string;
+        version?: string;
+        serverName?: string;
+        serverVersion?: string;
+    };
+    totalScore: number;
+    checks: CheckResult[];
+    generatedAt: string;
+    scorecardVersion: string;
+}

package/dist/types.js

@@ -0,0 +1,6 @@
+/**
+ * Shared types: the snapshot the probe captures and the shape of a check.
+ * Kept deliberately minimal so the audit pipeline stays portable.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "mcp-scorecard",
+  "version": "0.1.0",
+  "description": "Agent-readiness scorecard for any MCP server. Probes a target, runs 10 checks, outputs a 0-100 score with actionable findings.",
+  "type": "module",
+  "bin": {
+    "mcp-scorecard": "dist/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc --noEmit -p tsconfig.json",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts",
+    "test:checks": "node scripts/test-checks.mjs",
+    "test:self": "node scripts/smoke-self.mjs",
+    "test": "npm run typecheck && npm run build && npm run test:checks && npm run test:self",
+    "prepublishOnly": "npm test"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "scorecard",
+    "audit",
+    "agent-readiness",
+    "quality"
+  ],
+  "author": "David Mosiah",
+  "license": "MIT",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.21.0",
+    "ajv": "^8.17.1",
+    "zod": "^4.4.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsx": "^4.20.6",
+    "typescript": "^5.6.0"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/davidmosiah/mcp-scorecard.git"
+  },
+  "bugs": {
+    "url": "https://github.com/davidmosiah/mcp-scorecard/issues",
+    "email": "support@delx.ai"
+  },
+  "homepage": "https://github.com/davidmosiah/mcp-scorecard",
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "dist/index.js"
+}