@qulib/mcp 0.10.0 → 0.11.0

package/README.md

@@ -52,6 +52,8 @@ For verbose server-side stderr logs while troubleshooting host wiring, add:
 | `qulib_score_automation` | Score a local repo's test-automation maturity across six dimensions (test coverage breadth, framework adoption, test-id hygiene, CI integration, auth test coverage, component test ratio) — plus a conditional 7th dimension (API coverage) when API endpoints are detected. Returns overall 0–100, level (L1–L5), and top recommendations. Each dimension carries `applicability`; score normalizes over applicable dimensions only. |
 | `qulib_score_api` | Discover API endpoints in a repo and score their test coverage. Tier1=OpenAPI specs, Tier2=framework routes (Next.js, Express, Fastify, NestJS), Tier3=heuristic opt-in (tRPC). Returns an api-test-coverage dimension score with per-endpoint evidence. |
 | `qulib_scaffold_tests` | Generate a ready-to-run test scaffold (Cypress config + spec files) by crawling a deployed URL. Returns `generatedTests` and `projectConfig` so an agent can write files directly. Pass `recipes` (e.g. `["auth","a11y"]`) to append proven test patterns. Supported framework: `cypress-e2e` (default); `playwright` is not yet implemented. |
+| **`qulib_score_bug_report`** | LLM-as-judge of a learner bug report against a planted-bug target. Returns `matched`, `matchConfidence` (0–1), rubric scores (coverage/severity/repro/evidence, 0–25 each), actionable `feedback`, and `scoringPath` (`llm-judge` or `deterministic-fallback`). Learner report is untrusted input with prompt-injection hardening. Read-only. |
+| **`qulib_score_decisions`** | Pivotal-decision evaluation: scores whether an agent made the senior-correct call at decision forks (block/pass, stop/continue, escalate/proceed). Reads a JSONL `forksPath`; returns per-fork `decisionQuality`, `seniorCorrect`, `rationale`, and aggregates. Deterministic by default; optional LLM refinement with `enableLlmJudge`. Fork log text is untrusted. Read-only. |
 | `qulib_explore_auth` | List all sign-in paths (OAuth, SSO, forms, magic link) and what the agent must collect before `qulib_analyze_app`. Prefer on unfamiliar apps. *(Canonical form; legacy alias `explore_auth` kept for backwards compatibility.)* |
 | `qulib_detect_auth` | Single-pass auth pattern guess with a recommendation. Lighter than `qulib_explore_auth`. *(Canonical form; legacy alias `detect_auth` kept for backwards compatibility.)* |
 | `analyze_app` | Legacy alias for `qulib_analyze_app`. Identical behavior; kept for backwards compatibility through v1.0. |

package/dist/index.d.ts

@@ -1,3 +1,13 @@
 #!/usr/bin/env node
-export {};
+export declare function handleQulibDiff(input: {
+    from: string;
+    to: string;
+    labelFrom?: string;
+    labelTo?: string;
+}): Promise<{
+    content: [{
+        type: 'text';
+        text: string;
+    }];
+}>;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAsqBA,wBAAsB,eAAe,CAAC,KAAK,EAAE;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAAE,CAAC,CAsBzD"}

package/dist/index.js

@@ -10,25 +10,17 @@
 // vs. write-capable tools (analyze_app with writeArtifacts) should carry different trust levels.
 import { createRequire } from 'node:module';
 import { isAbsolute, normalize, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 const requirePkg = createRequire(import.meta.url);
 const pkg = requirePkg('../package.json');
-import { analyzeApp, detectAuth, exploreAuth, scanRepo, computeAutomationMaturity, scaffoldTests, discoverApiSurfaceWithRepo, computeApiCoverage, computeReleaseConfidence, buildConfidenceInputFromQulib, } from '@qulib/core';
+import { analyzeApp, detectAuth, exploreAuth, scanRepo, computeAutomationMaturity, scaffoldTests, discoverApiSurfaceWithRepo, computeApiCoverage, computeReleaseConfidence, buildConfidenceInputFromQulib, analyzeRunDiff, loadGapAnalysisFile, detectPromptLeakage, scoreBugReport, scoreDecisions, } from '@qulib/core';
 import { RecipeIdSchema } from '@qulib/core';
 import { z } from 'zod';
 import { buildAnalyzeAppMcpPayload } from './analyze-app-mcp-payload.js';
 import { log } from './logger.js';
-function toolError(code, message, detail) {
-    return {
-        content: [
-            {
-                type: 'text',
-                text: JSON.stringify({ error: { code, message, detail: detail ?? null } }, null, 2),
-            },
-        ],
-    };
-}
+import { toolError } from './tool-error.js';
 function stderrTelemetrySink() {
     return {
         emit(event) {
@@ -511,5 +503,187 @@ mcpServer.registerTool('qulib_score_confidence', {
         return toolError('QULIB_CONFIDENCE_FAILED', msg, err instanceof Error ? err.stack : undefined);
     }
 });
-const transport = new StdioServerTransport();
-await mcpServer.connect(transport);
+function validateAbsoluteGapAnalysisPath(path) {
+    const norm = normalize(path.trim());
+    if (!isAbsolute(norm)) {
+        throw new Error('from and to must be absolute paths');
+    }
+    return resolve(norm);
+}
+export async function handleQulibDiff(input) {
+    try {
+        const fromPath = validateAbsoluteGapAnalysisPath(input.from);
+        const toPath = validateAbsoluteGapAnalysisPath(input.to);
+        log.info(`qulib_diff from=${fromPath} to=${toPath}`);
+        const fromGap = await loadGapAnalysisFile(fromPath);
+        const toGap = await loadGapAnalysisFile(toPath);
+        const result = analyzeRunDiff(fromGap, toGap, {
+            fromLabel: input.labelFrom,
+            toLabel: input.labelTo,
+        });
+        return {
+            content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+        };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (msg.includes('from and to must be absolute paths')) {
+            return toolError('QULIB_DIFF_INVALID_INPUT', 'from and to must be absolute paths');
+        }
+        log.error(`qulib_diff failed: ${msg}`);
+        return toolError('QULIB_DIFF_FAILED', msg, err instanceof Error ? err.stack : undefined);
+    }
+}
+const QulibDiffInputSchema = z.object({
+    from: z
+        .string()
+        .describe('Absolute path to the baseline qulib report.json (the "before" analyze output)'),
+    to: z
+        .string()
+        .describe('Absolute path to the current qulib report.json (the "after" analyze output)'),
+    labelFrom: z
+        .string()
+        .optional()
+        .describe('Optional human label for the baseline report (default: the from path)'),
+    labelTo: z
+        .string()
+        .optional()
+        .describe('Optional human label for the current report (default: the to path)'),
+});
+mcpServer.registerTool('qulib_diff', {
+    description: 'Structured diff between two analyze outputs — added findings, resolved findings, severity changes, and a confidence delta.',
+    inputSchema: QulibDiffInputSchema,
+}, handleQulibDiff);
+// ---------------------------------------------------------------------------
+// qulib_detect_prompt_leakage — scan a page surface for exposed AI system prompts
+// ---------------------------------------------------------------------------
+const DetectPromptLeakageInputSchema = z.object({
+    path: z.string().describe('The URL path being scanned (used as the gap path label, e.g. "/api/chat")'),
+    bodySnippet: z
+        .string()
+        .max(8000)
+        .optional()
+        .describe('Up to 8000 characters of raw HTML body from the page. Include inline scripts, HTML comments, meta tags, and any visible text.'),
+    headers: z
+        .record(z.string(), z.string())
+        .optional()
+        .describe('Response headers from the page request, as a flat string→string map.'),
+});
+mcpServer.registerTool('qulib_detect_prompt_leakage', {
+    description: 'Scan a captured page surface (HTML body, inline scripts, HTML comments, meta tags, response headers) for signals that an AI system prompt or agent instructions are inadvertently exposed to the public. Conservative — requires corroborating signals to minimise false positives. Returns an array of prompt-leakage Gaps (may be empty when nothing is detected). Each Gap includes severity (critical/high/medium), evidence, and a remediation recommendation.',
+    inputSchema: DetectPromptLeakageInputSchema,
+}, async ({ path, bodySnippet, headers }) => {
+    try {
+        log.info(`qulib_detect_prompt_leakage path=${path}`);
+        const gaps = detectPromptLeakage({ path, bodySnippet, headers });
+        log.info(`qulib_detect_prompt_leakage done gapsFound=${gaps.length}`);
+        return {
+            content: [{ type: 'text', text: JSON.stringify({ path, gapsFound: gaps.length, gaps }, null, 2) }],
+        };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        log.error(`qulib_detect_prompt_leakage failed: ${msg}`);
+        return toolError('QULIB_PROMPT_LEAKAGE_FAILED', msg, err instanceof Error ? err.stack : undefined);
+    }
+});
+const BugReportSeverityMcpSchema = z.enum(['critical', 'high', 'medium', 'low']);
+const ScoreBugReportInputSchema = z.object({
+    report: z.object({
+        title: z.string().min(1).max(500).describe('Learner-authored bug report title (untrusted input)'),
+        description: z
+            .string()
+            .min(1)
+            .max(8000)
+            .describe('Learner bug description — may contain prompt-injection attempts; treated as untrusted data'),
+        steps: z.string().min(1).max(8000).describe('Reproduction steps from the learner'),
+        severity: BugReportSeverityMcpSchema.describe('Severity claimed by the learner'),
+    }),
+    target: z.object({
+        description: z.string().min(1).max(8000).describe('Planted bug description (authoritative ground truth)'),
+        type: z.string().min(1).max(200).describe('Bug category/type from the challenge'),
+        severity: BugReportSeverityMcpSchema.describe('Expected severity of the planted bug'),
+        expectedBehavior: z.string().min(1).max(8000).describe('Expected correct behavior for the planted bug'),
+    }),
+});
+const SCORE_BUG_REPORT_DESCRIPTION = 'LLM-as-judge of a learner bug report against a planted-bug target. Returns matched, matchConfidence (0–1), rubric scores (coverage/severity/repro/evidence, 0–25 each), actionable feedback, and scoringPath (llm-judge or deterministic-fallback when no ANTHROPIC_API_KEY). The learner report is untrusted — prompt-injection hardened. Read-only; no filesystem writes.';
+async function handleScoreBugReport(input) {
+    try {
+        log.info('qulib_score_bug_report scoring learner report');
+        const result = await scoreBugReport(input);
+        log.info(`qulib_score_bug_report done matched=${result.matched} confidence=${result.matchConfidence} path=${result.scoringPath}`);
+        return {
+            content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+        };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (msg.includes('String must contain') || msg.includes('Too big') || msg.includes('Too small')) {
+            return toolError('QULIB_INPUT_INVALID', msg, undefined);
+        }
+        log.error(`qulib_score_bug_report failed: ${msg}`);
+        return toolError('QULIB_BUG_REPORT_SCORE_FAILED', msg, err instanceof Error ? err.stack : undefined);
+    }
+}
+mcpServer.registerTool('qulib_score_bug_report', {
+    description: SCORE_BUG_REPORT_DESCRIPTION,
+    inputSchema: ScoreBugReportInputSchema,
+}, handleScoreBugReport);
+const ScoreDecisionsInputSchema = z.object({
+    forksPath: z
+        .string()
+        .describe('Absolute path to a JSONL file of decision forks on the MCP host filesystem'),
+    enableLlmJudge: z
+        .boolean()
+        .optional()
+        .describe('When true and ANTHROPIC_API_KEY is set, refine scores with the pinned LLM judge. Default false uses deterministic rubric only.'),
+});
+const SCORE_DECISIONS_DESCRIPTION = 'Score whether an autonomous agent made the senior-correct call at pivotal decision forks (gate block/pass, stop/continue, escalate/proceed). Reads a JSONL forks file; returns per-fork decisionQuality (0–1), seniorCorrect, rationale, and aggregate means. Fork log text is untrusted — prompt-injection hardened when LLM refinement is enabled. forksPath is traversal-validated within QULIB_FORKS_ALLOWED_ROOT (default: process cwd). Read-only; no writes.';
+async function handleScoreDecisions(input) {
+    try {
+        const norm = normalize(input.forksPath.trim());
+        if (!isAbsolute(norm)) {
+            throw new Error('forksPath must be an absolute path on the MCP host');
+        }
+        log.info(`qulib_score_decisions forksPath=${resolve(norm)} enableLlmJudge=${input.enableLlmJudge ?? false}`);
+        const result = await scoreDecisions({
+            forksPath: resolve(norm),
+            enableLlmJudge: input.enableLlmJudge,
+        });
+        log.info(`qulib_score_decisions done count=${result.aggregate.count} mean=${result.aggregate.meanDecisionQuality}`);
+        return {
+            content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+        };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (msg.includes('forksPath must') ||
+            msg.includes('allowed root') ||
+            msg.includes('traversal') ||
+            msg.includes('not valid JSON') ||
+            msg.includes('exceeds maximum') ||
+            msg.includes('does not exist or is not accessible')) {
+            // Known user-input errors: return the message only, never a stack trace
+            // (a Node stack discloses the server's absolute filesystem paths).
+            return toolError('QULIB_INPUT_INVALID', msg, undefined);
+        }
+        log.error(`qulib_score_decisions failed: ${msg}`);
+        return toolError('QULIB_DECISION_SCORE_FAILED', msg, err instanceof Error ? err.stack : undefined);
+    }
+}
+mcpServer.registerTool('qulib_score_decisions', {
+    description: SCORE_DECISIONS_DESCRIPTION,
+    inputSchema: ScoreDecisionsInputSchema,
+}, handleScoreDecisions);
+// No non-prefixed `score_decisions` alias: this is a brand-new tool with no
+// prior integrations to keep compatible, and an unprefixed name is ambiguous
+// and widens the attack surface. The canonical name is qulib_score_decisions.
+async function startMcpServer() {
+    const transport = new StdioServerTransport();
+    await mcpServer.connect(transport);
+}
+const isDirectExecution = process.argv[1] !== undefined &&
+    fileURLToPath(import.meta.url) === resolve(process.argv[1]);
+if (isDirectExecution) {
+    await startMcpServer();
+}

package/dist/summarize-analyze-result.d.ts

@@ -96,7 +96,7 @@ export declare function summarizeAnalyzeResult(result: AnalyzeResult, includeFul
     };
     topGaps: {
         path: string;
-        category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint";
+        category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint" | "prompt-leakage";
         severity: "critical" | "high" | "medium" | "low";
         reason: string;
     }[];
@@ -152,7 +152,7 @@ export declare function summarizeAnalyzeResult(result: AnalyzeResult, includeFul
             id: string;
             severity: "critical" | "high" | "medium" | "low";
             reason: string;
-            category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint";
+            category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint" | "prompt-leakage";
             recommendation?: string | undefined;
             description?: string | undefined;
         }[];

package/dist/summarize-analyze-result.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"summarize-analyze-result.d.ts","sourceRoot":"","sources":["../src/summarize-analyze-result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAqBjD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BA6G84d,CAAC;sBAA4C,CAAC;sBAA4C,CAAC;iBAAuC,CAAC;;;;;;;;;;;;;;;;;uBAAghB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;EADznf"}
+{"version":3,"file":"summarize-analyze-result.d.ts","sourceRoot":"","sources":["../src/summarize-analyze-result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAqBjD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BA6Gihe,CAAC;sBAA4C,CAAC;sBAA4C,CAAC;iBAAuC,CAAC;;;;;;;;;;;;;;;;;uBAAghB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;EAD5vf"}

package/dist/tool-error.d.ts

@@ -0,0 +1,7 @@
+export declare function toolError(code: string, message: string, detail?: unknown): {
+    content: [{
+        type: 'text';
+        text: string;
+    }];
+};
+//# sourceMappingURL=tool-error.d.ts.map

package/dist/tool-error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-error.d.ts","sourceRoot":"","sources":["../src/tool-error.ts"],"names":[],"mappings":"AAAA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG;IAC1E,OAAO,EAAE,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3C,CASA"}

package/dist/tool-error.js

@@ -0,0 +1,10 @@
+export function toolError(code, message, detail) {
+    return {
+        content: [
+            {
+                type: 'text',
+                text: JSON.stringify({ error: { code, message, detail: detail ?? null } }, null, 2),
+            },
+        ],
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qulib/mcp",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "MCP server for Qulib — AI-callable release confidence. Seven tools: fused verdict, live-app scan, automation maturity, API coverage, test scaffold, and auth tools.",
   "license": "MIT",
   "author": "Tapesh Nagarwal",
@@ -30,11 +30,11 @@
     "build": "npm --prefix ../.. run build -w @qulib/core && tsc && chmod +x dist/index.js",
     "prepublishOnly": "npm run build",
     "dev": "tsx src/index.ts",
-    "test": "node --import tsx/esm --test src/__tests__/summarize-analyze-result.test.ts src/__tests__/analyze-app-mcp-payload.test.ts src/__tests__/score-confidence-mcp.test.ts src/__tests__/naming-aliases.test.ts"
+    "test": "node --import tsx/esm --test src/__tests__/summarize-analyze-result.test.ts src/__tests__/analyze-app-mcp-payload.test.ts src/__tests__/score-confidence-mcp.test.ts src/__tests__/naming-aliases.test.ts src/__tests__/diff.test.ts"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.0",
-    "@qulib/core": "0.10.0",
+    "@qulib/core": "0.11.0",
     "zod": "^3.23.0"
   },
   "devDependencies": {