@qulib/mcp 0.2.1 → 0.3.0

package/README.md

@@ -6,8 +6,9 @@
 
 Tools:
 
-- **`analyze_app(url, auth?)`** — full quality scan (optional form-login auth).
-- **`detect_auth(url, timeoutMs?)`** — detect whether the site uses form login, OAuth, magic link, etc., and get a plain-language recommendation (including when to use manual `qulib auth init` and storage state).
+- **`explore_auth(url, timeoutMs?)`** — list all sign-in paths (OAuth, unknown SSO heuristics, forms, magic link) and what the agent must collect before `analyze_app`. Prefer this on unfamiliar apps.
+- **`analyze_app`** — quality scan (optional form-login or storage-state auth). **Default payload is summary-first:** `summary`, `topGaps`, `costIntelligenceSummary`, `nextDeterministicChecks`, small previews. Set **`includeFullReport: true`** for the full `analyzeApp` result (all scenarios). Optional harness overrides: **`llmMaxOutputTokensPerCall`**, **`llmTokenBudget`** (legacy), **`testGenerationLimit`**, **`enableLlmScenarios`** (default true when omitted).
+- **`detect_auth(url, timeoutMs?)`** — single-pattern auth guess with a short recommendation (lighter than `explore_auth`).
 
 Returns from `analyze_app`:
 
@@ -52,6 +53,38 @@ This is a one-time step. You'll only need to do it again if Playwright's browser
 
 If you skip this step, the first tool call will return a clear error telling you to run the command.
 
+## Agentic auth exploration (`explore_auth`)
+
+On unfamiliar apps, call **`explore_auth`** before **`analyze_app`**. The response lists each sign-in path (curated public OAuth/SSO, password forms, magic-link wording, and **heuristic** unknown buttons such as tenant-specific SSO). Each path includes **`requirements`** (e.g. storage-state vs credentials) and **`suggestedAgentBehavior`**.
+
+When the model sees **`unrecognizedButtons`**, it can ask the user to register a label on the **MCP host** with the CLI:
+
+`qulib auth providers add --id <kebab-id> --label "..." --pattern "..."` — patterns are saved under **`~/.qulib/providers.json`** and merged with the built-in list on the next `explore_auth` / `explore-auth`. Nothing is auto-written without an explicit `providers add`.
+
+## Compact vs full `analyze_app` response
+
+| | Default (`includeFullReport` omitted or false) | `includeFullReport: true` |
+|--|--|--|
+| Size | Small: top gaps, cost summary, next checks | Full `gapAnalysis` with every scenario |
+| When to use | Routine agent turns, chat context limits | Deep dives, exporting full scenario JSON |
+
+Example (full):
+
+```json
+{ "url": "https://example.com", "includeFullReport": true }
+```
+
+Example (tighter LLM envelope from MCP):
+
+```json
+{
+  "url": "https://example.com",
+  "llmMaxOutputTokensPerCall": 2048,
+  "testGenerationLimit": 5,
+  "enableLlmScenarios": true
+}
+```
+
 ## Example usage
 
 Ask Claude:

package/dist/compact-analyze-payload.d.ts

@@ -0,0 +1,144 @@
+import type { AnalyzeResult } from '@qulib/core';
+export declare function buildCompactAnalyzePayload(result: AnalyzeResult, includeFullReport: boolean): AnalyzeResult | {
+    includeFullReport: boolean;
+    note: string;
+    detectedAuth?: {
+        type: "unknown" | "form-login" | "none" | "oauth" | "magic-link";
+        loginUrl: string | null;
+        hasAuth: boolean;
+        provider: string | null;
+        observedSelectors: {
+            usernameSelector: string | null;
+            passwordSelector: string | null;
+            submitSelector: string | null;
+        } | null;
+        oauthButtons: {
+            provider: string;
+            text: string;
+        }[];
+        recommendation: string;
+    } | undefined;
+    summary: {
+        status: import("@qulib/core").AnalyzeStatus;
+        coverageScore: number | null;
+        releaseConfidence: number | null;
+        mode: "url-only" | "url-repo" | "auth-required";
+        coveragePagesScanned: number;
+        coverageBudgetExceeded: boolean;
+        coverageWarning: "auth-required" | "budget-exceeded" | "low-coverage" | "navigation-failures" | null;
+        gapCount: number;
+        scenarioCount: number;
+        generatedTestCount: number;
+        publicSurface: {
+            pageCount: number;
+            gapCount: number;
+            accessibilityViolationCount: number;
+            brokenLinkCount: number;
+        } | null;
+    };
+    topGaps: {
+        path: string;
+        category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage";
+        severity: "high" | "medium" | "low" | "critical";
+        reason: string;
+    }[];
+    costIntelligenceSummary: {
+        maxOutputTokensPerLlmCall: number;
+        usageDataQuality: "none" | "actual" | "estimated" | "mixed";
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        budgetWarningCount: number;
+        maturityLevel: number;
+        maturityLabel: string;
+    } | null;
+    costIntelligence: {
+        maxOutputTokensPerLlmCall: number;
+        budgetRole: "max-output-tokens-per-llm-call";
+        records: {
+            provider: string;
+            model: string;
+            inputTokens: number;
+            outputTokens: number;
+            operationType: "scenario-generation";
+            timestamp: string;
+            dataQuality: "none" | "actual" | "estimated" | "mixed";
+            estimatedCostUsd?: number | undefined;
+            promptHash?: string | undefined;
+            resultHash?: string | undefined;
+            notes?: string | undefined;
+        }[];
+        budgetWarnings: string[];
+        usageSummary: {
+            dataQuality: "none" | "actual" | "estimated" | "mixed";
+            totalInputTokens: number;
+            totalOutputTokens: number;
+        };
+        repeatedOperations: {
+            recommendation: string;
+            promptHash: string;
+            count: number;
+        }[];
+        deterministicMaturity: {
+            label: string;
+            level: number;
+            rationale: string;
+            ceilingNote?: string | undefined;
+        };
+        conversionRecommendations: string[];
+    } | null;
+    nextDeterministicChecks: string[];
+    gapAnalysisPreview: {
+        analyzedAt: string;
+        gapsSample: {
+            path: string;
+            id: string;
+            severity: "high" | "medium" | "low" | "critical";
+            reason: string;
+            category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage";
+            recommendation?: string | undefined;
+            description?: string | undefined;
+        }[];
+        scenariosOmitted: number;
+        generatedTestsOmitted: number;
+    };
+    routeInventorySummary: {
+        scannedAt: string;
+        baseUrl: string;
+        routeCount: number;
+        pagesSkipped: number;
+        budgetExceeded: boolean;
+    };
+    repoInventory: {
+        scannedAt: string;
+        routes: {
+            path: string;
+            method: "unknown" | "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+            file: string;
+        }[];
+        repoPath: string;
+        testFiles: {
+            type: "playwright" | "cypress-e2e" | "cypress-component" | "jest" | "vitest" | "other";
+            file: string;
+            coveredPaths: string[];
+        }[];
+        missingTestIds: string[];
+        cypressStructure: {
+            detected: boolean;
+            hasCommandsFile: boolean;
+            existingE2eFiles: string[];
+            existingComponentFiles: string[];
+            e2eFolder?: string | undefined;
+            componentFolder?: string | undefined;
+            fixturesFolder?: string | undefined;
+            supportFolder?: string | undefined;
+        };
+    } | null;
+    decisionLogPreview: {
+        timestamp: string;
+        reason: string;
+        phase: "observe" | "think" | "act" | "harness";
+        decision: string;
+        metadata?: Record<string, unknown> | undefined;
+    }[];
+};
+//# sourceMappingURL=compact-analyze-payload.d.ts.map

package/dist/compact-analyze-payload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compact-analyze-payload.d.ts","sourceRoot":"","sources":["../src/compact-analyze-payload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAqBjD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BA0E0igB,CAAC;sBAA4C,CAAC;sBAA4C,CAAC;iBAAuC,CAAC;;;;;;;;;;;;;;;;;uBAAghB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAAt7e,CAAC;2BAA6C,CAAC;0BAA4C,CAAC;yBAA2C,CAAC;;;;;;;;;;EAD3+C"}

package/dist/compact-analyze-payload.js

@@ -0,0 +1,87 @@
+const severityOrder = { critical: 0, high: 1, medium: 2, low: 3 };
+function topGapsBySeverity(gaps, limit) {
+    return [...gaps].sort((a, b) => severityOrder[a.severity] - severityOrder[b.severity]).slice(0, limit);
+}
+function nextDeterministicChecks(gaps, conversion) {
+    const out = [];
+    const byCat = new Map();
+    for (const g of gaps) {
+        byCat.set(g.category, (byCat.get(g.category) ?? 0) + 1);
+    }
+    for (const [cat, n] of [...byCat.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3)) {
+        out.push(`Add or tighten deterministic coverage for **${cat}** (${n} gap(s) in this scan).`);
+    }
+    out.push(...conversion.slice(0, 2));
+    return out.slice(0, 5);
+}
+export function buildCompactAnalyzePayload(result, includeFullReport) {
+    if (includeFullReport) {
+        return result;
+    }
+    const g = result.gapAnalysis;
+    const ci = g.costIntelligence;
+    const top = topGapsBySeverity(result.gaps, 5);
+    const costSummary = ci
+        ? {
+            maxOutputTokensPerLlmCall: ci.maxOutputTokensPerLlmCall,
+            usageDataQuality: ci.usageSummary.dataQuality,
+            totalInputTokens: ci.usageSummary.totalInputTokens,
+            totalOutputTokens: ci.usageSummary.totalOutputTokens,
+            budgetWarningCount: ci.budgetWarnings.length,
+            maturityLevel: ci.deterministicMaturity.level,
+            maturityLabel: ci.deterministicMaturity.label,
+        }
+        : null;
+    const ps = result.publicSurface;
+    return {
+        summary: {
+            status: result.status,
+            coverageScore: result.coverageScore,
+            releaseConfidence: g.releaseConfidence,
+            mode: g.mode,
+            coveragePagesScanned: g.coveragePagesScanned,
+            coverageBudgetExceeded: g.coverageBudgetExceeded,
+            coverageWarning: g.coverageWarning ?? null,
+            gapCount: g.gaps.length,
+            scenarioCount: g.scenarios.length,
+            generatedTestCount: g.generatedTests.length,
+            publicSurface: ps === null
+                ? null
+                : {
+                    pageCount: ps.pages.length,
+                    gapCount: ps.gaps.length,
+                    accessibilityViolationCount: ps.accessibilityViolations.length,
+                    brokenLinkCount: ps.brokenLinks.length,
+                },
+        },
+        topGaps: top.map((x) => ({
+            path: x.path,
+            category: x.category,
+            severity: x.severity,
+            reason: x.reason,
+        })),
+        costIntelligenceSummary: costSummary,
+        costIntelligence: ci ?? null,
+        nextDeterministicChecks: ci
+            ? nextDeterministicChecks(result.gaps, ci.conversionRecommendations)
+            : nextDeterministicChecks(result.gaps, []),
+        gapAnalysisPreview: {
+            analyzedAt: g.analyzedAt,
+            gapsSample: g.gaps.slice(0, 8),
+            scenariosOmitted: g.scenarios.length,
+            generatedTestsOmitted: g.generatedTests.length,
+        },
+        routeInventorySummary: {
+            scannedAt: result.routeInventory.scannedAt,
+            baseUrl: result.routeInventory.baseUrl,
+            routeCount: result.routeInventory.routes.length,
+            pagesSkipped: result.routeInventory.pagesSkipped,
+            budgetExceeded: result.routeInventory.budgetExceeded,
+        },
+        repoInventory: result.repoInventory,
+        decisionLogPreview: result.decisionLog.slice(-8),
+        ...(result.detectedAuth !== undefined && { detectedAuth: result.detectedAuth }),
+        includeFullReport: false,
+        note: 'Summary-first payload. Pass includeFullReport: true for full gapAnalysis (all scenarios and generatedTests).',
+    };
+}

package/dist/index.js

@@ -2,8 +2,16 @@
 import { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
-import { analyzeApp, detectAuth } from '@qulib/core';
+import { analyzeApp, detectAuth, exploreAuth } from '@qulib/core';
 import { z } from 'zod';
+import { buildCompactAnalyzePayload } from './compact-analyze-payload.js';
+import { log } from './logger.js';
+const mcpProgressLog = {
+    info: (message) => log.info(message),
+    warn: (message) => log.warn(message),
+    error: (message) => log.error(message),
+    debug: (message) => log.debug(message),
+};
 const FormLoginMcpAuthSchema = z.object({
     type: z.literal('form-login'),
     loginUrl: z.string().url(),
@@ -23,6 +31,11 @@ const AnalyzeInputSchema = z.object({
     maxPagesToScan: z.number().int().min(1).max(50).optional(),
     timeoutMs: z.number().int().positive().optional(),
     auth: z.discriminatedUnion('type', [FormLoginMcpAuthSchema, StorageStateMcpAuthSchema]).optional(),
+    includeFullReport: z.boolean().optional(),
+    llmTokenBudget: z.number().int().positive().optional(),
+    llmMaxOutputTokensPerCall: z.number().int().positive().optional(),
+    testGenerationLimit: z.number().int().positive().max(50).optional(),
+    enableLlmScenarios: z.boolean().optional(),
 });
 const server = new Server({
     name: 'qulib-mcp',
@@ -34,9 +47,21 @@ const server = new Server({
 });
 server.setRequestHandler(ListToolsRequestSchema, async () => ({
     tools: [
+        {
+            name: 'explore_auth',
+            description: 'Use this BEFORE analyze_app when scanning unfamiliar apps. Returns all detected sign-in paths with per-path requirements describing what credentials or actions the agent must collect from the user before calling analyze_app. Combines built-in OAuth/SSO labels, user-local patterns from ~/.qulib/providers.json, and heuristic unknown buttons.',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'Full URL of the deployed app or login page' },
+                    timeoutMs: { type: 'number', description: 'Navigation timeout in milliseconds (default 20000)' },
+                },
+                required: ['url'],
+            },
+        },
         {
             name: 'analyze_app',
-            description: 'Analyze a deployed web app for quality gaps. Returns a release confidence score (0-100), accessibility violations, broken links, and prioritized risks. Supports optional form-login or storage-state (Playwright) authentication.',
+            description: 'Analyze a deployed web app for quality gaps. Default response is summary-first (top gaps, cost summary, next checks). Set includeFullReport for the full gapAnalysis. Optional llmMaxOutputTokensPerCall / llmTokenBudget (legacy), testGenerationLimit, enableLlmScenarios align with @qulib/core HarnessConfig.',
             inputSchema: {
                 type: 'object',
                 properties: {
@@ -78,6 +103,23 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
                             },
                         ],
                     },
+                    includeFullReport: {
+                        type: 'boolean',
+                        description: 'When true, returns the full analyzeApp payload including all scenarios. Default false returns a summary-first shape.',
+                    },
+                    llmTokenBudget: {
+                        type: 'number',
+                        description: 'Legacy per-completion max output tokens (same as HarnessConfig.llmTokenBudget). Prefer llmMaxOutputTokensPerCall when both are set.',
+                    },
+                    llmMaxOutputTokensPerCall: {
+                        type: 'number',
+                        description: 'Optional override for per-completion max output tokens (maps to HarnessConfig.llmMaxOutputTokensPerCall).',
+                    },
+                    testGenerationLimit: { type: 'number', description: 'Max gaps fed into scenario generation (default 5).' },
+                    enableLlmScenarios: {
+                        type: 'boolean',
+                        description: 'When false, never calls an LLM for scenarios (default true when omitted).',
+                    },
                 },
                 required: ['url'],
             },
@@ -97,6 +139,20 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
     ],
 }));
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    if (request.params.name === 'explore_auth') {
+        const { url, timeoutMs } = z
+            .object({
+            url: z.string().url(),
+            timeoutMs: z.number().int().positive().optional(),
+        })
+            .parse(request.params.arguments ?? {});
+        log.info(`explore_auth tool url=${url} timeoutMs=${timeoutMs ?? 20000}`);
+        const result = await exploreAuth(url, timeoutMs, mcpProgressLog);
+        log.info(`explore_auth tool done authRequired=${result.authRequired} paths=${result.authPaths.length}`);
+        return {
+            content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+        };
+    }
     if (request.params.name === 'detect_auth') {
         const { url, timeoutMs } = z
             .object({
@@ -104,7 +160,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             timeoutMs: z.number().int().positive().optional(),
         })
             .parse(request.params.arguments ?? {});
-        const result = await detectAuth(url, timeoutMs);
+        log.info(`detect_auth tool url=${url} timeoutMs=${timeoutMs ?? 15000}`);
+        const result = await detectAuth(url, timeoutMs, mcpProgressLog);
+        const providerSummary = result.oauthButtons.length > 0
+            ? result.oauthButtons.map((b) => b.provider).join(', ')
+            : result.provider ?? 'none';
+        log.info(`detect_auth tool done type=${result.type} providers=${providerSummary} automatable=${result.type === 'form-login'}`);
         return {
             content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
         };
@@ -133,31 +194,36 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         : input.auth?.type === 'storage-state'
             ? { type: 'storage-state', path: input.auth.path }
             : undefined;
+    const harnessConfig = {
+        maxPagesToScan: input.maxPagesToScan ?? 10,
+        maxDepth: 3,
+        minPagesForConfidence: 3,
+        timeoutMs: input.timeoutMs ?? 30000,
+        retryCount: 0,
+        llmTokenBudget: input.llmTokenBudget ?? input.llmMaxOutputTokensPerCall ?? 4096,
+        llmMaxOutputTokensPerCall: input.llmMaxOutputTokensPerCall,
+        testGenerationLimit: input.testGenerationLimit ?? 5,
+        enableLlmScenarios: input.enableLlmScenarios !== false,
+        readOnlyMode: true,
+        requireHumanReview: false,
+        failOnConsoleError: false,
+        explorer: 'playwright',
+        defaultAdapter: 'playwright',
+        adapters: ['playwright'],
+        ...(authConfig && { auth: authConfig }),
+    };
     const result = await analyzeApp({
         url: input.url,
         writeArtifacts: false,
-        config: {
-            maxPagesToScan: input.maxPagesToScan ?? 10,
-            maxDepth: 3,
-            minPagesForConfidence: 3,
-            timeoutMs: input.timeoutMs ?? 30000,
-            retryCount: 0,
-            llmTokenBudget: 1,
-            testGenerationLimit: 1,
-            readOnlyMode: true,
-            requireHumanReview: false,
-            failOnConsoleError: false,
-            explorer: 'playwright',
-            defaultAdapter: 'playwright',
-            adapters: ['playwright'],
-            ...(authConfig && { auth: authConfig }),
-        },
+        config: harnessConfig,
+        progressLog: mcpProgressLog,
     });
+    const payload = buildCompactAnalyzePayload(result, input.includeFullReport === true);
     return {
         content: [
             {
                 type: 'text',
-                text: JSON.stringify(result, null, 2),
+                text: JSON.stringify(payload, null, 2),
             },
         ],
     };

package/dist/logger.d.ts

@@ -0,0 +1,7 @@
+export declare const log: {
+    info(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+    debug(message: string): void;
+};
+//# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAiBA,eAAO,MAAM,GAAG;kBACA,MAAM,GAAG,IAAI;kBAGb,MAAM,GAAG,IAAI;mBAGZ,MAAM,GAAG,IAAI;mBAGb,MAAM,GAAG,IAAI;CAK7B,CAAC"}

package/dist/logger.js

@@ -0,0 +1,29 @@
+function pad2(n) {
+    return String(n).padStart(2, '0');
+}
+function wallTime() {
+    const d = new Date();
+    return `${pad2(d.getHours())}:${pad2(d.getMinutes())}:${pad2(d.getSeconds())}`;
+}
+function isDebugEnabled() {
+    return process.env.QULIB_DEBUG === '1';
+}
+function emit(level, message) {
+    process.stderr.write(`[qulib ${wallTime()}] ${level}  ${message}\n`);
+}
+export const log = {
+    info(message) {
+        emit('INFO', message);
+    },
+    warn(message) {
+        emit('WARN', message);
+    },
+    error(message) {
+        emit('ERROR', message);
+    },
+    debug(message) {
+        if (isDebugEnabled()) {
+            emit('DEBUG', message);
+        }
+    },
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qulib/mcp",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "MCP server for Qulib — AI-callable QA gap analysis",
   "license": "MIT",
   "author": "Tapesh Nagarwal",
@@ -27,12 +27,13 @@
     "README.md"
   ],
   "scripts": {
-    "build": "tsc && chmod +x dist/index.js",
-    "dev": "tsx src/index.ts"
+    "build": "npm --prefix ../.. run build -w @qulib/core && tsc && chmod +x dist/index.js",
+    "dev": "tsx src/index.ts",
+    "test": "node --import tsx/esm --test src/compact-analyze-payload.test.ts"
   },
   "dependencies": {
-    "@qulib/core": "0.2.1",
     "@modelcontextprotocol/sdk": "^1.0.0",
+    "@qulib/core": "0.3.0",
     "zod": "^3.23.0"
   },
   "devDependencies": {