llm-cli-gateway 1.1.0 → 1.5.4

package/dist/validation-report.js

@@ -0,0 +1,129 @@
+export function buildValidationReport(input) {
+    const perModelOutputs = input.results.map(result => ({
+        provider: result.provider,
+        model: result.model,
+        status: result.status,
+        verdict: result.verdict,
+        rationale: result.rationale,
+        risks: result.risks,
+        jobId: result.rawJobReference?.jobId ?? null,
+        correlationId: result.rawJobReference?.correlationId ?? null,
+        warning: result.warning ?? null,
+        error: result.error,
+    }));
+    const jobIds = perModelOutputs.flatMap(output => (output.jobId ? [output.jobId] : []));
+    const disagreements = summarizeDisagreement(input.results);
+    const limitations = summarizeLimitations(input.results, input.synthesis);
+    const confidence = confidenceFor(input.results, disagreements.hasMaterialDisagreement);
+    const finalRecommendation = recommendationFor(input.results, disagreements.hasMaterialDisagreement);
+    const structuredContent = {
+        validationId: input.validationId,
+        status: input.status,
+        startedAt: input.startedAt,
+        intent: input.intent,
+        originalRequest: input.originalRequest,
+        modelList: input.modelList,
+        perModelOutputs,
+        disagreements,
+        finalRecommendation,
+        confidence,
+        limitations,
+        jobIds,
+        synthesis: input.synthesis,
+    };
+    return {
+        schemaVersion: "validation-report.v1",
+        humanReadable: renderHumanReport(structuredContent),
+        structuredContent,
+    };
+}
+function summarizeDisagreement(results) {
+    const completed = results.filter(result => result.status === "completed");
+    const terminalProblems = results.filter(result => ["failed", "canceled", "orphaned", "skipped"].includes(result.status));
+    const pending = results.filter(result => result.status === "running" || result.verdict === "pending");
+    const verdicts = new Set(completed
+        .map(result => normalizeVerdict(result.verdict))
+        .filter((verdict) => Boolean(verdict)));
+    const signals = [];
+    if (verdicts.size > 1)
+        signals.push(`Completed providers returned ${verdicts.size} different verdicts.`);
+    for (const result of terminalProblems)
+        signals.push(`${result.provider} is ${result.status}.`);
+    for (const result of pending)
+        signals.push(`${result.provider} is still pending.`);
+    const hasMaterialDisagreement = verdicts.size > 1 || terminalProblems.length > 0 || pending.length > 0;
+    return {
+        hasMaterialDisagreement,
+        summary: hasMaterialDisagreement
+            ? "Do not treat this validation as consensus; inspect the per-model outputs and unresolved states."
+            : completed.length > 0
+                ? "Completed providers do not show material verdict disagreement in the normalized report."
+                : "No completed provider outputs are available yet.",
+        signals,
+    };
+}
+function summarizeLimitations(results, synthesis) {
+    const limitations = [];
+    if (results.some(result => result.status === "running")) {
+        limitations.push("Some provider jobs are still running; poll job_status and job_result before treating the report as final.");
+    }
+    if (results.some(result => result.status !== "completed")) {
+        limitations.push("Only completed provider outputs are suitable as judge synthesis evidence.");
+    }
+    if (synthesis.status === "waiting_for_provider_results") {
+        limitations.push("Judge synthesis has not run because provider results still need to be collected.");
+    }
+    else if (synthesis.status === "skipped") {
+        limitations.push(`Judge synthesis skipped: ${synthesis.note}`);
+    }
+    else if (synthesis.status === "not_requested") {
+        limitations.push("No explicit judge synthesis was requested; use per-model outputs for the decision.");
+    }
+    limitations.push("Large raw outputs are intentionally kept behind job_result references to fit normal MCP client responses.");
+    return limitations;
+}
+function confidenceFor(results, hasMaterialDisagreement) {
+    const completedCount = results.filter(result => result.status === "completed").length;
+    if (completedCount === 0)
+        return "none";
+    if (hasMaterialDisagreement)
+        return "low";
+    if (completedCount === 1)
+        return "medium";
+    return "high";
+}
+function recommendationFor(results, hasMaterialDisagreement) {
+    const completedCount = results.filter(result => result.status === "completed").length;
+    if (completedCount === 0) {
+        return "Wait for at least one provider job to complete, then collect job_result before deciding.";
+    }
+    if (hasMaterialDisagreement) {
+        return "Review the per-model outputs and resolve disagreements manually before acting.";
+    }
+    return "Completed provider outputs show no normalized verdict disagreement; review rationales and risks before acting.";
+}
+function renderHumanReport(content) {
+    const lines = [
+        `Validation report ${content.validationId}`,
+        `Status: ${content.status}`,
+        `Models: ${content.modelList.join(", ") || "none"}`,
+        "",
+        "Per-model outputs:",
+        ...content.perModelOutputs.map(output => {
+            const job = output.jobId ? ` job=${output.jobId}` : "";
+            const verdict = output.verdict ? ` verdict=${output.verdict}` : "";
+            return `- ${output.provider}: ${output.status}${verdict}${job}`;
+        }),
+        "",
+        `Disagreement: ${content.disagreements.summary}`,
+        `Recommendation: ${content.finalRecommendation}`,
+        `Confidence: ${content.confidence}`,
+        "",
+        "Limitations:",
+        ...content.limitations.map(limitation => `- ${limitation}`),
+    ];
+    return lines.join("\n");
+}
+function normalizeVerdict(verdict) {
+    return verdict?.trim().toLowerCase() || null;
+}

package/dist/validation-tools.d.ts

@@ -0,0 +1,7 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { AsyncJobManager } from "./async-job-manager.js";
+import { type ValidationOrchestratorDeps } from "./validation-orchestrator.js";
+export interface ValidationToolDeps extends ValidationOrchestratorDeps {
+    asyncJobManager: AsyncJobManager;
+}
+export declare function registerValidationTools(server: McpServer, deps: ValidationToolDeps): void;

package/dist/validation-tools.js

@@ -0,0 +1,198 @@
+import { z } from "zod";
+import { getCliInfo } from "./model-registry.js";
+import { collectValidationJobResult, startJudgeSynthesis, startValidationRun, } from "./validation-orchestrator.js";
+const providerSchema = z.enum(["claude", "codex", "gemini", "grok", "mistral"]);
+const providerListSchema = z.array(providerSchema).min(1).default(["claude", "codex"]);
+const normalizedProviderResultSchema = z.object({
+    provider: providerSchema,
+    model: z.string().nullable(),
+    status: z.enum(["running", "completed", "failed", "canceled", "orphaned", "skipped"]),
+    verdict: z.string().nullable(),
+    rationale: z.string().nullable(),
+    risks: z.array(z.string()).default([]),
+    rawJobReference: z
+        .object({
+        jobId: z.string(),
+        correlationId: z.string(),
+        statusTool: z.literal("job_status"),
+        resultTool: z.literal("job_result"),
+    })
+        .nullable(),
+    error: z.string().nullable(),
+    warning: z.string().optional(),
+});
+function textResponse(body) {
+    const text = responseText(body);
+    return {
+        content: [{ type: "text", text }],
+        structuredContent: body,
+    };
+}
+function responseText(body) {
+    const report = findHumanReadableReport(body);
+    if (report)
+        return report;
+    return JSON.stringify(body, null, 2);
+}
+function findHumanReadableReport(value) {
+    if (typeof value !== "object" || value === null)
+        return null;
+    if ("humanReadable" in value &&
+        typeof value.humanReadable === "string") {
+        return value.humanReadable;
+    }
+    if ("report" in value) {
+        return findHumanReadableReport(value.report);
+    }
+    return null;
+}
+export function registerValidationTools(server, deps) {
+    server.tool("validate_with_models", {
+        question: z.string().min(1).describe("Question or content to validate."),
+        models: providerListSchema.describe("Providers to ask. Defaults to Claude and Codex."),
+        focus: z
+            .string()
+            .default("correctness, missing assumptions, and practical next steps")
+            .describe("What reviewers should pay attention to."),
+        judgeModel: providerSchema
+            .optional()
+            .describe("Optional provider to run an explicit judge synthesis job."),
+    }, async ({ question, models, focus, judgeModel }) => textResponse({
+        success: true,
+        tool: "validate_with_models",
+        readMostly: true,
+        report: startValidationRun(deps, {
+            intent: "validate",
+            question,
+            providers: models,
+            focus,
+            judgeProvider: judgeModel,
+        }),
+    }));
+    server.tool("second_opinion", {
+        answer: z.string().min(1).describe("Answer to review."),
+        question: z.string().optional().describe("Original question, if available."),
+        model: providerSchema.default("codex").describe("Provider to ask for the second opinion."),
+    }, async ({ answer, question, model }) => textResponse({
+        success: true,
+        tool: "second_opinion",
+        readMostly: true,
+        report: startValidationRun(deps, {
+            intent: "second_opinion",
+            question,
+            content: answer,
+            providers: [model],
+        }),
+    }));
+    server.tool("compare_answers", {
+        question: z.string().min(1).describe("Question the answers respond to."),
+        answers: z.array(z.string().min(1)).min(2).describe("Two or more answers to compare."),
+    }, async ({ question, answers }) => textResponse({
+        success: true,
+        tool: "compare_answers",
+        readMostly: true,
+        comparison: {
+            question,
+            answerCount: answers.length,
+            checks: ["agreement", "contradictions", "missing evidence", "actionable recommendation"],
+            status: "local_summary_only",
+            note: "Use validate_with_models when independent provider review is needed.",
+        },
+    }));
+    server.tool("red_team_review", {
+        content: z.string().min(1).describe("Plan, answer, or document to challenge."),
+        riskLevel: z
+            .enum(["normal", "high"])
+            .default("normal")
+            .describe("How aggressively to review."),
+        models: providerListSchema.describe("Providers to ask for adversarial review."),
+    }, async ({ content, riskLevel, models }) => textResponse({
+        success: true,
+        tool: "red_team_review",
+        readMostly: true,
+        report: startValidationRun(deps, {
+            intent: "red_team",
+            content,
+            providers: models,
+            riskLevel,
+        }),
+    }));
+    server.tool("consensus_check", {
+        claim: z.string().min(1).describe("Claim to check across providers."),
+        models: providerListSchema.describe("Providers to ask for agreement or disagreement."),
+    }, async ({ claim, models }) => textResponse({
+        success: true,
+        tool: "consensus_check",
+        readMostly: true,
+        report: startValidationRun(deps, {
+            intent: "consensus",
+            content: claim,
+            providers: models,
+        }),
+    }));
+    server.tool("ask_model", {
+        question: z.string().min(1).describe("Question for one provider."),
+        model: providerSchema.default("claude").describe("Provider to ask."),
+    }, async ({ question, model }) => textResponse({
+        success: true,
+        tool: "ask_model",
+        readMostly: true,
+        report: startValidationRun(deps, {
+            intent: "ask_model",
+            question,
+            providers: [model],
+        }),
+    }));
+    server.tool("synthesize_validation", {
+        question: z.string().min(1).describe("Original request that was validated."),
+        providerResults: z
+            .array(normalizedProviderResultSchema)
+            .min(1)
+            .describe("Terminal normalized provider results from job_result."),
+        judgeModel: providerSchema.default("codex").describe("Provider to run the judge synthesis."),
+    }, async ({ question, providerResults, judgeModel }) => textResponse({
+        success: true,
+        tool: "synthesize_validation",
+        readMostly: true,
+        synthesis: startJudgeSynthesis(deps, {
+            question,
+            providerResults,
+            judgeProvider: judgeModel,
+        }),
+    }));
+    server.tool("list_available_models", {}, async () => textResponse({ success: true, models: getCliInfo() }));
+    server.tool("job_status", {
+        jobId: z.string().min(1).describe("Validation job ID."),
+    }, async ({ jobId }) => {
+        const job = deps.asyncJobManager.getJobSnapshot(jobId);
+        if (!job) {
+            return textResponse({ success: false, error: "Job not found", jobId });
+        }
+        return textResponse({ success: true, job });
+    });
+    server.tool("job_result", {
+        jobId: z.string().min(1).describe("Validation job ID."),
+        provider: providerSchema
+            .optional()
+            .describe("Provider that produced the job, used for normalized validation output."),
+        maxChars: z
+            .number()
+            .int()
+            .min(1000)
+            .max(2000000)
+            .default(200000)
+            .describe("Maximum result size."),
+    }, async ({ jobId, provider, maxChars }) => {
+        const result = deps.asyncJobManager.getJobResult(jobId, maxChars);
+        if (!result) {
+            return textResponse({ success: false, error: "Job not found", jobId });
+        }
+        return textResponse({
+            success: true,
+            result,
+            normalized: provider !== undefined
+                ? collectValidationJobResult(deps, provider, jobId, null, maxChars)
+                : null,
+        });
+    });
+}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "llm-cli-gateway",
-  "version": "1.1.0",
+  "version": "1.5.4",
   "mcpName": "io.github.verivus-oss/llm-cli-gateway",
-  "description": "MCP server providing unified access to Claude Code, Codex, and Gemini CLIs with session management, retry logic, and async job orchestration.",
+  "description": "MCP server providing unified access to Claude Code, Codex, Gemini, Grok, and Mistral Vibe CLIs with session management, retry logic, async job orchestration, durable job results, and cross-LLM validation.",
   "license": "MIT",
   "author": {
     "name": "VerivusAI Labs",
@@ -10,7 +10,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/verivus-oss/llm-cli-gateway.git"
+    "url": "git+https://github.com/verivus-oss/llm-cli-gateway.git"
   },
   "homepage": "https://github.com/verivus-oss/llm-cli-gateway#readme",
   "bugs": {
@@ -33,10 +33,14 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
+    },
+    "./gateway-server": {
+      "types": "./dist/gateway-server.d.ts",
+      "default": "./dist/gateway-server.js"
     }
   },
   "bin": {
-    "llm-cli-gateway": "./dist/index.js"
+    "llm-cli-gateway": "dist/index.js"
   },
   "engines": {
     "node": ">=20.0.0"
@@ -45,6 +49,7 @@
     "dist/**/*.js",
     "dist/**/*.d.ts",
     "!dist/__tests__/**",
+    "setup/status.schema.json",
     "README.md",
     "CHANGELOG.md",
     "LICENSE"
@@ -54,6 +59,8 @@
     "build:all": "tsc",
     "prepublishOnly": "npm run build && npm test",
     "start": "node dist/index.js",
+    "start:http": "node dist/index.js --transport=http",
+    "doctor": "node dist/index.js doctor --json",
     "migrate": "node dist/migrate.js",
     "test": "vitest run",
     "test:coverage": "vitest run --coverage",
@@ -68,11 +75,14 @@
     "lint:fix": "eslint src/**/*.ts --fix",
     "format": "prettier --write 'src/**/*.ts'",
     "format:check": "prettier --check 'src/**/*.ts'",
-    "check": "npm run build && npm run lint && npm test"
+    "check": "npm run build && npm run lint && npm test",
+    "release:build": "bash installer/build-release.sh",
+    "release:checksums": "cd installer/dist && sha256sum --check SHA256SUMS",
+    "release:docker": "docker compose -f docker-compose.personal.yml build"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.0",
-    "better-sqlite3": "^11.0.0",
+    "better-sqlite3": "^12.9.0",
     "ioredis": "^5.4.1",
     "pg": "^8.12.0",
     "toml": "^3.0.0",

package/setup/status.schema.json

@@ -0,0 +1,271 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://verivus.ai/schemas/llm-cli-gateway/status.schema.json",
+  "title": "llm-cli-gateway doctor status",
+  "type": "object",
+  "required": [
+    "schema_version",
+    "ok",
+    "generated_at",
+    "system",
+    "gateway",
+    "transport",
+    "auth",
+    "providers",
+    "endpoint_exposure",
+    "client_config",
+    "next_actions"
+  ],
+  "properties": {
+    "schema_version": { "const": "1.0" },
+    "ok": { "type": "boolean" },
+    "generated_at": { "type": "string", "format": "date-time" },
+    "system": {
+      "type": "object",
+      "required": ["os", "arch", "release", "node_version"],
+      "properties": {
+        "os": { "type": "string" },
+        "arch": { "type": "string" },
+        "release": { "type": "string" },
+        "node_version": { "type": "string" }
+      },
+      "additionalProperties": false
+    },
+    "gateway": {
+      "type": "object",
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "const": "llm-cli-gateway" },
+        "version": { "type": "string" }
+      },
+      "additionalProperties": false
+    },
+    "transport": {
+      "type": "object",
+      "required": ["default", "http"],
+      "properties": {
+        "default": { "enum": ["stdio", "http"] },
+        "http": {
+          "type": "object",
+          "required": ["enabled", "host", "port", "path", "public_url_configured", "public_url"],
+          "properties": {
+            "enabled": { "type": "boolean" },
+            "host": { "type": "string" },
+            "port": { "type": "integer", "minimum": 1, "maximum": 65535 },
+            "path": { "type": "string" },
+            "public_url_configured": { "type": "boolean" },
+            "public_url": { "type": ["string", "null"] }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "auth": {
+      "type": "object",
+      "required": ["required", "token_configured", "source"],
+      "properties": {
+        "required": { "type": "boolean" },
+        "token_configured": { "type": "boolean" },
+        "source": { "enum": ["env", "disabled"] }
+      },
+      "additionalProperties": false
+    },
+    "providers": {
+      "type": "object",
+      "required": ["claude", "codex", "gemini", "grok", "mistral"],
+      "additionalProperties": {
+        "type": "object",
+        "required": [
+          "cli_available",
+          "version",
+          "login_status",
+          "version_command",
+          "login_check",
+          "install_guidance",
+          "login_guidance"
+        ],
+        "properties": {
+          "cli_available": { "type": "boolean" },
+          "version": { "type": ["string", "null"] },
+          "login_status": {
+            "enum": ["authenticated", "not_authenticated", "unknown", "not_checked"]
+          },
+          "version_command": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "login_check": {
+            "type": "object",
+            "required": ["method", "command", "credential_store", "detail"],
+            "properties": {
+              "method": { "enum": ["cli", "credential_store", "not_checked"] },
+              "command": {
+                "type": ["array", "null"],
+                "items": { "type": "string" }
+              },
+              "credential_store": { "enum": ["present", "not_found", "not_checked"] },
+              "detail": { "type": "string" }
+            },
+            "additionalProperties": false
+          },
+          "install_guidance": {
+            "type": "object",
+            "required": ["summary", "commands"],
+            "properties": {
+              "summary": { "type": "string" },
+              "commands": {
+                "type": "array",
+                "items": { "type": "string" }
+              },
+              "documentation_url": { "type": "string" }
+            },
+            "additionalProperties": false
+          },
+          "login_guidance": {
+            "type": "object",
+            "required": ["summary", "commands", "credential_handling"],
+            "properties": {
+              "summary": { "type": "string" },
+              "commands": {
+                "type": "array",
+                "items": { "type": "string" }
+              },
+              "credential_handling": { "type": "string" }
+            },
+            "additionalProperties": false
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "endpoint_exposure": {
+      "type": "object",
+      "required": [
+        "mode",
+        "local_url",
+        "public_url_configured",
+        "public_url",
+        "https_required_for_web",
+        "https_configured",
+        "web_clients_supported",
+        "tunnel_provider",
+        "reachable_from_web",
+        "verification",
+        "next_actions"
+      ],
+      "properties": {
+        "mode": {
+          "enum": ["local_only", "lan", "tunnel", "byo_reverse_proxy", "misconfigured"]
+        },
+        "local_url": { "type": "string" },
+        "public_url_configured": { "type": "boolean" },
+        "public_url": { "type": ["string", "null"] },
+        "https_required_for_web": { "type": "boolean" },
+        "https_configured": { "type": "boolean" },
+        "web_clients_supported": { "type": "boolean" },
+        "tunnel_provider": { "type": ["string", "null"] },
+        "reachable_from_web": { "enum": ["not_checked", "reachable", "unreachable"] },
+        "verification": {
+          "type": "object",
+          "required": ["method", "checked_url", "status_code", "error"],
+          "properties": {
+            "method": { "enum": ["not_checked", "http_head"] },
+            "checked_url": { "type": ["string", "null"] },
+            "status_code": { "type": ["integer", "null"] },
+            "error": { "type": ["string", "null"] }
+          },
+          "additionalProperties": false
+        },
+        "next_actions": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "client_config": {
+      "type": "object",
+      "required": [
+        "claude_desktop_config_present",
+        "codex_config_present",
+        "gemini_settings_present",
+        "gemini_config",
+        "vibe_session_logging"
+      ],
+      "properties": {
+        "claude_desktop_config_present": { "type": "boolean" },
+        "codex_config_present": { "type": "boolean" },
+        "gemini_settings_present": { "type": "boolean" },
+        "gemini_config": {
+          "type": "object",
+          "required": [
+            "project_gemini_md_present",
+            "project_gemini_md_path",
+            "user_gemini_md_present",
+            "user_gemini_md_path",
+            "settings_json_present",
+            "settings_json_path",
+            "mcp_servers_registered",
+            "mcp_reconciliation",
+            "next_actions"
+          ],
+          "properties": {
+            "project_gemini_md_present": { "type": "boolean" },
+            "project_gemini_md_path": { "type": "string" },
+            "user_gemini_md_present": { "type": "boolean" },
+            "user_gemini_md_path": { "type": "string" },
+            "settings_json_present": { "type": "boolean" },
+            "settings_json_path": { "type": "string" },
+            "mcp_servers_registered": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "mcp_reconciliation": {
+              "type": "object",
+              "required": ["whitelisted", "missing_from_settings"],
+              "properties": {
+                "whitelisted": {
+                  "type": "array",
+                  "items": { "type": "string" }
+                },
+                "missing_from_settings": {
+                  "type": "array",
+                  "items": { "type": "string" }
+                }
+              },
+              "additionalProperties": false
+            },
+            "next_actions": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          },
+          "additionalProperties": false
+        },
+        "vibe_session_logging": {
+          "type": "object",
+          "required": [
+            "config_path",
+            "config_present",
+            "session_logging_enabled",
+            "note"
+          ],
+          "properties": {
+            "config_path": { "type": "string" },
+            "config_present": { "type": "boolean" },
+            "session_logging_enabled": { "type": "boolean" },
+            "note": { "type": "string" }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "next_actions": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  },
+  "additionalProperties": false
+}