llm-cli-gateway 1.4.0 → 1.5.13

package/dist/validation-normalizer.js

@@ -0,0 +1,79 @@
+export function normalizeStartedJob(provider, model, snapshot, warning) {
+    return {
+        provider,
+        model,
+        status: snapshot.status,
+        verdict: snapshot.status === "running" ? "pending" : null,
+        rationale: snapshot.status === "running" ? "Provider job is running asynchronously." : null,
+        risks: [],
+        rawJobReference: {
+            jobId: snapshot.id,
+            correlationId: snapshot.correlationId,
+            statusTool: "job_status",
+            resultTool: "job_result",
+        },
+        error: snapshot.error,
+        warning,
+    };
+}
+export function normalizeSkippedProvider(provider, reason) {
+    return {
+        provider,
+        model: null,
+        status: "skipped",
+        verdict: "not_run",
+        rationale: reason,
+        risks: [reason],
+        rawJobReference: null,
+        error: reason,
+    };
+}
+export function normalizeJobResult(provider, model, result) {
+    const output = result.stdout.trim();
+    const error = result.error || (result.status === "failed" ? result.stderr.trim() : null);
+    return {
+        provider,
+        model,
+        status: result.status,
+        verdict: inferVerdict(output, result.status),
+        rationale: output ? excerpt(output, 1800) : error,
+        risks: extractRisks(output, error),
+        rawJobReference: {
+            jobId: result.id,
+            correlationId: result.correlationId,
+            statusTool: "job_status",
+            resultTool: "job_result",
+        },
+        error,
+    };
+}
+function inferVerdict(output, status) {
+    if (status === "running")
+        return "pending";
+    if (status === "canceled" || status === "orphaned")
+        return status;
+    if (status === "failed")
+        return "failed";
+    const verdictMatch = output.match(/(?:^|\n)\s*verdict\s*:\s*(.+)/i);
+    if (verdictMatch?.[1])
+        return excerpt(verdictMatch[1].trim(), 240);
+    if (output)
+        return "answered";
+    return null;
+}
+function extractRisks(output, error) {
+    const risks = output
+        .split(/\r?\n/)
+        .map(line => line.trim())
+        .filter(line => /^(?:[-*]\s*)?(?:risk|risks|concern|caution|limitation)\b/i.test(line))
+        .slice(0, 5)
+        .map(line => excerpt(line, 300));
+    if (error && risks.length === 0)
+        risks.push(excerpt(error, 300));
+    return risks;
+}
+function excerpt(value, max) {
+    if (value.length <= max)
+        return value;
+    return `${value.slice(0, max - 3)}...`;
+}

package/dist/validation-orchestrator.d.ts

@@ -0,0 +1,47 @@
+import type { AsyncJobManager } from "./async-job-manager.js";
+import { type ProviderRuntimeStatus } from "./provider-status.js";
+import { type NormalizedValidationResult, type ValidationProvider } from "./validation-normalizer.js";
+import { type ValidationReport } from "./validation-report.js";
+import { type ValidationIntent } from "./validation-prompts.js";
+export interface ValidationOrchestratorDeps {
+    asyncJobManager: AsyncJobManager;
+    getProviderRuntimeStatus?: (provider: ValidationProvider) => ProviderRuntimeStatus;
+}
+export interface StartValidationInput {
+    intent: ValidationIntent;
+    question?: string;
+    content?: string;
+    providers: ValidationProvider[];
+    focus?: string;
+    riskLevel?: "normal" | "high";
+    judgeProvider?: ValidationProvider;
+}
+export interface ValidationRunReport {
+    success: boolean;
+    validationId: string;
+    status: "running" | "partial" | "not_started";
+    startedAt: string;
+    intent: ValidationIntent;
+    originalRequest: {
+        question?: string;
+        content?: string;
+        focus?: string;
+    };
+    modelList: ValidationProvider[];
+    results: NormalizedValidationResult[];
+    synthesis: {
+        status: "not_requested" | "waiting_for_provider_results" | "running" | "skipped";
+        judgeModel: ValidationProvider | null;
+        rawJobReference: NormalizedValidationResult["rawJobReference"];
+        note: string;
+    };
+    report: ValidationReport;
+    next: string;
+}
+export declare function startValidationRun(deps: ValidationOrchestratorDeps, input: StartValidationInput): ValidationRunReport;
+export declare function startJudgeSynthesis(deps: ValidationOrchestratorDeps, input: {
+    question: string;
+    providerResults: NormalizedValidationResult[];
+    judgeProvider: ValidationProvider;
+}): ValidationRunReport["synthesis"];
+export declare function collectValidationJobResult(deps: ValidationOrchestratorDeps, provider: ValidationProvider, jobId: string, model: string | null, maxChars?: number): NormalizedValidationResult | null;

package/dist/validation-orchestrator.js

@@ -0,0 +1,145 @@
+import { randomUUID } from "node:crypto";
+import { getProviderRuntimeStatus } from "./provider-status.js";
+import { normalizeJobResult, normalizeSkippedProvider, normalizeStartedJob, } from "./validation-normalizer.js";
+import { buildValidationReport } from "./validation-report.js";
+import { buildJudgePrompt, buildValidationPrompt, } from "./validation-prompts.js";
+export function startValidationRun(deps, input) {
+    const validationId = randomUUID();
+    const startedAt = new Date().toISOString();
+    const prompt = buildValidationPrompt({
+        intent: input.intent,
+        question: input.question,
+        content: input.content,
+        focus: input.focus,
+        riskLevel: input.riskLevel,
+    });
+    const providers = uniqueProviders(input.providers);
+    const results = providers.map(provider => startProviderJob(deps, provider, prompt, validationId));
+    const runningCount = results.filter(result => result.status === "running").length;
+    const skippedCount = results.filter(result => result.status === "skipped").length;
+    const synthesis = plannedJudgeSynthesis(input);
+    const status = runningCount === 0 ? "not_started" : skippedCount > 0 ? "partial" : "running";
+    const reportInput = {
+        validationId,
+        status,
+        startedAt,
+        intent: input.intent,
+        originalRequest: {
+            question: input.question,
+            content: input.content,
+            focus: input.focus,
+        },
+        modelList: providers,
+        results,
+        synthesis,
+    };
+    return {
+        success: runningCount > 0,
+        validationId,
+        status,
+        startedAt,
+        intent: input.intent,
+        originalRequest: reportInput.originalRequest,
+        modelList: providers,
+        results,
+        synthesis,
+        report: buildValidationReport(reportInput),
+        next: "Use job_status to poll each rawJobReference.jobId, job_result to collect provider outputs, then synthesize_validation if a judge summary is needed.",
+    };
+}
+export function startJudgeSynthesis(deps, input) {
+    const pending = input.providerResults.find(result => result.status === "running" || result.verdict === "pending");
+    if (pending) {
+        return {
+            status: "waiting_for_provider_results",
+            judgeModel: input.judgeProvider,
+            rawJobReference: null,
+            note: `Provider result for ${pending.provider} is still pending; collect terminal provider results before judge synthesis.`,
+        };
+    }
+    const completedResults = input.providerResults.filter(result => result.status === "completed");
+    const omittedResults = input.providerResults.filter(result => result.status !== "completed");
+    if (completedResults.length === 0) {
+        return {
+            status: "skipped",
+            judgeModel: input.judgeProvider,
+            rawJobReference: null,
+            note: "Judge synthesis requires at least one completed provider result; skipped, failed, canceled, or orphaned results are preserved in the report but are not judge evidence.",
+        };
+    }
+    const runtimeStatus = deps.getProviderRuntimeStatus ?? getProviderRuntimeStatus;
+    const runtime = runtimeStatus(input.judgeProvider);
+    if (!runtime.installed) {
+        return {
+            status: "skipped",
+            judgeModel: input.judgeProvider,
+            rawJobReference: null,
+            note: `${runtime.displayName} was selected as judge but is not installed.`,
+        };
+    }
+    const snapshot = deps.asyncJobManager.startJob(input.judgeProvider, buildProviderArgs(input.judgeProvider, buildJudgePrompt({
+        question: input.question,
+        providerResults: completedResults,
+    })), `validation-judge-${randomUUID()}-${input.judgeProvider}`);
+    return {
+        status: "running",
+        judgeModel: input.judgeProvider,
+        rawJobReference: {
+            jobId: snapshot.id,
+            correlationId: snapshot.correlationId,
+            statusTool: "job_status",
+            resultTool: "job_result",
+        },
+        note: omittedResults.length > 0
+            ? `Judge synthesis is running on ${runtime.displayName} using ${completedResults.length} completed provider result(s); ${omittedResults.length} non-completed result(s) were preserved but omitted.`
+            : `Judge synthesis is running on ${runtime.displayName} using completed provider results.`,
+    };
+}
+export function collectValidationJobResult(deps, provider, jobId, model, maxChars = 200000) {
+    const result = deps.asyncJobManager.getJobResult(jobId, maxChars);
+    if (!result)
+        return null;
+    return normalizeJobResult(provider, model, result);
+}
+function startProviderJob(deps, provider, prompt, validationId) {
+    const runtimeStatus = deps.getProviderRuntimeStatus ?? getProviderRuntimeStatus;
+    const runtime = runtimeStatus(provider);
+    if (!runtime.installed) {
+        return normalizeSkippedProvider(provider, `${runtime.displayName} runtime is not installed.`);
+    }
+    const warning = runtime.loginStatus === "authenticated"
+        ? undefined
+        : `${runtime.displayName} login status is ${runtime.loginStatus}; the job may fail until login is complete.`;
+    const snapshot = deps.asyncJobManager.startJob(provider, buildProviderArgs(provider, prompt), `validation-${validationId}-${provider}`);
+    return normalizeStartedJob(provider, runtime.version, snapshot, warning);
+}
+function plannedJudgeSynthesis(input) {
+    if (!input.judgeProvider) {
+        return {
+            status: "not_requested",
+            judgeModel: null,
+            rawJobReference: null,
+            note: "No judge synthesis was requested; provider disagreement is preserved for the caller.",
+        };
+    }
+    return {
+        status: "waiting_for_provider_results",
+        judgeModel: input.judgeProvider,
+        rawJobReference: null,
+        note: "Collect provider results first, then call synthesize_validation with those results.",
+    };
+}
+function buildProviderArgs(provider, prompt) {
+    if (provider === "claude" || provider === "grok" || provider === "mistral") {
+        // Mistral Vibe mirrors Grok's `-p PROMPT` headless surface. Model selection
+        // is via VIBE_ACTIVE_MODEL env var (no --model flag); for validation runs we
+        // let the user's environment pick the active model.
+        return ["-p", prompt];
+    }
+    if (provider === "codex")
+        return ["exec", "--skip-git-repo-check", prompt];
+    return [prompt];
+}
+function uniqueProviders(providers) {
+    return Array.from(new Set(providers));
+}

package/dist/validation-prompts.d.ts

@@ -0,0 +1,15 @@
+import type { NormalizedValidationResult } from "./validation-normalizer.js";
+export type ValidationIntent = "validate" | "second_opinion" | "red_team" | "consensus" | "ask_model";
+interface BasePromptInput {
+    intent: ValidationIntent;
+    question?: string;
+    content?: string;
+    focus?: string;
+    riskLevel?: "normal" | "high";
+}
+export declare function buildValidationPrompt(input: BasePromptInput): string;
+export declare function buildJudgePrompt(input: {
+    question: string;
+    providerResults: NormalizedValidationResult[];
+}): string;
+export {};

package/dist/validation-prompts.js

@@ -0,0 +1,52 @@
+export function buildValidationPrompt(input) {
+    const focus = input.focus || "correctness, missing assumptions, and practical next steps";
+    const header = [
+        "You are one independent reviewer in a personal cross-LLM validation run.",
+        "Return a concise answer with these headings: Verdict, Rationale, Risks, Suggested next step.",
+        "Do not claim consensus; other model responses will be compared separately.",
+    ];
+    if (input.intent === "second_opinion") {
+        return [
+            ...header,
+            `Focus: ${focus}`,
+            "",
+            `Original question: ${input.question || "(not provided)"}`,
+            "",
+            "Answer to review:",
+            input.content || "",
+        ].join("\n");
+    }
+    if (input.intent === "red_team") {
+        return [
+            ...header,
+            `Review intensity: ${input.riskLevel || "normal"}`,
+            "Challenge assumptions, unsafe advice, unsupported claims, and likely failure modes.",
+            "",
+            input.content || "",
+        ].join("\n");
+    }
+    if (input.intent === "consensus") {
+        return [
+            ...header,
+            "Assess whether the claim is true, false, uncertain, or context-dependent.",
+            "",
+            `Claim: ${input.content || input.question || ""}`,
+        ].join("\n");
+    }
+    if (input.intent === "ask_model") {
+        return [input.question || input.content || ""].join("\n");
+    }
+    return [...header, `Focus: ${focus}`, "", input.question || input.content || ""].join("\n");
+}
+export function buildJudgePrompt(input) {
+    return [
+        "You are the explicit judge model for a personal cross-LLM validation run.",
+        "Synthesize only from the provider results below. Preserve material disagreement.",
+        "Return: Summary, Agreements, Disagreements, Recommendation, Confidence, Limitations.",
+        "",
+        `Original request: ${input.question}`,
+        "",
+        "Provider results:",
+        JSON.stringify(input.providerResults, null, 2),
+    ].join("\n");
+}

package/dist/validation-report.d.ts

@@ -0,0 +1,57 @@
+import type { NormalizedValidationResult, ValidationProvider } from "./validation-normalizer.js";
+import type { ValidationIntent } from "./validation-prompts.js";
+export type ValidationReportConfidence = "none" | "low" | "medium" | "high";
+export interface ValidationReportInput {
+    validationId: string;
+    status: "running" | "partial" | "not_started";
+    startedAt: string;
+    intent: ValidationIntent;
+    originalRequest: {
+        question?: string;
+        content?: string;
+        focus?: string;
+    };
+    modelList: ValidationProvider[];
+    results: NormalizedValidationResult[];
+    synthesis: {
+        status: "not_requested" | "waiting_for_provider_results" | "running" | "skipped";
+        judgeModel: ValidationProvider | null;
+        rawJobReference: NormalizedValidationResult["rawJobReference"];
+        note: string;
+    };
+}
+export interface ValidationReport {
+    schemaVersion: "validation-report.v1";
+    humanReadable: string;
+    structuredContent: {
+        validationId: string;
+        status: ValidationReportInput["status"];
+        startedAt: string;
+        intent: ValidationIntent;
+        originalRequest: ValidationReportInput["originalRequest"];
+        modelList: ValidationProvider[];
+        perModelOutputs: Array<{
+            provider: ValidationProvider;
+            model: string | null;
+            status: NormalizedValidationResult["status"];
+            verdict: string | null;
+            rationale: string | null;
+            risks: string[];
+            jobId: string | null;
+            correlationId: string | null;
+            warning: string | null;
+            error: string | null;
+        }>;
+        disagreements: {
+            hasMaterialDisagreement: boolean;
+            summary: string;
+            signals: string[];
+        };
+        finalRecommendation: string;
+        confidence: ValidationReportConfidence;
+        limitations: string[];
+        jobIds: string[];
+        synthesis: ValidationReportInput["synthesis"];
+    };
+}
+export declare function buildValidationReport(input: ValidationReportInput): ValidationReport;

package/dist/validation-report.js

@@ -0,0 +1,129 @@
+export function buildValidationReport(input) {
+    const perModelOutputs = input.results.map(result => ({
+        provider: result.provider,
+        model: result.model,
+        status: result.status,
+        verdict: result.verdict,
+        rationale: result.rationale,
+        risks: result.risks,
+        jobId: result.rawJobReference?.jobId ?? null,
+        correlationId: result.rawJobReference?.correlationId ?? null,
+        warning: result.warning ?? null,
+        error: result.error,
+    }));
+    const jobIds = perModelOutputs.flatMap(output => (output.jobId ? [output.jobId] : []));
+    const disagreements = summarizeDisagreement(input.results);
+    const limitations = summarizeLimitations(input.results, input.synthesis);
+    const confidence = confidenceFor(input.results, disagreements.hasMaterialDisagreement);
+    const finalRecommendation = recommendationFor(input.results, disagreements.hasMaterialDisagreement);
+    const structuredContent = {
+        validationId: input.validationId,
+        status: input.status,
+        startedAt: input.startedAt,
+        intent: input.intent,
+        originalRequest: input.originalRequest,
+        modelList: input.modelList,
+        perModelOutputs,
+        disagreements,
+        finalRecommendation,
+        confidence,
+        limitations,
+        jobIds,
+        synthesis: input.synthesis,
+    };
+    return {
+        schemaVersion: "validation-report.v1",
+        humanReadable: renderHumanReport(structuredContent),
+        structuredContent,
+    };
+}
+function summarizeDisagreement(results) {
+    const completed = results.filter(result => result.status === "completed");
+    const terminalProblems = results.filter(result => ["failed", "canceled", "orphaned", "skipped"].includes(result.status));
+    const pending = results.filter(result => result.status === "running" || result.verdict === "pending");
+    const verdicts = new Set(completed
+        .map(result => normalizeVerdict(result.verdict))
+        .filter((verdict) => Boolean(verdict)));
+    const signals = [];
+    if (verdicts.size > 1)
+        signals.push(`Completed providers returned ${verdicts.size} different verdicts.`);
+    for (const result of terminalProblems)
+        signals.push(`${result.provider} is ${result.status}.`);
+    for (const result of pending)
+        signals.push(`${result.provider} is still pending.`);
+    const hasMaterialDisagreement = verdicts.size > 1 || terminalProblems.length > 0 || pending.length > 0;
+    return {
+        hasMaterialDisagreement,
+        summary: hasMaterialDisagreement
+            ? "Do not treat this validation as consensus; inspect the per-model outputs and unresolved states."
+            : completed.length > 0
+                ? "Completed providers do not show material verdict disagreement in the normalized report."
+                : "No completed provider outputs are available yet.",
+        signals,
+    };
+}
+function summarizeLimitations(results, synthesis) {
+    const limitations = [];
+    if (results.some(result => result.status === "running")) {
+        limitations.push("Some provider jobs are still running; poll job_status and job_result before treating the report as final.");
+    }
+    if (results.some(result => result.status !== "completed")) {
+        limitations.push("Only completed provider outputs are suitable as judge synthesis evidence.");
+    }
+    if (synthesis.status === "waiting_for_provider_results") {
+        limitations.push("Judge synthesis has not run because provider results still need to be collected.");
+    }
+    else if (synthesis.status === "skipped") {
+        limitations.push(`Judge synthesis skipped: ${synthesis.note}`);
+    }
+    else if (synthesis.status === "not_requested") {
+        limitations.push("No explicit judge synthesis was requested; use per-model outputs for the decision.");
+    }
+    limitations.push("Large raw outputs are intentionally kept behind job_result references to fit normal MCP client responses.");
+    return limitations;
+}
+function confidenceFor(results, hasMaterialDisagreement) {
+    const completedCount = results.filter(result => result.status === "completed").length;
+    if (completedCount === 0)
+        return "none";
+    if (hasMaterialDisagreement)
+        return "low";
+    if (completedCount === 1)
+        return "medium";
+    return "high";
+}
+function recommendationFor(results, hasMaterialDisagreement) {
+    const completedCount = results.filter(result => result.status === "completed").length;
+    if (completedCount === 0) {
+        return "Wait for at least one provider job to complete, then collect job_result before deciding.";
+    }
+    if (hasMaterialDisagreement) {
+        return "Review the per-model outputs and resolve disagreements manually before acting.";
+    }
+    return "Completed provider outputs show no normalized verdict disagreement; review rationales and risks before acting.";
+}
+function renderHumanReport(content) {
+    const lines = [
+        `Validation report ${content.validationId}`,
+        `Status: ${content.status}`,
+        `Models: ${content.modelList.join(", ") || "none"}`,
+        "",
+        "Per-model outputs:",
+        ...content.perModelOutputs.map(output => {
+            const job = output.jobId ? ` job=${output.jobId}` : "";
+            const verdict = output.verdict ? ` verdict=${output.verdict}` : "";
+            return `- ${output.provider}: ${output.status}${verdict}${job}`;
+        }),
+        "",
+        `Disagreement: ${content.disagreements.summary}`,
+        `Recommendation: ${content.finalRecommendation}`,
+        `Confidence: ${content.confidence}`,
+        "",
+        "Limitations:",
+        ...content.limitations.map(limitation => `- ${limitation}`),
+    ];
+    return lines.join("\n");
+}
+function normalizeVerdict(verdict) {
+    return verdict?.trim().toLowerCase() || null;
+}

package/dist/validation-tools.d.ts

@@ -0,0 +1,7 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { AsyncJobManager } from "./async-job-manager.js";
+import { type ValidationOrchestratorDeps } from "./validation-orchestrator.js";
+export interface ValidationToolDeps extends ValidationOrchestratorDeps {
+    asyncJobManager: AsyncJobManager;
+}
+export declare function registerValidationTools(server: McpServer, deps: ValidationToolDeps): void;