@mcoda/codali 0.1.66

package/dist/agents/PhaseAgentSelector.js

@@ -0,0 +1,287 @@
+import { GlobalRepository } from "@mcoda/db";
+import { resolveAgentConfigFromRecord, } from "./AgentResolver.js";
+const PHASE_CAPABILITIES = {
+    librarian: ["docdex_query", "summarization", "keyword_extraction", "log_analysis"],
+    architect: ["plan", "system_architecture", "architectural_design", "deep_reasoning"],
+    builder: ["code_write", "complex_refactoring", "migration_assist", "debugging"],
+    critic: ["code_review", "pull_request_review", "final_code_review", "standard_compliance"],
+    interpreter: ["code_review", "pull_request_review", "final_code_review", "standard_compliance"],
+};
+const PHASE_REQUIRED_CAPS = {
+    librarian: ["docdex_query", "summarization", "keyword_extraction", "log_analysis"],
+    architect: ["plan", "system_architecture", "architectural_design", "deep_reasoning"],
+    builder: ["code_write", "simple_refactor", "iterative_coding", "migration_assist"],
+    critic: ["code_review", "pull_request_review", "final_code_review", "standard_compliance"],
+    interpreter: ["code_review", "pull_request_review", "final_code_review", "standard_compliance"],
+};
+const PHASE_BEST_USAGE = {
+    librarian: ["lightweight_tasks", "log_analysis", "summarization", "doc_generation"],
+    architect: ["system_architecture", "architectural_design", "deep_reasoning", "plan"],
+    builder: ["code_write", "coding_light", "iterative_coding", "rapid_prototyping"],
+    critic: ["code_review", "code_review_secondary", "production_verification"],
+    interpreter: ["code_review", "code_review_secondary", "production_verification"],
+};
+const STRUCTURED_OUTPUT_CAPABILITIES = [
+    "strict_instruction_following",
+    "json_formatting",
+    "schema_adherence",
+    "structured_output",
+];
+const PATCH_JSON_EXECUTION_CAPABILITIES = [
+    "iterative_coding",
+    "simple_refactor",
+];
+const PATCH_JSON_RELIABILITY_CAPABILITIES = [
+    "iterative_coding",
+    "strict_instruction_following",
+    "json_formatting",
+    "schema_adherence",
+    "structured_output",
+    "test_fixing",
+];
+const ADAPTERS_REQUIRING_AUTH = new Set([
+    "openai-api",
+    "zhipu-api",
+    "gemini-cli",
+    "codex-cli",
+]);
+const countMatches = (capabilities, required) => required.filter((cap) => capabilities.includes(cap)).length;
+const isManagedMswarmCloudAgent = (agent) => {
+    const config = agent.config;
+    if (!config || typeof config !== "object" || Array.isArray(config)) {
+        return false;
+    }
+    const managed = config.mswarmCloud;
+    return Boolean(managed &&
+        typeof managed === "object" &&
+        !Array.isArray(managed) &&
+        managed.managed === true);
+};
+const isCloudModel = (agent) => {
+    if (isManagedMswarmCloudAgent(agent)) {
+        return true;
+    }
+    return Boolean(agent.defaultModel?.toLowerCase().includes(":cloud"));
+};
+const normalizeAdapter = (adapter) => (adapter ?? "").trim().toLowerCase();
+const requiresConfiguredAuth = (agent) => ADAPTERS_REQUIRING_AUTH.has(normalizeAdapter(agent.adapter));
+const adjustScoreForReadiness = (score, agent, readiness) => {
+    let adjusted = score;
+    if (readiness.healthStatus === "degraded")
+        adjusted -= 25;
+    if (requiresConfiguredAuth(agent) && readiness.authConfigured === false)
+        adjusted -= 80;
+    return adjusted;
+};
+const scoreAgent = (phase, agent, capabilities, builderMode) => {
+    if (builderMode === "tool_calls" && phase === "builder" && agent.supportsTools === false) {
+        return Number.NEGATIVE_INFINITY;
+    }
+    const rating = agent.rating ?? 0;
+    const reasoning = agent.reasoningRating ?? rating;
+    const cost = agent.costPerMillion ?? 0;
+    const maxComplexity = agent.maxComplexity ?? 0;
+    const capHits = countMatches(capabilities, PHASE_CAPABILITIES[phase]);
+    const requiredHits = countMatches(capabilities, PHASE_REQUIRED_CAPS[phase]);
+    const structuredHits = countMatches(capabilities, STRUCTURED_OUTPUT_CAPABILITIES);
+    const patchExecutionHits = countMatches(capabilities, PATCH_JSON_EXECUTION_CAPABILITIES);
+    const patchReliabilityHits = countMatches(capabilities, PATCH_JSON_RELIABILITY_CAPABILITIES);
+    const hasIterativeCoding = capabilities.includes("iterative_coding");
+    const hasToolRunner = capabilities.includes("tool_runner");
+    const usageBoost = agent.bestUsage && PHASE_BEST_USAGE[phase].includes(agent.bestUsage) ? 2 : 0;
+    const patchJsonBuilder = phase === "builder" && builderMode === "patch_json";
+    let score = 0;
+    if (phase === "architect" || phase === "critic" || phase === "interpreter") {
+        score += reasoning * 3 + rating * 2;
+    }
+    else if (phase === "builder") {
+        score += rating * 2 + reasoning;
+    }
+    else {
+        score += rating + reasoning * 0.5;
+    }
+    score += capHits * 4;
+    if (capHits === 0)
+        score -= 3;
+    score += requiredHits * 2;
+    if (patchJsonBuilder)
+        score += requiredHits * 3;
+    score += usageBoost;
+    if (patchJsonBuilder) {
+        score += structuredHits * 14;
+        score += patchExecutionHits * 4;
+        score += patchReliabilityHits * 3;
+        if (structuredHits === 0)
+            score -= 24;
+        if (patchExecutionHits === 0)
+            score -= 8;
+        if (!hasIterativeCoding)
+            score -= 8;
+        else
+            score += 8;
+        if (requiredHits === 0)
+            score -= 12;
+        if (agent.supportsTools === false)
+            score -= 14;
+        else
+            score += 3;
+        if (hasToolRunner)
+            score += 2;
+    }
+    const prefersStructuredBuilder = patchJsonBuilder && structuredHits > 0;
+    const costPenalty = phase === "builder"
+        ? prefersStructuredBuilder ? 1.5 : patchJsonBuilder ? 2 : 5
+        : phase === "librarian" ? 4 : phase === "architect" ? 1.5 : 1;
+    if (phase === "builder") {
+        score -= cost * costPenalty;
+        score -= maxComplexity * 0.5;
+    }
+    else if (phase === "librarian") {
+        score -= cost * costPenalty;
+        score -= maxComplexity;
+    }
+    else {
+        score -= cost * costPenalty;
+    }
+    return score;
+};
+const resolveOverrideAgent = async (repo, agentRef) => {
+    const agent = (await repo.getAgentById(agentRef)) ?? (await repo.getAgentBySlug(agentRef));
+    if (!agent) {
+        throw new Error(`Agent ${agentRef} not found`);
+    }
+    const capabilities = await repo.getAgentCapabilities(agent.id);
+    const resolved = await resolveAgentConfigFromRecord(agent, repo);
+    return { agent, capabilities, resolved };
+};
+export const selectPhaseAgents = async (options) => {
+    const repo = await GlobalRepository.create();
+    try {
+        const agents = await repo.listAgents();
+        const capCache = new Map();
+        const readinessCache = new Map();
+        const getCaps = async (agent) => {
+            const cached = capCache.get(agent.id);
+            if (cached)
+                return cached;
+            const caps = await repo.getAgentCapabilities(agent.id);
+            capCache.set(agent.id, caps);
+            return caps;
+        };
+        const getReadiness = async (agent) => {
+            const cached = readinessCache.get(agent.id);
+            if (cached)
+                return cached;
+            const [auth, health] = await Promise.all([
+                repo.getAgentAuthMetadata(agent.id),
+                repo.getAgentHealth(agent.id),
+            ]);
+            const readiness = {
+                authConfigured: auth.configured,
+                healthStatus: health?.status,
+            };
+            readinessCache.set(agent.id, readiness);
+            return readiness;
+        };
+        const buildSelection = async (phase) => {
+            const excludedIds = new Set(options.excludeAgentIds?.[phase] ?? []);
+            const overrideRef = options.overrides[phase];
+            if (overrideRef) {
+                const resolvedOverride = await resolveOverrideAgent(repo, overrideRef);
+                return {
+                    phase,
+                    agent: resolvedOverride.agent,
+                    capabilities: resolvedOverride.capabilities,
+                    resolved: resolvedOverride.resolved,
+                    source: "override",
+                    reason: "routing.agent override",
+                };
+            }
+            const scored = [];
+            const patchJsonBuilder = phase === "builder" && options.builderMode === "patch_json";
+            for (const agent of agents) {
+                if (excludedIds.has(agent.id))
+                    continue;
+                if (!agent.defaultModel)
+                    continue;
+                if (!options.allowCloudModels && isCloudModel(agent))
+                    continue;
+                const readiness = await getReadiness(agent);
+                if (readiness.healthStatus === "unreachable")
+                    continue;
+                const caps = await getCaps(agent);
+                const requiredHits = countMatches(caps, PHASE_REQUIRED_CAPS[phase]);
+                const structuredHits = countMatches(caps, STRUCTURED_OUTPUT_CAPABILITIES);
+                const patchExecutionHits = countMatches(caps, PATCH_JSON_EXECUTION_CAPABILITIES);
+                const score = adjustScoreForReadiness(scoreAgent(phase, agent, caps, options.builderMode), agent, readiness);
+                if (!Number.isFinite(score))
+                    continue;
+                scored.push({
+                    agent,
+                    caps,
+                    score,
+                    requiredHits,
+                    structuredHits,
+                    patchExecutionHits,
+                });
+            }
+            const hasRequired = scored.some((candidate) => candidate.requiredHits > 0);
+            let candidates = hasRequired
+                ? scored.filter((candidate) => candidate.requiredHits > 0)
+                : scored;
+            if (patchJsonBuilder) {
+                const structuredCandidates = candidates.filter((candidate) => candidate.structuredHits > 0);
+                if (structuredCandidates.length > 0) {
+                    candidates = structuredCandidates;
+                }
+                else {
+                    const patchExecutionCandidates = candidates.filter((candidate) => candidate.patchExecutionHits > 0);
+                    if (patchExecutionCandidates.length > 0) {
+                        candidates = patchExecutionCandidates;
+                    }
+                }
+            }
+            candidates.sort((a, b) => b.score - a.score);
+            for (const candidate of candidates) {
+                try {
+                    const resolved = await resolveAgentConfigFromRecord(candidate.agent, repo);
+                    return {
+                        phase,
+                        agent: candidate.agent,
+                        capabilities: candidate.caps,
+                        resolved,
+                        source: "auto",
+                        score: candidate.score,
+                        reason: "scored capability match",
+                    };
+                }
+                catch {
+                    continue;
+                }
+            }
+            if (options.fallbackAgent) {
+                const fallbackCaps = await getCaps(options.fallbackAgent.agent);
+                return {
+                    phase,
+                    agent: options.fallbackAgent.agent,
+                    capabilities: fallbackCaps,
+                    resolved: options.fallbackAgent,
+                    source: "fallback",
+                    reason: "auto selection failed",
+                };
+            }
+            return { phase, capabilities: [], source: "none", reason: "no eligible agents" };
+        };
+        const result = {
+            librarian: await buildSelection("librarian"),
+            architect: await buildSelection("architect"),
+            builder: await buildSelection("builder"),
+            critic: await buildSelection("critic"),
+            interpreter: await buildSelection("interpreter"),
+        };
+        return result;
+    }
+    finally {
+        await repo.close();
+    }
+};

package/dist/cli/EvalCommand.d.ts

@@ -0,0 +1,37 @@
+export declare const EVAL_EXIT_CODES: {
+    readonly usage_error: 2;
+    readonly suite_validation_error: 3;
+    readonly run_failure: 4;
+    readonly gate_failure: 5;
+};
+type EvalExitCode = (typeof EVAL_EXIT_CODES)[keyof typeof EVAL_EXIT_CODES];
+export declare class EvalCommandError extends Error {
+    readonly exitCode: EvalExitCode;
+    constructor(message: string, exitCode: EvalExitCode);
+}
+export interface ParsedEvalArgs {
+    suite_path?: string;
+    output: "text" | "json";
+    baseline_path?: string;
+    report_dir?: string;
+    workspace_root?: string;
+    config_path?: string;
+    provider?: string;
+    model?: string;
+    api_key?: string;
+    base_url?: string;
+    agent?: string;
+    agent_id?: string;
+    agent_slug?: string;
+    workflow_profile?: string;
+    smart?: boolean;
+    no_deep_investigation?: boolean;
+    help?: boolean;
+}
+export declare const parseEvalArgs: (argv: string[]) => ParsedEvalArgs;
+export declare class EvalCommand {
+    static helpText(): string;
+    static run(argv: string[]): Promise<void>;
+}
+export {};
+//# sourceMappingURL=EvalCommand.d.ts.map

package/dist/cli/EvalCommand.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalCommand.d.ts","sourceRoot":"","sources":["../../src/cli/EvalCommand.ts"],"names":[],"mappings":"AAeA,eAAO,MAAM,eAAe;;;;;CAKlB,CAAC;AAEX,KAAK,YAAY,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAC;AAE3E,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;gBAEpB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY;CAKpD;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AA+BD,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,KAAG,cAkG9C,CAAC;AA2DF,qBAAa,WAAW;IACtB,MAAM,CAAC,QAAQ,IAAI,MAAM;WAIZ,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAqJhD"}

package/dist/cli/EvalCommand.js

@@ -0,0 +1,333 @@
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+import process from "node:process";
+import { loadConfig } from "../config/ConfigLoader.js";
+import { EvalTaskExecutor } from "../eval/EvalTaskExecutor.js";
+import { EvalRunner } from "../eval/EvalRunner.js";
+import { evaluateGates, resolveGateThresholds } from "../eval/GateEvaluator.js";
+import { aggregateMetrics } from "../eval/MetricsAggregator.js";
+import { compareAgainstBaseline } from "../eval/RegressionComparator.js";
+import { serializeEvalReport } from "../eval/ReportSerializer.js";
+import { ReportStore } from "../eval/ReportStore.js";
+import { loadSuiteFromFile } from "../eval/SuiteLoader.js";
+import { SuiteValidationError } from "../eval/SuiteSchema.js";
+import { resolveWorkspaceRoot } from "./RunCommand.js";
+export const EVAL_EXIT_CODES = {
+    usage_error: 2,
+    suite_validation_error: 3,
+    run_failure: 4,
+    gate_failure: 5,
+};
+export class EvalCommandError extends Error {
+    constructor(message, exitCode) {
+        super(message);
+        this.name = "EvalCommandError";
+        this.exitCode = exitCode;
+    }
+}
+const HELP_TEXT = "Usage: codali eval --suite <path> [options]\n\n"
+    + "Options:\n"
+    + "  --suite <path>               Path to eval suite JSON (required)\n"
+    + "  --output <text|json>         Output mode (default: text)\n"
+    + "  --baseline <path>            Optional baseline report for regression diff\n"
+    + "  --report-dir <path>          Override eval report output directory\n"
+    + "  --workspace-root <path>      Workspace root for task execution\n"
+    + "  --provider <name>            Provider override passed to task runs\n"
+    + "  --model <name>               Model override passed to task runs\n"
+    + "  --api-key <token>            API key override passed to task runs\n"
+    + "  --base-url <url>             Base URL override passed to task runs\n"
+    + "  --agent <slug>               Agent override passed to task runs\n"
+    + "  --agent-id <id>              Agent id override passed to task runs\n"
+    + "  --agent-slug <slug>          Agent slug override passed to task runs\n"
+    + "  --profile <name>             Workflow profile override for task runs\n"
+    + "  --smart                      Force smart mode for task runs\n"
+    + "  --no-deep-investigation      Disable deep investigation for task runs\n"
+    + "  --config <path>              Config file path\n"
+    + "  --help                       Show help\n";
+const expectValue = (argv, index, flag) => {
+    const value = argv[index + 1];
+    if (!value || value.startsWith("--")) {
+        throw new EvalCommandError(`Missing value for ${flag}.`, EVAL_EXIT_CODES.usage_error);
+    }
+    return value;
+};
+export const parseEvalArgs = (argv) => {
+    const parsed = {
+        output: "text",
+    };
+    for (let index = 0; index < argv.length; index += 1) {
+        const arg = argv[index];
+        if (arg === "--help" || arg === "-h") {
+            parsed.help = true;
+            continue;
+        }
+        if (arg === "--suite") {
+            parsed.suite_path = expectValue(argv, index, "--suite");
+            index += 1;
+            continue;
+        }
+        if (arg === "--output") {
+            const value = expectValue(argv, index, "--output").trim().toLowerCase();
+            if (value !== "text" && value !== "json") {
+                throw new EvalCommandError("Invalid --output value. Expected text|json.", EVAL_EXIT_CODES.usage_error);
+            }
+            parsed.output = value;
+            index += 1;
+            continue;
+        }
+        if (arg === "--baseline") {
+            parsed.baseline_path = expectValue(argv, index, "--baseline");
+            index += 1;
+            continue;
+        }
+        if (arg === "--report-dir") {
+            parsed.report_dir = expectValue(argv, index, "--report-dir");
+            index += 1;
+            continue;
+        }
+        if (arg === "--workspace-root") {
+            parsed.workspace_root = expectValue(argv, index, "--workspace-root");
+            index += 1;
+            continue;
+        }
+        if (arg === "--config") {
+            parsed.config_path = expectValue(argv, index, "--config");
+            index += 1;
+            continue;
+        }
+        if (arg === "--provider") {
+            parsed.provider = expectValue(argv, index, "--provider");
+            index += 1;
+            continue;
+        }
+        if (arg === "--model") {
+            parsed.model = expectValue(argv, index, "--model");
+            index += 1;
+            continue;
+        }
+        if (arg === "--api-key") {
+            parsed.api_key = expectValue(argv, index, "--api-key");
+            index += 1;
+            continue;
+        }
+        if (arg === "--base-url") {
+            parsed.base_url = expectValue(argv, index, "--base-url");
+            index += 1;
+            continue;
+        }
+        if (arg === "--agent") {
+            parsed.agent = expectValue(argv, index, "--agent");
+            index += 1;
+            continue;
+        }
+        if (arg === "--agent-id") {
+            parsed.agent_id = expectValue(argv, index, "--agent-id");
+            index += 1;
+            continue;
+        }
+        if (arg === "--agent-slug") {
+            parsed.agent_slug = expectValue(argv, index, "--agent-slug");
+            index += 1;
+            continue;
+        }
+        if (arg === "--profile" || arg === "--workflow-profile") {
+            parsed.workflow_profile = expectValue(argv, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--smart") {
+            parsed.smart = true;
+            continue;
+        }
+        if (arg === "--no-deep-investigation") {
+            parsed.no_deep_investigation = true;
+            continue;
+        }
+        throw new EvalCommandError(`Unknown eval flag: ${arg}`, EVAL_EXIT_CODES.usage_error);
+    }
+    return parsed;
+};
+const formatRate = (value) => value === null ? "n/a" : `${(value * 100).toFixed(2)}%`;
+const formatNullable = (value, unit = "") => value === null ? "n/a" : `${value.toFixed(2)}${unit}`;
+const printTextReport = (report, reportPath) => {
+    const metrics = report.metrics;
+    const gateSummary = report.gates.passed
+        ? "passed"
+        : `failed (${report.gates.failures.map((failure) => failure.code).join(", ")})`;
+    const lines = [
+        `Eval suite: ${report.suite.suite_name} (${report.suite.suite_id})`,
+        `Tasks: ${report.summary.task_passed}/${report.summary.task_total} passed`,
+        `Execution errors: ${report.summary.execution_errors}`,
+        `M-001 task success: ${formatRate(metrics.m001_task_success_rate.value)}`,
+        `M-002 first-pass success: ${formatRate(metrics.m002_first_pass_success_rate.value)}`,
+        `M-003 patch apply success: ${formatRate(metrics.m003_patch_apply_success_rate.value)}`,
+        `M-004 verification pass: ${formatRate(metrics.m004_verification_pass_rate.value)}`,
+        `M-005 hallucination rate: ${formatRate(metrics.m005_hallucination_rate.value)}`,
+        `M-006 scope violation rate: ${formatRate(metrics.m006_scope_violation_rate.value)}`,
+        `M-007 latency median/p95: ${formatNullable(metrics.m007_latency_ms.median, "ms")}/${formatNullable(metrics.m007_latency_ms.p95, "ms")}`,
+        `M-008 success tokens median/p95: ${formatNullable(metrics.m008_success_tokens.median)}/${formatNullable(metrics.m008_success_tokens.p95)}`,
+        `M-008 success cost median/p95: ${formatNullable(metrics.m008_success_cost_usd.median, " USD")}/${formatNullable(metrics.m008_success_cost_usd.p95, " USD")}`,
+        `Regression gates: ${gateSummary}`,
+        `Report: ${reportPath}`,
+    ];
+    // eslint-disable-next-line no-console
+    console.log(lines.join("\n"));
+};
+const loadBaselineReport = async (params) => {
+    const { parsed, suitePath, suiteBaselinePath, store } = params;
+    const suiteDir = path.dirname(suitePath);
+    if (parsed.baseline_path) {
+        const baselinePath = await store.resolvePath(parsed.baseline_path, process.cwd());
+        return { path: baselinePath, report: await store.read(baselinePath) };
+    }
+    if (suiteBaselinePath) {
+        const baselinePath = await store.resolvePath(suiteBaselinePath, suiteDir);
+        return { path: baselinePath, report: await store.read(baselinePath) };
+    }
+    const latest = await store.findLatestForSuite({
+        suite_fingerprint: params.suiteFingerprint,
+        exclude_report_id: params.reportId,
+    });
+    if (!latest)
+        return {};
+    return { path: latest.path, report: latest.report };
+};
+export class EvalCommand {
+    static helpText() {
+        return HELP_TEXT;
+    }
+    static async run(argv) {
+        const parsed = parseEvalArgs(argv);
+        if (parsed.help) {
+            // eslint-disable-next-line no-console
+            console.log(HELP_TEXT);
+            return;
+        }
+        if (!parsed.suite_path) {
+            throw new EvalCommandError("Missing required --suite <path>.", EVAL_EXIT_CODES.usage_error);
+        }
+        const resolvedWorkspaceRoot = resolveWorkspaceRoot(process.cwd(), parsed.workspace_root);
+        const cliConfig = {};
+        if (parsed.workspace_root)
+            cliConfig.workspaceRoot = parsed.workspace_root;
+        if (parsed.provider)
+            cliConfig.provider = parsed.provider;
+        if (parsed.model)
+            cliConfig.model = parsed.model;
+        if (parsed.api_key)
+            cliConfig.apiKey = parsed.api_key;
+        if (parsed.base_url)
+            cliConfig.baseUrl = parsed.base_url;
+        if (parsed.workflow_profile)
+            cliConfig.workflow = { profile: parsed.workflow_profile };
+        if (parsed.smart !== undefined)
+            cliConfig.smart = parsed.smart;
+        if (parsed.no_deep_investigation !== undefined) {
+            cliConfig.deepInvestigation = { enabled: !parsed.no_deep_investigation };
+        }
+        const config = await loadConfig({
+            cli: cliConfig,
+            cwd: resolvedWorkspaceRoot,
+            configPath: parsed.config_path,
+        });
+        let loadedSuite;
+        try {
+            loadedSuite = await loadSuiteFromFile(parsed.suite_path, process.cwd());
+        }
+        catch (error) {
+            if (error instanceof SuiteValidationError) {
+                const issueSummary = error.issues
+                    .map((issue) => `${issue.path}:${issue.code}`)
+                    .join(", ");
+                throw new EvalCommandError(`Suite validation failed: ${issueSummary}`, EVAL_EXIT_CODES.suite_validation_error);
+            }
+            throw error;
+        }
+        const executor = new EvalTaskExecutor({
+            workspace_root: config.workspaceRoot,
+            suite_dir: loadedSuite.suite_dir,
+            provider: parsed.provider ?? (config.provider || undefined),
+            model: parsed.model ?? (config.model || undefined),
+            api_key: parsed.api_key ?? config.apiKey,
+            base_url: parsed.base_url ?? config.baseUrl,
+            agent: parsed.agent,
+            agent_id: parsed.agent_id,
+            agent_slug: parsed.agent_slug,
+            workflow_profile: parsed.workflow_profile,
+            smart: parsed.smart,
+            no_deep_investigation: parsed.no_deep_investigation,
+        });
+        const runner = new EvalRunner({
+            suite_id: loadedSuite.suite.suite_id,
+            suite_fingerprint: loadedSuite.suite_fingerprint,
+            tasks: loadedSuite.suite.tasks,
+            executor,
+        });
+        const runResult = await runner.run();
+        const metrics = aggregateMetrics(runResult);
+        const reportId = randomUUID();
+        const reportStore = new ReportStore(config.workspaceRoot, parsed.report_dir ?? config.eval?.report_dir);
+        const suiteBaselinePath = loadedSuite.suite.baseline?.mode === "none"
+            ? undefined
+            : loadedSuite.suite.baseline?.report_path;
+        const baseline = await loadBaselineReport({
+            parsed,
+            suitePath: loadedSuite.suite_path,
+            suiteBaselinePath,
+            store: reportStore,
+            suiteFingerprint: loadedSuite.suite_fingerprint,
+            reportId,
+        });
+        const regression = compareAgainstBaseline({
+            current: metrics,
+            baseline: baseline.report?.metrics,
+            baseline_report_id: baseline.report?.report_id,
+            baseline_created_at: baseline.report?.created_at,
+        });
+        const thresholds = resolveGateThresholds(config.eval?.gates, loadedSuite.suite.thresholds);
+        const gates = evaluateGates({ metrics, thresholds, comparison: regression });
+        const hasRunFailures = runResult.summary.failed > 0 || runResult.summary.execution_errors > 0;
+        const exitCode = hasRunFailures
+            ? EVAL_EXIT_CODES.run_failure
+            : (!gates.passed ? EVAL_EXIT_CODES.gate_failure : 0);
+        const report = {
+            schema_version: 1,
+            report_id: reportId,
+            created_at: new Date().toISOString(),
+            suite: {
+                suite_id: loadedSuite.suite.suite_id,
+                suite_name: loadedSuite.suite.name,
+                suite_path: loadedSuite.suite_path,
+                suite_fingerprint: loadedSuite.suite_fingerprint,
+                task_count: loadedSuite.suite.tasks.length,
+            },
+            summary: {
+                exit_code: exitCode,
+                passed: exitCode === 0,
+                gate_passed: gates.passed,
+                task_total: runResult.summary.total,
+                task_passed: runResult.summary.passed,
+                task_failed: runResult.summary.failed,
+                execution_errors: runResult.summary.execution_errors,
+            },
+            run: runResult,
+            metrics,
+            regression,
+            gates,
+        };
+        const reportPath = await reportStore.save(report);
+        if (parsed.output === "json") {
+            // eslint-disable-next-line no-console
+            console.log(serializeEvalReport(report, true));
+        }
+        else {
+            printTextReport(report, reportPath);
+        }
+        if (exitCode === EVAL_EXIT_CODES.run_failure) {
+            throw new EvalCommandError("Eval run completed with failed tasks or execution errors.", EVAL_EXIT_CODES.run_failure);
+        }
+        if (exitCode === EVAL_EXIT_CODES.gate_failure) {
+            throw new EvalCommandError("Eval regression gates failed.", EVAL_EXIT_CODES.gate_failure);
+        }
+    }
+}

package/dist/cli/FeedbackCommand.d.ts

@@ -0,0 +1,22 @@
+import { MemoryWriteback } from "../cognitive/MemoryWriteback.js";
+import { PostMortemAnalyzer } from "../cognitive/PostMortemAnalyzer.js";
+import type { CodaliConfig } from "../config/Config.js";
+import { DocdexClient } from "../docdex/DocdexClient.js";
+import { createProvider } from "../providers/ProviderRegistry.js";
+import type { Provider } from "../providers/ProviderTypes.js";
+interface FeedbackCommandDependencies {
+    cwd?: () => string;
+    loadConfig?: (options: {
+        cwd: string;
+    }) => Promise<CodaliConfig>;
+    createDocdexClient?: (config: CodaliConfig) => DocdexClient;
+    createProvider?: typeof createProvider;
+    createMemoryWriteback?: (client: DocdexClient, options: ConstructorParameters<typeof MemoryWriteback>[1]) => Pick<MemoryWriteback, "persist">;
+    createAnalyzer?: (provider: Provider, workspaceRoot: string) => Pick<PostMortemAnalyzer, "analyze">;
+    log?: (line: string) => void;
+}
+export declare class FeedbackCommand {
+    static run(argv: string[], deps?: FeedbackCommandDependencies): Promise<void>;
+}
+export {};
+//# sourceMappingURL=FeedbackCommand.d.ts.map