@mcoda/mswarm 0.1.57 → 0.1.61

package/dist/vendor/codali/eval/ReportInputAdapter.js

@@ -0,0 +1,229 @@
+const asRecord = (value) => {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return undefined;
+    return value;
+};
+const asString = (value) => {
+    if (typeof value !== "string")
+        return undefined;
+    const trimmed = value.trim();
+    return trimmed.length ? trimmed : undefined;
+};
+const asBoolean = (value) => {
+    if (typeof value === "boolean")
+        return value;
+    if (value === null)
+        return null;
+    return null;
+};
+const asNumber = (value) => {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return null;
+    return value;
+};
+const uniqueSortedStrings = (value) => {
+    if (!Array.isArray(value))
+        return [];
+    return Array.from(new Set(value
+        .filter((entry) => typeof entry === "string")
+        .map((entry) => entry.trim())
+        .filter((entry) => entry.length > 0))).sort((left, right) => left.localeCompare(right));
+};
+const normalizeRunStatus = (value) => {
+    if (value === "pass" || value === "fail" || value === "degraded")
+        return value;
+    return "unknown";
+};
+const normalizePhaseStatus = (value) => {
+    if (value === "available" || value === "missing" || value === "degraded")
+        return value;
+    return "missing";
+};
+const normalizeVerificationOutcome = (value) => {
+    if (value === "verified_passed"
+        || value === "verified_failed"
+        || value === "unverified_with_reason") {
+        return value;
+    }
+    return null;
+};
+const normalizeArtifactReferences = (value) => {
+    if (!Array.isArray(value))
+        return [];
+    const results = [];
+    for (const entry of value) {
+        const record = asRecord(entry);
+        if (!record)
+            continue;
+        const phase = asString(record.phase);
+        const kind = asString(record.kind);
+        if (!phase || !kind)
+            continue;
+        results.push({
+            phase,
+            kind,
+            status: record.status === "missing" ? "missing" : "present",
+            path: asString(record.path) ?? null,
+            reason_code: asString(record.reason_code) ?? null,
+        });
+    }
+    return results.sort((left, right) => `${left.phase}:${left.kind}:${left.path ?? ""}`.localeCompare(`${right.phase}:${right.kind}:${right.path ?? ""}`));
+};
+const normalizePhaseTelemetry = (value) => {
+    if (!Array.isArray(value))
+        return [];
+    const sources = [];
+    for (const entry of value) {
+        const record = asRecord(entry);
+        if (!record)
+            continue;
+        const phase = asString(record.phase);
+        if (!phase)
+            continue;
+        const usage = asRecord(record.usage);
+        const cost = asRecord(record.cost);
+        const inputTokens = asNumber(usage?.input_tokens);
+        const outputTokens = asNumber(usage?.output_tokens);
+        const totalTokens = asNumber(usage?.total_tokens)
+            ?? (inputTokens !== null || outputTokens !== null
+                ? (inputTokens ?? 0) + (outputTokens ?? 0)
+                : null);
+        sources.push({
+            phase,
+            duration_ms: asNumber(record.duration_ms),
+            provider: asString(record.provider) ?? null,
+            model: asString(record.model) ?? null,
+            input_tokens: inputTokens,
+            output_tokens: outputTokens,
+            total_tokens: totalTokens,
+            cost_usd: asNumber(cost?.usd),
+            cost_source: asString(cost?.source) ?? null,
+            missing_usage_reason: asString(record.missing_usage_reason) ?? null,
+            missing_cost_reason: asString(record.missing_cost_reason) ?? null,
+        });
+    }
+    return sources.sort((left, right) => left.phase.localeCompare(right.phase));
+};
+const phaseKeyToSummaryPhase = (key) => {
+    if (key === "plan")
+        return "plan";
+    if (key === "retrieval")
+        return "retrieve";
+    if (key === "patch")
+        return "act";
+    if (key === "verification")
+        return "verify";
+    return undefined;
+};
+const buildPhaseOutcomes = (runSummary, telemetry) => {
+    const quality = asRecord(runSummary.quality_dimensions);
+    const phaseStatus = new Map();
+    if (quality) {
+        for (const [key, rawValue] of Object.entries(quality)) {
+            const phase = phaseKeyToSummaryPhase(key);
+            if (!phase)
+                continue;
+            phaseStatus.set(phase, normalizePhaseStatus(rawValue));
+        }
+    }
+    for (const entry of telemetry) {
+        if (!phaseStatus.has(entry.phase)) {
+            phaseStatus.set(entry.phase, "missing");
+        }
+    }
+    if (phaseStatus.size === 0) {
+        for (const phase of ["plan", "retrieve", "act", "verify"]) {
+            phaseStatus.set(phase, "missing");
+        }
+    }
+    const telemetryByPhase = new Map();
+    for (const entry of telemetry) {
+        telemetryByPhase.set(entry.phase, entry);
+    }
+    const outcomes = [];
+    for (const [phase, status] of phaseStatus.entries()) {
+        const source = telemetryByPhase.get(phase);
+        outcomes.push({
+            phase,
+            status,
+            duration_ms: source?.duration_ms ?? null,
+            provider: source?.provider ?? null,
+            model: source?.model ?? null,
+            input_tokens: source?.input_tokens ?? null,
+            output_tokens: source?.output_tokens ?? null,
+            total_tokens: source?.total_tokens ?? null,
+            cost_usd: source?.cost_usd ?? null,
+            cost_source: source?.cost_source ?? null,
+            missing_usage_reason: source?.missing_usage_reason ?? null,
+            missing_cost_reason: source?.missing_cost_reason ?? null,
+        });
+    }
+    return outcomes.sort((left, right) => left.phase.localeCompare(right.phase));
+};
+const sumNullable = (values) => {
+    const present = values.filter((entry) => entry !== null);
+    if (!present.length)
+        return null;
+    return present.reduce((sum, value) => sum + value, 0);
+};
+export const adaptRunSummaryForReport = (input = {}) => {
+    const runSummary = asRecord(input.runSummary);
+    const finalDisposition = asRecord(runSummary?.final_disposition);
+    const artifactReferences = normalizeArtifactReferences(runSummary?.artifact_references);
+    const phaseTelemetry = normalizePhaseTelemetry(runSummary?.phase_telemetry);
+    const phaseOutcomes = buildPhaseOutcomes(runSummary ?? {}, phaseTelemetry);
+    const topLevelUsage = asRecord(runSummary?.usage);
+    const topLevelTotalTokens = asNumber(topLevelUsage?.totalTokens)
+        ?? (asNumber(topLevelUsage?.inputTokens) !== null || asNumber(topLevelUsage?.outputTokens) !== null
+            ? (asNumber(topLevelUsage?.inputTokens) ?? 0) + (asNumber(topLevelUsage?.outputTokens) ?? 0)
+            : null);
+    const usageTokensTotal = topLevelTotalTokens ?? sumNullable(phaseOutcomes.map((phase) => phase.total_tokens));
+    const topLevelCost = asNumber(runSummary?.actualCost);
+    const phaseCost = sumNullable(phaseOutcomes.map((phase) => phase.cost_usd));
+    const costUsd = topLevelCost ?? phaseCost;
+    const missingArtifacts = uniqueSortedStrings(runSummary?.missing_artifacts
+        ?? artifactReferences
+            .filter((entry) => entry.status === "missing")
+            .map((entry) => `${entry.phase}:${entry.kind}`));
+    const verificationRecord = asRecord(runSummary?.verification);
+    const verificationOutcome = normalizeVerificationOutcome(input.verificationOutcome)
+        ?? normalizeVerificationOutcome(verificationRecord?.outcome);
+    const markers = new Set();
+    if (!runSummary)
+        markers.add("run_summary_missing");
+    if (!asString(runSummary?.run_id ?? runSummary?.runId ?? input.runId))
+        markers.add("run_id_missing");
+    if (!finalDisposition)
+        markers.add("final_disposition_missing");
+    if (!phaseTelemetry.length)
+        markers.add("phase_telemetry_missing");
+    if (verificationOutcome === null)
+        markers.add("verification_outcome_missing");
+    if (usageTokensTotal === null)
+        markers.add("usage_tokens_missing");
+    if (costUsd === null)
+        markers.add("cost_missing");
+    const touchedFiles = Array.from(new Set([
+        ...uniqueSortedStrings(runSummary?.touchedFiles),
+        ...(Array.isArray(input.touchedFiles) ? input.touchedFiles : []),
+    ])).sort((left, right) => left.localeCompare(right));
+    return {
+        schema_version: 1,
+        run_id: asString(runSummary?.run_id ?? runSummary?.runId ?? input.runId) ?? null,
+        task_id: asString(runSummary?.task_id ?? runSummary?.taskId ?? input.taskId) ?? null,
+        fingerprint: asString(runSummary?.fingerprint) ?? null,
+        duration_ms: asNumber(runSummary?.durationMs),
+        final_status: normalizeRunStatus(finalDisposition?.status),
+        failure_class: asString(finalDisposition?.failure_class ?? finalDisposition?.failureClass) ?? null,
+        reason_codes: uniqueSortedStrings(finalDisposition?.reason_codes ?? finalDisposition?.reasons),
+        retryable: asBoolean(finalDisposition?.retryable),
+        verification_outcome: verificationOutcome,
+        touched_files: touchedFiles,
+        artifact_references: artifactReferences,
+        missing_artifacts: missingArtifacts,
+        phase_outcomes: phaseOutcomes,
+        usage_tokens_total: usageTokensTotal,
+        cost_usd: costUsd,
+        missing_data_markers: Array.from(markers).sort((left, right) => left.localeCompare(right)),
+    };
+};

package/dist/vendor/codali/eval/ReportSerializer.d.ts

@@ -0,0 +1,32 @@
+import type { EvalGateResult } from "./GateEvaluator.js";
+import type { EvalMetrics } from "./MetricTypes.js";
+import type { EvalRegressionComparison } from "./RegressionComparator.js";
+import type { EvalRunResult } from "./EvalRunner.js";
+export interface EvalReport {
+    schema_version: 1;
+    report_id: string;
+    created_at: string;
+    suite: {
+        suite_id: string;
+        suite_name: string;
+        suite_path: string;
+        suite_fingerprint: string;
+        task_count: number;
+    };
+    summary: {
+        exit_code: number;
+        passed: boolean;
+        gate_passed: boolean;
+        task_total: number;
+        task_passed: number;
+        task_failed: number;
+        execution_errors: number;
+    };
+    run: EvalRunResult;
+    metrics: EvalMetrics;
+    regression: EvalRegressionComparison;
+    gates: EvalGateResult;
+}
+export declare const serializeEvalReport: (report: EvalReport, pretty?: boolean) => string;
+export declare const parseEvalReport: (content: string) => EvalReport;
+//# sourceMappingURL=ReportSerializer.d.ts.map

package/dist/vendor/codali/eval/ReportSerializer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReportSerializer.d.ts","sourceRoot":"","sources":["../../src/eval/ReportSerializer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,WAAW,UAAU;IACzB,cAAc,EAAE,CAAC,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,OAAO,CAAC;QAChB,WAAW,EAAE,OAAO,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,GAAG,EAAE,aAAa,CAAC;IACnB,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,wBAAwB,CAAC;IACrC,KAAK,EAAE,cAAc,CAAC;CACvB;AAOD,eAAO,MAAM,mBAAmB,GAAI,QAAQ,UAAU,EAAE,gBAAa,KAAG,MAKvE,CAAC;AAEF,eAAO,MAAM,eAAe,GAAI,SAAS,MAAM,KAAG,UAkBjD,CAAC"}

package/dist/vendor/codali/eval/ReportSerializer.js

@@ -0,0 +1,33 @@
+import { stableJsonStringify } from "./SuiteSchema.js";
+const asRecord = (value) => {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return undefined;
+    return value;
+};
+export const serializeEvalReport = (report, pretty = true) => {
+    const stable = stableJsonStringify(report);
+    if (!pretty)
+        return stable;
+    const parsed = JSON.parse(stable);
+    return `${JSON.stringify(parsed, null, 2)}\n`;
+};
+export const parseEvalReport = (content) => {
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch (error) {
+        throw new Error(`Report JSON parse failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const report = asRecord(parsed);
+    if (!report) {
+        throw new Error("Report payload must be an object.");
+    }
+    if (report.schema_version !== 1) {
+        throw new Error("Unsupported report schema_version.");
+    }
+    if (typeof report.report_id !== "string" || !report.report_id.trim()) {
+        throw new Error("Report report_id is required.");
+    }
+    return report;
+};

package/dist/vendor/codali/eval/ReportStore.d.ts

@@ -0,0 +1,18 @@
+import { type EvalReport } from "./ReportSerializer.js";
+export declare class ReportStore {
+    readonly report_dir: string;
+    constructor(workspaceRoot: string, reportDir?: string);
+    private ensureDir;
+    save(report: EvalReport): Promise<string>;
+    read(reportPath: string): Promise<EvalReport>;
+    resolvePath(reportPath: string, cwd: string): Promise<string>;
+    listReports(): Promise<string[]>;
+    findLatestForSuite(params: {
+        suite_fingerprint: string;
+        exclude_report_id?: string;
+    }): Promise<{
+        path: string;
+        report: EvalReport;
+    } | undefined>;
+}
+//# sourceMappingURL=ReportStore.d.ts.map

package/dist/vendor/codali/eval/ReportStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReportStore.d.ts","sourceRoot":"","sources":["../../src/eval/ReportStore.ts"],"names":[],"mappings":"AAIA,OAAO,EAAwC,KAAK,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAuC9F,qBAAa,WAAW;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;gBAEhB,aAAa,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;YAKvC,SAAS;IAIjB,IAAI,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAUzC,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAK7C,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK7D,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAmBhC,kBAAkB,CAAC,MAAM,EAAE;QAC/B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,CAAA;KAAE,GAAG,SAAS,CAAC;CAe9D"}

package/dist/vendor/codali/eval/ReportStore.js

@@ -0,0 +1,96 @@
+import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { getGlobalWorkspaceDir } from "../runtime/StoragePaths.js";
+import { DEFAULT_LOG_DIR } from "../config/Config.js";
+import { parseEvalReport, serializeEvalReport } from "./ReportSerializer.js";
+const sanitize = (value) => value
+    .trim()
+    .replace(/[^a-z0-9._-]+/gi, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "")
+    .toLowerCase() || "suite";
+const fileTimestamp = (filePath) => {
+    const match = /-(\d{13})-[a-z0-9._-]+\.json$/i.exec(path.basename(filePath));
+    if (!match)
+        return undefined;
+    const parsed = Number(match[1]);
+    return Number.isFinite(parsed) ? parsed : undefined;
+};
+const reportSortKey = async (filePath) => {
+    let mtimeMs = 0;
+    try {
+        const info = await stat(filePath);
+        mtimeMs = info.mtimeMs;
+    }
+    catch {
+        mtimeMs = 0;
+    }
+    return {
+        // Prefer timestamp encoded in the report filename to avoid OS-specific mtime granularity.
+        timestamp: fileTimestamp(filePath) ?? Math.floor(mtimeMs),
+        mtimeMs,
+        fileName: path.basename(filePath).toLowerCase(),
+    };
+};
+export class ReportStore {
+    constructor(workspaceRoot, reportDir) {
+        const storageRoot = getGlobalWorkspaceDir(workspaceRoot);
+        this.report_dir = path.resolve(storageRoot, reportDir ?? path.join(DEFAULT_LOG_DIR, "eval"));
+    }
+    async ensureDir() {
+        await mkdir(this.report_dir, { recursive: true });
+    }
+    async save(report) {
+        await this.ensureDir();
+        const suiteId = sanitize(report.suite.suite_id);
+        const timestamp = Date.now();
+        const fileName = `${suiteId}-${timestamp}-${sanitize(report.report_id)}.json`;
+        const filePath = path.join(this.report_dir, fileName);
+        await writeFile(filePath, serializeEvalReport(report, true), "utf8");
+        return filePath;
+    }
+    async read(reportPath) {
+        const content = await readFile(reportPath, "utf8");
+        return parseEvalReport(content);
+    }
+    async resolvePath(reportPath, cwd) {
+        const candidate = path.isAbsolute(reportPath) ? reportPath : path.resolve(cwd, reportPath);
+        return path.resolve(candidate);
+    }
+    async listReports() {
+        await this.ensureDir();
+        const entries = await readdir(this.report_dir);
+        const jsonEntries = entries
+            .filter((entry) => entry.endsWith(".json"))
+            .map((entry) => path.join(this.report_dir, entry));
+        const withKeys = await Promise.all(jsonEntries.map(async (entry) => ({ entry, key: await reportSortKey(entry) })));
+        withKeys.sort((left, right) => {
+            const byTimestamp = right.key.timestamp - left.key.timestamp;
+            if (byTimestamp !== 0)
+                return byTimestamp;
+            const byMtime = right.key.mtimeMs - left.key.mtimeMs;
+            if (byMtime !== 0)
+                return byMtime;
+            return right.key.fileName.localeCompare(left.key.fileName);
+        });
+        return withKeys.map((entry) => entry.entry);
+    }
+    async findLatestForSuite(params) {
+        const reportFiles = await this.listReports();
+        for (const reportFile of reportFiles) {
+            let parsed;
+            try {
+                parsed = await this.read(reportFile);
+            }
+            catch {
+                continue;
+            }
+            if (parsed.suite.suite_fingerprint !== params.suite_fingerprint)
+                continue;
+            if (params.exclude_report_id && parsed.report_id === params.exclude_report_id)
+                continue;
+            return { path: reportFile, report: parsed };
+        }
+        return undefined;
+    }
+}

package/dist/vendor/codali/eval/SuiteLoader.d.ts

@@ -0,0 +1,12 @@
+import { type EvalSuiteDefinition, type EvalTaskDefinition } from "./SuiteSchema.js";
+export interface LoadedEvalSuite {
+    suite_path: string;
+    suite_dir: string;
+    suite_fingerprint: string;
+    suite: EvalSuiteDefinition;
+}
+export declare const hashSuiteDefinition: (suite: EvalSuiteDefinition) => string;
+export declare const resolveSuitePath: (suitePath: string, cwd: string) => string;
+export declare const resolveTaskFilePath: (task: EvalTaskDefinition, suiteDir: string, workspaceRoot: string) => string | undefined;
+export declare const loadSuiteFromFile: (suitePath: string, cwd?: string) => Promise<LoadedEvalSuite>;
+//# sourceMappingURL=SuiteLoader.d.ts.map

package/dist/vendor/codali/eval/SuiteLoader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SuiteLoader.d.ts","sourceRoot":"","sources":["../../src/eval/SuiteLoader.ts"],"names":[],"mappings":"AAGA,OAAO,EAIL,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACxB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,mBAAmB,CAAC;CAC5B;AAgBD,eAAO,MAAM,mBAAmB,GAAI,OAAO,mBAAmB,KAAG,MACc,CAAC;AAEhF,eAAO,MAAM,gBAAgB,GAAI,WAAW,MAAM,EAAE,KAAK,MAAM,KAAG,MAMjE,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAC9B,MAAM,kBAAkB,EACxB,UAAU,MAAM,EAChB,eAAe,MAAM,KACpB,MAAM,GAAG,SAMX,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAC5B,WAAW,MAAM,EACjB,YAAmB,KAClB,OAAO,CAAC,eAAe,CAiCzB,CAAC"}

package/dist/vendor/codali/eval/SuiteLoader.js

@@ -0,0 +1,51 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { normalizeSuiteDefinition, stableJsonStringify, SuiteValidationError, } from "./SuiteSchema.js";
+const toSuiteValidationError = (label, message, code, pathLabel = label) => new SuiteValidationError(`${label} validation failed.`, [
+    {
+        path: pathLabel,
+        code,
+        message,
+    },
+]);
+export const hashSuiteDefinition = (suite) => createHash("sha256").update(stableJsonStringify(suite), "utf8").digest("hex");
+export const resolveSuitePath = (suitePath, cwd) => {
+    const trimmed = suitePath.trim();
+    if (!trimmed) {
+        throw toSuiteValidationError("suite", "Suite path must be non-empty.", "missing_suite_path");
+    }
+    return path.resolve(cwd, trimmed);
+};
+export const resolveTaskFilePath = (task, suiteDir, workspaceRoot) => {
+    if (!task.task_file)
+        return undefined;
+    const candidate = path.isAbsolute(task.task_file)
+        ? task.task_file
+        : path.resolve(suiteDir, task.task_file);
+    return path.resolve(workspaceRoot, path.relative(workspaceRoot, candidate));
+};
+export const loadSuiteFromFile = async (suitePath, cwd = process.cwd()) => {
+    const resolvedPath = resolveSuitePath(suitePath, cwd);
+    let content = "";
+    try {
+        content = await readFile(resolvedPath, "utf8");
+    }
+    catch (error) {
+        throw toSuiteValidationError("suite", `Unable to read suite file: ${error instanceof Error ? error.message : String(error)}`, "suite_read_failed", resolvedPath);
+    }
+    let raw;
+    try {
+        raw = JSON.parse(content);
+    }
+    catch (error) {
+        throw toSuiteValidationError("suite", `Suite file is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, "suite_invalid_json", resolvedPath);
+    }
+    const suite = normalizeSuiteDefinition(raw, resolvedPath);
+    return {
+        suite_path: resolvedPath,
+        suite_dir: path.dirname(resolvedPath),
+        suite_fingerprint: hashSuiteDefinition(suite),
+        suite,
+    };
+};

package/dist/vendor/codali/eval/SuiteSchema.d.ts

@@ -0,0 +1,56 @@
+export declare const EVAL_COMMANDS: readonly ["run", "fix", "review", "explain", "test"];
+export type EvalCommandName = (typeof EVAL_COMMANDS)[number];
+export declare const EVAL_VERIFICATION_EXPECTATIONS: readonly ["verified_passed", "verified_failed", "unverified_with_reason", "any"];
+export type EvalVerificationExpectation = (typeof EVAL_VERIFICATION_EXPECTATIONS)[number];
+export interface EvalGateThresholds {
+    patch_apply_drop_max: number;
+    verification_pass_rate_min: number;
+    hallucination_rate_max: number;
+    scope_violation_rate_max: number;
+}
+export interface EvalSuiteBaseline {
+    mode: "previous" | "none";
+    report_path?: string;
+}
+export interface EvalTaskAssertions {
+    expect_success: boolean;
+    expect_exit_code?: number;
+    expect_patch_apply?: boolean;
+    expect_verification?: EvalVerificationExpectation;
+    max_latency_ms?: number;
+    max_cost_usd?: number;
+    allow_hallucination: boolean;
+    allow_scope_violation: boolean;
+}
+export interface EvalTaskDefinition {
+    id: string;
+    title: string;
+    description?: string;
+    mode: "success" | "failure";
+    command: EvalCommandName;
+    task_file?: string;
+    inline_task?: string;
+    args: string[];
+    assertions: EvalTaskAssertions;
+}
+export interface EvalSuiteDefinition {
+    schema_version: 1;
+    suite_id: string;
+    name: string;
+    description?: string;
+    thresholds?: Partial<EvalGateThresholds>;
+    baseline?: EvalSuiteBaseline;
+    tasks: EvalTaskDefinition[];
+}
+export interface SuiteValidationIssue {
+    path: string;
+    code: string;
+    message: string;
+}
+export declare class SuiteValidationError extends Error {
+    readonly issues: SuiteValidationIssue[];
+    constructor(message: string, issues: SuiteValidationIssue[]);
+}
+export declare const normalizeSuiteDefinition: (raw: unknown, label?: string) => EvalSuiteDefinition;
+export declare const stableJsonStringify: (value: unknown) => string;
+//# sourceMappingURL=SuiteSchema.d.ts.map

package/dist/vendor/codali/eval/SuiteSchema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SuiteSchema.d.ts","sourceRoot":"","sources":["../../src/eval/SuiteSchema.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,aAAa,sDAAuD,CAAC;AAClF,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7D,eAAO,MAAM,8BAA8B,kFAKjC,CAAC;AACX,MAAM,MAAM,2BAA2B,GAAG,CAAC,OAAO,8BAA8B,CAAC,CAAC,MAAM,CAAC,CAAC;AAE1F,MAAM,WAAW,kBAAkB;IACjC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,0BAA0B,EAAE,MAAM,CAAC;IACnC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,UAAU,GAAG,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,mBAAmB,CAAC,EAAE,2BAA2B,CAAC;IAClD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,qBAAqB,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,SAAS,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,eAAe,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,CAAC,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACzC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,KAAK,EAAE,kBAAkB,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,MAAM,EAAE,oBAAoB,EAAE,CAAC;gBAE5B,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,EAAE;CAK5D;AAsUD,eAAO,MAAM,wBAAwB,GAAI,KAAK,OAAO,EAAE,cAAe,KAAG,mBAyExE,CAAC;AAYF,eAAO,MAAM,mBAAmB,GAAI,OAAO,OAAO,KAAG,MAClB,CAAC"}