@kweaver-ai/kweaver-sdk 0.7.4 → 0.8.2

package/dist/trace-ai/eval-set/output-writer.js

@@ -0,0 +1,126 @@
+/**
+ * M5 eval-set output writer — handles directory layout, index upsert, shard
+ * merge, on-conflict resolution (fail / skip / overwrite), and .bak preservation.
+ *
+ * MVP layout: always one shard named `cases.yaml`. Users can manually split
+ * into multi-shard later (re-write `index.yaml` to reference more shards)
+ * and call `kweaver trace schema validate` to verify.
+ */
+import { readFile, writeFile, copyFile, mkdir } from "node:fs/promises";
+import path from "node:path";
+import yaml from "js-yaml";
+import { EvalSetIndexSchema, EvalSetShardSchema } from "./schemas.js";
+export class WriterError extends Error {
+    conflictIds;
+    constructor(message, conflictIds) {
+        super(message);
+        this.conflictIds = conflictIds;
+        this.name = "WriterError";
+    }
+}
+const SHARD_NAME = "cases.yaml";
+const INDEX_NAME = "index.yaml";
+async function fileExists(p) {
+    try {
+        await readFile(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function readShardCases(shardPath) {
+    if (!(await fileExists(shardPath)))
+        return [];
+    const raw = await readFile(shardPath, "utf8");
+    const parsed = yaml.load(raw);
+    const r = EvalSetShardSchema.safeParse(parsed);
+    if (!r.success) {
+        throw new WriterError(`existing shard at ${shardPath} fails schema validation: ${r.error.issues[0].message}`);
+    }
+    return r.data.cases;
+}
+export async function writeEvalSet(opts) {
+    const { outDir, evalSetId, newCases, onConflict } = opts;
+    // intra-batch duplicate detection
+    const seenInBatch = new Set();
+    const dupInBatch = [];
+    for (const c of newCases) {
+        if (seenInBatch.has(c.query_id))
+            dupInBatch.push(c.query_id);
+        seenInBatch.add(c.query_id);
+    }
+    if (dupInBatch.length > 0) {
+        throw new WriterError(`intra-batch duplicate query_id(s): ${dupInBatch.join(", ")}`, dupInBatch);
+    }
+    await mkdir(outDir, { recursive: true });
+    const shardPath = path.join(outDir, SHARD_NAME);
+    const existingCases = await readShardCases(shardPath);
+    const existingIds = new Set(existingCases.map((c) => c.query_id));
+    const incomingByConflict = newCases.filter((c) => existingIds.has(c.query_id));
+    const incomingFresh = newCases.filter((c) => !existingIds.has(c.query_id));
+    if (incomingByConflict.length > 0 && onConflict === "fail") {
+        throw new WriterError(`query_id conflict(s): ${incomingByConflict.map((c) => c.query_id).join(", ")}`, incomingByConflict.map((c) => c.query_id));
+    }
+    let mergedCases;
+    let casesWritten = 0;
+    let casesSkipped = 0;
+    if (onConflict === "skip") {
+        mergedCases = [...existingCases, ...incomingFresh];
+        casesWritten = incomingFresh.length;
+        casesSkipped = incomingByConflict.length;
+    }
+    else if (onConflict === "overwrite") {
+        if (incomingByConflict.length > 0 && (await fileExists(shardPath))) {
+            await copyFile(shardPath, shardPath + ".bak");
+        }
+        const overwriteIds = new Set(incomingByConflict.map((c) => c.query_id));
+        const kept = existingCases.filter((c) => !overwriteIds.has(c.query_id));
+        mergedCases = [...kept, ...incomingFresh, ...incomingByConflict];
+        casesWritten = incomingFresh.length + incomingByConflict.length;
+        casesSkipped = 0;
+    }
+    else {
+        // "fail" strategy — no conflicts at this point (would have thrown above)
+        mergedCases = [...existingCases, ...incomingFresh];
+        casesWritten = incomingFresh.length;
+        casesSkipped = 0;
+    }
+    const shardDoc = {
+        schema_version: "trace-eval-set/v1",
+        cases: mergedCases,
+    };
+    const shardCheck = EvalSetShardSchema.safeParse(shardDoc);
+    if (!shardCheck.success) {
+        throw new WriterError(`merged shard fails schema validation: ${shardCheck.error.issues[0].message}`);
+    }
+    await writeFile(shardPath, yaml.dump(shardDoc, { lineWidth: 120, noRefs: true }), "utf8");
+    const indexPath = path.join(outDir, INDEX_NAME);
+    let indexDoc;
+    if (await fileExists(indexPath)) {
+        const raw = await readFile(indexPath, "utf8");
+        const parsed = yaml.load(raw);
+        const r = EvalSetIndexSchema.safeParse(parsed);
+        if (!r.success) {
+            throw new WriterError(`existing index.yaml fails schema validation: ${r.error.issues[0].message}`);
+        }
+        indexDoc = r.data;
+        if (!indexDoc.shards.some((s) => s.path === SHARD_NAME)) {
+            indexDoc.shards.push({ path: SHARD_NAME });
+        }
+    }
+    else {
+        indexDoc = {
+            schema_version: "trace-eval-set-index/v1",
+            eval_set_id: evalSetId,
+            shards: [{ path: SHARD_NAME }],
+        };
+    }
+    await writeFile(indexPath, yaml.dump(indexDoc, { lineWidth: 120, noRefs: true }), "utf8");
+    return {
+        cases_written: casesWritten,
+        cases_skipped: casesSkipped,
+        conflicts: incomingByConflict.map((c) => c.query_id),
+        shard_paths: [shardPath],
+    };
+}

package/dist/trace-ai/eval-set/query-picker.d.ts

@@ -0,0 +1,37 @@
+/**
+ * M5 eval-set query picker — two lift functions:
+ *   - liftFromQueriesFile(path)   reads `trace-eval-set-input/v1` simplified yaml
+ *   - liftFromDiagnosis(dir)      reads M4 diagnose report yamls (added in Task 4)
+ *
+ * Both return EvalCase[] (without query_id auto-fill — that happens in builder.ts).
+ */
+import type { EvalCase } from "./types.js";
+export declare class QueryPickerError extends Error {
+    readonly path?: string | undefined;
+    constructor(message: string, path?: string | undefined);
+}
+export declare function liftFromQueriesFile(filePath: string): Promise<EvalCase[]>;
+export interface LiftFromDiagnosisResult {
+    cases: EvalCase[];
+    skipped_findings_count: number;
+    skipped_findings_summary: string[];
+}
+/**
+ * Read all *.yaml / *.yml files in `dirPath`, validate each as `trace-diagnose-report/v1`,
+ * and extract `findings[*].verify_with.suggested_eval_case` as EvalCases.
+ *
+ * Skips findings where:
+ *   - `suggested_eval_case.query` is null (M4 has no user query → can't construct EvalCase.input)
+ *   - `suggested_eval_case.assertions` is empty (refinement would fail; no reference either)
+ *
+ * Lifts:
+ *   - `EvalCase.input.user_message = suggested_eval_case.query`
+ *   - `EvalCase.query_id = suggested_eval_case.query_id ?? ""` (empty → builder.ensureQueryId fills)
+ *   - `EvalCase.assertions` = M4 string templates wrapped as placeholder `contains` assertions
+ *     with `_note` flagging "convert to structured manually"
+ *   - `EvalCase.reference = undefined` (M4 doesn't emit reference)
+ *
+ * Files that fail to schema-validate cause a fail-fast error (all *.yaml in dir
+ * must be diagnose reports — picker doesn't filter by content).
+ */
+export declare function liftFromDiagnosis(dirPath: string): Promise<LiftFromDiagnosisResult>;

package/dist/trace-ai/eval-set/query-picker.js

@@ -0,0 +1,147 @@
+/**
+ * M5 eval-set query picker — two lift functions:
+ *   - liftFromQueriesFile(path)   reads `trace-eval-set-input/v1` simplified yaml
+ *   - liftFromDiagnosis(dir)      reads M4 diagnose report yamls (added in Task 4)
+ *
+ * Both return EvalCase[] (without query_id auto-fill — that happens in builder.ts).
+ */
+import { readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+import yaml from "js-yaml";
+import { EvalSetInputSchema } from "./schemas.js";
+import { ReportSchema } from "../diagnose/schemas.js";
+export class QueryPickerError extends Error {
+    path;
+    constructor(message, path) {
+        super(message);
+        this.path = path;
+        this.name = "QueryPickerError";
+    }
+}
+export async function liftFromQueriesFile(filePath) {
+    let raw;
+    try {
+        raw = await readFile(filePath, "utf8");
+    }
+    catch (e) {
+        const err = e;
+        if (err.code === "ENOENT") {
+            throw new QueryPickerError(`file not found: ${filePath}`, filePath);
+        }
+        throw new QueryPickerError(`failed to read ${filePath}: ${err.message}`, filePath);
+    }
+    let parsed;
+    try {
+        parsed = yaml.load(raw);
+    }
+    catch (e) {
+        throw new QueryPickerError(`failed to parse yaml ${filePath}: ${e.message}`, filePath);
+    }
+    const result = EvalSetInputSchema.safeParse(parsed);
+    if (!result.success) {
+        const firstIssue = result.error.issues[0];
+        const where = firstIssue.path.join(".");
+        throw new QueryPickerError(`schema validation failed for ${filePath} at '${where}': ${firstIssue.message}`, filePath);
+    }
+    return result.data.cases.map((c) => ({
+        query_id: c.query_id ?? "", // empty → builder.ensureQueryId fills it; undefined would break downstream
+        input: c.input,
+        reference: c.reference,
+        assertions: c.assertions,
+        tags: c.tags,
+    }));
+}
+/**
+ * Read all *.yaml / *.yml files in `dirPath`, validate each as `trace-diagnose-report/v1`,
+ * and extract `findings[*].verify_with.suggested_eval_case` as EvalCases.
+ *
+ * Skips findings where:
+ *   - `suggested_eval_case.query` is null (M4 has no user query → can't construct EvalCase.input)
+ *   - `suggested_eval_case.assertions` is empty (refinement would fail; no reference either)
+ *
+ * Lifts:
+ *   - `EvalCase.input.user_message = suggested_eval_case.query`
+ *   - `EvalCase.query_id = suggested_eval_case.query_id ?? ""` (empty → builder.ensureQueryId fills)
+ *   - `EvalCase.assertions` = M4 string templates wrapped as placeholder `contains` assertions
+ *     with `_note` flagging "convert to structured manually"
+ *   - `EvalCase.reference = undefined` (M4 doesn't emit reference)
+ *
+ * Files that fail to schema-validate cause a fail-fast error (all *.yaml in dir
+ * must be diagnose reports — picker doesn't filter by content).
+ */
+export async function liftFromDiagnosis(dirPath) {
+    let entries;
+    try {
+        entries = await readdir(dirPath);
+    }
+    catch (e) {
+        const err = e;
+        if (err.code === "ENOENT") {
+            throw new QueryPickerError(`directory not found: ${dirPath}`, dirPath);
+        }
+        throw new QueryPickerError(`failed to read directory ${dirPath}: ${err.message}`, dirPath);
+    }
+    const yamlFiles = entries
+        .filter((e) => e.endsWith(".yaml") || e.endsWith(".yml"))
+        .map((e) => path.join(dirPath, e));
+    // Accumulate by query_id so multiple findings from the same conversation
+    // collapse into one case with merged assertions (avoids intra-batch dup error).
+    const byQueryId = new Map();
+    let skipped = 0;
+    const skippedSummary = [];
+    for (const file of yamlFiles) {
+        let raw;
+        try {
+            raw = await readFile(file, "utf8");
+        }
+        catch (e) {
+            throw new QueryPickerError(`failed to read ${file}: ${e.message}`, file);
+        }
+        let parsed;
+        try {
+            parsed = yaml.load(raw);
+        }
+        catch (e) {
+            throw new QueryPickerError(`failed to parse yaml ${file}: ${e.message}`, file);
+        }
+        const result = ReportSchema.safeParse(parsed);
+        if (!result.success) {
+            const firstIssue = result.error.issues[0];
+            const where = firstIssue.path.join(".");
+            throw new QueryPickerError(`schema validation failed for ${file} at '${where}': ${firstIssue.message}`, file);
+        }
+        for (const finding of result.data.findings) {
+            const sec = finding.verify_with.suggested_eval_case;
+            if (sec.query === null) {
+                skipped += 1;
+                skippedSummary.push(`${path.basename(file)}: rule=${finding.rule_id} (query=null; upgrade M4 trace to populate input.messages)`);
+                continue;
+            }
+            if (sec.assertions.length === 0) {
+                skipped += 1;
+                skippedSummary.push(`${path.basename(file)}: rule=${finding.rule_id} (empty assertions; refinement would fail)`);
+                continue;
+            }
+            const placeholderAssertions = sec.assertions.map((t) => ({
+                type: "contains",
+                value: t,
+                _note: "auto-lifted from M4 assertion template; convert to structured assertion manually",
+            }));
+            const queryId = sec.query_id ?? "";
+            const existing = byQueryId.get(queryId);
+            if (existing) {
+                existing.assertions = [...(existing.assertions ?? []), ...placeholderAssertions];
+            }
+            else {
+                byQueryId.set(queryId, {
+                    query_id: queryId,
+                    input: { user_message: sec.query },
+                    reference: undefined,
+                    assertions: placeholderAssertions,
+                    tags: undefined,
+                });
+            }
+        }
+    }
+    return { cases: [...byQueryId.values()], skipped_findings_count: skipped, skipped_findings_summary: skippedSummary };
+}

package/dist/trace-ai/eval-set/redactor.d.ts

@@ -0,0 +1,42 @@
+/**
+ * M5 eval-set redactor — PII pattern matching + replacement.
+ *
+ * Three rule sources, in priority order (chain):
+ *   1. --redaction-rules=<path>      (CLI flag, highest)
+ *   2. <repo>/redaction-rules/*.yaml (repo-local)
+ *   3. BUILTIN_RULES                 (5 low-fidelity defaults)
+ *
+ * Builtin rules cover common Chinese-context PII: phone / email / id_card /
+ * bank_card / ip. Organizations write more rules in <repo>/redaction-rules/
+ * for their business-specific patterns.
+ *
+ * Rule yaml format:
+ *   rules:
+ *     - name: <id>
+ *       pattern: <regex source string>
+ *       replace: <replacement template; supports {hash6} placeholder>
+ *
+ * Malformed regex causes loadRules to throw RedactorError (no silent fallback).
+ */
+import type { RedactionRule } from "./types.js";
+export declare class RedactorError extends Error {
+    readonly path?: string | undefined;
+    constructor(message: string, path?: string | undefined);
+}
+/**
+ * 5 builtin low-fidelity PII patterns. Tuned for Chinese-context defaults;
+ * organizations override with their own rules in <repo>/redaction-rules/.
+ */
+export declare const BUILTIN_RULES: RedactionRule[];
+export interface LoadRulesOpts {
+    /** From `--redaction-rules=<path>`; highest priority */
+    cliFlag: string | undefined;
+    /** From `<repo>/redaction-rules/` (resolved by caller — usually `path.join(repoRoot, "redaction-rules")`) */
+    repoDir: string | undefined;
+}
+export interface LoadRulesResult {
+    rules: RedactionRule[];
+    source: "cli-flag" | "repo" | "builtin";
+}
+export declare function loadRules(opts: LoadRulesOpts): Promise<LoadRulesResult>;
+export declare function applyRules(text: string, rules: RedactionRule[]): string;

package/dist/trace-ai/eval-set/redactor.js

@@ -0,0 +1,133 @@
+/**
+ * M5 eval-set redactor — PII pattern matching + replacement.
+ *
+ * Three rule sources, in priority order (chain):
+ *   1. --redaction-rules=<path>      (CLI flag, highest)
+ *   2. <repo>/redaction-rules/*.yaml (repo-local)
+ *   3. BUILTIN_RULES                 (5 low-fidelity defaults)
+ *
+ * Builtin rules cover common Chinese-context PII: phone / email / id_card /
+ * bank_card / ip. Organizations write more rules in <repo>/redaction-rules/
+ * for their business-specific patterns.
+ *
+ * Rule yaml format:
+ *   rules:
+ *     - name: <id>
+ *       pattern: <regex source string>
+ *       replace: <replacement template; supports {hash6} placeholder>
+ *
+ * Malformed regex causes loadRules to throw RedactorError (no silent fallback).
+ */
+import { readFile, readdir, stat } from "node:fs/promises";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import yaml from "js-yaml";
+export class RedactorError extends Error {
+    path;
+    constructor(message, path) {
+        super(message);
+        this.path = path;
+        this.name = "RedactorError";
+    }
+}
+/**
+ * 5 builtin low-fidelity PII patterns. Tuned for Chinese-context defaults;
+ * organizations override with their own rules in <repo>/redaction-rules/.
+ */
+export const BUILTIN_RULES = [
+    {
+        name: "phone",
+        pattern: /1[3-9]\d{9}/g,
+        replace: "<phone:{hash6}>",
+    },
+    {
+        name: "email",
+        pattern: /[\w.+-]+@[\w.-]+\.\w+/g,
+        replace: "<email:{hash6}>",
+    },
+    {
+        name: "id_card",
+        pattern: /\b\d{17}[\dXx]\b/g,
+        replace: "<id_card:{hash6}>",
+    },
+    {
+        name: "bank_card",
+        pattern: /\b\d{15,19}\b/g, // 银行卡号长度 15-19 位
+        replace: "<bank_card:{hash6}>",
+    },
+    {
+        name: "ip",
+        pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g,
+        replace: "<ip:{hash6}>",
+    },
+];
+function compileRule(entry, srcPath) {
+    let pattern;
+    try {
+        pattern = new RegExp(entry.pattern, "g");
+    }
+    catch (e) {
+        throw new RedactorError(`invalid regex in rule '${entry.name}' at ${srcPath}: ${e.message}`, srcPath);
+    }
+    return { name: entry.name, pattern, replace: entry.replace };
+}
+async function readRulesFile(filePath) {
+    let raw;
+    try {
+        raw = await readFile(filePath, "utf8");
+    }
+    catch (e) {
+        throw new RedactorError(`failed to read rule file ${filePath}: ${e.message}`, filePath);
+    }
+    let parsed;
+    try {
+        parsed = yaml.load(raw);
+    }
+    catch (e) {
+        throw new RedactorError(`failed to parse yaml ${filePath}: ${e.message}`, filePath);
+    }
+    const doc = parsed;
+    if (!doc || !Array.isArray(doc.rules)) {
+        throw new RedactorError(`rule file ${filePath} must have top-level 'rules: []'`, filePath);
+    }
+    return doc.rules.map((e) => compileRule(e, filePath));
+}
+export async function loadRules(opts) {
+    if (opts.cliFlag) {
+        const rules = await readRulesFile(opts.cliFlag);
+        return { rules, source: "cli-flag" };
+    }
+    if (opts.repoDir) {
+        let stats;
+        try {
+            stats = await stat(opts.repoDir);
+        }
+        catch {
+            stats = null;
+        }
+        if (stats && stats.isDirectory()) {
+            const entries = await readdir(opts.repoDir);
+            const yamlFiles = entries
+                .filter((e) => e.endsWith(".yaml") || e.endsWith(".yml"))
+                .map((e) => path.join(opts.repoDir, e));
+            if (yamlFiles.length > 0) {
+                const allRules = [];
+                for (const f of yamlFiles) {
+                    allRules.push(...(await readRulesFile(f)));
+                }
+                return { rules: allRules, source: "repo" };
+            }
+        }
+    }
+    return { rules: BUILTIN_RULES, source: "builtin" };
+}
+function hash6(input) {
+    return createHash("sha256").update(input).digest("hex").slice(0, 6);
+}
+export function applyRules(text, rules) {
+    let out = text;
+    for (const rule of rules) {
+        out = out.replace(rule.pattern, (match) => rule.replace.replace("{hash6}", hash6(match)));
+    }
+    return out;
+}

package/dist/trace-ai/eval-set/rubric-templates/answer-match-reference.prompt.md

@@ -0,0 +1,19 @@
+You are evaluating whether a candidate answer is semantically equivalent to a reference answer for a knowledge-graph Q&A system.
+
+Question: {{question}}
+
+Reference answer (ground truth):
+{{reference_answer}}
+
+Candidate answer (from agent):
+{{candidate_answer}}
+
+Judge whether the candidate answer is semantically correct relative to the reference.
+A candidate passes if it conveys the same key facts, even if phrased differently.
+A candidate fails if it omits critical facts, states incorrect facts, or hallucinates information not in the reference.
+Partial answers that cover most key facts but miss minor details should pass.
+
+{{language_instruction}}
+
+Respond with valid JSON matching this schema exactly:
+{{output_schema}}

package/dist/trace-ai/eval-set/schemas.d.ts

@@ -0,0 +1,136 @@
+/**
+ * M5 eval-set zod schemas (PR-A, MVP-B scope).
+ *
+ * 4 schemas in this file:
+ *   - EvalSetIndexSchema:  trace-eval-set-index/v1   (eval-set dir's index.yaml)
+ *   - EvalSetShardSchema:  trace-eval-set/v1         (final shard yaml file)
+ *   - EvalSetInputSchema:  trace-eval-set-input/v1   (--queries simplified input)
+ *   - TestReportSchema:    trace-test-report/v1      (test report; PR-A defines schema only;
+ *                                                     PR-B consumer)
+ *
+ * EvalSetShardSchema and EvalSetInputSchema share the same refinement:
+ *   "for each case, at least one of {reference, non-empty assertions} must be present."
+ *
+ * The D5 builtin rubric `answer-match-reference` output schema is NOT here —
+ * it belongs to the rubric template definition (per spec doc §4.1).
+ */
+import { z } from "zod";
+export declare const EvalSetIndexSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-eval-set-index/v1">;
+    eval_set_id: z.ZodString;
+    shards: z.ZodArray<z.ZodObject<{
+        path: z.ZodString;
+        role: z.ZodOptional<z.ZodEnum<{
+            seed: "seed";
+            regression: "regression";
+            holdout: "holdout";
+        }>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const EvalSetShardSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-eval-set/v1">;
+    cases: z.ZodArray<z.ZodObject<{
+        query_id: z.ZodString;
+        input: z.ZodObject<{
+            user_message: z.ZodString;
+        }, z.core.$strip>;
+        reference: z.ZodOptional<z.ZodObject<{
+            answer: z.ZodString;
+        }, z.core.$strip>>;
+        assertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            type: z.ZodEnum<{
+                contains: "contains";
+                not_contains: "not_contains";
+                regex: "regex";
+                tool_call_count: "tool_call_count";
+                tool_call_order: "tool_call_order";
+                semantic_match: "semantic_match";
+                latency_ms: "latency_ms";
+            }>;
+        }, z.core.$loose>>>;
+        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const EvalSetInputSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-eval-set-input/v1">;
+    cases: z.ZodArray<z.ZodObject<{
+        input: z.ZodObject<{
+            user_message: z.ZodString;
+        }, z.core.$strip>;
+        query_id: z.ZodOptional<z.ZodString>;
+        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        reference: z.ZodOptional<z.ZodObject<{
+            answer: z.ZodString;
+        }, z.core.$strip>>;
+        assertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            type: z.ZodEnum<{
+                contains: "contains";
+                not_contains: "not_contains";
+                regex: "regex";
+                tool_call_count: "tool_call_count";
+                tool_call_order: "tool_call_order";
+                semantic_match: "semantic_match";
+                latency_ms: "latency_ms";
+            }>;
+        }, z.core.$loose>>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const TestReportSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-test-report/v1">;
+    meta: z.ZodObject<{
+        eval_set_dir: z.ZodString;
+        eval_set_id: z.ZodString;
+        candidate: z.ZodObject<{
+            agent_id: z.ZodString;
+            agent_version: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>;
+        cli_version: z.ZodString;
+        ran_at: z.ZodString;
+        duration_ms: z.ZodNumber;
+    }, z.core.$strip>;
+    summary: z.ZodObject<{
+        total: z.ZodNumber;
+        pass: z.ZodNumber;
+        fail: z.ZodNumber;
+        error: z.ZodNumber;
+        skip: z.ZodNumber;
+        by_assertion_type: z.ZodRecord<z.ZodString, z.ZodObject<{
+            pass: z.ZodNumber;
+            fail: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    cases: z.ZodArray<z.ZodObject<{
+        query_id: z.ZodString;
+        status: z.ZodEnum<{
+            error: "error";
+            pass: "pass";
+            fail: "fail";
+            skip: "skip";
+        }>;
+        conversation_id: z.ZodNullable<z.ZodString>;
+        trace_id: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        duration_ms: z.ZodOptional<z.ZodNumber>;
+        assertion_results: z.ZodArray<z.ZodObject<{
+            assertion: z.ZodObject<{
+                type: z.ZodEnum<{
+                    contains: "contains";
+                    not_contains: "not_contains";
+                    regex: "regex";
+                    tool_call_count: "tool_call_count";
+                    tool_call_order: "tool_call_order";
+                    semantic_match: "semantic_match";
+                    latency_ms: "latency_ms";
+                }>;
+            }, z.core.$loose>;
+            verdict: z.ZodEnum<{
+                pass: "pass";
+                fail: "fail";
+                skip: "skip";
+            }>;
+            actual: z.ZodOptional<z.ZodUnknown>;
+        }, z.core.$strip>>;
+        failure_reason: z.ZodOptional<z.ZodString>;
+        error_code: z.ZodOptional<z.ZodString>;
+        error_message: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;