@kweaver-ai/kweaver-sdk 0.8.1 → 0.8.3

package/dist/trace-ai/eval-set/test-runner.js

@@ -0,0 +1,153 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import yaml from "js-yaml";
+import { evaluateAssertion } from "./assertion-evaluator.js";
+import { EvalSetIndexSchema, EvalSetShardSchema, TestReportSchema } from "./schemas.js";
+// ── eval-set loader ───────────────────────────────────────────────────────────
+async function loadEvalCases(evalSetDir) {
+    const indexRaw = await fs.readFile(path.join(evalSetDir, "index.yaml"), "utf8");
+    const index = EvalSetIndexSchema.parse(yaml.load(indexRaw));
+    const cases = [];
+    for (const shard of index.shards) {
+        const shardRaw = await fs.readFile(path.join(evalSetDir, shard.path), "utf8");
+        const parsed = EvalSetShardSchema.parse(yaml.load(shardRaw));
+        cases.push(...parsed.cases);
+    }
+    return cases;
+}
+// ── case runner ───────────────────────────────────────────────────────────────
+async function runCase(evalCase, agentInfo, deps) {
+    const startMs = Date.now();
+    let conversationId = null;
+    let traceId = null;
+    let spans = [];
+    let answerText = "";
+    let stage = "chat";
+    try {
+        const chatResult = await deps.sendChat({
+            agentInfo,
+            query: evalCase.input.user_message,
+        });
+        answerText = chatResult.text;
+        conversationId = chatResult.conversationId ?? null;
+        if (conversationId) {
+            stage = "trace";
+            const traceResult = await deps.fetchTrace(conversationId);
+            spans = traceResult.spans;
+            traceId = spans[0]?.traceId ?? null;
+        }
+    }
+    catch (e) {
+        const durationMs = Date.now() - startMs;
+        return {
+            query_id: evalCase.query_id,
+            status: "error",
+            conversation_id: conversationId,
+            trace_id: traceId,
+            duration_ms: durationMs,
+            assertion_results: [],
+            error_message: e instanceof Error ? e.message : String(e),
+            error_code: stage === "trace" ? "trace-fetch-failed" : "chat-failed",
+        };
+    }
+    const durationMs = Date.now() - startMs;
+    const assertionResults = [];
+    for (const assertion of evalCase.assertions ?? []) {
+        try {
+            const result = await evaluateAssertion(assertion, {
+                answer: answerText,
+                spans,
+                reference: evalCase.reference,
+                durationMs,
+                question: evalCase.input.user_message,
+                semanticMatchProvider: deps.semanticMatchProvider,
+            });
+            assertionResults.push({ assertion, verdict: result.verdict, actual: result.actual });
+        }
+        catch (e) {
+            assertionResults.push({ assertion, verdict: "skip", actual: `assertion-eval-error: ${e.message}` });
+        }
+    }
+    // A case may pass schema with reference-only (no assertions), but without
+    // assertions there is no pass/fail signal — mark skip so it does not
+    // silently inflate the pass count.
+    if (assertionResults.length === 0) {
+        return {
+            query_id: evalCase.query_id,
+            status: "skip",
+            conversation_id: conversationId,
+            trace_id: traceId,
+            duration_ms: durationMs,
+            assertion_results: assertionResults,
+            failure_reason: "no assertions configured; case has reference but no judge (e.g. semantic_match) wired",
+        };
+    }
+    const hasFail = assertionResults.some((r) => r.verdict === "fail");
+    const allSkip = assertionResults.every((r) => r.verdict === "skip");
+    const status = hasFail ? "fail" : allSkip ? "skip" : "pass";
+    return {
+        query_id: evalCase.query_id,
+        status: status,
+        conversation_id: conversationId,
+        trace_id: traceId,
+        duration_ms: durationMs,
+        assertion_results: assertionResults,
+    };
+}
+// ── main runner ───────────────────────────────────────────────────────────────
+export async function run(opts) {
+    const { evalSetDir, candidateAgentId, candidateAgentVersion, outDir, deps } = opts;
+    const maxParallel = opts.maxParallel ?? 4;
+    const [cases, agentInfo] = await Promise.all([
+        loadEvalCases(evalSetDir),
+        deps.fetchAgent(candidateAgentId, candidateAgentVersion),
+    ]);
+    // Fetch eval_set_id from index
+    const indexRaw = await fs.readFile(path.join(evalSetDir, "index.yaml"), "utf8");
+    const index = EvalSetIndexSchema.parse(yaml.load(indexRaw));
+    const ranAt = new Date().toISOString();
+    const overallStart = Date.now();
+    const caseResults = new Array(cases.length);
+    let nextIdx = 0;
+    const workerCount = Math.max(1, Math.min(maxParallel, cases.length));
+    const workers = Array.from({ length: workerCount }, async () => {
+        while (true) {
+            const idx = nextIdx++;
+            if (idx >= cases.length)
+                return;
+            caseResults[idx] = await runCase(cases[idx], agentInfo, deps);
+        }
+    });
+    await Promise.all(workers);
+    const overallDurationMs = Date.now() - overallStart;
+    // Build summary
+    const counts = { total: caseResults.length, pass: 0, fail: 0, error: 0, skip: 0 };
+    const byType = {};
+    for (const cr of caseResults) {
+        counts[cr.status]++;
+        for (const ar of cr.assertion_results) {
+            const t = ar.assertion["type"];
+            if (!byType[t])
+                byType[t] = { pass: 0, fail: 0 };
+            if (ar.verdict === "pass")
+                byType[t].pass++;
+            else if (ar.verdict === "fail")
+                byType[t].fail++;
+        }
+    }
+    const report = TestReportSchema.parse({
+        schema_version: "trace-test-report/v1",
+        meta: {
+            eval_set_dir: evalSetDir,
+            eval_set_id: index.eval_set_id,
+            candidate: { agent_id: agentInfo.id, agent_version: agentInfo.version },
+            cli_version: "0.0.0",
+            ran_at: ranAt,
+            duration_ms: overallDurationMs,
+        },
+        summary: { ...counts, by_assertion_type: byType },
+        cases: caseResults,
+    });
+    await fs.mkdir(outDir, { recursive: true });
+    await fs.writeFile(path.join(outDir, "report.yaml"), yaml.dump(report, { lineWidth: 120 }), "utf8");
+}

package/dist/trace-ai/eval-set/types.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Internal types for the M5 eval-set module (PR-A).
+ *
+ * These mirror the zod schemas in `./schemas.ts` but are kept independent so
+ * non-validating code paths (builder / picker / redactor / output-writer) can
+ * import the types without paying the zod parse overhead at module load.
+ */
+export interface EvalCaseInput {
+    user_message: string;
+}
+export interface EvalReference {
+    answer: string;
+}
+export type AssertionType = "contains" | "not_contains" | "regex" | "tool_call_count" | "tool_call_order" | "semantic_match" | "latency_ms";
+export interface EvalAssertion {
+    type: AssertionType;
+    [key: string]: unknown;
+}
+export interface EvalCase {
+    query_id: string;
+    input: EvalCaseInput;
+    reference?: EvalReference;
+    assertions?: EvalAssertion[];
+    tags?: string[];
+}
+export interface EvalSetIndexShard {
+    path: string;
+    role?: "seed" | "regression" | "holdout";
+}
+export interface EvalSetIndex {
+    schema_version: "trace-eval-set-index/v1";
+    eval_set_id: string;
+    shards: EvalSetIndexShard[];
+}
+export interface BuildResult {
+    cases_written: number;
+    cases_skipped: number;
+    conflicts: string[];
+    shard_paths: string[];
+    redaction_rules_source: "cli-flag" | "repo" | "builtin";
+}
+export interface RedactionRule {
+    name: string;
+    pattern: RegExp;
+    replace: string;
+}

package/dist/trace-ai/eval-set/types.js

@@ -0,0 +1,8 @@
+/**
+ * Internal types for the M5 eval-set module (PR-A).
+ *
+ * These mirror the zod schemas in `./schemas.ts` but are kept independent so
+ * non-validating code paths (builder / picker / redactor / output-writer) can
+ * import the types without paying the zod parse overhead at module load.
+ */
+export {};

package/dist/trace-ai/exp/bundle-writer.d.ts

@@ -0,0 +1,10 @@
+import type { LineageEntry, RoundData } from "./schemas.js";
+interface WriteBundlesOpts {
+    expDir: string;
+    experimentId: string;
+    lineage: LineageEntry[];
+    rounds: RoundData[];
+    createdBy: string;
+}
+export declare function writeBundles(opts: WriteBundlesOpts): Promise<void>;
+export {};

package/dist/trace-ai/exp/bundle-writer.js

@@ -0,0 +1,54 @@
+// src/trace-ai/exp/bundle-writer.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+import crypto from "node:crypto";
+import yaml from "js-yaml";
+export async function writeBundles(opts) {
+    const { expDir, experimentId, lineage, rounds, createdBy } = opts;
+    const bestEntry = lineage.filter(e => e.status === "scored").at(-1) ?? lineage.at(-1);
+    const bestVersion = bestEntry?.version ?? 0;
+    const bundleId = `bundle_${crypto.randomBytes(4).toString("hex")}`;
+    const now = new Date().toISOString();
+    const bundle = {
+        schema_version: "trace-bundle/v1",
+        experiment_id: experimentId,
+        bundle_id: bundleId,
+        best_trial_version: bestVersion,
+        resources: {
+            agent_config: bestEntry?.next_change ?? {},
+            skills: [],
+        },
+        provenance: {
+            created_by: createdBy,
+            created_at: now,
+            evidence_traces: rounds.flatMap(r => (r.per_query_results ?? []).map(q => q.raw_trace_id ?? "").filter(Boolean)),
+            round_refs: rounds.map(r => `.trace-state/rounds/round-${r.round}.yaml`),
+        },
+    };
+    const lastRound = rounds.at(-1);
+    const manifest = {
+        schema_version: "trace-manifest/v1",
+        experiment_id: experimentId,
+        trial_version: bestVersion,
+        predictions: {
+            fixes: (lastRound?.per_query_results ?? [])
+                .filter(q => q.assertion_results.every(a => a.verdict === "pass"))
+                .map(q => ({ query_id: q.query_id, reason: "all assertions passed" })),
+            risks: (lastRound?.per_query_results ?? [])
+                .filter(q => q.assertion_results.some(a => a.verdict === "fail"))
+                .map(q => ({ query_id: q.query_id, reason: "assertions failed" })),
+        },
+    };
+    const provenance = {
+        experiment_id: experimentId,
+        generated_at: now,
+        rounds_count: rounds.length,
+        lineage_count: lineage.length,
+        round_verdicts: rounds.map(r => ({ round: r.round, verdict: r.triage_conclusion?.verdict ?? "pending" })),
+    };
+    const outDir = path.join(expDir, "outputs");
+    await fs.mkdir(outDir, { recursive: true });
+    await fs.writeFile(path.join(outDir, "bundle.yaml"), yaml.dump(bundle, { lineWidth: -1 }));
+    await fs.writeFile(path.join(outDir, "manifest.yaml"), yaml.dump(manifest, { lineWidth: -1 }));
+    await fs.writeFile(path.join(outDir, "provenance.yaml"), yaml.dump(provenance, { lineWidth: -1 }));
+}

package/dist/trace-ai/exp/claude-binary.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Resolves the path to the claude CLI binary.
+ * Priority: CLAUDE_BIN env → `which claude` → known install locations → bare "claude".
+ */
+export declare function resolveClaudeBinary(): string;

package/dist/trace-ai/exp/claude-binary.js

@@ -0,0 +1,30 @@
+import { execSync } from "node:child_process";
+import os from "node:os";
+/**
+ * Resolves the path to the claude CLI binary.
+ * Priority: CLAUDE_BIN env → `which claude` → known install locations → bare "claude".
+ */
+export function resolveClaudeBinary() {
+    if (process.env["CLAUDE_BIN"])
+        return process.env["CLAUDE_BIN"];
+    try {
+        const resolved = execSync("which claude", { encoding: "utf8", timeout: 3000 }).trim();
+        // Reject shell alias expansions like "claude: aliased to ..."
+        if (resolved && !resolved.includes(" "))
+            return resolved;
+    }
+    catch { /* fall through */ }
+    const home = os.homedir();
+    for (const p of [
+        `${home}/.local/bin/claude`,
+        "/opt/homebrew/bin/claude",
+        "/usr/local/bin/claude",
+    ]) {
+        try {
+            execSync(`test -x "${p}"`, { timeout: 1000 });
+            return p;
+        }
+        catch { /* try next */ }
+    }
+    return "claude";
+}

package/dist/trace-ai/exp/coordinator.d.ts

@@ -0,0 +1,45 @@
+import type { Mission, NextChange, QueryResult, RoundData } from "./schemas.js";
+export interface SynthesizerClient {
+    generate(input: {
+        mission: Mission;
+        candidateConfig: Record<string, unknown>;
+        prevRound?: RoundData;
+        prevRounds: RoundData[];
+        crossRoundMemoryRef?: string;
+    }): Promise<NextChange>;
+}
+export interface TriageClient {
+    triage(input: {
+        currentRound: RoundData;
+        prevRounds: RoundData[];
+        candidateConfig: Record<string, unknown>;
+        crossRoundMemoryRef?: string;
+    }): Promise<RoundData["triage_conclusion"] & {
+        new_memory_token: string;
+    }>;
+}
+export interface CoordinatorOpts {
+    expDir: string;
+    synthesizer: SynthesizerClient;
+    triage: TriageClient;
+    runEval: (opts: {
+        evalSetPaths: string[];
+        candidatePath: string;
+        expDir: string;
+        round: number;
+    }) => Promise<{
+        queryResults: QueryResult[];
+    }>;
+    experimentId?: string;
+}
+export declare class ExperimentCoordinator {
+    private opts;
+    private store;
+    private heartbeatTimer?;
+    constructor(opts: CoordinatorOpts);
+    run(): Promise<void>;
+    resume(): Promise<void>;
+    private runLoop;
+    private checkAbort;
+    private withRetry;
+}

package/dist/trace-ai/exp/coordinator.js

@@ -0,0 +1,203 @@
+// src/trace-ai/exp/coordinator.ts
+import path from "node:path";
+import fs from "node:fs/promises";
+import yaml from "js-yaml";
+import { ExpStore } from "./exp-store/index.js";
+import { applyPatch } from "./patch/index.js";
+import { computeScores } from "./scoring.js";
+import { writeBundles } from "./bundle-writer.js";
+export class ExperimentCoordinator {
+    opts;
+    store;
+    heartbeatTimer;
+    constructor(opts) {
+        this.opts = opts;
+        this.store = new ExpStore(opts.expDir);
+    }
+    async run() {
+        const replayed = await this.store.replayState();
+        if (replayed.isTerminal && !replayed.currentState.includes("Aborted")) {
+            throw new Error(`Experiment is in terminal state ${replayed.currentState}. Use --new-run to start fresh.`);
+        }
+        const mission = await this.store.readMission();
+        const expId = this.opts.experimentId ?? `exp_${Date.now()}`;
+        if (replayed.currentRound === 0) {
+            await this.store.initDir(mission);
+        }
+        await this.store.acquireLock();
+        this.heartbeatTimer = setInterval(() => { void this.store.updateHeartbeat(); }, 10_000);
+        // If previous run failed mid-round, retry that round (startRound = currentRound - 1)
+        const startRound = replayed.lastFailure && replayed.currentRound > 0
+            ? replayed.currentRound - 1
+            : replayed.currentRound;
+        try {
+            await this.runLoop(mission, startRound, expId);
+        }
+        finally {
+            clearInterval(this.heartbeatTimer);
+            await this.store.releaseLock();
+        }
+    }
+    async resume() {
+        const replayed = await this.store.replayState();
+        if (replayed.currentState !== "Deciding") {
+            throw new Error(`Cannot resume: experiment is in state ${replayed.currentState}, not Deciding. Only Deciding state supports resume.`);
+        }
+        await this.store.acquireLock();
+        this.heartbeatTimer = setInterval(() => { void this.store.updateHeartbeat(); }, 10_000);
+        try {
+            const mission = await this.store.readMission();
+            const expId = `exp_${replayed.currentRound}`;
+            await this.runLoop(mission, replayed.currentRound, expId);
+        }
+        finally {
+            clearInterval(this.heartbeatTimer);
+            await this.store.releaseLock();
+        }
+    }
+    async runLoop(mission, startRound, expId) {
+        const round = startRound + 1;
+        const maxRounds = mission.max_rounds ?? Infinity;
+        if (await this.checkAbort(round))
+            return;
+        // === Generating (Apply Phase) ===
+        await this.store.appendEvent({ type: "state_transition", from: "Deciding", to: "Generating", round });
+        const nextChange = mission.next_change;
+        if (!nextChange)
+            throw new Error("mission.md has no next_change — add one or let Synthesizer suggest");
+        const prevRounds = await this.store.readAllRounds();
+        // Load current candidate and apply patch
+        const currentCandidatePath = path.join(this.opts.expDir, mission.current_candidate.path);
+        const currentCandidate = yaml.load(await fs.readFile(currentCandidatePath, "utf8"));
+        const patched = applyPatch(currentCandidate, nextChange);
+        patched["candidate_version"] = `v${round}`;
+        const newCandidatePath = path.join(this.opts.expDir, "candidates", `candidate-v${round}.yaml`);
+        await fs.writeFile(newCandidatePath, yaml.dump(patched, { lineWidth: -1 }));
+        await this.store.appendLineage({
+            version: round,
+            candidate_path: `candidates/candidate-v${round}.yaml`,
+            next_change: nextChange,
+            status: "running",
+        });
+        if (await this.checkAbort(round))
+            return;
+        // === Executing ===
+        await this.store.appendEvent({ type: "state_transition", from: "Generating", to: "Executing", round });
+        const evalSetPaths = mission.eval_sets.map(e => path.join(this.opts.expDir, e.path));
+        let queryResults;
+        try {
+            const result = await this.withRetry(() => this.opts.runEval({ evalSetPaths, candidatePath: newCandidatePath, expDir: this.opts.expDir, round }), "Executing");
+            queryResults = result.queryResults;
+        }
+        catch {
+            return; // step_failed already written by withRetry
+        }
+        if (await this.checkAbort(round))
+            return;
+        // === Scoring ===
+        await this.store.appendEvent({ type: "state_transition", from: "Executing", to: "Scoring", round });
+        const guardrails = mission.guardrails ?? [];
+        const scores = computeScores(queryResults, guardrails);
+        if (scores.guardrail_hard_fail) {
+            await this.store.updateLineage(round, { status: "guardrail_failed" });
+            await this.store.writeRound(round, { round, trial_version: round, guardrail_failed: true, scores });
+            await this.store.appendEvent({ type: "state_transition", from: "Scoring", to: "Deciding", round });
+            process.stdout.write(`\nRound ${round}: Guardrail hard gate violated. Fix the candidate and run exp resume.\n`);
+            return;
+        }
+        await this.store.updateLineage(round, { status: "scored" });
+        await this.store.writeRound(round, { round, trial_version: round, scores, per_query_results: queryResults });
+        if (await this.checkAbort(round))
+            return;
+        // === Triaging ===
+        await this.store.appendEvent({ type: "state_transition", from: "Scoring", to: "Triaging", round });
+        const currentRoundData = (await this.store.readAllRounds()).find(r => r.round === round) ?? { round, trial_version: round };
+        const prevMemory = prevRounds.at(-1)?.triage_conclusion?.cross_round_memory_ref;
+        let triageResult;
+        try {
+            triageResult = await this.withRetry(() => this.opts.triage.triage({
+                currentRound: currentRoundData,
+                prevRounds,
+                candidateConfig: patched,
+                crossRoundMemoryRef: prevMemory,
+            }), "Triaging");
+        }
+        catch {
+            return;
+        }
+        await this.store.writeRound(round, {
+            triage_conclusion: {
+                diagnoses: triageResult.diagnoses,
+                hints: triageResult.hints,
+                verdict: triageResult.verdict,
+                cross_round_memory_ref: triageResult.new_memory_token,
+            },
+        });
+        await this.store.appendEvent({ type: "round_completed", round, verdict: triageResult.verdict });
+        // Generate next suggestion if continuing
+        if (triageResult.verdict === "continue" && round < maxRounds) {
+            const updatedMission = await this.store.readMission();
+            try {
+                const suggestion = await this.withRetry(() => this.opts.synthesizer.generate({
+                    mission: updatedMission,
+                    candidateConfig: patched,
+                    prevRound: currentRoundData,
+                    prevRounds,
+                    crossRoundMemoryRef: triageResult.new_memory_token,
+                }), "Triaging");
+                await this.store.writeSuggestedChange(suggestion);
+            }
+            catch {
+                return;
+            }
+        }
+        // === Deciding ===
+        await this.store.appendEvent({ type: "state_transition", from: "Triaging", to: "Deciding", round });
+        if (triageResult.verdict === "publish" || round >= maxRounds) {
+            // Publish immediately
+            await this.store.appendEvent({ type: "state_transition", from: "Deciding", to: "Publishing", round });
+            const allRounds = await this.store.readAllRounds();
+            const allLineage = await this.store.readLineage();
+            await writeBundles({ expDir: this.opts.expDir, experimentId: expId, lineage: allLineage, rounds: allRounds, createdBy: process.env["USER"] ?? "unknown" });
+            await this.store.appendEvent({ type: "state_transition", from: "Publishing", to: "Published", round });
+            process.stdout.write(`\nExperiment complete. Outputs written to ${path.join(this.opts.expDir, "outputs")}\n`);
+        }
+        else {
+            // Pause at Deciding — lock released by run()/resume() finally block
+            process.stdout.write(`\nRound ${round} complete.\n`);
+            process.stdout.write(`Scores: outcome=${scores.outcome.toFixed(2)}, trajectory=${scores.trajectory.toFixed(2)}\n`);
+            process.stdout.write(`Triage: ${triageResult.diagnoses.join("; ")}\n`);
+            process.stdout.write(`Next suggestion written to mission.md. Review and run exp resume to continue.\n`);
+        }
+    }
+    async checkAbort(round) {
+        if (await this.store.isAborted()) {
+            clearInterval(this.heartbeatTimer);
+            await this.store.appendEvent({ type: "aborted", round, reason: "user_abort" });
+            await this.store.releaseLock();
+            return true;
+        }
+        return false;
+    }
+    async withRetry(fn, state) {
+        let lastErr;
+        for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+                return await fn();
+            }
+            catch (err) {
+                lastErr = err;
+                if (attempt < 2) {
+                    await new Promise(r => setTimeout(r, 1000 * 2 ** attempt));
+                }
+            }
+        }
+        await this.store.appendEvent({
+            type: "step_failed",
+            state: state,
+            error: String(lastErr),
+            retryable: true,
+        });
+        throw lastErr;
+    }
+}

package/dist/trace-ai/exp/eval-runner.d.ts

@@ -0,0 +1,14 @@
+import type { QueryResult } from "./schemas.js";
+import type { RunnerDeps } from "../eval-set/test-runner.js";
+export interface EvalRunnerOpts {
+    evalSetPaths: string[];
+    candidatePath: string;
+    expDir: string;
+    round: number;
+    deps: RunnerDeps;
+    maxParallel?: number;
+}
+export interface EvalRunResult {
+    queryResults: QueryResult[];
+}
+export declare function runEval(opts: EvalRunnerOpts): Promise<EvalRunResult>;

package/dist/trace-ai/exp/eval-runner.js

@@ -0,0 +1,47 @@
+// src/trace-ai/exp/eval-runner.ts
+import path from "node:path";
+import yaml from "js-yaml";
+import fs from "node:fs/promises";
+import { run as evalSetRun } from "../eval-set/test-runner.js";
+export async function runEval(opts) {
+    const candidateRaw = yaml.load(await fs.readFile(opts.candidatePath, "utf8"));
+    const agentId = candidateRaw["agent_id"] ?? "candidate";
+    const agentVersion = candidateRaw["candidate_version"];
+    const roundEvalBase = path.join(opts.expDir, ".trace-state", "rounds", `round-${opts.round}-eval`);
+    // Run eval for each eval-set (sequentially for MVP-C single-path)
+    const allResults = [];
+    for (const evalSetDir of opts.evalSetPaths) {
+        // Each eval-set gets its own subdir so outputs from multiple sets don't overwrite each other
+        const outDir = path.join(roundEvalBase, path.basename(evalSetDir));
+        await fs.mkdir(outDir, { recursive: true });
+        await evalSetRun({
+            evalSetDir,
+            candidateAgentId: agentId,
+            candidateAgentVersion: agentVersion,
+            outDir,
+            maxParallel: opts.maxParallel ?? 4,
+            deps: opts.deps,
+        });
+        // Read report and convert to QueryResult[]
+        const reportPath = path.join(outDir, "report.yaml");
+        const report = yaml.load(await fs.readFile(reportPath, "utf8"));
+        for (const c of report.cases) {
+            allResults.push({
+                query_id: c.query_id,
+                assertion_results: c.assertion_results.map(ar => ({
+                    type: ar.assertion.type,
+                    verdict: ar.verdict,
+                    reason: typeof ar.actual === "string" ? ar.actual : undefined,
+                })),
+                trajectory_summary: {
+                    tool_call_sequence: [], // populated from trace if available
+                    retry_count: 0,
+                    latency_ms: c.duration_ms ?? 0,
+                    error_codes: [],
+                },
+                raw_trace_id: c.trace_id ?? undefined,
+            });
+        }
+    }
+    return { queryResults: allResults };
+}

package/dist/trace-ai/exp/exp-store/abort-signal.d.ts

@@ -0,0 +1,3 @@
+export declare function isAborted(expDir: string): Promise<boolean>;
+export declare function writeAbortSignal(expDir: string): Promise<void>;
+export declare function clearAbortSignal(expDir: string): Promise<void>;

package/dist/trace-ai/exp/exp-store/abort-signal.js

@@ -0,0 +1,27 @@
+// src/trace-ai/exp/exp-store/abort-signal.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+function signalPath(expDir) {
+    return path.join(expDir, ".trace-state", "abort.signal");
+}
+export async function isAborted(expDir) {
+    try {
+        await fs.access(signalPath(expDir));
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function writeAbortSignal(expDir) {
+    await fs.writeFile(signalPath(expDir), new Date().toISOString(), "utf8");
+}
+export async function clearAbortSignal(expDir) {
+    try {
+        await fs.unlink(signalPath(expDir));
+    }
+    catch (err) {
+        if (err.code !== "ENOENT")
+            throw err;
+    }
+}

package/dist/trace-ai/exp/exp-store/candidate-lineage-yaml.d.ts

@@ -0,0 +1,4 @@
+import type { LineageEntry } from "../schemas.js";
+export declare function appendLineage(expDir: string, entry: Omit<LineageEntry, "appended_at">): Promise<void>;
+export declare function updateLineage(expDir: string, version: number, patch: Partial<LineageEntry>): Promise<void>;
+export declare function readLineage(expDir: string): Promise<LineageEntry[]>;