@kweaver-ai/kweaver-sdk 0.7.4 → 0.8.2

package/dist/trace-ai/eval-set/schemas.js

@@ -0,0 +1,130 @@
+/**
+ * M5 eval-set zod schemas (PR-A, MVP-B scope).
+ *
+ * 4 schemas in this file:
+ *   - EvalSetIndexSchema:  trace-eval-set-index/v1   (eval-set dir's index.yaml)
+ *   - EvalSetShardSchema:  trace-eval-set/v1         (final shard yaml file)
+ *   - EvalSetInputSchema:  trace-eval-set-input/v1   (--queries simplified input)
+ *   - TestReportSchema:    trace-test-report/v1      (test report; PR-A defines schema only;
+ *                                                     PR-B consumer)
+ *
+ * EvalSetShardSchema and EvalSetInputSchema share the same refinement:
+ *   "for each case, at least one of {reference, non-empty assertions} must be present."
+ *
+ * The D5 builtin rubric `answer-match-reference` output schema is NOT here —
+ * it belongs to the rubric template definition (per spec doc §4.1).
+ */
+import { z } from "zod";
+const InputSchema = z.object({
+    user_message: z.string().min(1),
+});
+const ReferenceSchema = z.object({
+    answer: z.string().min(1),
+});
+const AssertionSchema = z.object({
+    type: z.enum([
+        "contains",
+        "not_contains",
+        "regex",
+        "tool_call_count",
+        "tool_call_order",
+        "semantic_match",
+        "latency_ms",
+    ]),
+}).passthrough(); // allow type-specific fields (value, pattern, tool, op, n, ...)
+// ── trace-eval-set-index/v1 ──────────────────────────────────────────────
+const ShardRefSchema = z.object({
+    path: z
+        .string()
+        .min(1)
+        .refine((p) => !p.includes("..") && !p.startsWith("/"), {
+        message: "shard path must be a relative path within the eval-set directory (no '..' / '/')",
+    }),
+    role: z.enum(["seed", "regression", "holdout"]).optional(),
+});
+export const EvalSetIndexSchema = z.object({
+    schema_version: z.literal("trace-eval-set-index/v1"),
+    eval_set_id: z.string().min(1),
+    shards: z.array(ShardRefSchema).min(1),
+});
+// ── trace-eval-set/v1 ────────────────────────────────────────────────────
+const refineCase = (data, ctx) => {
+    const hasReference = data.reference !== undefined && data.reference !== null;
+    const hasAssertions = Array.isArray(data.assertions) && data.assertions.length > 0;
+    if (!hasReference && !hasAssertions) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "each case must have either a 'reference' object or a non-empty 'assertions' array; both empty is not allowed (evaluator has no pass/fail signal)",
+        });
+    }
+};
+const FinalCaseSchema = z
+    .object({
+    query_id: z.string().min(1),
+    input: InputSchema,
+    reference: ReferenceSchema.optional(),
+    assertions: z.array(AssertionSchema).optional(),
+    tags: z.array(z.string()).optional(),
+})
+    .superRefine(refineCase);
+export const EvalSetShardSchema = z.object({
+    schema_version: z.literal("trace-eval-set/v1"),
+    cases: z.array(FinalCaseSchema).min(1),
+});
+// ── trace-eval-set-input/v1 (D1: same refinement as final) ───────────────
+const InputCaseSchema = z
+    .object({
+    input: InputSchema,
+    query_id: z.string().min(1).optional(),
+    tags: z.array(z.string()).optional(),
+    reference: ReferenceSchema.optional(),
+    assertions: z.array(AssertionSchema).optional(),
+})
+    .superRefine(refineCase);
+export const EvalSetInputSchema = z.object({
+    schema_version: z.literal("trace-eval-set-input/v1"),
+    cases: z.array(InputCaseSchema).min(1),
+});
+// ── trace-test-report/v1 (PR-A defines; PR-B writes) ─────────────────────
+const AssertionResultSchema = z.object({
+    assertion: AssertionSchema,
+    verdict: z.enum(["pass", "fail", "skip"]),
+    actual: z.unknown().optional(),
+});
+const CaseResultSchema = z.object({
+    query_id: z.string().min(1),
+    status: z.enum(["pass", "fail", "error", "skip"]),
+    conversation_id: z.string().nullable(),
+    trace_id: z.string().nullable().optional(),
+    duration_ms: z.number().nonnegative().optional(),
+    assertion_results: z.array(AssertionResultSchema),
+    failure_reason: z.string().optional(),
+    error_code: z.string().optional(),
+    error_message: z.string().optional(),
+});
+export const TestReportSchema = z.object({
+    schema_version: z.literal("trace-test-report/v1"),
+    meta: z.object({
+        eval_set_dir: z.string().min(1),
+        eval_set_id: z.string().min(1),
+        candidate: z.object({
+            agent_id: z.string().min(1),
+            agent_version: z.string().optional(),
+        }),
+        cli_version: z.string().min(1),
+        ran_at: z.string().min(1),
+        duration_ms: z.number().nonnegative(),
+    }),
+    summary: z.object({
+        total: z.number().int().nonnegative(),
+        pass: z.number().int().nonnegative(),
+        fail: z.number().int().nonnegative(),
+        error: z.number().int().nonnegative(),
+        skip: z.number().int().nonnegative(),
+        by_assertion_type: z.record(z.string(), z.object({
+            pass: z.number().int().nonnegative(),
+            fail: z.number().int().nonnegative(),
+        })),
+    }),
+    cases: z.array(CaseResultSchema),
+});

package/dist/trace-ai/eval-set/semantic-match-provider.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Builtin `semantic_match` judge for the eval-set test runner (M5 D5).
+ *
+ * Wraps an `AgentProvider` + the `builtin:answer-match-reference` prompt
+ * template + a small zod output schema into the `SemanticMatchProvider`
+ * surface the assertion-evaluator already speaks. The runner stays
+ * provider-agnostic; only this file knows how to render the rubric
+ * prompt and validate the LLM's reply.
+ *
+ * Output schema is intentionally local to this rubric (spec §4.1) —
+ * not in `schemas.ts`, which only carries the eval-set / report shapes.
+ */
+import { z } from "zod";
+import type { AgentProvider } from "../../agent-providers/types.js";
+import { type PromptTemplateRegistry, type AgentOutputLang } from "../../agent-providers/prompt-template.js";
+import type { SemanticMatchProvider } from "./assertion-evaluator.js";
+export declare const ANSWER_MATCH_REFERENCE_REF = "builtin:answer-match-reference";
+export declare const AnswerMatchOutputSchema: z.ZodObject<{
+    verdict: z.ZodEnum<{
+        pass: "pass";
+        fail: "fail";
+    }>;
+    reasoning: z.ZodString;
+}, z.core.$strip>;
+export interface CreateSemanticMatchProviderOpts {
+    provider: AgentProvider;
+    promptRegistry: PromptTemplateRegistry;
+    /** Output locale for the rubric's reasoning text. Default 'en'. */
+    lang?: AgentOutputLang;
+    /** Per-invoke timeout override. */
+    timeoutMs?: number;
+}
+export declare function createBuiltinSemanticMatchProvider(opts: CreateSemanticMatchProviderOpts): SemanticMatchProvider;

package/dist/trace-ai/eval-set/semantic-match-provider.js

@@ -0,0 +1,51 @@
+/**
+ * Builtin `semantic_match` judge for the eval-set test runner (M5 D5).
+ *
+ * Wraps an `AgentProvider` + the `builtin:answer-match-reference` prompt
+ * template + a small zod output schema into the `SemanticMatchProvider`
+ * surface the assertion-evaluator already speaks. The runner stays
+ * provider-agnostic; only this file knows how to render the rubric
+ * prompt and validate the LLM's reply.
+ *
+ * Output schema is intentionally local to this rubric (spec §4.1) —
+ * not in `schemas.ts`, which only carries the eval-set / report shapes.
+ */
+import { z } from "zod";
+import { render as renderPrompt, languageInstructionFor, } from "../../agent-providers/prompt-template.js";
+export const ANSWER_MATCH_REFERENCE_REF = "builtin:answer-match-reference";
+export const AnswerMatchOutputSchema = z.object({
+    verdict: z.enum(["pass", "fail"]),
+    reasoning: z.string(),
+});
+// JSON shape the LLM is told to emit; rendered into the prompt's
+// `{{output_schema}}` placeholder. Kept declarative so we don't try to
+// reflect a Zod schema into JSON at runtime.
+const OUTPUT_SCHEMA_DOC = {
+    type: "object",
+    required: ["verdict", "reasoning"],
+    properties: {
+        verdict: { type: "string", enum: ["pass", "fail"] },
+        reasoning: { type: "string" },
+    },
+};
+export function createBuiltinSemanticMatchProvider(opts) {
+    const { provider, promptRegistry, lang = "en", timeoutMs } = opts;
+    return {
+        async judgeSemanticMatch(question, candidateAnswer, referenceAnswer) {
+            const tpl = promptRegistry.get(ANSWER_MATCH_REFERENCE_REF);
+            const prompt = renderPrompt(tpl, {
+                question,
+                candidate_answer: candidateAnswer,
+                reference_answer: referenceAnswer,
+                language_instruction: languageInstructionFor(lang),
+                output_schema: OUTPUT_SCHEMA_DOC,
+            });
+            const resp = await provider.invoke({
+                prompt,
+                outputSchema: AnswerMatchOutputSchema,
+                timeoutMs,
+            });
+            return { verdict: resp.output.verdict, reasoning: resp.output.reasoning };
+        },
+    };
+}

package/dist/trace-ai/eval-set/test-runner.d.ts

@@ -0,0 +1,34 @@
+import type { TraceSpan } from "../../api/conversations.js";
+import type { SemanticMatchProvider } from "./assertion-evaluator.js";
+export interface RunnerDeps {
+    fetchAgent: (agentId: string, version?: string) => Promise<{
+        id: string;
+        key: string;
+        version: string;
+    }>;
+    sendChat: (opts: {
+        agentInfo: {
+            id: string;
+            key: string;
+            version: string;
+        };
+        query: string;
+        conversationId?: string;
+    }) => Promise<{
+        text: string;
+        conversationId?: string;
+    }>;
+    fetchTrace: (conversationId: string) => Promise<{
+        spans: TraceSpan[];
+    }>;
+    semanticMatchProvider?: SemanticMatchProvider;
+}
+export interface RunOpts {
+    evalSetDir: string;
+    candidateAgentId: string;
+    candidateAgentVersion?: string;
+    outDir: string;
+    maxParallel?: number;
+    deps: RunnerDeps;
+}
+export declare function run(opts: RunOpts): Promise<void>;

package/dist/trace-ai/eval-set/test-runner.js

@@ -0,0 +1,153 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import yaml from "js-yaml";
+import { evaluateAssertion } from "./assertion-evaluator.js";
+import { EvalSetIndexSchema, EvalSetShardSchema, TestReportSchema } from "./schemas.js";
+// ── eval-set loader ───────────────────────────────────────────────────────────
+async function loadEvalCases(evalSetDir) {
+    const indexRaw = await fs.readFile(path.join(evalSetDir, "index.yaml"), "utf8");
+    const index = EvalSetIndexSchema.parse(yaml.load(indexRaw));
+    const cases = [];
+    for (const shard of index.shards) {
+        const shardRaw = await fs.readFile(path.join(evalSetDir, shard.path), "utf8");
+        const parsed = EvalSetShardSchema.parse(yaml.load(shardRaw));
+        cases.push(...parsed.cases);
+    }
+    return cases;
+}
+// ── case runner ───────────────────────────────────────────────────────────────
+async function runCase(evalCase, agentInfo, deps) {
+    const startMs = Date.now();
+    let conversationId = null;
+    let traceId = null;
+    let spans = [];
+    let answerText = "";
+    let stage = "chat";
+    try {
+        const chatResult = await deps.sendChat({
+            agentInfo,
+            query: evalCase.input.user_message,
+        });
+        answerText = chatResult.text;
+        conversationId = chatResult.conversationId ?? null;
+        if (conversationId) {
+            stage = "trace";
+            const traceResult = await deps.fetchTrace(conversationId);
+            spans = traceResult.spans;
+            traceId = spans[0]?.traceId ?? null;
+        }
+    }
+    catch (e) {
+        const durationMs = Date.now() - startMs;
+        return {
+            query_id: evalCase.query_id,
+            status: "error",
+            conversation_id: conversationId,
+            trace_id: traceId,
+            duration_ms: durationMs,
+            assertion_results: [],
+            error_message: e instanceof Error ? e.message : String(e),
+            error_code: stage === "trace" ? "trace-fetch-failed" : "chat-failed",
+        };
+    }
+    const durationMs = Date.now() - startMs;
+    const assertionResults = [];
+    for (const assertion of evalCase.assertions ?? []) {
+        try {
+            const result = await evaluateAssertion(assertion, {
+                answer: answerText,
+                spans,
+                reference: evalCase.reference,
+                durationMs,
+                question: evalCase.input.user_message,
+                semanticMatchProvider: deps.semanticMatchProvider,
+            });
+            assertionResults.push({ assertion, verdict: result.verdict, actual: result.actual });
+        }
+        catch (e) {
+            assertionResults.push({ assertion, verdict: "skip", actual: `assertion-eval-error: ${e.message}` });
+        }
+    }
+    // A case may pass schema with reference-only (no assertions), but without
+    // assertions there is no pass/fail signal — mark skip so it does not
+    // silently inflate the pass count.
+    if (assertionResults.length === 0) {
+        return {
+            query_id: evalCase.query_id,
+            status: "skip",
+            conversation_id: conversationId,
+            trace_id: traceId,
+            duration_ms: durationMs,
+            assertion_results: assertionResults,
+            failure_reason: "no assertions configured; case has reference but no judge (e.g. semantic_match) wired",
+        };
+    }
+    const hasFail = assertionResults.some((r) => r.verdict === "fail");
+    const allSkip = assertionResults.every((r) => r.verdict === "skip");
+    const status = hasFail ? "fail" : allSkip ? "skip" : "pass";
+    return {
+        query_id: evalCase.query_id,
+        status: status,
+        conversation_id: conversationId,
+        trace_id: traceId,
+        duration_ms: durationMs,
+        assertion_results: assertionResults,
+    };
+}
+// ── main runner ───────────────────────────────────────────────────────────────
+export async function run(opts) {
+    const { evalSetDir, candidateAgentId, candidateAgentVersion, outDir, deps } = opts;
+    const maxParallel = opts.maxParallel ?? 4;
+    const [cases, agentInfo] = await Promise.all([
+        loadEvalCases(evalSetDir),
+        deps.fetchAgent(candidateAgentId, candidateAgentVersion),
+    ]);
+    // Fetch eval_set_id from index
+    const indexRaw = await fs.readFile(path.join(evalSetDir, "index.yaml"), "utf8");
+    const index = EvalSetIndexSchema.parse(yaml.load(indexRaw));
+    const ranAt = new Date().toISOString();
+    const overallStart = Date.now();
+    const caseResults = new Array(cases.length);
+    let nextIdx = 0;
+    const workerCount = Math.max(1, Math.min(maxParallel, cases.length));
+    const workers = Array.from({ length: workerCount }, async () => {
+        while (true) {
+            const idx = nextIdx++;
+            if (idx >= cases.length)
+                return;
+            caseResults[idx] = await runCase(cases[idx], agentInfo, deps);
+        }
+    });
+    await Promise.all(workers);
+    const overallDurationMs = Date.now() - overallStart;
+    // Build summary
+    const counts = { total: caseResults.length, pass: 0, fail: 0, error: 0, skip: 0 };
+    const byType = {};
+    for (const cr of caseResults) {
+        counts[cr.status]++;
+        for (const ar of cr.assertion_results) {
+            const t = ar.assertion["type"];
+            if (!byType[t])
+                byType[t] = { pass: 0, fail: 0 };
+            if (ar.verdict === "pass")
+                byType[t].pass++;
+            else if (ar.verdict === "fail")
+                byType[t].fail++;
+        }
+    }
+    const report = TestReportSchema.parse({
+        schema_version: "trace-test-report/v1",
+        meta: {
+            eval_set_dir: evalSetDir,
+            eval_set_id: index.eval_set_id,
+            candidate: { agent_id: agentInfo.id, agent_version: agentInfo.version },
+            cli_version: "0.0.0",
+            ran_at: ranAt,
+            duration_ms: overallDurationMs,
+        },
+        summary: { ...counts, by_assertion_type: byType },
+        cases: caseResults,
+    });
+    await fs.mkdir(outDir, { recursive: true });
+    await fs.writeFile(path.join(outDir, "report.yaml"), yaml.dump(report, { lineWidth: 120 }), "utf8");
+}

package/dist/trace-ai/eval-set/types.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Internal types for the M5 eval-set module (PR-A).
+ *
+ * These mirror the zod schemas in `./schemas.ts` but are kept independent so
+ * non-validating code paths (builder / picker / redactor / output-writer) can
+ * import the types without paying the zod parse overhead at module load.
+ */
+export interface EvalCaseInput {
+    user_message: string;
+}
+export interface EvalReference {
+    answer: string;
+}
+export type AssertionType = "contains" | "not_contains" | "regex" | "tool_call_count" | "tool_call_order" | "semantic_match" | "latency_ms";
+export interface EvalAssertion {
+    type: AssertionType;
+    [key: string]: unknown;
+}
+export interface EvalCase {
+    query_id: string;
+    input: EvalCaseInput;
+    reference?: EvalReference;
+    assertions?: EvalAssertion[];
+    tags?: string[];
+}
+export interface EvalSetIndexShard {
+    path: string;
+    role?: "seed" | "regression" | "holdout";
+}
+export interface EvalSetIndex {
+    schema_version: "trace-eval-set-index/v1";
+    eval_set_id: string;
+    shards: EvalSetIndexShard[];
+}
+export interface BuildResult {
+    cases_written: number;
+    cases_skipped: number;
+    conflicts: string[];
+    shard_paths: string[];
+    redaction_rules_source: "cli-flag" | "repo" | "builtin";
+}
+export interface RedactionRule {
+    name: string;
+    pattern: RegExp;
+    replace: string;
+}

package/dist/trace-ai/eval-set/types.js

@@ -0,0 +1,8 @@
+/**
+ * Internal types for the M5 eval-set module (PR-A).
+ *
+ * These mirror the zod schemas in `./schemas.ts` but are kept independent so
+ * non-validating code paths (builder / picker / redactor / output-writer) can
+ * import the types without paying the zod parse overhead at module load.
+ */
+export {};

package/dist/trace-ai/exp/bundle-writer.d.ts

@@ -0,0 +1,10 @@
+import type { LineageEntry, RoundData } from "./schemas.js";
+interface WriteBundlesOpts {
+    expDir: string;
+    experimentId: string;
+    lineage: LineageEntry[];
+    rounds: RoundData[];
+    createdBy: string;
+}
+export declare function writeBundles(opts: WriteBundlesOpts): Promise<void>;
+export {};

package/dist/trace-ai/exp/bundle-writer.js

@@ -0,0 +1,54 @@
+// src/trace-ai/exp/bundle-writer.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+import crypto from "node:crypto";
+import yaml from "js-yaml";
+export async function writeBundles(opts) {
+    const { expDir, experimentId, lineage, rounds, createdBy } = opts;
+    const bestEntry = lineage.filter(e => e.status === "scored").at(-1) ?? lineage.at(-1);
+    const bestVersion = bestEntry?.version ?? 0;
+    const bundleId = `bundle_${crypto.randomBytes(4).toString("hex")}`;
+    const now = new Date().toISOString();
+    const bundle = {
+        schema_version: "trace-bundle/v1",
+        experiment_id: experimentId,
+        bundle_id: bundleId,
+        best_trial_version: bestVersion,
+        resources: {
+            agent_config: bestEntry?.next_change ?? {},
+            skills: [],
+        },
+        provenance: {
+            created_by: createdBy,
+            created_at: now,
+            evidence_traces: rounds.flatMap(r => (r.per_query_results ?? []).map(q => q.raw_trace_id ?? "").filter(Boolean)),
+            round_refs: rounds.map(r => `.trace-state/rounds/round-${r.round}.yaml`),
+        },
+    };
+    const lastRound = rounds.at(-1);
+    const manifest = {
+        schema_version: "trace-manifest/v1",
+        experiment_id: experimentId,
+        trial_version: bestVersion,
+        predictions: {
+            fixes: (lastRound?.per_query_results ?? [])
+                .filter(q => q.assertion_results.every(a => a.verdict === "pass"))
+                .map(q => ({ query_id: q.query_id, reason: "all assertions passed" })),
+            risks: (lastRound?.per_query_results ?? [])
+                .filter(q => q.assertion_results.some(a => a.verdict === "fail"))
+                .map(q => ({ query_id: q.query_id, reason: "assertions failed" })),
+        },
+    };
+    const provenance = {
+        experiment_id: experimentId,
+        generated_at: now,
+        rounds_count: rounds.length,
+        lineage_count: lineage.length,
+        round_verdicts: rounds.map(r => ({ round: r.round, verdict: r.triage_conclusion?.verdict ?? "pending" })),
+    };
+    const outDir = path.join(expDir, "outputs");
+    await fs.mkdir(outDir, { recursive: true });
+    await fs.writeFile(path.join(outDir, "bundle.yaml"), yaml.dump(bundle, { lineWidth: -1 }));
+    await fs.writeFile(path.join(outDir, "manifest.yaml"), yaml.dump(manifest, { lineWidth: -1 }));
+    await fs.writeFile(path.join(outDir, "provenance.yaml"), yaml.dump(provenance, { lineWidth: -1 }));
+}

package/dist/trace-ai/exp/claude-binary.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Resolves the path to the claude CLI binary.
+ * Priority: CLAUDE_BIN env → `which claude` → known install locations → bare "claude".
+ */
+export declare function resolveClaudeBinary(): string;

package/dist/trace-ai/exp/claude-binary.js

@@ -0,0 +1,30 @@
+import { execSync } from "node:child_process";
+import os from "node:os";
+/**
+ * Resolves the path to the claude CLI binary.
+ * Priority: CLAUDE_BIN env → `which claude` → known install locations → bare "claude".
+ */
+export function resolveClaudeBinary() {
+    if (process.env["CLAUDE_BIN"])
+        return process.env["CLAUDE_BIN"];
+    try {
+        const resolved = execSync("which claude", { encoding: "utf8", timeout: 3000 }).trim();
+        // Reject shell alias expansions like "claude: aliased to ..."
+        if (resolved && !resolved.includes(" "))
+            return resolved;
+    }
+    catch { /* fall through */ }
+    const home = os.homedir();
+    for (const p of [
+        `${home}/.local/bin/claude`,
+        "/opt/homebrew/bin/claude",
+        "/usr/local/bin/claude",
+    ]) {
+        try {
+            execSync(`test -x "${p}"`, { timeout: 1000 });
+            return p;
+        }
+        catch { /* try next */ }
+    }
+    return "claude";
+}

package/dist/trace-ai/exp/coordinator.d.ts

@@ -0,0 +1,45 @@
+import type { Mission, NextChange, QueryResult, RoundData } from "./schemas.js";
+export interface SynthesizerClient {
+    generate(input: {
+        mission: Mission;
+        candidateConfig: Record<string, unknown>;
+        prevRound?: RoundData;
+        prevRounds: RoundData[];
+        crossRoundMemoryRef?: string;
+    }): Promise<NextChange>;
+}
+export interface TriageClient {
+    triage(input: {
+        currentRound: RoundData;
+        prevRounds: RoundData[];
+        candidateConfig: Record<string, unknown>;
+        crossRoundMemoryRef?: string;
+    }): Promise<RoundData["triage_conclusion"] & {
+        new_memory_token: string;
+    }>;
+}
+export interface CoordinatorOpts {
+    expDir: string;
+    synthesizer: SynthesizerClient;
+    triage: TriageClient;
+    runEval: (opts: {
+        evalSetPaths: string[];
+        candidatePath: string;
+        expDir: string;
+        round: number;
+    }) => Promise<{
+        queryResults: QueryResult[];
+    }>;
+    experimentId?: string;
+}
+export declare class ExperimentCoordinator {
+    private opts;
+    private store;
+    private heartbeatTimer?;
+    constructor(opts: CoordinatorOpts);
+    run(): Promise<void>;
+    resume(): Promise<void>;
+    private runLoop;
+    private checkAbort;
+    private withRetry;
+}