@kweaver-ai/kweaver-sdk 0.8.1 → 0.8.3

package/dist/trace-ai/eval-set/redactor.js

@@ -0,0 +1,133 @@
+/**
+ * M5 eval-set redactor — PII pattern matching + replacement.
+ *
+ * Three rule sources, in priority order (chain):
+ *   1. --redaction-rules=<path>      (CLI flag, highest)
+ *   2. <repo>/redaction-rules/*.yaml (repo-local)
+ *   3. BUILTIN_RULES                 (5 low-fidelity defaults)
+ *
+ * Builtin rules cover common Chinese-context PII: phone / email / id_card /
+ * bank_card / ip. Organizations write more rules in <repo>/redaction-rules/
+ * for their business-specific patterns.
+ *
+ * Rule yaml format:
+ *   rules:
+ *     - name: <id>
+ *       pattern: <regex source string>
+ *       replace: <replacement template; supports {hash6} placeholder>
+ *
+ * Malformed regex causes loadRules to throw RedactorError (no silent fallback).
+ */
+import { readFile, readdir, stat } from "node:fs/promises";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import yaml from "js-yaml";
+export class RedactorError extends Error {
+    path;
+    constructor(message, path) {
+        super(message);
+        this.path = path;
+        this.name = "RedactorError";
+    }
+}
+/**
+ * 5 builtin low-fidelity PII patterns. Tuned for Chinese-context defaults;
+ * organizations override with their own rules in <repo>/redaction-rules/.
+ */
+export const BUILTIN_RULES = [
+    {
+        name: "phone",
+        pattern: /1[3-9]\d{9}/g,
+        replace: "<phone:{hash6}>",
+    },
+    {
+        name: "email",
+        pattern: /[\w.+-]+@[\w.-]+\.\w+/g,
+        replace: "<email:{hash6}>",
+    },
+    {
+        name: "id_card",
+        pattern: /\b\d{17}[\dXx]\b/g,
+        replace: "<id_card:{hash6}>",
+    },
+    {
+        name: "bank_card",
+        pattern: /\b\d{15,19}\b/g, // 银行卡号长度 15-19 位
+        replace: "<bank_card:{hash6}>",
+    },
+    {
+        name: "ip",
+        pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g,
+        replace: "<ip:{hash6}>",
+    },
+];
+function compileRule(entry, srcPath) {
+    let pattern;
+    try {
+        pattern = new RegExp(entry.pattern, "g");
+    }
+    catch (e) {
+        throw new RedactorError(`invalid regex in rule '${entry.name}' at ${srcPath}: ${e.message}`, srcPath);
+    }
+    return { name: entry.name, pattern, replace: entry.replace };
+}
+async function readRulesFile(filePath) {
+    let raw;
+    try {
+        raw = await readFile(filePath, "utf8");
+    }
+    catch (e) {
+        throw new RedactorError(`failed to read rule file ${filePath}: ${e.message}`, filePath);
+    }
+    let parsed;
+    try {
+        parsed = yaml.load(raw);
+    }
+    catch (e) {
+        throw new RedactorError(`failed to parse yaml ${filePath}: ${e.message}`, filePath);
+    }
+    const doc = parsed;
+    if (!doc || !Array.isArray(doc.rules)) {
+        throw new RedactorError(`rule file ${filePath} must have top-level 'rules: []'`, filePath);
+    }
+    return doc.rules.map((e) => compileRule(e, filePath));
+}
+export async function loadRules(opts) {
+    if (opts.cliFlag) {
+        const rules = await readRulesFile(opts.cliFlag);
+        return { rules, source: "cli-flag" };
+    }
+    if (opts.repoDir) {
+        let stats;
+        try {
+            stats = await stat(opts.repoDir);
+        }
+        catch {
+            stats = null;
+        }
+        if (stats && stats.isDirectory()) {
+            const entries = await readdir(opts.repoDir);
+            const yamlFiles = entries
+                .filter((e) => e.endsWith(".yaml") || e.endsWith(".yml"))
+                .map((e) => path.join(opts.repoDir, e));
+            if (yamlFiles.length > 0) {
+                const allRules = [];
+                for (const f of yamlFiles) {
+                    allRules.push(...(await readRulesFile(f)));
+                }
+                return { rules: allRules, source: "repo" };
+            }
+        }
+    }
+    return { rules: BUILTIN_RULES, source: "builtin" };
+}
+function hash6(input) {
+    return createHash("sha256").update(input).digest("hex").slice(0, 6);
+}
+export function applyRules(text, rules) {
+    let out = text;
+    for (const rule of rules) {
+        out = out.replace(rule.pattern, (match) => rule.replace.replace("{hash6}", hash6(match)));
+    }
+    return out;
+}

package/dist/trace-ai/eval-set/rubric-templates/answer-match-reference.prompt.md

@@ -0,0 +1,19 @@
+You are evaluating whether a candidate answer is semantically equivalent to a reference answer for a knowledge-graph Q&A system.
+
+Question: {{question}}
+
+Reference answer (ground truth):
+{{reference_answer}}
+
+Candidate answer (from agent):
+{{candidate_answer}}
+
+Judge whether the candidate answer is semantically correct relative to the reference.
+A candidate passes if it conveys the same key facts, even if phrased differently.
+A candidate fails if it omits critical facts, states incorrect facts, or hallucinates information not in the reference.
+Partial answers that cover most key facts but miss minor details should pass.
+
+{{language_instruction}}
+
+Respond with valid JSON matching this schema exactly:
+{{output_schema}}

package/dist/trace-ai/eval-set/schemas.d.ts

@@ -0,0 +1,136 @@
+/**
+ * M5 eval-set zod schemas (PR-A, MVP-B scope).
+ *
+ * 4 schemas in this file:
+ *   - EvalSetIndexSchema:  trace-eval-set-index/v1   (eval-set dir's index.yaml)
+ *   - EvalSetShardSchema:  trace-eval-set/v1         (final shard yaml file)
+ *   - EvalSetInputSchema:  trace-eval-set-input/v1   (--queries simplified input)
+ *   - TestReportSchema:    trace-test-report/v1      (test report; PR-A defines schema only;
+ *                                                     PR-B consumer)
+ *
+ * EvalSetShardSchema and EvalSetInputSchema share the same refinement:
+ *   "for each case, at least one of {reference, non-empty assertions} must be present."
+ *
+ * The D5 builtin rubric `answer-match-reference` output schema is NOT here —
+ * it belongs to the rubric template definition (per spec doc §4.1).
+ */
+import { z } from "zod";
+export declare const EvalSetIndexSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-eval-set-index/v1">;
+    eval_set_id: z.ZodString;
+    shards: z.ZodArray<z.ZodObject<{
+        path: z.ZodString;
+        role: z.ZodOptional<z.ZodEnum<{
+            seed: "seed";
+            regression: "regression";
+            holdout: "holdout";
+        }>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const EvalSetShardSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-eval-set/v1">;
+    cases: z.ZodArray<z.ZodObject<{
+        query_id: z.ZodString;
+        input: z.ZodObject<{
+            user_message: z.ZodString;
+        }, z.core.$strip>;
+        reference: z.ZodOptional<z.ZodObject<{
+            answer: z.ZodString;
+        }, z.core.$strip>>;
+        assertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            type: z.ZodEnum<{
+                contains: "contains";
+                not_contains: "not_contains";
+                regex: "regex";
+                tool_call_count: "tool_call_count";
+                tool_call_order: "tool_call_order";
+                semantic_match: "semantic_match";
+                latency_ms: "latency_ms";
+            }>;
+        }, z.core.$loose>>>;
+        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const EvalSetInputSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-eval-set-input/v1">;
+    cases: z.ZodArray<z.ZodObject<{
+        input: z.ZodObject<{
+            user_message: z.ZodString;
+        }, z.core.$strip>;
+        query_id: z.ZodOptional<z.ZodString>;
+        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        reference: z.ZodOptional<z.ZodObject<{
+            answer: z.ZodString;
+        }, z.core.$strip>>;
+        assertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            type: z.ZodEnum<{
+                contains: "contains";
+                not_contains: "not_contains";
+                regex: "regex";
+                tool_call_count: "tool_call_count";
+                tool_call_order: "tool_call_order";
+                semantic_match: "semantic_match";
+                latency_ms: "latency_ms";
+            }>;
+        }, z.core.$loose>>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const TestReportSchema: z.ZodObject<{
+    schema_version: z.ZodLiteral<"trace-test-report/v1">;
+    meta: z.ZodObject<{
+        eval_set_dir: z.ZodString;
+        eval_set_id: z.ZodString;
+        candidate: z.ZodObject<{
+            agent_id: z.ZodString;
+            agent_version: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>;
+        cli_version: z.ZodString;
+        ran_at: z.ZodString;
+        duration_ms: z.ZodNumber;
+    }, z.core.$strip>;
+    summary: z.ZodObject<{
+        total: z.ZodNumber;
+        pass: z.ZodNumber;
+        fail: z.ZodNumber;
+        error: z.ZodNumber;
+        skip: z.ZodNumber;
+        by_assertion_type: z.ZodRecord<z.ZodString, z.ZodObject<{
+            pass: z.ZodNumber;
+            fail: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    cases: z.ZodArray<z.ZodObject<{
+        query_id: z.ZodString;
+        status: z.ZodEnum<{
+            error: "error";
+            pass: "pass";
+            fail: "fail";
+            skip: "skip";
+        }>;
+        conversation_id: z.ZodNullable<z.ZodString>;
+        trace_id: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        duration_ms: z.ZodOptional<z.ZodNumber>;
+        assertion_results: z.ZodArray<z.ZodObject<{
+            assertion: z.ZodObject<{
+                type: z.ZodEnum<{
+                    contains: "contains";
+                    not_contains: "not_contains";
+                    regex: "regex";
+                    tool_call_count: "tool_call_count";
+                    tool_call_order: "tool_call_order";
+                    semantic_match: "semantic_match";
+                    latency_ms: "latency_ms";
+                }>;
+            }, z.core.$loose>;
+            verdict: z.ZodEnum<{
+                pass: "pass";
+                fail: "fail";
+                skip: "skip";
+            }>;
+            actual: z.ZodOptional<z.ZodUnknown>;
+        }, z.core.$strip>>;
+        failure_reason: z.ZodOptional<z.ZodString>;
+        error_code: z.ZodOptional<z.ZodString>;
+        error_message: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;

package/dist/trace-ai/eval-set/schemas.js

@@ -0,0 +1,130 @@
+/**
+ * M5 eval-set zod schemas (PR-A, MVP-B scope).
+ *
+ * 4 schemas in this file:
+ *   - EvalSetIndexSchema:  trace-eval-set-index/v1   (eval-set dir's index.yaml)
+ *   - EvalSetShardSchema:  trace-eval-set/v1         (final shard yaml file)
+ *   - EvalSetInputSchema:  trace-eval-set-input/v1   (--queries simplified input)
+ *   - TestReportSchema:    trace-test-report/v1      (test report; PR-A defines schema only;
+ *                                                     PR-B consumer)
+ *
+ * EvalSetShardSchema and EvalSetInputSchema share the same refinement:
+ *   "for each case, at least one of {reference, non-empty assertions} must be present."
+ *
+ * The D5 builtin rubric `answer-match-reference` output schema is NOT here —
+ * it belongs to the rubric template definition (per spec doc §4.1).
+ */
+import { z } from "zod";
+const InputSchema = z.object({
+    user_message: z.string().min(1),
+});
+const ReferenceSchema = z.object({
+    answer: z.string().min(1),
+});
+const AssertionSchema = z.object({
+    type: z.enum([
+        "contains",
+        "not_contains",
+        "regex",
+        "tool_call_count",
+        "tool_call_order",
+        "semantic_match",
+        "latency_ms",
+    ]),
+}).passthrough(); // allow type-specific fields (value, pattern, tool, op, n, ...)
+// ── trace-eval-set-index/v1 ──────────────────────────────────────────────
+const ShardRefSchema = z.object({
+    path: z
+        .string()
+        .min(1)
+        .refine((p) => !p.includes("..") && !p.startsWith("/"), {
+        message: "shard path must be a relative path within the eval-set directory (no '..' / '/')",
+    }),
+    role: z.enum(["seed", "regression", "holdout"]).optional(),
+});
+export const EvalSetIndexSchema = z.object({
+    schema_version: z.literal("trace-eval-set-index/v1"),
+    eval_set_id: z.string().min(1),
+    shards: z.array(ShardRefSchema).min(1),
+});
+// ── trace-eval-set/v1 ────────────────────────────────────────────────────
+const refineCase = (data, ctx) => {
+    const hasReference = data.reference !== undefined && data.reference !== null;
+    const hasAssertions = Array.isArray(data.assertions) && data.assertions.length > 0;
+    if (!hasReference && !hasAssertions) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "each case must have either a 'reference' object or a non-empty 'assertions' array; both empty is not allowed (evaluator has no pass/fail signal)",
+        });
+    }
+};
+const FinalCaseSchema = z
+    .object({
+    query_id: z.string().min(1),
+    input: InputSchema,
+    reference: ReferenceSchema.optional(),
+    assertions: z.array(AssertionSchema).optional(),
+    tags: z.array(z.string()).optional(),
+})
+    .superRefine(refineCase);
+export const EvalSetShardSchema = z.object({
+    schema_version: z.literal("trace-eval-set/v1"),
+    cases: z.array(FinalCaseSchema).min(1),
+});
+// ── trace-eval-set-input/v1 (D1: same refinement as final) ───────────────
+const InputCaseSchema = z
+    .object({
+    input: InputSchema,
+    query_id: z.string().min(1).optional(),
+    tags: z.array(z.string()).optional(),
+    reference: ReferenceSchema.optional(),
+    assertions: z.array(AssertionSchema).optional(),
+})
+    .superRefine(refineCase);
+export const EvalSetInputSchema = z.object({
+    schema_version: z.literal("trace-eval-set-input/v1"),
+    cases: z.array(InputCaseSchema).min(1),
+});
+// ── trace-test-report/v1 (PR-A defines; PR-B writes) ─────────────────────
+const AssertionResultSchema = z.object({
+    assertion: AssertionSchema,
+    verdict: z.enum(["pass", "fail", "skip"]),
+    actual: z.unknown().optional(),
+});
+const CaseResultSchema = z.object({
+    query_id: z.string().min(1),
+    status: z.enum(["pass", "fail", "error", "skip"]),
+    conversation_id: z.string().nullable(),
+    trace_id: z.string().nullable().optional(),
+    duration_ms: z.number().nonnegative().optional(),
+    assertion_results: z.array(AssertionResultSchema),
+    failure_reason: z.string().optional(),
+    error_code: z.string().optional(),
+    error_message: z.string().optional(),
+});
+export const TestReportSchema = z.object({
+    schema_version: z.literal("trace-test-report/v1"),
+    meta: z.object({
+        eval_set_dir: z.string().min(1),
+        eval_set_id: z.string().min(1),
+        candidate: z.object({
+            agent_id: z.string().min(1),
+            agent_version: z.string().optional(),
+        }),
+        cli_version: z.string().min(1),
+        ran_at: z.string().min(1),
+        duration_ms: z.number().nonnegative(),
+    }),
+    summary: z.object({
+        total: z.number().int().nonnegative(),
+        pass: z.number().int().nonnegative(),
+        fail: z.number().int().nonnegative(),
+        error: z.number().int().nonnegative(),
+        skip: z.number().int().nonnegative(),
+        by_assertion_type: z.record(z.string(), z.object({
+            pass: z.number().int().nonnegative(),
+            fail: z.number().int().nonnegative(),
+        })),
+    }),
+    cases: z.array(CaseResultSchema),
+});

package/dist/trace-ai/eval-set/semantic-match-provider.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Builtin `semantic_match` judge for the eval-set test runner (M5 D5).
+ *
+ * Wraps an `AgentProvider` + the `builtin:answer-match-reference` prompt
+ * template + a small zod output schema into the `SemanticMatchProvider`
+ * surface the assertion-evaluator already speaks. The runner stays
+ * provider-agnostic; only this file knows how to render the rubric
+ * prompt and validate the LLM's reply.
+ *
+ * Output schema is intentionally local to this rubric (spec §4.1) —
+ * not in `schemas.ts`, which only carries the eval-set / report shapes.
+ */
+import { z } from "zod";
+import type { AgentProvider } from "../../agent-providers/types.js";
+import { type PromptTemplateRegistry, type AgentOutputLang } from "../../agent-providers/prompt-template.js";
+import type { SemanticMatchProvider } from "./assertion-evaluator.js";
+export declare const ANSWER_MATCH_REFERENCE_REF = "builtin:answer-match-reference";
+export declare const AnswerMatchOutputSchema: z.ZodObject<{
+    verdict: z.ZodEnum<{
+        pass: "pass";
+        fail: "fail";
+    }>;
+    reasoning: z.ZodString;
+}, z.core.$strip>;
+export interface CreateSemanticMatchProviderOpts {
+    provider: AgentProvider;
+    promptRegistry: PromptTemplateRegistry;
+    /** Output locale for the rubric's reasoning text. Default 'en'. */
+    lang?: AgentOutputLang;
+    /** Per-invoke timeout override. */
+    timeoutMs?: number;
+}
+export declare function createBuiltinSemanticMatchProvider(opts: CreateSemanticMatchProviderOpts): SemanticMatchProvider;

package/dist/trace-ai/eval-set/semantic-match-provider.js

@@ -0,0 +1,51 @@
+/**
+ * Builtin `semantic_match` judge for the eval-set test runner (M5 D5).
+ *
+ * Wraps an `AgentProvider` + the `builtin:answer-match-reference` prompt
+ * template + a small zod output schema into the `SemanticMatchProvider`
+ * surface the assertion-evaluator already speaks. The runner stays
+ * provider-agnostic; only this file knows how to render the rubric
+ * prompt and validate the LLM's reply.
+ *
+ * Output schema is intentionally local to this rubric (spec §4.1) —
+ * not in `schemas.ts`, which only carries the eval-set / report shapes.
+ */
+import { z } from "zod";
+import { render as renderPrompt, languageInstructionFor, } from "../../agent-providers/prompt-template.js";
+export const ANSWER_MATCH_REFERENCE_REF = "builtin:answer-match-reference";
+export const AnswerMatchOutputSchema = z.object({
+    verdict: z.enum(["pass", "fail"]),
+    reasoning: z.string(),
+});
+// JSON shape the LLM is told to emit; rendered into the prompt's
+// `{{output_schema}}` placeholder. Kept declarative so we don't try to
+// reflect a Zod schema into JSON at runtime.
+const OUTPUT_SCHEMA_DOC = {
+    type: "object",
+    required: ["verdict", "reasoning"],
+    properties: {
+        verdict: { type: "string", enum: ["pass", "fail"] },
+        reasoning: { type: "string" },
+    },
+};
+export function createBuiltinSemanticMatchProvider(opts) {
+    const { provider, promptRegistry, lang = "en", timeoutMs } = opts;
+    return {
+        async judgeSemanticMatch(question, candidateAnswer, referenceAnswer) {
+            const tpl = promptRegistry.get(ANSWER_MATCH_REFERENCE_REF);
+            const prompt = renderPrompt(tpl, {
+                question,
+                candidate_answer: candidateAnswer,
+                reference_answer: referenceAnswer,
+                language_instruction: languageInstructionFor(lang),
+                output_schema: OUTPUT_SCHEMA_DOC,
+            });
+            const resp = await provider.invoke({
+                prompt,
+                outputSchema: AnswerMatchOutputSchema,
+                timeoutMs,
+            });
+            return { verdict: resp.output.verdict, reasoning: resp.output.reasoning };
+        },
+    };
+}

package/dist/trace-ai/eval-set/test-runner.d.ts

@@ -0,0 +1,34 @@
+import type { TraceSpan } from "../../api/conversations.js";
+import type { SemanticMatchProvider } from "./assertion-evaluator.js";
+export interface RunnerDeps {
+    fetchAgent: (agentId: string, version?: string) => Promise<{
+        id: string;
+        key: string;
+        version: string;
+    }>;
+    sendChat: (opts: {
+        agentInfo: {
+            id: string;
+            key: string;
+            version: string;
+        };
+        query: string;
+        conversationId?: string;
+    }) => Promise<{
+        text: string;
+        conversationId?: string;
+    }>;
+    fetchTrace: (conversationId: string) => Promise<{
+        spans: TraceSpan[];
+    }>;
+    semanticMatchProvider?: SemanticMatchProvider;
+}
+export interface RunOpts {
+    evalSetDir: string;
+    candidateAgentId: string;
+    candidateAgentVersion?: string;
+    outDir: string;
+    maxParallel?: number;
+    deps: RunnerDeps;
+}
+export declare function run(opts: RunOpts): Promise<void>;