@kweaver-ai/kweaver-sdk 0.7.4 → 0.8.2

package/dist/trace-ai/diagnose/trace-shaper.js

@@ -0,0 +1,73 @@
+// Map from OTel GenAI `gen_ai.operation.name` (the cross-runtime standard) to
+// the diagnostic SpanKind buckets rules filter on. `agent.trace.type` is kept
+// as a fallback for runtimes/fixtures that pre-tag spans with our own taxonomy.
+const KIND_MAP = {
+    // OTel GenAI semconv operation names
+    chat: "llm",
+    text_completion: "llm",
+    embeddings: "retrieval",
+    execute_tool: "tool",
+    // pre-existing custom taxonomy (synthetic fixtures, optional runtime tag)
+    model: "llm",
+    llm: "llm",
+    tool: "tool",
+    retrieval: "retrieval",
+    reasoning: "reasoning",
+};
+function deriveKind(attrs) {
+    const op = attrs["gen_ai.operation.name"];
+    if (typeof op === "string" && op in KIND_MAP)
+        return KIND_MAP[op];
+    const t = attrs["agent.trace.type"];
+    if (typeof t === "string" && t in KIND_MAP)
+        return KIND_MAP[t];
+    return "unknown";
+}
+function deriveStatus(raw) {
+    const code = raw?.code?.toUpperCase();
+    if (code === "OK")
+        return "ok";
+    if (code === "ERROR")
+        return "error";
+    return "unset";
+}
+function durationMs(start, end) {
+    if (!start || !end)
+        return 0;
+    // string nanos → BigInt to avoid precision loss, then convert to ms.
+    const s = BigInt(start);
+    const e = BigInt(end);
+    return Number((e - s) / 1000000n);
+}
+export function assembleTraceTree(traceId, raw) {
+    const spans = raw.map((r) => {
+        const attrs = r.attributes ?? {};
+        return {
+            spanId: r.spanId,
+            parentSpanId: r.parentSpanId ?? null,
+            name: r.name ?? "",
+            kind: deriveKind(attrs),
+            startTimeUnixNano: r.startTimeUnixNano ?? "0",
+            endTimeUnixNano: r.endTimeUnixNano ?? "0",
+            durationMs: durationMs(r.startTimeUnixNano, r.endTimeUnixNano),
+            status: deriveStatus(r.status),
+            attributes: attrs,
+            events: r.events,
+        };
+    });
+    const byId = new Map();
+    const parentToChildren = new Map();
+    const byKind = new Map();
+    for (const s of spans) {
+        byId.set(s.spanId, s);
+        const arr = parentToChildren.get(s.parentSpanId) ?? [];
+        arr.push(s);
+        parentToChildren.set(s.parentSpanId, arr);
+        const kindArr = byKind.get(s.kind) ?? [];
+        kindArr.push(s);
+        byKind.set(s.kind, kindArr);
+    }
+    const roots = parentToChildren.get(null) ?? [];
+    const root = roots.length > 0 ? roots[0] : null;
+    return { traceId, spans, byId, parentToChildren, byKind, root };
+}

package/dist/trace-ai/diagnose/types.d.ts

@@ -0,0 +1,173 @@
+export interface SpanAttributes {
+    [key: string]: unknown;
+}
+export interface Span {
+    spanId: string;
+    parentSpanId: string | null;
+    name: string;
+    kind: SpanKind;
+    startTimeUnixNano: string;
+    endTimeUnixNano: string;
+    durationMs: number;
+    status: 'ok' | 'error' | 'unset';
+    attributes: SpanAttributes;
+    events?: Array<{
+        name?: string;
+        time?: string;
+        attributes?: Record<string, unknown>;
+    }>;
+}
+export type SpanKind = 'tool' | 'llm' | 'retrieval' | 'reasoning' | 'unknown';
+export interface TraceTree {
+    traceId: string;
+    spans: Span[];
+    byId: Map<string, Span>;
+    parentToChildren: Map<string | null, Span[]>;
+    byKind: Map<SpanKind, Span[]>;
+    root: Span | null;
+}
+export interface RuleTaxonomy {
+    signalsAxis: 'interaction' | 'execution' | 'environment';
+    msClass: 'retry_loop' | 'tool_misuse' | 'context_loss' | 'goal_drift' | 'cascading_error' | 'silent_quality_degradation';
+}
+export interface RubricInputSpec {
+    kind: string;
+    source: string;
+}
+export interface RubricSpec {
+    judgeQuestion: string;
+    inputs: RubricInputSpec[];
+    /** Original JSON-Schema-ish blob (kept for YAML round-trips / debug). */
+    outputSchemaRaw: Record<string, unknown>;
+    /** Compiled zod schema (built once at load time via output-schema-converter). */
+    outputZodSchema: import("zod").ZodTypeAny;
+    agentBinding: {
+        provider: string;
+        promptTemplateRef: string;
+    };
+    /** Optional gating; see RuleSchema.rubric.gates_on. */
+    gatesOn?: string[];
+}
+export interface Rule {
+    schemaVersion: 'diagnosis-rule/v1';
+    id: string;
+    severity: 'low' | 'medium' | 'high';
+    symptom: string;
+    taxonomy: RuleTaxonomy;
+    suggestedFix: {
+        target: string;
+        changeTemplate: string;
+    };
+    verifyWith: {
+        assertionTemplates: string[];
+    };
+    /** Exactly one of `predicateRef` or `rubric` is non-null (XOR enforced at load). */
+    predicateRef: string | null;
+    rubric: RubricSpec | null;
+    params: Record<string, unknown>;
+    sourcePath: string;
+}
+export type JudgmentKind = 'symbolic' | 'rubric';
+export interface Hit {
+    evidenceSpans: string[];
+    excerpt: string;
+    bindings: Record<string, unknown>;
+}
+export type Predicate = (trace: TraceTree, params: Record<string, unknown>) => Hit[];
+export interface Finding {
+    ruleId: string;
+    judgmentKind: JudgmentKind;
+    severity: 'low' | 'medium' | 'high';
+    symptom: string;
+    likelyCause: string;
+    evidence: {
+        spans: string[];
+        excerpt: string;
+    };
+    suggestedFix: {
+        target: string;
+        change: string;
+    };
+    /** Symbolic always 'low' (no semantic basis); rubric carries agent confidence. */
+    confidence: 'low' | 'medium' | 'high';
+    verifyWith: {
+        suggestedEvalCase: {
+            queryId: string | null;
+            query: string | null;
+            assertions: string[];
+        };
+    };
+}
+export interface SummaryRootCause {
+    findingIds: number[];
+    description: string;
+    targetForFix: string;
+}
+export interface SummaryFixPriority {
+    findingId: number;
+    reason: string;
+}
+export interface SummaryCrossLink {
+    findingIds: number[];
+    relation: string;
+}
+export interface Summary {
+    headline: string;
+    primaryRootCause: SummaryRootCause | null;
+    fixPriority: SummaryFixPriority[];
+    crossFindingLinks: SummaryCrossLink[];
+}
+export interface Report {
+    schemaVersion: 'trace-diagnose-report/v1';
+    trace: {
+        traceId: string;
+        agentId: string | null;
+        tenant: string | null;
+    };
+    run: {
+        diagnosedAt: string;
+        cliVersion: string;
+        mode: 'symbolic-only' | 'rubric-only' | 'hybrid';
+        rulesApplied: string[];
+        rulesSkipped: {
+            ruleId: string;
+            reason: string;
+        }[];
+        synthesizerMode: 'template' | 'agent';
+    };
+    summary: Summary;
+    findings: Finding[];
+}
+export interface DiagnoseOpts {
+    out: string | null;
+    rulesDir: string | null;
+    noBuiltin: boolean;
+    /** PR-B: when true, skip rubric rules (warn + record in rules_skipped) AND
+     *  fall the synthesizer back from agent → template. Default is now false
+     *  (both pillars on). */
+    noLlm: boolean;
+    /** Skip artifact persistence. Default false (artifacts ARE written). */
+    noArtifacts?: boolean;
+    /** Override default provider used by the agent synthesizer (rubric rules
+     *  pick their own provider via `agent_binding.provider`). null = registry default. */
+    agentProvider: string | null;
+    timeoutMs: number;
+    baseUrl: string;
+    token: string;
+    businessDomain: string;
+    /**
+     * Output format(s). yaml is the source of truth (always re-derivable into
+     * markdown). When `--out` is a file path, `both` writes <stem>.yaml +
+     * <stem>.md side by side; `yaml` or `markdown` writes a single file at the
+     * given path. When `--out` is null (stdout), `both` collapses to yaml only —
+     * piping markdown to a downstream YAML consumer would silently corrupt it.
+     * Default: 'both' when out is a file, 'yaml' when stdout.
+     */
+    format?: 'yaml' | 'markdown' | 'both';
+    /**
+     * Output locale for agent-judged natural-language fields (rubric reasoning,
+     * synthesizer headline / description / fix_priority reason). Default 'en'.
+     * Affects only prose; JSON keys / enum values / span IDs always stay English.
+     */
+    lang?: 'en' | 'zh';
+}

package/dist/trace-ai/diagnose/types.js

@@ -0,0 +1 @@
+export {};

package/dist/trace-ai/eval-set/assertion-evaluator.d.ts

@@ -0,0 +1,29 @@
+import type { TraceSpan } from "../../api/conversations.js";
+import type { EvalAssertion, EvalReference } from "./types.js";
+export interface SemanticMatchVerdict {
+    verdict: "pass" | "fail";
+    reasoning: string;
+}
+export interface SemanticMatchProvider {
+    judgeSemanticMatch(question: string, candidateAnswer: string, referenceAnswer: string): Promise<SemanticMatchVerdict>;
+}
+export interface AssertionContext {
+    answer: string;
+    spans: TraceSpan[];
+    reference?: EvalReference;
+    durationMs?: number;
+    /**
+     * The user message that produced `answer`. Used as the default
+     * `{{question}}` for `semantic_match` when the assertion doesn't
+     * override it — case authors should not have to repeat user_message
+     * inside every semantic_match block.
+     */
+    question?: string;
+    semanticMatchProvider?: SemanticMatchProvider;
+}
+export interface AssertionResult {
+    verdict: "pass" | "fail" | "skip";
+    actual?: unknown;
+    reason?: string;
+}
+export declare function evaluateAssertion(assertion: EvalAssertion, ctx: AssertionContext): Promise<AssertionResult>;

package/dist/trace-ai/eval-set/assertion-evaluator.js

@@ -0,0 +1,100 @@
+function applyOp(actual, op, expected) {
+    switch (op) {
+        case "eq": return actual === expected;
+        case "lt": return actual < expected;
+        case "lte": return actual <= expected;
+        case "gt": return actual > expected;
+        case "gte": return actual >= expected;
+    }
+}
+function toolCallsFor(spans, toolName) {
+    return spans.filter((s) => s.kind === "tool" && s.attributes?.["gen_ai.tool.name"] === toolName);
+}
+function sortedToolNames(spans) {
+    return spans
+        .filter((s) => s.kind === "tool")
+        .slice()
+        .sort((a, b) => (a.startTime < b.startTime ? -1 : a.startTime > b.startTime ? 1 : 0))
+        .map((s) => String(s.attributes?.["gen_ai.tool.name"] ?? ""));
+}
+function isSubsequence(sequence, actual) {
+    let si = 0;
+    for (const name of actual) {
+        if (name === sequence[si])
+            si++;
+        if (si === sequence.length)
+            return true;
+    }
+    return false;
+}
+export async function evaluateAssertion(assertion, ctx) {
+    const { answer, spans, durationMs } = ctx;
+    const a = assertion;
+    switch (assertion.type) {
+        case "contains": {
+            const value = String(a["value"] ?? "");
+            return answer.includes(value)
+                ? { verdict: "pass" }
+                : { verdict: "fail", actual: answer };
+        }
+        case "not_contains": {
+            const value = String(a["value"] ?? "");
+            return answer.includes(value)
+                ? { verdict: "fail", actual: answer }
+                : { verdict: "pass" };
+        }
+        case "regex": {
+            const pattern = String(a["pattern"] ?? "");
+            let re;
+            try {
+                re = new RegExp(pattern);
+            }
+            catch {
+                return { verdict: "skip", reason: "invalid-regex: " + pattern };
+            }
+            return re.test(answer) ? { verdict: "pass" } : { verdict: "fail", actual: answer };
+        }
+        case "tool_call_count": {
+            const tool = String(a["tool"] ?? "");
+            const op = a["op"] ?? "eq";
+            const value = Number(a["value"] ?? 0);
+            const count = toolCallsFor(spans, tool).length;
+            return applyOp(count, op, value)
+                ? { verdict: "pass", actual: count }
+                : { verdict: "fail", actual: count };
+        }
+        case "tool_call_order": {
+            const sequence = Array.isArray(a["sequence"])
+                ? a["sequence"].map(String)
+                : [];
+            const actual = sortedToolNames(spans);
+            return isSubsequence(sequence, actual)
+                ? { verdict: "pass", actual }
+                : { verdict: "fail", actual };
+        }
+        case "latency_ms": {
+            if (durationMs === undefined || durationMs === null) {
+                return { verdict: "skip", reason: "durationMs not available" };
+            }
+            const op = a["op"] ?? "lte";
+            const value = Number(a["value"] ?? 0);
+            return applyOp(durationMs, op, value)
+                ? { verdict: "pass", actual: durationMs }
+                : { verdict: "fail", actual: durationMs };
+        }
+        case "semantic_match": {
+            const provider = ctx.semanticMatchProvider;
+            if (!provider) {
+                return { verdict: "skip", reason: "semantic_match requires a provider; pass semanticMatchProvider in context" };
+            }
+            if (!ctx.reference?.answer) {
+                return { verdict: "skip", reason: "semantic_match requires reference.answer on the eval case" };
+            }
+            const question = String(a["question"] ?? ctx.question ?? "");
+            const smv = await provider.judgeSemanticMatch(question, answer, ctx.reference.answer);
+            return { verdict: smv.verdict, actual: smv.reasoning };
+        }
+        default:
+            return { verdict: "skip", reason: `unknown assertion type: ${assertion.type}` };
+    }
+}

package/dist/trace-ai/eval-set/builder.d.ts

@@ -0,0 +1,36 @@
+/**
+ * M5 eval-set builder — orchestrates build:
+ *   picker → ensureQueryId → redact → write (with conflict resolution) → validate
+ *
+ * `ensureQueryId` is the deterministic hash-based ID generator (inline here,
+ * not a separate file — spec doc §9 "反过度工程" decision).
+ */
+import type { BuildResult } from "./types.js";
+import { type ConflictStrategy } from "./output-writer.js";
+export declare class BuilderError extends Error {
+    readonly cause?: Error | undefined;
+    constructor(message: string, cause?: Error | undefined);
+}
+export type BuildSource = {
+    kind: "diagnosis";
+    path: string;
+} | {
+    kind: "queries";
+    path: string;
+};
+export interface BuildOpts {
+    source: BuildSource;
+    outDir: string;
+    evalSetId: string;
+    onConflict: ConflictStrategy;
+    /** From `--redaction-rules=<path>` */
+    redactionRulesCliFlag: string | undefined;
+    /** From CWD: usually `path.join(process.cwd(), "redaction-rules")` — caller passes resolved path */
+    repoDir: string | undefined;
+}
+export declare function ensureQueryId(c: {
+    query_id: string;
+    input: unknown;
+    tags?: string[];
+}): string;
+export declare function build(opts: BuildOpts): Promise<BuildResult>;

package/dist/trace-ai/eval-set/builder.js

@@ -0,0 +1,126 @@
+/**
+ * M5 eval-set builder — orchestrates build:
+ *   picker → ensureQueryId → redact → write (with conflict resolution) → validate
+ *
+ * `ensureQueryId` is the deterministic hash-based ID generator (inline here,
+ * not a separate file — spec doc §9 "反过度工程" decision).
+ */
+import { createHash } from "node:crypto";
+import { liftFromQueriesFile, liftFromDiagnosis, QueryPickerError } from "./query-picker.js";
+import { loadRules, applyRules, RedactorError } from "./redactor.js";
+import { writeEvalSet, WriterError } from "./output-writer.js";
+export class BuilderError extends Error {
+    cause;
+    constructor(message, cause) {
+        super(message);
+        this.cause = cause;
+        this.name = "BuilderError";
+    }
+}
+/**
+ * Canonical JSON serialization for hashing — keys sorted, no whitespace.
+ * Ensures hash(case) is stable across runs.
+ */
+function canonicalJson(value) {
+    if (value === null || typeof value !== "object")
+        return JSON.stringify(value);
+    if (Array.isArray(value))
+        return "[" + value.map(canonicalJson).join(",") + "]";
+    const obj = value;
+    const keys = Object.keys(obj).sort();
+    return "{" + keys.map((k) => JSON.stringify(k) + ":" + canonicalJson(obj[k])).join(",") + "}";
+}
+export function ensureQueryId(c) {
+    if (c.query_id && c.query_id.length > 0)
+        return c.query_id;
+    const seed = canonicalJson({ input: c.input, tags: c.tags ?? [] });
+    return createHash("sha256").update(seed).digest("hex").slice(0, 12);
+}
+function redactCase(c, applyFn) {
+    const redacted = {
+        query_id: c.query_id,
+        input: { user_message: applyFn(c.input.user_message) },
+        tags: c.tags,
+    };
+    if (c.reference) {
+        redacted.reference = { answer: applyFn(c.reference.answer) };
+    }
+    if (c.assertions) {
+        redacted.assertions = c.assertions; // assertions strings (regex / value) intentionally NOT redacted
+        // — they are user-authored test expectations, not raw PII
+    }
+    return redacted;
+}
+export async function build(opts) {
+    // Stage 1: pick cases
+    let lifted;
+    let skippedFindingsCount = 0;
+    try {
+        if (opts.source.kind === "queries") {
+            lifted = await liftFromQueriesFile(opts.source.path);
+        }
+        else {
+            const r = await liftFromDiagnosis(opts.source.path);
+            lifted = r.cases;
+            skippedFindingsCount = r.skipped_findings_count;
+        }
+    }
+    catch (e) {
+        if (e instanceof QueryPickerError) {
+            throw new BuilderError(`picker failed: ${e.message}`, e);
+        }
+        throw e;
+    }
+    // Stage 2: ensure query_id
+    const withIds = lifted.map((c) => ({ ...c, query_id: ensureQueryId(c) }));
+    // Stage 3: redact
+    let rulesResult;
+    try {
+        rulesResult = await loadRules({
+            cliFlag: opts.redactionRulesCliFlag,
+            repoDir: opts.repoDir,
+        });
+    }
+    catch (e) {
+        if (e instanceof RedactorError) {
+            throw new BuilderError(`redactor failed: ${e.message}`, e);
+        }
+        throw e;
+    }
+    const apply = (s) => applyRules(s, rulesResult.rules);
+    const redacted = withIds.map((c) => redactCase(c, apply));
+    // Stage 3.5: guard against 0-cases lift (better UX than letting writer fail
+    // with cryptic "Too small: expected array to have >=1 items"). Common cause:
+    // --diagnosis= where every finding has query=null (e.g. runtime doesn't emit
+    // gen_ai.input.messages) or empty assertions[].
+    if (redacted.length === 0) {
+        const sourceLabel = opts.source.kind === "diagnosis" ? "--diagnosis=" : "--queries=";
+        const skippedNote = skippedFindingsCount > 0
+            ? `\n  Skipped ${skippedFindingsCount} finding(s) — common causes:\n    - findings have query: null (M4 trace runtime doesn't emit gen_ai.input.messages)\n    - findings have empty assertions[]`
+            : "";
+        throw new BuilderError(`lifted 0 eval-cases from ${sourceLabel}${opts.source.path}.${skippedNote}\n  Alternatives: use --queries=<file> to provide queries manually, or upgrade M4 trace runtime to emit gen_ai.input.messages.`);
+    }
+    // Stage 4: write + conflict + validate
+    let writeRes;
+    try {
+        writeRes = await writeEvalSet({
+            outDir: opts.outDir,
+            evalSetId: opts.evalSetId,
+            newCases: redacted,
+            onConflict: opts.onConflict,
+        });
+    }
+    catch (e) {
+        if (e instanceof WriterError) {
+            throw new BuilderError(`writer failed: ${e.message}`, e);
+        }
+        throw e;
+    }
+    return {
+        cases_written: writeRes.cases_written,
+        cases_skipped: writeRes.cases_skipped + skippedFindingsCount,
+        conflicts: writeRes.conflicts,
+        shard_paths: writeRes.shard_paths,
+        redaction_rules_source: rulesResult.source,
+    };
+}

package/dist/trace-ai/eval-set/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * M5 eval-set module — public exports.
+ *
+ * Consumers (commands/trace.ts, tests, future M6 reuse) import from this
+ * barrel; internal modules cross-import via direct paths.
+ */
+export type { EvalCase, EvalCaseInput, EvalReference, EvalAssertion, AssertionType, EvalSetIndex, EvalSetIndexShard, BuildResult, RedactionRule, } from "./types.js";
+export { build, ensureQueryId, BuilderError } from "./builder.js";
+export type { BuildOpts, BuildSource } from "./builder.js";
+export { run as runTest } from "./test-runner.js";
+export type { RunOpts, RunnerDeps } from "./test-runner.js";
+export { evaluateAssertion } from "./assertion-evaluator.js";
+export type { AssertionContext, AssertionResult, SemanticMatchProvider, SemanticMatchVerdict, } from "./assertion-evaluator.js";
+export { createBuiltinSemanticMatchProvider, ANSWER_MATCH_REFERENCE_REF, AnswerMatchOutputSchema, } from "./semantic-match-provider.js";
+export type { CreateSemanticMatchProviderOpts } from "./semantic-match-provider.js";

package/dist/trace-ai/eval-set/index.js

@@ -0,0 +1,10 @@
+/**
+ * M5 eval-set module — public exports.
+ *
+ * Consumers (commands/trace.ts, tests, future M6 reuse) import from this
+ * barrel; internal modules cross-import via direct paths.
+ */
+export { build, ensureQueryId, BuilderError } from "./builder.js";
+export { run as runTest } from "./test-runner.js";
+export { evaluateAssertion } from "./assertion-evaluator.js";
+export { createBuiltinSemanticMatchProvider, ANSWER_MATCH_REFERENCE_REF, AnswerMatchOutputSchema, } from "./semantic-match-provider.js";

package/dist/trace-ai/eval-set/output-writer.d.ts

@@ -0,0 +1,27 @@
+/**
+ * M5 eval-set output writer — handles directory layout, index upsert, shard
+ * merge, on-conflict resolution (fail / skip / overwrite), and .bak preservation.
+ *
+ * MVP layout: always one shard named `cases.yaml`. Users can manually split
+ * into multi-shard later (re-write `index.yaml` to reference more shards)
+ * and call `kweaver trace schema validate` to verify.
+ */
+import type { EvalCase } from "./types.js";
+export declare class WriterError extends Error {
+    readonly conflictIds?: string[] | undefined;
+    constructor(message: string, conflictIds?: string[] | undefined);
+}
+export type ConflictStrategy = "fail" | "skip" | "overwrite";
+export interface WriteEvalSetOpts {
+    outDir: string;
+    evalSetId: string;
+    newCases: EvalCase[];
+    onConflict: ConflictStrategy;
+}
+export interface WriteEvalSetResult {
+    cases_written: number;
+    cases_skipped: number;
+    conflicts: string[];
+    shard_paths: string[];
+}
+export declare function writeEvalSet(opts: WriteEvalSetOpts): Promise<WriteEvalSetResult>;