@kweaver-ai/kweaver-sdk 0.8.1 → 0.8.3

package/dist/trace-ai/diagnose/schemas.js

@@ -0,0 +1,154 @@
+import { z } from "zod";
+const TaxonomySchema = z.object({
+    signals_axis: z.enum(["interaction", "execution", "environment"]),
+    ms_class: z.enum([
+        "retry_loop",
+        "tool_misuse",
+        "context_loss",
+        "goal_drift",
+        "cascading_error",
+        "silent_quality_degradation",
+    ]),
+});
+const SuggestedFixSchema = z.object({
+    target: z.string().min(1),
+    change_template: z.string().min(1),
+});
+const VerifyWithSchema = z.object({
+    assertion_templates: z.array(z.string()).default([]),
+});
+/**
+ * Rubric input source descriptor. The supported source prefixes are
+ * resolved by `diagnose/agent-binding.ts` against the in-memory TraceTree:
+ *
+ *   - `extract_from_root_attr:<dot.path>`   → root span attribute by name
+ *   - `filter_by_kind:[kind1,kind2,...]`    → ordered span subset by kind
+ *   - `literal:<json>`                      → constant blob (debug / fixtures)
+ *
+ * Authors describe **which slice of the trace** the agent needs as context;
+ * the binding does the actual extraction so rule YAML stays declarative.
+ */
+const RubricInputSchema = z.object({
+    kind: z.string().min(1),
+    source: z.string().min(1),
+});
+/**
+ * Minimal JSON-Schema-ish shape we accept for rubric output_schema. We
+ * convert to a zod schema at load time (see `output-schema-converter.ts`);
+ * keeping this loose here lets authors paste literal JSON Schema without
+ * us re-implementing the whole spec — just the subset we need (object
+ * with required[] + properties{type,enum,items}).
+ */
+const RubricOutputSchemaSchema = z.object({
+    type: z.literal("object"),
+    required: z.array(z.string()).default([]),
+    properties: z.record(z.string(), z.record(z.string(), z.unknown())),
+});
+const AgentBindingSchema = z.object({
+    provider: z.string().min(1),
+    prompt_template_ref: z.string().regex(/^builtin:[a-zA-Z0-9_-]+$/),
+});
+const RubricSchema = z.object({
+    judge_question: z.string().min(1),
+    inputs: z.array(RubricInputSchema).default([]),
+    output_schema: RubricOutputSchemaSchema,
+    agent_binding: AgentBindingSchema,
+    /**
+     * Optional symbolic rule_ids that act as gate for this rubric in batch mode.
+     * Empty/missing → rubric runs on all traces (PR-B fallback). In single-trace
+     * mode this field is ignored; rubric always runs.
+     */
+    gates_on: z.array(z.string()).optional(),
+});
+/**
+ * The convergence contract between Stage-1 (symbolic) and Stage-2 (rubric):
+ * every rubric verdict MUST emit `first_violating_step_id` so cross-finding
+ * links can correlate rubric findings with the spans symbolic rules cite.
+ *
+ * Enforced as a YAML-load-time check rather than at runtime so authors
+ * see the violation in `trace diagnose rules validate <path>`.
+ */
+const FIRST_VIOLATING_STEP_ID = "first_violating_step_id";
+export const RuleSchema = z
+    .object({
+    schema_version: z.literal("diagnosis-rule/v1"),
+    id: z.string().regex(/^[a-z][a-z0-9_]*$/),
+    severity: z.enum(["low", "medium", "high"]),
+    symptom: z.string().min(1),
+    taxonomy: TaxonomySchema,
+    suggested_fix: SuggestedFixSchema,
+    verify_with: VerifyWithSchema,
+    predicate: z.string().regex(/^builtin:[a-z][a-z0-9_]*$/).optional(),
+    rubric: RubricSchema.optional(),
+    params: z.record(z.string(), z.unknown()).default({}),
+})
+    .refine((r) => Boolean(r.predicate) !== Boolean(r.rubric), { message: "exactly one of `predicate` or `rubric` must be present" })
+    .refine((r) => !r.rubric || r.rubric.output_schema.required.includes(FIRST_VIOLATING_STEP_ID), {
+    message: `rubric.output_schema.required must include '${FIRST_VIOLATING_STEP_ID}' (Stage-1↔Stage-2 convergence contract)`,
+    path: ["rubric", "output_schema", "required"],
+});
+const FindingSchema = z.object({
+    rule_id: z.string(),
+    judgment_kind: z.enum(["symbolic", "rubric"]),
+    severity: z.enum(["low", "medium", "high"]),
+    symptom: z.string(),
+    likely_cause: z.string(),
+    evidence: z.object({
+        spans: z.array(z.string()),
+        excerpt: z.string(),
+    }),
+    suggested_fix: z.object({
+        target: z.string(),
+        change: z.string(),
+    }),
+    // Symbolic findings always emit `low` (no semantic basis for higher).
+    // Rubric agent supplies its own confidence; rule-loader propagates the
+    // value the agent returned in the rubric output. Accept the union.
+    confidence: z.enum(["low", "medium", "high"]),
+    verify_with: z.object({
+        suggested_eval_case: z.object({
+            query_id: z.string().nullable(),
+            query: z.string().nullable(),
+            assertions: z.array(z.string()),
+        }),
+    }),
+});
+const SummarySchema = z.object({
+    headline: z.string().max(160),
+    primary_root_cause: z
+        .object({
+        finding_ids: z.array(z.number().int().nonnegative()).min(1),
+        description: z.string(),
+        target_for_fix: z.string(),
+    })
+        .nullable(),
+    fix_priority: z.array(z.object({
+        finding_id: z.number().int().nonnegative(),
+        reason: z.string(),
+    })),
+    cross_finding_links: z.array(z.object({
+        finding_ids: z.array(z.number().int().nonnegative()).min(2),
+        relation: z.string(),
+    })),
+});
+export const ReportSchema = z.object({
+    schema_version: z.literal("trace-diagnose-report/v1"),
+    trace: z.object({
+        trace_id: z.string(),
+        agent_id: z.string().nullable(),
+        tenant: z.string().nullable(),
+    }),
+    run: z.object({
+        diagnosed_at: z.string(),
+        cli_version: z.string(),
+        mode: z.enum(["symbolic-only", "rubric-only", "hybrid"]),
+        rules_applied: z.array(z.string()),
+        rules_skipped: z.array(z.object({ rule_id: z.string(), reason: z.string() })),
+        synthesizer_mode: z.enum(["template", "agent"]),
+    }),
+    summary: SummarySchema,
+    findings: z.array(FindingSchema),
+});
+/** The Summary section in isolation — exported so the agent synthesizer
+ *  can validate its LLM output against the same shape the report uses. */
+export const SummaryOutputSchema = SummarySchema;

package/dist/trace-ai/diagnose/signal-probe.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Stage-1 (symbolic) runner. Rubric rules are handled separately in
+ * `agent-binding.ts` and merged into the findings list by `index.ts`.
+ *
+ * Rationale for keeping the split here: symbolic predicates are cheap,
+ * deterministic, sync; rubric judgments are slow, non-deterministic,
+ * async. Running them in one loop would entangle backpressure,
+ * timeout, and retry concerns that only apply to one of the two paths.
+ */
+import type { Hit, Rule, TraceTree } from "./types.js";
+export declare class RuleProbeError extends Error {
+    constructor(ruleId: string, cause: Error);
+}
+export declare function runRules(rules: Rule[], tree: TraceTree): Promise<Map<string, Hit[]>>;
+/** Helpers that split a rule list by which stage owns them. */
+export declare function symbolicRules(rules: Rule[]): Rule[];
+export declare function rubricRules(rules: Rule[]): Rule[];

package/dist/trace-ai/diagnose/signal-probe.js

@@ -0,0 +1,39 @@
+/**
+ * Stage-1 (symbolic) runner. Rubric rules are handled separately in
+ * `agent-binding.ts` and merged into the findings list by `index.ts`.
+ *
+ * Rationale for keeping the split here: symbolic predicates are cheap,
+ * deterministic, sync; rubric judgments are slow, non-deterministic,
+ * async. Running them in one loop would entangle backpressure,
+ * timeout, and retry concerns that only apply to one of the two paths.
+ */
+import { resolvePredicate } from "./predicate-registry.js";
+export class RuleProbeError extends Error {
+    constructor(ruleId, cause) {
+        super(`predicate failed for rule '${ruleId}': ${cause.message}`);
+        this.name = "RuleProbeError";
+    }
+}
+export async function runRules(rules, tree) {
+    const out = new Map();
+    for (const rule of rules) {
+        if (!rule.predicateRef)
+            continue; // rubric rule — handled by agent-binding
+        const fn = resolvePredicate(rule.predicateRef);
+        try {
+            const hits = fn(tree, rule.params);
+            out.set(rule.id, hits);
+        }
+        catch (e) {
+            throw new RuleProbeError(rule.id, e);
+        }
+    }
+    return out;
+}
+/** Helpers that split a rule list by which stage owns them. */
+export function symbolicRules(rules) {
+    return rules.filter((r) => r.predicateRef !== null);
+}
+export function rubricRules(rules) {
+    return rules.filter((r) => r.rubric !== null);
+}

package/dist/trace-ai/diagnose/synthesizer-agent.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Stage-3 — agent-driven within-trace synthesizer.
+ *
+ * Takes the N findings produced by Stages 1+2 and asks the LLM to compose
+ * a `Summary` (headline + root cause + ordered fix priority + cross-finding
+ * links). Falls back to the deterministic `templateSynthesize` if:
+ *   - findings.length === 0 (no narrative needed)
+ *   - no provider registered / provider unavailable
+ *   - the agent invocation fails for any reason (we still want a usable
+ *     report even when the LLM judge times out)
+ *
+ * The agent path remains a *narrative* layer — symbolic and rubric findings
+ * are already in hand; the synthesizer doesn't fabricate new findings, only
+ * organizes the ones it was given. This keeps the contract small and the
+ * failure modes containable.
+ */
+import type { Finding, Summary } from "./types.js";
+import type { AgentProvider } from "../../agent-providers/types.js";
+import { PromptTemplateRegistry, type AgentOutputLang } from "../../agent-providers/prompt-template.js";
+import type { ArtifactWriter } from "../scan/artifacts/writer.js";
+export interface AgentSynthesizeOpts {
+    findings: Finding[];
+    traceId: string;
+    agentId: string | null;
+    provider: AgentProvider | null;
+    promptRegistry: PromptTemplateRegistry;
+    promptRef?: string;
+    timeoutMs?: number;
+    /** Output locale for synthesizer prose. Default 'en'. */
+    lang?: AgentOutputLang;
+    /** When provided, writes Stage-3 prompt/response artifacts. */
+    artifacts?: ArtifactWriter;
+}
+export interface AgentSynthesizeResult {
+    summary: Summary;
+    mode: "agent" | "template";
+    /** Reason set when mode === 'template' under a non-default branch. */
+    fallbackReason?: string;
+}
+export declare function agentSynthesize(opts: AgentSynthesizeOpts): Promise<AgentSynthesizeResult>;

package/dist/trace-ai/diagnose/synthesizer-agent.js

@@ -0,0 +1,158 @@
+/**
+ * Stage-3 — agent-driven within-trace synthesizer.
+ *
+ * Takes the N findings produced by Stages 1+2 and asks the LLM to compose
+ * a `Summary` (headline + root cause + ordered fix priority + cross-finding
+ * links). Falls back to the deterministic `templateSynthesize` if:
+ *   - findings.length === 0 (no narrative needed)
+ *   - no provider registered / provider unavailable
+ *   - the agent invocation fails for any reason (we still want a usable
+ *     report even when the LLM judge times out)
+ *
+ * The agent path remains a *narrative* layer — symbolic and rubric findings
+ * are already in hand; the synthesizer doesn't fabricate new findings, only
+ * organizes the ones it was given. This keeps the contract small and the
+ * failure modes containable.
+ */
+import { AgentProviderError } from "../../agent-providers/types.js";
+import { render as renderPrompt, languageInstructionFor, } from "../../agent-providers/prompt-template.js";
+import { SummaryOutputSchema } from "./schemas.js";
+import { templateSynthesize } from "./synthesizer-template.js";
+/** Map zod-validated agent output (snake_case Summary) → internal camelCase Summary. */
+function toInternalSummary(out) {
+    return {
+        headline: out.headline,
+        primaryRootCause: out.primary_root_cause === null
+            ? null
+            : {
+                findingIds: out.primary_root_cause.finding_ids,
+                description: out.primary_root_cause.description,
+                targetForFix: out.primary_root_cause.target_for_fix,
+            },
+        fixPriority: out.fix_priority.map((p) => ({ findingId: p.finding_id, reason: p.reason })),
+        crossFindingLinks: out.cross_finding_links.map((l) => ({
+            findingIds: l.finding_ids,
+            relation: l.relation,
+        })),
+    };
+}
+/** Snake-case projection of Finding for the prompt — matches the YAML
+ *  representation users already see, so the model doesn't have to translate. */
+function findingForPrompt(f, idx) {
+    return {
+        index: idx,
+        rule_id: f.ruleId,
+        judgment_kind: f.judgmentKind,
+        severity: f.severity,
+        symptom: f.symptom,
+        likely_cause: f.likelyCause,
+        evidence_spans: f.evidence.spans,
+        excerpt: f.evidence.excerpt,
+        suggested_fix_target: f.suggestedFix.target,
+        suggested_fix_change: f.suggestedFix.change,
+        confidence: f.confidence,
+    };
+}
+/**
+ * The output_schema rendered into the prompt is a JSON-Schema-ish
+ * description of the Summary contract. Authored inline rather than
+ * derived from the zod schema to keep the prompt human-readable.
+ */
+const SUMMARY_OUTPUT_SCHEMA_DESCRIPTION = {
+    type: "object",
+    required: ["headline", "primary_root_cause", "fix_priority", "cross_finding_links"],
+    properties: {
+        headline: { type: "string", maxLength: 160 },
+        primary_root_cause: {
+            type: "object_or_null",
+            required: ["finding_ids", "description", "target_for_fix"],
+            properties: {
+                finding_ids: { type: "array", items: { type: "integer" } },
+                description: { type: "string" },
+                target_for_fix: { type: "string" },
+            },
+        },
+        fix_priority: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["finding_id", "reason"],
+                properties: { finding_id: { type: "integer" }, reason: { type: "string" } },
+            },
+        },
+        cross_finding_links: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["finding_ids", "relation"],
+                properties: {
+                    finding_ids: { type: "array", items: { type: "integer" }, minItems: 2 },
+                    relation: { type: "string" },
+                },
+            },
+        },
+    },
+};
+export async function agentSynthesize(opts) {
+    // Empty findings: no narrative to compose. Both modes produce the same
+    // summary here, so default to `template` so reports don't claim the
+    // agent ran when it didn't.
+    if (opts.findings.length === 0) {
+        return { summary: templateSynthesize([]), mode: "template" };
+    }
+    if (!opts.provider) {
+        return {
+            summary: templateSynthesize(opts.findings),
+            mode: "template",
+            fallbackReason: "no-provider-registered",
+        };
+    }
+    const ref = opts.promptRef ?? "builtin:within-trace-synthesizer-v1";
+    if (!opts.promptRegistry.has(ref)) {
+        return {
+            summary: templateSynthesize(opts.findings),
+            mode: "template",
+            fallbackReason: `prompt-template-missing:${ref}`,
+        };
+    }
+    const tpl = opts.promptRegistry.get(ref);
+    const prompt = renderPrompt(tpl, {
+        trace_id: opts.traceId,
+        agent_id: opts.agentId ?? "<unknown>",
+        findings: opts.findings.map((f, i) => findingForPrompt(f, i)),
+        output_schema: SUMMARY_OUTPUT_SCHEMA_DESCRIPTION,
+        language_instruction: languageInstructionFor(opts.lang ?? "en"),
+    });
+    try {
+        if (!(await opts.provider.isAvailable())) {
+            return {
+                summary: templateSynthesize(opts.findings),
+                mode: "template",
+                fallbackReason: `provider-not-available:${opts.provider.name}`,
+            };
+        }
+        if (opts.artifacts) {
+            await opts.artifacts.writeStageThreeSynthPrompt(prompt);
+        }
+        const resp = await opts.provider.invoke({
+            prompt,
+            outputSchema: SummaryOutputSchema,
+            timeoutMs: opts.timeoutMs,
+            correlationId: `synthesize:${opts.traceId}`,
+        });
+        if (opts.artifacts) {
+            await opts.artifacts.writeStageThreeSynthResponse(resp.output);
+        }
+        return { summary: toInternalSummary(resp.output), mode: "agent" };
+    }
+    catch (e) {
+        if (e instanceof AgentProviderError) {
+            return {
+                summary: templateSynthesize(opts.findings),
+                mode: "template",
+                fallbackReason: `agent-error:${e.kind}`,
+            };
+        }
+        throw e;
+    }
+}

package/dist/{trace-core → trace-ai}/diagnose/trace-shaper.js

@@ -52,6 +52,7 @@ export function assembleTraceTree(traceId, raw) {
             durationMs: durationMs(r.startTimeUnixNano, r.endTimeUnixNano),
             status: deriveStatus(r.status),
             attributes: attrs,
+            events: r.events,
         };
     });
     const byId = new Map();

package/dist/{trace-core → trace-ai}/diagnose/types.d.ts

@@ -11,6 +11,11 @@ export interface Span {
     durationMs: number;
     status: 'ok' | 'error' | 'unset';
     attributes: SpanAttributes;
+    events?: Array<{
+        name?: string;
+        time?: string;
+        attributes?: Record<string, unknown>;
+    }>;
 }
 export type SpanKind = 'tool' | 'llm' | 'retrieval' | 'reasoning' | 'unknown';
 export interface TraceTree {
@@ -25,6 +30,24 @@ export interface RuleTaxonomy {
     signalsAxis: 'interaction' | 'execution' | 'environment';
     msClass: 'retry_loop' | 'tool_misuse' | 'context_loss' | 'goal_drift' | 'cascading_error' | 'silent_quality_degradation';
 }
+export interface RubricInputSpec {
+    kind: string;
+    source: string;
+}
+export interface RubricSpec {
+    judgeQuestion: string;
+    inputs: RubricInputSpec[];
+    /** Original JSON-Schema-ish blob (kept for YAML round-trips / debug). */
+    outputSchemaRaw: Record<string, unknown>;
+    /** Compiled zod schema (built once at load time via output-schema-converter). */
+    outputZodSchema: import("zod").ZodTypeAny;
+    agentBinding: {
+        provider: string;
+        promptTemplateRef: string;
+    };
+    /** Optional gating; see RuleSchema.rubric.gates_on. */
+    gatesOn?: string[];
+}
 export interface Rule {
     schemaVersion: 'diagnosis-rule/v1';
     id: string;
@@ -38,10 +61,13 @@ export interface Rule {
     verifyWith: {
         assertionTemplates: string[];
     };
-    predicateRef: string;
+    /** Exactly one of `predicateRef` or `rubric` is non-null (XOR enforced at load). */
+    predicateRef: string | null;
+    rubric: RubricSpec | null;
     params: Record<string, unknown>;
     sourcePath: string;
 }
+export type JudgmentKind = 'symbolic' | 'rubric';
 export interface Hit {
     evidenceSpans: string[];
     excerpt: string;
@@ -50,7 +76,7 @@ export interface Hit {
 export type Predicate = (trace: TraceTree, params: Record<string, unknown>) => Hit[];
 export interface Finding {
     ruleId: string;
-    judgmentKind: 'symbolic';
+    judgmentKind: JudgmentKind;
     severity: 'low' | 'medium' | 'high';
     symptom: string;
     likelyCause: string;
@@ -62,7 +88,8 @@ export interface Finding {
         target: string;
         change: string;
     };
-    confidence: 'low';
+    /** Symbolic always 'low' (no semantic basis); rubric carries agent confidence. */
+    confidence: 'low' | 'medium' | 'high';
     verifyWith: {
         suggestedEvalCase: {
             queryId: string | null;
@@ -100,13 +127,13 @@ export interface Report {
     run: {
         diagnosedAt: string;
         cliVersion: string;
-        mode: 'symbolic-only';
+        mode: 'symbolic-only' | 'rubric-only' | 'hybrid';
         rulesApplied: string[];
         rulesSkipped: {
             ruleId: string;
             reason: string;
         }[];
-        synthesizerMode: 'template';
+        synthesizerMode: 'template' | 'agent';
     };
     summary: Summary;
     findings: Finding[];
@@ -115,10 +142,32 @@ export interface DiagnoseOpts {
     out: string | null;
     rulesDir: string | null;
     noBuiltin: boolean;
-    noLlm: true;
+    /** PR-B: when true, skip rubric rules (warn + record in rules_skipped) AND
+     *  fall the synthesizer back from agent → template. Default is now false
+     *  (both pillars on). */
+    noLlm: boolean;
+    /** Skip artifact persistence. Default false (artifacts ARE written). */
+    noArtifacts?: boolean;
+    /** Override default provider used by the agent synthesizer (rubric rules
+     *  pick their own provider via `agent_binding.provider`). null = registry default. */
     agentProvider: string | null;
     timeoutMs: number;
     baseUrl: string;
     token: string;
     businessDomain: string;
+    /**
+     * Output format(s). yaml is the source of truth (always re-derivable into
+     * markdown). When `--out` is a file path, `both` writes <stem>.yaml +
+     * <stem>.md side by side; `yaml` or `markdown` writes a single file at the
+     * given path. When `--out` is null (stdout), `both` collapses to yaml only —
+     * piping markdown to a downstream YAML consumer would silently corrupt it.
+     * Default: 'both' when out is a file, 'yaml' when stdout.
+     */
+    format?: 'yaml' | 'markdown' | 'both';
+    /**
+     * Output locale for agent-judged natural-language fields (rubric reasoning,
+     * synthesizer headline / description / fix_priority reason). Default 'en'.
+     * Affects only prose; JSON keys / enum values / span IDs always stay English.
+     */
+    lang?: 'en' | 'zh';
 }

package/dist/trace-ai/eval-set/assertion-evaluator.d.ts

@@ -0,0 +1,29 @@
+import type { TraceSpan } from "../../api/conversations.js";
+import type { EvalAssertion, EvalReference } from "./types.js";
+export interface SemanticMatchVerdict {
+    verdict: "pass" | "fail";
+    reasoning: string;
+}
+export interface SemanticMatchProvider {
+    judgeSemanticMatch(question: string, candidateAnswer: string, referenceAnswer: string): Promise<SemanticMatchVerdict>;
+}
+export interface AssertionContext {
+    answer: string;
+    spans: TraceSpan[];
+    reference?: EvalReference;
+    durationMs?: number;
+    /**
+     * The user message that produced `answer`. Used as the default
+     * `{{question}}` for `semantic_match` when the assertion doesn't
+     * override it — case authors should not have to repeat user_message
+     * inside every semantic_match block.
+     */
+    question?: string;
+    semanticMatchProvider?: SemanticMatchProvider;
+}
+export interface AssertionResult {
+    verdict: "pass" | "fail" | "skip";
+    actual?: unknown;
+    reason?: string;
+}
+export declare function evaluateAssertion(assertion: EvalAssertion, ctx: AssertionContext): Promise<AssertionResult>;

package/dist/trace-ai/eval-set/assertion-evaluator.js

@@ -0,0 +1,100 @@
+function applyOp(actual, op, expected) {
+    switch (op) {
+        case "eq": return actual === expected;
+        case "lt": return actual < expected;
+        case "lte": return actual <= expected;
+        case "gt": return actual > expected;
+        case "gte": return actual >= expected;
+    }
+}
+function toolCallsFor(spans, toolName) {
+    return spans.filter((s) => s.kind === "tool" && s.attributes?.["gen_ai.tool.name"] === toolName);
+}
+function sortedToolNames(spans) {
+    return spans
+        .filter((s) => s.kind === "tool")
+        .slice()
+        .sort((a, b) => (a.startTime < b.startTime ? -1 : a.startTime > b.startTime ? 1 : 0))
+        .map((s) => String(s.attributes?.["gen_ai.tool.name"] ?? ""));
+}
+function isSubsequence(sequence, actual) {
+    let si = 0;
+    for (const name of actual) {
+        if (name === sequence[si])
+            si++;
+        if (si === sequence.length)
+            return true;
+    }
+    return false;
+}
+export async function evaluateAssertion(assertion, ctx) {
+    const { answer, spans, durationMs } = ctx;
+    const a = assertion;
+    switch (assertion.type) {
+        case "contains": {
+            const value = String(a["value"] ?? "");
+            return answer.includes(value)
+                ? { verdict: "pass" }
+                : { verdict: "fail", actual: answer };
+        }
+        case "not_contains": {
+            const value = String(a["value"] ?? "");
+            return answer.includes(value)
+                ? { verdict: "fail", actual: answer }
+                : { verdict: "pass" };
+        }
+        case "regex": {
+            const pattern = String(a["pattern"] ?? "");
+            let re;
+            try {
+                re = new RegExp(pattern);
+            }
+            catch {
+                return { verdict: "skip", reason: "invalid-regex: " + pattern };
+            }
+            return re.test(answer) ? { verdict: "pass" } : { verdict: "fail", actual: answer };
+        }
+        case "tool_call_count": {
+            const tool = String(a["tool"] ?? "");
+            const op = a["op"] ?? "eq";
+            const value = Number(a["value"] ?? 0);
+            const count = toolCallsFor(spans, tool).length;
+            return applyOp(count, op, value)
+                ? { verdict: "pass", actual: count }
+                : { verdict: "fail", actual: count };
+        }
+        case "tool_call_order": {
+            const sequence = Array.isArray(a["sequence"])
+                ? a["sequence"].map(String)
+                : [];
+            const actual = sortedToolNames(spans);
+            return isSubsequence(sequence, actual)
+                ? { verdict: "pass", actual }
+                : { verdict: "fail", actual };
+        }
+        case "latency_ms": {
+            if (durationMs === undefined || durationMs === null) {
+                return { verdict: "skip", reason: "durationMs not available" };
+            }
+            const op = a["op"] ?? "lte";
+            const value = Number(a["value"] ?? 0);
+            return applyOp(durationMs, op, value)
+                ? { verdict: "pass", actual: durationMs }
+                : { verdict: "fail", actual: durationMs };
+        }
+        case "semantic_match": {
+            const provider = ctx.semanticMatchProvider;
+            if (!provider) {
+                return { verdict: "skip", reason: "semantic_match requires a provider; pass semanticMatchProvider in context" };
+            }
+            if (!ctx.reference?.answer) {
+                return { verdict: "skip", reason: "semantic_match requires reference.answer on the eval case" };
+            }
+            const question = String(a["question"] ?? ctx.question ?? "");
+            const smv = await provider.judgeSemanticMatch(question, answer, ctx.reference.answer);
+            return { verdict: smv.verdict, actual: smv.reasoning };
+        }
+        default:
+            return { verdict: "skip", reason: `unknown assertion type: ${assertion.type}` };
+    }
+}