@kweaver-ai/kweaver-sdk 0.8.1 → 0.8.2

package/dist/trace-ai/diagnose/index.js

@@ -0,0 +1,246 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import yaml from "js-yaml";
+import { fileURLToPath } from "node:url";
+import { getSpansByConversationId } from "../../api/trace.js";
+import { assembleTraceTree } from "./trace-shaper.js";
+import { loadRules, RuleLoadError } from "./rule-loader.js";
+import { runRules, RuleProbeError, rubricRules } from "./signal-probe.js";
+import { agentSynthesize } from "./synthesizer-agent.js";
+import { evaluateRubricRules } from "./agent-binding.js";
+import { assembleReport, reportToYamlObject, symbolicHitsToFindings } from "./report-assembler.js";
+import { renderReportMarkdown } from "./report-markdown.js";
+import { defaultRegistry } from "../../agent-providers/registry.js";
+import { defaultPromptRegistry, } from "../../agent-providers/prompt-template.js";
+import { ArtifactWriter } from "../scan/artifacts/writer.js";
+import { resolveArtifactsBase } from "../scan/artifacts/paths.js";
+import { extractUserQueryFromTrace } from "./query-extractor.js";
+import "./builtin-rules/register.js"; // side effect: registers all builtin predicates
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const BUILTIN_DIR = path.join(__dirname, "builtin-rules");
+// Prompts moved to top-level agent-providers/ when the trace-core/ container
+// was split (refactor 2026-05-12). diagnose/ now sits two levels under src/,
+// so we go up two and across.
+const SHARED_PROMPT_DIR = path.join(__dirname, "..", "..", "agent-providers", "prompts");
+export class TraceNotFoundError extends Error {
+    constructor(conversationId) {
+        super(`no spans found for conversation: ${conversationId}`);
+        this.name = "TraceNotFoundError";
+    }
+}
+let sharedPromptsLoaded = false;
+async function ensureBuiltinPromptsLoaded(reg) {
+    if (reg !== defaultPromptRegistry) {
+        // Caller-provided registry: load on every call so test-specific
+        // overrides see their content (cheap; ENOENT is no-op).
+        await reg.loadBuiltinDir(SHARED_PROMPT_DIR);
+        return;
+    }
+    if (sharedPromptsLoaded)
+        return;
+    await reg.loadBuiltinDir(SHARED_PROMPT_DIR);
+    sharedPromptsLoaded = true;
+}
+export async function diagnose(conversationId, opts, internal = {}) {
+    const t_start = Date.now();
+    const cwdRulesDir = opts.rulesDir ?? path.join(process.cwd(), "diagnosis-rules");
+    const registry = internal.registry ?? defaultRegistry;
+    const promptRegistry = internal.promptRegistry ?? defaultPromptRegistry;
+    await ensureBuiltinPromptsLoaded(promptRegistry);
+    // ── Artifact writer setup ────────────────────────────────────────────────
+    const artifactsEnabled = !(opts.noArtifacts ?? false) && opts.out !== null;
+    const artifactsBase = artifactsEnabled
+        ? resolveArtifactsBase({ mode: "single", out: opts.out })
+        : "";
+    const artifacts = new ArtifactWriter({ base: artifactsBase, enabled: artifactsEnabled });
+    // ── 1. Fetch + shape spans ──────────────────────────────────────────────
+    const fetched = await getSpansByConversationId({
+        baseUrl: opts.baseUrl,
+        token: opts.token,
+        businessDomain: opts.businessDomain,
+        conversationId,
+    });
+    const rawSpans = fetched.spans;
+    if (rawSpans.length === 0)
+        throw new TraceNotFoundError(conversationId);
+    const observedTraceIds = fetched.traceIds.length > 0
+        ? fetched.traceIds
+        : [...new Set(rawSpans.map((s) => s.traceId).filter((t) => Boolean(t)))];
+    const primaryTraceId = observedTraceIds[0] ?? conversationId;
+    if (observedTraceIds.length > 1) {
+        process.stderr.write(`warning: conversation ${conversationId} has ${observedTraceIds.length} traces; diagnosing the first (${primaryTraceId})\n`);
+    }
+    const spansForPrimary = observedTraceIds.length > 0
+        ? rawSpans.filter((s) => !s.traceId || s.traceId === primaryTraceId)
+        : rawSpans;
+    const tree = assembleTraceTree(primaryTraceId, spansForPrimary);
+    // ── 1b. Extract user query for suggested_eval_case population ───────────
+    const userQuery = extractUserQueryFromTrace(tree);
+    const queryId = conversationId;
+    // ── 2. Load rules + run Stage-1 (symbolic) ──────────────────────────────
+    const rules = await loadRules({
+        builtinDir: BUILTIN_DIR,
+        cwdRulesDir,
+        extraRulesDir: null,
+        noBuiltin: opts.noBuiltin,
+    });
+    const hits = await runRules(rules, tree);
+    const symbolicFindings = symbolicHitsToFindings(rules, hits, userQuery, queryId);
+    // ── 3. Stage-2 (rubric) — skip everything when --no-llm ─────────────────
+    const haveRubric = rubricRules(rules).length > 0;
+    let rubricFindings = [];
+    let rulesSkipped = [];
+    if (haveRubric) {
+        const r = await evaluateRubricRules({
+            rules,
+            tree,
+            registry,
+            promptRegistry,
+            noLlm: opts.noLlm,
+            timeoutMs: opts.timeoutMs,
+            lang: opts.lang,
+            artifacts,
+            userQuery,
+            queryId,
+        });
+        rubricFindings = r.findings;
+        rulesSkipped = r.skipped;
+    }
+    const allFindings = [...symbolicFindings, ...rubricFindings];
+    // ── 4. Stage-3 — agent synthesizer (template fallback) ──────────────────
+    const synthProvider = opts.noLlm
+        ? null
+        : registry.resolve({ preferred: opts.agentProvider ?? undefined });
+    const synth = await agentSynthesize({
+        findings: allFindings,
+        traceId: primaryTraceId,
+        agentId: extractAgentId(tree),
+        provider: synthProvider,
+        promptRegistry,
+        timeoutMs: opts.timeoutMs,
+        lang: opts.lang,
+        artifacts,
+    });
+    // ── 5. Assemble report ──────────────────────────────────────────────────
+    const haveSymbolic = rules.some((r) => r.predicateRef !== null);
+    const ranRubric = haveRubric && !opts.noLlm;
+    const mode = haveSymbolic && ranRubric
+        ? "hybrid"
+        : ranRubric
+            ? "rubric-only"
+            : "symbolic-only";
+    const version = await cliVersion();
+    const report = assembleReport({
+        traceId: primaryTraceId,
+        agentId: extractAgentId(tree),
+        tenant: extractTenant(tree),
+        cliVersion: version,
+        rules,
+        hits,
+        extraFindings: rubricFindings,
+        summary: synth.summary,
+        mode,
+        rulesSkipped,
+        synthesizerMode: synth.mode,
+        userQuery,
+        queryId,
+    });
+    // ── 6. Write run-metadata artifact ─────────────────────────────────────
+    const t_total = Date.now() - t_start;
+    await artifacts.writeRunMetadata({
+        cli_args: { conv_id: conversationId, out: opts.out, lang: opts.lang ?? "en" },
+        agent_id: extractAgentId(tree) ?? "",
+        rule_load_summary: {
+            rules_applied: rules.map((r) => r.id),
+            rules_skipped_at_load: [],
+            rules_dir: opts.rulesDir ?? "builtin",
+        },
+        single_agent_validation: { checked_conv_ids: 1, agent_id_resolved: extractAgentId(tree) ?? "" },
+        timing: { stage_1_ms: 0, stage_2_ms: 0, stage_3_ms: 0, stage_4_ms: 0, total_ms: t_total },
+        llm_calls: {
+            stage_2_chunks: rubricFindings.length > 0 ? 1 : 0,
+            stage_3: synth.mode === "agent" ? 1 : 0,
+            stage_4: 0,
+            total: (rubricFindings.length > 0 ? 1 : 0) + (synth.mode === "agent" ? 1 : 0),
+        },
+        cost_estimate_usd: { stage_2: 0, stage_4: 0, total: 0, model_price_table_version: "2026-05" },
+    });
+    // ── 7. Emit ──────────────────────────────────────────────────────────────
+    const yamlText = yaml.dump(reportToYamlObject(report));
+    // Markdown renderer also receives the conversation_id + business_domain so
+    // the "How to verify" section can emit runnable CLI commands. These two
+    // values are NOT in the yaml schema (yaml stays CLI-agnostic) — they live
+    // only in the md projection.
+    const mdOpts = { conversationId, businessDomain: opts.businessDomain };
+    const format = opts.format ?? (opts.out !== null ? "both" : "yaml");
+    if (opts.out !== null) {
+        await fs.mkdir(path.dirname(opts.out), { recursive: true });
+        const { yamlPath, mdPath } = derivePaths(opts.out, format);
+        if (yamlPath !== null)
+            await fs.writeFile(yamlPath, yamlText, "utf8");
+        if (mdPath !== null)
+            await fs.writeFile(mdPath, renderReportMarkdown(report, mdOpts), "utf8");
+    }
+    else {
+        // stdout — markdown to stdout would corrupt downstream `yq` / yaml consumers, so
+        // 'both' degrades to yaml-only. Users who want md on stdout pass --format=markdown.
+        if (format === "markdown") {
+            process.stdout.write(renderReportMarkdown(report, mdOpts));
+        }
+        else {
+            process.stdout.write(yamlText);
+        }
+    }
+    if (report.findings.length === 0) {
+        process.stderr.write("no findings\n");
+    }
+    return report;
+}
+/** Resolve which file paths to write given the user-supplied --out and format.
+ *  Both: derive the missing extension from the given one; if --out had no
+ *  recognized extension, append .yaml / .md. Single-format: write to --out
+ *  verbatim (caller's extension is honored as-is). */
+export function derivePaths(out, format) {
+    if (format === "yaml")
+        return { yamlPath: out, mdPath: null };
+    if (format === "markdown")
+        return { yamlPath: null, mdPath: out };
+    // both
+    const lower = out.toLowerCase();
+    if (lower.endsWith(".yaml") || lower.endsWith(".yml")) {
+        const stem = out.slice(0, out.lastIndexOf("."));
+        return { yamlPath: out, mdPath: `${stem}.md` };
+    }
+    if (lower.endsWith(".md") || lower.endsWith(".markdown")) {
+        const stem = out.slice(0, out.lastIndexOf("."));
+        return { yamlPath: `${stem}.yaml`, mdPath: out };
+    }
+    return { yamlPath: `${out}.yaml`, mdPath: `${out}.md` };
+}
+function extractAgentId(tree) {
+    for (const s of tree.spans) {
+        const v = s.attributes["gen_ai.agent.id"];
+        if (typeof v === "string")
+            return v;
+    }
+    return null;
+}
+function extractTenant(tree) {
+    for (const s of tree.spans) {
+        const v = s.attributes["tenant"];
+        if (typeof v === "string")
+            return v;
+    }
+    return null;
+}
+async function cliVersion() {
+    try {
+        const pkgPath = path.join(__dirname, "..", "..", "..", "package.json");
+        const txt = await fs.readFile(pkgPath, "utf8");
+        return JSON.parse(txt).version ?? "0.0.0";
+    }
+    catch {
+        return "0.0.0";
+    }
+}
+export { TraceNotFoundError as DiagnoseTraceNotFound, RuleLoadError, RuleProbeError };

package/dist/trace-ai/diagnose/output-schema-converter.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Convert a rubric YAML's `output_schema` (a JSON-Schema-ish blob) into a
+ * zod schema the agent provider validates LLM responses against.
+ *
+ * We don't pull in a full JSON-Schema-to-Zod converter — rubric YAMLs use
+ * a deliberately narrow subset: `type: object` with `required[]` and
+ * `properties{type, enum, items}`. Anything richer is rejected at load
+ * time so authors don't accidentally rely on full JSON Schema semantics
+ * we haven't implemented.
+ *
+ * Supported per-property `type` values: `string`, `number`, `boolean`,
+ * `array` (homogeneous items by `items.type`), `object` (recursive).
+ * `enum` (string-only) is supported on `string` properties.
+ *
+ * Unsupported / rejected at conversion time: `type: integer` (use number),
+ * `anyOf`/`oneOf`, `$ref`, `additionalProperties: false`, `format`.
+ */
+import { z } from "zod";
+import type { RubricYaml } from "./schemas.js";
+export declare class OutputSchemaConversionError extends Error {
+    readonly path: string;
+    constructor(message: string, path: string);
+}
+export declare function rubricOutputToZod(rubric: RubricYaml): z.ZodTypeAny;

package/dist/trace-ai/diagnose/output-schema-converter.js

@@ -0,0 +1,81 @@
+/**
+ * Convert a rubric YAML's `output_schema` (a JSON-Schema-ish blob) into a
+ * zod schema the agent provider validates LLM responses against.
+ *
+ * We don't pull in a full JSON-Schema-to-Zod converter — rubric YAMLs use
+ * a deliberately narrow subset: `type: object` with `required[]` and
+ * `properties{type, enum, items}`. Anything richer is rejected at load
+ * time so authors don't accidentally rely on full JSON Schema semantics
+ * we haven't implemented.
+ *
+ * Supported per-property `type` values: `string`, `number`, `boolean`,
+ * `array` (homogeneous items by `items.type`), `object` (recursive).
+ * `enum` (string-only) is supported on `string` properties.
+ *
+ * Unsupported / rejected at conversion time: `type: integer` (use number),
+ * `anyOf`/`oneOf`, `$ref`, `additionalProperties: false`, `format`.
+ */
+import { z } from "zod";
+export class OutputSchemaConversionError extends Error {
+    path;
+    constructor(message, path) {
+        super(`${message} (at ${path})`);
+        this.path = path;
+        this.name = "OutputSchemaConversionError";
+    }
+}
+function convertProp(spec, path) {
+    const t = spec.type;
+    if (typeof t !== "string") {
+        throw new OutputSchemaConversionError(`property is missing 'type' string`, path);
+    }
+    switch (t) {
+        case "string": {
+            if (Array.isArray(spec.enum)) {
+                if (spec.enum.length === 0) {
+                    throw new OutputSchemaConversionError(`empty enum`, path);
+                }
+                for (const v of spec.enum) {
+                    if (typeof v !== "string") {
+                        throw new OutputSchemaConversionError(`enum supports string values only`, path);
+                    }
+                }
+                return z.enum(spec.enum);
+            }
+            return z.string();
+        }
+        case "number": return z.number();
+        case "boolean": return z.boolean();
+        case "array": {
+            const items = spec.items;
+            if (!items) {
+                throw new OutputSchemaConversionError(`array property requires 'items'`, path);
+            }
+            return z.array(convertProp(items, `${path}.items`));
+        }
+        case "object": {
+            const subProps = spec.properties ?? {};
+            const subRequired = spec.required ?? [];
+            return buildObject(subProps, subRequired, path);
+        }
+        default:
+            throw new OutputSchemaConversionError(`unsupported type '${t}'`, path);
+    }
+}
+function buildObject(properties, required, path) {
+    const shape = {};
+    const requiredSet = new Set(required);
+    for (const [key, spec] of Object.entries(properties)) {
+        const sub = convertProp(spec, `${path}.${key}`);
+        shape[key] = requiredSet.has(key) ? sub : sub.optional();
+    }
+    for (const req of required) {
+        if (!(req in properties)) {
+            throw new OutputSchemaConversionError(`required key '${req}' is not present in properties`, path);
+        }
+    }
+    return z.object(shape);
+}
+export function rubricOutputToZod(rubric) {
+    return buildObject(rubric.output_schema.properties, rubric.output_schema.required, "output_schema");
+}

package/dist/trace-ai/diagnose/query-extractor.d.ts

@@ -0,0 +1,14 @@
+import type { TraceTree } from "./types.js";
+/**
+ * Extract the most recent user-role message from a trace's input.messages.
+ *
+ * Scans spans for `gen_ai.input.messages` (a JSON-stringified array of
+ * {role, content}), checking two locations in order:
+ *   1. span.events[*].attributes  — emitted by dolphin otel_listener as the
+ *      "gen_ai.client.inference.operation.details" event (primary path)
+ *   2. span.attributes             — fallback for runtimes that promote the
+ *      field directly onto the span
+ *
+ * Returns the last `role === "user"` message content, or null if not found.
+ */
+export declare function extractUserQueryFromTrace(tree: TraceTree): string | null;

package/dist/trace-ai/diagnose/query-extractor.js

@@ -0,0 +1,45 @@
+/**
+ * Extract the most recent user-role message from a trace's input.messages.
+ *
+ * Scans spans for `gen_ai.input.messages` (a JSON-stringified array of
+ * {role, content}), checking two locations in order:
+ *   1. span.events[*].attributes  — emitted by dolphin otel_listener as the
+ *      "gen_ai.client.inference.operation.details" event (primary path)
+ *   2. span.attributes             — fallback for runtimes that promote the
+ *      field directly onto the span
+ *
+ * Returns the last `role === "user"` message content, or null if not found.
+ */
+export function extractUserQueryFromTrace(tree) {
+    for (const span of tree.spans) {
+        const candidates = [];
+        // Primary: event attributes (dolphin otel_listener path)
+        for (const ev of span.events ?? []) {
+            const v = ev.attributes?.["gen_ai.input.messages"];
+            if (typeof v === "string")
+                candidates.push(v);
+        }
+        // Fallback: span attributes
+        const spanAttr = span.attributes?.["gen_ai.input.messages"];
+        if (typeof spanAttr === "string")
+            candidates.push(spanAttr);
+        for (const raw of candidates) {
+            let parsed;
+            try {
+                parsed = JSON.parse(raw);
+            }
+            catch {
+                continue;
+            }
+            if (!Array.isArray(parsed))
+                continue;
+            for (let i = parsed.length - 1; i >= 0; i--) {
+                const m = parsed[i];
+                if (m?.role === "user" && typeof m.content === "string" && m.content.length > 0) {
+                    return m.content;
+                }
+            }
+        }
+    }
+    return null;
+}

package/dist/trace-ai/diagnose/report-assembler.d.ts

@@ -0,0 +1,31 @@
+import type { Finding, Hit, Report, Rule, Summary } from "./types.js";
+export interface AssembleReportOpts {
+    traceId: string;
+    agentId: string | null;
+    tenant: string | null;
+    cliVersion: string;
+    rules: Rule[];
+    hits: Map<string, Hit[]>;
+    /** Additional pre-built findings (rubric judgments come from agent-binding). */
+    extraFindings?: Finding[];
+    summary: Summary;
+    /** Run mode. Default `symbolic-only` for backward compat. */
+    mode?: 'symbolic-only' | 'rubric-only' | 'hybrid';
+    /** Rubric rules skipped due to --no-llm / unavailable provider / etc. */
+    rulesSkipped?: {
+        ruleId: string;
+        reason: string;
+    }[];
+    /** Stage-3 synthesizer that produced `summary`. */
+    synthesizerMode?: 'template' | 'agent';
+    /** User query extracted from trace input.messages (2026-05-13). */
+    userQuery?: string | null;
+    /** Conversation/query ID for suggested_eval_case correlation (2026-05-13). */
+    queryId?: string | null;
+}
+/** Build symbolic-pillar findings from rule+hit pairs.
+ *  Exported so callers (e.g. tests, index.ts) can compose findings from
+ *  multiple sources before handing them to a custom summary path. */
+export declare function symbolicHitsToFindings(rules: Rule[], hits: Map<string, Hit[]>, userQuery?: string | null, queryId?: string | null): Finding[];
+export declare function assembleReport(opts: AssembleReportOpts): Report;
+export declare function reportToYamlObject(r: Report): unknown;

package/dist/{trace-core → trace-ai}/diagnose/report-assembler.js

@@ -4,17 +4,22 @@ function renderTemplate(tpl, bindings) {
         return v === undefined ? `{{${key}}}` : String(v);
     });
 }
-export function assembleReport(opts) {
+/** Build symbolic-pillar findings from rule+hit pairs.
+ *  Exported so callers (e.g. tests, index.ts) can compose findings from
+ *  multiple sources before handing them to a custom summary path. */
+export function symbolicHitsToFindings(rules, hits, userQuery = null, queryId = null) {
     const findings = [];
-    for (const rule of opts.rules) {
-        const ruleHits = opts.hits.get(rule.id) ?? [];
+    for (const rule of rules) {
+        if (rule.predicateRef === null)
+            continue;
+        const ruleHits = hits.get(rule.id) ?? [];
         for (const hit of ruleHits) {
             findings.push({
                 ruleId: rule.id,
                 judgmentKind: "symbolic",
                 severity: rule.severity,
                 symptom: rule.symptom,
-                likelyCause: rule.symptom, // PR-A: no LLM, so we mirror symptom; PR-B agent overrides this
+                likelyCause: rule.symptom, // symbolic: no LLM, so mirror symptom; rubric agent overrides
                 evidence: { spans: hit.evidenceSpans, excerpt: hit.excerpt },
                 suggestedFix: {
                     target: rule.suggestedFix.target,
@@ -23,24 +28,29 @@ export function assembleReport(opts) {
                 confidence: "low",
                 verifyWith: {
                     suggestedEvalCase: {
-                        queryId: null, // PR-A: no query extraction yet (deferred per spec)
-                        query: null,
+                        queryId,
+                        query: userQuery,
                         assertions: rule.verifyWith.assertionTemplates.map((t) => renderTemplate(t, hit.bindings)),
                     },
                 },
             });
         }
     }
+    return findings;
+}
+export function assembleReport(opts) {
+    const symbolicFindings = symbolicHitsToFindings(opts.rules, opts.hits, opts.userQuery ?? null, opts.queryId ?? null);
+    const findings = [...symbolicFindings, ...(opts.extraFindings ?? [])];
     return {
         schemaVersion: "trace-diagnose-report/v1",
         trace: { traceId: opts.traceId, agentId: opts.agentId, tenant: opts.tenant },
         run: {
             diagnosedAt: new Date().toISOString(),
             cliVersion: opts.cliVersion,
-            mode: "symbolic-only",
+            mode: opts.mode ?? "symbolic-only",
             rulesApplied: opts.rules.map((r) => r.id),
-            rulesSkipped: [],
-            synthesizerMode: "template",
+            rulesSkipped: opts.rulesSkipped ?? [],
+            synthesizerMode: opts.synthesizerMode ?? "template",
         },
         summary: opts.summary,
         findings,

package/dist/trace-ai/diagnose/report-markdown.d.ts

@@ -0,0 +1,18 @@
+import type { Report } from "./types.js";
+/**
+ * Optional context the md renderer uses to build runnable verification
+ * commands. None of these are in the yaml schema (which stays v1-locked and
+ * CLI-agnostic) — they live only in the markdown view so users who paste the
+ * md into a ticket / PR have copy-pasteable shell commands without needing to
+ * remember the trace's conversation context.
+ */
+export interface MarkdownRenderOpts {
+    /** The conversation_id passed to `kweaver trace diagnose`. Used to render
+     *  the "re-run diagnosis" command. When undefined, that command is rendered
+     *  with a `<conversation_id>` placeholder. */
+    conversationId?: string;
+    /** Business domain (`-bd` flag). When undefined, commands omit the flag and
+     *  inherit kweaver's default (`bd_public`). */
+    businessDomain?: string;
+}
+export declare function renderReportMarkdown(r: Report, opts?: MarkdownRenderOpts): string;