@kweaver-ai/kweaver-sdk 0.8.3 → 0.8.4

package/dist/trace-ai/exp/providers/synthesizer-client.js

@@ -1,39 +1,57 @@
-// src/trace-ai/exp/providers/synthesizer-client.ts
-import yaml from "js-yaml";
-import { defaultRegistry } from "../../../agent-providers/registry.js";
-import { NextChangeSchema } from "../schemas.js";
-export class ClaudeCodeSynthesizer {
-    async generate(input) {
-        const provider = defaultRegistry.resolve({ preferred: "claude-code" });
-        if (!provider)
-            throw new Error("claude-code provider not available");
-        const prevSummary = input.prevRounds.map(r => `Round ${r.round}: outcome=${r.scores?.outcome.toFixed(2) ?? "?"}, hints=${r.triage_conclusion?.hints.join("; ") ?? "none"}`).join("\n");
-        const prompt = `You are an agent optimization assistant. Given an experiment goal and round results, suggest the next change to try.
+export function buildKnContextPrompt(ctx) {
+    const existingTypes = ctx.existing_schema.object_types
+        .map(t => `  - ${t.concept_name}: fields=[${t.fields.map(f => `${f.name}(${f.type})`).join(", ")}]`)
+        .join("\n");
+    const existingRelations = ctx.existing_schema.relation_types
+        .map(r => `  - ${r.concept_name}: ${r.source} → ${r.target}, join_key="${r.join_key}"`)
+        .join("\n") || "  (none)";
+    const dataviews = ctx.available_dataviews
+        .map(dv => `  - id="${dv.id}" name="${dv.name}"\n    columns=[${dv.columns.map(c => `${c.name}(${c.type})`).join(", ")}]`)
+        .join("\n");
+    const probes = ctx.data_probes && ctx.data_probes.length > 0
+        ? "\n## Data Probes (live record counts from KN)\n" + ctx.data_probes
+            .map(p => `  - ${p.concept_name} (data_view: ${p.data_view_id}) → ${p.total_records} records`)
+            .join("\n")
+        : "";
+    return `
+## Existing KN Schema (kn_id: ${ctx.kn_id})
+Object types:
+${existingTypes || "  (none)"}
+Relation types:
+${existingRelations}
 
-GOAL: ${input.mission.goal}
+## Available Vega Dataviews
+${dataviews}${probes}
 
-CURRENT CANDIDATE CONFIG:
-${yaml.dump(input.candidateConfig, { lineWidth: 80 })}
-
-PREVIOUS ROUNDS:
-${prevSummary || "None (first round)"}
-
-${input.prevRound?.triage_conclusion ? `TRIAGE HINTS FROM LAST ROUND:\n${input.prevRound.triage_conclusion.hints.join("\n")}` : ""}
-
-${input.crossRoundMemoryRef ? `CROSS-ROUND CONTEXT: ${input.crossRoundMemoryRef}` : ""}
-
-Respond with a JSON object with exactly these fields:
-- "target": one of "agent.system_prompt", "agent.temperature", "agent.model", "skill.add", "skill.remove", "skill.swap"
-- "hypothesis": brief explanation of why this change might help
-- "patch": a JSON Merge Patch string to apply to the candidate config
+## Instructions for generating the KN patch:
+1. Find the dataview whose name contains the missing concept_name as a suffix substring
+   (e.g. concept "vehicle_sales" matches "ht_data_513_vehicle_sales" because name.endsWith("vehicle_sales") after split by "_" heuristic).
+   If multiple candidates exist, pick the closest suffix match.
+2. Extract data_properties from that dataview's columns.
+3. Infer primary_keys: look for a column named "{concept_name}_id". If not found, use "id".
+   Include only the primary key column(s), not all columns.
+4. For relation_type inference:
+   - Normalize field names: lowercase + remove underscores (e.g. "VEHICLEID" → "vehicleid", "vehicle_id" → "vehicleid").
+   - If any existing KN object_type has a field whose normalized name equals any column in the new dataview:
+     propose a relation_type with join_key = "{existing_field_name} → {new_column_name}".
+5. Set kn_id to "${ctx.kn_id}".
+`.trim();
+}
+export function buildSkillContextPrompt(ctx) {
+    const skillDocs = ctx.bound_skills
+        .map(s => `### skill_id: "${s.id}" (version: ${s.version})\n${s.content}`)
+        .join("\n\n---\n\n");
+    return `
+## Currently Bound Skills
+${skillDocs}
 
-Example for changing system_prompt:
-{"target": "agent.system_prompt", "hypothesis": "Add explicit stop condition", "patch": "{\"agent\":{\"system_prompt\":\"New prompt here\"}}"}`;
-        const response = await provider.invoke({
-            prompt,
-            outputSchema: NextChangeSchema,
-            correlationId: `synthesizer-${Date.now()}`,
-        });
-        return response.output;
-    }
+## Instructions for generating the skill.content patch:
+1. Read the failure evidence carefully. It names a specific tool (e.g. "query_object_instance").
+2. Search each skill's content for that tool name. The skill that documents the tool is the one to patch.
+   Set skill_id to that skill's id.
+3. Generate append_section that fixes the capability gap described in the failure evidence.
+   - Match the existing document style (heading level, bullet format, code examples if any).
+   - Be specific: if the issue is "no sort_by", show exactly how to pass sort_by.
+   - If the issue is "no pagination", show the search_after loop pattern.
+`.trim();
 }

package/dist/trace-ai/exp/providers/triage-client.d.ts

@@ -1,19 +1,32 @@
-import type { RoundData } from "../schemas.js";
+import type { RoundData, FailureAttribution, QueryFailureAnalysis, Mission, NextChange, KnContext, SkillContext } from "../schemas.js";
 export interface TriageInput {
     currentRound: RoundData;
     prevRounds: RoundData[];
     candidateConfig: Record<string, unknown>;
     crossRoundMemoryRef?: string;
+    failureAnalysis?: QueryFailureAnalysis[];
+    mission: Mission;
+    kn_context?: KnContext;
+    skill_context?: SkillContext;
 }
 export interface TriageResult {
+    verdict: "continue" | "publish" | "abort";
+    summary: string;
+    failure_attribution: FailureAttribution[];
+    /** Next change to apply. Present when verdict === "continue"; null/undefined otherwise. */
+    next_change?: NextChange;
+    /** Diagnoses extracted from the LLM response (defaults to [summary]) */
     diagnoses: string[];
+    /** Hints extracted from the LLM response */
     hints: string[];
-    verdict: "continue" | "publish";
+    /** Opaque memory token passed across rounds */
     new_memory_token: string;
 }
 export interface TriageClient {
     triage(input: TriageInput): Promise<TriageResult>;
 }
+export declare function parseTriageOutput(raw: string): TriageResult;
+export declare function buildTriagePrompt(input: TriageInput): string;
 export declare class ClaudeCodeTriageClient implements TriageClient {
     triage(input: TriageInput): Promise<TriageResult>;
 }

package/dist/trace-ai/exp/providers/triage-client.js

@@ -1,51 +1,166 @@
 // src/trace-ai/exp/providers/triage-client.ts
+import yaml from "js-yaml";
 import { z } from "zod";
 import { defaultRegistry } from "../../../agent-providers/registry.js";
-const TriageOutputSchema = z.object({
-    diagnoses: z.array(z.string()),
-    hints: z.array(z.string()),
-    verdict: z.enum(["continue", "publish"]),
-    new_memory_token: z.string(),
-});
-export class ClaudeCodeTriageClient {
-    async triage(input) {
-        const provider = defaultRegistry.resolve({ preferred: "claude-code" });
-        if (!provider)
-            throw new Error("claude-code provider not available");
-        const r = input.currentRound;
-        const scoresSummary = r.scores
-            ? `outcome=${r.scores.outcome.toFixed(2)}, trajectory=${r.scores.trajectory.toFixed(2)}, guardrail=${r.scores.guardrail.toFixed(2)}`
-            : "no scores";
+import { FailureAttributionSchema, NextChangeSchema } from "../schemas.js";
+import { buildKnContextPrompt, buildSkillContextPrompt } from "./synthesizer-client.js";
+/**
+ * Unwrap a Markdown code fence the LLM commonly puts around its JSON
+ * (```json … ``` or bare ``` … ```), tolerating prose before/after it.
+ * Falls back to the trimmed input when no fence is present.
+ */
+function unwrapJson(raw) {
+    const fence = raw.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
+    return (fence ? fence[1] : raw).trim();
+}
+export function parseTriageOutput(raw) {
+    let obj;
+    try {
+        obj = JSON.parse(unwrapJson(raw));
+    }
+    catch {
+        throw new Error(`Triage output is not valid JSON: ${raw.slice(0, 200)}`);
+    }
+    const data = obj;
+    const verdict = data["verdict"];
+    if (verdict !== "continue" && verdict !== "publish" && verdict !== "abort") {
+        throw new Error(`Invalid verdict in triage output: ${String(verdict)}`);
+    }
+    const summary = typeof data["summary"] === "string" ? data["summary"] : "";
+    const rawAttribution = Array.isArray(data["failure_attribution"]) ? data["failure_attribution"] : [];
+    const failure_attribution = rawAttribution.map((item) => {
+        const parsed = FailureAttributionSchema.safeParse(item);
+        if (!parsed.success)
+            throw new Error(`Invalid failure_attribution item: ${JSON.stringify(item)}`);
+        return parsed.data;
+    });
+    let next_change;
+    if (verdict === "continue") {
+        if (data["next_change"] === undefined || data["next_change"] === null) {
+            throw new Error(`verdict=continue requires next_change in triage output`);
+        }
+        const parsed = NextChangeSchema.safeParse(data["next_change"]);
+        if (!parsed.success) {
+            throw new Error(`Invalid next_change in triage output: ${parsed.error.message}`);
+        }
+        next_change = parsed.data;
+    }
+    const diagnoses = Array.isArray(data["diagnoses"])
+        ? data["diagnoses"].filter((d) => typeof d === "string")
+        : [summary];
+    const hints = Array.isArray(data["hints"])
+        ? data["hints"].filter((h) => typeof h === "string")
+        : [];
+    const new_memory_token = typeof data["new_memory_token"] === "string" ? data["new_memory_token"] : summary;
+    return { verdict, summary, failure_attribution, next_change, diagnoses, hints, new_memory_token };
+}
+export function buildTriagePrompt(input) {
+    const r = input.currentRound;
+    const scoresSummary = r.scores
+        ? `outcome=${r.scores.outcome.toFixed(2)}, trajectory=${r.scores.trajectory.toFixed(2)}, guardrail=${r.scores.guardrail.toFixed(2)}`
+        : "no scores";
+    let failureSection;
+    if (input.failureAnalysis && input.failureAnalysis.length > 0) {
+        const lines = input.failureAnalysis.map(fa => {
+            let entry = `${fa.query_id} [${fa.verdict}]: ${fa.assertion_reason}`;
+            if (fa.tool_call_summary.length > 0) {
+                entry += `\n  Tools: ${fa.tool_call_summary.join("; ")}`;
+            }
+            return entry;
+        }).join("\n");
+        failureSection = `FAILURE ANALYSIS:\n${lines}`;
+    }
+    else {
         const failedQueries = (r.per_query_results ?? [])
             .filter(q => q.assertion_results.some(a => a.verdict === "fail"))
             .map(q => `${q.query_id}: ${q.assertion_results.filter(a => a.verdict === "fail").map(a => a.type).join(", ")}`)
             .join("\n");
-        // candidateConfig is available for future prompt enrichment; omitted here to keep the prompt focused on scores.
-        const prompt = `You are an agent evaluation triager. Analyze the current round results and recommend next steps.
+        failureSection = `FAILED QUERIES:\n${failedQueries || "None"}`;
+    }
+    let contextSection = "";
+    if (input.kn_context)
+        contextSection += "\n\n" + buildKnContextPrompt(input.kn_context);
+    if (input.skill_context)
+        contextSection += "\n\n" + buildSkillContextPrompt(input.skill_context);
+    // Restrict suggested_target choices + output examples to layers this mission opted into.
+    // Empty / missing enabled_targets defaults to ["agent.system_prompt"] (handled by MissionSchema).
+    const enabled = new Set(input.mission.enabled_targets);
+    const targetEnumLine = [...enabled].map(t => `"${t}"`).join(" | ");
+    const allExamples = [
+        ["agent.system_prompt", `# agent.system_prompt — patch is a JSON string (escaped) or object with {agent:{system_prompt}}
+{"target":"agent.system_prompt","hypothesis":"Add explicit stop condition","patch":"{\\"agent\\":{\\"system_prompt\\":\\"New prompt here\\"}}"}`],
+        ["agent.skills", `# agent.skills — patch is structured {unbind:[skill_id...], bind:[{id,version}...]}
+{"target":"agent.skills","hypothesis":"Swap retrieval skill to v2","patch":{"unbind":["retrieval-v1"],"bind":[{"id":"retrieval-v2","version":"v2"}]}}`],
+        ["kn.object_type", `# kn.object_type — patch is {kn_id, add_object_types:[{concept_name, dataview_id, primary_keys, data_properties}], add_relation_types:[]}
+{"target":"kn.object_type","hypothesis":"Add vehicle_sales concept","patch":{"kn_id":"kn-x","add_object_types":[{"concept_name":"vehicle_sales","dataview_id":"dv-001","primary_keys":["vehicle_sales_id"],"data_properties":[{"name":"sales","type":"integer"},{"name":"month","type":"string"}]}],"add_relation_types":[]}}`],
+        ["kn.relation_type", `# kn.relation_type — patch is {kn_id, add_object_types:[], add_relation_types:[{concept_name, source_object_type, target_object_type, join_key}]}
+{"target":"kn.relation_type","hypothesis":"Link sales to vehicle","patch":{"kn_id":"kn-x","add_object_types":[],"add_relation_types":[{"concept_name":"sold_for","source_object_type":"vehicle_sales","target_object_type":"vehicle","join_key":"vehicle_id"}]}}`],
+        ["skill.content", `# skill.content — patch is {skill_id, append_section}
+{"target":"skill.content","hypothesis":"Document sort_by usage","patch":{"skill_id":"query-sop","append_section":"## Sort_by usage\\nPass sort_by=[{field, order}] to query_object_instance for ordering."}}`],
+    ];
+    const examplesBlock = allExamples
+        .filter(([target]) => enabled.has(target))
+        .map(([, body]) => body)
+        .join("\n\n");
+    return `You are an agent evaluation planner. Analyze the current round results, decide whether to continue/publish/abort, and (if continuing) propose the next change to try in one pass.
+
+GOAL: ${input.mission.goal}
+
+CURRENT CANDIDATE CONFIG:
+${yaml.dump(input.candidateConfig, { lineWidth: 80 })}
 
 ROUND ${r.round} SCORES: ${scoresSummary}
 
-FAILED QUERIES:
-${failedQueries || "None"}
+${failureSection}
 
 TRAJECTORY ISSUES:
 ${(r.per_query_results ?? []).filter(q => q.trajectory_summary.retry_count > 1).map(q => `${q.query_id}: ${q.trajectory_summary.retry_count} retries`).join("\n") || "None"}
 
 PREVIOUS ROUND HISTORY:
-${input.prevRounds.map(pr => `Round ${pr.round}: outcome=${pr.scores?.outcome.toFixed(2) ?? "?"}, verdict=${pr.triage_conclusion?.verdict ?? "?"}`).join("\n") || "None"}
+${input.prevRounds.map(pr => `Round ${pr.round}: outcome=${pr.scores?.outcome.toFixed(2) ?? "?"}, verdict=${pr.triage_conclusion?.verdict ?? "?"}, hints=${pr.triage_conclusion?.hints?.join("; ") || "none"}`).join("\n") || "None"}
+
+${input.crossRoundMemoryRef ? `CROSS-ROUND CONTEXT: ${input.crossRoundMemoryRef}` : ""}${contextSection}
+
+Respond with a single JSON object containing these fields:
+- "verdict": "continue" | "publish" | "abort"
+    * continue = more rounds needed, you must also provide next_change
+    * publish = current candidate is good enough
+    * abort = experiment cannot improve further
+- "summary": one-sentence summary of key findings
+- "failure_attribution": array of root-cause attributions (sorted by affected_queries count desc),
+  empty array when verdict is publish or abort.
+  Each entry shape:
+  {
+    "layer": "kn" | "skill" | "agent",
+    "evidence": "<one sentence citing specific tool call or return value>",
+    "affected_queries": ["<query_id>", ...],
+    "suggested_target": ${targetEnumLine}
+  }
+- "next_change": REQUIRED when verdict=continue, omit/null otherwise. Use failure_attribution[0].suggested_target.
+  This mission has enabled_targets = [${[...enabled].join(", ")}]; do NOT propose any other target.
+- "hints": array of short actionable hints for the next round (carried forward to future PREVIOUS ROUND HISTORY); use [] if none.
+- "new_memory_token": short string (≤ 200 chars) summarizing what to remember across rounds; will appear as CROSS-ROUND CONTEXT next round.
+
+Attribution rules:
+- "kn":    agent queried KN but concept/relation was missing or returned empty unexpectedly
+- "skill": agent had the right intent but the tool usage pattern was wrong (no pagination, no sort_by, wrong filter)
+- "agent": agent misidentified the concept to query, or made an orchestration error
 
-${input.crossRoundMemoryRef ? `CONTEXT FROM PREVIOUS TRIAGE: ${input.crossRoundMemoryRef}` : ""}
+NEXT_CHANGE OUTPUT EXAMPLES (pick the one matching your chosen target):
 
-Respond with JSON:
-- "diagnoses": list of root cause observations
-- "hints": list of specific suggestions for next change
-- "verdict": "continue" if more rounds needed, "publish" if this candidate is good enough
-- "new_memory_token": brief summary of key findings to carry forward (1-2 sentences)`;
+${examplesBlock}`;
+}
+export class ClaudeCodeTriageClient {
+    async triage(input) {
+        const provider = defaultRegistry.resolve({ preferred: "claude-code" });
+        if (!provider)
+            throw new Error("claude-code provider not available");
+        const prompt = buildTriagePrompt(input);
         const response = await provider.invoke({
             prompt,
-            outputSchema: TriageOutputSchema,
+            outputSchema: z.unknown(),
             correlationId: `triage-${Date.now()}`,
         });
-        return response.output;
+        return parseTriageOutput(response.rawText);
     }
 }

package/dist/trace-ai/exp/run-preflight.d.ts

@@ -0,0 +1,19 @@
+import { type AgentConfigFetcher } from "./capture-fingerprint.js";
+export interface RunPreflightOpts {
+    expDir: string;
+    agentId: string;
+    fetchConfig: AgentConfigFetcher;
+    evalSetPaths: string[];
+}
+/**
+ * Run the preflight reconciliation. Throws PreflightMismatchError if the live
+ * agent's KN binding does not match the eval set's target_kn.
+ *
+ * The expected fingerprint is re-captured every round: without an executor that
+ * deploys patches, the loop cannot tell an intentional agent change from
+ * unintended drift, so it does not gate on config drift. It records the live
+ * config as the per-round provenance fingerprint and lets preflightCheck
+ * enforce the load-bearing invariant — the KN binding. (When an executor
+ * exists, pass the deployed fingerprint as `expected` to also gate on drift.)
+ */
+export declare function runPreflight(opts: RunPreflightOpts): Promise<void>;

package/dist/trace-ai/exp/run-preflight.js

@@ -0,0 +1,56 @@
+// src/trace-ai/exp/run-preflight.ts
+//
+// Preflight orchestration: before an eval round runs, reconcile the live agent
+// against expectation. On first run it captures a baseline expected fingerprint;
+// thereafter it verifies the live agent still matches that baseline and that the
+// agent's KN binding matches every eval set's declared target_kn.
+import fs from "node:fs/promises";
+import path from "node:path";
+import yaml from "js-yaml";
+import { EvalSetIndexSchema } from "../eval-set/schemas.js";
+import { captureAgentFingerprint } from "./capture-fingerprint.js";
+import { preflightCheck } from "./preflight.js";
+import { writeExpectedFingerprint } from "./exp-store/expected-fingerprint.js";
+/**
+ * Resolve the single KN the eval sets expect the agent to be bound to.
+ * Returns undefined when no eval set declares a target_kn (invariant 4 is then
+ * skipped). Throws when eval sets declare conflicting target_kn values.
+ */
+async function resolveEvalTargetKn(evalSetPaths) {
+    const found = new Set();
+    for (const p of evalSetPaths) {
+        let raw;
+        try {
+            raw = await fs.readFile(path.join(p, "index.yaml"), "utf8");
+        }
+        catch {
+            continue; // no index here — the eval runner surfaces that, not preflight
+        }
+        const parsed = EvalSetIndexSchema.safeParse(yaml.load(raw));
+        if (parsed.success && parsed.data.target_kn)
+            found.add(parsed.data.target_kn);
+    }
+    if (found.size === 0)
+        return undefined;
+    if (found.size > 1) {
+        throw new Error(`eval sets declare conflicting target_kn: ${[...found].sort().join(", ")}`);
+    }
+    return [...found][0];
+}
+/**
+ * Run the preflight reconciliation. Throws PreflightMismatchError if the live
+ * agent's KN binding does not match the eval set's target_kn.
+ *
+ * The expected fingerprint is re-captured every round: without an executor that
+ * deploys patches, the loop cannot tell an intentional agent change from
+ * unintended drift, so it does not gate on config drift. It records the live
+ * config as the per-round provenance fingerprint and lets preflightCheck
+ * enforce the load-bearing invariant — the KN binding. (When an executor
+ * exists, pass the deployed fingerprint as `expected` to also gate on drift.)
+ */
+export async function runPreflight(opts) {
+    const actual = await captureAgentFingerprint(opts.fetchConfig, opts.agentId, "latest");
+    await writeExpectedFingerprint(opts.expDir, actual);
+    const targetKn = await resolveEvalTargetKn(opts.evalSetPaths);
+    preflightCheck(actual, actual, targetKn);
+}