@kweaver-ai/kweaver-sdk 0.8.3 → 0.8.4

package/dist/trace-ai/exp/index.js

@@ -4,7 +4,6 @@ import fs from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import { ExpStore } from "./exp-store/index.js";
 import { ExperimentCoordinator } from "./coordinator.js";
-import { ClaudeCodeSynthesizer } from "./providers/synthesizer-client.js";
 import { ClaudeCodeTriageClient } from "./providers/triage-client.js";
 import { runEval } from "./eval-runner.js";
 import { defaultRegistry } from "../../agent-providers/registry.js";
@@ -13,17 +12,40 @@ import { PromptTemplateRegistry } from "../../agent-providers/prompt-template.js
 import { createBuiltinSemanticMatchProvider } from "../eval-set/semantic-match-provider.js";
 import { ensureValidToken } from "../../auth/oauth.js";
 import { fetchAgentInfo, sendChatRequest } from "../../api/agent-chat.js";
+import { getAgent } from "../../api/agent-list.js";
 import { getTracesByConversation } from "../../api/conversations.js";
 import { upsertRegistry, listRegistry } from "./exp-store/exp-registry.js";
 import { runInfo, runList, getHealthChecks } from "./info.js";
 import { resolveClaudeBinary } from "./claude-binary.js";
+import { KweaverKnSchemaClient } from "./context/kn-schema-client.js";
+import { ContextAssembler } from "./context/context-assembler.js";
+import { probeObjectTypes } from "./context/kn-data-prober.js";
+import { queryResource } from "../../api/resources.js";
+import { KweaverVegaCatalogClient } from "./context/vega-catalog-client.js";
+import { KweaverKnApiClient } from "./patch/kn-api-client.js";
+import { KweaverSkillApiClient } from "./patch/skill-api-client.js";
+const MCP_PATH = "/api/agent-retrieval/v1/mcp";
 const __expIndexDir = path.dirname(fileURLToPath(import.meta.url));
+export function formatFailureAttribution(attribution) {
+    if (attribution.length === 0)
+        return "";
+    const lines = attribution.map((a) => {
+        const queries = a.affected_queries.join(", ");
+        const evidence = a.evidence.length > 55 ? a.evidence.slice(0, 52) + "..." : a.evidence;
+        const layerTag = `[${a.layer}]`.padEnd(7);
+        return `  ${layerTag} ${evidence.padEnd(55)}  → ${a.suggested_target}  (${queries})`;
+    });
+    return "Failure attribution:\n" + lines.join("\n");
+}
 const EVAL_SET_RUBRIC_DIR = path.join(__expIndexDir, "..", "eval-set", "rubric-templates");
 function ensureProvider() {
     if (!defaultRegistry.has("claude-code")) {
         defaultRegistry.register(new ClaudeCodeSubprocessProvider({
             binary: resolveClaudeBinary(),
-            defaultTimeoutMs: 120_000,
+            // 10 min: the merged Triage+Synthesizer call carries a whole round of
+            // eval context (per-query failures, KN schema, skill content) and the
+            // semantic-match evaluator runs on every case — 120s timed both out.
+            defaultTimeoutMs: 600_000,
         }), { setAsDefault: true });
     }
 }
@@ -110,6 +132,7 @@ export async function runExpCommand(argv) {
             const rounds = await store.readAllRounds();
             const lineage = await store.readLineage();
             const mission = await store.readMission().catch(() => null);
+            const events = await store.readAllEvents().catch(() => []);
             process.stdout.write(`State: ${replayed.currentState}  Round: ${replayed.currentRound}\n`);
             if (mission?.next_change) {
                 process.stdout.write(`Suggested next change:\n  target: ${mission.next_change.target}\n  hypothesis: ${mission.next_change.hypothesis}\n`);
@@ -121,6 +144,15 @@ export async function runExpCommand(argv) {
                     process.stdout.write(`Triage: ${last.triage_conclusion.diagnoses.join("; ")}\n`);
                 }
             }
+            // Read last TriageComplete event for failure_attribution
+            const lastTriage = events.filter((e) => e["type"] === "TriageComplete").at(-1);
+            const attribution = Array.isArray(lastTriage?.["failure_attribution"])
+                ? lastTriage["failure_attribution"]
+                : [];
+            const attrText = formatFailureAttribution(attribution);
+            if (attrText) {
+                process.stdout.write(`${attrText}\n`);
+            }
             process.stdout.write(`Lineage: ${lineage.length} versions\n`);
             return 0;
         }
@@ -202,10 +234,43 @@ async function makeCoordinator(expDir) {
     catch {
         process.stderr.write("warn: could not create semantic-match provider — semantic_match assertions will be skipped\n");
     }
+    // Read mission upfront so KN/Skill clients are only constructed when the
+    // experiment actually enables those layers. Avoids exposing stub clients
+    // (KweaverKnApiClient, KweaverSkillApiClient — both throw "not yet implemented")
+    // to missions that don't need them.
+    const mission = await new ExpStore(expDir).readMission();
+    const enabled = new Set(mission.enabled_targets);
+    const needsKn = enabled.has("kn.object_type") || enabled.has("kn.relation_type");
+    const needsSkill = enabled.has("skill.content");
+    const mcpUrl = baseUrl.replace(/\/+$/, "") + MCP_PATH;
+    const knSchemaClient = new KweaverKnSchemaClient(mcpUrl, token);
+    const vegaCatalogClient = new KweaverVegaCatalogClient(baseUrl, token);
+    // Wire probeObjectTypes with auth + businessDomain
+    const boundProbe = (schema, failures) => probeObjectTypes(schema, failures, queryResource, { baseUrl, accessToken: token });
+    // No-op SkillApiClient lets ContextAssembler pre-fetch bound_skill stubs even when
+    // skill.content isn't enabled (the bound list is informational for the planner).
+    const noopSkillContextClient = {
+        async getSkillContent(_id) { return ""; },
+        async publishSkillVersion(_id, _content) { return { version: "noop", content: "" }; },
+    };
+    const contextAssembler = new ContextAssembler(knSchemaClient, vegaCatalogClient, noopSkillContextClient, boundProbe);
     return new ExperimentCoordinator({
         expDir,
-        synthesizer: new ClaudeCodeSynthesizer(),
         triage: new ClaudeCodeTriageClient(),
+        contextAssembler,
+        // Full agent config (system_prompt / llms / skills.tools) comes from
+        // GET /v3/agent/{id}, which wraps the config object under a `config` key.
+        fetchAgentConfig: async (agentId) => {
+            const body = await getAgent({ baseUrl, accessToken: token, agentId, businessDomain: bd });
+            const parsed = JSON.parse(body);
+            return (parsed["config"] ?? parsed);
+        },
+        knClient: needsKn ? new KweaverKnApiClient(baseUrl, token) : undefined,
+        skillClient: needsSkill ? new KweaverSkillApiClient(baseUrl, token) : undefined,
+        fetchTrace: async (conversationId) => {
+            const r = await getTracesByConversation({ baseUrl, accessToken: token, conversationId, businessDomain: bd });
+            return { spans: r.spans };
+        },
         runEval: ({ evalSetPaths, candidatePath, round }) => runEval({
             evalSetPaths,
             candidatePath,

package/dist/trace-ai/exp/info.js

@@ -57,7 +57,7 @@ export async function buildExpSnapshot(expDir) {
     const scores = lastRound?.scores ?? null;
     const triage_summary = lastRound?.triage_conclusion?.diagnoses.join("; ") ?? null;
     const suggested_next = mission?.next_change
-        ? { target: mission.next_change.target, hypothesis: mission.next_change.hypothesis }
+        ? { target: mission.next_change.target, hypothesis: mission.next_change.hypothesis ?? "" }
         : null;
     return {
         workspace: expDir,

package/dist/trace-ai/exp/patch/index.d.ts

@@ -1,2 +1,13 @@
-import type { NextChange } from "../schemas.js";
-export declare function applyPatch(candidate: Record<string, unknown>, change: NextChange): Record<string, unknown>;
+import type { KnApiClient } from "./kn-api-client.js";
+import type { SkillApiClient } from "./skill-api-client.js";
+export interface ApplyResult {
+    candidate: Record<string, unknown>;
+    skillVersion?: string;
+}
+export declare class PatchApplier {
+    private workDir;
+    private knClient?;
+    private skillClient?;
+    constructor(workDir: string, knClient?: KnApiClient | undefined, skillClient?: SkillApiClient | undefined);
+    apply(candidate: Record<string, unknown>, rawNextChange: unknown): Promise<ApplyResult>;
+}

package/dist/trace-ai/exp/patch/index.js

@@ -1,13 +1,68 @@
+import { NextChangeSchema } from "../schemas.js";
 import { applyAgentConfigPatch } from "./agent-config.js";
-import { applySkillPatch } from "./skill.js";
-export function applyPatch(candidate, change) {
-    const prefix = change.target.split(".")[0];
-    switch (prefix) {
-        case "agent":
-            return applyAgentConfigPatch(candidate, change.patch);
-        case "skill":
-            return applySkillPatch(candidate, change.patch);
-        default:
-            throw new Error(`Unsupported target prefix "${prefix}" — only agent.* and skill.* are supported in MVP-C`);
+import { KnPatcher } from "./kn.js";
+import { SkillContentPatcher } from "./skill-content.js";
+export class PatchApplier {
+    workDir;
+    knClient;
+    skillClient;
+    constructor(workDir, knClient, skillClient) {
+        this.workDir = workDir;
+        this.knClient = knClient;
+        this.skillClient = skillClient;
     }
+    async apply(candidate, rawNextChange) {
+        const nc = NextChangeSchema.parse(rawNextChange);
+        const next = structuredClone(candidate);
+        switch (nc.target) {
+            case "agent.system_prompt": {
+                const patchStr = typeof nc.patch === "string" ? nc.patch : JSON.stringify(nc.patch);
+                return { candidate: applyAgentConfigPatch(next, patchStr) };
+            }
+            case "agent.skills": {
+                return { candidate: applyAgentSkillsPatch(next, nc.patch) };
+            }
+            case "kn.object_type":
+            case "kn.relation_type": {
+                if (!this.knClient)
+                    throw new Error("KnApiClient not provided for kn.* patch");
+                await new KnPatcher(this.knClient, this.workDir).apply(nc.patch);
+                const existingKn = next["kn"]
+                    ?? { id: nc.patch.kn_id, object_types: [], relation_types: [] };
+                next["kn"] = {
+                    ...existingKn,
+                    object_types: [...(existingKn["object_types"] ?? []), ...nc.patch.add_object_types],
+                    relation_types: [...(existingKn["relation_types"] ?? []), ...nc.patch.add_relation_types],
+                };
+                return { candidate: next };
+            }
+            case "skill.content": {
+                if (!this.skillClient)
+                    throw new Error("SkillApiClient not provided for skill.content patch");
+                const { newVersion } = await new SkillContentPatcher(this.skillClient).apply(nc.patch);
+                const agent = next["agent"] ?? {};
+                const skills = agent["skills"] ?? [];
+                agent["skills"] = skills.map(s => s.id === nc.patch.skill_id ? { ...s, version: newVersion } : s);
+                next["agent"] = agent;
+                return { candidate: next, skillVersion: newVersion };
+            }
+        }
+        throw new Error(`Unhandled patch target: ${String(nc.target)}`);
+    }
+}
+function applyAgentSkillsPatch(candidate, patch) {
+    const agent = candidate["agent"] ?? {};
+    const skills = (agent["skills"] ?? []).slice();
+    const unbindSet = new Set(patch.unbind);
+    let updated = skills.filter(s => !unbindSet.has(s.id));
+    for (const bind of patch.bind) {
+        const idx = updated.findIndex(s => s.id === bind.id);
+        if (idx >= 0)
+            updated[idx] = bind;
+        else
+            updated.push(bind);
+    }
+    agent["skills"] = updated;
+    candidate["agent"] = agent;
+    return candidate;
 }

package/dist/trace-ai/exp/patch/kn-api-client.d.ts

@@ -0,0 +1,40 @@
+import type { KnObjectTypeDef, KnRelationTypeDef } from "../schemas.js";
+export interface KnApiClient {
+    validateObjectType(knId: string, spec: KnObjectTypeDef): Promise<{
+        valid: boolean;
+        error?: string;
+    }>;
+    addObjectType(knId: string, spec: KnObjectTypeDef): Promise<{
+        concept_id: string;
+    }>;
+    validateRelationType(knId: string, spec: KnRelationTypeDef): Promise<{
+        valid: boolean;
+        error?: string;
+    }>;
+    addRelationType(knId: string, spec: KnRelationTypeDef): Promise<{
+        relation_id: string;
+    }>;
+    objectTypeExists(knId: string, conceptName: string): Promise<boolean>;
+    relationTypeExists(knId: string, conceptName: string): Promise<boolean>;
+}
+export declare class KweaverKnApiClient implements KnApiClient {
+    private baseUrl;
+    private token;
+    constructor(baseUrl: string, token: string);
+    validateObjectType(_knId: string, _spec: KnObjectTypeDef): Promise<{
+        valid: boolean;
+        error?: string;
+    }>;
+    addObjectType(_knId: string, _spec: KnObjectTypeDef): Promise<{
+        concept_id: string;
+    }>;
+    validateRelationType(_knId: string, _spec: KnRelationTypeDef): Promise<{
+        valid: boolean;
+        error?: string;
+    }>;
+    addRelationType(_knId: string, _spec: KnRelationTypeDef): Promise<{
+        relation_id: string;
+    }>;
+    objectTypeExists(_knId: string, _name: string): Promise<boolean>;
+    relationTypeExists(_knId: string, _name: string): Promise<boolean>;
+}

package/dist/trace-ai/exp/patch/kn-api-client.js

@@ -0,0 +1,14 @@
+export class KweaverKnApiClient {
+    baseUrl;
+    token;
+    constructor(baseUrl, token) {
+        this.baseUrl = baseUrl;
+        this.token = token;
+    }
+    async validateObjectType(_knId, _spec) { throw new Error("KweaverKnApiClient not yet implemented"); }
+    async addObjectType(_knId, _spec) { throw new Error("KweaverKnApiClient not yet implemented"); }
+    async validateRelationType(_knId, _spec) { throw new Error("KweaverKnApiClient not yet implemented"); }
+    async addRelationType(_knId, _spec) { throw new Error("KweaverKnApiClient not yet implemented"); }
+    async objectTypeExists(_knId, _name) { throw new Error("KweaverKnApiClient not yet implemented"); }
+    async relationTypeExists(_knId, _name) { throw new Error("KweaverKnApiClient not yet implemented"); }
+}

package/dist/trace-ai/exp/patch/kn.d.ts

@@ -0,0 +1,8 @@
+import type { KnPatch } from "../schemas.js";
+import type { KnApiClient } from "./kn-api-client.js";
+export declare class KnPatcher {
+    private client;
+    private rollback;
+    constructor(client: KnApiClient, workDir: string);
+    apply(patch: KnPatch): Promise<void>;
+}

package/dist/trace-ai/exp/patch/kn.js

@@ -0,0 +1,36 @@
+import { RollbackYaml } from "../exp-store/rollback-yaml.js";
+export class KnPatcher {
+    client;
+    rollback;
+    constructor(client, workDir) {
+        this.client = client;
+        this.rollback = new RollbackYaml(workDir);
+    }
+    async apply(patch) {
+        // Phase 1: dry-run all
+        for (const spec of patch.add_object_types) {
+            const r = await this.client.validateObjectType(patch.kn_id, spec);
+            if (!r.valid)
+                throw new Error(`KN dry-run failed for '${spec.concept_name}': ${r.error ?? "unknown"}`);
+        }
+        for (const spec of patch.add_relation_types) {
+            const r = await this.client.validateRelationType(patch.kn_id, spec);
+            if (!r.valid)
+                throw new Error(`KN dry-run failed for relation '${spec.concept_name}': ${r.error ?? "unknown"}`);
+        }
+        // Phase 2: apply object types (write rollback BEFORE each KN call)
+        for (const spec of patch.add_object_types) {
+            if (await this.client.objectTypeExists(patch.kn_id, spec.concept_name))
+                continue;
+            await this.rollback.appendStep({ op: "remove_object_type", kn_id: patch.kn_id, concept_name: spec.concept_name });
+            await this.client.addObjectType(patch.kn_id, spec);
+        }
+        // Phase 3: apply relation types
+        for (const spec of patch.add_relation_types) {
+            if (await this.client.relationTypeExists(patch.kn_id, spec.concept_name))
+                continue;
+            await this.rollback.appendStep({ op: "remove_relation_type", kn_id: patch.kn_id, concept_name: spec.concept_name });
+            await this.client.addRelationType(patch.kn_id, spec);
+        }
+    }
+}

package/dist/trace-ai/exp/patch/skill-api-client.d.ts

@@ -0,0 +1,17 @@
+export interface SkillApiClient {
+    getSkillContent(skillId: string): Promise<string>;
+    publishSkillVersion(skillId: string, content: string): Promise<{
+        version: string;
+        content: string;
+    }>;
+}
+export declare class KweaverSkillApiClient implements SkillApiClient {
+    private baseUrl;
+    private token;
+    constructor(baseUrl: string, token: string);
+    getSkillContent(_skillId: string): Promise<string>;
+    publishSkillVersion(_skillId: string, _content: string): Promise<{
+        version: string;
+        content: string;
+    }>;
+}

package/dist/trace-ai/exp/patch/skill-api-client.js

@@ -0,0 +1,14 @@
+export class KweaverSkillApiClient {
+    baseUrl;
+    token;
+    constructor(baseUrl, token) {
+        this.baseUrl = baseUrl;
+        this.token = token;
+    }
+    async getSkillContent(_skillId) {
+        throw new Error("KweaverSkillApiClient not yet implemented — use mock in tests");
+    }
+    async publishSkillVersion(_skillId, _content) {
+        throw new Error("KweaverSkillApiClient not yet implemented");
+    }
+}

package/dist/trace-ai/exp/patch/skill-content.d.ts

@@ -0,0 +1,9 @@
+import type { SkillContentPatch } from "../schemas.js";
+import type { SkillApiClient } from "./skill-api-client.js";
+export declare class SkillContentPatcher {
+    private client;
+    constructor(client: SkillApiClient);
+    apply(patch: SkillContentPatch): Promise<{
+        newVersion: string;
+    }>;
+}

package/dist/trace-ai/exp/patch/skill-content.js

@@ -0,0 +1,12 @@
+export class SkillContentPatcher {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    async apply(patch) {
+        const existing = await this.client.getSkillContent(patch.skill_id);
+        const updated = existing.trimEnd() + "\n\n" + patch.append_section;
+        const result = await this.client.publishSkillVersion(patch.skill_id, updated);
+        return { newVersion: result.version };
+    }
+}

package/dist/trace-ai/exp/preflight.d.ts

@@ -0,0 +1,77 @@
+/** A single tool the agent has bound, identified by tool + toolbox. */
+export interface ToolRef {
+    tool_id: string;
+    tool_box_id: string;
+}
+/**
+ * A kn_id/knId tool input whose `map_type` is not `"fixedValue"` — i.e. the KN id
+ * is not a fixed binding. `map_type: "auto"` means the LLM generates the value at
+ * runtime; `"model"`/`"var"` are likewise resolved dynamically. In all of these
+ * the configured `map_value` is NOT what the runtime queries, so a check that
+ * trusts `map_value` would be a false positive.
+ */
+export interface NonFixedKnBinding {
+    input_name: string;
+    map_type: string;
+    /** The configured value — recorded for diagnostics; the runtime ignores it. */
+    map_value: string;
+}
+/**
+ * The material subset of an agent's configuration — the fields that affect eval
+ * outcomes. Both the loop-owned "expected" record and the per-round "actual"
+ * read-back are normalized into this shape so they can be compared field-by-field.
+ */
+export interface AgentFingerprint {
+    agent_id: string;
+    version: string;
+    system_prompt: string;
+    model: string;
+    temperature: number;
+    /** Bound tools, sorted by tool_id. */
+    tools: ToolRef[];
+    /**
+     * KN ids the agent's tools deterministically query — extracted only from
+     * kn_id/knId inputs bound with `map_type: "fixedValue"` (or no map_type, for
+     * older configs). Deduped and sorted.
+     */
+    kn_ids: string[];
+    /**
+     * kn_id/knId inputs bound with a non-fixed `map_type` (e.g. `"auto"`). Empty
+     * when every KN binding is fixed. A non-empty list means the agent does NOT
+     * deterministically query a known KN — the value is resolved at runtime.
+     */
+    non_fixed_kn_bindings: NonFixedKnBinding[];
+}
+/** One field that failed reconciliation. */
+export interface PreflightMismatch {
+    field: string;
+    expected: string;
+    actual: string;
+}
+/** Thrown by preflightCheck when the agent under test does not match expectation. */
+export declare class PreflightMismatchError extends Error {
+    readonly mismatches: PreflightMismatch[];
+    constructor(mismatches: PreflightMismatch[]);
+}
+/**
+ * Build an AgentFingerprint from a raw agent-factory config object (the JSON
+ * returned by `agent get`). Format-specific extraction lives here so the rest of
+ * the loop only ever deals with the normalized fingerprint.
+ */
+export declare function fingerprintFromAgentConfig(agentId: string, version: string, raw: Record<string, unknown>): AgentFingerprint;
+/**
+ * Reconcile the agent under test against expectation. Throws PreflightMismatchError
+ * listing every failed invariant. Checks four invariants:
+ *   1. identity         — agent_id matches
+ *   2. version          — pinned version matches
+ *   3. config           — system_prompt / model / temperature / tools match
+ *   4. question↔patient — agent's KN binding is exactly the eval set's target_kn,
+ *                          AND that binding is deterministic (map_type "fixedValue").
+ *                          A non-fixed binding (e.g. "auto" = model-generated)
+ *                          fails this invariant even if map_value holds the
+ *                          right id, because the runtime does not use map_value.
+ *
+ * Invariant 4 is skipped when evalTargetKn is undefined (eval set has not yet
+ * declared a target_kn) — it cannot check what is not declared.
+ */
+export declare function preflightCheck(expected: AgentFingerprint, actual: AgentFingerprint, evalTargetKn?: string): void;

package/dist/trace-ai/exp/preflight.js

@@ -0,0 +1,148 @@
+// src/trace-ai/exp/preflight.ts
+//
+// Preflight reconciliation: before an eval round measures an agent, verify the
+// agent under test actually matches what the experiment expects. The exp loop
+// has no actuator/read-back of its own, so without this guard a whole round can
+// silently measure the wrong agent, the wrong version, or an agent bound to the
+// wrong KN. This turns those silent failures into a fail-fast at ~1s cost.
+/** Tool-input names that carry a KN id. The platform uses both spellings. */
+const KN_INPUT_NAMES = new Set(["kn_id", "knId"]);
+/** Thrown by preflightCheck when the agent under test does not match expectation. */
+export class PreflightMismatchError extends Error {
+    mismatches;
+    constructor(mismatches) {
+        const lines = mismatches.map(m => `  - ${m.field}: expected ${m.expected}, actual ${m.actual}`);
+        super(`Preflight check failed — agent under test does not match expectation:\n${lines.join("\n")}`);
+        this.name = "PreflightMismatchError";
+        this.mismatches = mismatches;
+    }
+}
+/**
+ * Build an AgentFingerprint from a raw agent-factory config object (the JSON
+ * returned by `agent get`). Format-specific extraction lives here so the rest of
+ * the loop only ever deals with the normalized fingerprint.
+ */
+export function fingerprintFromAgentConfig(agentId, version, raw) {
+    const system_prompt = typeof raw["system_prompt"] === "string" ? raw["system_prompt"] : "";
+    const llms = Array.isArray(raw["llms"]) ? raw["llms"] : [];
+    const defaultLlm = llms.find(l => l["is_default"] === true) ?? llms[0];
+    const llmConfig = defaultLlm?.["llm_config"] ?? {};
+    const model = typeof llmConfig["name"] === "string" ? llmConfig["name"] : "";
+    const temperature = typeof llmConfig["temperature"] === "number" ? llmConfig["temperature"] : NaN;
+    const skills = raw["skills"] ?? {};
+    const rawTools = Array.isArray(skills["tools"]) ? skills["tools"] : [];
+    const tools = rawTools
+        .map(t => ({ tool_id: String(t["tool_id"] ?? ""), tool_box_id: String(t["tool_box_id"] ?? "") }))
+        .sort((a, b) => a.tool_id.localeCompare(b.tool_id));
+    // Separate KN bindings by whether they are deterministic. A kn_id/knId input
+    // contributes to kn_ids only when bound with map_type "fixedValue" (or no
+    // map_type at all — older configs predate the field). Anything else (notably
+    // "auto" = model-generated) is recorded as a non-fixed binding instead, so a
+    // downstream check does not mistake the configured map_value for a real binding.
+    const knSet = new Set();
+    const nonFixed = new Map();
+    for (const t of rawTools) {
+        const inputs = Array.isArray(t["tool_input"]) ? t["tool_input"] : [];
+        for (const inp of inputs) {
+            if (!KN_INPUT_NAMES.has(String(inp["input_name"])))
+                continue;
+            const inputName = String(inp["input_name"]);
+            const mapValue = typeof inp["map_value"] === "string" ? inp["map_value"] : "";
+            const mapType = typeof inp["map_type"] === "string" ? inp["map_type"] : "";
+            const isFixed = mapType === "" || mapType === "fixedValue";
+            if (isFixed) {
+                if (mapValue !== "")
+                    knSet.add(mapValue);
+            }
+            else {
+                nonFixed.set(`${inputName}|${mapType}|${mapValue}`, { input_name: inputName, map_type: mapType, map_value: mapValue });
+            }
+        }
+    }
+    const kn_ids = [...knSet].sort();
+    const non_fixed_kn_bindings = [...nonFixed.values()].sort((a, b) => `${a.input_name}|${a.map_type}`.localeCompare(`${b.input_name}|${b.map_type}`));
+    return { agent_id: agentId, version, system_prompt, model, temperature, tools, kn_ids, non_fixed_kn_bindings };
+}
+/** Render a value for a diff message, truncating long strings so the message stays readable. */
+function repr(value) {
+    if (value.length > 80)
+        return `"${value.slice(0, 60)}…"(${value.length} chars)`;
+    return JSON.stringify(value);
+}
+function toolsEqual(a, b) {
+    if (a.length !== b.length)
+        return false;
+    return a.every((t, i) => {
+        const bt = b[i];
+        return bt !== undefined && t.tool_id === bt.tool_id && t.tool_box_id === bt.tool_box_id;
+    });
+}
+function reprTools(tools) {
+    return tools.length === 0 ? "(none)" : tools.map(t => t.tool_id).join(",");
+}
+/**
+ * Reconcile the agent under test against expectation. Throws PreflightMismatchError
+ * listing every failed invariant. Checks four invariants:
+ *   1. identity         — agent_id matches
+ *   2. version          — pinned version matches
+ *   3. config           — system_prompt / model / temperature / tools match
+ *   4. question↔patient — agent's KN binding is exactly the eval set's target_kn,
+ *                          AND that binding is deterministic (map_type "fixedValue").
+ *                          A non-fixed binding (e.g. "auto" = model-generated)
+ *                          fails this invariant even if map_value holds the
+ *                          right id, because the runtime does not use map_value.
+ *
+ * Invariant 4 is skipped when evalTargetKn is undefined (eval set has not yet
+ * declared a target_kn) — it cannot check what is not declared.
+ */
+export function preflightCheck(expected, actual, evalTargetKn) {
+    const mismatches = [];
+    if (expected.agent_id !== actual.agent_id) {
+        mismatches.push({ field: "agent_id", expected: repr(expected.agent_id), actual: repr(actual.agent_id) });
+    }
+    if (expected.version !== actual.version) {
+        mismatches.push({ field: "version", expected: repr(expected.version), actual: repr(actual.version) });
+    }
+    if (expected.system_prompt !== actual.system_prompt) {
+        mismatches.push({ field: "system_prompt", expected: repr(expected.system_prompt), actual: repr(actual.system_prompt) });
+    }
+    if (expected.model !== actual.model) {
+        mismatches.push({ field: "model", expected: repr(expected.model), actual: repr(actual.model) });
+    }
+    // NaN marks an absent temperature; two absent temperatures are equal.
+    const tempEqual = expected.temperature === actual.temperature
+        || (Number.isNaN(expected.temperature) && Number.isNaN(actual.temperature));
+    if (!tempEqual) {
+        mismatches.push({ field: "temperature", expected: String(expected.temperature), actual: String(actual.temperature) });
+    }
+    if (!toolsEqual(expected.tools, actual.tools)) {
+        mismatches.push({ field: "tools", expected: reprTools(expected.tools), actual: reprTools(actual.tools) });
+    }
+    if (evalTargetKn !== undefined) {
+        const nonFixed = actual.non_fixed_kn_bindings ?? [];
+        if (nonFixed.length > 0) {
+            // The agent's KN id is resolved at runtime (e.g. map_type "auto" =
+            // model-generated), so it does not deterministically query evalTargetKn —
+            // even if map_value happens to hold the right id. This is the exact
+            // false-positive a map_value-only check would miss.
+            const detail = nonFixed.map(b => `${b.input_name}(map_type=${b.map_type})`).join(", ");
+            mismatches.push({
+                field: "kn_binding",
+                expected: repr(evalTargetKn),
+                actual: `non-deterministic — ${detail}: the KN id is resolved at runtime, the configured value is ignored. Set map_type to "fixedValue".`,
+            });
+        }
+        else {
+            const actualKn = actual.kn_ids;
+            if (actualKn.length !== 1 || actualKn[0] !== evalTargetKn) {
+                mismatches.push({
+                    field: "kn_binding",
+                    expected: repr(evalTargetKn),
+                    actual: actualKn.length === 0 ? "(none)" : actualKn.map(repr).join(","),
+                });
+            }
+        }
+    }
+    if (mismatches.length > 0)
+        throw new PreflightMismatchError(mismatches);
+}

package/dist/trace-ai/exp/providers/synthesizer-client.d.ts

@@ -1,14 +1,3 @@
-import type { Mission, NextChange, RoundData } from "../schemas.js";
-export interface SynthesizerInput {
-    mission: Mission;
-    candidateConfig: Record<string, unknown>;
-    prevRound?: RoundData;
-    prevRounds: RoundData[];
-    crossRoundMemoryRef?: string;
-}
-export interface SynthesizerClient {
-    generate(input: SynthesizerInput): Promise<NextChange>;
-}
-export declare class ClaudeCodeSynthesizer implements SynthesizerClient {
-    generate(input: SynthesizerInput): Promise<NextChange>;
-}
+import type { KnContext, SkillContext } from "../schemas.js";
+export declare function buildKnContextPrompt(ctx: KnContext): string;
+export declare function buildSkillContextPrompt(ctx: SkillContext): string;