caik-cli 0.1.1 → 0.6.0

package/dist/chunk-5MHNQAV4.js

@@ -0,0 +1,317 @@
+#!/usr/bin/env node
+import {
+  getTraceCount,
+  getTraces
+} from "./chunk-7AIZTHHZ.js";
+import {
+  runSingleCase
+} from "./chunk-D4IM3YRX.js";
+import {
+  callAnthropic,
+  parseLLMJson
+} from "./chunk-DJJHS7KK.js";
+
+// src/daemon/eval-generator.ts
+import { randomUUID } from "crypto";
+function rowToEvalCase(row) {
+  const value = JSON.parse(row.assertion_value);
+  let assertion;
+  switch (row.assertion_type) {
+    case "must_contain":
+      assertion = { type: "must_contain", pattern: value.pattern };
+      break;
+    case "must_not_contain":
+      assertion = { type: "must_not_contain", pattern: value.pattern };
+      break;
+    case "max_output_length":
+      assertion = { type: "max_output_length", tokens: value.tokens };
+      break;
+    case "llm_judge":
+      assertion = { type: "llm_judge", prompt: value.prompt };
+      break;
+    case "code_check":
+      assertion = { type: "code_check", fn: value.fn };
+      break;
+    // Behavioral types
+    case "behavioral_must_contain":
+      assertion = { type: "behavioral_must_contain", scenario: value.scenario, pattern: value.pattern };
+      break;
+    case "behavioral_must_not_contain":
+      assertion = { type: "behavioral_must_not_contain", scenario: value.scenario, pattern: value.pattern };
+      break;
+    case "behavioral_max_length":
+      assertion = { type: "behavioral_max_length", scenario: value.scenario, tokens: value.tokens };
+      break;
+    case "behavioral_judge":
+      assertion = { type: "behavioral_judge", scenario: value.scenario, criteria: value.criteria };
+      break;
+    case "behavioral_code_check":
+      assertion = { type: "behavioral_code_check", scenario: value.scenario, fn: value.fn };
+      break;
+    default:
+      assertion = { type: "must_contain", pattern: "UNKNOWN" };
+  }
+  return {
+    id: row.id,
+    suiteSlug: row.suite_slug,
+    traceId: row.trace_id ?? void 0,
+    source: row.source,
+    category: row.category,
+    assertion,
+    description: row.description
+  };
+}
+function getEvalSuite(db, slug) {
+  const suiteRow = db.prepare("SELECT * FROM eval_suites WHERE slug = ?").get(slug);
+  if (!suiteRow) return null;
+  const caseRows = db.prepare("SELECT * FROM eval_cases WHERE suite_slug = ? ORDER BY created_at ASC").all(slug);
+  return {
+    slug: suiteRow.slug,
+    version: suiteRow.version,
+    cases: caseRows.map(rowToEvalCase),
+    tpr: suiteRow.tpr ?? void 0,
+    tnr: suiteRow.tnr ?? void 0,
+    validationSampleSize: suiteRow.validation_sample_size ?? void 0,
+    validatedAt: suiteRow.validated_at ?? void 0
+  };
+}
+function shouldGenerateEvals(db, slug) {
+  const counts = getTraceCount(db, slug);
+  if (counts.corrections < 3) return false;
+  const suite = getEvalSuite(db, slug);
+  if (!suite) return true;
+  const processedTraceIds = new Set(
+    suite.cases.filter((c) => c.traceId).map((c) => c.traceId)
+  );
+  const allTraces = getTraces(db, slug, { kind: "correction" });
+  const unprocessed = allTraces.filter((t) => !processedTraceIds.has(t.id));
+  return unprocessed.length > 0;
+}
+var EVAL_GEN_SYSTEM_STRUCTURAL = `You generate eval test cases for Claude Code artifacts (skills, prompts, rules, subagents, commands).
+Given a correction trace \u2014 what the user asked, what the skill produced, how the user corrected it \u2014
+generate ONE eval assertion that would catch this failure pattern.
+
+Prefer deterministic assertions over LLM judges:
+- "must_contain" \u2014 regex pattern that MUST be present in the skill. assertion_value: {"pattern": "regex_here"}
+- "must_not_contain" \u2014 regex pattern that must NOT be present. assertion_value: {"pattern": "regex_here"}
+- "code_check" \u2014 JavaScript function body receiving 'content' argument. assertion_value: {"fn": "return content.includes('...')"}
+- "llm_judge" \u2014 ONLY when pattern-based is impossible. assertion_value: {"prompt": "judge prompt here"}
+
+Return ONLY a JSON object:
+{"assertion_type": "...", "assertion_value": {...}, "category": "...", "description": "One line describing the failure"}`;
+var EVAL_GEN_SYSTEM_BEHAVIORAL = `You generate BEHAVIORAL eval test cases for Claude Code artifacts (skills, prompts, rules, subagents, commands).
+A behavioral eval works by: (1) using the artifact content as a system prompt for an LLM, (2) sending a test scenario as the user message, (3) checking the simulated output.
+
+Given a correction trace, generate ONE behavioral eval assertion with a realistic test scenario.
+
+Available types:
+- "behavioral_must_contain" \u2014 simulated output MUST match regex. assertion_value: {"scenario": "user request that triggers the skill", "pattern": "regex_for_expected_output"}
+- "behavioral_must_not_contain" \u2014 simulated output must NOT match. assertion_value: {"scenario": "...", "pattern": "regex_for_unwanted_output"}
+- "behavioral_max_length" \u2014 output must be under N tokens. assertion_value: {"scenario": "...", "tokens": 200}
+- "behavioral_code_check" \u2014 JS function on output string. assertion_value: {"scenario": "...", "fn": "return content.split('\\n').length < 20"}
+- "behavioral_judge" \u2014 ONLY when pattern-based is impossible. assertion_value: {"scenario": "...", "criteria": "what good output looks like"}
+
+The scenario should be a realistic user request that would expose the weakness found in the correction.
+Prefer behavioral_must_contain and behavioral_must_not_contain over behavioral_judge.
+
+Return ONLY a JSON object:
+{"assertion_type": "behavioral_...", "assertion_value": {...}, "category": "...", "description": "One line describing what the output should/shouldn't do"}`;
+function isBehavioralCorrection(correctionType) {
+  return ["scope_adjusted", "format_changed", "output_modified"].includes(correctionType ?? "");
+}
+async function generateEvalCase(trace, apiKey, model = "claude-haiku-4-5-20251001") {
+  if (!trace.correctionType && !trace.correctionPrompt) return null;
+  const useBehavioral = isBehavioralCorrection(trace.correctionType);
+  const userMessage = `Correction trace:
+- Skill slug: ${trace.slug}
+- Correction type: ${trace.correctionType ?? "unknown"}
+- User's correction message: ${trace.correctionPrompt ?? "(no message captured)"}
+- Tool that was used: ${trace.toolName}
+- Tool input: ${JSON.stringify(trace.toolInput, null, 2).slice(0, 2e3)}
+- Tool response: ${JSON.stringify(trace.toolResponse, null, 2).slice(0, 2e3)}
+
+Generate an eval assertion that would detect this failure pattern.`;
+  try {
+    const result = await callAnthropic(apiKey, {
+      model,
+      system: useBehavioral ? EVAL_GEN_SYSTEM_BEHAVIORAL : EVAL_GEN_SYSTEM_STRUCTURAL,
+      userMessage,
+      maxTokens: 512
+    });
+    const parsed = parseLLMJson(result.text);
+    if (!parsed.assertion_type || !parsed.assertion_value || !parsed.category || !parsed.description) {
+      return null;
+    }
+    let assertion;
+    switch (parsed.assertion_type) {
+      case "must_contain":
+        assertion = { type: "must_contain", pattern: String(parsed.assertion_value.pattern ?? "") };
+        break;
+      case "must_not_contain":
+        assertion = { type: "must_not_contain", pattern: String(parsed.assertion_value.pattern ?? "") };
+        break;
+      case "code_check":
+        assertion = { type: "code_check", fn: String(parsed.assertion_value.fn ?? "") };
+        break;
+      case "llm_judge":
+        assertion = { type: "llm_judge", prompt: String(parsed.assertion_value.prompt ?? "") };
+        break;
+      case "max_output_length":
+        assertion = { type: "max_output_length", tokens: Number(parsed.assertion_value.tokens ?? 500) };
+        break;
+      // Behavioral types
+      case "behavioral_must_contain":
+        assertion = { type: "behavioral_must_contain", scenario: String(parsed.assertion_value.scenario ?? ""), pattern: String(parsed.assertion_value.pattern ?? "") };
+        break;
+      case "behavioral_must_not_contain":
+        assertion = { type: "behavioral_must_not_contain", scenario: String(parsed.assertion_value.scenario ?? ""), pattern: String(parsed.assertion_value.pattern ?? "") };
+        break;
+      case "behavioral_max_length":
+        assertion = { type: "behavioral_max_length", scenario: String(parsed.assertion_value.scenario ?? ""), tokens: Number(parsed.assertion_value.tokens ?? 500) };
+        break;
+      case "behavioral_judge":
+        assertion = { type: "behavioral_judge", scenario: String(parsed.assertion_value.scenario ?? ""), criteria: String(parsed.assertion_value.criteria ?? "") };
+        break;
+      case "behavioral_code_check":
+        assertion = { type: "behavioral_code_check", scenario: String(parsed.assertion_value.scenario ?? ""), fn: String(parsed.assertion_value.fn ?? "") };
+        break;
+      default:
+        return null;
+    }
+    return {
+      evalCase: {
+        traceId: trace.id,
+        source: "trace",
+        category: parsed.category,
+        assertion,
+        description: parsed.description
+      },
+      llmCalls: 1
+    };
+  } catch {
+    return null;
+  }
+}
+function serializeAssertionValue(assertion) {
+  switch (assertion.type) {
+    case "must_contain":
+    case "must_not_contain":
+      return JSON.stringify({ pattern: assertion.pattern });
+    case "code_check":
+      return JSON.stringify({ fn: assertion.fn });
+    case "llm_judge":
+      return JSON.stringify({ prompt: assertion.prompt });
+    case "max_output_length":
+      return JSON.stringify({ tokens: assertion.tokens });
+    case "behavioral_must_contain":
+    case "behavioral_must_not_contain":
+      return JSON.stringify({ scenario: assertion.scenario, pattern: assertion.pattern });
+    case "behavioral_code_check":
+      return JSON.stringify({ scenario: assertion.scenario, fn: assertion.fn });
+    case "behavioral_judge":
+      return JSON.stringify({ scenario: assertion.scenario, criteria: assertion.criteria });
+    case "behavioral_max_length":
+      return JSON.stringify({ scenario: assertion.scenario, tokens: assertion.tokens });
+  }
+}
+var MAX_EVAL_CASES = 20;
+async function generateEvalSuite(db, slug, apiKey, model = "claude-haiku-4-5-20251001") {
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  let totalLLMCalls = 0;
+  let suite = getEvalSuite(db, slug);
+  if (!suite) {
+    db.prepare(
+      "INSERT INTO eval_suites (slug, version, created_at, updated_at) VALUES (?, 1, ?, ?)"
+    ).run(slug, now, now);
+    suite = { slug, version: 1, cases: [] };
+  }
+  const processedTraceIds = new Set(
+    suite.cases.filter((c) => c.traceId).map((c) => c.traceId)
+  );
+  const correctionTraces = getTraces(db, slug, { kind: "correction" });
+  const unprocessed = correctionTraces.filter((t) => !processedTraceIds.has(t.id));
+  const existingCategories = new Set(suite.cases.map((c) => c.category));
+  for (const trace of unprocessed) {
+    if (suite.cases.length >= MAX_EVAL_CASES) break;
+    if (totalLLMCalls >= MAX_EVAL_CASES) break;
+    const result = await generateEvalCase(trace, apiKey, model);
+    if (!result) continue;
+    totalLLMCalls += result.llmCalls;
+    if (existingCategories.has(result.evalCase.category)) continue;
+    existingCategories.add(result.evalCase.category);
+    const caseId = randomUUID();
+    const assertionValue = serializeAssertionValue(result.evalCase.assertion);
+    db.prepare(
+      `INSERT INTO eval_cases (id, suite_slug, trace_id, source, category, assertion_type, assertion_value, description, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    ).run(caseId, slug, trace.id, "trace", result.evalCase.category, result.evalCase.assertion.type, assertionValue, result.evalCase.description, now);
+    suite.cases.push({
+      id: caseId,
+      suiteSlug: slug,
+      traceId: trace.id,
+      source: "trace",
+      category: result.evalCase.category,
+      assertion: result.evalCase.assertion,
+      description: result.evalCase.description
+    });
+  }
+  if (suite.cases.length > MAX_EVAL_CASES) {
+    const toRemove = suite.cases.slice(0, suite.cases.length - MAX_EVAL_CASES);
+    for (const c of toRemove) {
+      db.prepare("DELETE FROM eval_cases WHERE id = ?").run(c.id);
+    }
+    suite.cases = suite.cases.slice(suite.cases.length - MAX_EVAL_CASES);
+  }
+  db.prepare("UPDATE eval_suites SET version = version + 1, updated_at = ? WHERE slug = ?").run(now, slug);
+  suite.version++;
+  return { suite, llmCalls: totalLLMCalls };
+}
+async function validateJudges(db, slug, apiKey, skillContent, simulationModel) {
+  const suite = getEvalSuite(db, slug);
+  if (!suite || suite.cases.length === 0) {
+    return { tpr: void 0, tnr: void 0, sampleSize: 0 };
+  }
+  const corrections = getTraces(db, slug, { kind: "correction" });
+  const simulationOptions = simulationModel ? { model: simulationModel, cache: /* @__PURE__ */ new Map() } : void 0;
+  let truePositives = 0;
+  for (const trace of corrections) {
+    const badContent = JSON.stringify(trace.toolResponse);
+    let anyFailed = false;
+    for (const evalCase of suite.cases) {
+      const result = await runSingleCase(badContent, evalCase, apiKey, simulationOptions);
+      if (!result.passed) {
+        anyFailed = true;
+        break;
+      }
+    }
+    if (anyFailed) truePositives++;
+  }
+  let tnr;
+  if (skillContent) {
+    let allPassed = true;
+    for (const evalCase of suite.cases) {
+      const result = await runSingleCase(skillContent, evalCase, apiKey, simulationOptions);
+      if (!result.passed) {
+        allPassed = false;
+        break;
+      }
+    }
+    tnr = allPassed ? 1 : 0;
+  }
+  const tpr = corrections.length > 0 ? truePositives / corrections.length : void 0;
+  const sampleSize = corrections.length + (skillContent ? 1 : 0);
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  db.prepare(
+    "UPDATE eval_suites SET tpr = ?, tnr = ?, validation_sample_size = ?, validated_at = ? WHERE slug = ?"
+  ).run(tpr ?? null, tnr ?? null, sampleSize, now, slug);
+  return { tpr, tnr, sampleSize };
+}
+
+export {
+  getEvalSuite,
+  shouldGenerateEvals,
+  isBehavioralCorrection,
+  generateEvalCase,
+  generateEvalSuite,
+  validateJudges
+};

package/dist/chunk-7AIZTHHZ.js

@@ -0,0 +1,152 @@
+#!/usr/bin/env node
+
+// src/daemon/trace-store.ts
+import { randomUUID } from "crypto";
+function bufferToolCall(db, sessionId, toolName, toolInput, toolResponse, success, slug) {
+  db.prepare(
+    `INSERT INTO session_buffer (session_id, type, timestamp, tool_name, tool_input, tool_response, success, slug)
+     VALUES (?, 'tool_call', ?, ?, ?, ?, ?, ?)`
+  ).run(
+    sessionId,
+    (/* @__PURE__ */ new Date()).toISOString(),
+    toolName,
+    JSON.stringify(toolInput),
+    JSON.stringify(toolResponse),
+    success ? 1 : 0,
+    slug
+  );
+}
+function bufferPrompt(db, sessionId, prompt, correctionType, slug) {
+  const type = correctionType ? "correction" : "prompt";
+  db.prepare(
+    `INSERT INTO session_buffer (session_id, type, timestamp, prompt, correction_type, slug)
+     VALUES (?, ?, ?, ?, ?, ?)`
+  ).run(sessionId, type, (/* @__PURE__ */ new Date()).toISOString(), prompt, correctionType ?? null, slug ?? null);
+}
+function buildTraces(db, sessionId) {
+  const rows = db.prepare("SELECT * FROM session_buffer WHERE session_id = ? ORDER BY id ASC").all(sessionId);
+  const traces = [];
+  const lastToolCall = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    if (row.type === "tool_call" && row.slug) {
+      lastToolCall.set(row.slug, row);
+    }
+    if (row.type === "correction" && row.correction_type) {
+      const slug = row.slug ?? "unknown";
+      const toolCall = lastToolCall.get(slug);
+      traces.push({
+        id: randomUUID(),
+        sessionId,
+        slug,
+        timestamp: row.timestamp,
+        kind: "correction",
+        toolName: toolCall?.tool_name ?? "unknown",
+        toolInput: toolCall?.tool_input ? JSON.parse(toolCall.tool_input) : {},
+        toolResponse: toolCall?.tool_response ? JSON.parse(toolCall.tool_response) : {},
+        correctionType: row.correction_type,
+        correctionPrompt: row.prompt ?? void 0
+      });
+    }
+  }
+  const successfulCalls = rows.filter(
+    (r) => r.type === "tool_call" && r.success === 1 && r.slug
+  );
+  for (const row of successfulCalls) {
+    if (Math.random() < 0.2) {
+      traces.push({
+        id: randomUUID(),
+        sessionId,
+        slug: row.slug,
+        timestamp: row.timestamp,
+        kind: "success",
+        toolName: row.tool_name ?? "unknown",
+        toolInput: row.tool_input ? JSON.parse(row.tool_input) : {},
+        toolResponse: row.tool_response ? JSON.parse(row.tool_response) : {}
+      });
+    }
+  }
+  return traces;
+}
+function persistTraces(db, traces) {
+  const stmt = db.prepare(
+    `INSERT INTO traces (id, session_id, slug, timestamp, kind, tool_name, tool_input, tool_response,
+     correction_type, correction_prompt, skill_content_hash, created_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+  );
+  const insertMany = db.transaction((items) => {
+    for (const t of items) {
+      stmt.run(
+        t.id,
+        t.sessionId,
+        t.slug,
+        t.timestamp,
+        t.kind,
+        t.toolName,
+        JSON.stringify(t.toolInput),
+        JSON.stringify(t.toolResponse),
+        t.correctionType ?? null,
+        t.correctionPrompt ?? null,
+        t.skillContentHash ?? null,
+        (/* @__PURE__ */ new Date()).toISOString()
+      );
+    }
+  });
+  insertMany(traces);
+}
+function rowToTrace(row) {
+  return {
+    id: row.id,
+    sessionId: row.session_id,
+    slug: row.slug,
+    timestamp: row.timestamp,
+    kind: row.kind,
+    toolName: row.tool_name,
+    toolInput: row.tool_input ? JSON.parse(row.tool_input) : {},
+    toolResponse: row.tool_response ? JSON.parse(row.tool_response) : {},
+    correctionType: row.correction_type ?? void 0,
+    correctionPrompt: row.correction_prompt ?? void 0,
+    skillContentHash: row.skill_content_hash ?? void 0
+  };
+}
+function getTraces(db, slug, opts) {
+  let sql = "SELECT * FROM traces WHERE slug = ?";
+  const params = [slug];
+  if (opts?.kind) {
+    sql += " AND kind = ?";
+    params.push(opts.kind);
+  }
+  sql += " ORDER BY timestamp DESC";
+  if (opts?.limit) {
+    sql += " LIMIT ?";
+    params.push(opts.limit);
+  }
+  const rows = db.prepare(sql).all(...params);
+  return rows.map(rowToTrace);
+}
+function getTraceCount(db, slug) {
+  const row = db.prepare(
+    `SELECT
+        COUNT(*) as total,
+        SUM(CASE WHEN kind = 'correction' THEN 1 ELSE 0 END) as corrections,
+        SUM(CASE WHEN kind = 'success' THEN 1 ELSE 0 END) as successes
+       FROM traces WHERE slug = ?`
+  ).get(slug);
+  return {
+    total: row.total,
+    corrections: row.corrections ?? 0,
+    successes: row.successes ?? 0
+  };
+}
+function clearSessionBuffer(db, sessionId) {
+  db.prepare("DELETE FROM session_buffer WHERE session_id = ?").run(sessionId);
+}
+
+export {
+  bufferToolCall,
+  bufferPrompt,
+  buildTraces,
+  persistTraces,
+  getTraces,
+  getTraceCount,
+  clearSessionBuffer
+};

package/dist/chunk-D4IM3YRX.js

@@ -0,0 +1,166 @@
+#!/usr/bin/env node
+import {
+  callAnthropic
+} from "./chunk-DJJHS7KK.js";
+
+// src/daemon/eval-runner.ts
+import { createHash } from "crypto";
+function simCacheKey(skillContent, scenario) {
+  return createHash("sha256").update(skillContent + "\0" + scenario).digest("hex").slice(0, 16);
+}
+async function simulateSkill(skillContent, scenario, apiKey, simOpts) {
+  const key = simCacheKey(skillContent, scenario);
+  const cached = simOpts.cache.get(key);
+  if (cached !== void 0) return cached;
+  const result = await callAnthropic(apiKey, {
+    model: simOpts.model,
+    system: skillContent,
+    userMessage: scenario,
+    maxTokens: 2048
+  });
+  simOpts.cache.set(key, result.text);
+  return result.text;
+}
+function checkPatternAssertion(content, type, pattern) {
+  try {
+    let flags = "";
+    let cleanPattern = pattern;
+    if (cleanPattern.startsWith("(?i)")) {
+      flags = "i";
+      cleanPattern = cleanPattern.slice(4);
+    }
+    const regex = new RegExp(cleanPattern, flags);
+    const found = regex.test(content);
+    if (type === "must_contain") {
+      return found ? { passed: true, reason: `Pattern "${pattern}" found` } : { passed: false, reason: `Pattern "${pattern}" not found in content` };
+    } else {
+      return found ? { passed: false, reason: `Pattern "${pattern}" found in content (should be absent)` } : { passed: true, reason: `Pattern "${pattern}" correctly absent` };
+    }
+  } catch (err) {
+    return {
+      passed: false,
+      reason: `Invalid regex pattern: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+}
+function checkCodeAssertion(content, fn) {
+  try {
+    const check = new Function("content", fn);
+    const result = check(content);
+    return result ? { passed: true, reason: "Code check passed" } : { passed: false, reason: "Code check returned false" };
+  } catch (err) {
+    return {
+      passed: false,
+      reason: `Code check error: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+}
+async function checkLLMJudge(skillContent, judgePrompt, apiKey) {
+  try {
+    const result = await callAnthropic(apiKey, {
+      model: "claude-haiku-4-5-20251001",
+      system: `You are an eval judge for Claude Code skills. Given a skill and a judge prompt, determine if the skill PASSES or FAILS the criteria. Respond with ONLY a JSON object: {"passed": true/false, "reason": "brief explanation"}`,
+      userMessage: `Skill content:
+<skill>
+${skillContent}
+</skill>
+
+Judge criteria: ${judgePrompt}`,
+      maxTokens: 256
+    });
+    const parsed = JSON.parse(result.text);
+    return parsed;
+  } catch (err) {
+    return {
+      passed: false,
+      reason: `LLM judge error: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+}
+async function runSingleCase(skillContent, evalCase, apiKey, simOpts) {
+  const assertion = evalCase.assertion;
+  switch (assertion.type) {
+    // ── Structural (check SKILL.md text directly) ──────────────────
+    case "must_contain":
+      return checkPatternAssertion(skillContent, "must_contain", assertion.pattern);
+    case "must_not_contain":
+      return checkPatternAssertion(skillContent, "must_not_contain", assertion.pattern);
+    case "code_check":
+      return checkCodeAssertion(skillContent, assertion.fn);
+    case "llm_judge":
+      return checkLLMJudge(skillContent, assertion.prompt, apiKey);
+    case "max_output_length":
+      return {
+        passed: skillContent.length <= assertion.tokens * 4,
+        reason: skillContent.length <= assertion.tokens * 4 ? `Content length ${skillContent.length} within limit` : `Content length ${skillContent.length} exceeds token limit ${assertion.tokens}`
+      };
+    // ── Behavioral (simulate skill, check output) ──────────────────
+    case "behavioral_must_contain": {
+      if (!simOpts) return { passed: false, reason: "No simulation config for behavioral eval" };
+      const output = await simulateSkill(skillContent, assertion.scenario, apiKey, simOpts);
+      return checkPatternAssertion(output, "must_contain", assertion.pattern);
+    }
+    case "behavioral_must_not_contain": {
+      if (!simOpts) return { passed: false, reason: "No simulation config for behavioral eval" };
+      const output = await simulateSkill(skillContent, assertion.scenario, apiKey, simOpts);
+      return checkPatternAssertion(output, "must_not_contain", assertion.pattern);
+    }
+    case "behavioral_max_length": {
+      if (!simOpts) return { passed: false, reason: "No simulation config for behavioral eval" };
+      const output = await simulateSkill(skillContent, assertion.scenario, apiKey, simOpts);
+      const tokenEstimate = Math.round(output.length / 4);
+      return {
+        passed: tokenEstimate <= assertion.tokens,
+        reason: tokenEstimate <= assertion.tokens ? `Simulated output ~${tokenEstimate} tokens, within limit ${assertion.tokens}` : `Simulated output ~${tokenEstimate} tokens, exceeds limit ${assertion.tokens}`
+      };
+    }
+    case "behavioral_judge": {
+      if (!simOpts) return { passed: false, reason: "No simulation config for behavioral eval" };
+      const output = await simulateSkill(skillContent, assertion.scenario, apiKey, simOpts);
+      return checkLLMJudge(output, assertion.criteria, apiKey);
+    }
+    case "behavioral_code_check": {
+      if (!simOpts) return { passed: false, reason: "No simulation config for behavioral eval" };
+      const output = await simulateSkill(skillContent, assertion.scenario, apiKey, simOpts);
+      return checkCodeAssertion(output, assertion.fn);
+    }
+    default:
+      return { passed: false, reason: `Unknown assertion type` };
+  }
+}
+async function runEvalSuite(skillContent, suite, baselineContent, apiKey, simulationModel) {
+  if (suite.cases.length === 0) {
+    return {
+      passRate: 1,
+      passCount: 0,
+      totalCases: 0,
+      failedCaseIds: [],
+      lengthRatio: baselineContent.length > 0 ? skillContent.length / baselineContent.length : 1
+    };
+  }
+  let passCount = 0;
+  const failedCaseIds = [];
+  const simOpts = simulationModel ? { model: simulationModel, cache: /* @__PURE__ */ new Map() } : void 0;
+  for (const evalCase of suite.cases) {
+    const result = await runSingleCase(skillContent, evalCase, apiKey, simOpts);
+    if (result.passed) {
+      passCount++;
+    } else {
+      failedCaseIds.push(evalCase.id);
+    }
+  }
+  return {
+    passRate: passCount / suite.cases.length,
+    passCount,
+    totalCases: suite.cases.length,
+    failedCaseIds,
+    lengthRatio: baselineContent.length > 0 ? skillContent.length / baselineContent.length : 1
+  };
+}
+
+export {
+  checkPatternAssertion,
+  checkCodeAssertion,
+  runSingleCase,
+  runEvalSuite
+};

package/dist/chunk-DJJHS7KK.js

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+
+// src/daemon/llm.ts
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+var ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages";
+function getAnthropicApiKey() {
+  if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY;
+  try {
+    const configPath = join(homedir(), ".caik", "config.json");
+    if (existsSync(configPath)) {
+      const raw = JSON.parse(readFileSync(configPath, "utf-8"));
+      if (typeof raw.anthropicApiKey === "string") return raw.anthropicApiKey;
+    }
+  } catch {
+  }
+  return void 0;
+}
+async function callAnthropic(apiKey, opts) {
+  const body = JSON.stringify({
+    model: opts.model,
+    max_tokens: opts.maxTokens ?? 4096,
+    system: opts.system,
+    messages: [{ role: "user", content: opts.userMessage }]
+  });
+  const res = await fetch(ANTHROPIC_API_URL, {
+    method: "POST",
+    headers: {
+      "x-api-key": apiKey,
+      "anthropic-version": "2023-06-01",
+      "content-type": "application/json"
+    },
+    body
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new Error(`Anthropic API ${res.status}: ${text}`);
+  }
+  const data = await res.json();
+  const textBlock = data.content.find((b) => b.type === "text");
+  if (!textBlock?.text) throw new Error("No text in Anthropic response");
+  return {
+    text: textBlock.text,
+    inputTokens: data.usage?.input_tokens ?? 0,
+    outputTokens: data.usage?.output_tokens ?? 0
+  };
+}
+function parseLLMJson(text) {
+  let cleaned = text.trim();
+  if (cleaned.startsWith("```")) {
+    cleaned = cleaned.replace(/^```[a-z]*\n?/i, "");
+    cleaned = cleaned.replace(/\n?```\s*$/, "");
+  }
+  return JSON.parse(cleaned);
+}
+
+export {
+  getAnthropicApiKey,
+  callAnthropic,
+  parseLLMJson
+};