kc-beta 0.1.0

package/src/agent/tools/rule-catalog.js

@@ -0,0 +1,113 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BaseTool, ToolResult } from "./base.js";
+
+const REQUIRED_FIELDS = new Set(["id", "source_ref", "description"]);
+const RECOMMENDED_FIELDS = new Set(["falsifiability_statement", "test_case_stub", "applicable_sections"]);
+
+/**
+ * CRUD on the rule registry with schema enforcement.
+ * Enforces required fields (id, source_ref, description) on create/update.
+ * Persists to rules/catalog.json.
+ */
+export class RuleCatalogTool extends BaseTool {
+  constructor(workspace) {
+    super();
+    this._workspace = workspace;
+    this._catalogPath = path.join(workspace.cwd, "rules", "catalog.json");
+  }
+
+  get name() { return "rule_catalog"; }
+  get description() {
+    return (
+      "CRUD on the rule registry. Operations: create, read, update, delete, list. " +
+      "Enforces required fields (id, source_ref, description) on create/update. " +
+      "Persists to rules/catalog.json."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        operation: { type: "string", enum: ["create", "read", "update", "delete", "list"], description: "Operation to perform" },
+        rule_id: { type: "string", description: "Rule ID (for read/update/delete)" },
+        data: { type: "object", description: "Rule data (for create/update). Must include: id, source_ref, description" },
+      },
+      required: ["operation"],
+    };
+  }
+
+  async execute(input) {
+    const op = input.operation || "";
+    const ruleId = input.rule_id || "";
+    const data = input.data || {};
+
+    if (op === "list") return this._list();
+    if (op === "read") return this._read(ruleId || data.id || "");
+    if (op === "create") return this._create(data);
+    if (op === "update") return this._update(ruleId || data.id || "", data);
+    if (op === "delete") return this._delete(ruleId || data.id || "");
+    return new ToolResult(`Unknown operation: ${op}`, true);
+  }
+
+  _load() {
+    if (!fs.existsSync(this._catalogPath)) return [];
+    try {
+      const data = JSON.parse(fs.readFileSync(this._catalogPath, "utf-8"));
+      return Array.isArray(data) ? data : [];
+    } catch { return []; }
+  }
+
+  _save(rules) {
+    fs.mkdirSync(path.dirname(this._catalogPath), { recursive: true });
+    fs.writeFileSync(this._catalogPath, JSON.stringify(rules, null, 2), "utf-8");
+  }
+
+  _list() {
+    const rules = this._load();
+    if (rules.length === 0) return new ToolResult("Catalog is empty. Use create to add rules.");
+    const summary = rules.map((r) => `- ${r.id || "?"}: ${(r.description || "(no description)").slice(0, 80)}`);
+    return new ToolResult(`${rules.length} rule(s):\n${summary.join("\n")}`);
+  }
+
+  _read(ruleId) {
+    if (!ruleId) return new ToolResult("rule_id required for read", true);
+    const rule = this._load().find((r) => r.id === ruleId);
+    if (!rule) return new ToolResult(`Rule not found: ${ruleId}`, true);
+    return new ToolResult(JSON.stringify(rule, null, 2));
+  }
+
+  _create(data) {
+    const missing = [...REQUIRED_FIELDS].filter((f) => !(f in data));
+    if (missing.length > 0) return new ToolResult(`Missing required fields: ${missing.join(", ")}`, true);
+    const rules = this._load();
+    if (rules.some((r) => r.id === data.id)) return new ToolResult(`Rule already exists: ${data.id}. Use update.`, true);
+    const warnings = [...RECOMMENDED_FIELDS].filter((f) => !(f in data));
+    rules.push(data);
+    this._save(rules);
+    let msg = `Created rule: ${data.id}`;
+    if (warnings.length > 0) msg += `\nMissing recommended fields: ${warnings.join(", ")}`;
+    return new ToolResult(msg);
+  }
+
+  _update(ruleId, data) {
+    if (!ruleId) return new ToolResult("rule_id required for update", true);
+    const rules = this._load();
+    const idx = rules.findIndex((r) => r.id === ruleId);
+    if (idx < 0) return new ToolResult(`Rule not found: ${ruleId}`, true);
+    Object.assign(rules[idx], data);
+    rules[idx].id = ruleId;
+    this._save(rules);
+    return new ToolResult(`Updated rule: ${ruleId}`);
+  }
+
+  _delete(ruleId) {
+    if (!ruleId) return new ToolResult("rule_id required for delete", true);
+    const rules = this._load();
+    const newRules = rules.filter((r) => r.id !== ruleId);
+    if (newRules.length === rules.length) return new ToolResult(`Rule not found: ${ruleId}`, true);
+    this._save(newRules);
+    return new ToolResult(`Deleted rule: ${ruleId}`);
+  }
+}

package/src/agent/tools/sandbox-exec.js

@@ -0,0 +1,106 @@
+import { spawn } from "node:child_process";
+import { BaseTool, ToolResult } from "./base.js";
+
+const MAX_OUTPUT = 10_000;
+
+/**
+ * Execute shell commands in the workspace directory.
+ * Uses child_process.spawn so pipes, redirects, && all work.
+ * Output (stdout + stderr combined) is capped at 10K chars.
+ */
+export class SandboxExecTool extends BaseTool {
+  /**
+   * @param {import('../workspace.js').Workspace} workspace
+   * @param {number} [timeout=30]
+   */
+  constructor(workspace, timeout = 30) {
+    super();
+    this._workspace = workspace;
+    this._timeout = timeout;
+  }
+
+  get name() { return "sandbox_exec"; }
+
+  get description() {
+    return (
+      "Execute a shell command in the workspace directory. " +
+      "Use for running scripts, installing packages, listing files, etc. " +
+      "Pipes, redirects, and chained commands (&&) are supported."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        command: {
+          type: "string",
+          description: "The shell command to execute (e.g. 'python script.py', 'ls -la')",
+        },
+      },
+      required: ["command"],
+    };
+  }
+
+  async execute(input) {
+    const command = input.command || "";
+    if (!command.trim()) {
+      return new ToolResult("No command provided", true);
+    }
+
+    try {
+      const { output, code } = await this._run(command);
+      let result = output;
+      if (result.length > MAX_OUTPUT) {
+        result = result.slice(0, MAX_OUTPUT) + "\n[truncated]";
+      }
+      if (code !== 0) {
+        result += `\n[exit code: ${code}]`;
+      }
+      return new ToolResult(result, code !== 0);
+    } catch (err) {
+      if (err.message === "timeout") {
+        return new ToolResult(`Command timed out after ${this._timeout}s`, true);
+      }
+      return new ToolResult(`Execution error: ${err.message}`, true);
+    }
+  }
+
+  /**
+   * @param {string} command
+   * @returns {Promise<{output: string, code: number}>}
+   */
+  _run(command) {
+    return new Promise((resolve, reject) => {
+      const controller = new AbortController();
+      const proc = spawn("sh", ["-c", command], {
+        cwd: this._workspace.cwd,
+        stdio: ["ignore", "pipe", "pipe"],
+        signal: controller.signal,
+      });
+
+      let output = "";
+      proc.stdout.on("data", (d) => { output += d.toString(); });
+      proc.stderr.on("data", (d) => { output += d.toString(); });
+
+      const timer = setTimeout(() => {
+        controller.abort();
+        reject(new Error("timeout"));
+      }, this._timeout * 1000);
+
+      proc.on("close", (code) => {
+        clearTimeout(timer);
+        resolve({ output, code: code ?? 1 });
+      });
+
+      proc.on("error", (err) => {
+        clearTimeout(timer);
+        if (err.name === "AbortError") {
+          reject(new Error("timeout"));
+        } else {
+          reject(err);
+        }
+      });
+    });
+  }
+}

package/src/agent/tools/tier-downgrade.js

@@ -0,0 +1,114 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BaseTool, ToolResult } from "./base.js";
+
+/**
+ * Test a workflow step at a lower worker LLM tier.
+ * Compares accuracy at target tier against current baseline.
+ * Used during distillation to find the minimum viable tier.
+ */
+export class TierDowngradeTool extends BaseTool {
+  constructor(workspace, workerLlm) {
+    super();
+    this._workspace = workspace;
+    this._worker = workerLlm;
+  }
+
+  get name() { return "tier_downgrade"; }
+  get description() {
+    return (
+      "Test a workflow step at a lower worker LLM tier. Compares accuracy " +
+      "against baseline. Use during distillation to find the cheapest tier " +
+      "that meets accuracy threshold."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        rule_id: { type: "string", description: "Rule ID being tested" },
+        prompt: { type: "string", description: "The extraction/judgment prompt to test" },
+        test_inputs: { type: "array", items: { type: "string" }, description: "List of document text chunks to test" },
+        expected_outputs: { type: "array", items: { type: "string" }, description: "Expected correct outputs" },
+        current_tier: { type: "string", description: "Current tier (baseline)" },
+        target_tier: { type: "string", description: "Lower tier to test" },
+      },
+      required: ["rule_id", "prompt", "test_inputs", "expected_outputs", "current_tier", "target_tier"],
+    };
+  }
+
+  async execute(input) {
+    const ruleId = input.rule_id || "";
+    const prompt = input.prompt || "";
+    const testInputs = input.test_inputs || [];
+    const expected = input.expected_outputs || [];
+    const currentTier = input.current_tier || "tier1";
+    const targetTier = input.target_tier || "tier2";
+
+    if (!testInputs.length || !expected.length || testInputs.length !== expected.length) {
+      return new ToolResult("test_inputs and expected_outputs must be non-empty and same length", true);
+    }
+
+    const currentResults = await this._runTier(currentTier, prompt, testInputs);
+    const targetResults = await this._runTier(targetTier, prompt, testInputs);
+
+    if (!currentResults) return new ToolResult(`Failed to run at ${currentTier}`, true);
+    if (!targetResults) return new ToolResult(`Failed to run at ${targetTier}`, true);
+
+    const currentAcc = this._accuracy(currentResults, expected);
+    const targetAcc = this._accuracy(targetResults, expected);
+    const delta = currentAcc - targetAcc;
+
+    // Read threshold from .env
+    let threshold = 0.9;
+    const envPath = path.join(this._workspace.cwd, ".env");
+    if (fs.existsSync(envPath)) {
+      for (const line of fs.readFileSync(envPath, "utf-8").split("\n")) {
+        if (line.startsWith("WORKFLOW_ACCURACY=")) {
+          try { threshold = parseFloat(line.split("=")[1].trim()); }
+          catch { /* ignore */ }
+        }
+      }
+    }
+
+    const recommend = targetAcc >= threshold && delta <= 0.05 ? "downgrade" : "keep_current";
+
+    const report = {
+      rule_id: ruleId, current_tier: currentTier, target_tier: targetTier,
+      current_accuracy: Math.round(currentAcc * 1000) / 1000,
+      target_accuracy: Math.round(targetAcc * 1000) / 1000,
+      accuracy_delta: Math.round(delta * 1000) / 1000,
+      threshold, recommendation: recommend, test_count: testInputs.length,
+    };
+    return new ToolResult(JSON.stringify(report, null, 2));
+  }
+
+  async _runTier(tier, prompt, inputs) {
+    const results = [];
+    for (const text of inputs) {
+      const fullPrompt = `${prompt}\n\nDocument text:\n${text}`;
+      const result = await this._worker.execute({ tier, prompt: fullPrompt, max_tokens: 2048 });
+      if (result.isError) return null;
+      try {
+        const data = JSON.parse(result.content);
+        results.push(data.response || "");
+      } catch {
+        results.push(result.content);
+      }
+    }
+    return results;
+  }
+
+  _accuracy(outputs, expected) {
+    if (!expected.length) return 0;
+    let matches = 0;
+    for (let i = 0; i < expected.length; i++) {
+      if (outputs[i] && expected[i].trim().toLowerCase() !== "" &&
+          outputs[i].toLowerCase().includes(expected[i].trim().toLowerCase())) {
+        matches++;
+      }
+    }
+    return matches / expected.length;
+  }
+}

package/src/agent/tools/worker-llm-call.js

@@ -0,0 +1,109 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BaseTool, ToolResult } from "./base.js";
+
+/**
+ * Call a worker LLM at a specified tier for verification tasks.
+ * Reads tier-to-model mapping from workspace .env. Routes through
+ * the configured API provider.
+ */
+export class WorkerLLMCallTool extends BaseTool {
+  constructor(workspace, { apiKey, baseUrl } = {}) {
+    super();
+    this._workspace = workspace;
+    this._apiKey = apiKey || "";
+    this._baseUrl = (baseUrl || "https://api.siliconflow.cn/v1").replace(/\/+$/, "");
+    this._tierModels = {};
+    this._loadTiers();
+  }
+
+  _loadTiers() {
+    const envPath = path.join(this._workspace.cwd, ".env");
+    if (!fs.existsSync(envPath)) return;
+    const lines = fs.readFileSync(envPath, "utf-8").split("\n");
+    for (const line of lines) {
+      for (const tier of ["TIER1", "TIER2", "TIER3", "TIER4"]) {
+        if (line.startsWith(`${tier}=`)) {
+          const models = line.split("=")[1].split(",").map((m) => m.trim()).filter(Boolean);
+          this._tierModels[tier.toLowerCase()] = models;
+        }
+      }
+    }
+  }
+
+  get name() { return "worker_llm_call"; }
+
+  get description() {
+    return (
+      "Call a worker LLM at a specified tier (tier1-tier4) for extraction, " +
+      "judgment, or other verification tasks. Tier1 is most capable/expensive, " +
+      "tier4 is cheapest. Returns response with model used and token counts."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        tier: { type: "string", enum: ["tier1", "tier2", "tier3", "tier4"], description: "Worker LLM tier to use" },
+        prompt: { type: "string", description: "The user/task prompt to send" },
+        system_prompt: { type: "string", description: "Optional system prompt for context" },
+        max_tokens: { type: "integer", description: "Maximum tokens in response (default 4096)" },
+      },
+      required: ["tier", "prompt"],
+    };
+  }
+
+  async execute(input) {
+    const tier = input.tier || "tier2";
+    const prompt = input.prompt || "";
+    const systemPrompt = input.system_prompt;
+    const maxTokens = input.max_tokens || 4096;
+
+    if (!prompt) return new ToolResult("No prompt provided", true);
+    if (!this._apiKey) return new ToolResult("Worker LLM API key not configured", true);
+
+    this._loadTiers();
+    const models = this._tierModels[tier] || [];
+    if (models.length === 0) {
+      return new ToolResult(`No models configured for ${tier}. Check .env TIER1-TIER4 settings.`, true);
+    }
+
+    const messages = [];
+    if (systemPrompt) messages.push({ role: "system", content: systemPrompt });
+    messages.push({ role: "user", content: prompt });
+
+    let lastError = "";
+    for (const model of models) {
+      try {
+        const resp = await fetch(`${this._baseUrl}/chat/completions`, {
+          method: "POST",
+          headers: {
+            "Authorization": `Bearer ${this._apiKey}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({ model, messages, max_tokens: maxTokens }),
+          signal: AbortSignal.timeout(120000),
+        });
+
+        if (resp.ok) {
+          const data = await resp.json();
+          const usage = data.usage || {};
+          const result = {
+            response: data.choices[0].message.content,
+            model_used: model,
+            tier,
+            tokens_in: usage.prompt_tokens || 0,
+            tokens_out: usage.completion_tokens || 0,
+          };
+          return new ToolResult(JSON.stringify(result, null, 2));
+        }
+        lastError = `${model}: HTTP ${resp.status}`;
+      } catch (e) {
+        lastError = `${model}: ${e.message}`;
+      }
+    }
+
+    return new ToolResult(`All models for ${tier} failed. Last error: ${lastError}`, true);
+  }
+}

package/src/agent/tools/workflow-run.js

@@ -0,0 +1,138 @@
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { BaseTool, ToolResult } from "./base.js";
+
+/**
+ * Execute a distilled workflow script against a document.
+ * Runs the workflow script in sandbox, attaches ConfidenceScorer
+ * result and trace ID automatically. Saves structured result to output/results/.
+ */
+export class WorkflowRunTool extends BaseTool {
+  constructor(workspace, versionManager, confidenceScorer, { timeout = 120 } = {}) {
+    super();
+    this._workspace = workspace;
+    this._versionMgr = versionManager;
+    this._confidence = confidenceScorer;
+    this._timeout = timeout;
+  }
+
+  get name() { return "workflow_run"; }
+
+  get description() {
+    return (
+      "Execute a distilled workflow against a document. Runs the workflow " +
+      "script, attaches confidence scores and trace IDs automatically. " +
+      "Results saved to output/results/."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        rule_id: { type: "string", description: "Rule ID whose workflow to execute" },
+        document_path: { type: "string", description: "Relative path to document in workspace" },
+        workflow_version: { type: "integer", description: "Workflow version to run (default: latest)" },
+      },
+      required: ["rule_id", "document_path"],
+    };
+  }
+
+  async execute(input) {
+    const ruleId = input.rule_id || "";
+    const docPath = input.document_path || "";
+    const wfVersion = input.workflow_version;
+
+    if (!ruleId || !docPath) return new ToolResult("rule_id and document_path required", true);
+
+    // Find workflow script
+    const wfDir = path.join(this._workspace.cwd, "workflows", ruleId);
+    const wfScript = this._findWorkflow(
+      fs.existsSync(wfDir) ? wfDir : path.join(this._workspace.cwd, "workflows"),
+      ruleId, wfVersion,
+    );
+    if (!wfScript) return new ToolResult(`No workflow found for ${ruleId} in workflows/`, true);
+
+    let docResolved;
+    try { docResolved = this._workspace.resolvePath(docPath); }
+    catch (e) { return new ToolResult(e.message, true); }
+    if (!fs.existsSync(docResolved)) return new ToolResult(`Document not found: ${docPath}`, true);
+
+    // Run workflow in subprocess
+    const cmd = `python ${wfScript} ${docResolved}`;
+    let output;
+    try {
+      output = await this._exec(cmd);
+    } catch (e) {
+      return new ToolResult(e.message, true);
+    }
+
+    // Parse output
+    let resultData;
+    try {
+      const lines = output.trim().split("\n");
+      resultData = JSON.parse(lines[lines.length - 1]);
+    } catch {
+      resultData = { raw_output: output.slice(0, 5000) };
+    }
+
+    // Attach confidence score
+    const extractedValue = String(resultData.extracted_value || resultData.value || "");
+    const method = resultData.extraction_method || "llm";
+    const confidence = this._confidence.score({
+      ruleId, extractedValue, extractionMethod: method, documentName: path.basename(docResolved),
+    });
+    resultData.confidence = confidence;
+    resultData.confidence_band = this._confidence.getBand(confidence);
+
+    // Attach trace ID
+    resultData.trace_id = this._versionMgr.generateTraceId(ruleId, "workflow_result");
+    resultData.rule_id = ruleId;
+    resultData.document = docPath;
+
+    // Save result
+    const resultsDir = path.join(this._workspace.cwd, "output", "results");
+    fs.mkdirSync(resultsDir, { recursive: true });
+    const resultFile = path.join(resultsDir, `${ruleId}_${path.parse(docResolved).name}.json`);
+    fs.writeFileSync(resultFile, JSON.stringify(resultData, null, 2), "utf-8");
+
+    return new ToolResult(JSON.stringify(resultData, null, 2));
+  }
+
+  _findWorkflow(wfDir, ruleId, version) {
+    if (!fs.existsSync(wfDir) || !fs.statSync(wfDir).isDirectory()) return null;
+
+    if (version) {
+      const target = path.join(wfDir, `workflow_v${version}.py`);
+      if (fs.existsSync(target)) return target;
+    }
+
+    const entries = fs.readdirSync(wfDir).sort();
+    const versioned = entries.filter((f) => /^workflow_v\d+\.py$/.test(f));
+    if (versioned.length > 0) return path.join(wfDir, versioned[versioned.length - 1]);
+
+    const any = entries.find((f) => f.endsWith(".py") && f.toLowerCase().includes("workflow"));
+    if (any) return path.join(wfDir, any);
+
+    const pyFile = entries.find((f) => f.endsWith(".py"));
+    return pyFile ? path.join(wfDir, pyFile) : null;
+  }
+
+  _exec(command) {
+    return new Promise((resolve, reject) => {
+      const controller = new AbortController();
+      const proc = spawn("sh", ["-c", command], {
+        cwd: this._workspace.cwd,
+        stdio: ["ignore", "pipe", "pipe"],
+        signal: controller.signal,
+      });
+      let output = "";
+      proc.stdout.on("data", (d) => { output += d.toString(); });
+      proc.stderr.on("data", (d) => { output += d.toString(); });
+      const timer = setTimeout(() => { controller.abort(); reject(new Error(`Workflow timed out after ${this._timeout}s`)); }, this._timeout * 1000);
+      proc.on("close", (code) => { clearTimeout(timer); resolve(output); });
+      proc.on("error", (err) => { clearTimeout(timer); reject(err); });
+    });
+  }
+}