kc-beta 0.1.0

package/src/agent/tools/document-parse.js

@@ -0,0 +1,184 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BaseTool, ToolResult } from "./base.js";
+
+const MAX_OUTPUT = 50_000;
+const MIN_CHARS_PER_PAGE = 50;
+
+/**
+ * Parse documents through a hard-coded escalation chain.
+ * Level 1: pdfjs-dist (free, local) — text extraction
+ * Level 2: MineRU API (if configured) — for scanned/complex documents
+ * Level 3: OCR models via SiliconFlow — fallback via vision models
+ */
+export class DocumentParseTool extends BaseTool {
+  constructor(workspace, { mineruApiUrl, mineruApiKey, siliconflowApiKey, siliconflowBaseUrl, ocrModel } = {}) {
+    super();
+    this._workspace = workspace;
+    this._mineruApiUrl = mineruApiUrl || "";
+    this._mineruApiKey = mineruApiKey || "";
+    this._sfApiKey = siliconflowApiKey || "";
+    this._sfBaseUrl = siliconflowBaseUrl || "https://api.siliconflow.cn/v1";
+    this._ocrModel = ocrModel || "";
+  }
+
+  get name() { return "document_parse"; }
+
+  get description() {
+    return (
+      "Parse a document (PDF, DOCX, TXT) and extract its text content. " +
+      "Internally uses an escalation chain: text extraction → API parser → OCR models. " +
+      "Starts cheap, escalates if needed. Use force_method to skip the chain."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        path: { type: "string", description: "Relative path to the document in the workspace" },
+        pages: { type: "string", description: "Page range to extract, e.g. '1-5', '3', '10-20'. Omit for all pages." },
+        force_method: {
+          type: "string",
+          enum: ["pdfjs", "mineru", "ocr"],
+          description: "Force a specific parsing method, skipping the escalation chain.",
+        },
+      },
+      required: ["path"],
+    };
+  }
+
+  async execute(input) {
+    const pathStr = input.path || "";
+    const pages = input.pages;
+    const force = input.force_method;
+
+    if (!pathStr) return new ToolResult("No path provided", true);
+
+    let resolved;
+    try { resolved = this._workspace.resolvePath(pathStr); }
+    catch (e) { return new ToolResult(e.message, true); }
+
+    if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) {
+      return new ToolResult(`File not found: ${pathStr}`, true);
+    }
+
+    const pageRange = this._parsePageRange(pages);
+
+    // Plain text files — read directly
+    const ext = path.extname(resolved).toLowerCase();
+    if ([".txt", ".md", ".csv", ".json", ".env"].includes(ext)) {
+      let text = fs.readFileSync(resolved, "utf-8");
+      if (text.length > MAX_OUTPUT) text = text.slice(0, MAX_OUTPUT) + "\n[truncated]";
+      return new ToolResult(`[Parsed via text read]\n\n${text}`);
+    }
+
+    if (force) return this._runMethod(force, resolved, pageRange);
+
+    // Escalation chain
+    // Level 1: pdfjs-dist
+    let result = await this._tryPdfjs(resolved, pageRange);
+    if (result && this._qualityOk(result)) {
+      return new ToolResult(this._formatOutput(result, "pdfjs", resolved));
+    }
+
+    // Level 2: MineRU API
+    if (this._mineruApiUrl) {
+      result = await this._tryMineru(resolved, pageRange);
+      if (result && this._qualityOk(result)) {
+        return new ToolResult(this._formatOutput(result, "mineru", resolved));
+      }
+    }
+
+    // Level 3: OCR via SiliconFlow
+    if (this._sfApiKey && this._ocrModel) {
+      result = await this._tryOcr(resolved, pageRange);
+      if (result) return new ToolResult(this._formatOutput(result, "ocr", resolved));
+    }
+
+    if (result) return new ToolResult(this._formatOutput(result, "pdfjs (low quality)", resolved));
+
+    return new ToolResult(
+      `Could not extract text from ${pathStr}. Configure OCR models in .env for image-based documents.`,
+      true,
+    );
+  }
+
+  async _runMethod(method, filePath, pageRange) {
+    let result;
+    if (method === "pdfjs") result = await this._tryPdfjs(filePath, pageRange);
+    else if (method === "mineru") result = await this._tryMineru(filePath, pageRange);
+    else if (method === "ocr") result = await this._tryOcr(filePath, pageRange);
+    else return new ToolResult(`Unknown method: ${method}`, true);
+
+    if (result) return new ToolResult(this._formatOutput(result, method, filePath));
+    return new ToolResult(`Method '${method}' failed for this document`, true);
+  }
+
+  async _tryPdfjs(filePath, pageRange) {
+    try {
+      const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs");
+      const data = new Uint8Array(fs.readFileSync(filePath));
+      const doc = await pdfjsLib.getDocument({ data, useSystemFonts: true }).promise;
+
+      const start = pageRange ? pageRange[0] : 0;
+      const end = pageRange ? pageRange[1] : doc.numPages - 1;
+      const pages = [];
+
+      for (let i = Math.max(0, start); i <= Math.min(end, doc.numPages - 1); i++) {
+        const page = await doc.getPage(i + 1); // 1-indexed
+        const content = await page.getTextContent();
+        const text = content.items.map((item) => item.str).join(" ");
+        if (text.trim()) {
+          pages.push(`--- Page ${i + 1} ---\n${text.trim()}`);
+        }
+      }
+
+      return pages.length > 0 ? pages.join("\n\n") : "";
+    } catch (e) {
+      return null;
+    }
+  }
+
+  async _tryMineru(filePath, pageRange) {
+    // TODO: Implement MineRU API call when available
+    return null;
+  }
+
+  async _tryOcr(filePath, pageRange) {
+    // OCR requires sending page images to a vision model API.
+    // Without a native image renderer, we delegate to the agent
+    // to use sandbox_exec for custom OCR pipelines.
+    // For now, return null to signal OCR is not available natively.
+    return null;
+  }
+
+  _qualityOk(text) {
+    if (!text || !text.trim()) return false;
+    const pages = (text.match(/--- Page \d+ ---/g) || []).length || 1;
+    const charsPerPage = text.length / pages;
+    if (charsPerPage < MIN_CHARS_PER_PAGE) return false;
+    const replacementRatio = (text.match(/\uFFFD/g) || []).length / Math.max(text.length, 1);
+    if (replacementRatio > 0.1) return false;
+    return true;
+  }
+
+  _formatOutput(text, method, filePath) {
+    if (text.length > MAX_OUTPUT) text = text.slice(0, MAX_OUTPUT) + "\n[truncated]";
+    return `[Parsed via ${method}]\n\n${text}`;
+  }
+
+  _parsePageRange(pages) {
+    if (!pages) return null;
+    pages = pages.trim();
+    if (pages.includes("-")) {
+      const parts = pages.split("-", 2);
+      try { return [parseInt(parts[0]) - 1, parseInt(parts[1]) - 1]; }
+      catch { return null; }
+    }
+    try {
+      const p = parseInt(pages) - 1;
+      return [p, p];
+    } catch { return null; }
+  }
+}

package/src/agent/tools/document-search.js

@@ -0,0 +1,111 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BaseTool, ToolResult } from "./base.js";
+
+const MAX_RESULTS = 20;
+const CONTEXT_CHARS = 200;
+const TEXT_EXTENSIONS = new Set([".txt", ".md", ".json", ".py", ".csv", ".env", ".log", ".js"]);
+
+/**
+ * Full-text search across documents in the workspace.
+ * Searches text files and parsed document outputs. Returns passages
+ * with source coordinates (file, line number).
+ */
+export class DocumentSearchTool extends BaseTool {
+  constructor(workspace) {
+    super();
+    this._workspace = workspace;
+  }
+
+  get name() { return "document_search"; }
+
+  get description() {
+    return (
+      "Search for text across documents in the workspace. Returns matching " +
+      "passages with file path and context. Supports plain text and regex queries."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "Search query (plain text or regex pattern)" },
+        path: { type: "string", description: "Subdirectory to search in (default: entire workspace)" },
+        max_results: { type: "integer", description: `Maximum results to return (default: ${MAX_RESULTS})` },
+        regex: { type: "boolean", description: "Treat query as regex pattern (default: false)" },
+      },
+      required: ["query"],
+    };
+  }
+
+  async execute(input) {
+    const query = input.query || "";
+    const searchPath = input.path || ".";
+    const maxResults = input.max_results || MAX_RESULTS;
+    const useRegex = input.regex || false;
+
+    if (!query) return new ToolResult("No query provided", true);
+
+    let searchDir;
+    try { searchDir = this._workspace.resolvePath(searchPath); }
+    catch (e) { return new ToolResult(e.message, true); }
+
+    if (!fs.existsSync(searchDir) || !fs.statSync(searchDir).isDirectory()) {
+      return new ToolResult(`Not a directory: ${searchPath}`, true);
+    }
+
+    let pattern;
+    try {
+      pattern = useRegex ? new RegExp(query, "gi") : new RegExp(query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "gi");
+    } catch (e) {
+      return new ToolResult(`Invalid regex: ${e.message}`, true);
+    }
+
+    const results = [];
+    this._searchDir(searchDir, pattern, results, maxResults);
+
+    if (results.length === 0) return new ToolResult(`No matches found for: ${query}`);
+
+    const lines = [];
+    for (const r of results) {
+      lines.push(`--- ${r.file}:${r.line} ---`);
+      lines.push(r.context);
+      lines.push("");
+    }
+    return new ToolResult(`Found ${results.length} match(es):\n\n${lines.join("\n")}`);
+  }
+
+  _searchDir(dir, pattern, results, maxResults) {
+    let entries;
+    try { entries = fs.readdirSync(dir, { withFileTypes: true }); }
+    catch { return; }
+
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+      if (results.length >= maxResults) break;
+      const fullPath = path.join(dir, entry.name);
+
+      if (entry.isDirectory()) {
+        if (entry.name.startsWith(".") || entry.name === "node_modules" || entry.name === "__pycache__") continue;
+        this._searchDir(fullPath, pattern, results, maxResults);
+      } else if (entry.isFile() && TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) {
+        let content;
+        try { content = fs.readFileSync(fullPath, "utf-8"); }
+        catch { continue; }
+
+        let match;
+        pattern.lastIndex = 0;
+        while ((match = pattern.exec(content)) !== null) {
+          const start = Math.max(0, match.index - CONTEXT_CHARS);
+          const end = Math.min(content.length, match.index + match[0].length + CONTEXT_CHARS);
+          const context = content.slice(start, end).trim();
+          const lineNum = content.slice(0, match.index).split("\n").length;
+          const relPath = path.relative(this._workspace.cwd, fullPath);
+
+          results.push({ file: relPath, line: lineNum, match: match[0], context });
+          if (results.length >= maxResults) break;
+        }
+      }
+    }
+  }
+}

package/src/agent/tools/evolution-cycle.js

@@ -0,0 +1,150 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BaseTool, ToolResult } from "./base.js";
+
+/**
+ * Run one structured iteration of the evolution loop.
+ * Enforces: diagnose -> classify -> fix instructions -> log.
+ * Classification is CODE: counts failure rate, applies threshold.
+ * Routes corner cases to CornerCaseRegistry automatically.
+ */
+export class EvolutionCycleTool extends BaseTool {
+  constructor(workspace, cornerCases) {
+    super();
+    this._workspace = workspace;
+    this._cornerCases = cornerCases;
+  }
+
+  get name() { return "evolution_cycle"; }
+  get description() {
+    return (
+      "Run one structured iteration of diagnose -> classify -> fix -> log. " +
+      "Provide test results with failures. The tool classifies failures as " +
+      "systemic (>10%) or corner case (<10%), routes corner cases to the " +
+      "registry, and saves a structured evolution log entry."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        rule_id: { type: "string", description: "Rule being evolved" },
+        total_test_docs: { type: "integer", description: "Total number of documents tested" },
+        failed_docs: {
+          type: "array",
+          items: {
+            type: "object",
+            properties: {
+              doc_id: { type: "string" },
+              diagnosis: { type: "string", enum: ["parsing", "extraction", "judgment", "scope"] },
+              root_cause: { type: "string" },
+            },
+          },
+          description: "List of failed documents with diagnosis",
+        },
+        accuracy_before: { type: "number", description: "Accuracy before this cycle" },
+        fix_applied: { type: "string", description: "Description of the fix applied (or planned)" },
+      },
+      required: ["rule_id", "total_test_docs", "failed_docs", "accuracy_before"],
+    };
+  }
+
+  async execute(input) {
+    const ruleId = input.rule_id || "";
+    const total = input.total_test_docs || 0;
+    const failures = input.failed_docs || [];
+    const accuracyBefore = input.accuracy_before || 0;
+    const fixApplied = input.fix_applied || "";
+
+    if (!ruleId || total <= 0) return new ToolResult("rule_id and total_test_docs required", true);
+
+    // Read systemic threshold from .env
+    let systemicThreshold = 0.10;
+    const envPath = path.join(this._workspace.cwd, ".env");
+    if (fs.existsSync(envPath)) {
+      for (const line of fs.readFileSync(envPath, "utf-8").split("\n")) {
+        if (line.startsWith("SYSTEMIC_THRESHOLD=")) {
+          try { systemicThreshold = parseFloat(line.split("=")[1].trim()); }
+          catch { /* ignore */ }
+        }
+      }
+    }
+
+    const failureRate = failures.length / total;
+    const classification = failureRate >= systemicThreshold ? "systemic" : "corner_case";
+
+    // Check repeated patterns
+    const repeatedPatterns = this._checkRepeatedPatterns(ruleId, failures);
+
+    // Route corner cases to registry
+    const cornerCasesAdded = [];
+    if (classification === "corner_case") {
+      for (const f of failures) {
+        const c = {
+          id: `CC_${ruleId}_${f.doc_id || "unknown"}`,
+          ruleId,
+          detectionPattern: f.root_cause || "unknown pattern",
+          resolution: fixApplied || "pending fix",
+          affectedDocuments: [f.doc_id || ""],
+          discoveryDate: new Date().toISOString(),
+          status: "active",
+        };
+        this._cornerCases.add(c);
+        cornerCasesAdded.push(c.id);
+      }
+    }
+
+    // Count iteration
+    const logDir = path.join(this._workspace.cwd, "logs", "evolution");
+    fs.mkdirSync(logDir, { recursive: true });
+    const existing = fs.readdirSync(logDir).filter((f) => f.startsWith(`${ruleId}_iter_`));
+    const iteration = existing.length + 1;
+
+    // Build log entry
+    const logEntry = {
+      iteration, rule_id: ruleId, timestamp: new Date().toISOString(),
+      accuracy_before: accuracyBefore, total_docs: total,
+      failed_docs: failures.length, failure_rate: Math.round(failureRate * 1000) / 1000,
+      classification, failures, fix_applied: fixApplied,
+      corner_cases_added: cornerCasesAdded, repeated_patterns: repeatedPatterns,
+    };
+
+    const logPath = path.join(logDir, `${ruleId}_iter_${String(iteration).padStart(3, "0")}.json`);
+    fs.writeFileSync(logPath, JSON.stringify(logEntry, null, 2), "utf-8");
+
+    const response = {
+      iteration, classification,
+      failure_rate: `${(failureRate * 100).toFixed(1)}%`,
+      action: classification === "systemic"
+        ? "REWRITE component — systemic issue affecting >10% of documents"
+        : `Recorded ${cornerCasesAdded.length} corner case(s) — do NOT patch main workflow`,
+      repeated_patterns: repeatedPatterns,
+      log_saved: path.relative(this._workspace.cwd, logPath),
+    };
+
+    if (repeatedPatterns.length > 0) {
+      response.warning = "Repeated failure patterns detected. Consider escalating approach.";
+    }
+
+    return new ToolResult(JSON.stringify(response, null, 2));
+  }
+
+  _checkRepeatedPatterns(ruleId, failures) {
+    const logDir = path.join(this._workspace.cwd, "logs", "evolution");
+    if (!fs.existsSync(logDir)) return [];
+    const currentCauses = new Set(failures.map((f) => (f.root_cause || "").toLowerCase()).filter(Boolean));
+    const repeated = [];
+
+    for (const f of fs.readdirSync(logDir).filter((f) => f.startsWith(`${ruleId}_iter_`)).sort()) {
+      try {
+        const data = JSON.parse(fs.readFileSync(path.join(logDir, f), "utf-8"));
+        const prevCauses = new Set((data.failures || []).map((f) => (f.root_cause || "").toLowerCase()).filter(Boolean));
+        for (const cause of currentCauses) {
+          if (prevCauses.has(cause) && !repeated.includes(cause)) repeated.push(cause);
+        }
+      } catch { /* skip */ }
+    }
+    return repeated;
+  }
+}

package/src/agent/tools/qc-sample.js

@@ -0,0 +1,94 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BaseTool, ToolResult } from "./base.js";
+
+const FREQUENCY_RATES = { high: 0.5, mid: 0.3, low: 0.1 };
+
+/**
+ * Draw adaptive sample from production results for quality review.
+ * Stratifies by confidence band: review ALL low, sample medium, spot-check high.
+ */
+export class QCSampleTool extends BaseTool {
+  constructor(workspace) {
+    super();
+    this._workspace = workspace;
+  }
+
+  get name() { return "qc_sample"; }
+  get description() {
+    return (
+      "Draw an adaptive sample from production results for quality review. " +
+      "Stratifies by confidence band (low=review all, medium=sample, high=spot-check). " +
+      "Returns list of document IDs to review."
+    );
+  }
+
+  get inputSchema() {
+    return {
+      type: "object",
+      properties: {
+        results_path: { type: "string", description: "Path to results directory (default: output/results/)" },
+      },
+    };
+  }
+
+  async execute(input) {
+    const resultsPath = input.results_path || "output/results";
+    let resultsDir;
+    try { resultsDir = this._workspace.resolvePath(resultsPath); }
+    catch (e) { return new ToolResult(e.message, true); }
+
+    if (!fs.existsSync(resultsDir) || !fs.statSync(resultsDir).isDirectory()) {
+      return new ToolResult(`Results directory not found: ${resultsPath}`, true);
+    }
+
+    // Load MONITOR_FREQUENCY from .env
+    let mediumRate = 0.3;
+    const envPath = path.join(this._workspace.cwd, ".env");
+    if (fs.existsSync(envPath)) {
+      for (const line of fs.readFileSync(envPath, "utf-8").split("\n")) {
+        if (line.startsWith("MONITOR_FREQUENCY=")) {
+          const freq = line.split("=")[1].trim().toLowerCase();
+          mediumRate = FREQUENCY_RATES[freq] ?? 0.3;
+        }
+      }
+    }
+
+    const low = [], medium = [], high = [];
+    const files = fs.readdirSync(resultsDir).filter((f) => f.endsWith(".json")).sort();
+    for (const f of files) {
+      try {
+        const data = JSON.parse(fs.readFileSync(path.join(resultsDir, f), "utf-8"));
+        const band = data.confidence_band || "medium";
+        const entry = { file: f, rule_id: data.rule_id || "", confidence: data.confidence || 0 };
+        if (band === "low") low.push(entry);
+        else if (band === "high") high.push(entry);
+        else medium.push(entry);
+      } catch { /* skip */ }
+    }
+
+    const toReview = [...low];
+    if (medium.length > 0) {
+      const sampleSize = Math.max(1, Math.floor(medium.length * mediumRate));
+      toReview.push(...this._sample(medium, sampleSize));
+    }
+    if (high.length > 0) {
+      const spotSize = Math.max(1, Math.floor(high.length * 0.1));
+      toReview.push(...this._sample(high, spotSize));
+    }
+
+    const report = {
+      total_results: low.length + medium.length + high.length,
+      distribution: { low: low.length, medium: medium.length, high: high.length },
+      sampling_rate_medium: mediumRate,
+      to_review: toReview.length,
+      review_list: toReview,
+    };
+    return new ToolResult(JSON.stringify(report, null, 2));
+  }
+
+  _sample(arr, n) {
+    const shuffled = [...arr].sort(() => Math.random() - 0.5);
+    return shuffled.slice(0, Math.min(n, shuffled.length));
+  }
+}

package/src/agent/tools/registry.js

@@ -0,0 +1,55 @@
+import { ToolResult } from "./base.js";
+
+/**
+ * Manages tool registration and dispatch.
+ * Tools register themselves; the engine loop discovers them via schemasOpenai()
+ * and dispatches to execute() when the LLM invokes a tool.
+ */
+export class ToolRegistry {
+  constructor() {
+    /** @type {Map<string, import('./base.js').BaseTool>} */
+    this._tools = new Map();
+  }
+
+  /**
+   * Register a tool instance.
+   * @param {import('./base.js').BaseTool} tool
+   */
+  register(tool) {
+    this._tools.set(tool.name, tool);
+  }
+
+  /**
+   * Return tool schemas in OpenAI function-calling format.
+   * @returns {Array<object>}
+   */
+  schemasOpenai() {
+    return Array.from(this._tools.values()).map((t) => ({
+      type: "function",
+      function: {
+        name: t.name,
+        description: t.description,
+        parameters: t.inputSchema,
+      },
+    }));
+  }
+
+  /**
+   * Execute a tool by name.
+   * @param {string} name
+   * @param {object} input
+   * @returns {Promise<ToolResult>}
+   */
+  async execute(name, input) {
+    const tool = this._tools.get(name);
+    if (!tool) {
+      return new ToolResult(`Unknown tool: ${name}`, true);
+    }
+    return tool.execute(input);
+  }
+
+  /** @returns {number} Number of registered tools */
+  get size() {
+    return this._tools.size;
+  }
+}