kc-beta 0.1.2 → 0.3.0

package/bin/kc-beta.js

@@ -1,16 +1,28 @@
 #!/usr/bin/env node
 
-const subcommand = process.argv[2];
+// Parse --en / --zh from anywhere in argv (session-only language override)
+const args = process.argv.slice(2);
+let languageOverride = null;
+const filtered = [];
+for (const arg of args) {
+  if (arg === "--en") languageOverride = "en";
+  else if (arg === "--zh") languageOverride = "zh";
+  else filtered.push(arg);
+}
+const subcommand = filtered[0];
 
 (async () => {
   if (subcommand === "onboard" || subcommand === "setup") {
     const { onboard } = await import("../src/cli/onboard.js");
     await onboard();
+  } else if (subcommand === "config") {
+    const { configEditor } = await import("../src/cli/config.js");
+    await configEditor();
   } else if (subcommand === "init") {
     const { init } = await import("../src/cli/init.js");
     await init();
   } else {
     const { main } = await import("../src/cli/index.js");
-    await main();
+    await main({ languageOverride });
   }
 })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kc-beta",
-  "version": "0.1.2",
+  "version": "0.3.0",
   "description": "KC Agent — LLM document verification agent (pure Node.js CLI)",
   "type": "module",
   "bin": {

package/src/agent/context-window.js

@@ -0,0 +1,151 @@
+import { estimateTokens, estimateMessagesTokens } from "./token-counter.js";
+
+/**
+ * Automatic context windowing for long conversations.
+ * When messages approach the model's context limit, older messages
+ * are compressed into summaries while keeping recent messages intact.
+ */
+export class ContextWindow {
+  /**
+   * @param {object} opts
+   * @param {number} opts.contextLimit - Total model context limit in tokens
+   * @param {number} [opts.reserveForResponse=8192] - Tokens reserved for model output
+   * @param {number} [opts.recentWindowSize=30] - Number of recent messages to always keep
+   */
+  constructor({ contextLimit, reserveForResponse = 8192, recentWindowSize = 30 }) {
+    this.contextLimit = contextLimit;
+    this.reserveForResponse = reserveForResponse;
+    this.recentWindowSize = recentWindowSize;
+  }
+
+  /**
+   * Apply windowing to a message array if it exceeds the token budget.
+   * @param {Array<object>} messages - Full message history
+   * @param {string[]} [phaseSummaries] - Summaries from completed pipeline phases
+   * @returns {{ messages: Array, wasWindowed: boolean, removedCount: number }}
+   */
+  window(messages, phaseSummaries = []) {
+    const totalTokens = estimateMessagesTokens(messages);
+    const budget = this.contextLimit - this.reserveForResponse;
+
+    // If within budget, return as-is
+    if (totalTokens <= budget * 0.85) {
+      return { messages, wasWindowed: false, removedCount: 0 };
+    }
+
+    // Split into older and recent
+    const splitPoint = Math.max(0, messages.length - this.recentWindowSize);
+    const recentMessages = messages.slice(splitPoint);
+    const olderMessages = messages.slice(0, splitPoint);
+
+    if (olderMessages.length === 0) {
+      return { messages, wasWindowed: false, removedCount: 0 };
+    }
+
+    // Build a compact summary of older messages
+    const recentTokens = estimateMessagesTokens(recentMessages);
+    const summaryBudget = budget - recentTokens - 500; // 500 tokens buffer
+    const compactedSummary = this._compactMessages(olderMessages, phaseSummaries, summaryBudget);
+
+    const windowedMessages = [
+      {
+        role: "user",
+        content: `[Context Summary - Earlier conversation compressed]\n\n${compactedSummary}`,
+      },
+      {
+        role: "assistant",
+        content: "Understood. I have the context from the summary above. Continuing with the current work.",
+      },
+      ...recentMessages,
+    ];
+
+    return {
+      messages: windowedMessages,
+      wasWindowed: true,
+      removedCount: olderMessages.length,
+    };
+  }
+
+  /**
+   * Create a mechanical compact summary of messages.
+   * Groups into conversational turns and extracts key info.
+   * @param {Array<object>} messages
+   * @param {string[]} phaseSummaries
+   * @param {number} tokenBudget
+   * @returns {string}
+   */
+  _compactMessages(messages, phaseSummaries, tokenBudget) {
+    const parts = [];
+
+    // Phase summaries first (high signal)
+    if (phaseSummaries.length > 0) {
+      parts.push("## Phase History");
+      for (const s of phaseSummaries) {
+        parts.push(`- ${s}`);
+      }
+      parts.push("");
+    }
+
+    // Extract key events from older messages
+    parts.push("## Conversation Summary");
+    const turns = this._groupIntoTurns(messages);
+
+    for (const turn of turns) {
+      const line = this._summarizeTurn(turn);
+      if (line) {
+        parts.push(`- ${line}`);
+        // Check budget
+        if (estimateTokens(parts.join("\n")) > tokenBudget * 0.9) {
+          parts.push("- [earlier history truncated]");
+          break;
+        }
+      }
+    }
+
+    return parts.join("\n");
+  }
+
+  /**
+   * Group messages into user-turn blocks.
+   * Each turn: { user: string, tools: [{name, summary}], assistantSummary: string }
+   */
+  _groupIntoTurns(messages) {
+    const turns = [];
+    let current = null;
+
+    for (const msg of messages) {
+      if (msg.role === "user") {
+        if (current) turns.push(current);
+        current = { user: msg.content || "", tools: [], assistant: "" };
+      } else if (msg.role === "assistant" && current) {
+        if (msg.content) current.assistant = msg.content;
+        if (msg.tool_calls) {
+          for (const tc of msg.tool_calls) {
+            current.tools.push(tc.function?.name || "unknown");
+          }
+        }
+      }
+      // tool results are captured implicitly via tool names
+    }
+    if (current) turns.push(current);
+    return turns;
+  }
+
+  /**
+   * Summarize a single conversational turn into one line.
+   */
+  _summarizeTurn(turn) {
+    const userSnippet = (turn.user || "").slice(0, 80).replace(/\n/g, " ");
+    if (!userSnippet) return null;
+
+    let line = `User: "${userSnippet}"`;
+    if (turn.tools.length > 0) {
+      line += ` → Tools: ${turn.tools.join(", ")}`;
+    }
+    if (turn.assistant) {
+      const aSnippet = turn.assistant.slice(0, 60).replace(/\n/g, " ");
+      line += ` → "${aSnippet}..."`;
+    }
+    return line;
+  }
+}

package/src/agent/context.js

@@ -32,9 +32,11 @@ outcome. Handle ambiguity explicitly — note it, ask the developer user. After
 audit which regulation sections are not yet covered.
 
 ### Entity Extraction
-Prefer regex/Python for predictable formats. Use LLM only when semantic understanding \
-is required. Every extraction captures: value, evidence, source location, confidence, \
-method used.
+Choose the cheapest method that meets accuracy threshold. Regex is the smallest \
+"model" — zero cost, instant, deterministic. Worker LLM handles semantic tasks \
+regex cannot (contextual interpretation, misleading language, adequacy judgment). \
+Try different methods, find the cost-accuracy balance. Every extraction captures: \
+value, evidence, source location, confidence, method used.
 
 ### Skill Authoring
 Write each rule into a skill folder following the Anthropic skill-creator format. A \
@@ -79,13 +81,15 @@ unclear regulations with them. Present results and let them judge.`;
 export class ContextAssembler {
   /**
    * @param {object} [opts]
+   * @param {string} [opts.agentMd] - Content of workspace AGENT.md (per-project context)
    * @param {string} [opts.pipelineState]
    * @param {string} [opts.workspaceState]
    * @param {string} [opts.skillIndex] - Brief index of available meta skills
    * @returns {string}
    */
-  build({ pipelineState, workspaceState, skillIndex } = {}) {
+  build({ agentMd, pipelineState, workspaceState, skillIndex } = {}) {
     const parts = [AGENT_IDENTITY];
+    if (agentMd) parts.push(agentMd);
     if (skillIndex) parts.push(skillIndex);
     if (pipelineState) parts.push(pipelineState);
     if (workspaceState) parts.push(workspaceState);

package/src/agent/engine.js

@@ -1,3 +1,5 @@
+import fs from "node:fs";
+import path from "node:path";
 import { AgentEvent } from "./events.js";
 import { ContextAssembler } from "./context.js";
 import { ConversationHistory } from "./history.js";
@@ -18,6 +20,7 @@ import { DashboardRenderTool } from "./tools/dashboard-render.js";
 import { EvolutionCycleTool } from "./tools/evolution-cycle.js";
 import { TierDowngradeTool } from "./tools/tier-downgrade.js";
 import { AgentTool } from "./tools/agent-tool.js";
+import { WebSearchTool } from "./tools/web-search.js";
 import { SkillLoader } from "./skill-loader.js";
 import { Phase } from "./pipelines/index.js";
 import { ProjectInitializer } from "./pipelines/initializer.js";
@@ -26,6 +29,10 @@ import { SkillAuthoringPipeline } from "./pipelines/skill-authoring.js";
 import { SkillTestingPipeline } from "./pipelines/skill-testing.js";
 import { DistillationEngine as DistillationPipeline } from "./pipelines/distillation.js";
 import { ProductionQCPipeline } from "./pipelines/production-qc.js";
+import { EventLog } from "./event-log.js";
+import { ContextWindow } from "./context-window.js";
+import { SessionState } from "./session-state.js";
+import { estimateTokens, estimateMessagesTokens } from "./token-counter.js";
 
 // Phases where worker LLM tools are available (DISTILL mode)
 const DISTILL_PHASES = new Set([Phase.DISTILLATION, Phase.PRODUCTION_QC]);
@@ -51,14 +58,27 @@ export class AgentEngine {
     this.context = new ContextAssembler();
 
     // Workspace + structural components
-    this.workspace = new Workspace(config.kcWorkspaceRoot, sessionId);
+    this.workspace = new Workspace(config.kcWorkspaceRoot, sessionId, config.projectDir);
     this.history = new ConversationHistory(this.workspace.cwd);
     this.versionManager = new VersionManager(this.workspace.cwd);
     this.cornerCases = new CornerCaseRegistry(this.workspace.cwd);
     this.confidence = new ConfidenceScorer(this.workspace.cwd, this.cornerCases);
 
+    // Event log (append-only JSONL, source of truth)
+    this.eventLog = new EventLog(this.workspace.cwd);
+
+    // Context windowing
+    this.contextWindow = new ContextWindow({
+      contextLimit: config.kcContextLimit || 200000,
+      reserveForResponse: config.kcMaxTokens || 65536,
+    });
+
+    // Session state persistence
+    this.sessionState = new SessionState(this.workspace.cwd);
+
     // Build all tool instances (but register phase-appropriate ones)
     this._buildTools = this._createAllTools();
+    this._phaseSummaries = [];
 
     // Pipeline system (meta-meta skills as code)
     this.currentPhase = Phase.BOOTSTRAP;
@@ -84,11 +104,20 @@ export class AgentEngine {
    * re-register per phase without recreating.
    */
   _createAllTools() {
+    // Worker LLM uses separate config if set, otherwise falls back to conductor
+    const workerApiKey = this.config.effectiveWorkerApiKey();
+    const workerBaseUrl = this.config.effectiveWorkerBaseUrl();
+    const workerAuthType = this.config.effectiveWorkerAuthType();
+
     const workerLlm = new WorkerLLMCallTool(this.workspace, {
-      apiKey: this.config.siliconflowApiKey,
-      baseUrl: this.config.siliconflowBaseUrl,
+      apiKey: workerApiKey,
+      baseUrl: workerBaseUrl,
+      authType: workerAuthType,
     });
 
+    // OCR/VLM uses worker config (VLM is a type of worker LLM)
+    const vlmModel = this.config.vlmTier1 || "";
+
     return {
       // Always available (BUILD + DISTILL)
       core: [
@@ -97,9 +126,9 @@ export class AgentEngine {
         new DocumentParseTool(this.workspace, {
           mineruApiUrl: this.config.mineruApiUrl,
           mineruApiKey: this.config.mineruApiKey,
-          siliconflowApiKey: this.config.siliconflowApiKey,
-          siliconflowBaseUrl: this.config.siliconflowBaseUrl,
-          ocrModel: this.config.ocrModelTier1,
+          llmApiKey: workerApiKey,
+          llmBaseUrl: workerBaseUrl,
+          ocrModel: vlmModel,
         }),
         new DocumentSearchTool(this.workspace),
         new RuleCatalogTool(this.workspace),
@@ -108,6 +137,7 @@ export class AgentEngine {
         new AgentTool(this.workspace, (sid) => new AgentEngine({
           client: this.client, config: this.config, sessionId: sid,
         })),
+        new WebSearchTool(this.config.tavilyApiKey),
       ],
       // Distillation+ only (DISTILL mode)
       distill: [
@@ -136,6 +166,189 @@ export class AgentEngine {
     }
   }
 
+  /**
+   * Read AGENT.md from workspace (per-project context).
+   * Returns content string or empty string if not found.
+   */
+  _readAgentMd() {
+    const agentMdPath = path.join(this.workspace.cwd, "AGENT.md");
+    try {
+      if (fs.existsSync(agentMdPath)) {
+        return fs.readFileSync(agentMdPath, "utf-8");
+      }
+    } catch { /* ignore */ }
+    return "";
+  }
+
+  /**
+   * Build the workspace/project directory state string for the system prompt.
+   */
+  _buildWorkspaceState() {
+    const lines = [
+      `## Directory Layout`,
+      `**KC Workspace:** ${this.workspace.cwd}`,
+      `  Use scope="workspace" (default). Write all working files here (rules, skills, workflows, results, logs).`,
+    ];
+    if (this.workspace.projectDir) {
+      lines.push(
+        `**Project Directory:** ${this.workspace.projectDir}`,
+        `  Use scope="project" to read/write files in the user's project folder.`,
+        `  This is where the user's source regulations, samples, and reference documents are.`,
+        ``,
+        `Read source documents from the project directory. Write KC outputs to the workspace.`,
+        `Write user-facing exports (reports, results) to the project directory when the user asks.`,
+      );
+    }
+    return lines.join("\n");
+  }
+
+  /**
+   * Get current context usage statistics.
+   * @returns {{ totalTokens: number, limit: number, percentage: number }}
+   */
+  getContextStats() {
+    const systemPrompt = this.context.build({
+      agentMd: this._readAgentMd(),
+      skillIndex: this._skillLoader.formatForContext(),
+      pipelineState: this.pipelines[this.currentPhase]?.describeState?.() || null,
+      workspaceState: this._buildWorkspaceState(),
+    });
+    const systemTokens = estimateTokens(systemPrompt);
+    const messageTokens = estimateMessagesTokens(this.history.messages);
+    const totalTokens = systemTokens + messageTokens;
+    const limit = this.config.kcContextLimit || 200000;
+    return {
+      totalTokens,
+      limit,
+      percentage: Math.round((totalTokens / limit) * 100),
+    };
+  }
+
+  /**
+   * Compact conversation history by summarizing older messages via LLM.
+   * Keeps the most recent messages intact.
+   * @param {object} [opts]
+   * @param {number} [opts.recentCount=20] - Number of recent messages to keep
+   * @returns {Promise<{removedCount: number, retainedCount: number, summaryTokens: number}|null>}
+   */
+  async compact({ recentCount = 20 } = {}) {
+    if (this.history.messages.length <= recentCount) return null;
+
+    const olderMessages = this.history.messages.slice(0, -recentCount);
+    const recentMessages = this.history.messages.slice(-recentCount);
+
+    let summary;
+    try {
+      const summaryResp = await this.client.chat({
+        model: this.config.kcModel,
+        messages: [
+          {
+            role: "system",
+            content:
+              "You are a conversation summarizer. Produce a concise summary of the following conversation. " +
+              "Focus on: decisions made, files created or modified, current state of work, key findings, " +
+              "unresolved questions. Be specific about file paths, rule IDs, and results. Keep under 2000 tokens.",
+          },
+          {
+            role: "user",
+            content: `Summarize this conversation:\n\n${JSON.stringify(olderMessages)}`,
+          },
+        ],
+        maxTokens: 2048,
+      });
+      summary = summaryResp.choices?.[0]?.message?.content || null;
+    } catch {
+      // LLM summary failed — do mechanical fallback
+      summary = null;
+    }
+
+    if (!summary) {
+      // Mechanical fallback: extract tool names and outcomes
+      const lines = ["Previous conversation summary (mechanical):"];
+      for (const msg of olderMessages) {
+        if (msg.role === "user") {
+          lines.push(`- User: ${(msg.content || "").slice(0, 100)}`);
+        } else if (msg.role === "assistant" && msg.tool_calls) {
+          for (const tc of msg.tool_calls) {
+            lines.push(`- Tool call: ${tc.function?.name}`);
+          }
+        }
+      }
+      summary = lines.join("\n");
+    }
+
+    // Replace history
+    this.history._messages = [
+      { role: "user", content: `[Previous conversation summary]\n${summary}` },
+      { role: "assistant", content: "Understood. I have the context from the summary above. Continuing from where we left off." },
+      ...recentMessages,
+    ];
+    this.history._save();
+
+    // Log compaction event
+    this.eventLog.append("compact", {
+      removedCount: olderMessages.length,
+      retainedCount: recentMessages.length,
+      summary,
+    });
+
+    return {
+      removedCount: olderMessages.length,
+      retainedCount: recentMessages.length,
+      summaryTokens: estimateTokens(summary),
+    };
+  }
+
+  /**
+   * Restore an engine from a persisted session.
+   * @param {object} opts
+   * @param {import('./llm-client.js').LLMClient} opts.client
+   * @param {object} opts.config
+   * @param {string} opts.sessionId
+   * @returns {Promise<AgentEngine>}
+   */
+  static async resume({ client, config, sessionId }) {
+    const engine = new AgentEngine({ client, config, sessionId });
+    const state = engine.sessionState;
+
+    if (state.exists) {
+      const data = state.load();
+      engine.currentPhase = data.currentPhase || Phase.BOOTSTRAP;
+      engine._phaseSummaries = data.phaseSummaries || [];
+      engine._registerToolsForPhase(engine.currentPhase);
+
+      // Restore project directory from saved state
+      if (data.projectDir) {
+        if (fs.existsSync(data.projectDir)) {
+          engine.workspace.projectDir = data.projectDir;
+        }
+        // If dir no longer exists, projectDir stays as whatever was passed at launch
+      }
+
+      // Restore pipeline milestones
+      const milestones = data.pipelineMilestones || {};
+      for (const [phase, mData] of Object.entries(milestones)) {
+        if (engine.pipelines[phase]?.importState) {
+          engine.pipelines[phase].importState(mData);
+        }
+      }
+
+      engine.eventLog.append("session_resume", {
+        resumedPhase: engine.currentPhase,
+        resumedFromSeq: data.lastEventSeq,
+      });
+    }
+
+    return engine;
+  }
+
+  /**
+   * Save current session state for future resume.
+   */
+  saveState() {
+    this.sessionState.save(this);
+  }
+
   /**
    * Run one conversation turn. Yields AgentEvent objects.
    * Loops: LLM call -> tool execution -> LLM call ... until no tool calls.
@@ -144,20 +357,36 @@ export class AgentEngine {
    */
   async *runTurn(userMessage) {
     this.history.addUser(userMessage);
+    this.eventLog.append("user_message", { content: userMessage });
 
     // Pipeline state injection
     const pipeline = this.pipelines[this.currentPhase];
     const pipelineState = pipeline?.describeState?.() || null;
 
     const systemPrompt = this.context.build({
+      agentMd: this._readAgentMd(),
       skillIndex: this._skillLoader.formatForContext(),
       pipelineState,
-      workspaceState: `Your workspace directory is: ${this.workspace.cwd}`,
+      workspaceState: this._buildWorkspaceState(),
     });
     const tools = this.toolRegistry.schemasOpenai();
 
     while (true) {
-      const messages = [{ role: "system", content: systemPrompt }, ...this.history.messages];
+      // Apply context windowing before sending to LLM
+      const windowed = this.contextWindow.window(this.history.messages, this._phaseSummaries);
+      const messages = [{ role: "system", content: systemPrompt }, ...windowed.messages];
+
+      if (windowed.wasWindowed) {
+        this.eventLog.append("context_windowed", {
+          removedCount: windowed.removedCount,
+          totalBefore: this.history.messages.length,
+        });
+      }
+
+      this.eventLog.append("llm_start", {
+        model: this.config.kcModel,
+        messageCount: messages.length,
+      });
 
       try {
         let collectedText = "";
@@ -194,6 +423,7 @@ export class AgentEngine {
           }
         }
 
+        // Log the complete assistant message (coalesced, not per-delta)
         const assistantMsg = { role: "assistant", content: collectedText || null };
         if (toolCallsAcc.size > 0) {
           assistantMsg.tool_calls = Array.from(toolCallsAcc.values()).map((tc) => ({
@@ -203,8 +433,14 @@ export class AgentEngine {
           }));
         }
         this.history.addRaw(assistantMsg);
+        this.eventLog.append("assistant_message", {
+          content: collectedText || null,
+          toolCalls: assistantMsg.tool_calls || [],
+        });
 
         if (toolCallsAcc.size === 0) {
+          this.eventLog.append("turn_complete", {});
+          this.saveState();
           yield new AgentEvent({ type: "turn_complete" });
           return;
         }
@@ -216,8 +452,16 @@ export class AgentEngine {
             inputData = tc.arguments ? JSON.parse(tc.arguments) : {};
           } catch { /* ignore */ }
 
+          this.eventLog.append("tool_start", { name: tc.name, input: inputData });
           yield new AgentEvent({ type: "tool_start", name: tc.name, input: inputData });
+
           const result = await this.toolRegistry.execute(tc.name, inputData);
+
+          this.eventLog.append("tool_result", {
+            name: tc.name,
+            output: result.content?.slice(0, 5000) || "",
+            isError: result.isError,
+          });
           yield new AgentEvent({
             type: "tool_result",
             name: tc.name,
@@ -236,8 +480,16 @@ export class AgentEngine {
             const pEvent = pipeline.onToolResult(tc.name, inputData, result);
             if (pEvent) {
               if (pEvent.type === "phase_ready" && pEvent.nextPhase) {
+                const phaseSummary = `[${this.currentPhase.toUpperCase()} completed]: ${pEvent.message || ""}`;
+                this._phaseSummaries.push(phaseSummary);
+                this.eventLog.append("phase_transition", {
+                  from: this.currentPhase,
+                  to: pEvent.nextPhase,
+                  summary: phaseSummary,
+                });
                 this.currentPhase = pEvent.nextPhase;
                 this._registerToolsForPhase(this.currentPhase);
+                this.saveState();
               }
               yield new AgentEvent({
                 type: "pipeline_event",
@@ -248,6 +500,7 @@ export class AgentEngine {
         }
 
       } catch (err) {
+        this.eventLog.append("error", { message: err.message });
         yield new AgentEvent({ type: "error", message: err.message });
         return;
       }