kc-beta 0.1.1 → 0.2.1

package/src/agent/engine.js

@@ -18,6 +18,8 @@ import { DashboardRenderTool } from "./tools/dashboard-render.js";
 import { EvolutionCycleTool } from "./tools/evolution-cycle.js";
 import { TierDowngradeTool } from "./tools/tier-downgrade.js";
 import { AgentTool } from "./tools/agent-tool.js";
+import { WebSearchTool } from "./tools/web-search.js";
+import { SkillLoader } from "./skill-loader.js";
 import { Phase } from "./pipelines/index.js";
 import { ProjectInitializer } from "./pipelines/initializer.js";
 import { RuleExtractionPipeline } from "./pipelines/extraction.js";
@@ -25,6 +27,13 @@ import { SkillAuthoringPipeline } from "./pipelines/skill-authoring.js";
 import { SkillTestingPipeline } from "./pipelines/skill-testing.js";
 import { DistillationEngine as DistillationPipeline } from "./pipelines/distillation.js";
 import { ProductionQCPipeline } from "./pipelines/production-qc.js";
+import { EventLog } from "./event-log.js";
+import { ContextWindow } from "./context-window.js";
+import { SessionState } from "./session-state.js";
+import { estimateTokens, estimateMessagesTokens } from "./token-counter.js";
+
+// Phases where worker LLM tools are available (DISTILL mode)
+const DISTILL_PHASES = new Set([Phase.DISTILLATION, Phase.PRODUCTION_QC]);
 
 /**
  * The KC Agent conversation engine.
@@ -32,7 +41,7 @@ import { ProductionQCPipeline } from "./pipelines/production-qc.js";
  * Core loop: user message -> context assembly -> LLM API (streaming) ->
  * tool execution (if any) -> repeat until no tool calls -> turn complete.
  *
- * Uses OpenAI-compatible API. Yields AgentEvent objects.
+ * Tools are phase-gated: worker LLM tools only available in DISTILL mode.
  */
 export class AgentEngine {
   /**
@@ -45,7 +54,6 @@ export class AgentEngine {
     this.client = client;
     this.config = config;
     this.context = new ContextAssembler();
-    this.toolRegistry = new ToolRegistry();
 
     // Workspace + structural components
     this.workspace = new Workspace(config.kcWorkspaceRoot, sessionId);
@@ -54,30 +62,21 @@ export class AgentEngine {
     this.cornerCases = new CornerCaseRegistry(this.workspace.cwd);
     this.confidence = new ConfidenceScorer(this.workspace.cwd, this.cornerCases);
 
-    // Register tools
-    this.toolRegistry.register(new SandboxExecTool(this.workspace, config.kcExecTimeout));
-    this.toolRegistry.register(new WorkspaceFileTool(this.workspace, this.versionManager));
-    this.toolRegistry.register(new DocumentParseTool(this.workspace, {
-      mineruApiUrl: config.mineruApiUrl,
-      mineruApiKey: config.mineruApiKey,
-      siliconflowApiKey: config.siliconflowApiKey,
-      siliconflowBaseUrl: config.siliconflowBaseUrl,
-      ocrModel: config.ocrModelTier1,
-    }));
-    this.toolRegistry.register(new DocumentSearchTool(this.workspace));
+    // Event log (append-only JSONL, source of truth)
+    this.eventLog = new EventLog(this.workspace.cwd);
 
-    const workerLlm = new WorkerLLMCallTool(this.workspace, {
-      apiKey: config.siliconflowApiKey,
-      baseUrl: config.siliconflowBaseUrl,
+    // Context windowing
+    this.contextWindow = new ContextWindow({
+      contextLimit: config.kcContextLimit || 200000,
+      reserveForResponse: config.kcMaxTokens || 65536,
     });
-    this.toolRegistry.register(workerLlm);
-    this.toolRegistry.register(new WorkflowRunTool(this.workspace, this.versionManager, this.confidence));
-    this.toolRegistry.register(new TierDowngradeTool(this.workspace, workerLlm));
-    this.toolRegistry.register(new EvolutionCycleTool(this.workspace, this.cornerCases));
-    this.toolRegistry.register(new RuleCatalogTool(this.workspace));
-    this.toolRegistry.register(new QCSampleTool(this.workspace));
-    this.toolRegistry.register(new DashboardRenderTool(this.workspace));
-    this.toolRegistry.register(new AgentTool(this.workspace, (sid) => new AgentEngine({ client, config, sessionId: sid })));
+
+    // Session state persistence
+    this.sessionState = new SessionState(this.workspace.cwd);
+
+    // Build all tool instances (but register phase-appropriate ones)
+    this._buildTools = this._createAllTools();
+    this._phaseSummaries = [];
 
     // Pipeline system (meta-meta skills as code)
     this.currentPhase = Phase.BOOTSTRAP;
@@ -89,14 +88,210 @@ export class AgentEngine {
       [Phase.DISTILLATION]: new DistillationPipeline(this.workspace),
       [Phase.PRODUCTION_QC]: new ProductionQCPipeline(this.workspace),
     };
+
+    // Skill discovery (Claude Code pattern: index in context, full content on demand)
+    this._skillLoader = new SkillLoader(config.language);
+
+    // Register tools for initial phase
+    this.toolRegistry = new ToolRegistry();
+    this._registerToolsForPhase(this.currentPhase);
   }
 
   /**
-   * Register additional tools (called after construction for tools that need extra deps).
-   * @param {import('./tools/base.js').BaseTool} tool
+   * Create all tool instances. Separated from registration so we can
+   * re-register per phase without recreating.
    */
-  registerTool(tool) {
-    this.toolRegistry.register(tool);
+  _createAllTools() {
+    const workerLlm = new WorkerLLMCallTool(this.workspace, {
+      apiKey: this.config.llmApiKey,
+      baseUrl: this.config.llmBaseUrl,
+      authType: this.config.authType,
+    });
+
+    return {
+      // Always available (BUILD + DISTILL)
+      core: [
+        new SandboxExecTool(this.workspace, this.config.kcExecTimeout),
+        new WorkspaceFileTool(this.workspace, this.versionManager),
+        new DocumentParseTool(this.workspace, {
+          mineruApiUrl: this.config.mineruApiUrl,
+          mineruApiKey: this.config.mineruApiKey,
+          llmApiKey: this.config.llmApiKey,
+          llmBaseUrl: this.config.llmBaseUrl,
+          ocrModel: this.config.ocrModelTier1,
+        }),
+        new DocumentSearchTool(this.workspace),
+        new RuleCatalogTool(this.workspace),
+        new EvolutionCycleTool(this.workspace, this.cornerCases),
+        new DashboardRenderTool(this.workspace),
+        new AgentTool(this.workspace, (sid) => new AgentEngine({
+          client: this.client, config: this.config, sessionId: sid,
+        })),
+        new WebSearchTool(this.config.tavilyApiKey),
+      ],
+      // Distillation+ only (DISTILL mode)
+      distill: [
+        workerLlm,
+        new WorkflowRunTool(this.workspace, this.versionManager, this.confidence),
+        new TierDowngradeTool(this.workspace, workerLlm),
+        new QCSampleTool(this.workspace),
+      ],
+    };
+  }
+
+  /**
+   * Register tools appropriate for the given phase.
+   * BUILD phases get core tools only.
+   * DISTILL phases get core + worker LLM tools.
+   */
+  _registerToolsForPhase(phase) {
+    this.toolRegistry = new ToolRegistry();
+    for (const tool of this._buildTools.core) {
+      this.toolRegistry.register(tool);
+    }
+    if (DISTILL_PHASES.has(phase)) {
+      for (const tool of this._buildTools.distill) {
+        this.toolRegistry.register(tool);
+      }
+    }
+  }
+
+  /**
+   * Get current context usage statistics.
+   * @returns {{ totalTokens: number, limit: number, percentage: number }}
+   */
+  getContextStats() {
+    const systemPrompt = this.context.build({
+      skillIndex: this._skillLoader.formatForContext(),
+      pipelineState: this.pipelines[this.currentPhase]?.describeState?.() || null,
+      workspaceState: `Your workspace directory is: ${this.workspace.cwd}`,
+    });
+    const systemTokens = estimateTokens(systemPrompt);
+    const messageTokens = estimateMessagesTokens(this.history.messages);
+    const totalTokens = systemTokens + messageTokens;
+    const limit = this.config.kcContextLimit || 200000;
+    return {
+      totalTokens,
+      limit,
+      percentage: Math.round((totalTokens / limit) * 100),
+    };
+  }
+
+  /**
+   * Compact conversation history by summarizing older messages via LLM.
+   * Keeps the most recent messages intact.
+   * @param {object} [opts]
+   * @param {number} [opts.recentCount=20] - Number of recent messages to keep
+   * @returns {Promise<{removedCount: number, retainedCount: number, summaryTokens: number}|null>}
+   */
+  async compact({ recentCount = 20 } = {}) {
+    if (this.history.messages.length <= recentCount) return null;
+
+    const olderMessages = this.history.messages.slice(0, -recentCount);
+    const recentMessages = this.history.messages.slice(-recentCount);
+
+    let summary;
+    try {
+      const summaryResp = await this.client.chat({
+        model: this.config.kcModel,
+        messages: [
+          {
+            role: "system",
+            content:
+              "You are a conversation summarizer. Produce a concise summary of the following conversation. " +
+              "Focus on: decisions made, files created or modified, current state of work, key findings, " +
+              "unresolved questions. Be specific about file paths, rule IDs, and results. Keep under 2000 tokens.",
+          },
+          {
+            role: "user",
+            content: `Summarize this conversation:\n\n${JSON.stringify(olderMessages)}`,
+          },
+        ],
+        maxTokens: 2048,
+      });
+      summary = summaryResp.choices?.[0]?.message?.content || null;
+    } catch {
+      // LLM summary failed — do mechanical fallback
+      summary = null;
+    }
+
+    if (!summary) {
+      // Mechanical fallback: extract tool names and outcomes
+      const lines = ["Previous conversation summary (mechanical):"];
+      for (const msg of olderMessages) {
+        if (msg.role === "user") {
+          lines.push(`- User: ${(msg.content || "").slice(0, 100)}`);
+        } else if (msg.role === "assistant" && msg.tool_calls) {
+          for (const tc of msg.tool_calls) {
+            lines.push(`- Tool call: ${tc.function?.name}`);
+          }
+        }
+      }
+      summary = lines.join("\n");
+    }
+
+    // Replace history
+    this.history._messages = [
+      { role: "user", content: `[Previous conversation summary]\n${summary}` },
+      { role: "assistant", content: "Understood. I have the context from the summary above. Continuing from where we left off." },
+      ...recentMessages,
+    ];
+    this.history._save();
+
+    // Log compaction event
+    this.eventLog.append("compact", {
+      removedCount: olderMessages.length,
+      retainedCount: recentMessages.length,
+      summary,
+    });
+
+    return {
+      removedCount: olderMessages.length,
+      retainedCount: recentMessages.length,
+      summaryTokens: estimateTokens(summary),
+    };
+  }
+
+  /**
+   * Restore an engine from a persisted session.
+   * @param {object} opts
+   * @param {import('./llm-client.js').LLMClient} opts.client
+   * @param {object} opts.config
+   * @param {string} opts.sessionId
+   * @returns {Promise<AgentEngine>}
+   */
+  static async resume({ client, config, sessionId }) {
+    const engine = new AgentEngine({ client, config, sessionId });
+    const state = engine.sessionState;
+
+    if (state.exists) {
+      const data = state.load();
+      engine.currentPhase = data.currentPhase || Phase.BOOTSTRAP;
+      engine._phaseSummaries = data.phaseSummaries || [];
+      engine._registerToolsForPhase(engine.currentPhase);
+
+      // Restore pipeline milestones
+      const milestones = data.pipelineMilestones || {};
+      for (const [phase, mData] of Object.entries(milestones)) {
+        if (engine.pipelines[phase]?.importState) {
+          engine.pipelines[phase].importState(mData);
+        }
+      }
+
+      engine.eventLog.append("session_resume", {
+        resumedPhase: engine.currentPhase,
+        resumedFromSeq: data.lastEventSeq,
+      });
+    }
+
+    return engine;
+  }
+
+  /**
+   * Save current session state for future resume.
+   */
+  saveState() {
+    this.sessionState.save(this);
   }
 
   /**
@@ -107,19 +302,35 @@ export class AgentEngine {
    */
   async *runTurn(userMessage) {
     this.history.addUser(userMessage);
+    this.eventLog.append("user_message", { content: userMessage });
 
     // Pipeline state injection
     const pipeline = this.pipelines[this.currentPhase];
     const pipelineState = pipeline?.describeState?.() || null;
 
     const systemPrompt = this.context.build({
+      skillIndex: this._skillLoader.formatForContext(),
       pipelineState,
       workspaceState: `Your workspace directory is: ${this.workspace.cwd}`,
     });
     const tools = this.toolRegistry.schemasOpenai();
 
     while (true) {
-      const messages = [{ role: "system", content: systemPrompt }, ...this.history.messages];
+      // Apply context windowing before sending to LLM
+      const windowed = this.contextWindow.window(this.history.messages, this._phaseSummaries);
+      const messages = [{ role: "system", content: systemPrompt }, ...windowed.messages];
+
+      if (windowed.wasWindowed) {
+        this.eventLog.append("context_windowed", {
+          removedCount: windowed.removedCount,
+          totalBefore: this.history.messages.length,
+        });
+      }
+
+      this.eventLog.append("llm_start", {
+        model: this.config.kcModel,
+        messageCount: messages.length,
+      });
 
       try {
         let collectedText = "";
@@ -137,13 +348,11 @@ export class AgentEngine {
           const delta = chunk.choices?.[0]?.delta;
           if (!delta) continue;
 
-          // Stream text content
           if (delta.content) {
             yield new AgentEvent({ type: "text_delta", text: delta.content });
             collectedText += delta.content;
           }
 
-          // Accumulate tool calls from deltas
           if (delta.tool_calls) {
             for (const tcDelta of delta.tool_calls) {
               const idx = tcDelta.index;
@@ -158,7 +367,7 @@ export class AgentEngine {
           }
         }
 
-        // Build assistant message for history
+        // Log the complete assistant message (coalesced, not per-delta)
         const assistantMsg = { role: "assistant", content: collectedText || null };
         if (toolCallsAcc.size > 0) {
           assistantMsg.tool_calls = Array.from(toolCallsAcc.values()).map((tc) => ({
@@ -168,9 +377,14 @@ export class AgentEngine {
           }));
         }
         this.history.addRaw(assistantMsg);
+        this.eventLog.append("assistant_message", {
+          content: collectedText || null,
+          toolCalls: assistantMsg.tool_calls || [],
+        });
 
-        // No tool calls → turn complete
         if (toolCallsAcc.size === 0) {
+          this.eventLog.append("turn_complete", {});
+          this.saveState();
           yield new AgentEvent({ type: "turn_complete" });
           return;
         }
@@ -180,10 +394,18 @@ export class AgentEngine {
           let inputData = {};
           try {
             inputData = tc.arguments ? JSON.parse(tc.arguments) : {};
-          } catch { /* ignore parse errors */ }
+          } catch { /* ignore */ }
 
+          this.eventLog.append("tool_start", { name: tc.name, input: inputData });
           yield new AgentEvent({ type: "tool_start", name: tc.name, input: inputData });
+
           const result = await this.toolRegistry.execute(tc.name, inputData);
+
+          this.eventLog.append("tool_result", {
+            name: tc.name,
+            output: result.content?.slice(0, 5000) || "",
+            isError: result.isError,
+          });
           yield new AgentEvent({
             type: "tool_result",
             name: tc.name,
@@ -191,19 +413,27 @@ export class AgentEngine {
             isError: result.isError,
           });
 
-          // Add tool result message
           this.history.addRaw({
             role: "tool",
             tool_call_id: tc.id,
             content: result.content,
           });
 
-          // Pipeline controller: update state after tool execution
+          // Pipeline controller: update state and re-register tools on phase change
           if (pipeline?.onToolResult) {
             const pEvent = pipeline.onToolResult(tc.name, inputData, result);
             if (pEvent) {
               if (pEvent.type === "phase_ready" && pEvent.nextPhase) {
+                const phaseSummary = `[${this.currentPhase.toUpperCase()} completed]: ${pEvent.message || ""}`;
+                this._phaseSummaries.push(phaseSummary);
+                this.eventLog.append("phase_transition", {
+                  from: this.currentPhase,
+                  to: pEvent.nextPhase,
+                  summary: phaseSummary,
+                });
                 this.currentPhase = pEvent.nextPhase;
+                this._registerToolsForPhase(this.currentPhase);
+                this.saveState();
               }
               yield new AgentEvent({
                 type: "pipeline_event",
@@ -213,9 +443,8 @@ export class AgentEngine {
           }
         }
 
-        // Loop continues — send tool results back to LLM
-
       } catch (err) {
+        this.eventLog.append("error", { message: err.message });
         yield new AgentEvent({ type: "error", message: err.message });
         return;
       }

package/src/agent/event-log.js

@@ -0,0 +1,111 @@
+import fs from "node:fs";
+import path from "node:path";
+import { estimateTokens } from "./token-counter.js";
+
+/**
+ * Append-only JSONL event log for KC agent sessions.
+ * Each line is a JSON object: { seq, ts, type, data }
+ *
+ * This is the source of truth for session history. ConversationHistory
+ * and display logs become views over this log.
+ */
+export class EventLog {
+  /**
+   * @param {string} workspacePath - Session workspace directory
+   */
+  constructor(workspacePath) {
+    this._dir = path.join(workspacePath, "logs");
+    this._logPath = path.join(this._dir, "events.jsonl");
+    this._seq = 0;
+    this._estimatedTokens = 0;
+    this._initFromExisting();
+  }
+
+  /** Current sequence number */
+  get currentSeq() { return this._seq; }
+
+  /** Estimated total tokens across all events */
+  get estimatedTokens() { return this._estimatedTokens; }
+
+  /** Path to the log file */
+  get logPath() { return this._logPath; }
+
+  /**
+   * Initialize sequence counter and token estimate from existing log file.
+   */
+  _initFromExisting() {
+    if (!fs.existsSync(this._logPath)) return;
+    try {
+      const content = fs.readFileSync(this._logPath, "utf-8");
+      const lines = content.split("\n").filter((l) => l.trim());
+      for (const line of lines) {
+        try {
+          const event = JSON.parse(line);
+          if (event.seq > this._seq) this._seq = event.seq;
+          this._estimatedTokens += this._eventTokens(event);
+        } catch { /* skip malformed lines */ }
+      }
+    } catch { /* file read error, start fresh */ }
+  }
+
+  /**
+   * Append an event to the log.
+   * @param {string} type - Event type
+   * @param {object} [data] - Event payload
+   * @returns {number} The sequence number of the appended event
+   */
+  append(type, data = {}) {
+    this._seq++;
+    const event = {
+      seq: this._seq,
+      ts: new Date().toISOString(),
+      type,
+      data,
+    };
+
+    fs.mkdirSync(this._dir, { recursive: true });
+    fs.appendFileSync(this._logPath, JSON.stringify(event) + "\n", "utf-8");
+
+    this._estimatedTokens += this._eventTokens(event);
+    return this._seq;
+  }
+
+  /**
+   * Read events from the log with optional filtering.
+   * @param {object} [opts]
+   * @param {number} [opts.fromSeq] - Start reading from this sequence (inclusive)
+   * @param {number} [opts.toSeq] - Stop reading at this sequence (inclusive)
+   * @param {string[]} [opts.types] - Only return events of these types
+   * @returns {Array<object>}
+   */
+  read({ fromSeq = 0, toSeq = Infinity, types } = {}) {
+    if (!fs.existsSync(this._logPath)) return [];
+
+    const events = [];
+    const content = fs.readFileSync(this._logPath, "utf-8");
+    const lines = content.split("\n").filter((l) => l.trim());
+
+    for (const line of lines) {
+      try {
+        const event = JSON.parse(line);
+        if (event.seq < fromSeq || event.seq > toSeq) continue;
+        if (types && !types.includes(event.type)) continue;
+        events.push(event);
+      } catch { /* skip */ }
+    }
+
+    return events;
+  }
+
+  /**
+   * Estimate tokens for a single event (for running total).
+   * @param {object} event
+   * @returns {number}
+   */
+  _eventTokens(event) {
+    const dataStr = typeof event.data === "string"
+      ? event.data
+      : JSON.stringify(event.data || {});
+    return estimateTokens(dataStr);
+  }
+}