chainlesschain 0.42.3 → 0.43.0

package/src/lib/hierarchical-memory.js

@@ -15,9 +15,83 @@ export const MEMORY_CONFIG = {
 };
 
 // ─── In-memory layers ────────────────────────────────────────────
-// Map<id, { id, content, type, importance, accessCount, createdAt, lastAccessed }>
-export const _working = new Map();
-export const _shortTerm = new Map();
+// Internal storage: Map<namespace, Map<id, entry>>
+// Default namespace "global" preserves backward compatibility.
+const _workingNS = new Map();
+const _shortTermNS = new Map();
+const DEFAULT_NS = "global";
+
+function _getWorkingNS(namespace) {
+  const ns = namespace || DEFAULT_NS;
+  if (!_workingNS.has(ns)) _workingNS.set(ns, new Map());
+  return _workingNS.get(ns);
+}
+
+function _getShortTermNS(namespace) {
+  const ns = namespace || DEFAULT_NS;
+  if (!_shortTermNS.has(ns)) _shortTermNS.set(ns, new Map());
+  return _shortTermNS.get(ns);
+}
+
+// ─── Backward-compatible proxy ──────────────────────────────────
+// Existing code (and tests) access _working/_shortTerm as flat Maps:
+//   _working.size, _working.get(id), _working.clear(), _working.delete(id)
+// We proxy these to the default namespace while exposing namespaced internals.
+function _createCompatProxy(nsMap, getNS) {
+  return {
+    // Flat access — routes to default namespace
+    get size() {
+      const ns = getNS(DEFAULT_NS);
+      return ns.size;
+    },
+    get(key) {
+      const ns = getNS(DEFAULT_NS);
+      return ns.get(key);
+    },
+    set(key, value) {
+      const ns = getNS(DEFAULT_NS);
+      return ns.set(key, value);
+    },
+    has(key) {
+      const ns = getNS(DEFAULT_NS);
+      return ns.has(key);
+    },
+    delete(key) {
+      const ns = getNS(DEFAULT_NS);
+      return ns.delete(key);
+    },
+    values() {
+      const ns = getNS(DEFAULT_NS);
+      return ns.values();
+    },
+    entries() {
+      const ns = getNS(DEFAULT_NS);
+      return ns.entries();
+    },
+    keys() {
+      const ns = getNS(DEFAULT_NS);
+      return ns.keys();
+    },
+    forEach(callback) {
+      const ns = getNS(DEFAULT_NS);
+      return ns.forEach(callback);
+    },
+    [Symbol.iterator]() {
+      const ns = getNS(DEFAULT_NS);
+      return ns[Symbol.iterator]();
+    },
+    // Clear ALL namespaces (for test cleanup)
+    clear() {
+      nsMap.clear();
+    },
+    // ─── Namespace-aware internals (used by this module) ──────
+    _nsMap: nsMap,
+    _getNS: getNS,
+  };
+}
+
+export const _working = _createCompatProxy(_workingNS, _getWorkingNS);
+export const _shortTerm = _createCompatProxy(_shortTermNS, _getShortTermNS);
 
 // ─── Helpers ─────────────────────────────────────────────────────
 function generateId() {
@@ -91,6 +165,17 @@ export function ensureMemoryTables(db) {
  * Store a memory at the appropriate layer based on importance.
  * core >= 0.9, long-term >= 0.6, short-term >= 0.3, working < 0.3
  */
+/**
+ * Store a memory at the appropriate layer based on importance.
+ * core >= 0.9, long-term >= 0.6, short-term >= 0.3, working < 0.3
+ *
+ * @param {object} db - Database instance
+ * @param {string} content - Memory content
+ * @param {object} [options]
+ * @param {number} [options.importance=0.5]
+ * @param {string} [options.type="episodic"]
+ * @param {string} [options.namespace] - Namespace for in-memory isolation (default: "global")
+ */
 export function storeMemory(db, content, options = {}) {
   if (!content || !content.trim()) {
     throw new Error("Memory content cannot be empty");
@@ -101,6 +186,7 @@ export function storeMemory(db, content, options = {}) {
     Math.min(1, parseFloat(options.importance) || 0.5),
   );
   const type = options.type || "episodic";
+  const namespace = options.namespace || DEFAULT_NS;
   const id = generateId();
   const now = nowISO();
 
@@ -119,14 +205,15 @@ export function storeMemory(db, content, options = {}) {
     ).run(id, content, type, importance, "long-term", now, now);
   } else if (importance >= 0.3) {
     layer = "short-term";
-    if (_shortTerm.size >= MEMORY_CONFIG.shortTermCapacity) {
+    const nsMap = _getShortTermNS(namespace);
+    if (nsMap.size >= MEMORY_CONFIG.shortTermCapacity) {
       // Evict oldest
-      const oldest = [..._shortTerm.entries()].sort(
+      const oldest = [...nsMap.entries()].sort(
         (a, b) => new Date(a[1].lastAccessed) - new Date(b[1].lastAccessed),
       )[0];
-      if (oldest) _shortTerm.delete(oldest[0]);
+      if (oldest) nsMap.delete(oldest[0]);
     }
-    _shortTerm.set(id, {
+    nsMap.set(id, {
       id,
       content,
       type,
@@ -137,13 +224,14 @@ export function storeMemory(db, content, options = {}) {
     });
   } else {
     layer = "working";
-    if (_working.size >= MEMORY_CONFIG.workingCapacity) {
-      const oldest = [..._working.entries()].sort(
+    const nsMap = _getWorkingNS(namespace);
+    if (nsMap.size >= MEMORY_CONFIG.workingCapacity) {
+      const oldest = [...nsMap.entries()].sort(
         (a, b) => new Date(a[1].lastAccessed) - new Date(b[1].lastAccessed),
       )[0];
-      if (oldest) _working.delete(oldest[0]);
+      if (oldest) nsMap.delete(oldest[0]);
     }
-    _working.set(id, {
+    nsMap.set(id, {
       id,
       content,
       type,
@@ -162,16 +250,22 @@ export function storeMemory(db, content, options = {}) {
 /**
  * Search all memory layers with Ebbinghaus forgetting curve.
  * Strengthens recalled memories (spacing effect).
+ *
+ * When options.namespace is set, searches that namespace's in-memory maps
+ * plus the shared long-term/core DB layers. Without namespace, searches
+ * the default "global" namespace (backward compatible).
  */
 export function recallMemory(db, query, options = {}) {
   if (!query || !query.trim()) return [];
 
   const limit = Math.max(1, parseInt(options.limit) || 20);
   const pattern = query.toLowerCase();
+  const namespace = options.namespace || DEFAULT_NS;
   const results = [];
 
-  // Search working memory
-  for (const mem of _working.values()) {
+  // Search working memory (namespace-scoped)
+  const workingNS = _getWorkingNS(namespace);
+  for (const mem of workingNS.values()) {
     if (mem.content.toLowerCase().includes(pattern)) {
       const retention = calcRetention(mem.lastAccessed);
       if (retention >= MEMORY_CONFIG.recallThreshold) {
@@ -182,8 +276,9 @@ export function recallMemory(db, query, options = {}) {
     }
   }
 
-  // Search short-term memory
-  for (const mem of _shortTerm.values()) {
+  // Search short-term memory (namespace-scoped)
+  const shortTermNS = _getShortTermNS(namespace);
+  for (const mem of shortTermNS.values()) {
     if (mem.content.toLowerCase().includes(pattern)) {
       const retention = calcRetention(mem.lastAccessed);
       if (retention >= MEMORY_CONFIG.recallThreshold) {
@@ -260,41 +355,46 @@ export function consolidateMemory(db) {
   let promoted = 0;
   let forgotten = 0;
 
-  // Promote working → short-term
-  for (const [id, mem] of _working) {
-    const retention = calcRetention(mem.lastAccessed);
-    if (retention < MEMORY_CONFIG.recallThreshold) {
-      _working.delete(id);
-      forgotten++;
-    } else if (mem.accessCount >= 3) {
-      _working.delete(id);
-      _shortTerm.set(id, { ...mem, lastAccessed: nowISO() });
-      promoted++;
+  // Promote working → short-term (across all namespaces)
+  for (const [ns, nsMap] of _workingNS) {
+    const shortTermNS = _getShortTermNS(ns);
+    for (const [id, mem] of nsMap) {
+      const retention = calcRetention(mem.lastAccessed);
+      if (retention < MEMORY_CONFIG.recallThreshold) {
+        nsMap.delete(id);
+        forgotten++;
+      } else if (mem.accessCount >= 3) {
+        nsMap.delete(id);
+        shortTermNS.set(id, { ...mem, lastAccessed: nowISO() });
+        promoted++;
+      }
     }
   }
 
-  // Promote short-term → long-term
-  for (const [id, mem] of _shortTerm) {
-    const retention = calcRetention(mem.lastAccessed);
-    if (retention < MEMORY_CONFIG.recallThreshold) {
-      _shortTerm.delete(id);
-      forgotten++;
-    } else if (mem.accessCount >= 5) {
-      _shortTerm.delete(id);
-      const now = nowISO();
-      db.prepare(
-        `INSERT INTO memory_long_term (id, content, type, importance, access_count, layer, created_at, last_accessed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
-      ).run(
-        id,
-        mem.content,
-        mem.type,
-        mem.importance,
-        mem.accessCount,
-        "long-term",
-        mem.createdAt,
-        now,
-      );
-      promoted++;
+  // Promote short-term → long-term (across all namespaces)
+  for (const [, nsMap] of _shortTermNS) {
+    for (const [id, mem] of nsMap) {
+      const retention = calcRetention(mem.lastAccessed);
+      if (retention < MEMORY_CONFIG.recallThreshold) {
+        nsMap.delete(id);
+        forgotten++;
+      } else if (mem.accessCount >= 5) {
+        nsMap.delete(id);
+        const now = nowISO();
+        db.prepare(
+          `INSERT INTO memory_long_term (id, content, type, importance, access_count, layer, created_at, last_accessed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+        ).run(
+          id,
+          mem.content,
+          mem.type,
+          mem.importance,
+          mem.accessCount,
+          "long-term",
+          mem.createdAt,
+          now,
+        );
+        promoted++;
+      }
     }
   }
 
@@ -324,15 +424,19 @@ function _searchByType(db, query, type, options = {}) {
   const pattern = query.toLowerCase();
   const results = [];
 
-  // In-memory layers
-  for (const mem of _working.values()) {
-    if (mem.type === type && mem.content.toLowerCase().includes(pattern)) {
-      results.push({ ...mem, layer: "working" });
+  // In-memory layers (search all namespaces)
+  for (const [, nsMap] of _workingNS) {
+    for (const mem of nsMap.values()) {
+      if (mem.type === type && mem.content.toLowerCase().includes(pattern)) {
+        results.push({ ...mem, layer: "working" });
+      }
     }
   }
-  for (const mem of _shortTerm.values()) {
-    if (mem.type === type && mem.content.toLowerCase().includes(pattern)) {
-      results.push({ ...mem, layer: "short-term" });
+  for (const [, nsMap] of _shortTermNS) {
+    for (const mem of nsMap.values()) {
+      if (mem.type === type && mem.content.toLowerCase().includes(pattern)) {
+        results.push({ ...mem, layer: "short-term" });
+      }
     }
   }
 
@@ -412,19 +516,23 @@ export function pruneMemory(db, options = {}) {
   const maxAgeHours = parseFloat(options.maxAge) || 720; // 30 days default
   let pruned = 0;
 
-  // Prune in-memory layers by retention
-  for (const [id, mem] of _working) {
-    const retention = calcRetention(mem.lastAccessed);
-    if (retention < MEMORY_CONFIG.recallThreshold) {
-      _working.delete(id);
-      pruned++;
+  // Prune in-memory layers by retention (across all namespaces)
+  for (const [, nsMap] of _workingNS) {
+    for (const [id, mem] of nsMap) {
+      const retention = calcRetention(mem.lastAccessed);
+      if (retention < MEMORY_CONFIG.recallThreshold) {
+        nsMap.delete(id);
+        pruned++;
+      }
     }
   }
-  for (const [id, mem] of _shortTerm) {
-    const retention = calcRetention(mem.lastAccessed);
-    if (retention < MEMORY_CONFIG.recallThreshold) {
-      _shortTerm.delete(id);
-      pruned++;
+  for (const [, nsMap] of _shortTermNS) {
+    for (const [id, mem] of nsMap) {
+      const retention = calcRetention(mem.lastAccessed);
+      if (retention < MEMORY_CONFIG.recallThreshold) {
+        nsMap.delete(id);
+        pruned++;
+      }
     }
   }
 
@@ -460,12 +568,22 @@ export function getMemoryStats(db) {
     .prepare(`SELECT COUNT(*) as count FROM memory_sharing`)
     .get();
 
+  // Sum across all namespaces
+  let workingTotal = 0;
+  for (const [, nsMap] of _workingNS) workingTotal += nsMap.size;
+  let shortTermTotal = 0;
+  for (const [, nsMap] of _shortTermNS) shortTermTotal += nsMap.size;
+
   return {
-    working: _working.size,
-    shortTerm: _shortTerm.size,
+    working: workingTotal,
+    shortTerm: shortTermTotal,
     longTerm: ltCount.count,
     core: coreCount.count,
     shared: shareCount.count,
-    total: _working.size + _shortTerm.size + ltCount.count + coreCount.count,
+    namespaces: {
+      working: [..._workingNS.keys()],
+      shortTerm: [..._shortTermNS.keys()],
+    },
+    total: workingTotal + shortTermTotal + ltCount.count + coreCount.count,
   };
 }

package/src/lib/sub-agent-context.js

@@ -0,0 +1,296 @@
+/**
+ * Sub-Agent Context — isolated execution context for child agents.
+ *
+ * Provides message isolation, independent context engineering, tool whitelisting,
+ * iteration limits, and result summarization. Sub-agents run in their own
+ * context and only return a summary to the parent agent.
+ *
+ * @module sub-agent-context
+ */
+
+import crypto from "crypto";
+import { CLIContextEngineering } from "./cli-context-engineering.js";
+import { agentLoop, buildSystemPrompt, AGENT_TOOLS } from "./agent-core.js";
+
+// ─── Constants ──────────────────────────────────────────────────────────────
+
+const DEFAULT_MAX_ITERATIONS = 8;
+const SUMMARY_DIRECT_THRESHOLD = 500; // chars — below this, use result as-is
+const SUMMARY_SECTION_PATTERN =
+  /^##\s*(Summary|Result|Output|Conclusion|Answer)/im;
+const TRUNCATE_LENGTH = 500;
+
+// ─── SubAgentContext ────────────────────────────────────────────────────────
+
+export class SubAgentContext {
+  /**
+   * Factory method — creates an isolated sub-agent context.
+   *
+   * @param {object} options
+   * @param {string} options.role - Sub-agent role (e.g. "code-review", "summarizer")
+   * @param {string} options.task - Task description for the sub-agent
+   * @param {string} [options.parentId] - Parent context ID (null for root)
+   * @param {string|null} [options.inheritedContext] - Condensed context from parent
+   * @param {string[]} [options.allowedTools] - Tool whitelist (null = all tools)
+   * @param {number} [options.maxIterations] - Iteration limit
+   * @param {number} [options.tokenBudget] - Optional token budget
+   * @param {object} [options.db] - Database instance
+   * @param {object} [options.permanentMemory] - Permanent memory instance
+   * @param {object} [options.llmOptions] - LLM provider/model/key options
+   * @param {string} [options.cwd] - Working directory
+   * @returns {SubAgentContext}
+   */
+  static create(options = {}) {
+    return new SubAgentContext(options);
+  }
+
+  constructor(options = {}) {
+    this.id = `sub-${crypto.randomUUID().slice(0, 12)}`;
+    this.parentId = options.parentId || null;
+    this.role = options.role || "general";
+    this.task = options.task || "";
+    this.maxIterations = options.maxIterations || DEFAULT_MAX_ITERATIONS;
+    this.tokenBudget = options.tokenBudget || null;
+    this.inheritedContext = options.inheritedContext || null;
+    this.allowedTools = options.allowedTools || null; // null = all
+    this.cwd = options.cwd || process.cwd();
+    this.status = "active";
+    this.result = null;
+    this.createdAt = new Date().toISOString();
+    this.completedAt = null;
+
+    // ── Isolated state ──────────────────────────────────────────────
+    // Independent message history — never shared with parent
+    this.messages = [];
+
+    // Independent context engine — does not inherit parent's compaction/errors
+    this.contextEngine = new CLIContextEngineering({
+      db: options.db || null,
+      permanentMemory: options.permanentMemory || null,
+      scope: {
+        taskId: this.id,
+        role: this.role,
+        parentObjective: this.task,
+      },
+    });
+
+    // Track tool usage and token consumption
+    this._toolsUsed = [];
+    this._tokenCount = 0;
+    this._iterationCount = 0;
+
+    // LLM options for chatWithTools
+    this._llmOptions = options.llmOptions || {};
+
+    // Build isolated system prompt
+    const basePrompt = buildSystemPrompt(this.cwd);
+    const rolePrompt = `\n\n## Sub-Agent Role: ${this.role}\nYou are a focused sub-agent with the role "${this.role}". Your task is:\n${this.task}\n\nStay focused on this specific task. Be concise and return results directly.`;
+    const contextSection = this.inheritedContext
+      ? `\n\n## Parent Context\n${this.inheritedContext}`
+      : "";
+
+    this.messages.push({
+      role: "system",
+      content: basePrompt + rolePrompt + contextSection,
+    });
+  }
+
+  /**
+   * Run the sub-agent loop with the given user prompt.
+   * Collects events, enforces iteration limit, and returns a structured result.
+   *
+   * @param {string} userPrompt - The task prompt for this sub-agent
+   * @param {object} [loopOptions] - Additional options for agentLoop
+   * @returns {Promise<{ summary: string, artifacts: Array, tokenCount: number, toolsUsed: string[], iterationCount: number }>}
+   */
+  async run(userPrompt, loopOptions = {}) {
+    if (this.status !== "active") {
+      throw new Error(
+        `SubAgentContext ${this.id} is not active (status: ${this.status})`,
+      );
+    }
+
+    // Add user message
+    this.messages.push({ role: "user", content: userPrompt });
+
+    const artifacts = [];
+    let lastContent = "";
+
+    // Build filtered tool list
+    const tools = this._getFilteredTools();
+
+    // Merge LLM options
+    const options = {
+      ...this._llmOptions,
+      contextEngine: this.contextEngine,
+      cwd: this.cwd,
+      ...loopOptions,
+    };
+
+    try {
+      // Use a separate messages array for the agent loop
+      // The agentLoop will append to this.messages directly
+      const gen = agentLoop(this.messages, options);
+
+      for await (const event of gen) {
+        this._iterationCount++;
+
+        if (event.type === "tool-executing") {
+          this._toolsUsed.push(event.tool);
+        }
+
+        if (event.type === "tool-result") {
+          // Store large tool results as artifacts
+          const resultStr = JSON.stringify(event.result);
+          // Estimate token count from tool result (~4 chars per token)
+          this._tokenCount += Math.ceil(resultStr.length / 4);
+          if (resultStr.length > 2000) {
+            artifacts.push({
+              type: "tool-output",
+              tool: event.tool,
+              content: resultStr,
+              truncated: resultStr.length > 10000,
+            });
+          }
+        }
+
+        if (event.type === "response-complete") {
+          lastContent = event.content || "";
+          // Estimate token count from response content (~4 chars per token)
+          this._tokenCount += Math.ceil((lastContent.length || 0) / 4);
+        }
+
+        // Enforce token budget
+        if (this.tokenBudget && this._tokenCount >= this.tokenBudget) {
+          this.forceComplete("token-budget-exceeded");
+          break;
+        }
+
+        // Enforce iteration limit
+        if (this._iterationCount >= this.maxIterations * 3) {
+          // 3 events per iteration (executing + result + potential response)
+          break;
+        }
+      }
+    } catch (err) {
+      this.status = "failed";
+      this.completedAt = new Date().toISOString();
+      this.result = {
+        summary: `Sub-agent failed: ${err.message}`,
+        artifacts: [],
+        tokenCount: this._tokenCount,
+        toolsUsed: [...new Set(this._toolsUsed)],
+        iterationCount: this._iterationCount,
+      };
+      return this.result;
+    }
+
+    // Summarize the result
+    const summary = this.summarize(lastContent);
+
+    this.status = "completed";
+    this.completedAt = new Date().toISOString();
+    this.result = {
+      summary,
+      artifacts,
+      tokenCount: this._tokenCount,
+      toolsUsed: [...new Set(this._toolsUsed)],
+      iterationCount: this._iterationCount,
+    };
+
+    return this.result;
+  }
+
+  /**
+   * Three-level summarization strategy.
+   *
+   * 1. Direct use — result ≤ 500 chars → return as-is
+   * 2. Section extraction — if result contains ## Summary/Result → extract that section
+   * 3. Truncate + artifact — take first 500 chars, store full output as artifact
+   *
+   * @param {string} content - Raw result content
+   * @returns {string} Summarized content
+   */
+  summarize(content) {
+    if (!content || content.length === 0) {
+      return "(No output from sub-agent)";
+    }
+
+    // Strategy 1: Direct use for short content
+    if (content.length <= SUMMARY_DIRECT_THRESHOLD) {
+      return content;
+    }
+
+    // Strategy 2: Extract structured section
+    const match = content.match(SUMMARY_SECTION_PATTERN);
+    if (match) {
+      const sectionStart = match.index;
+      // Find end of section (next ## heading or end of string)
+      const rest = content.slice(sectionStart + match[0].length);
+      const nextHeading = rest.search(/^##\s/m);
+      const section =
+        nextHeading >= 0 ? rest.slice(0, nextHeading).trim() : rest.trim();
+      if (section.length > 0 && section.length <= 1000) {
+        return section;
+      }
+    }
+
+    // Strategy 3: Truncate + note
+    return (
+      content.substring(0, TRUNCATE_LENGTH) +
+      `\n...[truncated, full output: ${content.length} chars]`
+    );
+  }
+
+  /**
+   * Get filtered tools based on allowedTools whitelist.
+   * @returns {Array} Filtered AGENT_TOOLS
+   */
+  _getFilteredTools() {
+    if (!this.allowedTools || this.allowedTools.length === 0) {
+      return AGENT_TOOLS;
+    }
+    return AGENT_TOOLS.filter((t) =>
+      this.allowedTools.includes(t.function.name),
+    );
+  }
+
+  /**
+   * Force-complete this sub-agent (e.g. on timeout or parent cancellation).
+   * @param {string} [reason] - Reason for force-completion
+   */
+  forceComplete(reason = "forced") {
+    if (this.status === "active") {
+      this.status = "completed";
+      this.completedAt = new Date().toISOString();
+      if (!this.result) {
+        this.result = {
+          summary: `(Sub-agent force-completed: ${reason})`,
+          artifacts: [],
+          tokenCount: this._tokenCount,
+          toolsUsed: [...new Set(this._toolsUsed)],
+          iterationCount: this._iterationCount,
+        };
+      }
+    }
+  }
+
+  /**
+   * Get a serializable snapshot of this context (for debugging/logging).
+   */
+  toJSON() {
+    return {
+      id: this.id,
+      parentId: this.parentId,
+      role: this.role,
+      task: this.task,
+      status: this.status,
+      messageCount: this.messages.length,
+      toolsUsed: [...new Set(this._toolsUsed)],
+      tokenCount: this._tokenCount,
+      iterationCount: this._iterationCount,
+      createdAt: this.createdAt,
+      completedAt: this.completedAt,
+    };
+  }
+}