chainlesschain 0.162.30 → 0.162.32

package/src/runtime/agent-core.js

@@ -338,9 +338,12 @@ function _buildPersonaPrompt(persona, envLines, cwd) {
  *  4. Default hardcoded prompt → fallback when no persona
  *
  * @param {string} [cwd] - working directory
+ * @param {object} [opts]
+ * @param {string[]} [opts.additionalDirectories] - extra workspace roots
+ *   (absolute paths) the agent may read/search/edit beyond `cwd`.
  * @returns {string} complete system prompt
  */
-export function buildSystemPrompt(cwd) {
+export function buildSystemPrompt(cwd, opts = {}) {
   const dir = cwd || process.cwd();
 
   // Check for project persona
@@ -393,6 +396,19 @@ export function buildSystemPrompt(cwd) {
     // Non-critical
   }
 
+  // Advertise extra workspace roots (--add-dir) so the model knows it may
+  // reach beyond cwd and which absolute paths to use.
+  const extraDirs = Array.isArray(opts.additionalDirectories)
+    ? opts.additionalDirectories.filter(Boolean)
+    : [];
+  if (extraDirs.length > 0) {
+    prompt +=
+      `\n\n## Additional working directories\n` +
+      `Beyond the current working directory, you may read, search, and edit ` +
+      `files under these absolute roots. Pass absolute paths to access them:\n` +
+      extraDirs.map((d) => `- ${d}`).join("\n");
+  }
+
   return prompt;
 }
 
@@ -529,6 +545,7 @@ export async function executeTool(name, args, context = {}) {
       shellPolicyOverrides: context.shellPolicyOverrides || null,
       approvalGate: context.approvalGate || null,
       shellConfirm: context.shellConfirm || null,
+      additionalDirectories: context.additionalDirectories || null,
     });
   } catch (err) {
     if (hookDb) {
@@ -598,6 +615,7 @@ async function executeToolInner(
     shellPolicyOverrides,
     approvalGate,
     shellConfirm,
+    additionalDirectories,
   },
 ) {
   const localToolDescriptor =
@@ -962,41 +980,52 @@ async function executeToolInner(
     }
 
     case "search_files": {
-      const dir = args.directory ? path.resolve(cwd, args.directory) : cwd;
-      try {
-        if (args.content_search) {
-          const cmd =
-            process.platform === "win32"
-              ? `findstr /s /i /n "${args.pattern}" *`
-              : `grep -r -l -i "${args.pattern}" . --include="*" 2>/dev/null | head -20`;
-          const output = execSync(cmd, {
-            cwd: dir,
-            encoding: "utf8",
-            timeout: 10000,
-          });
-          return attachDescriptor({
-            matches: output.trim().split("\n").slice(0, 20),
-          });
-        } else {
-          const cmd =
-            process.platform === "win32"
-              ? `dir /s /b *${args.pattern}* 2>NUL`
-              : `find . -name "*${args.pattern}*" -type f 2>/dev/null | head -20`;
+      // An explicit directory scopes the search to one root; otherwise span
+      // cwd plus any --add-dir roots so cross-package searches find matches.
+      const extraRoots = Array.isArray(additionalDirectories)
+        ? additionalDirectories.filter(Boolean)
+        : [];
+      const roots = args.directory
+        ? [path.resolve(cwd, args.directory)]
+        : [cwd, ...extraRoots];
+      const isContent = Boolean(args.content_search);
+      const cmd = isContent
+        ? process.platform === "win32"
+          ? `findstr /s /i /n "${args.pattern}" *`
+          : `grep -r -l -i "${args.pattern}" . --include="*" 2>/dev/null | head -20`
+        : process.platform === "win32"
+          ? `dir /s /b *${args.pattern}* 2>NUL`
+          : `find . -name "*${args.pattern}*" -type f 2>/dev/null | head -20`;
+
+      const hits = [];
+      const seen = new Set();
+      for (const root of roots) {
+        if (hits.length >= 20) break;
+        try {
+          if (!fs.existsSync(root)) continue;
           const output = execSync(cmd, {
-            cwd: dir,
+            cwd: root,
             encoding: "utf8",
             timeout: 10000,
           });
-          return attachDescriptor({
-            files: output.trim().split("\n").filter(Boolean).slice(0, 20),
-          });
+          for (const line of output.trim().split("\n")) {
+            const v = line.trim();
+            if (!v || seen.has(v)) continue;
+            // Qualify with the root so multi-root results stay unambiguous.
+            const labeled = roots.length > 1 ? `${root}: ${v}` : v;
+            seen.add(v);
+            hits.push(labeled);
+            if (hits.length >= 20) break;
+          }
+        } catch {
+          // No matches in this root — continue to the next.
         }
-      } catch {
-        return attachDescriptor({
-          files: [],
-          message: "No matches found",
-        });
       }
+
+      if (hits.length === 0) {
+        return attachDescriptor({ files: [], message: "No matches found" });
+      }
+      return attachDescriptor(isContent ? { matches: hits } : { files: hits });
     }
 
     case "list_dir": {
@@ -1608,9 +1637,16 @@ export async function chatWithTools(rawMessages, options) {
   } = options;
 
   const persona = _loadProjectPersona(options.cwd);
+  // Merge the project-persona deny-list with any caller-supplied deny-list
+  // (e.g. headless `--disallowed-tools`). Without this merge the caller's
+  // deny-list is silently dropped and the tool stays callable.
+  const mergedDisabledTools = [
+    ...(Array.isArray(persona?.toolsDisabled) ? persona.toolsDisabled : []),
+    ...(Array.isArray(options.disabledTools) ? options.disabledTools : []),
+  ];
   const tools = getAgentToolDefinitions({
     names: options.enabledToolNames,
-    disabledTools: persona?.toolsDisabled,
+    disabledTools: mergedDisabledTools,
     extraTools: [
       ...(options.hostManagedToolPolicy?.toolDefinitions || []),
       ...(options.extraToolDefinitions || []),
@@ -1627,7 +1663,21 @@ export async function chatWithTools(rawMessages, options) {
   throwIfAborted(signal);
 
   if (provider === "ollama") {
-    const response = await fetch(`${baseUrl}/api/chat`, {
+    const apiUrl = `${baseUrl}/api/chat`;
+    // Real-time token deltas (Claude-Code `--include-partial-messages`): when
+    // the caller supplies an onToken hook, stream the response and forward each
+    // content chunk as it arrives. Tool calls + usage are accumulated and the
+    // same {message, usage} shape is returned, so the agent loop is unchanged.
+    // Without onToken we keep the cheaper single-shot non-streaming request.
+    if (typeof options.onToken === "function") {
+      return await _chatOllamaStreaming(
+        apiUrl,
+        { model, messages, tools },
+        options.onToken,
+        signal,
+      );
+    }
+    const response = await fetch(apiUrl, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       signal,
@@ -1781,6 +1831,109 @@ export async function chatWithTools(rawMessages, options) {
   return out;
 }
 
+// ─── Ollama streaming (token deltas for --include-partial-messages) ─────────
+//
+// Ollama `/api/chat` with `stream:true` returns NDJSON: one JSON object per
+// line, each carrying an incremental `message.content` chunk, optional
+// `message.tool_calls` (emitted whole, not byte-streamed), and a final line
+// with `done:true` + `prompt_eval_count`/`eval_count` token totals. We reduce
+// the stream line-by-line so onToken fires live, then finalize into the same
+// {message, usage} shape the non-streaming branch returns.
+
+function _ollamaInitState() {
+  return {
+    role: "assistant",
+    content: "",
+    toolCalls: null,
+    promptEval: 0,
+    evalCount: 0,
+  };
+}
+
+function _ollamaReduceLine(state, line, onToken) {
+  const s = (line || "").trim();
+  if (!s) return state;
+  let obj;
+  try {
+    obj = JSON.parse(s);
+  } catch {
+    return state; // tolerate partial/garbage lines mid-stream
+  }
+  const msg = obj.message;
+  if (msg) {
+    if (msg.role) state.role = msg.role;
+    if (typeof msg.content === "string" && msg.content) {
+      state.content += msg.content;
+      if (typeof onToken === "function") {
+        try {
+          onToken(msg.content);
+        } catch {
+          // A failing UI hook must never break the agent run.
+        }
+      }
+    }
+    if (Array.isArray(msg.tool_calls) && msg.tool_calls.length) {
+      state.toolCalls = (state.toolCalls || []).concat(msg.tool_calls);
+    }
+  }
+  if (obj.prompt_eval_count) state.promptEval = obj.prompt_eval_count;
+  if (obj.eval_count) state.evalCount = obj.eval_count;
+  return state;
+}
+
+function _ollamaFinalize(state) {
+  const message = { role: state.role, content: state.content };
+  if (state.toolCalls && state.toolCalls.length) {
+    message.tool_calls = state.toolCalls;
+  }
+  const data = { message };
+  if (state.promptEval || state.evalCount) {
+    data.usage = {
+      input_tokens: state.promptEval,
+      output_tokens: state.evalCount,
+    };
+  }
+  return data;
+}
+
+/**
+ * Pure reducer over an iterable of Ollama NDJSON lines. Exported for tests so
+ * the parse/accumulate logic can be exercised without a live HTTP stream.
+ */
+export function _accumulateOllamaStream(lines, onToken) {
+  const state = _ollamaInitState();
+  for (const line of lines) _ollamaReduceLine(state, line, onToken);
+  return _ollamaFinalize(state);
+}
+
+async function _chatOllamaStreaming(apiUrl, body, onToken, signal) {
+  const response = await fetch(apiUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    signal,
+    body: JSON.stringify({ ...body, stream: true }),
+  });
+  if (!response.ok) {
+    throw new Error(`Ollama error: ${response.status}`);
+  }
+  const state = _ollamaInitState();
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buf = "";
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buf += decoder.decode(value, { stream: true });
+    let idx;
+    while ((idx = buf.indexOf("\n")) >= 0) {
+      _ollamaReduceLine(state, buf.slice(0, idx), onToken);
+      buf = buf.slice(idx + 1);
+    }
+  }
+  if (buf.trim()) _ollamaReduceLine(state, buf, onToken);
+  return _ollamaFinalize(state);
+}
+
 function _normalizeAnthropicResponse(data) {
   const content = data.content || [];
   const textBlocks = content.filter((b) => b.type === "text");
@@ -1807,11 +1960,58 @@ function _normalizeAnthropicResponse(data) {
 
 // ─── Agent loop (async generator) ─────────────────────────────────────────
 
+// Tools that never mutate the workspace — auto-checkpoint skips these.
+const _CHECKPOINT_READ_ONLY = new Set([
+  "read_file",
+  "search_files",
+  "list_dir",
+  "list_skills",
+  "search_sessions",
+]);
+
+let _checkpointStoreP = null;
+function _loadCheckpointStore() {
+  if (!_checkpointStoreP) {
+    _checkpointStoreP = import("../lib/checkpoint-store.js");
+  }
+  return _checkpointStoreP;
+}
+
+/**
+ * Best-effort auto-checkpoint of the working tree BEFORE a mutating tool runs,
+ * so a later `cc checkpoint restore` can roll back to just before that tool.
+ * Enabled via toolContext.autoCheckpoint; uses the git engine only (no-op
+ * outside a git work tree). Never throws — checkpointing must not block a tool.
+ *
+ * @returns {Promise<string|null>} the checkpoint id, or null when skipped
+ */
+async function _autoCheckpointBeforeTool(toolContext, toolName, toolArgs) {
+  if (!toolContext?.autoCheckpoint) return null;
+  if (_CHECKPOINT_READ_ONLY.has(toolName)) return null;
+  const cwd = toolContext.cwd || process.cwd();
+  try {
+    const store = await _loadCheckpointStore();
+    if (!store.isCheckpointAvailable(cwd)) return null;
+    const res = store.createCheckpoint(cwd, {
+      session: toolContext.checkpointSession || "agent",
+      label: `before ${toolName}: ${formatToolArgs(toolName, toolArgs)}`.slice(
+        0,
+        120,
+      ),
+      skipIfUnchanged: true,
+    });
+    return res?.id || null;
+  } catch {
+    return null; // checkpoint failure must never block the tool
+  }
+}
+
 /**
  * Async generator that drives the agentic tool-use loop.
  *
  * Yields events:
  *   { type: "slot-filling", slot, question }  — when asking user for missing info
+ *   { type: "checkpoint", id, tool }          — auto-checkpoint before a mutating tool
  *   { type: "tool-executing", tool, args }
  *   { type: "tool-result", tool, result, error }
  *   { type: "response-complete", content }
@@ -1819,6 +2019,38 @@ function _normalizeAnthropicResponse(data) {
  * @param {Array} messages - mutable messages array (will be appended to)
  * @param {object} options - provider, model, baseUrl, apiKey, contextEngine, hookDb, skillLoader, cwd, slotFiller, interaction
  */
+/**
+ * Lazily build (and cache on `options`) the PromptCompressor used for in-loop
+ * auto-compaction. Returns null when the feature is off or the module can't be
+ * loaded — callers treat that as "don't compact". Cached (including null) so we
+ * import once per run, not once per iteration.
+ */
+async function _getAutoCompactor(options) {
+  if (Object.prototype.hasOwnProperty.call(options, "_autoCompactor")) {
+    return options._autoCompactor;
+  }
+  let compressor = null;
+  try {
+    const { feature } = await import("../lib/feature-flags.js");
+    if (feature("PROMPT_COMPRESSOR")) {
+      const { PromptCompressor } =
+        await import("../harness/prompt-compressor.js");
+      compressor = new PromptCompressor({
+        model: options.model,
+        provider: options.provider,
+      });
+    }
+  } catch {
+    compressor = null;
+  }
+  try {
+    options._autoCompactor = compressor;
+  } catch {
+    // options may be frozen — fine, we just re-import next iteration
+  }
+  return compressor;
+}
+
 export async function* agentLoop(messages, options) {
   // Shared iteration budget — replaces hardcoded MAX_ITERATIONS.
   // When options.iterationBudget is provided (e.g. from parent agent),
@@ -1842,6 +2074,10 @@ export async function* agentLoop(messages, options) {
     shellPolicyOverrides: options.shellPolicyOverrides || null,
     approvalGate: options.approvalGate || null,
     shellConfirm: options.shellConfirm || null,
+    additionalDirectories: options.additionalDirectories || null,
+    autoCheckpoint: options.autoCheckpoint || false,
+    checkpointSession:
+      options.checkpointSession || options.sessionId || "agent",
   };
 
   throwIfAborted(signal);
@@ -1942,6 +2178,42 @@ export async function* agentLoop(messages, options) {
       };
     }
 
+    // Headless auto-compaction (Claude-Code `--print` parity). Keeps long
+    // `-p` / `--resume` runs under the model's context window instead of
+    // growing until the provider rejects the request. Opt-out with
+    // `autoCompact: false` (the interactive REPL does this — it compacts on its
+    // own schedule). Default-on, gated by the PROMPT_COMPRESSOR flag + a size
+    // threshold inside the compressor, so it only fires for genuinely large
+    // contexts. Safe to compact here: the previous iteration always finishes
+    // its full tool_call→tool_result cycle before we loop, so `messages` has no
+    // dangling call; `preserveToolPairs` then guarantees compaction never
+    // orphans a tool result. Best-effort — a failure never aborts the run.
+    if (options.autoCompact !== false && messages.length > 4) {
+      try {
+        const compactor = await _getAutoCompactor(options);
+        if (compactor && compactor.shouldAutoCompact(messages)) {
+          const { messages: compacted, stats } = await compactor.compress(
+            messages,
+            { preserveToolPairs: true },
+          );
+          if (stats.saved > 0 && compacted.length < messages.length) {
+            messages.splice(0, messages.length, ...compacted);
+            if (typeof options.onCompaction === "function") {
+              try {
+                options.onCompaction(stats, compacted);
+              } catch {
+                // persistence is best-effort
+              }
+            }
+            yield { type: "compaction", stats, runId };
+          }
+        }
+      } catch (_e) {
+        if (isAbortError(_e) || signal?.aborted) throw _e;
+        // Compaction is best-effort — proceed with the uncompacted messages.
+      }
+    }
+
     // Turn-scoped context injection (open-agents prepareCall parity).
     // prepareCall runs fresh each iteration and returns an ephemeral
     // system-message supplement that is NOT persisted to messages history.
@@ -2013,6 +2285,15 @@ export async function* agentLoop(messages, options) {
         toolArgs = {};
       }
 
+      // Auto-checkpoint the work tree before a mutating tool (opt-in), so the
+      // user can `cc checkpoint restore` back to just before this call.
+      const cpId = await _autoCheckpointBeforeTool(
+        toolContext,
+        toolName,
+        toolArgs,
+      );
+      if (cpId) yield { type: "checkpoint", id: cpId, tool: toolName };
+
       yield { type: "tool-executing", tool: toolName, args: toolArgs };
 
       let toolResult;

package/src/runtime/fallback-model.js

@@ -0,0 +1,109 @@
+/**
+ * `--fallback-model` support — Claude-Code parity for unattended runs.
+ *
+ * Wraps the agent loop's LLM call so a single request that fails with a
+ * *retryable* error (overload / rate-limit / transient network) is transparently
+ * re-issued once with a backup model. Because it sits at the chatFn seam
+ * (`agentLoop` uses `options.chatFn || chatWithTools`), fallback needs no changes
+ * to the runners — the wrapped fn is passed in via `options.chatFn`.
+ *
+ * Same-provider only: the fallback swaps `options.model`, keeping the configured
+ * provider / baseUrl / apiKey. Cross-provider fallback is a larger feature.
+ */
+
+import { chatWithTools } from "./agent-core.js";
+
+// Heuristics for "try again on a different model" — overloaded backends,
+// rate limits, and transient connectivity. Deliberately conservative: a 4xx
+// that is not 429 (bad request / auth) is NOT retried.
+const RETRYABLE_PATTERNS = [
+  /overload/i,
+  /rate.?limit/i,
+  /too many requests/i,
+  /temporarily unavailable/i,
+  /\b429\b/,
+  /\b50[0234]\b/,
+  /\b529\b/,
+  /timeout/i,
+  /timed out/i,
+  /ETIMEDOUT/i,
+  /ECONNREFUSED/i,
+  /ECONNRESET/i,
+  /ENOTFOUND/i,
+  /EAI_AGAIN/i,
+  /socket hang up/i,
+  /fetch failed/i,
+  /network error/i,
+];
+
+/**
+ * Decide whether an error from an LLM call warrants a fallback retry.
+ * @param {any} err
+ * @returns {boolean}
+ */
+export function isRetryableModelError(err) {
+  if (!err) return false;
+  const status =
+    typeof err.status === "number"
+      ? err.status
+      : typeof err.statusCode === "number"
+        ? err.statusCode
+        : null;
+  if (status === 429) return true;
+  if (status !== null && status >= 500 && status <= 599) return true;
+
+  const parts = [
+    err.message,
+    typeof err.code === "string" ? err.code : "",
+    err.cause?.message,
+    err.cause?.code,
+  ]
+    .filter(Boolean)
+    .join(" ");
+  return RETRYABLE_PATTERNS.some((re) => re.test(parts));
+}
+
+/**
+ * Build a chatFn that retries once on the fallback model.
+ *
+ * @param {object} opts
+ * @param {string} opts.fallbackModel              backup model name (required)
+ * @param {Function} [opts.baseChatFn=chatWithTools] underlying LLM call
+ * @param {Function} [opts.isRetryable]            error predicate (testing seam)
+ * @param {Function} [opts.onFallback]             notified ({from,to,error}) on retry
+ * @returns {Function} a (messages, options) => Promise<result> chatFn
+ */
+export function makeFallbackChatFn(opts = {}) {
+  const fallbackModel = opts.fallbackModel;
+  const baseChatFn = opts.baseChatFn || chatWithTools;
+  const isRetryable = opts.isRetryable || isRetryableModelError;
+  const onFallback = opts.onFallback;
+
+  return async function chatWithFallback(messages, options = {}) {
+    try {
+      return await baseChatFn(messages, options);
+    } catch (err) {
+      const primaryModel = options.model;
+      // Skip a no-op retry when the fallback is the same model as the primary.
+      if (
+        !fallbackModel ||
+        fallbackModel === primaryModel ||
+        !isRetryable(err)
+      ) {
+        throw err;
+      }
+      if (typeof onFallback === "function") {
+        try {
+          onFallback({
+            from: primaryModel,
+            to: fallbackModel,
+            error: err?.message || String(err),
+          });
+        } catch {
+          // Notification is best-effort — never mask the retry.
+        }
+      }
+      return await baseChatFn(messages, { ...options, model: fallbackModel });
+    }
+  };
+}