chainlesschain 0.162.30 → 0.162.31

package/src/repl/agent-repl.js

@@ -65,6 +65,9 @@ import {
   agentLoop as coreAgentLoop,
   formatToolArgs,
 } from "../runtime/agent-core.js";
+import { expandFileRefs } from "../runtime/file-ref-expander.js";
+import { composeSystemPrompt } from "../runtime/system-prompt.js";
+import { makeFallbackChatFn } from "../runtime/fallback-model.js";
 
 /**
  * Reference to the runtime DB for hook execution (set during startAgentRepl)
@@ -150,6 +153,26 @@ export async function startAgentRepl(options = {}) {
   let provider = options.provider || "ollama";
   const baseUrl = options.baseUrl || "http://localhost:11434";
   const apiKey = options.apiKey || null;
+  // Extra workspace roots (--add-dir): advertised in the system prompt and
+  // spanned by search_files.
+  const additionalDirectories = Array.isArray(options.additionalDirectories)
+    ? options.additionalDirectories
+    : [];
+
+  // --fallback-model: retry a turn's LLM call once on a backup model when the
+  // primary errors out (overload / network). Built once; passed into every
+  // agentLoop call via chatFn. Undefined when no fallback configured.
+  const _fallbackChatFn = options.fallbackModel
+    ? makeFallbackChatFn({
+        fallbackModel: options.fallbackModel,
+        onFallback: ({ from, to, error }) =>
+          logger.info(
+            chalk.yellow(
+              `[fallback] model "${from}" failed (${error}); retrying with "${to}"`,
+            ),
+          ),
+      })
+    : undefined;
 
   // Bootstrap runtime (best-effort, DB not required)
   let db = null;
@@ -299,7 +322,18 @@ export async function startAgentRepl(options = {}) {
   }
 
   const messages = [
-    { role: "system", content: buildSystemPrompt(process.cwd()) },
+    {
+      role: "system",
+      // --system-prompt replaces the built-in prompt; --append-system-prompt
+      // extends it (parity with the headless runners).
+      content: composeSystemPrompt(
+        buildSystemPrompt(process.cwd(), { additionalDirectories }),
+        {
+          systemPrompt: options.systemPrompt,
+          appendSystemPrompt: options.appendSystemPrompt,
+        },
+      ),
+    },
   ];
 
   // Deep Agents Deploy Phase 1 — load agent bundle if --bundle provided.
@@ -1418,8 +1452,28 @@ export async function startAgentRepl(options = {}) {
       logger.verbose(`[hook] prompt rewritten by UserPromptSubmit hook`);
     }
 
+    // Expand @path file references into context blocks (Claude-Code parity),
+    // so `review @src/x.js` injects the file contents. Typo'd paths are warned
+    // about and left as-is.
+    let userContent = effectivePrompt;
+    try {
+      const fileRefs = expandFileRefs(effectivePrompt, { cwd: process.cwd() });
+      userContent = fileRefs.prompt;
+      for (const w of fileRefs.warnings) {
+        logger.info(chalk.yellow(`[@ref] ${w}`));
+      }
+      if (fileRefs.refs.length > 0) {
+        const summary = fileRefs.refs
+          .map((r) => `${r.rel}${r.kind === "dir" ? "/" : ""}`)
+          .join(", ");
+        logger.verbose(`[@ref] injected: ${summary}`);
+      }
+    } catch (err) {
+      logger.verbose(`[@ref] expansion skipped: ${err.message}`);
+    }
+
     // Add user message
-    messages.push({ role: "user", content: effectivePrompt });
+    messages.push({ role: "user", content: userContent });
 
     // Slot-filling: detect intent and fill missing parameters interactively
     try {
@@ -1476,9 +1530,11 @@ export async function startAgentRepl(options = {}) {
         iterationBudget,
         sessionId,
         cwd: process.cwd(),
+        additionalDirectories,
         prepareCall: defaultPrepareCall,
         approvalGate: _approvalGate,
         mcpClient: _bundleMcpClient || undefined,
+        chatFn: _fallbackChatFn,
       });
 
       if (sessionId && usageEvents?.length) {

package/src/repl/chat-repl.js

@@ -12,6 +12,7 @@ import readline from "readline";
 import chalk from "chalk";
 import { logger } from "../lib/logger.js";
 import { BUILT_IN_PROVIDERS } from "../lib/llm-providers.js";
+import { expandFileRefs } from "../runtime/file-ref-expander.js";
 import {
   streamOllama,
   streamOpenAI,
@@ -153,8 +154,22 @@ export async function startChatRepl(options = {}) {
       }
     }
 
+    // Expand @path file references into context blocks (Claude-Code parity),
+    // so `summarize @notes.md` injects the file. The model sees the expanded
+    // content; the JSONL log keeps the original line for readability.
+    let userContent = trimmed;
+    try {
+      const fileRefs = expandFileRefs(trimmed, { cwd: process.cwd() });
+      userContent = fileRefs.prompt;
+      for (const w of fileRefs.warnings) {
+        logger.info(chalk.yellow(`[@ref] ${w}`));
+      }
+    } catch (err) {
+      logger.verbose?.(`[@ref] expansion skipped: ${err.message}`);
+    }
+
     // Add user message
-    messages.push({ role: "user", content: trimmed });
+    messages.push({ role: "user", content: userContent });
 
     // Stream the response
     process.stdout.write(chalk.blue("ai>  "));

package/src/runtime/agent-core.js

@@ -338,9 +338,12 @@ function _buildPersonaPrompt(persona, envLines, cwd) {
  *  4. Default hardcoded prompt → fallback when no persona
  *
  * @param {string} [cwd] - working directory
+ * @param {object} [opts]
+ * @param {string[]} [opts.additionalDirectories] - extra workspace roots
+ *   (absolute paths) the agent may read/search/edit beyond `cwd`.
  * @returns {string} complete system prompt
  */
-export function buildSystemPrompt(cwd) {
+export function buildSystemPrompt(cwd, opts = {}) {
   const dir = cwd || process.cwd();
 
   // Check for project persona
@@ -393,6 +396,19 @@ export function buildSystemPrompt(cwd) {
     // Non-critical
   }
 
+  // Advertise extra workspace roots (--add-dir) so the model knows it may
+  // reach beyond cwd and which absolute paths to use.
+  const extraDirs = Array.isArray(opts.additionalDirectories)
+    ? opts.additionalDirectories.filter(Boolean)
+    : [];
+  if (extraDirs.length > 0) {
+    prompt +=
+      `\n\n## Additional working directories\n` +
+      `Beyond the current working directory, you may read, search, and edit ` +
+      `files under these absolute roots. Pass absolute paths to access them:\n` +
+      extraDirs.map((d) => `- ${d}`).join("\n");
+  }
+
   return prompt;
 }
 
@@ -529,6 +545,7 @@ export async function executeTool(name, args, context = {}) {
       shellPolicyOverrides: context.shellPolicyOverrides || null,
       approvalGate: context.approvalGate || null,
       shellConfirm: context.shellConfirm || null,
+      additionalDirectories: context.additionalDirectories || null,
     });
   } catch (err) {
     if (hookDb) {
@@ -598,6 +615,7 @@ async function executeToolInner(
     shellPolicyOverrides,
     approvalGate,
     shellConfirm,
+    additionalDirectories,
   },
 ) {
   const localToolDescriptor =
@@ -962,41 +980,52 @@ async function executeToolInner(
     }
 
     case "search_files": {
-      const dir = args.directory ? path.resolve(cwd, args.directory) : cwd;
-      try {
-        if (args.content_search) {
-          const cmd =
-            process.platform === "win32"
-              ? `findstr /s /i /n "${args.pattern}" *`
-              : `grep -r -l -i "${args.pattern}" . --include="*" 2>/dev/null | head -20`;
-          const output = execSync(cmd, {
-            cwd: dir,
-            encoding: "utf8",
-            timeout: 10000,
-          });
-          return attachDescriptor({
-            matches: output.trim().split("\n").slice(0, 20),
-          });
-        } else {
-          const cmd =
-            process.platform === "win32"
-              ? `dir /s /b *${args.pattern}* 2>NUL`
-              : `find . -name "*${args.pattern}*" -type f 2>/dev/null | head -20`;
+      // An explicit directory scopes the search to one root; otherwise span
+      // cwd plus any --add-dir roots so cross-package searches find matches.
+      const extraRoots = Array.isArray(additionalDirectories)
+        ? additionalDirectories.filter(Boolean)
+        : [];
+      const roots = args.directory
+        ? [path.resolve(cwd, args.directory)]
+        : [cwd, ...extraRoots];
+      const isContent = Boolean(args.content_search);
+      const cmd = isContent
+        ? process.platform === "win32"
+          ? `findstr /s /i /n "${args.pattern}" *`
+          : `grep -r -l -i "${args.pattern}" . --include="*" 2>/dev/null | head -20`
+        : process.platform === "win32"
+          ? `dir /s /b *${args.pattern}* 2>NUL`
+          : `find . -name "*${args.pattern}*" -type f 2>/dev/null | head -20`;
+
+      const hits = [];
+      const seen = new Set();
+      for (const root of roots) {
+        if (hits.length >= 20) break;
+        try {
+          if (!fs.existsSync(root)) continue;
           const output = execSync(cmd, {
-            cwd: dir,
+            cwd: root,
             encoding: "utf8",
             timeout: 10000,
           });
-          return attachDescriptor({
-            files: output.trim().split("\n").filter(Boolean).slice(0, 20),
-          });
+          for (const line of output.trim().split("\n")) {
+            const v = line.trim();
+            if (!v || seen.has(v)) continue;
+            // Qualify with the root so multi-root results stay unambiguous.
+            const labeled = roots.length > 1 ? `${root}: ${v}` : v;
+            seen.add(v);
+            hits.push(labeled);
+            if (hits.length >= 20) break;
+          }
+        } catch {
+          // No matches in this root — continue to the next.
         }
-      } catch {
-        return attachDescriptor({
-          files: [],
-          message: "No matches found",
-        });
       }
+
+      if (hits.length === 0) {
+        return attachDescriptor({ files: [], message: "No matches found" });
+      }
+      return attachDescriptor(isContent ? { matches: hits } : { files: hits });
     }
 
     case "list_dir": {
@@ -1608,9 +1637,16 @@ export async function chatWithTools(rawMessages, options) {
   } = options;
 
   const persona = _loadProjectPersona(options.cwd);
+  // Merge the project-persona deny-list with any caller-supplied deny-list
+  // (e.g. headless `--disallowed-tools`). Without this merge the caller's
+  // deny-list is silently dropped and the tool stays callable.
+  const mergedDisabledTools = [
+    ...(Array.isArray(persona?.toolsDisabled) ? persona.toolsDisabled : []),
+    ...(Array.isArray(options.disabledTools) ? options.disabledTools : []),
+  ];
   const tools = getAgentToolDefinitions({
     names: options.enabledToolNames,
-    disabledTools: persona?.toolsDisabled,
+    disabledTools: mergedDisabledTools,
     extraTools: [
       ...(options.hostManagedToolPolicy?.toolDefinitions || []),
       ...(options.extraToolDefinitions || []),
@@ -1842,6 +1878,7 @@ export async function* agentLoop(messages, options) {
     shellPolicyOverrides: options.shellPolicyOverrides || null,
     approvalGate: options.approvalGate || null,
     shellConfirm: options.shellConfirm || null,
+    additionalDirectories: options.additionalDirectories || null,
   };
 
   throwIfAborted(signal);

package/src/runtime/fallback-model.js

@@ -0,0 +1,109 @@
+/**
+ * `--fallback-model` support — Claude-Code parity for unattended runs.
+ *
+ * Wraps the agent loop's LLM call so a single request that fails with a
+ * *retryable* error (overload / rate-limit / transient network) is transparently
+ * re-issued once with a backup model. Because it sits at the chatFn seam
+ * (`agentLoop` uses `options.chatFn || chatWithTools`), fallback needs no changes
+ * to the runners — the wrapped fn is passed in via `options.chatFn`.
+ *
+ * Same-provider only: the fallback swaps `options.model`, keeping the configured
+ * provider / baseUrl / apiKey. Cross-provider fallback is a larger feature.
+ */
+
+import { chatWithTools } from "./agent-core.js";
+
+// Heuristics for "try again on a different model" — overloaded backends,
+// rate limits, and transient connectivity. Deliberately conservative: a 4xx
+// that is not 429 (bad request / auth) is NOT retried.
+const RETRYABLE_PATTERNS = [
+  /overload/i,
+  /rate.?limit/i,
+  /too many requests/i,
+  /temporarily unavailable/i,
+  /\b429\b/,
+  /\b50[0234]\b/,
+  /\b529\b/,
+  /timeout/i,
+  /timed out/i,
+  /ETIMEDOUT/i,
+  /ECONNREFUSED/i,
+  /ECONNRESET/i,
+  /ENOTFOUND/i,
+  /EAI_AGAIN/i,
+  /socket hang up/i,
+  /fetch failed/i,
+  /network error/i,
+];
+
+/**
+ * Decide whether an error from an LLM call warrants a fallback retry.
+ * @param {any} err
+ * @returns {boolean}
+ */
+export function isRetryableModelError(err) {
+  if (!err) return false;
+  const status =
+    typeof err.status === "number"
+      ? err.status
+      : typeof err.statusCode === "number"
+        ? err.statusCode
+        : null;
+  if (status === 429) return true;
+  if (status !== null && status >= 500 && status <= 599) return true;
+
+  const parts = [
+    err.message,
+    typeof err.code === "string" ? err.code : "",
+    err.cause?.message,
+    err.cause?.code,
+  ]
+    .filter(Boolean)
+    .join(" ");
+  return RETRYABLE_PATTERNS.some((re) => re.test(parts));
+}
+
+/**
+ * Build a chatFn that retries once on the fallback model.
+ *
+ * @param {object} opts
+ * @param {string} opts.fallbackModel              backup model name (required)
+ * @param {Function} [opts.baseChatFn=chatWithTools] underlying LLM call
+ * @param {Function} [opts.isRetryable]            error predicate (testing seam)
+ * @param {Function} [opts.onFallback]             notified ({from,to,error}) on retry
+ * @returns {Function} a (messages, options) => Promise<result> chatFn
+ */
+export function makeFallbackChatFn(opts = {}) {
+  const fallbackModel = opts.fallbackModel;
+  const baseChatFn = opts.baseChatFn || chatWithTools;
+  const isRetryable = opts.isRetryable || isRetryableModelError;
+  const onFallback = opts.onFallback;
+
+  return async function chatWithFallback(messages, options = {}) {
+    try {
+      return await baseChatFn(messages, options);
+    } catch (err) {
+      const primaryModel = options.model;
+      // Skip a no-op retry when the fallback is the same model as the primary.
+      if (
+        !fallbackModel ||
+        fallbackModel === primaryModel ||
+        !isRetryable(err)
+      ) {
+        throw err;
+      }
+      if (typeof onFallback === "function") {
+        try {
+          onFallback({
+            from: primaryModel,
+            to: fallbackModel,
+            error: err?.message || String(err),
+          });
+        } catch {
+          // Notification is best-effort — never mask the retry.
+        }
+      }
+      return await baseChatFn(messages, { ...options, model: fallbackModel });
+    }
+  };
+}

package/src/runtime/file-ref-expander.js

@@ -0,0 +1,258 @@
+/**
+ * `@file` reference expander — Claude-Code-style prompt file injection.
+ *
+ * Scans a prompt for `@<path>` tokens and, for each one that resolves to a real
+ * file/dir under the working directory, appends the file contents (or a dir
+ * listing) as a `<referenced-files>` context block. The original `@token` stays
+ * in the prose so the message remains readable and the model knows what the
+ * user pointed at.
+ *
+ * Design notes:
+ *  - A token only matches when `@` is at start-of-input or preceded by
+ *    whitespace / an opening bracket-quote. That skips email addresses
+ *    (`foo@bar.com`) and decorative `@` — the `@` there is preceded by a word
+ *    char and never matches.
+ *  - Non-existent paths are NOT expanded and are surfaced as warnings (so a
+ *    typo'd path is visible rather than silently dropped). Decorative `@words`
+ *    that happen not to be files simply produce no ref and no warning unless
+ *    they look path-like (contain a slash or a dot-extension).
+ *  - Files are read as UTF-8 (encoding.md rule). Binary files are skipped with
+ *    a note. Oversized files are truncated with a marker.
+ *  - All filesystem access goes through an injectable `deps` seam so unit tests
+ *    never touch the real disk (mirrors the project's `_deps` philosophy).
+ */
+
+import fsDefault from "fs";
+import pathDefault from "path";
+
+export const DEFAULT_MAX_BYTES = 100 * 1024; // 100 KB per referenced file
+export const DEFAULT_MAX_DIR_ENTRIES = 200;
+
+// `@` preceded by start / whitespace / opening bracket-quote, then a path run.
+// We capture greedily up to the next whitespace or a closing bracket-quote and
+// trim trailing sentence punctuation afterwards.
+const TOKEN_RE = /(^|[\s("'`[{])@([^\s"'`)\]}]+)/g;
+
+/** A raw token looks path-like if it has a directory separator or a file ext. */
+function looksPathLike(raw) {
+  return /[\\/]/.test(raw) || /\.[A-Za-z0-9]+$/.test(raw);
+}
+
+/** Strip trailing sentence punctuation that is unlikely to be part of a path. */
+function trimTrailingPunct(raw) {
+  return raw.replace(/[),;:!?'".]+$/g, "");
+}
+
+/**
+ * Find unique `@path` candidates in the text, in first-seen order.
+ * @returns {Array<{raw:string}>}
+ */
+export function findFileRefTokens(text) {
+  const seen = new Set();
+  const out = [];
+  let m;
+  TOKEN_RE.lastIndex = 0;
+  while ((m = TOKEN_RE.exec(text)) !== null) {
+    const raw = m[2];
+    if (!raw || seen.has(raw)) continue;
+    seen.add(raw);
+    out.push({ raw });
+  }
+  return out;
+}
+
+/** True when a buffer/string looks binary (has a NUL in the sampled prefix). */
+function looksBinary(buf) {
+  const sample = buf.slice(0, 8000);
+  for (let i = 0; i < sample.length; i++) {
+    if (sample[i] === 0) return true;
+  }
+  return false;
+}
+
+/**
+ * Resolve a single raw token to an injectable ref descriptor, or null when it
+ * does not resolve to an existing path. Tries the literal path first, then a
+ * trailing-punctuation-stripped variant (handles "see @config.json.").
+ */
+function resolveRef(raw, { cwd, fs, path, maxBytes, maxDirEntries }) {
+  const candidates = [raw];
+  const trimmed = trimTrailingPunct(raw);
+  if (trimmed && trimmed !== raw) candidates.push(trimmed);
+
+  for (const cand of candidates) {
+    const abs = path.resolve(cwd, cand);
+    let stat;
+    try {
+      stat = fs.statSync(abs);
+    } catch {
+      continue; // not this candidate
+    }
+    if (stat.isDirectory()) {
+      let entries;
+      try {
+        entries = fs.readdirSync(abs, { withFileTypes: true });
+      } catch (err) {
+        return {
+          kind: "error",
+          raw: cand,
+          rel: cand,
+          message: `cannot read directory: ${err.message}`,
+        };
+      }
+      const names = entries
+        .slice(0, maxDirEntries)
+        .map((e) => (e.isDirectory() ? `${e.name}/` : e.name))
+        .sort();
+      const truncated = entries.length > maxDirEntries;
+      return {
+        kind: "dir",
+        raw: cand,
+        rel: cand,
+        entries: names,
+        total: entries.length,
+        truncated,
+      };
+    }
+    if (stat.isFile()) {
+      let buf;
+      try {
+        buf = fs.readFileSync(abs);
+      } catch (err) {
+        return {
+          kind: "error",
+          raw: cand,
+          rel: cand,
+          message: `cannot read file: ${err.message}`,
+        };
+      }
+      if (looksBinary(buf)) {
+        return {
+          kind: "binary",
+          raw: cand,
+          rel: cand,
+          bytes: stat.size,
+        };
+      }
+      const truncated = buf.length > maxBytes;
+      const content = (truncated ? buf.slice(0, maxBytes) : buf).toString(
+        "utf-8",
+      );
+      return {
+        kind: "file",
+        raw: cand,
+        rel: cand,
+        bytes: stat.size,
+        content,
+        truncated,
+      };
+    }
+    // Something exotic (socket/fifo) — treat as non-expandable.
+    return null;
+  }
+  return null;
+}
+
+function escapeAttr(s) {
+  return String(s).replace(/"/g, "&quot;");
+}
+
+/** Render the resolved refs into a single `<referenced-files>` block. */
+function renderBlock(refs) {
+  const parts = [
+    '<referenced-files note="auto-injected from @path references">',
+  ];
+  for (const ref of refs) {
+    if (ref.kind === "file") {
+      const attrs =
+        `path="${escapeAttr(ref.rel)}" bytes="${ref.bytes}"` +
+        (ref.truncated
+          ? ` truncated="true" shown-bytes="${DEFAULT_MAX_BYTES}"`
+          : "");
+      parts.push(`<file ${attrs}>`);
+      parts.push(ref.content);
+      if (ref.truncated) {
+        parts.push(`\n… [truncated — file is ${ref.bytes} bytes]`);
+      }
+      parts.push("</file>");
+    } else if (ref.kind === "dir") {
+      const attrs =
+        `path="${escapeAttr(ref.rel)}" entries="${ref.total}"` +
+        (ref.truncated ? ` truncated="true"` : "");
+      parts.push(`<dir ${attrs}>`);
+      parts.push(ref.entries.join("\n"));
+      if (ref.truncated) {
+        parts.push(`… [truncated — ${ref.total} entries total]`);
+      }
+      parts.push("</dir>");
+    } else if (ref.kind === "binary") {
+      parts.push(
+        `<file path="${escapeAttr(ref.rel)}" bytes="${ref.bytes}" binary="true" note="binary file — contents omitted" />`,
+      );
+    }
+  }
+  parts.push("</referenced-files>");
+  return parts.join("\n");
+}
+
+/**
+ * Expand `@file` references in a prompt.
+ *
+ * @param {string} prompt
+ * @param {object} [opts]
+ * @param {string} [opts.cwd=process.cwd()]
+ * @param {number} [opts.maxBytes]
+ * @param {number} [opts.maxDirEntries]
+ * @param {object} [opts.deps]            Injection seam: { fs, path }.
+ * @returns {{ prompt:string, refs:Array, warnings:string[] }}
+ *          `prompt` is unchanged when nothing resolved; otherwise it is the
+ *          original text + a trailing `<referenced-files>` block.
+ */
+export function expandFileRefs(prompt, opts = {}) {
+  const text = typeof prompt === "string" ? prompt : "";
+  if (!text || !text.includes("@")) {
+    return { prompt: text, refs: [], warnings: [] };
+  }
+  const fs = opts.deps?.fs || fsDefault;
+  const path = opts.deps?.path || pathDefault;
+  const cwd = opts.cwd || process.cwd();
+  const maxBytes = Number.isFinite(opts.maxBytes)
+    ? opts.maxBytes
+    : DEFAULT_MAX_BYTES;
+  const maxDirEntries = Number.isFinite(opts.maxDirEntries)
+    ? opts.maxDirEntries
+    : DEFAULT_MAX_DIR_ENTRIES;
+
+  const tokens = findFileRefTokens(text);
+  const refs = [];
+  const warnings = [];
+
+  for (const { raw } of tokens) {
+    const ref = resolveRef(raw, { cwd, fs, path, maxBytes, maxDirEntries });
+    if (!ref) {
+      // Only warn for path-like tokens; bare @mentions are left alone.
+      if (looksPathLike(raw)) {
+        warnings.push(`@${raw} — no such file or directory (left as-is)`);
+      }
+      continue;
+    }
+    if (ref.kind === "error") {
+      warnings.push(`@${ref.raw} — ${ref.message} (left as-is)`);
+      continue;
+    }
+    refs.push(ref);
+  }
+
+  if (refs.length === 0) {
+    return { prompt: text, refs: [], warnings };
+  }
+
+  const block = renderBlock(refs);
+  return {
+    prompt: `${text}\n\n${block}`,
+    refs,
+    warnings,
+  };
+}
+
+export const _deps = { fs: fsDefault, path: pathDefault };