@semalt-ai/code 1.8.5 → 1.20.0

package/lib/api.js

@@ -6,6 +6,16 @@ const { URL } = require('url');
 
 const { buildToolsSchema, isUIActive } = require('./tools');
 const { TOOL_SPECS } = require('./tool_specs');
+const { dynamicToolSpecs } = require('./tool_registry');
+const { resolveApiKey } = require('./secrets');
+const { applyPromptCaching, applyReasoningEffort } = require('./payload');
+const {
+  messagesHaveImages,
+  countImages,
+  selectImageFormat,
+  resolveVisionCapability,
+  buildProviderMessages,
+} = require('./images');
 const writer = require('./ui/writer');
 const messages = require('./ui/messages');
 const dbg = require('./debug');
@@ -66,6 +76,105 @@ function debugDumpMessages(msgs) {
   }
 }
 
+// Strip client-only sibling keys from messages right before the wire. Today
+// that is the Phase 6a `_display` descriptor core (persisted on native tool
+// messages for replay fidelity). Returns the array unchanged when no message
+// carries one, so the common path allocates nothing.
+function stripInternalKeys(messages) {
+  if (!Array.isArray(messages) || !messages.some((m) => m && m._display !== undefined)) return messages;
+  return messages.map((m) => {
+    if (m && m._display !== undefined) {
+      const { _display, ...rest } = m;
+      return rest;
+    }
+    return m;
+  });
+}
+
+// Fit messages into tokenBudget tokens.
+// Uses chars/4 — aligned with estimateTokens; a deliberate under-estimate
+// for token-dense content (code, JSON, HTML) but consistent across the
+// codebase.
+//
+// Always keeps: system prompt + first non-system message (original task).
+// Drops intermediate messages oldest-first, then truncates the last tail
+// message (typically a large tool result) if still over budget.
+//
+// Pure function (no closure dependencies) — lives at module scope so it can be
+// unit-tested in isolation. Called from chatStream's proactive-trim and
+// 400/413 self-healing paths.
+function trimToTokenBudget(msgs, tokenBudget) {
+  const CHARS_PER_TOKEN = 4;
+  const system = msgs.filter((m) => m.role === 'system');
+  const nonSystem = msgs.filter((m) => m.role !== 'system');
+  if (nonSystem.length === 0) return [...system];
+
+  const pinned = nonSystem[0]; // original task — never dropped
+  let tail = nonSystem.slice(1);
+
+  const estimate = () => {
+    const all = tail.length > 0 ? [...system, pinned, ...tail] : [...system, pinned];
+    return Math.floor(JSON.stringify(all).length / CHARS_PER_TOKEN);
+  };
+
+  while (tail.length > 1 && estimate() > tokenBudget) {
+    tail = tail.slice(1);
+  }
+
+  if (tail.length === 1 && estimate() > tokenBudget) {
+    const msg = tail[0];
+    const otherChars = JSON.stringify([...system, pinned]).length;
+    const available = tokenBudget * CHARS_PER_TOKEN - otherChars - 200;
+    if (available > 0 && typeof msg.content === 'string' && msg.content.length > available) {
+      tail = [{ ...msg, content: '[…content truncated to fit model limit…]\n' + msg.content.slice(-available) }];
+    }
+  }
+
+  if (tail.length === 0 && estimate() > tokenBudget) {
+    const systemChars = JSON.stringify(system).length;
+    const available = tokenBudget * CHARS_PER_TOKEN - systemChars - 200;
+    if (available > 0 && typeof pinned.content === 'string' && pinned.content.length > available) {
+      return [...system, { ...pinned, content: '[…content truncated to fit model limit…]\n' + pinned.content.slice(-available) }];
+    }
+  }
+
+  return tail.length > 0 ? [...system, pinned, ...tail] : [...system, pinned];
+}
+
+// Estimate the context split for the counter (Variant B, display-only).
+//
+// The API returns usage.prompt_tokens PRE-SUMMED — it never breaks the prompt
+// into base (system prompt + tool specs) vs working (history + tool results).
+// So the split cannot be measured; it is ESTIMATED here from the assembled
+// payload. Both halves use the SAME char/4 estimator so they sum consistently
+// (the point of Variant B — no "real minus estimate" mixing where working would
+// look measured but secretly carry the base estimate's error). The real
+// prompt_tokens remains the authoritative anchor shown alongside this split.
+//
+//   base    = estimate(system messages) + estimate(serialized tool schema)
+//   working = estimate(every non-system message)  ← the part that grows
+//
+// Recompute PER REQUEST (cheap): the base is NOT eternally constant — it shifts
+// with native-vs-XML mode (tools live in payload.tools vs inside the system
+// prompt), dynamic tools (MCP connecting/failing mid-session), and plan-mode
+// toggling (PLAN_MODE_NOTICE). In XML mode `tools` is absent and the tool weight
+// lives inside the system prompt string, so estimating the actual system message
+// still captures it — base is never silently zero. Pure; unit-tested.
+function estimateContextSplit(msgs, tools) {
+  let systemChars = 0;
+  let workingChars = 0;
+  for (const m of (Array.isArray(msgs) ? msgs : [])) {
+    const len = JSON.stringify(m == null ? '' : m).length;
+    if (m && m.role === 'system') systemChars += len;
+    else workingChars += len;
+  }
+  const toolChars = tools ? JSON.stringify(tools).length : 0;
+  return {
+    base: Math.floor((systemChars + toolChars) / 4),
+    working: Math.floor(workingChars / 4),
+  };
+}
+
 function createApiClient({ getConfig, saveConfig, ui }) {
   const {
     BOLD,
@@ -252,6 +361,26 @@ function createApiClient({ getConfig, saveConfig, ui }) {
     });
   }
 
+  // Web search (Task W.2b). Calls the backend POST /api/search — which
+  // authenticates the Bearer token, queries SearXNG, and returns
+  // { results: [{title,url,snippet}, …] } (or an {error} envelope on failure,
+  // mapped to a thrown Error by requestJson). Modeled byte-for-byte on
+  // dashboardListModels: requireAuthToken() → requestJson(...). The optional
+  // `count` is forwarded so the backend can clamp it. The caller (the
+  // web_search tool) is responsible for catching every failure mode and
+  // surfacing a clean tool error — nothing here is special-cased.
+  function dashboardSearch(query, { count, timeout } = {}) {
+    const authToken = requireAuthToken();
+    const body = { query };
+    if (count != null) body.count = count;
+    return requestJson(dashboardUrl('/api/search'), {
+      method: 'POST',
+      timeout: timeout || 15000,
+      headers: { 'Authorization': `Bearer ${authToken}` },
+      body,
+    });
+  }
+
   function dashboardGetModelForCli(id) {
     const authToken = requireAuthToken();
     return requestJson(dashboardUrl(`/api/models/${encodeURIComponent(String(id))}/cli`), {
@@ -301,13 +430,33 @@ function createApiClient({ getConfig, saveConfig, ui }) {
     });
   }
 
-  async function chatStream(messages, { model, temperature, maxTokens, linePrefix = '', showThink = false, onToken = null, silent = false, signal = null, onTrim = null, nativeTools = true } = {}) {
+  async function chatStream(messages, { model, temperature, maxTokens, linePrefix = '', showThink = false, onToken = null, onReasoning = null, silent = false, signal = null, onTrim = null, nativeTools = true } = {}) {
     // nativeTools is plumbed through for downstream use (tools param + tool_calls parsing); no behavior change yet.
     const config = getConfig();
     const resolvedModel = model || config.default_model;
 
     if (signal && signal.aborted) throw new Error('Aborted');
 
+    // Multimodal image input (Task 5.4). When any turn carries attached images,
+    // resolve the provider content-part shape (Anthropic-style vs OpenAI-style)
+    // and FAIL LOUD for a known text-only model — never silently drop the image
+    // from the payload (constraint #2). An unknown capability (null) proceeds and
+    // lets the endpoint reject cleanly.
+    const imagesPresent = messagesHaveImages(messages);
+    let imageFormat = null;
+    if (imagesPresent) {
+      imageFormat = selectImageFormat(config, resolvedModel);
+      const vision = resolveVisionCapability(config, resolvedModel);
+      if (vision === false) {
+        const n = countImages(messages);
+        throw new Error(
+          `Model "${resolvedModel}" is not vision-capable, but ${n} image${n === 1 ? '' : 's'} ` +
+          `${n === 1 ? 'was' : 'were'} attached. Select a vision-capable model, or set ` +
+          `vision:true on the model profile if this endpoint does accept images.`,
+        );
+      }
+    }
+
     let trimNotified = false;
     function notifyTrim(info) {
       if (trimNotified) return;
@@ -317,51 +466,8 @@ function createApiClient({ getConfig, saveConfig, ui }) {
       }
     }
 
-    // Fit messages into tokenBudget tokens.
-    // Uses chars/4 — aligned with estimateTokens; a deliberate under-estimate
-    // for token-dense content (code, JSON, HTML) but consistent across the
-    // codebase.
-    //
-    // Always keeps: system prompt + first non-system message (original task).
-    // Drops intermediate messages oldest-first, then truncates the last tail
-    // message (typically a large tool result) if still over budget.
-    function trimToTokenBudget(msgs, tokenBudget) {
-      const CHARS_PER_TOKEN = 4;
-      const system = msgs.filter((m) => m.role === 'system');
-      const nonSystem = msgs.filter((m) => m.role !== 'system');
-      if (nonSystem.length === 0) return [...system];
-
-      const pinned = nonSystem[0]; // original task — never dropped
-      let tail = nonSystem.slice(1);
-
-      const estimate = () => {
-        const all = tail.length > 0 ? [...system, pinned, ...tail] : [...system, pinned];
-        return Math.floor(JSON.stringify(all).length / CHARS_PER_TOKEN);
-      };
-
-      while (tail.length > 1 && estimate() > tokenBudget) {
-        tail = tail.slice(1);
-      }
-
-      if (tail.length === 1 && estimate() > tokenBudget) {
-        const msg = tail[0];
-        const otherChars = JSON.stringify([...system, pinned]).length;
-        const available = tokenBudget * CHARS_PER_TOKEN - otherChars - 200;
-        if (available > 0 && typeof msg.content === 'string' && msg.content.length > available) {
-          tail = [{ ...msg, content: '[…content truncated to fit model limit…]\n' + msg.content.slice(-available) }];
-        }
-      }
-
-      if (tail.length === 0 && estimate() > tokenBudget) {
-        const systemChars = JSON.stringify(system).length;
-        const available = tokenBudget * CHARS_PER_TOKEN - systemChars - 200;
-        if (available > 0 && typeof pinned.content === 'string' && pinned.content.length > available) {
-          return [...system, { ...pinned, content: '[…content truncated to fit model limit…]\n' + pinned.content.slice(-available) }];
-        }
-      }
-
-      return tail.length > 0 ? [...system, pinned, ...tail] : [...system, pinned];
-    }
+    // trimToTokenBudget is a pure, module-scope helper (lifted out of this
+    // closure in Task 1.1 so it can be unit-tested directly; body unchanged).
 
     // Proactive trim: prefer a limit learned from a prior 400 overflow; otherwise
     // fall back to config.context_length (with a ~10% safety margin) as a hint.
@@ -409,7 +515,10 @@ function createApiClient({ getConfig, saveConfig, ui }) {
       const callable = Object.fromEntries(
         Object.entries(TOOL_SPECS).filter(([, spec]) => !spec.wrapper)
       );
-      payload.tools = buildToolsSchema(callable);
+      // Dynamic MCP tools (Task 3.3) advertise their schema here too, so the
+      // model can emit native tool_calls against `mcp__server__tool` names that
+      // dispatch through the same registry path as built-ins.
+      payload.tools = buildToolsSchema({ ...callable, ...dynamicToolSpecs() });
       payload.tool_choice = 'auto';
     }
 
@@ -418,14 +527,26 @@ function createApiClient({ getConfig, saveConfig, ui }) {
     async function doRequest(msgs) {
       if (dbg.isFile()) debugDumpMessages(msgs);
       validateToolCallInvariant(msgs);
-      const reqPayload = { ...payload, messages: msgs };
+      // Transform any image-bearing turn into the provider-specific multimodal
+      // content[] shape right before the wire (Task 5.4); the internal `images`
+      // field never leaves the client. The Phase 6a `_display` descriptor sibling
+      // (persisted on native tool messages for replay) is likewise client-only —
+      // strip it here so it is never fed to the model.
+      const wireMsgs = stripInternalKeys(imagesPresent ? buildProviderMessages(msgs, imageFormat) : msgs);
+      const reqPayload = { ...payload, messages: wireMsgs };
+      // Optional payload augmentations (Task 2.7): reasoning_effort for models
+      // that support it, and prompt-caching markers on the stable prefix when
+      // the user has opted in (config.prompt_caching). Both no-op otherwise.
+      applyReasoningEffort(reqPayload, config.reasoning_effort, resolvedModel, { force: !!config.reasoning_effort_force });
+      applyPromptCaching(reqPayload, config.prompt_caching === true);
       const reqBody = JSON.stringify(reqPayload);
       const res = await httpRequest(endpoint, {
         method: 'POST',
         timeout: config.request_timeout_ms,
         headers: {
           'Content-Type': 'application/json',
-          'Authorization': `Bearer ${config.api_key}`,
+          // Precedence: SEMALT_API_KEY env → OS keychain → config.api_key.
+          'Authorization': `Bearer ${resolveApiKey(config)}`,
           'Content-Length': Buffer.byteLength(reqBody),
         },
         signal,
@@ -592,10 +713,18 @@ function createApiClient({ getConfig, saveConfig, ui }) {
           };
         }
         const elapsedMs = Date.now() - startTime;
+        // Estimated base/working split (Variant B, display-only) computed from
+        // the payload ACTUALLY sent — trimmedMessages holds the final value
+        // after any 413/400-overflow retry, and payload.tools is present only in
+        // native mode (XML mode embeds tools in the system prompt, captured by
+        // the system-message estimate). Recomputed every request so it stays
+        // correct when MCP connects or plan mode toggles mid-session.
+        const contextEstimate = estimateContextSplit(trimmedMessages, payload.tools);
         resolve({
           content: fullText,
           toolCalls: nativeTools ? validToolCalls : [],
           usage,
+          context_estimate: contextEstimate,
           usage_from_provider: !!streamUsage,
           tool_calls_count: validToolCalls.length,
           finish_reason: streamFinishReason,
@@ -676,6 +805,14 @@ function createApiClient({ getConfig, saveConfig, ui }) {
               const uiActive = isUIActive();
               if (!inReasoning) {
                 inReasoning = true;
+                // Live-narration safety signal (a): the model demonstrably uses
+                // the structured reasoning_content channel this turn, so any
+                // delta.content that follows is narration, not inlined reasoning.
+                // Fire once per stream so the UI can eager-open its live gate on
+                // the native rail. Failures here must never break the stream.
+                if (typeof onReasoning === 'function') {
+                  try { onReasoning(); } catch { /* UI signal is best-effort */ }
+                }
                 if (showThink && !uiActive) {
                   // audit: allowed — non-TUI thinking output, interleaves with StreamRenderer sync writes.
                   process.stdout.write(`\n  ${FG_DARK}${DIM}⟨thinking⟩${RST}`);
@@ -773,7 +910,8 @@ function createApiClient({ getConfig, saveConfig, ui }) {
         timeout: config.request_timeout_ms,
         headers: {
           'Content-Type': 'application/json',
-          'Authorization': `Bearer ${config.api_key}`,
+          // Precedence: SEMALT_API_KEY env → OS keychain → config.api_key.
+          'Authorization': `Bearer ${resolveApiKey(config)}`,
           'Content-Length': Buffer.byteLength(body),
         },
       }, body);
@@ -812,9 +950,55 @@ function createApiClient({ getConfig, saveConfig, ui }) {
     });
   }
 
+  // Quiet, non-streaming completion. Unlike chatSync it does NOT write to
+  // scrollback or route errors through the UI — it returns the assistant text
+  // or THROWS, so a programmatic caller (the web-fetch secondary summarizer,
+  // Task W.1) can decide its own fallback. No native tools, no streaming chrome.
+  async function chatComplete(messages, { model, temperature, signal } = {}) {
+    const config = getConfig();
+    const payload = {
+      model: model || config.default_model,
+      messages,
+      temperature: typeof temperature === 'number' ? temperature : config.temperature,
+      stream: false,
+    };
+    const body = JSON.stringify(payload);
+    const res = await httpRequest(apiUrl('/v1/chat/completions'), {
+      method: 'POST',
+      timeout: config.request_timeout_ms,
+      signal: signal || undefined,
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${resolveApiKey(config)}`,
+        'Content-Length': Buffer.byteLength(body),
+      },
+    }, body);
+    return new Promise((resolve, reject) => {
+      let data = '';
+      res.setEncoding('utf8');
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        if (res.statusCode !== 200) {
+          reject(new Error(`HTTP ${res.statusCode} — ${String(data).slice(0, 200)}`));
+          return;
+        }
+        try {
+          const parsed = JSON.parse(data);
+          const content = parsed && parsed.choices && parsed.choices[0] && parsed.choices[0].message
+            ? parsed.choices[0].message.content : '';
+          resolve(content || '');
+        } catch (error) {
+          reject(new Error(`Parse error: ${error.message}`));
+        }
+      });
+      res.on('error', reject);
+    });
+  }
+
   return {
     chatStream,
     chatSync,
+    chatComplete,
     dashboardCreateChat,
     dashboardGetChat,
     dashboardGetModelForCli,
@@ -822,6 +1006,7 @@ function createApiClient({ getConfig, saveConfig, ui }) {
     dashboardListModels,
     dashboardLogout,
     dashboardSaveMessages,
+    dashboardSearch,
     dashboardWhoAmI,
     estimateTokens,
     getCliLoginStatus,
@@ -832,4 +1017,8 @@ function createApiClient({ getConfig, saveConfig, ui }) {
 
 module.exports = {
   createApiClient,
+  // Exported for unit testing (Task 1.1). Pure helper, no runtime behavior change.
+  trimToTokenBudget,
+  // Exported for unit testing the split-context counter (Variant B). Pure helper.
+  estimateContextSplit,
 };

package/lib/args.js

@@ -17,6 +17,12 @@ function parseArgs(argv) {
       case '--file':
         (opts.file = opts.file || []).push(argv[++i]);
         break;
+      case '--image':
+        // Multimodal image input (Task 5.4). Repeatable: attach one or more
+        // images (PNG/JPEG/WebP/GIF) to the user turn. Read through isPathSafe,
+        // size-checked, base64-encoded by the entry point (lib/images.js).
+        (opts.image = opts.image || []).push(argv[++i]);
+        break;
       case '-a':
       case '--analyze':
         opts.analyze = true;
@@ -24,6 +30,28 @@ function parseArgs(argv) {
       case '--dry-run':
         opts.dryRun = true;
         break;
+      case '-p':
+      case '--print':
+        opts.print = true;
+        break;
+      case '-b':
+      case '--background':
+        // Launch the task as a detached background process (Task 5.3). Used by
+        // `semalt-code run --background <prompt>`. The permission policy is fixed
+        // from the other flags at launch and cannot change after detach.
+        opts.background = true;
+        break;
+      case '--output-format': {
+        const v = argv[++i];
+        const allowed = ['text', 'json', 'stream-json'];
+        if (!allowed.includes(v)) {
+          process.stderr.write(`Error: --output-format must be one of ${allowed.join(', ')}.\n`);
+          process.exit(1);
+        }
+        opts.outputFormat = v;
+        opts.print = true; // selecting a machine format implies headless
+        break;
+      }
       case '--api-base':
         opts.apiBase = argv[++i];
         break;
@@ -55,8 +83,52 @@ function parseArgs(argv) {
       case '--readonly':
         opts.readonly = true;
         break;
-      case '--new':
-        opts.new = true;
+      case '--plan':
+        opts.plan = true;
+        break;
+      case '--no-verify':
+        // One-off skip of self-verification (Task 4.2) for this invocation, in
+        // BOTH advisory and enforcing modes. Threaded into runAgentLoop opts.
+        opts.noVerify = true;
+        break;
+      case '--max-iterations': {
+        // Cap on agent-loop iterations per turn. A positive integer caps the
+        // loop; 0 or 'unlimited' removes the cap (power-user choice). The value
+        // also flows through flagsConfigLayer (config.js) into config.max_iterations;
+        // it's consumed here so it isn't mis-parsed as a positional.
+        const v = argv[++i];
+        const ok = v !== undefined && (v === 'unlimited' || /^\d+$/.test(v));
+        if (!ok) {
+          process.stderr.write(`Error: --max-iterations requires a non-negative integer or "unlimited".\n`);
+          process.exit(1);
+        }
+        opts.maxIterations = v;
+        break;
+      }
+      case '--reasoning-effort':
+        // Consumed here so the value isn't mis-parsed as a positional; the
+        // runtime override flows through flagsConfigLayer (config.js).
+        opts.reasoningEffort = argv[++i];
+        break;
+      case '--prompt-caching':
+        opts.promptCaching = true;
+        break;
+      case '--allow-anywhere':
+        opts.allowAnywhere = true;
+        break;
+      case '--no-network':
+        // Binary network isolation (Task 4.4b): force kernel-level no-network for
+        // sandboxed commands (bwrap --unshare-net / Seatbelt deny network*). A
+        // human-only opt-in; the model can never reach it. The sandbox decision
+        // (lib/sandbox.js resolveSandboxedSpawn) reads the flag from argv directly,
+        // so this just records intent + keeps it out of the positional args.
+        opts.noNetwork = true;
+        break;
+      case '--dangerously-skip-permissions':
+        // The single explicit opt-out of ALL safety: disables the destructive
+        // command deny-list and the config-file read guard, and fully
+        // auto-approves every tool call. Pre-scanned in index.js too.
+        opts.dangerouslySkipPermissions = true;
         break;
       case '--show-think':
         opts.showThink = true;

package/lib/audit.js

@@ -28,4 +28,26 @@ function logToolCall(tag, input, approved, resultStatus) {
   }
 }
 
-module.exports = { AUDIT_LOG, logToolCall };
+// Checkpoint activity (Task 4.3). Recorded as a `checkpoint` row so the audit
+// log shows when prior file state was snapshotted before a mutation (and on
+// rewind). `seq` is the per-session checkpoint sequence number; `note` carries
+// the action + affected path(s) or the rewind outcome. Like logToolCall this
+// never throws.
+function logCheckpoint(seq, note) {
+  try {
+    let noteStr = typeof note === 'string' ? note : JSON.stringify(note);
+    if (noteStr.length > 200) noteStr = noteStr.slice(0, 197) + '...';
+    const entry = JSON.stringify({
+      ts: new Date().toISOString(),
+      tag: 'checkpoint',
+      input: `checkpoint:${seq} ${noteStr}`,
+      approved: true,
+      result: 'ok',
+    });
+    fs.appendFileSync(AUDIT_LOG, entry + '\n');
+  } catch {
+    // never throw
+  }
+}
+
+module.exports = { AUDIT_LOG, logToolCall, logCheckpoint };