@semalt-ai/code 1.8.5 → 1.20.0

package/lib/sdk.js

@@ -0,0 +1,328 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Embedding SDK — the STABLE public facade (Task 5.2)
+// ---------------------------------------------------------------------------
+//
+// `createAgent(options)` is the supported, semver-stable way to embed the agent
+// in another program. It encapsulates assembly — api client, tool registry,
+// permission manager, sandbox, config — behind a small object:
+//
+//     const { createAgent } = require('@semalt-ai/code');
+//     const agent = createAgent({ apiBase, apiKey, model });
+//     const res = await agent.run('Summarise README.md');
+//     await agent.close();
+//
+// The shape returned by `run()` is the same structured envelope headless mode
+// produces: { result, toolCalls, usage, cost, stopReason, verifyStatus }
+// (plus `messages` for multi-turn continuation).
+//
+// SECURITY — defaults safe (constraint 2). There is no TTY in embedded use, so
+// the facade takes a programmatic permission policy:
+//   * `approve(call) => boolean|Promise<boolean>` — an async approver, OR
+//   * `rules: [...]`  — preset allow/deny/ask rules (the Task 4.1 engine).
+// With NEITHER provided the default is to REFUSE every mutating/effectful tool
+// (mirroring non-TTY), never auto-approve. The OS sandbox and the destructive-
+// command deny-list stay ON; disabling them is an explicit, documented opt-in
+// (`sandbox: { mode: 'off' }`, and the process-global
+// `--dangerously-skip-permissions`), never a side effect of embedding.
+//
+// LIFECYCLE (constraint 3). `createAgent` may open resources (MCP servers).
+// Hosts MUST call `close()` to release them.
+//
+// The unstable building blocks (runAgentLoop, createApiClient, the registries)
+// live behind the separate `@semalt-ai/code/internals` subpath — see
+// lib/internals.js.
+
+const { EventEmitter } = require('events');
+
+const { DEFAULT_CONFIG } = require('./constants');
+const { normalizeConfig, readUserConfig, loadProjectConfig, resolveMaxIterations, isNativeToolsActive } = require('./config');
+const { loadRuleLayers } = require('./permission-rules');
+const ui = require('./ui');
+const { createPermissionManager } = require('./permissions');
+const { createToolExecutor, extractToolCalls, setUIActive, isUIActive } = require('./tools');
+const { createApiClient } = require('./api');
+const { createAgentRunner } = require('./agent');
+const { createHeadlessSink } = require('./headless');
+
+const TIER_NAMES = new Set(['fs', 'exec', 'net', 'sys']);
+
+// Resolve the requested permission tiers into the allowedTiers array the
+// PermissionManager expects. Accepts 'all', a single tier string, or an array.
+function resolveTiers(allow) {
+  if (!allow) return [];
+  if (allow === 'all') return ['fs', 'exec', 'net', 'sys'];
+  const arr = Array.isArray(allow) ? allow : [allow];
+  const out = [];
+  for (const a of arr) {
+    if (a === 'all') return ['fs', 'exec', 'net', 'sys'];
+    if (TIER_NAMES.has(a)) out.push(a);
+  }
+  return out;
+}
+
+// Build the user-layer rule set from a host-supplied `rules` option. Host rules
+// are TRUSTED (the embedding program authored them), so they map to the USER
+// layer — they may widen, unlike project rules. A project `.semalt/config.json`
+// (loaded when `loadProjectRules` is on) can still only NARROW, exactly as the
+// CLI enforces.
+function buildRuleLayers({ rules, cwd, loadProjectRules }) {
+  let userRules = [];
+  if (Array.isArray(rules)) userRules = rules;
+  else if (rules && Array.isArray(rules.user)) userRules = rules.user;
+  const projectRules = (rules && Array.isArray(rules.project)) ? rules.project : [];
+  const userCfg = { permissions: { rules: userRules } };
+  let projectCfg = { permissions: { rules: projectRules } };
+  if (loadProjectRules) {
+    const fromDisk = loadProjectConfig(cwd) || {};
+    const diskRules = (fromDisk.permissions && Array.isArray(fromDisk.permissions.rules)) ? fromDisk.permissions.rules : [];
+    projectCfg = { permissions: { rules: projectRules.concat(diskRules) } };
+  }
+  return loadRuleLayers(userCfg, projectCfg, () => {});
+}
+
+// Assemble a per-instance, normalized config object. By default the host's
+// ~/.semalt-ai/config.json is NOT read (a server embedding the SDK wants
+// isolation, not the operator's personal defaults); pass `loadUserConfig: true`
+// to layer it in. Explicit options always win.
+function buildConfig(options) {
+  const {
+    apiBase, apiKey, dashboardUrl, model, contextLength,
+    config: configOverride = {},
+    loadUserConfig = false,
+    sandbox,
+    maxIterations,
+  } = options;
+
+  const base = loadUserConfig ? readUserConfig() : {};
+  const merged = { ...DEFAULT_CONFIG, ...base, ...configOverride };
+  if (apiBase != null) merged.api_base = apiBase;
+  if (apiKey != null) merged.api_key = apiKey;
+  if (dashboardUrl != null) merged.dashboard_url = dashboardUrl;
+  if (model != null) merged.default_model = model;
+  if (contextLength != null) merged.context_length = contextLength;
+  if (sandbox && typeof sandbox === 'object') merged.sandbox = { ...(merged.sandbox || {}), ...sandbox };
+  if (maxIterations != null) merged.max_iterations = maxIterations;
+  return normalizeConfig(merged);
+}
+
+function createAgent(options = {}) {
+  const cwd = options.cwd || process.cwd();
+
+  // Per-instance config behind a closure (NOT the module-global in index.js), so
+  // two createAgent() instances never share config state.
+  let config = buildConfig(options);
+  const getConfig = () => config;
+  // setConfig stays in-memory only — the SDK never writes the operator's
+  // config.json (e.g. a learned context-length is kept for this instance alone).
+  const setConfig = (next) => { config = normalizeConfig({ ...config, ...next }); };
+
+  const emitter = new EventEmitter();
+  // Don't let a missing/throwing 'error' listener crash the host process — the
+  // run result already carries failures; events are advisory.
+  emitter.on('error', () => {});
+
+  // ── Permission perimeter (defaults safe) ────────────────────────────────
+  const allowedTiers = resolveTiers(options.allow);
+  const ruleLayers = buildRuleLayers({
+    rules: options.rules,
+    cwd,
+    loadProjectRules: options.loadProjectRules === true,
+  });
+  // The host-supplied async approver. We ALWAYS install one (the host's, or a
+  // refuse-by-default fallback) so the embedded gate decision is deterministic
+  // and never falls through to the interactive TTY prompt — even if the host
+  // process happens to have a TTY. The approver is consulted only AFTER an allow
+  // tier / allow rule has had its say, so with no policy at all it refuses every
+  // mutating tool: the safe embedded default.
+  const approver = typeof options.approve === 'function'
+    ? (call) => options.approve(call)
+    : () => false;
+  // dangerouslySkipPermissions is the programmatic equivalent of the human-only
+  // CLI flag: it auto-approves the GATE. NOTE it does NOT disable the deny-list
+  // / secret-read / config-write guards — those read the process argv once at
+  // module load (see lib/tools.js); turning them off requires launching the host
+  // process with --dangerously-skip-permissions. This asymmetry is documented in
+  // the README ("Multi-instance & process-global state").
+  const skipPermissions = options.dangerouslySkipPermissions === true;
+
+  const permissionManager = createPermissionManager(ui, {
+    allowedTiers,
+    readonly: options.readonly === true,
+    skipPermissions,
+    rules: ruleLayers,
+    cwd,
+    approver,
+    // Embedded: keep stdout byte-clean — the host gets denials in the result.
+    quiet: true,
+  });
+
+  // ── OS-sandbox fallback approver ─────────────────────────────────────────
+  // When the kernel sandbox is unavailable in `auto` mode, this decides whether
+  // to run a command unsandboxed. It lives here (NOT anywhere the model can
+  // reach). Default: refuse (return false) — never a silent unsandboxed run.
+  // A host opts into running unsandboxed by passing `onUnsandboxed`.
+  const onUnsandboxed = typeof options.onUnsandboxed === 'function'
+    ? options.onUnsandboxed
+    : async () => false;
+
+  const apiClient = createApiClient({ getConfig, saveConfig: setConfig, ui });
+
+  const { runAgentLoop } = createAgentRunner({
+    chatStream: apiClient.chatStream,
+    extractToolCalls: (reply, opts = {}) => extractToolCalls(reply, {
+      repairMalformedXml: !!getConfig().repair_malformed_tool_xml,
+      ...opts,
+    }),
+    ...createToolExecutor(permissionManager, ui, getConfig, {
+      onUnsandboxed,
+      // Web-fetch secondary summarizer (Task W.1) — same chatComplete-backed
+      // call the CLI wires; http_get summarizes extracted content in isolation.
+      webChat: (messages, opts) => apiClient.chatComplete(messages, opts),
+      // Web search (Task W.2b) — the web_search tool calls the backend
+      // /api/search via dashboardSearch; compact snippets, not full pages.
+      webSearch: (query, opts) => apiClient.dashboardSearch(query, opts),
+    }),
+    permissionManager,
+    ui,
+    getConfig,
+    onUnsandboxed,
+  });
+
+  // ── MCP (optional, lazily connected) ─────────────────────────────────────
+  // Connected on first run() when config.mcp.servers is non-empty, so close()
+  // has something to tear down. NOTE: the dynamic tool registry MCP tools land
+  // in is PROCESS-GLOBAL (lib/tool_registry.js _dynamic) — concurrent instances
+  // with different MCP servers would see each other's tools. Documented limit.
+  let mcpManager = null;
+  let mcpConnected = false;
+  async function ensureMcp() {
+    const servers = (getConfig().mcp && getConfig().mcp.servers) || {};
+    if (!Object.keys(servers).length) return;
+    if (!mcpManager) {
+      const { createMcpManager } = require('./mcp/client');
+      mcpManager = createMcpManager({ getConfig, logger: (m) => emitter.emit('warning', m) });
+    }
+    if (!mcpConnected) {
+      await mcpManager.connectAll();
+      mcpConnected = true;
+    }
+  }
+
+  let closed = false;
+  let memoryWarned = false;
+
+  async function run(prompt, runOpts = {}) {
+    if (closed) throw new Error('createAgent: this agent has been close()d');
+    await ensureMcp();
+
+    // Fail-loud (memory truncation): the agent's system prompt loads project
+    // memory (AGENTS.md/CLAUDE.md) from disk; if a file was cut at the cap, warn
+    // the host once via the 'warning' event (same channel as MCP warnings) —
+    // user-facing, never added to the model prompt. Once per agent, not per run.
+    if (!memoryWarned) {
+      memoryWarned = true;
+      try {
+        const { loadProjectMemory, memoryTruncationWarnings } = require('./memory');
+        for (const w of memoryTruncationWarnings(loadProjectMemory())) emitter.emit('warning', w);
+      } catch { /* best-effort; never block a run */ }
+    }
+
+    const model = runOpts.model || getConfig().default_model;
+    const messages = Array.isArray(runOpts.messages)
+      ? runOpts.messages.slice()
+      : [{ role: 'user', content: String(prompt == null ? '' : prompt) }];
+
+    // Multimodal image input (Task 5.4). `images` accepts file paths or
+    // already-encoded { media_type, data } records; each is read through the
+    // size + path guards and attached to the latest user turn. A bad image
+    // throws here (the host gets a clear error), never a silent drop.
+    if (runOpts.images && runOpts.images.length) {
+      const { resolveImageInputs, attachImagesToLastUser } = require('./images');
+      const { isPathSafe } = require('./tools');
+      const imgs = resolveImageInputs(runOpts.images, { maxBytes: getConfig().image_max_bytes, isPathSafe });
+      attachImagesToLastUser(messages, imgs);
+    }
+    const maxIter = runOpts.maxIterations != null
+      ? resolveMaxIterations(runOpts.maxIterations)
+      : resolveMaxIterations(getConfig().max_iterations);
+    const tokenLimit = runOpts.tokenLimit != null ? runOpts.tokenLimit : (getConfig().context_length || null);
+
+    // Reuse the headless json sink to build the documented result envelope
+    // (result / toolCalls / usage / cost / stopReason / verifyStatus). We
+    // capture the object instead of writing a JSON line to stdout.
+    let envelope = null;
+    const sink = createHeadlessSink('json', (obj) => { envelope = obj; }, {
+      model,
+      priceOverrides: getConfig().pricing || {},
+    });
+
+    // Forward streaming activity to the event emitter (advisory; the result is
+    // authoritative). Merge AFTER the sink's machine callbacks so both run.
+    const eventCallbacks = {
+      onToken: (t) => emitter.emit('token', t),
+      onAssistantMessage: (m) => emitter.emit('assistant', m),
+      onToolStart: (ctx) => emitter.emit('tool-start', ctx),
+      onToolEnd: (tag, resultStr, ms, meta) => emitter.emit('tool', { tag, result: resultStr, ms, meta }),
+      onError: (e) => emitter.emit('error', e),
+    };
+    const callbacks = {};
+    for (const key of new Set([...Object.keys(eventCallbacks), ...Object.keys(sink.callbacks)])) {
+      const a = eventCallbacks[key];
+      const b = sink.callbacks[key];
+      callbacks[key] = (...args) => { if (a) a(...args); if (b) b(...args); };
+    }
+
+    // Keep the host's stdout byte-clean: suppress the tool ✓/✗ lines and the
+    // write/append permission diff for the duration (the same flag headless
+    // machine modes use). NOTE this flag is process-global (lib/tools.js) — two
+    // instances running CONCURRENTLY share it; documented under multi-instance.
+    const prevUIActive = isUIActive();
+    setUIActive(true);
+    let res;
+    try {
+      res = await runAgentLoop(messages, model, maxIter, tokenLimit, {
+        callbacks,
+        systemPrompt: runOpts.systemPrompt != null ? runOpts.systemPrompt : null,
+        systemPromptMode: getConfig().system_prompt_mode || 'system_role',
+        planMode: !!runOpts.planMode,
+        noVerify: !!runOpts.noVerify,
+        debug: false,
+      });
+    } finally {
+      setUIActive(prevUIActive);
+    }
+    sink.finalize(res);
+    emitter.emit('done', envelope);
+    // Attach the running message history so a host can continue the conversation
+    // by passing { messages } back into a subsequent run().
+    return { ...envelope, messages: res.messages };
+  }
+
+  function on(event, cb) { emitter.on(event, cb); return handle; }
+  function off(event, cb) { emitter.off(event, cb); return handle; }
+
+  async function close() {
+    if (closed) return;
+    closed = true;
+    if (mcpManager) {
+      try { await mcpManager.shutdown(); } catch { /* best-effort */ }
+    }
+    emitter.removeAllListeners();
+  }
+
+  const handle = {
+    run,
+    on,
+    off,
+    close,
+    // Read-only introspection — handy for hosts; not a stability-critical surface.
+    getConfig,
+    get cwd() { return cwd; },
+    get closed() { return closed; },
+  };
+  return handle;
+}
+
+module.exports = { createAgent };

package/lib/secrets.js

@@ -0,0 +1,211 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// API-key sourcing — env var → OS keychain → config file
+// ---------------------------------------------------------------------------
+//
+// Precedence (highest first):
+//   1. SEMALT_API_KEY environment variable
+//   2. OS keychain (macOS Keychain / Linux libsecret / Windows Credential Mgr)
+//   3. api_key in ~/.semalt-ai/config.json  (plaintext fallback)
+//
+// Keys sourced from (1) or (2) are NEVER written back to config.json and are
+// reported as redacted by configShow — only the source label is surfaced.
+//
+// Keychain access shells out to the platform's native tool (no npm deps). If
+// the tool is missing or fails, we degrade gracefully and fall through to the
+// next source rather than crashing.
+// ---------------------------------------------------------------------------
+
+const { spawnSync } = require('child_process');
+
+const ENV_VAR = 'SEMALT_API_KEY';
+const KEYCHAIN_SERVICE = 'semalt-code';
+const KEYCHAIN_ACCOUNT = 'api_key';
+
+// Process-lifetime cache so we don't re-shell to the keychain on every request.
+// undefined = not yet resolved. { source, key } once resolved.
+let _cache;
+
+function _run(cmd, args, input) {
+  try {
+    const res = spawnSync(cmd, args, {
+      input: input === undefined ? undefined : input,
+      encoding: 'utf8',
+      timeout: 5000,
+      windowsHide: true,
+    });
+    if (!res || res.error) return null;
+    if (res.status !== 0) return null;
+    return typeof res.stdout === 'string' ? res.stdout : null;
+  } catch {
+    return null;
+  }
+}
+
+// Strip only a single trailing CR/LF that the keychain tool appends — never
+// trim interior or leading characters, which could be meaningful in a key.
+function _stripTrailingNewline(s) {
+  return typeof s === 'string' ? s.replace(/\r?\n$/, '') : s;
+}
+
+// Generic keychain read for an arbitrary (service, account). Returns the stored
+// secret string or null. The platform branches mirror the API-key path; the
+// MCP OAuth token store (lib/mcp/oauth.js) reuses this so OAuth tokens land in
+// the OS keychain, never in plaintext config.
+function keychainGetItem(service, account) {
+  if (!service || !account) return null;
+  const platform = process.platform;
+  if (platform === 'darwin') {
+    const out = _run('security', [
+      'find-generic-password', '-s', service, '-a', account, '-w',
+    ]);
+    const v = _stripTrailingNewline(out);
+    return v && v.length ? v : null;
+  }
+  if (platform === 'linux') {
+    const out = _run('secret-tool', [
+      'lookup', 'service', service, 'account', account,
+    ]);
+    const v = _stripTrailingNewline(out);
+    return v && v.length ? v : null;
+  }
+  if (platform === 'win32') {
+    const ps =
+      `try {` +
+      `[void][Windows.Security.Credentials.PasswordVault,Windows.Security.Credentials,ContentType=WindowsRuntime];` +
+      `$v = New-Object Windows.Security.Credentials.PasswordVault;` +
+      `$c = $v.Retrieve('${service}','${account}');` +
+      `$c.RetrievePassword(); Write-Output $c.Password` +
+      `} catch { exit 1 }`;
+    const out = _run('powershell', ['-NoProfile', '-NonInteractive', '-Command', ps]);
+    const v = _stripTrailingNewline(out);
+    return v && v.trim().length ? v.trim() : null;
+  }
+  return null;
+}
+
+// Generic keychain write for an arbitrary (service, account). Returns true on
+// success, false if the platform tool is unavailable or the store failed.
+function keychainSetItem(service, account, value, label) {
+  if (!service || !account || typeof value !== 'string' || !value) return false;
+  const platform = process.platform;
+  if (platform === 'darwin') {
+    const out = _run('security', [
+      'add-generic-password', '-U', '-s', service, '-a', account, '-w', value,
+    ]);
+    return out !== null;
+  }
+  if (platform === 'linux') {
+    const out = _run('secret-tool', [
+      'store', '--label', label || `${service} ${account}`,
+      'service', service, 'account', account,
+    ], value);
+    return out !== null;
+  }
+  if (platform === 'win32') {
+    const escaped = value.replace(/'/g, "''");
+    const ps =
+      `try {` +
+      `[void][Windows.Security.Credentials.PasswordVault,Windows.Security.Credentials,ContentType=WindowsRuntime];` +
+      `$v = New-Object Windows.Security.Credentials.PasswordVault;` +
+      `$cred = New-Object Windows.Security.Credentials.PasswordCredential('${service}','${account}','${escaped}');` +
+      `$v.Add($cred)` +
+      `} catch { exit 1 }`;
+    const out = _run('powershell', ['-NoProfile', '-NonInteractive', '-Command', ps]);
+    return out !== null;
+  }
+  return false;
+}
+
+// Generic keychain delete for an arbitrary (service, account). Best-effort;
+// returns true if the platform tool reported success.
+function keychainDeleteItem(service, account) {
+  if (!service || !account) return false;
+  const platform = process.platform;
+  if (platform === 'darwin') {
+    const out = _run('security', ['delete-generic-password', '-s', service, '-a', account]);
+    return out !== null;
+  }
+  if (platform === 'linux') {
+    const out = _run('secret-tool', ['clear', 'service', service, 'account', account]);
+    return out !== null;
+  }
+  if (platform === 'win32') {
+    const ps =
+      `try {` +
+      `[void][Windows.Security.Credentials.PasswordVault,Windows.Security.Credentials,ContentType=WindowsRuntime];` +
+      `$v = New-Object Windows.Security.Credentials.PasswordVault;` +
+      `$c = $v.Retrieve('${service}','${account}');` +
+      `$v.Remove($c)` +
+      `} catch { exit 1 }`;
+    const out = _run('powershell', ['-NoProfile', '-NonInteractive', '-Command', ps]);
+    return out !== null;
+  }
+  return false;
+}
+
+// Read the API key from the OS keychain. Returns the key string or null.
+function keychainGet() {
+  return keychainGetItem(KEYCHAIN_SERVICE, KEYCHAIN_ACCOUNT);
+}
+
+// Store the API key in the OS keychain. Returns true on success, false if the
+// platform tool is unavailable or the store failed.
+function keychainSet(key) {
+  if (typeof key !== 'string' || !key) return false;
+  const stored = keychainSetItem(KEYCHAIN_SERVICE, KEYCHAIN_ACCOUNT, key, 'semalt-code API key');
+  if (stored) { _cache = undefined; return true; }
+  return false;
+}
+
+// Resolve the active API key following the documented precedence. `config` is
+// the loaded config object (used only for the plaintext fallback). The result
+// is cached for the env/keychain sources; the config fallback is re-read each
+// call so a runtime `config set api_key` still takes effect.
+function resolveApiKey(config) {
+  if (_cache === undefined) {
+    const env = process.env[ENV_VAR];
+    if (env && env.trim()) {
+      _cache = { source: 'env', key: env };
+    } else {
+      const kc = keychainGet();
+      if (kc && kc.trim()) {
+        _cache = { source: 'keychain', key: kc };
+      } else {
+        _cache = { source: null, key: null };
+      }
+    }
+  }
+  if (_cache.key) return _cache.key;
+  return (config && typeof config.api_key === 'string') ? config.api_key : '';
+}
+
+// Report which source the active API key comes from: 'env' | 'keychain' |
+// 'config' | 'none'. Used by configShow so users can see where their key
+// resolves from without ever printing the key itself.
+function apiKeySource(config) {
+  if (_cache === undefined) resolveApiKey(config);
+  if (_cache.source) return _cache.source;
+  const cfgKey = config && typeof config.api_key === 'string' ? config.api_key : '';
+  return cfgKey ? 'config' : 'none';
+}
+
+// Test/CLI helper: forget the cached resolution (e.g. right after storing a new
+// key) so the next resolveApiKey() re-reads env/keychain.
+function _clearCache() { _cache = undefined; }
+
+module.exports = {
+  ENV_VAR,
+  KEYCHAIN_SERVICE,
+  KEYCHAIN_ACCOUNT,
+  resolveApiKey,
+  apiKeySource,
+  keychainGet,
+  keychainSet,
+  // Generic keychain item access (Task 3.3) — used by the MCP OAuth token store.
+  keychainGetItem,
+  keychainSetItem,
+  keychainDeleteItem,
+  _clearCache,
+};