@newsails/veil-cli 1.0.1

package/core/agent.js

@@ -0,0 +1,329 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const Ajv = require('ajv');
+const paths = require('../utils/paths');
+
+const ajv = new Ajv({ allErrors: true });
+let _agentSchema = null;
+
+/**
+ * Load and cache the agent JSON schema.
+ * @returns {Function} Compiled ajv validator
+ */
+function getAgentValidator() {
+  if (_agentSchema) return _agentSchema;
+  const schema = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'schemas', 'agent.json'), 'utf8'));
+  _agentSchema = ajv.compile(schema);
+  return _agentSchema;
+}
+
+/**
+ * Resolve $AGENT_FOLDER variable in a string.
+ * @param {string} text
+ * @param {string} agentFolderPath
+ * @returns {string}
+ */
+function resolveAgentFolder(text, agentFolderPath) {
+  return text.replace(/\$AGENT_FOLDER/g, agentFolderPath);
+}
+
+/**
+ * Load a single agent from its folder.
+ * Validates agent.json against schema. Throws on invalid.
+ *
+ * @param {string} agentFolderPath - Absolute path to the agent folder
+ * @returns {Object} Loaded and validated agent definition
+ */
+function loadAgentFromFolder(agentFolderPath) {
+  const configPath = path.join(agentFolderPath, 'agent.json');
+  if (!fs.existsSync(configPath)) {
+    throw new Error(`Agent folder ${agentFolderPath} is missing agent.json`);
+  }
+
+  let config;
+  try {
+    config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+  } catch (err) {
+    throw new Error(`Failed to parse agent.json at ${configPath}: ${err.message}`);
+  }
+
+  const validate = getAgentValidator();
+  if (!validate(config)) {
+    const errors = validate.errors.map(e => `  ${e.instancePath || '(root)'}: ${e.message}`).join('\n');
+    throw new Error(`Invalid agent.json at ${configPath}:\n${errors}`);
+  }
+
+  // Read AGENT.md
+  const agentMdPath = path.join(agentFolderPath, 'AGENT.md');
+  let agentMd = '';
+  if (fs.existsSync(agentMdPath)) {
+    agentMd = resolveAgentFolder(fs.readFileSync(agentMdPath, 'utf8'), agentFolderPath);
+  }
+
+  // Read SOUL.md (optional)
+  const soulMdPath = path.join(agentFolderPath, 'SOUL.md');
+  let soulMd = null;
+  if (fs.existsSync(soulMdPath)) {
+    soulMd = resolveAgentFolder(fs.readFileSync(soulMdPath, 'utf8'), agentFolderPath);
+  }
+
+  return {
+    ...config,
+    agentFolder: agentFolderPath,
+    agentMd,
+    soulMd,
+  };
+}
+
+/**
+ * Find and load a named agent. Checks project-level first, then global.
+ *
+ * @param {string} cwd - Project root directory
+ * @param {string} name - Agent name
+ * @returns {Object} Loaded agent definition
+ */
+function loadAgent({ cwd, name }) {
+  const projectAgentDir = paths.getAgentDir(cwd, name);
+  if (fs.existsSync(projectAgentDir)) {
+    return { ...loadAgentFromFolder(projectAgentDir), source: 'project' };
+  }
+
+  const globalAgentDir = path.join(paths.getGlobalAgentsDir(), name);
+  if (fs.existsSync(globalAgentDir)) {
+    return { ...loadAgentFromFolder(globalAgentDir), source: 'global' };
+  }
+
+  const bundledAgentDir = path.join(__dirname, '..', '.veil', 'agents', name);
+  if (fs.existsSync(bundledAgentDir)) {
+    return { ...loadAgentFromFolder(bundledAgentDir), source: 'bundled' };
+  }
+
+  throw new Error(`Agent "${name}" not found. Looked in:\n  ${projectAgentDir}\n  ${globalAgentDir}\n  ${bundledAgentDir}`);
+}
+
+/**
+ * List all available agents (project-level + global, deduped by name).
+ *
+ * @param {string} cwd - Project root directory
+ * @returns {Object[]} Array of { name, description, folder, source } objects
+ */
+function listAgents({ cwd }) {
+  const found = new Map();
+
+  // Project-level agents first (take priority)
+  const projectDir = paths.getProjectAgentsDir(cwd);
+  if (fs.existsSync(projectDir)) {
+    for (const entry of fs.readdirSync(projectDir, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      const agentFolder = path.join(projectDir, entry.name);
+      const configPath = path.join(agentFolder, 'agent.json');
+      if (!fs.existsSync(configPath)) continue;
+      try {
+        const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+        found.set(entry.name, {
+          name: entry.name,
+          description: config.description || '',
+          model: config.model || '',
+          modes: config.modes ? Object.keys(config.modes).filter(m => config.modes[m]?.enabled) : [],
+          folder: agentFolder,
+          source: 'project',
+        });
+      } catch {}
+    }
+  }
+
+  // Global agents (only add if not already in project)
+  const globalDir = paths.getGlobalAgentsDir();
+  if (fs.existsSync(globalDir)) {
+    for (const entry of fs.readdirSync(globalDir, { withFileTypes: true })) {
+      if (!entry.isDirectory() || found.has(entry.name)) continue;
+      const agentFolder = path.join(globalDir, entry.name);
+      const configPath = path.join(agentFolder, 'agent.json');
+      if (!fs.existsSync(configPath)) continue;
+      try {
+        const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+        found.set(entry.name, {
+          name: entry.name,
+          description: config.description || '',
+          model: config.model || '',
+          modes: config.modes ? Object.keys(config.modes).filter(m => config.modes[m]?.enabled) : [],
+          folder: agentFolder,
+          source: 'global',
+        });
+      } catch {}
+    }
+  }
+
+  // Bundled agents (shipped with CLI, lowest priority)
+  const bundledDir = path.join(__dirname, '..', '.veil', 'agents');
+  if (fs.existsSync(bundledDir)) {
+    for (const entry of fs.readdirSync(bundledDir, { withFileTypes: true })) {
+      if (!entry.isDirectory() || found.has(entry.name)) continue;
+      const agentFolder = path.join(bundledDir, entry.name);
+      const configPath = path.join(agentFolder, 'agent.json');
+      if (!fs.existsSync(configPath)) continue;
+      try {
+        const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+        found.set(entry.name, {
+          name: entry.name,
+          description: config.description || '',
+          model: config.model || '',
+          modes: config.modes ? Object.keys(config.modes).filter(m => config.modes[m]?.enabled) : [],
+          folder: agentFolder,
+          source: 'bundled',
+        });
+      } catch {}
+    }
+  }
+
+  return Array.from(found.values());
+}
+
+/**
+ * Get effective mode config for an agent (merges top-level defaults with mode-specific overrides).
+ *
+ * @param {Object} agent - Loaded agent definition
+ * @param {string} mode - 'chat'|'task'|'daemon'|'subagent'
+ * @param {Object} settings - Global/project settings
+ * @returns {Object} Effective mode config
+ */
+function getEffectiveModeConfig(agent, mode, settings) {
+  const modeConfig = (agent.modes && agent.modes[mode]) || {};
+  return {
+    tools: modeConfig.tools || [],
+    disallowedTools: modeConfig.disallowedTools || [],
+    skills: modeConfig.skills || [],
+    autoLoadSkills: modeConfig.autoLoadSkills || [],
+    mcpServers: modeConfig.mcpServers || [],
+    allowedAgents: modeConfig.allowedAgents || [],
+    disallowedAgents: modeConfig.disallowedAgents || [],
+    permissions: modeConfig.permissions || { allow: [], deny: [] },
+    maxIterations: modeConfig.maxIterations || settings.maxIterations || 50,
+    maxDurationSeconds: modeConfig.maxDurationSeconds || settings.maxDurationSeconds || 300,
+    onExhausted: modeConfig.onExhausted || 'fail',
+  };
+}
+
+/**
+ * Resolve the agent directory for a given level.
+ * @param {'project'|'global'} level
+ * @param {string} cwd
+ * @param {string} name
+ * @returns {string}
+ */
+function resolveAgentDirForLevel(level, cwd, name) {
+  if (level === 'global') return path.join(paths.getGlobalAgentsDir(), name);
+  return paths.getAgentDir(cwd, name);
+}
+
+/**
+ * Check all levels for an existing agent with the given name.
+ * @param {string} name
+ * @param {string} cwd
+ * @returns {{ exists: boolean, level: string|null, folder: string|null }}
+ */
+function findExistingAgent(name, cwd) {
+  const checks = [
+    { level: 'project', folder: paths.getAgentDir(cwd, name) },
+    { level: 'global', folder: path.join(paths.getGlobalAgentsDir(), name) },
+    { level: 'bundled', folder: path.join(__dirname, '..', '.veil', 'agents', name) },
+  ];
+  for (const { level, folder } of checks) {
+    if (fs.existsSync(path.join(folder, 'agent.json'))) {
+      return { exists: true, level, folder };
+    }
+  }
+  return { exists: false, level: null, folder: null };
+}
+
+/**
+ * Create a new agent at the specified level.
+ *
+ * @param {{ name: string, level?: 'project'|'global', cwd: string, config: Object, agentMd?: string }} opts
+ * @returns {{ name: string, level: string, folder: string }}
+ */
+function createAgent({ name, level = 'project', cwd, config, agentMd = '' }) {
+  if (!name || typeof name !== 'string' || !/^[a-z0-9_-]+$/i.test(name)) {
+    throw Object.assign(new Error(`Invalid agent name "${name}". Use only letters, numbers, hyphens and underscores.`), { code: 'INVALID_NAME' });
+  }
+
+  const existing = findExistingAgent(name, cwd);
+  if (existing.exists) {
+    throw Object.assign(new Error(`Agent "${name}" already exists at ${existing.level} level (${existing.folder})`), { code: 'AGENT_EXISTS', existingLevel: existing.level });
+  }
+
+  // Merge name into config and validate
+  const fullConfig = { ...config, name };
+  const validate = getAgentValidator();
+  if (!validate(fullConfig)) {
+    const errors = validate.errors.map(e => `  ${e.instancePath || '(root)'}: ${e.message}`).join('\n');
+    throw Object.assign(new Error(`Invalid agent config:\n${errors}`), { code: 'INVALID_CONFIG' });
+  }
+
+  const agentDir = resolveAgentDirForLevel(level, cwd, name);
+  fs.mkdirSync(agentDir, { recursive: true });
+  fs.writeFileSync(path.join(agentDir, 'agent.json'), JSON.stringify(fullConfig, null, 2), 'utf8');
+  if (agentMd) {
+    fs.writeFileSync(path.join(agentDir, 'AGENT.md'), agentMd, 'utf8');
+  }
+
+  return { name, level, folder: agentDir };
+}
+
+/**
+ * Update an existing agent's config and/or AGENT.md.
+ * Finds the agent across all writable levels. Bundled agents cannot be updated.
+ *
+ * @param {{ name: string, cwd: string, config?: Object, agentMd?: string }} opts
+ * @returns {{ name: string, level: string, folder: string }}
+ */
+function updateAgent({ name, cwd, config, agentMd }) {
+  const existing = findExistingAgent(name, cwd);
+  if (!existing.exists) {
+    throw Object.assign(new Error(`Agent "${name}" not found`), { code: 'AGENT_NOT_FOUND' });
+  }
+  if (existing.level === 'bundled') {
+    throw Object.assign(new Error(`Agent "${name}" is a bundled agent and cannot be modified`), { code: 'AGENT_READ_ONLY' });
+  }
+
+  if (config !== undefined) {
+    const configPath = path.join(existing.folder, 'agent.json');
+    const current = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    const merged = { ...current, ...config, name };
+    const validate = getAgentValidator();
+    if (!validate(merged)) {
+      const errors = validate.errors.map(e => `  ${e.instancePath || '(root)'}: ${e.message}`).join('\n');
+      throw Object.assign(new Error(`Invalid agent config:\n${errors}`), { code: 'INVALID_CONFIG' });
+    }
+    fs.writeFileSync(configPath, JSON.stringify(merged, null, 2), 'utf8');
+  }
+
+  if (agentMd !== undefined) {
+    fs.writeFileSync(path.join(existing.folder, 'AGENT.md'), agentMd, 'utf8');
+  }
+
+  return { name, level: existing.level, folder: existing.folder };
+}
+
+/**
+ * Delete an agent folder. Bundled agents cannot be deleted.
+ *
+ * @param {{ name: string, cwd: string }} opts
+ * @returns {{ name: string, level: string, folder: string }}
+ */
+function deleteAgent({ name, cwd }) {
+  const existing = findExistingAgent(name, cwd);
+  if (!existing.exists) {
+    throw Object.assign(new Error(`Agent "${name}" not found`), { code: 'AGENT_NOT_FOUND' });
+  }
+  if (existing.level === 'bundled') {
+    throw Object.assign(new Error(`Agent "${name}" is a bundled agent and cannot be deleted`), { code: 'AGENT_READ_ONLY' });
+  }
+
+  fs.rmSync(existing.folder, { recursive: true, force: true });
+  return { name, level: existing.level, folder: existing.folder };
+}
+
+module.exports = { loadAgent, listAgents, getEffectiveModeConfig, createAgent, updateAgent, deleteAgent };

package/core/cancel.js

@@ -0,0 +1,38 @@
+'use strict';
+
+/**
+ * Cooperative cancellation registry.
+ * Maps taskId → AbortController so that API routes can signal running loops to stop.
+ */
+
+const _signals = new Map();
+
+/**
+ * Register a cancellation signal for a task.
+ * @param {string} taskId
+ * @returns {AbortSignal}
+ */
+function register(taskId) {
+  const ac = new AbortController();
+  _signals.set(taskId, ac);
+  return ac.signal;
+}
+
+/**
+ * Signal a running task to cancel.
+ * @param {string} taskId
+ */
+function cancel(taskId) {
+  const ac = _signals.get(taskId);
+  if (ac) ac.abort();
+}
+
+/**
+ * Remove the signal after task completion.
+ * @param {string} taskId
+ */
+function cleanup(taskId) {
+  _signals.delete(taskId);
+}
+
+module.exports = { register, cancel, cleanup };

package/core/compaction.js

@@ -0,0 +1,176 @@
+'use strict';
+
+const { callLLM, extractMessage } = require('../llm/client');
+const { buildCompactionPrompt } = require('./prompt');
+const { getModelConfig } = require('../utils/settings');
+const { writeMemory } = require('./memory');
+const F = require('../settings/fields');
+
+/**
+ * Apply observation masking: replace tool outputs older than N turns with a placeholder.
+ * @param {Object[]} messages - Conversation messages array (mutated in place)
+ * @param {number} maskingTurns - How many recent turns to keep unmasked (default: 10)
+ * @returns {Object[]} Mutated messages array
+ */
+function applyObservationMasking(messages, maskingTurns = F.DEFAULT_OBSERVATION_MASKING_TURNS) {
+  // Count assistant turns from the end
+  let assistantTurnCount = 0;
+  const masked = [];
+
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role === 'assistant') assistantTurnCount++;
+
+    if (assistantTurnCount > maskingTurns && msg.role === 'tool' && msg.content && msg.content !== '[output hidden]') {
+      masked.push(i);
+    }
+  }
+
+  for (const idx of masked) {
+    messages[idx] = { ...messages[idx], content: '[output hidden]' };
+  }
+
+  return messages;
+}
+
+/**
+ * Truncate large tool outputs in recent turns.
+ * @param {Object[]} messages - Conversation messages (mutated in place)
+ * @param {number} maxChars - Max chars before truncation (default: 500)
+ * @returns {Object[]}
+ */
+function applyToolResultClearing(messages, maxChars = 500) {
+  for (const msg of messages) {
+    if (msg.role === 'tool' && msg.content && msg.content !== '[output hidden]' && msg.content.length > maxChars) {
+      msg.content = msg.content.slice(0, maxChars) + '\n... [truncated]';
+    }
+  }
+  return messages;
+}
+
+/**
+ * Estimate context usage as a fraction (0-1).
+ * Uses a rough 4 chars/token approximation.
+ * @param {Object[]} messages
+ * @param {number} contextWindow - Model's context window size in tokens (default: 100000)
+ * @returns {number} Usage fraction 0-1
+ */
+function estimateContextUsage(messages, contextWindow = 100000) {
+  const totalChars = messages.reduce((sum, m) => {
+    let chars = (m.content || '').length;
+    if (m.tool_calls) chars += JSON.stringify(m.tool_calls).length;
+    return sum + chars;
+  }, 0);
+  const estimatedTokens = Math.ceil(totalChars / 4);
+  return estimatedTokens / contextWindow;
+}
+
+/**
+ * Pre-compaction memory extraction: ask the model what should be remembered long-term.
+ * @param {{ messages: Object[], settings: Object, cwd: string, agentName: string }} opts
+ */
+async function extractMemoriesBeforeCompaction({ messages, settings, cwd, agentName }) {
+  const modelConfig = getModelConfig(settings, F.MODEL_MAIN);
+  if (!modelConfig[F.MODEL_API_KEY] || !modelConfig[F.MODEL_NAME]) return;
+
+  const extractionMessages = [
+    ...messages.slice(-20), // last 20 messages for context
+    {
+      role: 'user',
+      content: 'What key facts, decisions, or learnings from this conversation should be remembered long-term? Reply with a concise markdown list. If nothing important, reply with exactly: NO_MEMORY',
+    },
+  ];
+
+  try {
+    const response = await callLLM({
+      baseUrl: modelConfig[F.MODEL_BASE_URL],
+      apiKey: modelConfig[F.MODEL_API_KEY],
+      model: modelConfig[F.MODEL_NAME],
+      messages: extractionMessages,
+      tools: [],
+    });
+    const { content } = extractMessage(response);
+    if (content && content.trim() !== 'NO_MEMORY' && content.trim().length > 10) {
+      writeMemory({ cwd, agentName, scope: 'agent', content: content.trim() });
+    }
+  } catch {
+    // Non-fatal — compaction continues without memory extraction
+  }
+}
+
+/**
+ * Run full context compaction: summarize conversation history.
+ * @param {{ messages: Object[], settings: Object, taskBrief?: string }} opts
+ * @returns {Promise<Object[]>} New compressed messages array
+ */
+async function compactMessages({ messages, settings, taskBrief }) {
+  const modelConfig = getModelConfig(settings, F.MODEL_COMPACT);
+  if (!modelConfig[F.MODEL_API_KEY] || !modelConfig[F.MODEL_NAME]) {
+    // No compact model — just trim to last 20 messages
+    const systemMessages = messages.filter(m => m.role === 'system');
+    const recent = messages.filter(m => m.role !== 'system').slice(-20);
+    return [...systemMessages, ...recent];
+  }
+
+  const compactionPrompt = buildCompactionPrompt({ messages, taskBrief });
+
+  const compactionMessages = [
+    { role: 'system', content: compactionPrompt },
+    ...messages.filter(m => m.role !== 'system'),
+  ];
+
+  try {
+    const response = await callLLM({
+      baseUrl: modelConfig[F.MODEL_BASE_URL],
+      apiKey: modelConfig[F.MODEL_API_KEY],
+      model: modelConfig[F.MODEL_NAME],
+      messages: compactionMessages,
+      tools: [],
+    });
+    const { content } = extractMessage(response);
+
+    // New messages: preserve system prompt(s), replace rest with summary
+    const systemMessages = messages.filter(m => m.role === 'system');
+    return [
+      ...systemMessages,
+      { role: 'user', content: '[Previous conversation compacted]' },
+      { role: 'assistant', content: content || '[compaction produced no summary]' },
+    ];
+  } catch {
+    // Compaction failed — fall back to trimming
+    const systemMessages = messages.filter(m => m.role === 'system');
+    const recent = messages.filter(m => m.role !== 'system').slice(-20);
+    return [...systemMessages, ...recent];
+  }
+}
+
+/**
+ * Full context management pipeline: masking → clearing → compaction if needed.
+ * @param {{ messages: Object[], settings: Object, taskBrief?: string, cwd: string, agentName: string }} opts
+ * @returns {Promise<{ messages: Object[], compacted: boolean }>}
+ */
+async function manageContext({ messages, settings, taskBrief, cwd, agentName }) {
+  const threshold = settings.compaction?.threshold || F.DEFAULT_COMPACTION_THRESHOLD;
+  const maskingTurns = settings.compaction?.observationMaskingTurns || F.DEFAULT_OBSERVATION_MASKING_TURNS;
+
+  // Step 1: Observation masking
+  applyObservationMasking(messages, maskingTurns);
+
+  // Step 2: Tool result clearing
+  applyToolResultClearing(messages);
+
+  // Step 3: Check if compaction needed
+  const usage = estimateContextUsage(messages);
+  if (usage < threshold) {
+    return { messages, compacted: false };
+  }
+
+  // Step 3a: Pre-compaction memory extraction
+  await extractMemoriesBeforeCompaction({ messages, settings, cwd, agentName });
+
+  // Step 3b: Compact
+  const compacted = await compactMessages({ messages, settings, taskBrief });
+  return { messages: compacted, compacted: true };
+}
+
+module.exports = { manageContext };

package/core/events.js

@@ -0,0 +1,13 @@
+'use strict';
+
+/**
+ * In-process event bus for real-time agent events.
+ * WebSocket connections subscribe to this bus and receive all runtime events.
+ */
+
+const { EventEmitter } = require('events');
+
+const bus = new EventEmitter();
+bus.setMaxListeners(200);
+
+module.exports = bus;