@newsails/veil-cli 1.0.1

package/core/prompt.js

@@ -0,0 +1,185 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const paths = require('../utils/paths');
+const context = require('../utils/context');
+
+const PROMPTS_DIR = path.join(__dirname, '..', 'system-prompts');
+
+/**
+ * Read a system prompt template file.
+ * @param {string} filename - Relative to system-prompts/
+ * @returns {string}
+ */
+function readPromptFile(filename) {
+  try {
+    return fs.readFileSync(path.join(PROMPTS_DIR, filename), 'utf8');
+  } catch {
+    return '';
+  }
+}
+
+/**
+ * Replace template variables in a prompt string.
+ * @param {string} text
+ * @param {Object} vars - Key/value map of variables
+ * @returns {string}
+ */
+function interpolate(text, vars) {
+  return text.replace(/\{\{(\w+)\}\}/g, (_, key) => {
+    return vars[key] !== undefined ? String(vars[key]) : `{{${key}}}`;
+  });
+}
+
+/**
+ * Read a markdown file safely.
+ * @param {string} filePath
+ * @returns {string}
+ */
+function readMd(filePath) {
+  try {
+    return fs.readFileSync(filePath, 'utf8').trim();
+  } catch {
+    return '';
+  }
+}
+
+/**
+ * Get git status for the project directory.
+ * @param {string} cwd
+ * @returns {string}
+ */
+function getGitStatus(cwd) {
+  try {
+    const { execSync } = require('child_process');
+    const branch = execSync('git branch --show-current 2>/dev/null', { cwd, timeout: 3000 }).toString().trim();
+    const status = execSync('git status --short 2>/dev/null | head -20', { cwd, timeout: 3000 }).toString().trim();
+    if (!branch) return 'Not a git repository';
+    return `Branch: ${branch}\n${status || '(clean)'}`;
+  } catch {
+    return 'Not a git repository';
+  }
+}
+
+/**
+ * Assemble the 7-layer system prompt for an agent session.
+ *
+ * @param {{ agent: Object, mode: string, sessionId: string, taskId?: string, cwd: string, settings: Object, memory?: string, todos?: Object[], taskBrief?: string }} opts
+ * @returns {string} Full system prompt text
+ */
+function assembleSystemPrompt({ agent, mode, sessionId, taskId, cwd, settings, memory, todos, taskBrief }) {
+  const layers = [];
+  const version = context.getVersion();
+
+  // ── Layer 1: Base Core (static) ───────────────────────────────────────────
+  const baseCoreVars = {
+    VERSION: version,
+    AGENT_NAME: agent.name,
+    MODE: mode,
+    SESSION_ID: sessionId || '',
+  };
+  layers.push(interpolate(readPromptFile('base-core.md'), baseCoreVars));
+
+  // ── Layer 2: Safety & Rules (static) ─────────────────────────────────────
+  // layers.push(readPromptFile('safety-rules.md'));
+
+  // ── Layer 3: Environment (dynamic) ───────────────────────────────────────
+  const mcpServersList = settings.mcpServers && Object.keys(settings.mcpServers).length > 0
+    ? Object.keys(settings.mcpServers).map(s => `- ${s}`).join('\n')
+    : 'None configured';
+
+  const envVars = {
+    OS: `${os.type()} ${os.release()}`,
+    SHELL: process.env.SHELL || '/bin/sh',
+    CWD: cwd,
+    DATETIME: new Date().toISOString(),
+    DATEONLY: new Date().toISOString().split('T')[0],
+    NODE_VERSION: process.version,
+    GIT_STATUS: getGitStatus(cwd),
+    MCP_SERVERS: mcpServersList,
+  };
+  layers.push(interpolate(readPromptFile('environment.md'), envVars));
+
+  // ── Layer 4: Agent Instructions ───────────────────────────────────────────
+  const agentInstructions = [];
+
+  // Global project AGENT.md
+  const projectAgentMd = readMd(paths.getProjectAgentMdPath(cwd));
+  if (projectAgentMd) agentInstructions.push(`## Project Instructions\n\n${projectAgentMd}`);
+
+  // Agent-specific AGENT.md
+  if (agent.agentMd) agentInstructions.push(`## Agent Instructions\n\n${agent.agentMd}`);
+
+  // SOUL.md
+  if (agent.soulMd) agentInstructions.push(`## Persona\n\n${agent.soulMd}`);
+
+  // Memory
+  const agentMemoryPath = paths.getAgentMemoryDir(cwd, agent.name);
+  const agentMemoryMd = readMd(path.join(agentMemoryPath, 'MEMORY.md'));
+  const projectMemoryMd = readMd(path.join(paths.getProjectMemoryDir(cwd), 'MEMORY.md'));
+  if (agentMemoryMd) agentInstructions.push(`## Agent Memory\n\n${agentMemoryMd}`);
+  if (projectMemoryMd) agentInstructions.push(`## Project Memory\n\n${projectMemoryMd}`);
+  if (memory) agentInstructions.push(`## Injected Memory\n\n${memory}`);
+
+  if (agentInstructions.length > 0) {
+    layers.push(agentInstructions.join('\n\n---\n\n'));
+  }
+
+  // ── Layer 5: Tools & Capabilities ────────────────────────────────────────
+  // Tool descriptions are injected at the API level (tools parameter), not in system prompt.
+  // Only listing custom tools summary here as Tier 2.
+  // (Registry handles Tier 1 full schemas via buildToolsForLLM)
+
+  // ── Layer 6: Task Heuristics ──────────────────────────────────────────────
+  if (mode === 'task' || mode === 'daemon' || mode === 'subagent') {
+    layers.push(readPromptFile('task-heuristics.md'));
+  }
+
+  // ── Layer 7: Session Context ──────────────────────────────────────────────
+  const sessionCtx = [];
+  if (taskBrief) sessionCtx.push(`## Current Task\n\n${taskBrief}`);
+  if (todos && todos.length > 0) {
+    const todoList = todos.map(t => `- [${t.status === 'completed' ? 'x' : ' '}] ${t.content}`).join('\n');
+    sessionCtx.push(`## Active TODOs\n\n${todoList}`);
+  }
+  if (sessionCtx.length > 0) {
+    layers.push(sessionCtx.join('\n\n'));
+  }
+
+  return layers.filter(Boolean).join('\n\n---\n\n').trim();
+}
+
+/**
+ * Build a compaction prompt — summarizes conversation for context compression.
+ * @param {{ messages: Object[], taskBrief?: string }} opts
+ * @returns {string}
+ */
+function buildCompactionPrompt({ messages, taskBrief }) {
+  return `You are summarizing a conversation to reduce context length.
+
+${taskBrief ? `The task was: ${taskBrief}\n\n` : ''}Summarize the conversation below, preserving:
+- Key decisions made
+- Important findings and facts discovered
+- Files read or modified (with their paths)
+- Active problems or unresolved issues
+- Any errors encountered and how they were handled
+- Current state and what step comes next
+
+Be concise but complete. The summary will replace the full conversation history.
+
+Format as structured markdown with clear sections.`;
+}
+
+/**
+ * Build a reminder injection for mid-conversation.
+ * @param {{ type: 'anti-drift'|'stall-recovery', vars?: Object }} opts
+ * @returns {string}
+ */
+function buildReminder({ type, vars = {} }) {
+  const filename = `reminders/${type}.md`;
+  return interpolate(readPromptFile(filename), vars);
+}
+
+module.exports = { assembleSystemPrompt, buildCompactionPrompt, buildReminder };

package/core/queue.js

@@ -0,0 +1,96 @@
+'use strict';
+
+const db = require('../infrastructure/database');
+
+/**
+ * In-memory agent message queue helper.
+ * The actual queue is in SQLite (agent_messages table).
+ * This module provides higher-level queue operations.
+ */
+
+/**
+ * Get and consume all non-followup pending messages for an agent.
+ * Returns them as user-role messages to inject into conversation.
+ * Messages with a target_session_id are only drained by that specific session.
+ * @param {string} agentName
+ * @param {string|null} sessionId  - The current session ID (used to match target_session_id)
+ * @returns {{ messages: Object[], correlationIds: string[] }}
+ */
+function drainNonFollowup(agentName, sessionId = null) {
+  const pending = db.getPendingAgentMessages(agentName);
+  const messages = [];
+  const correlationIds = [];
+
+  for (const msg of pending) {
+    if (msg.followup) continue;
+    if (msg.target_session_id && msg.target_session_id !== sessionId) continue;
+    messages.push({
+      role: 'user',
+      content: `[Message from ${msg.from_agent}]: ${msg.content}`,
+      _correlationId: msg.correlation_id || null,
+      _messageId: msg.id,
+    });
+    if (msg.correlation_id) correlationIds.push(msg.correlation_id);
+    db.markAgentMessageDelivered(msg.id);
+  }
+
+  return { messages, correlationIds };
+}
+
+/**
+ * Get and consume all followup pending messages for an agent.
+ * Messages with a target_session_id are only drained by that specific session.
+ * @param {string} agentName
+ * @param {string|null} sessionId  - The current session ID (used to match target_session_id)
+ * @returns {Object[]}
+ */
+function drainFollowup(agentName, sessionId = null) {
+  const pending = db.getPendingAgentMessages(agentName);
+  const messages = [];
+
+  for (const msg of pending) {
+    if (!msg.followup) continue;
+    if (msg.target_session_id && msg.target_session_id !== sessionId) continue;
+    messages.push({
+      role: 'user',
+      content: `[Message from ${msg.from_agent}]: ${msg.content}`,
+    });
+    db.markAgentMessageDelivered(msg.id);
+  }
+
+  return messages;
+}
+
+/**
+ * Post a response to a correlated agent_message (sync messaging).
+ * @param {string} correlationId
+ * @param {string} response
+ */
+function postCorrelatedResponse(correlationId, response) {
+  if (!correlationId) return;
+  db.getDb().prepare(
+    "UPDATE agent_messages SET response = ?, delivered_at = ? WHERE correlation_id = ? AND status = 'delivered'"
+  ).run(response, new Date().toISOString(), correlationId);
+}
+
+/**
+ * Notify all durable subscribers of a completed task.
+ * Injects a message into each subscriber's session.
+ * @param {{ taskId: string, task: Object }} opts
+ */
+function notifyTaskSubscribers({ taskId, task }) {
+  const subs = db.getPendingSubscriptions(taskId);
+  for (const sub of subs) {
+    const content = `[Task ${taskId} completed] Agent: ${task.agent_name}, Status: ${task.status}${task.output ? `\nOutput: ${String(task.output).slice(0, 500)}` : ''}`;
+    db.enqueueAgentMessage({
+      targetAgent: sub.subscriber_agent,
+      targetSessionId: sub.subscriber_session_id,
+      fromAgent: 'system',
+      content,
+      followup: false,
+    });
+    db.markSubscriptionDelivered(sub.id);
+  }
+}
+
+module.exports = { drainNonFollowup, drainFollowup, postCorrelatedResponse, notifyTaskSubscribers };

package/core/registry.js

@@ -0,0 +1,291 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const Ajv = require('ajv');
+const addFormats = require('ajv-formats');
+const paths = require('../utils/paths');
+
+const ajv = new Ajv({ allErrors: true });
+addFormats(ajv);
+
+// Cache of compiled validators per tool name
+const _validators = new Map();
+
+// Registry of all built-in tools: name → { schema, execute }
+const _builtins = new Map();
+
+/**
+ * Register a built-in tool.
+ * @param {string} name
+ * @param {{ schema: Object, execute: Function }} tool
+ */
+function registerBuiltin(name, tool) {
+  if (!tool.schema || !tool.execute) {
+    throw new Error(`Tool "${name}" must have schema and execute`);
+  }
+  _builtins.set(name, tool);
+  if (tool.schema.input_schema) {
+    _validators.set(name, ajv.compile(tool.schema.input_schema));
+  }
+}
+
+/**
+ * Load and register all built-in tools from the tools/ directory.
+ * @param {string} toolsDir - Absolute path to the tools/ directory
+ */
+function loadBuiltinTools(toolsDir) {
+  const files = fs.readdirSync(toolsDir).filter(f => f.endsWith('.js'));
+  for (const file of files) {
+    try {
+      const tool = require(path.join(toolsDir, file));
+      if (tool.schema && tool.execute) {
+        registerBuiltin(tool.schema.name, tool);
+      }
+    } catch (err) {
+      console.error(`Failed to load built-in tool ${file}: ${err.message}`);
+    }
+  }
+}
+
+/**
+ * Load a custom JS tool from a folder.
+ * @param {string} toolFolder - Absolute path to the tool folder
+ * @returns {{ schema: Object, execute: Function }|null}
+ */
+function loadCustomTool(toolFolder) {
+  const manifestPath = path.join(toolFolder, 'tool.json');
+  const indexPath = path.join(toolFolder, 'index.js');
+  if (!fs.existsSync(manifestPath) || !fs.existsSync(indexPath)) return null;
+  try {
+    const schema = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+    // Hot reload: delete require cache before loading
+    delete require.cache[require.resolve(indexPath)];
+    const execute = require(indexPath);
+    return { schema, execute };
+  } catch (err) {
+    console.error(`Failed to load custom tool at ${toolFolder}: ${err.message}`);
+    return null;
+  }
+}
+
+/**
+ * Discover custom tools from a directory (each subdirectory is a tool).
+ * @param {string} dir
+ * @returns {Map<string, { schema: Object, execute: Function }>}
+ */
+function discoverCustomTools(dir) {
+  const result = new Map();
+  if (!fs.existsSync(dir)) return result;
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const tool = loadCustomTool(path.join(dir, entry.name));
+    if (tool && tool.schema && tool.schema.name) {
+      result.set(tool.schema.name, tool);
+    }
+  }
+  return result;
+}
+
+/**
+ * Resolve a tool by name, following the resolution order:
+ * built-in → agent-level → project-level → global → not found
+ *
+ * @param {{ name: string, agentConfig: Object, cwd: string }} opts
+ * @returns {{ schema: Object, execute: Function }|null}
+ */
+function resolveTool({ name, agentConfig, cwd }) {
+  // 1. Built-in
+  if (_builtins.has(name)) return _builtins.get(name);
+
+  // 2. Agent-level
+  if (agentConfig && agentConfig.agentFolder) {
+    const agentToolDir = path.join(agentConfig.agentFolder, 'tools', name);
+    const agentTool = loadCustomTool(agentToolDir);
+    if (agentTool) return agentTool;
+  }
+
+  // 3. Project-level
+  const projectTool = loadCustomTool(path.join(paths.getProjectToolsDir(cwd), name));
+  if (projectTool) return projectTool;
+
+  // 4. Global
+  const globalTool = loadCustomTool(path.join(paths.getGlobalToolsDir(), name));
+  if (globalTool) return globalTool;
+
+  return null;
+}
+
+/**
+ * Build the tools list to send to the LLM.
+ * Tier 1 (built-in): full schemas
+ * Tier 2 (custom): names + one-line descriptions only (full schema loaded via tool_search)
+ *
+ * @param {{ agentConfig: Object, cwd: string, modeConfig?: Object }} opts
+ * @returns {Object[]} OpenAI-format tool definitions
+ */
+function buildToolsForLLM({ agentConfig, cwd, modeConfig = {} }) {
+  const allowedTools = modeConfig.tools || [];
+  const disallowedTools = new Set(modeConfig.disallowedTools || []);
+
+  // Auto-disable memory tools if memory is disabled for this agent
+  const memoryEnabled = agentConfig?.memory?.enabled !== false;
+  if (!memoryEnabled) {
+    disallowedTools.add('memory_write');
+    disallowedTools.add('memory_read');
+  }
+
+  const result = [];
+
+  // Tier 1: built-in tools — full schemas
+  for (const [name, tool] of _builtins) {
+    if (disallowedTools.has(name)) continue;
+    if (allowedTools.length > 0 && !allowedTools.includes(name)) continue;
+    result.push(toolToOpenAIFormat(tool.schema));
+  }
+
+  return result;
+}
+
+/**
+ * Get names + descriptions of all custom tools (Tier 2 listing).
+ * @param {{ agentConfig: Object, cwd: string }} opts
+ * @returns {Array<{ name: string, description: string }>}
+ */
+function listCustomToolSummaries({ agentConfig, cwd }) {
+  const summaries = [];
+
+  // Agent-level
+  if (agentConfig && agentConfig.agentFolder) {
+    const agentToolsDir = path.join(agentConfig.agentFolder, 'tools');
+    for (const [name, tool] of discoverCustomTools(agentToolsDir)) {
+      summaries.push({ name, description: tool.schema.description || '' });
+    }
+  }
+
+  // Project-level
+  for (const [name, tool] of discoverCustomTools(paths.getProjectToolsDir(cwd))) {
+    summaries.push({ name, description: tool.schema.description || '' });
+  }
+
+  // Global
+  for (const [name, tool] of discoverCustomTools(paths.getGlobalToolsDir())) {
+    summaries.push({ name, description: tool.schema.description || '' });
+  }
+
+  return summaries;
+}
+
+/**
+ * Search tools by query string (name or description match).
+ * Returns full schemas for matching tools.
+ * @param {{ query: string, agentConfig: Object, cwd: string }} opts
+ * @returns {Object[]} Full tool schemas
+ */
+function searchTools({ query, agentConfig, cwd }) {
+  const lq = query.toLowerCase();
+  const results = [];
+
+  for (const [name, tool] of _builtins) {
+    if (name.includes(lq) || (tool.schema.description || '').toLowerCase().includes(lq)) {
+      results.push(tool.schema);
+    }
+  }
+
+  if (agentConfig && agentConfig.agentFolder) {
+    for (const [, tool] of discoverCustomTools(path.join(agentConfig.agentFolder, 'tools'))) {
+      const n = tool.schema.name || '';
+      if (n.includes(lq) || (tool.schema.description || '').toLowerCase().includes(lq)) {
+        results.push(tool.schema);
+      }
+    }
+  }
+
+  for (const [, tool] of discoverCustomTools(paths.getProjectToolsDir(cwd))) {
+    const n = tool.schema.name || '';
+    if (n.includes(lq) || (tool.schema.description || '').toLowerCase().includes(lq)) {
+      results.push(tool.schema);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Validate tool input arguments against the tool's JSON schema.
+ * @param {string} toolName
+ * @param {Object} input
+ * @returns {{ valid: boolean, errors?: string[] }}
+ */
+function validateToolInput(toolName, input) {
+  const validator = _validators.get(toolName);
+  if (!validator) return { valid: true };
+  const valid = validator(input);
+  if (!valid) {
+    return { valid: false, errors: validator.errors.map(e => `${e.instancePath || '(root)'}: ${e.message}`) };
+  }
+  return { valid: true };
+}
+
+/**
+ * Convert our tool schema format to OpenAI function-calling format.
+ * @param {Object} schema
+ * @returns {Object}
+ */
+function toolToOpenAIFormat(schema) {
+  return {
+    type: 'function',
+    function: {
+      name: schema.name,
+      description: schema.description || '',
+      parameters: schema.input_schema || { type: 'object', properties: {} },
+    },
+  };
+}
+
+/**
+ * Resolve a custom tool by name and return its OpenAI-format schema, applying
+ * the same whitelist/blocklist rules as buildToolsForLLM.
+ * Also registers an AJV validator for the tool if not already registered.
+ * Returns null if the tool is blocked, not in the whitelist, or not found.
+ *
+ * @param {string} name - Tool name to activate
+ * @param {Object} agentConfig
+ * @param {string} cwd
+ * @param {Object} modeConfig
+ * @returns {Object|null} OpenAI-format tool definition, or null
+ */
+function activateToolByName(name, agentConfig, cwd, modeConfig = {}) {
+  const allowedTools    = modeConfig.tools || [];
+  const disallowedTools = new Set(modeConfig.disallowedTools || []);
+
+  if (disallowedTools.has(name)) return null;
+  if (allowedTools.length > 0 && !allowedTools.includes(name)) return null;
+
+  const tool = resolveTool({ name, agentConfig, cwd });
+  if (!tool) return null;
+
+  if (tool.schema.input_schema && !_validators.has(name)) {
+    try { _validators.set(name, ajv.compile(tool.schema.input_schema)); } catch {}
+  }
+
+  return toolToOpenAIFormat(tool.schema);
+}
+
+/**
+ * Get all registered built-in tool names.
+ * @returns {string[]}
+ */
+function getBuiltinNames() {
+  return Array.from(_builtins.keys());
+}
+
+module.exports = {
+  loadBuiltinTools,
+  resolveTool,
+  buildToolsForLLM,
+  listCustomToolSummaries,
+  searchTools,
+  validateToolInput,
+  activateToolByName,
+};

package/core/remote-methods.js

@@ -0,0 +1,124 @@
+'use strict';
+
+const { randomUUID } = require('crypto');
+
+/**
+ * In-memory Remote Method Execution queue.
+ *
+ * Allows custom tools to call remoteMethodExecution({ method, data, timeoutMs })
+ * which blocks until a connected UI client posts a result via
+ * POST /remote-methods/:id/result.
+ *
+ * Methods do NOT survive a server restart (by design).
+ * SSE clients are notified of new pending methods and of completed ones
+ * (so secondary clients know to discard them).
+ */
+
+/** @type {Map<string, { id: string, method: string, data: any, createdAt: string, resolve: Function, reject: Function, timer: NodeJS.Timeout }>} */
+const _pending = new Map();
+
+/** @type {Set<import('http').ServerResponse>} */
+const _clients = new Set();
+
+/**
+ * Send an SSE event to all connected clients.
+ * @param {{ type: string, [key: string]: any }} payload
+ */
+function _broadcast(payload) {
+  const line = `data: ${JSON.stringify(payload)}\n\n`;
+  for (const res of _clients) {
+    try {
+      res.write(line);
+    } catch {
+      _clients.delete(res);
+    }
+  }
+}
+
+/**
+ * Register a new SSE client.  Immediately flushes all currently-pending
+ * methods so the client is fully caught up.
+ * @param {import('http').ServerResponse} res
+ */
+function addClient(res) {
+  _clients.add(res);
+  // Flush current pending methods to the new client
+  for (const entry of _pending.values()) {
+    try {
+      res.write(`data: ${JSON.stringify({
+        type: 'method.pending',
+        id: entry.id,
+        method: entry.method,
+        data: entry.data,
+        createdAt: entry.createdAt,
+      })}\n\n`);
+    } catch { /* client may have already gone */ }
+  }
+}
+
+/**
+ * Remove an SSE client (on disconnect).
+ * @param {import('http').ServerResponse} res
+ */
+function removeClient(res) {
+  _clients.delete(res);
+}
+
+/**
+ * Enqueue a remote method call.  Returns a Promise that resolves when the UI
+ * posts a result, or rejects on timeout.
+ *
+ * @param {{ method: string, data?: any, timeoutMs?: number }} opts
+ * @returns {Promise<any>} Resolves with whatever the UI sends as `result`
+ */
+function enqueue({ method, data = null, timeoutMs = 120_000 }) {
+  const id = randomUUID();
+  const createdAt = new Date().toISOString();
+
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      if (_pending.has(id)) {
+        _pending.delete(id);
+        _broadcast({ type: 'method.done', id, timedOut: true });
+        reject(new Error(`remoteMethodExecution timed out after ${timeoutMs}ms (id=${id})`));
+      }
+    }, timeoutMs);
+
+    _pending.set(id, { id, method, data, createdAt, resolve, reject, timer });
+
+    // Notify all connected UIs
+    _broadcast({ type: 'method.pending', id, method, data, createdAt });
+  });
+}
+
+/**
+ * Resolve a pending remote method call with a result from the UI.
+ *
+ * @param {string} id
+ * @param {any} result
+ * @returns {'ok'|'not_found'|'already_resolved'}
+ */
+function resolve(id, result) {
+  const entry = _pending.get(id);
+  if (!entry) return 'not_found';
+
+  clearTimeout(entry.timer);
+  _pending.delete(id);
+
+  // Notify ALL clients (including the one that just responded) so multi-client
+  // setups can discard the pending method immediately.
+  _broadcast({ type: 'method.done', id, timedOut: false });
+
+  entry.resolve(result);
+  return 'ok';
+}
+
+/**
+ * Return a snapshot of all currently-pending method IDs (for diagnostics).
+ * @returns {string[]}
+ */
+function pendingIds() {
+  return Array.from(_pending.keys());
+}
+
+module.exports = { addClient, removeClient, enqueue, resolve, pendingIds };