@stevederico/dotbot 0.16.0

package/core/compaction.js

@@ -0,0 +1,261 @@
+// core/compaction.js
+// Session auto-compaction: summarizes old messages when context nears provider limits.
+// Runs before trimMessages as an intelligent alternative to hard truncation.
+// Operates on standard-format messages (see normalize.js).
+
+import { AI_PROVIDERS } from "../utils/providers.js";
+import { toStandardFormat, toProviderFormat } from "./normalize.js";
+
+/** Max context window tokens per provider family. */
+const CONTEXT_LIMITS = {
+  anthropic: 180000,
+  openai: 120000,
+  xai: 120000,
+  ollama: 6000,
+  dottie_desktop: 6000,
+};
+
+/** Number of recent messages to always preserve verbatim. */
+const RECENT_TO_KEEP = 20;
+
+/** Compaction triggers when estimated tokens exceed this fraction of the context limit. */
+const COMPACT_THRESHOLD = 0.7;
+
+/** Cheapest model per provider, used for summarization calls. */
+const CHEAP_MODELS = {
+  xai: "grok-4-1-fast-non-reasoning",
+  anthropic: "claude-3-5-haiku-20241022",
+  openai: "gpt-5-nano",
+};
+
+/**
+ * Estimate token count for an array of standard-format messages.
+ *
+ * Uses a 4-chars-per-token heuristic. Counts content, toolCalls
+ * (name + input + result), and thinking fields.
+ *
+ * @param {Array} messages - Conversation messages in standard format.
+ * @returns {number} Estimated token count.
+ */
+export function estimateTokens(messages) {
+  let chars = 0;
+
+  for (const msg of messages) {
+    if (typeof msg.content === "string") {
+      chars += msg.content.length;
+    }
+
+    if (msg.toolCalls) {
+      for (const tc of msg.toolCalls) {
+        chars += (tc.name || "").length;
+        chars += JSON.stringify(tc.input || {}).length;
+        if (tc.result) {
+          chars += (typeof tc.result === "string" ? tc.result : JSON.stringify(tc.result)).length;
+        }
+      }
+    }
+
+    if (msg.thinking) {
+      chars += msg.thinking.length;
+    }
+  }
+
+  return Math.ceil(chars / 4);
+}
+
+/**
+ * Naive fallback summary when no AI provider is available.
+ *
+ * Extracts the first 100 characters of each user message, tool call names,
+ * and tool results, producing a bullet-point summary.
+ *
+ * @param {Array} messages - Standard-format messages to summarize.
+ * @returns {string} Plain-text summary.
+ */
+function naiveSummary(messages) {
+  const lines = [];
+
+  for (const msg of messages) {
+    if (msg.role === "user" && msg.content) {
+      lines.push(`- User: ${msg.content.slice(0, 100)}`);
+    } else if (msg.role === "assistant") {
+      if (msg.toolCalls) {
+        for (const tc of msg.toolCalls) {
+          lines.push(`- Tool: ${tc.name}`);
+          if (tc.result) {
+            const resultStr = typeof tc.result === "string" ? tc.result : JSON.stringify(tc.result);
+            lines.push(`- Result: ${resultStr.slice(0, 100)}`);
+          }
+        }
+      }
+    }
+  }
+
+  return lines.length > 0
+    ? lines.join("\n")
+    : "Previous conversation context (details unavailable).";
+}
+
+/**
+ * Summarize old messages using the cheapest available AI provider.
+ *
+ * Checks provider keys in order: xAI, Anthropic, OpenAI.
+ * Falls back to naiveSummary() if no provider is configured.
+ * Messages are expected in standard format; converted to provider format for the API call.
+ *
+ * @param {Array} oldMessages - Standard-format messages to summarize.
+ * @param {Object} [options={}]
+ * @param {AbortSignal} [options.signal] - Optional abort signal.
+ * @param {Object} [options.providers={}] - Provider config with API keys.
+ * @returns {Promise<string>} Summary text.
+ */
+async function summarizeMessages(oldMessages, { signal, providers = {} } = {}) {
+  // Build plain-text transcript from standard-format messages
+  const transcript = oldMessages
+    .map((msg) => {
+      const role = msg.role || "unknown";
+      const parts = [];
+
+      if (msg.content) {
+        parts.push(msg.content);
+      }
+
+      if (msg.toolCalls) {
+        for (const tc of msg.toolCalls) {
+          parts.push(`[tool: ${tc.name}]`);
+          if (tc.result) {
+            const resultStr = typeof tc.result === "string" ? tc.result : JSON.stringify(tc.result);
+            parts.push(`[result: ${resultStr.slice(0, 200)}]`);
+          }
+        }
+      }
+
+      return `${role}: ${parts.join(" ")}`;
+    })
+    .join("\n");
+
+  // Find cheapest available provider
+  const providerOrder = ["xai", "anthropic", "openai"];
+
+  let selectedProvider = null;
+  let selectedModel = null;
+  let apiKey = null;
+  for (const id of providerOrder) {
+    if (providers[id]?.apiKey) {
+      selectedProvider = AI_PROVIDERS[id];
+      selectedModel = CHEAP_MODELS[id];
+      apiKey = providers[id].apiKey;
+      break;
+    }
+  }
+
+  if (!selectedProvider || !apiKey) {
+    return naiveSummary(oldMessages);
+  }
+
+  const summaryPrompt =
+    "Summarize this conversation concisely. Preserve: key decisions, user facts, tool results, and any important outcomes. Omit: greetings, thinking steps, redundant tool calls. Output only the summary, no preamble.";
+
+  // Build summarization request in standard format, then convert for the provider
+  const summaryMessages = [
+    { role: "system", content: summaryPrompt },
+    { role: "user", content: transcript },
+  ];
+
+  const targetFormat = selectedProvider.id === "anthropic" ? "anthropic" : "openai";
+  const providerMessages = toProviderFormat(summaryMessages, targetFormat);
+
+  // Anthropic doesn't support system role in messages — use top-level system param
+  let requestBody;
+  if (targetFormat === "anthropic") {
+    requestBody = {
+      model: selectedModel,
+      max_tokens: 1024,
+      system: summaryPrompt,
+      messages: providerMessages.filter((m) => m.role !== "system"),
+    };
+  } else {
+    requestBody = {
+      model: selectedModel,
+      max_tokens: 1024,
+      messages: providerMessages,
+    };
+  }
+
+  try {
+    const url = `${selectedProvider.apiUrl}${selectedProvider.endpoint}`;
+    const headers = selectedProvider.headers(apiKey);
+
+    const res = await fetch(url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(requestBody),
+      signal,
+    });
+
+    if (!res.ok) {
+      console.error(`[compaction] summarization failed (${res.status}), using naive fallback`);
+      return naiveSummary(oldMessages);
+    }
+
+    const data = await res.json();
+    const text = selectedProvider.formatResponse(data);
+    return text || naiveSummary(oldMessages);
+  } catch (err) {
+    if (err.name === "AbortError") throw err;
+    console.error("[compaction] summarization error, using naive fallback:", err.message);
+    return naiveSummary(oldMessages);
+  }
+}
+
+/**
+ * Compact a conversation's messages if approaching the provider's context limit.
+ *
+ * Splits messages into [system prompt] + [old messages] + [recent N messages],
+ * summarizes old messages via the cheapest AI provider, and returns a compacted
+ * array. If under the threshold, returns the original messages unchanged.
+ *
+ * @param {Array} messages - Full conversation history including system prompt.
+ * @param {Object} [options={}]
+ * @param {string} [options.providerId='ollama'] - Provider ID for context limit lookup.
+ * @param {AbortSignal} [options.signal] - Optional abort signal.
+ * @param {Object} [options.providers] - Provider configuration with API keys: { anthropic: { apiKey }, openai: { apiKey }, xai: { apiKey } }
+ * @returns {Promise<{messages: Array, compacted: boolean}>} Compacted result.
+ */
+export async function compactMessages(messages, { providerId = "ollama", signal, providers = {} } = {}) {
+  const tokens = estimateTokens(messages);
+  const limit = CONTEXT_LIMITS[providerId] || CONTEXT_LIMITS.ollama;
+
+  if (tokens < limit * COMPACT_THRESHOLD) {
+    return { messages, compacted: false };
+  }
+
+  // Need at least system + RECENT_TO_KEEP + 1 old message to compact
+  if (messages.length <= RECENT_TO_KEEP + 2) {
+    return { messages, compacted: false };
+  }
+
+  const systemPrompt = messages[0];
+  const recent = messages.slice(-RECENT_TO_KEEP);
+  const old = messages.slice(1, messages.length - RECENT_TO_KEEP);
+
+  if (old.length === 0) {
+    return { messages, compacted: false };
+  }
+
+  console.log(`[compaction] ${tokens} tokens (~${Math.round((tokens / limit) * 100)}% of ${providerId} limit), compacting ${old.length} old messages`);
+
+  const summary = await summarizeMessages(old, { signal, providers });
+
+  const summaryMessage = {
+    role: "user",
+    content: `[Context Summary]\n${summary}`,
+    _compaction: true,
+    _ts: Date.now(),
+  };
+
+  return {
+    messages: [systemPrompt, summaryMessage, ...recent],
+    compacted: true,
+  };
+}

package/core/cron_handler.js

@@ -0,0 +1,262 @@
+/**
+ * Cron task handler for dotbot.
+ *
+ * Extracted from dottie-os server.js to provide a reusable cron task executor
+ * that handles session resolution, stale user gates, task injection, and
+ * notification hooks.
+ */
+
+import { compactMessages } from './compaction.js';
+
+/**
+ * Create a cron task handler function.
+ *
+ * @param {Object} options
+ * @param {Object} options.sessionStore - Session store instance
+ * @param {Object} options.cronStore - Cron store instance
+ * @param {Object} options.taskStore - Task store instance (optional)
+ * @param {Object} options.memoryStore - Memory store instance (optional)
+ * @param {Object} options.providers - Provider API keys for compaction
+ * @param {number} [options.staleThresholdMs=86400000] - Skip heartbeat if user idle longer than this (default: 24h)
+ * @param {Object} [options.hooks] - Host-specific hooks
+ * @param {Function} [options.hooks.onNotification] - async (userId, { title, body, type }) => void
+ * @param {Function} [options.hooks.taskFetcher] - async (userId, taskId) => task object
+ * @param {Function} [options.hooks.tasksFinder] - async (userId, filter) => tasks array
+ * @returns {Function} Async handler for cron task execution
+ */
+export function createCronHandler({
+  sessionStore,
+  cronStore,
+  taskStore,
+  memoryStore,
+  providers = {},
+  staleThresholdMs = 24 * 60 * 60 * 1000,
+  hooks = {},
+}) {
+  // Agent reference - will be set after init() creates the agent
+  let agent = null;
+
+  /**
+   * Set the agent instance (called by init() after agent creation).
+   * @param {Object} agentInstance
+   */
+  function setAgent(agentInstance) {
+    agent = agentInstance;
+  }
+
+  /**
+   * Handle a cron task firing.
+   *
+   * @param {Object} task - Cron task object
+   * @param {string} task.name - Task name (e.g., "heartbeat", "task_step")
+   * @param {string} [task.userId] - User ID for user-level tasks
+   * @param {string} [task.sessionId] - Session ID for session-scoped tasks
+   * @param {string} [task.taskId] - Task ID for task_step cron jobs
+   * @param {string} [task.prompt] - Task prompt
+   */
+  async function handleTaskFire(task) {
+    console.log(`[cron] processing task ${task.name} (userId=${task.userId || 'none'}, sessionId=${task.sessionId || 'none'})`);
+
+    // Skip if agent not initialized yet
+    if (!agent) {
+      console.warn('[cron] task skipped - agent not initialized');
+      return;
+    }
+
+    // Resolve session: user-level heartbeats find the most recent session,
+    // session-scoped tasks use the explicit sessionId.
+    let session;
+    if (task.userId) {
+      session = await sessionStore.getOrCreateDefaultSession(task.userId);
+    } else if (task.sessionId) {
+      session = await sessionStore.getSessionInternal(task.sessionId);
+    }
+
+    if (!session) {
+      console.log(`[cron] no session found for task ${task.name}, skipping`);
+      return;
+    }
+
+    // Stale user check: skip heartbeat if user hasn't interacted recently
+    if (task.name === 'heartbeat' && session.updatedAt) {
+      const idleMs = Date.now() - new Date(session.updatedAt).getTime();
+      if (idleMs > staleThresholdMs) {
+        console.log(`[cron] skipping heartbeat for stale user ${session.owner} (idle ${Math.round(idleMs / 3600000)}h)`);
+        return;
+      }
+    }
+
+    // Build task content depending on task type
+    const taskContent = await buildTaskContent(task, session);
+    if (!taskContent) return;
+
+    // Add message to session
+    await sessionStore.addMessage(session.id, {
+      role: 'user',
+      content: taskContent,
+    });
+
+    // Re-fetch session to pick up the added message
+    const updatedSession = await sessionStore.getSessionInternal(session.id);
+    const providerId = updatedSession.provider || 'ollama';
+
+    // Compact old messages before running agent loop
+    const compacted = await compactMessages(updatedSession.messages, {
+      providerId,
+      providers,
+    });
+
+    if (compacted.compacted) {
+      updatedSession.messages = compacted.messages;
+      updatedSession.messages.push({
+        role: 'user',
+        content: '[Compaction] Compressed conversation history',
+        _ts: Date.now(),
+      });
+    }
+
+    // Run the agent chat loop
+    let finalText = '';
+    for await (const event of agent.chat({
+      sessionId: updatedSession.id,
+      message: '', // Message already added to session
+      provider: providerId,
+      model: updatedSession.model,
+      context: {
+        userID: updatedSession.owner,
+        sessionId: updatedSession.id,
+        memoryStore,
+        taskStore,
+      },
+    })) {
+      if (event.type === 'text_delta' && event.text) {
+        finalText += event.text;
+      }
+    }
+
+    // Create notification if the agent produced meaningful output
+    const trimmed = finalText.trim();
+    if (trimmed && trimmed.length > 10 && updatedSession.owner && hooks.onNotification) {
+      try {
+        await hooks.onNotification(updatedSession.owner, {
+          title: 'Dottie',
+          body: trimmed.slice(0, 500),
+          type: task.name === 'heartbeat' ? 'heartbeat' : 'cron',
+        });
+        console.log(`[cron] notification created for ${updatedSession.owner} (${trimmed.length} chars)`);
+      } catch (err) {
+        console.error('[cron] failed to create notification:', err.message);
+      }
+    } else if (task.name === 'heartbeat') {
+      console.log(`[cron] heartbeat for ${updatedSession.owner} produced no meaningful output (${finalText.trim().length} chars)`);
+    }
+  }
+
+  /**
+   * Build the content for a cron task based on its type.
+   *
+   * @param {Object} task - Cron task
+   * @param {Object} session - User session
+   * @returns {Promise<string|null>} Task content or null to skip
+   */
+  async function buildTaskContent(task, session) {
+    if (task.name === 'task_step' && task.taskId) {
+      // Task step continuation — inject targeted prompt for the specific task
+      return await buildTaskStepContent(task, session);
+    }
+
+    if (task.name === 'heartbeat' && session.owner) {
+      // Heartbeat — check for auto-mode tasks with pending steps
+      return await buildHeartbeatContent(task, session);
+    }
+
+    // Default: use the task prompt
+    return `[Heartbeat] ${task.prompt}`;
+  }
+
+  /**
+   * Build content for a task_step cron job.
+   */
+  async function buildTaskStepContent(task, session) {
+    try {
+      let taskDoc;
+      if (hooks.taskFetcher) {
+        taskDoc = await hooks.taskFetcher(session.owner, task.taskId);
+      } else if (taskStore) {
+        taskDoc = await taskStore.getTask(session.owner, task.taskId);
+      }
+
+      if (!taskDoc) {
+        console.log(`[cron] task_step: task ${task.taskId} not found, skipping`);
+        return null;
+      }
+
+      if (taskDoc.status === 'completed' || taskDoc.status === 'abandoned') {
+        console.log(`[cron] task_step: task ${task.taskId} already ${taskDoc.status}, skipping`);
+        return null;
+      }
+
+      const nextStep = taskDoc.steps?.find(s => !s.done);
+      if (!nextStep) {
+        console.log(`[cron] task_step: all steps done for task ${task.taskId}, skipping`);
+        return null;
+      }
+
+      const doneCount = taskDoc.steps.filter(s => s.done).length;
+      return `[Task Work] Continue auto-executing task "${taskDoc.description}" (${doneCount}/${taskDoc.steps.length} steps done). Call task_work with task_id "${task.taskId}" to execute the next step.`;
+    } catch (err) {
+      console.error('[cron] task_step error:', err.message);
+      return null;
+    }
+  }
+
+  /**
+   * Build content for a heartbeat cron job.
+   */
+  async function buildHeartbeatContent(task, session) {
+    let taskContent = `[Heartbeat] ${task.prompt}`;
+
+    try {
+      let tasks = [];
+      if (hooks.tasksFinder) {
+        tasks = await hooks.tasksFinder(session.owner, { status: ['pending', 'in_progress'] });
+      } else if (taskStore) {
+        tasks = await taskStore.findTasks(session.owner, { status: ['pending', 'in_progress'] });
+      }
+
+      if (tasks.length > 0) {
+        // Check if any task is in auto mode with pending steps
+        const autoTask = tasks.find(t => t.mode === 'auto' && t.steps?.some(s => !s.done));
+        if (autoTask) {
+          const doneCount = autoTask.steps.filter(s => s.done).length;
+          const nextStep = autoTask.steps.find(s => !s.done);
+          taskContent = `[Heartbeat] Auto-mode task "${autoTask.description}" has pending steps (${doneCount}/${autoTask.steps.length} done). Call task_work with task_id "${autoTask._id || autoTask.id}" to execute: "${nextStep.text}"`;
+        } else {
+          // List all active tasks
+          const lines = tasks.map(t => {
+            let line = `• [${t.priority}] ${t.description}`;
+            if (t.mode) line += ` [${t.mode}]`;
+            if (t.deadline) line += ` (due: ${t.deadline})`;
+            if (t.steps && t.steps.length > 0) {
+              const done = t.steps.filter(s => s.done).length;
+              line += ` (${done}/${t.steps.length} steps)`;
+              for (const step of t.steps) {
+                line += `\n  ${step.done ? '[x]' : '[ ]'} ${step.text}`;
+              }
+            }
+            return line;
+          });
+          taskContent += `\n\nActive tasks:\n${lines.join('\n')}`;
+        }
+      }
+    } catch (err) {
+      console.error('[cron] failed to fetch tasks for heartbeat:', err.message);
+    }
+
+    return taskContent;
+  }
+
+  // Return handler with setAgent method attached
+  handleTaskFire.setAgent = setAgent;
+  return handleTaskFire;
+}