daemora 1.0.3 → 1.0.5

package/src/services/cleanup.js

@@ -0,0 +1,106 @@
+/**
+ * cleanup.js - Data retention management.
+ *
+ * Cleans up old task files, audit logs, cost logs, and stale sub-agent sessions.
+ * Configurable via CLEANUP_AFTER_DAYS env var (default: 30, 0 = never delete).
+ */
+import { readdirSync, statSync, unlinkSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { config } from "../config/default.js";
+
+const CLEANUP_DAYS = parseInt(process.env.CLEANUP_AFTER_DAYS || "30", 10);
+
+/**
+ * Run cleanup across all data directories.
+ * @param {number} [days] - Override retention days (0 = skip)
+ * @returns {{ tasks: number, audit: number, costs: number, sessions: number, total: number }}
+ */
+export function runCleanup(days = CLEANUP_DAYS) {
+  if (days <= 0) return { tasks: 0, audit: 0, costs: 0, sessions: 0, total: 0 };
+
+  const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+  const results = {
+    tasks: cleanDir(config.tasksDir, cutoff, ".json"),
+    audit: cleanDir(config.auditDir, cutoff, ".jsonl"),
+    costs: cleanDir(config.costsDir, cutoff, ".jsonl"),
+    sessions: cleanStaleSessions(cutoff),
+    total: 0,
+  };
+  results.total = results.tasks + results.audit + results.costs + results.sessions;
+  return results;
+}
+
+/**
+ * Delete files older than cutoff in a directory.
+ */
+function cleanDir(dirPath, cutoffMs, ext) {
+  if (!existsSync(dirPath)) return 0;
+  let deleted = 0;
+  try {
+    const files = readdirSync(dirPath).filter(f => f.endsWith(ext));
+    for (const file of files) {
+      const filePath = join(dirPath, file);
+      try {
+        const mtime = statSync(filePath).mtimeMs;
+        if (mtime < cutoffMs) {
+          unlinkSync(filePath);
+          deleted++;
+        }
+      } catch {}
+    }
+  } catch {}
+  return deleted;
+}
+
+/**
+ * Clean sub-agent sessions (telegram-123--coder.json) that are stale.
+ * Main user sessions (no "--") are kept regardless.
+ */
+function cleanStaleSessions(cutoffMs) {
+  if (!existsSync(config.sessionsDir)) return 0;
+  let deleted = 0;
+  try {
+    const files = readdirSync(config.sessionsDir).filter(f => f.endsWith(".json"));
+    for (const file of files) {
+      const name = file.slice(0, -5);
+      // Only clean sub-agent sessions (contain "--")
+      if (!name.includes("--")) continue;
+      const filePath = join(config.sessionsDir, file);
+      try {
+        const mtime = statSync(filePath).mtimeMs;
+        if (mtime < cutoffMs) {
+          unlinkSync(filePath);
+          deleted++;
+        }
+      } catch {}
+    }
+  } catch {}
+  return deleted;
+}
+
+/**
+ * Get storage stats without deleting anything.
+ */
+export function getStorageStats() {
+  return {
+    tasks: countDir(config.tasksDir, ".json"),
+    audit: countDir(config.auditDir, ".jsonl"),
+    costs: countDir(config.costsDir, ".jsonl"),
+    sessions: countDir(config.sessionsDir, ".json"),
+    retentionDays: CLEANUP_DAYS || "never",
+  };
+}
+
+function countDir(dirPath, ext) {
+  if (!existsSync(dirPath)) return { files: 0, sizeKB: 0 };
+  try {
+    const files = readdirSync(dirPath).filter(f => f.endsWith(ext));
+    let totalSize = 0;
+    for (const file of files) {
+      try { totalSize += statSync(join(dirPath, file)).size; } catch {}
+    }
+    return { files: files.length, sizeKB: Math.round(totalSize / 1024) };
+  } catch {
+    return { files: 0, sizeKB: 0 };
+  }
+}

package/src/services/sessions.js

@@ -1,5 +1,5 @@
 import { v4 as uuidv4 } from "uuid";
-import { writeFileSync, readFileSync, existsSync, mkdirSync } from "fs";
+import { writeFileSync, readFileSync, existsSync, mkdirSync, readdirSync, unlinkSync } from "fs";
 import { config } from "../config/default.js";
 
 const SESSIONS_DIR = config.sessionsDir;
@@ -17,6 +17,7 @@ export function createSession(existingId = null) {
     messages: [],
   };
   sessions.set(sessionId, session);
+  saveSession(session);
   return session;
 }
 
@@ -59,6 +60,43 @@ export function setMessages(sessionId, messages) {
   return session;
 }
 
+/**
+ * List sub-agent session IDs. If prefix given, only returns sessions starting with `{prefix}--`.
+ */
+export function listSessions(prefix = null) {
+  try {
+    const files = readdirSync(SESSIONS_DIR).filter(f => f.endsWith(".json"));
+    let sessionIds = files.map(f => f.slice(0, -5));
+    if (prefix) {
+      // Return only sub-agent sessions for this parent
+      sessionIds = sessionIds.filter(id => id.startsWith(prefix + "--"));
+    } else {
+      // Exclude sub-agent sessions (contain "--") from top-level listing
+      sessionIds = sessionIds.filter(id => !id.includes("--"));
+    }
+    return sessionIds;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Clear a session — removes messages from memory and deletes file from disk.
+ */
+export function clearSession(sessionId) {
+  let found = false;
+  if (sessions.has(sessionId)) {
+    sessions.delete(sessionId);
+    found = true;
+  }
+  const filePath = `${SESSIONS_DIR}/${sessionId}.json`;
+  if (existsSync(filePath)) {
+    unlinkSync(filePath);
+    found = true;
+  }
+  return found;
+}
+
 function saveSession(session) {
   const filePath = `${SESSIONS_DIR}/${session.sessionId}.json`;
   writeFileSync(filePath, JSON.stringify(session, null, 2));

package/src/setup/wizard.js

@@ -7,7 +7,53 @@ import { banner, stepHeader, kv, summaryTable, completeBanner, t, S } from "./th
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT_DIR = join(__dirname, "..", "..");
-const TOTAL_STEPS = 8;
+const TOTAL_STEPS = 9;
+const OLLAMA_EMBED_MODEL = "all-minilm";
+
+/**
+ * Pull all-minilm embedding model (if Ollama available) and pre-embed all skills.
+ * This runs during setup so the agent has instant skill matching from first task.
+ */
+async function setupSkillEmbeddings(provider, envConfig, spin) {
+  const { execSync } = await import("child_process");
+  const hasOllama = (() => {
+    try { execSync("ollama --version", { stdio: "ignore" }); return true; } catch { return false; }
+  })();
+
+  // Pull local embedding model as primary (no API key) or fallback (has API key)
+  const hasApiEmbedding = envConfig.OPENAI_API_KEY || envConfig.GOOGLE_AI_API_KEY;
+
+  if (hasOllama) {
+    const purpose = hasApiEmbedding ? "offline fallback" : "skill matching";
+    spin.message(`Pulling ${OLLAMA_EMBED_MODEL} for ${purpose} (22M params, ~45MB)`);
+    try {
+      execSync(`ollama pull ${OLLAMA_EMBED_MODEL}`, { stdio: "ignore", timeout: 120_000 });
+      p.log.success(`${S.check}  Embedding model ${t.bold(OLLAMA_EMBED_MODEL)} ready (${purpose})`);
+    } catch {
+      if (!hasApiEmbedding) {
+        p.log.warn(`Could not pull ${OLLAMA_EMBED_MODEL}. Skill matching will use built-in TF-IDF.`);
+      }
+    }
+  }
+
+  // Pre-embed all skills (works with any provider: API, Ollama, or TF-IDF)
+  spin.message("Pre-embedding skills for instant matching");
+  try {
+    // Temporarily set env vars so the embedding provider can detect them
+    if (envConfig.OPENAI_API_KEY) process.env.OPENAI_API_KEY = envConfig.OPENAI_API_KEY;
+    if (envConfig.GOOGLE_AI_API_KEY) process.env.GOOGLE_AI_API_KEY = envConfig.GOOGLE_AI_API_KEY;
+    if (provider === "ollama" || hasOllama) process.env.OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
+
+    const skillLoader = (await import("../skills/SkillLoader.js")).default;
+    skillLoader.load();
+    await skillLoader.embedSkills();
+
+    const count = skillLoader.list().length;
+    p.log.success(`${S.check}  ${count} skills embedded for instant matching`);
+  } catch {
+    p.log.info("Skill embedding deferred to first startup.");
+  }
+}
 
 function cancelled() {
   p.cancel("Setup cancelled.");
@@ -646,8 +692,25 @@ export async function runSetupWizard() {
 
   p.log.success(`Daemon: ${t.bold(daemonMode ? "Enabled" : "Disabled")}`);
 
-  // ━━━ Step 7: MCP Servers ━━━
-  stepHeader(7, TOTAL_STEPS, "MCP Tool Servers");
+  // ━━━ Step 7: Data Cleanup ━━━
+  stepHeader(7, TOTAL_STEPS, "Data Cleanup");
+
+  const cleanupDays = guard(await p.select({
+    message: "Auto-delete old tasks, logs & sessions after how many days?",
+    options: [
+      { value: "30",  label: "30 days",  hint: "recommended" },
+      { value: "7",   label: "7 days",   hint: "aggressive — saves most space" },
+      { value: "90",  label: "90 days",  hint: "keep 3 months of history" },
+      { value: "365", label: "1 year",   hint: "long-term retention" },
+      { value: "0",   label: "Never",    hint: "keep everything forever" },
+    ],
+  }));
+  envConfig.CLEANUP_AFTER_DAYS = cleanupDays;
+
+  p.log.success(`Cleanup: ${t.bold(cleanupDays === "0" ? "Never" : cleanupDays + " days")}`);
+
+  // ━━━ Step 8: MCP Servers ━━━
+  stepHeader(8, TOTAL_STEPS, "MCP Tool Servers");
 
   p.note(
     [
@@ -962,7 +1025,7 @@ export async function runSetupWizard() {
   }
 
   // ━━━ Step 8: Secret Vault ━━━
-  stepHeader(8, TOTAL_STEPS, "Secret Vault");
+  stepHeader(9, TOTAL_STEPS, "Secret Vault");
 
   p.note(
     [
@@ -1101,6 +1164,14 @@ export async function runSetupWizard() {
     }
   }
 
+  // Pull embedding model & pre-embed skills
+  spin.message("Setting up skill embeddings");
+  try {
+    await setupSkillEmbeddings(provider, envConfig, spin);
+  } catch {
+    // Non-fatal — TF-IDF fallback will handle it
+  }
+
   spin.stop(`${S.check}  Configuration saved`);
 
   // ━━━ Summary ━━━

package/src/skills/SkillLoader.js

@@ -1,8 +1,8 @@
-import { readFileSync, writeFileSync, existsSync, readdirSync, mkdirSync } from "fs";
-import { join } from "path";
+import { readFileSync, writeFileSync, existsSync, readdirSync, mkdirSync, statSync, accessSync, constants } from "fs";
+import { join, delimiter } from "path";
 import { createHash } from "node:crypto";
 import { config } from "../config/default.js";
-import { generateEmbedding, getEmbeddingProvider } from "../utils/Embeddings.js";
+import { generateEmbedding, getEmbeddingProvider, buildTfidfVocab } from "../utils/Embeddings.js";
 
 /**
  * Skill Loader - auto-discovers .md skill files from the skills/ directory.
@@ -47,24 +47,12 @@ class SkillLoader {
       return;
     }
 
-    const files = readdirSync(skillsDir).filter((f) => f.endsWith(".md"));
     this.skills.clear();
-
-    for (const file of files) {
-      try {
-        const filePath = join(skillsDir, file);
-        const content = readFileSync(filePath, "utf-8");
-        const skill = this.parseSkill(content, file);
-        if (skill) {
-          this.skills.set(skill.name, skill);
-        }
-      } catch (error) {
-        console.log(`[SkillLoader] Error loading ${file}: ${error.message}`);
-      }
-    }
+    this._loadFromDir(skillsDir);
 
     this.loaded = true;
     this._loadSkillVectors();
+    this._buildTfidfIndex();
     console.log(
       `[SkillLoader] Loaded ${this.skills.size} skills: ${[...this.skills.keys()].join(", ") || "(none)"}`
     );
@@ -104,10 +92,56 @@ class SkillLoader {
       triggers: meta.triggers
         ? meta.triggers.split(",").map((t) => t.trim().toLowerCase())
         : [],
+      // Eligibility fields (OpenClaw-style filtering)
+      os: meta.os ? meta.os.split(",").map((s) => s.trim().toLowerCase()) : [],
+      requires: meta.requires ? meta.requires.split(",").map((s) => s.trim()) : [],
+      env: meta.env ? meta.env.split(",").map((s) => s.trim()) : [],
       content: body,
     };
   }
 
+  /**
+   * Load skills from a directory — supports flat .md files and subdirectories with SKILL.md.
+   * Scans one level deep: skills/foo.md and skills/bar/SKILL.md both work.
+   */
+  _loadFromDir(dir) {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      try {
+        const entryPath = join(dir, entry);
+        const stat = statSync(entryPath);
+
+        if (stat.isFile() && entry.endsWith(".md")) {
+          // Flat file: skills/coding.md
+          const content = readFileSync(entryPath, "utf-8");
+          const skill = this.parseSkill(content, entry);
+          if (skill) this.skills.set(skill.name, skill);
+        } else if (stat.isDirectory()) {
+          // Subdirectory: skills/webapp-testing/SKILL.md
+          const skillMd = join(entryPath, "SKILL.md");
+          if (existsSync(skillMd)) {
+            const content = readFileSync(skillMd, "utf-8");
+            const skill = this.parseSkill(content, `${entry}/SKILL.md`);
+            if (skill) this.skills.set(skill.name, skill);
+          }
+        }
+      } catch (error) {
+        console.log(`[SkillLoader] Error loading ${entry}: ${error.message}`);
+      }
+    }
+  }
+
+  /**
+   * Build TF-IDF vocabulary from all loaded skills (zero-cost local embeddings fallback).
+   */
+  _buildTfidfIndex() {
+    const docs = [];
+    for (const [, skill] of this.skills) {
+      docs.push(`${skill.name} ${skill.description} ${skill.triggers.join(" ")} ${skill.content.slice(0, 500)}`);
+    }
+    buildTfidfVocab(docs);
+  }
+
   // ── Embedding helpers ────────────────────────────────────────────────────────
 
   _contentHash(skill) {
@@ -237,6 +271,59 @@ class SkillLoader {
     return this.getSkillPrompts(taskInput);
   }
 
+  /**
+   * Get matched skill summaries (name + description + path) for lazy loading.
+   * Uses hybrid ranking: embeddings (API or local) → keyword fallback → list all.
+   * Returns up to `limit` skills, sorted by relevance.
+   */
+  async getMatchedSkillSummaries(taskInput, limit = 20) {
+    if (!this.loaded) this.load();
+    if (this.skills.size === 0) return [];
+
+    const toSummary = (skill) => ({
+      name: skill.name,
+      description: skill.description,
+      path: `skills/${skill.name}.md`,
+    });
+
+    // 1. Embedding match (OpenAI/Google/Ollama/TF-IDF — whatever is available)
+    if (taskInput) {
+      const vectorsAvailable = Object.keys(this._skillVectors).length > 0;
+      if (getEmbeddingProvider() && vectorsAvailable) {
+        const queryVector = await this._generateEmbedding(taskInput);
+        if (queryVector) {
+          const scored = [];
+          for (const [name, skill] of this.skills) {
+            const cached = this._skillVectors[name];
+            if (!cached?.vector) continue;
+            const score = this._cosineSim(queryVector, cached.vector);
+            scored.push({ skill, score });
+          }
+          scored.sort((a, b) => b.score - a.score);
+          const top = scored.slice(0, limit);
+          if (top.length > 0) {
+            console.log(`[SkillLoader] Ranked top ${top.length}/${this.skills.size} skills (embedding)`);
+            return top.map((s) => toSummary(s.skill));
+          }
+        }
+      }
+
+      // 2. Keyword fallback — matched first, then fill remaining up to limit
+      const keywordMatched = this.matchSkills(taskInput);
+      if (keywordMatched.length > 0) {
+        const matchedNames = new Set(keywordMatched.map((s) => s.name));
+        const rest = [...this.skills.values()].filter((s) => !matchedNames.has(s.name));
+        const combined = [...keywordMatched, ...rest].slice(0, limit);
+        console.log(`[SkillLoader] Ranked top ${combined.length}/${this.skills.size} skills (keyword)`);
+        return combined.map(toSummary);
+      }
+    }
+
+    // 3. No match or no input — return first N alphabetically
+    const all = [...this.skills.values()].slice(0, limit);
+    return all.map(toSummary);
+  }
+
   // ── Sync keyword API (fallback) ───────────────────────────────────────────────
 
   /**

package/src/storage/TaskStore.js

@@ -23,7 +23,7 @@ export function loadTask(taskId) {
 /**
  * List recent tasks (sorted by createdAt descending).
  */
-export function listTasks({ limit = 20, status = null } = {}) {
+export function listTasks({ limit = 20, status = null, type = null } = {}) {
   const files = readdirSync(TASKS_DIR).filter((f) => f.endsWith(".json"));
   let tasks = files.map((f) => {
     try {
@@ -37,10 +37,28 @@ export function listTasks({ limit = 20, status = null } = {}) {
     tasks = tasks.filter((t) => t.status === status);
   }
 
+  if (type) {
+    tasks = tasks.filter((t) => (t.type || "chat") === type);
+  }
+
   tasks.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt));
   return tasks.slice(0, limit);
 }
 
+/**
+ * List child tasks of a given parent task.
+ */
+export function listChildTasks(parentTaskId) {
+  const files = readdirSync(TASKS_DIR).filter((f) => f.endsWith(".json"));
+  return files
+    .map((f) => {
+      try { return JSON.parse(readFileSync(`${TASKS_DIR}/${f}`, "utf-8")); }
+      catch { return null; }
+    })
+    .filter((t) => t && t.parentTaskId === parentTaskId)
+    .sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt));
+}
+
 /**
  * On startup, find tasks stuck in "running" state and reset to "pending".
  * Returns the count of tasks reset so the caller can log accordingly.