neoagent 1.4.1 → 1.4.3

package/server/routes/memory.js

@@ -186,7 +186,9 @@ router.get('/conversations', (req, res) => {
 
 router.post('/conversations/search', (req, res) => {
   const mm = req.app.locals.memoryManager;
-  const results = mm.searchConversations(req.session.userId, req.body.query);
+  const results = mm.searchConversations(req.session.userId, req.body.query, {
+    sessions: parseInt(req.body.limit) || 8
+  });
   res.json(results);
 });
 

package/server/routes/settings.js

@@ -5,6 +5,7 @@ const db = require('../db/database');
 const { requireAuth } = require('../middleware/auth');
 const { normalizeWhatsAppWhitelist } = require('../utils/whatsapp');
 const { UPDATE_STATUS_FILE, APP_DIR } = require('../../runtime/paths');
+const { ensureDefaultAiSettings, DEFAULT_AI_SETTINGS } = require('../services/ai/settings');
 
 router.use(requireAuth);
 
@@ -35,8 +36,9 @@ router.get('/meta/models', (req, res) => {
 
 // Get all settings
 router.get('/', (req, res) => {
+  ensureDefaultAiSettings(req.session.userId);
   const rows = db.prepare('SELECT key, value FROM user_settings WHERE user_id = ?').all(req.session.userId);
-  const settings = {};
+  const settings = { ...DEFAULT_AI_SETTINGS };
   for (const row of rows) {
     try {
       settings[row.key] = JSON.parse(row.value);
@@ -123,6 +125,31 @@ router.get('/token-usage/summary', (req, res) => {
     return acc;
   }, { tokens: 0, runs: 0 });
 
+  const promptMetricRows = db.prepare(`
+    SELECT prompt_metrics
+    FROM agent_runs
+    WHERE user_id = ? AND prompt_metrics IS NOT NULL
+    ORDER BY created_at DESC
+    LIMIT 20
+  `).all(userId);
+
+  const parsedMetrics = promptMetricRows
+    .map((row) => {
+      try { return JSON.parse(row.prompt_metrics); } catch { return null; }
+    })
+    .filter(Boolean);
+
+  const metricTotals = parsedMetrics.reduce((acc, item) => {
+    const last = item.lastEstimate || {};
+    acc.count += 1;
+    acc.system += Number(last.systemPromptTokens || 0);
+    acc.tools += Number(last.toolSchemaTokens || 0);
+    acc.history += Number(last.historyTokens || 0);
+    acc.recall += Number(last.recalledMemoryTokens || 0);
+    acc.replay += Number(last.toolReplayTokens || 0);
+    return acc;
+  }, { count: 0, system: 0, tools: 0, history: 0, recall: 0, replay: 0 });
+
   res.json({
     totals: {
       totalTokens: Number(totals?.totalTokens || 0),
@@ -131,7 +158,18 @@ router.get('/token-usage/summary', (req, res) => {
       last7DaysTokens: last7Totals.tokens,
       last7DaysRuns: last7Totals.runs
     },
-    last7Days
+    last7Days,
+    promptMetrics: {
+      sampleCount: metricTotals.count,
+      average: metricTotals.count === 0 ? null : {
+        systemPromptTokens: Math.round(metricTotals.system / metricTotals.count),
+        toolSchemaTokens: Math.round(metricTotals.tools / metricTotals.count),
+        historyTokens: Math.round(metricTotals.history / metricTotals.count),
+        recalledMemoryTokens: Math.round(metricTotals.recall / metricTotals.count),
+        toolReplayTokens: Math.round(metricTotals.replay / metricTotals.count)
+      },
+      latest: parsedMetrics[0] || null
+    }
   });
 });
 

package/server/routes/skills.js

@@ -1,108 +1,147 @@
 const express = require('express');
 const router = express.Router();
-const fs = require('fs');
-const path = require('path');
 const { requireAuth } = require('../middleware/auth');
-const { AGENT_DATA_DIR } = require('../../runtime/paths');
-
-const SKILLS_DIR = path.join(AGENT_DATA_DIR, 'skills');
-
-/**
- * Resolve a client-supplied filename to an absolute path inside SKILLS_DIR.
- * Returns null if the resolved path escapes the directory (path traversal attempt).
- */
-function safeSkillPath(filename) {
-  if (!filename || typeof filename !== 'string') return null;
-  // Strip any directory components – only the basename is allowed
-  const base = path.basename(filename);
-  if (!base || base === '.' || base === '..') return null;
-  const resolved = path.resolve(SKILLS_DIR, base);
-  // Ensure the resolved path stays inside SKILLS_DIR
-  if (!resolved.startsWith(SKILLS_DIR + path.sep) && resolved !== SKILLS_DIR) return null;
-  return resolved;
-}
+const { sanitizeError } = require('../utils/security');
+const { SkillRunner } = require('../services/ai/toolRunner');
 
 router.use(requireAuth);
 
-// List all skills
-router.get('/', (req, res) => {
-  if (!fs.existsSync(SKILLS_DIR)) fs.mkdirSync(SKILLS_DIR, { recursive: true });
+async function getSkillRunner(app) {
+  if (app.locals?.skillRunner) return app.locals.skillRunner;
+  const runner = new SkillRunner();
+  await runner.loadSkills();
+  return runner;
+}
 
-  const files = fs.readdirSync(SKILLS_DIR).filter(f => f.endsWith('.md'));
-  const skills = files.map(f => {
-    const content = fs.readFileSync(path.join(SKILLS_DIR, f), 'utf-8');
-    const meta = parseSkillMeta(content);
+function parseSkillDocument(content) {
+  const match = String(content || '').match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  if (!match) {
     return {
-      filename: f,
-      name: meta.name || f.replace('.md', ''),
-      description: meta.description || '',
-      trigger: meta.trigger || '',
-      enabled: meta.enabled !== false,
-      category: meta.category || 'general'
+      error: 'Skill files must start with frontmatter delimited by ---'
     };
-  });
-
-  res.json(skills);
-});
-
-// Get a specific skill
-router.get('/:filename', (req, res) => {
-  const fp = safeSkillPath(req.params.filename);
-  if (!fp || !fs.existsSync(fp)) return res.status(404).json({ error: 'Skill not found' });
-
-  const content = fs.readFileSync(fp, 'utf-8');
-  res.json({ filename: req.params.filename, content, meta: parseSkillMeta(content) });
-});
+  }
 
-// Create a new skill
-router.post('/', (req, res) => {
-  const { filename, content } = req.body;
-  if (!filename || !content) return res.status(400).json({ error: 'filename and content required' });
+  const frontmatter = {};
+  for (const line of match[1].split('\n')) {
+    const colon = line.indexOf(':');
+    if (colon === -1) continue;
+    const key = line.slice(0, colon).trim();
+    let value = line.slice(colon + 1).trim();
+    if (value === 'true') value = true;
+    else if (value === 'false') value = false;
+    else if ((value.startsWith('{') && value.endsWith('}')) || (value.startsWith('[') && value.endsWith(']'))) {
+      try { value = JSON.parse(value); } catch { /* keep raw */ }
+    }
+    frontmatter[key] = value;
+  }
 
-  const baseName = filename.replace(/\.md$/i, '').replace(/[^a-zA-Z0-9_.-]/g, '');
-  const safeName = baseName + '.md';
-  const fp = path.join(SKILLS_DIR, safeName);
+  if (!frontmatter.name) {
+    return { error: 'Skill frontmatter must include name' };
+  }
 
-  if (!fs.existsSync(SKILLS_DIR)) fs.mkdirSync(SKILLS_DIR, { recursive: true });
+  return {
+    name: String(frontmatter.name),
+    description: String(frontmatter.description || ''),
+    instructions: match[2].trim(),
+    metadata: Object.fromEntries(
+      Object.entries(frontmatter).filter(([key]) => !['name', 'description'].includes(key))
+    )
+  };
+}
 
-  fs.writeFileSync(fp, content, 'utf-8');
-  res.status(201).json({ filename: safeName, meta: parseSkillMeta(content) });
+router.get('/', async (req, res) => {
+  try {
+    const runner = await getSkillRunner(req.app);
+    const skills = runner.getAll().map((skill) => ({
+      name: skill.name,
+      description: skill.description,
+      enabled: skill.enabled,
+      draft: skill.metadata?.draft === true,
+      category: skill.metadata?.category || 'general',
+      trigger: skill.metadata?.trigger || '',
+      source: skill.metadata?.source || 'local',
+      autoCreated: skill.metadata?.auto_created === true,
+      filePath: skill.filePath
+    }));
+    res.json(skills.sort((a, b) => {
+      if (a.draft !== b.draft) return a.draft ? -1 : 1;
+      return a.name.localeCompare(b.name);
+    }));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
 });
 
-// Update a skill
-router.put('/:filename', (req, res) => {
-  const fp = safeSkillPath(req.params.filename);
-  if (!fp || !fs.existsSync(fp)) return res.status(404).json({ error: 'Skill not found' });
-
-  fs.writeFileSync(fp, req.body.content, 'utf-8');
-  res.json({ filename: path.basename(fp), meta: parseSkillMeta(req.body.content) });
+router.get('/:name', async (req, res) => {
+  try {
+    const runner = await getSkillRunner(req.app);
+    const skill = runner.getSkill(req.params.name);
+    if (!skill) return res.status(404).json({ error: 'Skill not found' });
+    const fs = require('fs');
+    const content = fs.readFileSync(skill.filePath, 'utf-8');
+    res.json({
+      name: skill.name,
+      content,
+      meta: skill.metadata,
+      enabled: skill.metadata?.enabled !== false
+    });
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
 });
 
-// Delete a skill
-router.delete('/:filename', (req, res) => {
-  const fp = safeSkillPath(req.params.filename);
-  if (!fp || !fs.existsSync(fp)) return res.status(404).json({ error: 'Skill not found' });
-
-  fs.unlinkSync(fp);
-  res.json({ success: true });
+router.post('/', async (req, res) => {
+  try {
+    const runner = await getSkillRunner(req.app);
+    const parsed = parseSkillDocument(req.body.content);
+    if (parsed.error) return res.status(400).json({ error: parsed.error });
+
+    const result = runner.createSkill(
+      req.body.filename || parsed.name,
+      parsed.description,
+      parsed.instructions,
+      parsed.metadata
+    );
+
+    if (result.error) return res.status(400).json(result);
+    res.status(201).json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
 });
 
-function parseSkillMeta(content) {
-  const meta = {};
-  const match = content.match(/^---\n([\s\S]*?)\n---/);
-  if (!match) return meta;
+router.put('/:name', async (req, res) => {
+  try {
+    const runner = await getSkillRunner(req.app);
+    if (typeof req.body.enabled === 'boolean' && !req.body.content) {
+      const result = runner.setSkillEnabled(req.params.name, req.body.enabled);
+      if (result.error) return res.status(404).json(result);
+      return res.json(result);
+    }
+
+    const parsed = parseSkillDocument(req.body.content);
+    if (parsed.error) return res.status(400).json({ error: parsed.error });
+    const result = runner.updateSkill(req.params.name, {
+      description: parsed.description,
+      instructions: parsed.instructions,
+      metadata: parsed.metadata
+    });
+    if (result.error) return res.status(404).json(result);
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
 
-  const lines = match[1].split('\n');
-  for (const line of lines) {
-    const colon = line.indexOf(':');
-    if (colon === -1) continue;
-    const key = line.slice(0, colon).trim();
-    let val = line.slice(colon + 1).trim();
-    if (val === 'true') val = true;
-    else if (val === 'false') val = false;
-    meta[key] = val;
+router.delete('/:name', async (req, res) => {
+  try {
+    const runner = await getSkillRunner(req.app);
+    const result = runner.deleteSkill(req.params.name);
+    if (result.error) return res.status(404).json(result);
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
   }
-  return meta;
-}
+});
 
 module.exports = router;

package/server/routes/store.js

@@ -317,6 +317,46 @@ Run in the user's specified directory (default: current working directory):
 Present as a structured git dashboard. Note any uncommitted changes or detached HEAD.`
   },
 
+  {
+    id: 'pdf-toolkit',
+    name: 'PDF Toolkit',
+    description: 'Inspect, extract, split, merge, compress, and rearrange PDF files.',
+    category: 'productivity',
+    icon: '📄',
+    content: `---
+name: pdf-toolkit
+description: Inspect, extract, split, merge, compress, and rearrange PDF files
+category: productivity
+icon: 📄
+enabled: true
+---
+
+Work with PDF files using whatever is available on the machine. Prefer tools in this order when relevant:
+- \`pdfinfo\`, \`pdftotext\`, \`pdftoppm\`, \`pdftocairo\` from Poppler for inspection, text extraction, and page rendering
+- \`qpdf\` for splitting, merging, rotating, decrypting, and page selection
+- \`pdftk\` for merge/split/stamp workflows if available
+- \`mutool\` for extracting text, objects, and cleaning PDFs
+- \`gs\` (Ghostscript) for compression or PDF regeneration
+
+Workflow:
+1. Verify the input file exists.
+2. Check which PDF utilities are installed with \`which\`.
+3. Choose the safest tool for the requested operation.
+4. Write outputs next to the source file unless the user specifies a destination.
+5. Report the output path, page counts, and any limitations clearly.
+
+Common tasks:
+- Inspect a PDF with \`pdfinfo <file.pdf>\`
+- Extract text with \`pdftotext <file.pdf> -\` or \`pdftotext -layout <file.pdf> -\`
+- Merge PDFs with \`qpdf --empty --pages a.pdf b.pdf -- out.pdf\`
+- Split pages with \`qpdf in.pdf --pages in.pdf 1-5 -- out.pdf\`
+- Reorder or remove pages with \`qpdf in.pdf --pages in.pdf 1,3,5-7 -- out.pdf\`
+- Compress via Ghostscript:
+\`gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/ebook -dNOPAUSE -dQUIET -dBATCH -sOutputFile=out.pdf in.pdf\`
+
+Preserve the original file unless the user explicitly asks to overwrite it.`
+  },
+
   {
     id: 'docker-status',
     name: 'Docker Status',
@@ -1047,6 +1087,66 @@ ruby <file>.rb             # Ruby
 - No unused imports or dead code left behind`
   },
 
+  {
+    id: 'csv-toolkit',
+    name: 'CSV Toolkit',
+    description: 'Inspect, clean, summarize, filter, and transform CSV and TSV files.',
+    category: 'productivity',
+    icon: '📊',
+    content: `---
+name: csv-toolkit
+description: Inspect, clean, summarize, filter, and transform CSV and TSV files
+category: productivity
+icon: 📊
+enabled: true
+---
+
+Work with local CSV or TSV data using lightweight shell tools.
+
+Preferred tools:
+- \`python3\` with the standard \`csv\` module for reliable parsing
+- \`mlr\` (Miller) if installed for fast tabular summaries
+- \`csvkit\` commands such as \`csvlook\`, \`csvcut\`, \`csvstat\`, \`csvgrep\` if installed
+
+Workflow:
+1. Detect delimiter and header structure from the file.
+2. For quick inspection, show row count, column count, header names, and a 5-row preview.
+3. For analysis requests, compute exactly what the user asked for: filtering, grouping, aggregations, sorting, or conversion to JSON or Markdown.
+4. Save transformed output to a sibling file unless the user asked for inline output only.
+
+Avoid naive comma-splitting because quoted fields can break it. If the file is large, sample first and then run the full transform once the operation is clear.`
+  },
+
+  {
+    id: 'markdown-workbench',
+    name: 'Markdown Workbench',
+    description: 'Format, lint, outline, and convert Markdown notes and docs.',
+    category: 'productivity',
+    icon: '📝',
+    content: `---
+name: markdown-workbench
+description: Format, lint, outline, and convert Markdown notes and docs
+category: productivity
+icon: 📝
+enabled: true
+---
+
+Help turn rough notes into usable docs quickly.
+
+Workflow:
+1. Read the source Markdown or note file.
+2. Decide whether the user needs cleanup and formatting, heading structure, summary or outline, checklist extraction, or conversion to HTML or PDF.
+3. Preserve meaning while improving structure and readability.
+
+Common tasks:
+- Normalize headings, lists, code fences, and spacing
+- Pull out tasks into a checklist grouped by topic or urgency
+- Build a short linked outline from headings
+- If available, use \`pandoc\` to convert Markdown to HTML, DOCX, or PDF
+
+Prefer preserving the user's tone unless they ask for a rewrite.`
+  },
+
   // ── MAKER ────────────────────────────────────────────────────────────────────
   {
     id: 'psa-car-controller',

package/server/services/ai/compaction.js

@@ -2,50 +2,38 @@ async function compact(messages, provider, model) {
   const systemMsg = messages.find(m => m.role === 'system');
   const nonSystem = messages.filter(m => m.role !== 'system');
 
-  // Only compact once history is clearly old enough to avoid touching recent context.
   if (nonSystem.length < 12) return messages;
 
-  const keepRecent = 6;
+  const keepRecent = 8;
   const toCompact = nonSystem.slice(0, -keepRecent);
   const recent = nonSystem.slice(-keepRecent);
 
-  let compactionText = '';
-  for (const msg of toCompact) {
-    if (msg.role === 'user') {
-      compactionText += `User: ${(msg.content || '').slice(0, 500)}\n`;
-    } else if (msg.role === 'assistant') {
-      const content = (msg.content || '').slice(0, 500);
-      if (msg.tool_calls) {
-        const tools = msg.tool_calls.map(tc => tc.function.name).join(', ');
-        compactionText += `Assistant: [used tools: ${tools}] ${content}\n`;
-      } else {
-        compactionText += `Assistant: ${content}\n`;
-      }
-    } else if (msg.role === 'tool') {
-      const result = (msg.content || '').slice(0, 200);
-      compactionText += `Tool result: ${result}\n`;
+  const compactionText = toCompact.map((msg) => {
+    if (msg.role === 'assistant' && msg.tool_calls) {
+      const tools = msg.tool_calls.map((tc) => tc.function.name).join(', ');
+      return `assistant(tools:${tools}) ${(msg.content || '').slice(0, 320)}`;
     }
-  }
+    if (msg.role === 'tool') {
+      return `tool:${msg.name || 'tool'} ${(msg.content || '').slice(0, 220)}`;
+    }
+    return `${msg.role}: ${(msg.content || '').slice(0, 360)}`;
+  }).join('\n');
 
   const summaryPrompt = [
-    { role: 'system', content: 'Summarize this conversation history concisely. Preserve: key decisions, facts learned, user preferences, task outcomes, and any errors or important results. Be thorough but compact.' },
+    { role: 'system', content: 'Compress conversation context. Preserve goals, constraints, tool outcomes, errors, and unresolved work. Keep it compact.' },
     { role: 'user', content: `Summarize this conversation:\n\n${compactionText}` }
   ];
 
   try {
-    const response = await provider.chat(summaryPrompt, [], { model, maxTokens: 1000 });
+    const response = await provider.chat(summaryPrompt, [], { model, maxTokens: 320 });
     const summary = response.content || 'Previous conversation context (summary unavailable).';
 
     const compactedMessages = [];
     if (systemMsg) compactedMessages.push(systemMsg);
     compactedMessages.push({
-      role: 'user',
+      role: 'system',
       content: `[Previous conversation summary]\n${summary}`
     });
-    compactedMessages.push({
-      role: 'assistant',
-      content: 'Understood. I have the context from our previous conversation. Continuing.'
-    });
     compactedMessages.push(...recent);
 
     return compactedMessages;
@@ -54,13 +42,9 @@ async function compact(messages, provider, model) {
     const trimmed = [];
     if (systemMsg) trimmed.push(systemMsg);
     trimmed.push({
-      role: 'user',
+      role: 'system',
       content: '[Earlier conversation context was trimmed due to length]'
     });
-    trimmed.push({
-      role: 'assistant',
-      content: 'Understood. Some earlier context was trimmed. Continuing with recent messages.'
-    });
     trimmed.push(...recent);
     return trimmed;
   }