neoagent 1.4.0 → 1.4.3

package/.env.example

@@ -26,3 +26,8 @@ OPENAI_API_KEY=your-openai-api-key-here
 #   • Gemini models (e.g. gemini-3.1-flash-lite-preview)
 # Get your key at: https://aistudio.google.com/app/apikey
 GOOGLE_AI_KEY=your-google-ai-key-here
+
+# Brave Search API key — used for:
+#   • Native web_search tool (search the web without driving the browser)
+# Get your key at: https://api.search.brave.com/
+BRAVE_SEARCH_API_KEY=your-brave-search-api-key-here

package/com.neoagent.plist

@@ -7,21 +7,23 @@
 
     <key>ProgramArguments</key>
     <array>
-        <string>/usr/local/bin/node</string>
-        <string>/Users/neo/NeoAgent/server/index.js</string>
+        <string>__NODE_BIN__</string>
+        <string>__APP_DIR__/server/index.js</string>
     </array>
 
     <key>WorkingDirectory</key>
-    <string>/Users/neo/NeoAgent</string>
+    <string>__APP_DIR__</string>
 
     <key>EnvironmentVariables</key>
     <dict>
         <key>PATH</key>
         <string>/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin</string>
         <key>HOME</key>
-        <string>/Users/neo</string>
+        <string>__HOME__</string>
         <key>NODE_ENV</key>
         <string>production</string>
+        <key>NEOAGENT_HOME</key>
+        <string>__RUNTIME_HOME__</string>
     </dict>
 
     <!-- Auto-restart if the process exits for any reason -->
@@ -37,9 +39,9 @@
     <integer>10</integer>
 
     <key>StandardOutPath</key>
-    <string>/Users/neo/NeoAgent/data/logs/neoagent.log</string>
+    <string>__LOG_DIR__/neoagent.log</string>
 
     <key>StandardErrorPath</key>
-    <string>/Users/neo/NeoAgent/data/logs/neoagent.error.log</string>
+    <string>__LOG_DIR__/neoagent.error.log</string>
 </dict>
 </plist>

package/docs/configuration.md

@@ -1,6 +1,7 @@
 # Configuration
 
-All settings live in `.env` at the project root. Run `neoagent setup` to regenerate interactively.
+All settings live in `~/.neoagent/.env` by default. Run `neoagent setup` to regenerate interactively.
+You can override the runtime root with `NEOAGENT_HOME`.
 
 ## Variables
 
@@ -24,6 +25,7 @@ At least one API key is required. The active provider and model are configured i
 | `OPENAI_API_KEY` | GPT-4o / Whisper (OpenAI) |
 | `XAI_API_KEY` | Grok (xAI) |
 | `GOOGLE_AI_KEY` | Gemini (Google) |
+| `BRAVE_SEARCH_API_KEY` | Brave Search API for the native `web_search` tool |
 | `OLLAMA_URL` | Local Ollama (`http://localhost:11434`) |
 
 ### Messaging
@@ -34,6 +36,12 @@ At least one API key is required. The active provider and model are configured i
 
 Telegram, Discord, and WhatsApp tokens are stored in the database via the web UI Settings page — not in `.env`.
 
+## Runtime data paths
+
+- Config: `~/.neoagent/.env`
+- Database/session/logs: `~/.neoagent/data/`
+- Skills/soul/daily memory files: `~/.neoagent/agent-data/`
+
 ---
 
 ## Minimal `.env` example

package/docs/skills.md

@@ -1,6 +1,6 @@
 # Skills
 
-Skills are Markdown files in `agent-data/skills/` that describe capabilities the agent can use when responding to tasks. They are loaded at runtime — no restart needed after editing.
+Skills are Markdown files in `~/.neoagent/agent-data/skills/` by default. They are loaded at runtime — no restart needed after editing.
 
 ## Built-in skills
 
@@ -23,10 +23,14 @@ Skills are Markdown files in `agent-data/skills/` that describe capabilities the
 | `tail-log.md` | Tail any log file |
 | `news-hackernews.md` | Fetch Hacker News top stories |
 | `qr-code.md` | Generate QR codes |
+| `pdf-toolkit.md` | Inspect, extract, merge, split, and compress PDF files |
+| `git-summary.md` | Summarize git status, branches, commits, and diffs |
+| `csv-toolkit.md` | Inspect and transform CSV/TSV data files |
+| `markdown-workbench.md` | Clean up, outline, and convert Markdown notes/docs |
 
 ## Adding a skill
 
-Create a Markdown file in `agent-data/skills/`:
+Create a Markdown file in `~/.neoagent/agent-data/skills/`:
 
 ```markdown
 # My Skill Name

package/lib/manager.js

@@ -5,15 +5,22 @@ const net = require('net');
 const crypto = require('crypto');
 const readline = require('readline');
 const { spawn, spawnSync } = require('child_process');
+const {
+  APP_DIR,
+  RUNTIME_HOME,
+  DATA_DIR,
+  LOG_DIR,
+  ENV_FILE,
+  PID_FILE,
+  ensureRuntimeDirs,
+  migrateLegacyRuntime
+} = require('../runtime/paths');
 
-const APP_DIR = path.resolve(__dirname, '..');
 const APP_NAME = 'NeoAgent';
 const SERVICE_LABEL = 'com.neoagent';
 const PLIST_SRC = path.join(APP_DIR, 'com.neoagent.plist');
 const PLIST_DST = path.join(os.homedir(), 'Library', 'LaunchAgents', 'com.neoagent.plist');
 const SYSTEMD_UNIT = path.join(os.homedir(), '.config', 'systemd', 'user', 'neoagent.service');
-const LOG_DIR = path.join(APP_DIR, 'data', 'logs');
-const ENV_FILE = path.join(APP_DIR, '.env');
 
 const COLORS = process.stdout.isTTY
   ? {
@@ -136,7 +143,18 @@ function commandExists(cmd) {
 }
 
 function ensureLogDir() {
-  fs.mkdirSync(LOG_DIR, { recursive: true });
+  ensureRuntimeDirs();
+}
+
+function backupRuntimeData() {
+  const backupsDir = path.join(RUNTIME_HOME, 'backups');
+  fs.mkdirSync(backupsDir, { recursive: true });
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const target = path.join(backupsDir, `pre-update-${stamp}`);
+  fs.mkdirSync(target, { recursive: true });
+
+  if (fs.existsSync(ENV_FILE)) fs.copyFileSync(ENV_FILE, path.join(target, '.env'));
+  if (fs.existsSync(DATA_DIR)) fs.cpSync(DATA_DIR, path.join(target, 'data'), { recursive: true, force: false, errorOnExist: false });
 }
 
 function killByPort(port) {
@@ -198,6 +216,7 @@ async function ask(question, fallback = '') {
 
 async function cmdSetup() {
   heading('Environment Setup');
+  ensureRuntimeDirs();
 
   const current = {};
   if (fs.existsSync(ENV_FILE)) {
@@ -217,6 +236,7 @@ async function cmdSetup() {
   const openai = await ask('OpenAI API key', current.OPENAI_API_KEY || '');
   const xai = await ask('xAI API key', current.XAI_API_KEY || '');
   const google = await ask('Google API key', current.GOOGLE_AI_KEY || '');
+  const brave = await ask('Brave Search API key', current.BRAVE_SEARCH_API_KEY || '');
   const ollama = await ask('Ollama URL', current.OLLAMA_URL || 'http://localhost:11434');
   const origins = await ask('Allowed CORS origins', current.ALLOWED_ORIGINS || '');
 
@@ -228,6 +248,7 @@ async function cmdSetup() {
     openai ? `OPENAI_API_KEY=${openai}` : '',
     xai ? `XAI_API_KEY=${xai}` : '',
     google ? `GOOGLE_AI_KEY=${google}` : '',
+    brave ? `BRAVE_SEARCH_API_KEY=${brave}` : '',
     ollama ? `OLLAMA_URL=${ollama}` : '',
     origins ? `ALLOWED_ORIGINS=${origins}` : ''
   ].filter(Boolean);
@@ -253,9 +274,11 @@ function installMacService() {
   const nodeBin = process.execPath;
   const content = fs
     .readFileSync(PLIST_SRC, 'utf8')
-    .replace(/\/usr\/local\/bin\/node/g, nodeBin)
-    .replace(/\/Users\/neo\/NeoAgent/g, APP_DIR)
-    .replace(/\/Users\/neo/g, os.homedir());
+    .replace(/__NODE_BIN__/g, nodeBin)
+    .replace(/__APP_DIR__/g, APP_DIR)
+    .replace(/__HOME__/g, os.homedir())
+    .replace(/__RUNTIME_HOME__/g, RUNTIME_HOME)
+    .replace(/__LOG_DIR__/g, LOG_DIR);
 
   fs.writeFileSync(PLIST_DST, content);
 
@@ -290,8 +313,7 @@ function startFallback() {
   });
   child.unref();
 
-  fs.mkdirSync(path.join(APP_DIR, 'data'), { recursive: true });
-  fs.writeFileSync(path.join(APP_DIR, 'data', 'neoagent.pid'), String(child.pid));
+  fs.writeFileSync(PID_FILE, String(child.pid));
   logOk(`Started detached process (pid ${child.pid})`);
 }
 
@@ -352,7 +374,7 @@ function cmdStop() {
     return;
   }
 
-  const pidPath = path.join(APP_DIR, 'data', 'neoagent.pid');
+  const pidPath = PID_FILE;
   let stopped = false;
   if (fs.existsSync(pidPath)) {
     const pid = Number(fs.readFileSync(pidPath, 'utf8').trim());
@@ -435,6 +457,8 @@ function cmdLogs() {
 
 function cmdUpdate() {
   heading(`Update ${APP_NAME}`);
+  migrateLegacyRuntime((msg) => logInfo(msg));
+  ensureRuntimeDirs();
 
   if (fs.existsSync(path.join(APP_DIR, '.git')) && commandExists('git')) {
     const branch = runQuiet('git', ['rev-parse', '--abbrev-ref', 'HEAD']);
@@ -458,6 +482,7 @@ function cmdUpdate() {
     logWarn('No git repo detected; attempting npm global update.');
     if (commandExists('npm')) {
       try {
+        backupRuntimeData();
         runOrThrow('npm', ['install', '-g', 'neoagent@latest']);
         logOk('npm global update completed');
       } catch {
@@ -529,6 +554,8 @@ function printHelp() {
 }
 
 async function runCLI(argv) {
+  migrateLegacyRuntime((msg) => logInfo(msg));
+  ensureRuntimeDirs();
   const command = argv[0] || 'help';
 
   switch (command) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neoagent",
-  "version": "1.4.0",
+  "version": "1.4.3",
   "description": "Proactive personal AI agent with no limits",
   "license": "MIT",
   "main": "server/index.js",
@@ -10,6 +10,7 @@
   "files": [
     "bin",
     "lib",
+    "runtime",
     "server",
     "docs",
     "com.neoagent.plist",
@@ -21,6 +22,8 @@
     "start": "node server/index.js",
     "dev": "node --watch server/index.js",
     "manage": "node bin/neoagent.js",
+    "test": "node --test",
+    "benchmark:tokens": "node scripts/benchmark-token-cost.js",
     "release": "npx semantic-release"
   },
   "repository": {

package/runtime/paths.js

@@ -0,0 +1,80 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const APP_DIR = path.resolve(__dirname, '..');
+const HOME_DIR = os.homedir();
+const RUNTIME_HOME = path.resolve(process.env.NEOAGENT_HOME || path.join(HOME_DIR, '.neoagent'));
+const DATA_DIR = path.resolve(process.env.NEOAGENT_DATA_DIR || path.join(RUNTIME_HOME, 'data'));
+const AGENT_DATA_DIR = path.resolve(process.env.NEOAGENT_AGENT_DATA_DIR || path.join(RUNTIME_HOME, 'agent-data'));
+const LOG_DIR = path.join(DATA_DIR, 'logs');
+const ENV_FILE = path.resolve(process.env.NEOAGENT_ENV_FILE || path.join(RUNTIME_HOME, '.env'));
+const UPDATE_STATUS_FILE = path.join(DATA_DIR, 'update-status.json');
+const PID_FILE = path.join(DATA_DIR, 'neoagent.pid');
+
+const LEGACY_ENV_FILE = path.join(APP_DIR, '.env');
+const LEGACY_DATA_DIR = path.join(APP_DIR, 'data');
+const LEGACY_AGENT_DATA_DIR = path.join(APP_DIR, 'agent-data');
+
+function ensureRuntimeDirs() {
+  for (const dir of [RUNTIME_HOME, DATA_DIR, LOG_DIR, AGENT_DATA_DIR]) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+function copyFileIfMissing(src, dest) {
+  if (!fs.existsSync(src) || fs.existsSync(dest)) return false;
+  fs.mkdirSync(path.dirname(dest), { recursive: true });
+  fs.copyFileSync(src, dest);
+  return true;
+}
+
+function copyDirMerge(src, dest) {
+  if (!fs.existsSync(src)) return false;
+  if (path.resolve(src) === path.resolve(dest)) return false;
+  if (fs.existsSync(dest)) {
+    const existing = fs.readdirSync(dest);
+    if (existing.length > 0) return false;
+  }
+  fs.mkdirSync(dest, { recursive: true });
+  fs.cpSync(src, dest, { recursive: true, force: false, errorOnExist: false });
+  return true;
+}
+
+function migrateLegacyRuntime(logger = () => {}) {
+  ensureRuntimeDirs();
+  let changed = false;
+
+  if (copyFileIfMissing(LEGACY_ENV_FILE, ENV_FILE)) {
+    try { fs.chmodSync(ENV_FILE, 0o600); } catch {}
+    logger(`migrated ${LEGACY_ENV_FILE} -> ${ENV_FILE}`);
+    changed = true;
+  }
+  if (copyDirMerge(LEGACY_DATA_DIR, DATA_DIR)) {
+    logger(`migrated ${LEGACY_DATA_DIR} -> ${DATA_DIR}`);
+    changed = true;
+  }
+  if (copyDirMerge(LEGACY_AGENT_DATA_DIR, AGENT_DATA_DIR)) {
+    logger(`migrated ${LEGACY_AGENT_DATA_DIR} -> ${AGENT_DATA_DIR}`);
+    changed = true;
+  }
+
+  return changed;
+}
+
+module.exports = {
+  APP_DIR,
+  HOME_DIR,
+  RUNTIME_HOME,
+  DATA_DIR,
+  AGENT_DATA_DIR,
+  LOG_DIR,
+  ENV_FILE,
+  UPDATE_STATUS_FILE,
+  PID_FILE,
+  LEGACY_ENV_FILE,
+  LEGACY_DATA_DIR,
+  LEGACY_AGENT_DATA_DIR,
+  ensureRuntimeDirs,
+  migrateLegacyRuntime
+};

package/server/db/database.js

@@ -1,9 +1,7 @@
 const Database = require('better-sqlite3');
 const path = require('path');
-const fs = require('fs');
-
-const DATA_DIR = path.join(__dirname, '../..', 'data');
-if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
+const { DATA_DIR, ensureRuntimeDirs } = require('../../runtime/paths');
+ensureRuntimeDirs();
 
 const DB_PATH = path.join(DATA_DIR, 'neoagent.db');
 const db = new Database(DB_PATH);
@@ -40,6 +38,7 @@ db.exec(`
     trigger_source TEXT,
     model TEXT,
     total_tokens INTEGER DEFAULT 0,
+    prompt_metrics TEXT,
     error TEXT,
     created_at TEXT DEFAULT (datetime('now')),
     updated_at TEXT DEFAULT (datetime('now')),
@@ -144,7 +143,10 @@ db.exec(`
     model TEXT,
     total_tokens INTEGER DEFAULT 0,
     compaction_count INTEGER DEFAULT 0,
+    summary TEXT,
+    summary_message_count INTEGER DEFAULT 0,
     last_compaction TEXT,
+    last_summary TEXT,
     created_at TEXT DEFAULT (datetime('now')),
     updated_at TEXT DEFAULT (datetime('now')),
     FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
@@ -228,12 +230,84 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_core_memory_user ON core_memory(user_id, key);
 `);
 
+try {
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS conversation_history_fts USING fts5(
+      content,
+      role UNINDEXED,
+      user_id UNINDEXED,
+      agent_run_id UNINDEXED,
+      tokenize = 'porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS conversation_history_fts_ai AFTER INSERT ON conversation_history BEGIN
+      INSERT INTO conversation_history_fts(rowid, content, role, user_id, agent_run_id)
+      VALUES (new.id, COALESCE(new.content, ''), new.role, new.user_id, COALESCE(new.agent_run_id, ''));
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS conversation_history_fts_ad AFTER DELETE ON conversation_history BEGIN
+      DELETE FROM conversation_history_fts WHERE rowid = old.id;
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS conversation_history_fts_au AFTER UPDATE ON conversation_history BEGIN
+      DELETE FROM conversation_history_fts WHERE rowid = old.id;
+      INSERT INTO conversation_history_fts(rowid, content, role, user_id, agent_run_id)
+      VALUES (new.id, COALESCE(new.content, ''), new.role, new.user_id, COALESCE(new.agent_run_id, ''));
+    END;
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
+      content,
+      role UNINDEXED,
+      user_id UNINDEXED,
+      run_id UNINDEXED,
+      platform UNINDEXED,
+      platform_chat_id UNINDEXED,
+      tokenize = 'porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS messages_fts_ai AFTER INSERT ON messages BEGIN
+      INSERT INTO messages_fts(rowid, content, role, user_id, run_id, platform, platform_chat_id)
+      VALUES (new.id, COALESCE(new.content, ''), new.role, new.user_id, COALESCE(new.run_id, ''), COALESCE(new.platform, ''), COALESCE(new.platform_chat_id, ''));
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS messages_fts_ad AFTER DELETE ON messages BEGIN
+      DELETE FROM messages_fts WHERE rowid = old.id;
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS messages_fts_au AFTER UPDATE ON messages BEGIN
+      DELETE FROM messages_fts WHERE rowid = old.id;
+      INSERT INTO messages_fts(rowid, content, role, user_id, run_id, platform, platform_chat_id)
+      VALUES (new.id, COALESCE(new.content, ''), new.role, new.user_id, COALESCE(new.run_id, ''), COALESCE(new.platform, ''), COALESCE(new.platform_chat_id, ''));
+    END;
+  `);
+} catch {
+  // FTS5 is optional. The app still works with LIKE-based fallbacks if unavailable.
+}
+
 // Migrations for existing databases
 for (const col of [
   "ALTER TABLE scheduled_tasks ADD COLUMN run_at TEXT",
   "ALTER TABLE scheduled_tasks ADD COLUMN one_time INTEGER DEFAULT 0",
+  "ALTER TABLE agent_runs ADD COLUMN prompt_metrics TEXT",
+  "ALTER TABLE conversations ADD COLUMN summary TEXT",
+  "ALTER TABLE conversations ADD COLUMN summary_message_count INTEGER DEFAULT 0",
+  "ALTER TABLE conversations ADD COLUMN last_summary TEXT",
 ]) {
   try { db.exec(col); } catch { /* column already exists */ }
 }
 
+try {
+  db.exec(`
+    INSERT OR REPLACE INTO conversation_history_fts(rowid, content, role, user_id, agent_run_id)
+    SELECT id, COALESCE(content, ''), role, user_id, COALESCE(agent_run_id, '')
+    FROM conversation_history;
+
+    INSERT OR REPLACE INTO messages_fts(rowid, content, role, user_id, run_id, platform, platform_chat_id)
+    SELECT id, COALESCE(content, ''), role, user_id, COALESCE(run_id, ''), COALESCE(platform, ''), COALESCE(platform_chat_id, '')
+    FROM messages;
+  `);
+} catch {
+  // Older SQLite builds without FTS5 should still let the app boot.
+}
+
 module.exports = db;

package/server/index.js

@@ -1,4 +1,7 @@
-require('dotenv').config();
+const { ENV_FILE, DATA_DIR, APP_DIR, migrateLegacyRuntime, ensureRuntimeDirs } = require('../runtime/paths');
+require('dotenv').config({ path: ENV_FILE });
+migrateLegacyRuntime();
+ensureRuntimeDirs();
 
 const express = require('express');
 const session = require('express-session');
@@ -8,7 +11,6 @@ const { Server: SocketIO } = require('socket.io');
 const helmet = require('helmet');
 const cors = require('cors');
 const path = require('path');
-const fs = require('fs');
 
 const db = require('./db/database');
 const { requireAuth, requireNoAuth } = require('./middleware/auth');
@@ -35,8 +37,6 @@ if (!process.env.SESSION_SECRET) {
 }
 
 const PORT = process.env.PORT || 3333;
-const DATA_DIR = path.join(__dirname, '../data');
-if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
 
 // ── Middleware ──
 
@@ -159,7 +159,7 @@ app.get('/api/version', requireAuth, (req, res) => {
   try {
     const { execSync } = require('child_process');
     gitSha = execSync('git rev-parse --short HEAD', {
-      cwd: path.join(__dirname, '..'),
+      cwd: APP_DIR,
       encoding: 'utf8',
       stdio: ['ignore', 'pipe', 'ignore']
     }).trim();