neoagent 2.3.1-beta.2 → 2.3.1-beta.3

package/server/services/ai/systemPrompt.js

@@ -138,7 +138,7 @@ When drafting on behalf of the user, match their likely voice from available con
 If the user approves a previously shown draft, send that draft rather than silently rewriting it.
 
 TASKS
-Use one-time schedule triggers for single reminders or delayed actions, recurring schedule triggers for repeating automation, and official integration triggers when the task should react to connected Gmail, Outlook, Slack, Teams, or WhatsApp Personal events. When calling task tools, prefer one unified trigger section: trigger={ type, config }. Make task prompts self-contained: who/what to check, exact action to take, when to notify, and which channel to use if known.
+Use manual triggers for run-on-demand tasks, one-time schedule triggers for single reminders or delayed actions, recurring schedule triggers for repeating automation, and official integration triggers when the task should react to connected Gmail, Outlook, Slack, Teams, or WhatsApp Personal events. When calling task tools, prefer one unified trigger section: trigger={ type, config }. Make task prompts self-contained: who/what to check, exact action to take, when to notify, and which channel to use if known.
 Do not create vague tasks like "check this" when the future run would not know what "this" means. Resolve references into names, links, file paths, IDs, dates, and success criteria before saving the task.
 For notification tasks, distinguish between notifying the user in their current messaging channel, emailing the user, and contacting someone else. Default reminders should notify the user through the active messaging channel unless the user explicitly asks for email, phone, or a third party.
 When creating or updating a task, include whether it should notify every time, only on change, only on errors, or only when a condition is met. If unspecified, choose the least noisy useful behavior and say what you chose.

package/server/services/ai/tools.js

@@ -973,8 +973,8 @@ function getAvailableTools(app, options = {}) {
                 type: 'object',
                 properties: {
                     name: { type: 'string', description: 'Short descriptive name for the task.' },
-                    trigger: { type: 'object', description: 'Unified trigger object. Prefer { type: "schedule" | integration_trigger_type, config: {...} }.' },
-                    trigger_type: { type: 'string', description: 'Trigger type such as schedule, gmail_message_received, outlook_email_received, slack_message_received, teams_message_received, weather_event, or whatsapp_personal_message_received.' },
+                    trigger: { type: 'object', description: 'Unified trigger object. Prefer { type: "manual" | "schedule" | integration_trigger_type, config: {...} }.' },
+                    trigger_type: { type: 'string', description: 'Trigger type such as manual, schedule, gmail_message_received, outlook_email_received, slack_message_received, teams_message_received, weather_event, or whatsapp_personal_message_received.' },
                     trigger_config: { type: 'object', description: 'Trigger-specific configuration object. For schedule triggers prefer { mode: "recurring", cronExpression: "m h dom mon dow" } or { mode: "one_time", runAt: ISO datetime }. 5-field cron only (seconds unsupported).' },
                     prompt: { type: 'string', description: 'The instructions the agent will run when the trigger fires.' },
                     enabled: { type: 'boolean', description: 'Whether to activate immediately.' },
@@ -1010,7 +1010,7 @@ function getAvailableTools(app, options = {}) {
                     task_id: { type: 'number', description: 'The numeric ID of the task to update.' },
                     name: { type: 'string', description: 'New name for the task.' },
                     trigger: { type: 'object', description: 'Unified trigger object. Use { type, config } to update trigger in one section.' },
-                    trigger_type: { type: 'string', description: 'Updated trigger type.' },
+                    trigger_type: { type: 'string', description: 'Updated trigger type, e.g. manual, schedule, or integration trigger type.' },
                     trigger_config: { type: 'object', description: 'Updated trigger-specific configuration. For schedule triggers use mode+cronExpression (recurring) or mode+runAt (one_time).' },
                     prompt: { type: 'string', description: 'Updated task prompt.' },
                     enabled: { type: 'boolean', description: 'Enable or disable the task.' },
@@ -1152,6 +1152,17 @@ function getAvailableTools(app, options = {}) {
                 required: ['image_path']
             }
         },
+        {
+            name: 'ocr_extract',
+            description: 'Extract raw text from an image locally using Tesseract OCR. This is faster and completely offline compared to analyze_image.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    image_path: { type: 'string', description: 'Absolute path to the image file' }
+                },
+                required: ['image_path']
+            }
+        },
         {
             name: 'read_health_data',
             description: 'Read the user\'s synced mobile health data. Omit metric_type for a summary of all available metrics. With metric_type, returns an aggregate summary (total, avg, min, max over all stored data) plus the most recent individual records. Always report the summary figures — avoid listing every raw record.',
@@ -2311,13 +2322,17 @@ async function executeTool(toolName, args, context, engine) {
                 if (!resolvedTrigger.hasType || !resolvedTrigger.triggerType) {
                     return { error: 'Task trigger type is required (use trigger.type or trigger_type).' };
                 }
-                if (!resolvedTrigger.hasConfig || resolvedTrigger.triggerConfig === undefined) {
+                const normalizedTriggerType = String(resolvedTrigger.triggerType || '').trim();
+                const triggerConfig = (!resolvedTrigger.hasConfig || resolvedTrigger.triggerConfig === undefined)
+                    ? (normalizedTriggerType === 'manual' ? {} : undefined)
+                    : resolvedTrigger.triggerConfig;
+                if (triggerConfig === undefined) {
                     return { error: 'Task trigger config is required (use trigger.config or trigger_config).' };
                 }
                 const task = await s.createTask(userId, {
                     name: args.name,
-                    triggerType: resolvedTrigger.triggerType,
-                    triggerConfig: resolvedTrigger.triggerConfig,
+                    triggerType: normalizedTriggerType,
+                    triggerConfig,
                     prompt: args.prompt,
                     enabled: args.enabled !== false,
                     model: args.model || null,
@@ -2580,6 +2595,20 @@ async function executeTool(toolName, args, context, engine) {
             }
         }
 
+        case 'ocr_extract': {
+            try {
+                const fs = require('fs');
+                if (!fs.existsSync(args.image_path)) {
+                    return { error: 'File not found: ' + args.image_path };
+                }
+                const Tesseract = require('tesseract.js');
+                const result = await Tesseract.recognize(args.image_path, 'eng');
+                return { text: result.data.text, confidence: result.data.confidence };
+            } catch (err) {
+                return { error: err.message };
+            }
+        }
+
         case 'spawn_subagent': {
             try {
                 const task = args.context ? `${args.task}\n\nContext: ${args.context}` : args.task;

package/server/services/browser/controller.js

@@ -6,11 +6,11 @@ const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
 if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
 
 const USER_AGENTS = [
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
   'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
 ];
 
 const VIEWPORTS = [
@@ -148,6 +148,50 @@ class BrowserController {
             return arr;
           }
         });
+
+        // WebGL Spoofing
+        const getParameterProxyHandler = {
+          apply: function(target, ctx, args) {
+            const param = args[0];
+            // UNMASKED_VENDOR_WEBGL
+            if (param === 37445) return 'Google Inc. (Apple)';
+            // UNMASKED_RENDERER_WEBGL
+            if (param === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)';
+            return Reflect.apply(target, ctx, args);
+          }
+        };
+        const getParam = WebGLRenderingContext.prototype.getParameter;
+        WebGLRenderingContext.prototype.getParameter = new Proxy(getParam, getParameterProxyHandler);
+        if (typeof WebGL2RenderingContext !== 'undefined') {
+          const getParam2 = WebGL2RenderingContext.prototype.getParameter;
+          WebGL2RenderingContext.prototype.getParameter = new Proxy(getParam2, getParameterProxyHandler);
+        }
+
+        // Canvas Spoofing (slight noise)
+        const originalFillText = CanvasRenderingContext2D.prototype.fillText;
+        CanvasRenderingContext2D.prototype.fillText = function(...args) {
+            if (!this._spoofing_applied) {
+                this._spoofing_applied = true;
+                const r = Math.random() * 0.0001;
+                const g = Math.random() * 0.0001;
+                const b = Math.random() * 0.0001;
+                this.fillStyle = \`rgba(\${Math.floor(r * 255)}, \${Math.floor(g * 255)}, \${Math.floor(b * 255)}, 0.01)\`;
+                originalFillText.call(this, "spoof", 0, 0);
+            }
+            return originalFillText.apply(this, args);
+        };
+
+        // Media Devices Spoofing
+        if (navigator.mediaDevices && navigator.mediaDevices.enumerateDevices) {
+            const originalEnumerateDevices = navigator.mediaDevices.enumerateDevices.bind(navigator.mediaDevices);
+            navigator.mediaDevices.enumerateDevices = async () => {
+                return [
+                    { kind: 'audioinput', deviceId: 'default', groupId: 'a', label: 'MacBook Pro Microphone' },
+                    { kind: 'audiooutput', deviceId: 'default', groupId: 'b', label: 'MacBook Pro Speakers' },
+                    { kind: 'videoinput', deviceId: 'default', groupId: 'c', label: 'FaceTime HD Camera' }
+                ];
+            };
+        }
       })();
     `);
   }

package/server/services/desktop/screenRecorder.js

@@ -0,0 +1,126 @@
+'use strict';
+
+const fs = require('fs/promises');
+const path = require('path');
+const os = require('os');
+const { exec } = require('child_process');
+const { promisify } = require('util');
+const tesseract = require('tesseract.js');
+const db = require('../../db/database');
+const { getErrorMessage } = require('../bootstrap_helpers');
+
+const execAsync = promisify(exec);
+
+class ScreenRecorder {
+  constructor() {
+    this.intervalMs = 10000; // 10 seconds
+    this.intervalId = null;
+    this.cleanupIntervalId = null;
+    this.isRecording = false;
+    this.isProcessing = false;
+    this.tempFilePath = path.join(os.tmpdir(), `neoagent-screen-${Date.now()}.png`);
+  }
+
+  start() {
+    if (process.platform !== 'darwin') {
+      console.log('[ScreenRecorder] Not starting: Screen recording is currently macOS only.');
+      return;
+    }
+
+    if (this.isRecording) return;
+    this.isRecording = true;
+
+    console.log('[ScreenRecorder] Starting continuous screen recording (10s interval)');
+    
+    // Start the recording loop
+    this.intervalId = setInterval(() => this.captureAndProcess(), this.intervalMs);
+    
+    // Run an initial capture
+    this.captureAndProcess();
+
+    // Start daily cleanup of old records (7 days)
+    this.cleanupIntervalId = setInterval(() => this.cleanupOldRecords(), 24 * 60 * 60 * 1000);
+    this.cleanupOldRecords();
+  }
+
+  stop() {
+    this.isRecording = false;
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+    if (this.cleanupIntervalId) {
+      clearInterval(this.cleanupIntervalId);
+      this.cleanupIntervalId = null;
+    }
+    console.log('[ScreenRecorder] Stopped continuous screen recording');
+  }
+
+  async captureAndProcess() {
+    if (this.isProcessing || !this.isRecording) return;
+    this.isProcessing = true;
+
+    try {
+      // Capture screen silently (-x) to file
+      await execAsync(`screencapture -x "${this.tempFilePath}"`);
+
+      // Verify file exists
+      await fs.access(this.tempFilePath);
+
+      // Extract text via local OCR
+      const { data } = await tesseract.recognize(this.tempFilePath, 'eng+deu', {
+        logger: () => {} // Silence verbose OCR logs
+      });
+
+      const textContent = data.text.trim();
+
+      // Only store if meaningful text was found
+      if (textContent.length > 5) {
+        // We need a user ID. For the local desktop agent, usually user 1 or we query the active user.
+        const userRow = db.prepare('SELECT id FROM users ORDER BY id ASC LIMIT 1').get();
+        if (userRow) {
+          // Identify the active foreground app via AppleScript
+          let appName = 'Unknown';
+          try {
+            const { stdout } = await execAsync(`osascript -e 'tell application "System Events" to get name of first application process whose frontmost is true'`);
+            appName = stdout.trim();
+          } catch (e) {
+            // Ignore AppleScript errors
+          }
+
+          db.prepare(\`
+            INSERT INTO screen_history (user_id, app_name, text_content)
+            VALUES (?, ?, ?)
+          \`).run(userRow.id, appName, textContent);
+        }
+      }
+
+    } catch (err) {
+      console.error('[ScreenRecorder] Capture/OCR failed:', getErrorMessage(err));
+    } finally {
+      // Always cleanup the screenshot image immediately
+      try {
+        await fs.unlink(this.tempFilePath);
+      } catch (e) {
+        // Ignore unlink errors if file didn't exist
+      }
+      this.isProcessing = false;
+    }
+  }
+
+  cleanupOldRecords() {
+    try {
+      const result = db.prepare(\`
+        DELETE FROM screen_history 
+        WHERE timestamp < datetime('now', '-7 days')
+      \`).run();
+      if (result.changes > 0) {
+        console.log(\`[ScreenRecorder] Purged \${result.changes} old screen history records.\`);
+      }
+    } catch (err) {
+      console.error('[ScreenRecorder] Cleanup failed:', getErrorMessage(err));
+    }
+  }
+}
+
+module.exports = { ScreenRecorder };

package/server/services/integrations/whatsapp/provider.js

@@ -545,7 +545,7 @@ class WhatsAppPersonalProvider extends EventEmitter {
             chatCount: client.chats.size,
             cachedMessageChats: client.messages.size,
             syncNote:
-              'Read tools only expose chats and messages synchronized through this personal integration session.',
+              'This account is linked and connected. Read tools expose only chats and messages currently available in this session cache.',
           },
         };
       case 'whatsapp_personal_list_chats': {
@@ -557,10 +557,14 @@ class WhatsAppPersonalProvider extends EventEmitter {
           .slice(0, limit);
         return {
           result: {
+            account: client.accountEmail || connection.account_email || null,
+            connected: client.status === 'connected',
             chats,
             count: chats.length,
             syncNote:
-              'Only chats synchronized after linking this personal integration are available here.',
+              chats.length > 0
+                ? 'Chats shown here come from the current personal WhatsApp cache.'
+                : 'The personal WhatsApp account is connected, but no chats are cached yet. The cache populates as history/events are synchronized.',
           },
         };
       }

package/server/services/manager.js

@@ -24,6 +24,7 @@ const { RuntimeManager } = require('./runtime/manager');
 const { BrowserExtensionRegistry } = require('./browser/extension/registry');
 const { DesktopCompanionRegistry } = require('./desktop/registry');
 const { DesktopProvider } = require('./desktop/provider');
+const { ScreenRecorder } = require('./desktop/screenRecorder');
 const { assertRuntimeValidation, getRuntimeValidation } = require('./runtime/validation');
 const {
   getErrorMessage,
@@ -428,6 +429,17 @@ function createWidgetService(app) {
   return widgetService;
 }
 
+function createScreenRecorder(app) {
+  const screenRecorder = registerLocal(
+    app,
+    'screenRecorder',
+    new ScreenRecorder(),
+  );
+  screenRecorder.start();
+  logServiceReady('Screen recorder started');
+  return screenRecorder;
+}
+
 function restoreMessagingConnections(messagingManager) {
   void runBackgroundTask('[Messaging] Restore error:', () =>
     messagingManager.restoreConnections(),
@@ -513,6 +525,7 @@ async function startServices(app, io) {
     const messagingManager = createMessagingManager(app, io, agentEngine);
     const recordingManager = createRecordingManager(app, io);
     createWidgetService(app);
+    createScreenRecorder(app);
 
     restoreMessagingConnections(messagingManager);
     restoreMcpClients(mcpClient);
@@ -639,6 +652,15 @@ async function stopServices(app) {
     }
   }
 
+  if (app.locals.screenRecorder) {
+    try {
+      app.locals.screenRecorder.stop();
+      logServiceReady('Screen recorder stopped');
+    } catch (err) {
+      console.error('[ScreenRecorder] Stop error:', getErrorMessage(err));
+    }
+  }
+
   if (app.locals.browserExtensionRegistry) {
     try {
       app.locals.browserExtensionRegistry.closeAll();

package/server/services/skills/base_catalog.js

@@ -1161,6 +1161,39 @@ Auto-orients and arranges the STL, applies settings from JSON files, slices all
 3. Build the command from the flags above
 4. Run it and check for errors in the output (debug level 2 is a good default)
 5. Report the output file location and any warnings`
+  },
+
+  {
+    id: 'package-tracker',
+    name: 'Package Tracker',
+    description: 'Track packages by automatically detecting the carrier and fetching the tracking link.',
+    category: 'info',
+    icon: '📦',
+    content: `---
+name: package-tracker
+description: Track packages by automatically detecting the carrier and fetching the tracking link.
+category: info
+icon: 📦
+enabled: true
+---
+
+You are a package tracking assistant. When the user gives you a tracking number, you should:
+
+1. Identify the carrier using common regex patterns:
+   - UPS: \`\\b(1Z[0-9A-Z]{16})\\b\`
+   - FedEx: \`\\b([0-9]{12,15})\\b\` (usually 12 or 15 digits)
+   - USPS: \`\\b(94[0-9]{20})\\b\` or \`\\b([A-Z]{2}[0-9]{9}US)\\b\`
+   - DHL: \`\\b([0-9]{10})\\b\` or \`\\b([0-9]{20})\\b\`
+
+2. Generate the direct tracking link for the user:
+   - UPS: \`https://www.ups.com/track?tracknum={number}\`
+   - FedEx: \`https://www.fedex.com/fedextrack/?trknbr={number}\`
+   - USPS: \`https://tools.usps.com/go/TrackConfirmAction?tLabels={number}\`
+   - DHL: \`https://www.dhl.com/global-en/home/tracking/tracking-express.html?submit=1&tracking-id={number}\`
+
+3. If the carrier is not obvious, use \`web_search\` with the query "track package {number}" to identify the carrier.
+4. Present the carrier and the direct tracking link to the user clearly.
+5. Optionally use \`browser_navigate\` or \`http_request\` to fetch the current status from the tracking link, but note that many carriers block automated scraping. The direct link is the most important output.`
   }
 ];
 

package/server/services/tasks/adapters/index.js

@@ -2,6 +2,7 @@
 
 module.exports = [
   require('./schedule'),
+  require('./manual'),
   require('./gmail_message_received'),
   require('./outlook_email_received'),
   require('./slack_message_received'),

package/server/services/tasks/adapters/manual.js

@@ -0,0 +1,12 @@
+'use strict';
+
+module.exports = {
+  type: 'manual',
+  label: 'Manual Trigger',
+  async validateConfig() {
+    return {};
+  },
+  summarize() {
+    return 'Manual run only';
+  },
+};

package/server/services/tasks/runtime.js

@@ -66,7 +66,7 @@ class TaskRuntime {
       label: adapter.label,
       providerKey: adapter.providerKey || null,
       appKey: adapter.appKey || null,
-      available: adapter.type === 'schedule'
+      available: adapter.type === 'schedule' || adapter.type === 'manual'
         ? true
         : this._hasConnectedApp(userId, agentId, adapter.providerKey, adapter.appKey),
     }));

package/server/services/widgets/service.js

@@ -243,7 +243,11 @@ class WidgetService {
        WHERE id = ? AND user_id = ?`
     ).get(widgetId, userId);
     if (!row) return null;
-    return this._serializeWidget(row, this._loadLatestSnapshotMap([widgetId]).get(widgetId) || null);
+    return this._serializeWidget(
+      row,
+      this._loadLatestSnapshotMap([widgetId]).get(widgetId) || null,
+      this._loadWidgetTasksMap([widgetId], userId).get(widgetId) || []
+    );
   }
 
   listWidgets(userId, { agentId = null } = {}) {
@@ -262,7 +266,8 @@ class WidgetService {
          ORDER BY updated_at DESC, created_at DESC`
       ).all(userId);
     const snapshotMap = this._loadLatestSnapshotMap(rows.map((row) => row.id));
-    return rows.map((row) => this._serializeWidget(row, snapshotMap.get(row.id) || null));
+    const tasksMap = this._loadWidgetTasksMap(rows.map((row) => row.id), userId);
+    return rows.map((row) => this._serializeWidget(row, snapshotMap.get(row.id) || null, tasksMap.get(row.id) || []));
   }
 
   listLatestSnapshots(userId, { agentId = null } = {}) {
@@ -343,7 +348,7 @@ class WidgetService {
       throw new Error('Widget not found.');
     }
 
-    const current = this._serializeWidget(existingRow, null);
+    const current = this._serializeWidget(existingRow, null, []);
     const normalized = normalizeWidgetInput({
       name: input.name ?? current.name,
       template: input.template ?? current.template,
@@ -581,7 +586,46 @@ class WidgetService {
     return map;
   }
 
-  _serializeWidget(row, latestSnapshot) {
+  _loadWidgetTasksMap(widgetIds, userId) {
+    const ids = Array.from(new Set(widgetIds.filter(Boolean)));
+    const map = new Map();
+    if (!ids.length) return map;
+    
+    const placeholders = ids.map(() => '?').join(', ');
+    const params = [userId, ...ids];
+    
+    // We filter tasks where task_type is NOT 'widget_refresh' 
+    // and where the task_config contains the widgetId.
+    const rows = db.prepare(
+      `SELECT id, name, trigger_type, enabled, task_config
+       FROM scheduled_tasks
+       WHERE user_id = ?
+         AND task_type != 'widget_refresh'
+         AND json_extract(task_config, '$.widgetId') IN (${placeholders})
+       ORDER BY created_at ASC`
+    ).all(...params);
+    
+    for (const row of rows) {
+      const config = parseJsonObject(row.task_config, {});
+      const widgetId = config.widgetId;
+      if (!widgetId) continue;
+      
+      const task = {
+        id: row.id,
+        name: row.name,
+        triggerType: row.trigger_type,
+        enabled: row.enabled !== 0 && row.enabled !== false,
+      };
+      
+      if (!map.has(widgetId)) {
+        map.set(widgetId, []);
+      }
+      map.get(widgetId).push(task);
+    }
+    return map;
+  }
+
+  _serializeWidget(row, latestSnapshot, tasks = []) {
     const definition = parseJsonObject(row.definition_json, {});
     return {
       id: row.id,
@@ -600,6 +644,7 @@ class WidgetService {
       updatedAt: row.updated_at || null,
       nextRefresh: row.refresh_cron ? findNextRun(row.refresh_cron)?.toISOString() || null : null,
       latestSnapshot,
+      tasks,
     };
   }
 }