npm - neoagent - Versions diffs - 2.3.1-beta.0 → 2.3.1-beta.3 - Mend

neoagent 2.3.1-beta.0 → 2.3.1-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (38) hide show

package/README.md +2 -0
package/docs/automation.md +1 -1
package/docs/capabilities.md +2 -2
package/docs/configuration.md +6 -1
package/docs/integrations.md +4 -0
package/package.json +2 -1
package/server/db/database.js +57 -0
package/server/http/routes/screenHistory.js +46 -0
package/server/http/routes/triggers.js +81 -0
package/server/http/routes.js +3 -1
package/server/public/assets/NOTICES +61 -0
package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
package/server/public/flutter_bootstrap.js +1 -1
package/server/public/main.dart.js +46519 -46294
package/server/routes/integrations.js +11 -8
package/server/services/ai/engine.js +9 -0
package/server/services/ai/systemPrompt.js +1 -1
package/server/services/ai/tools.js +150 -11
package/server/services/browser/controller.js +47 -3
package/server/services/desktop/screenRecorder.js +126 -0
package/server/services/integrations/env.js +5 -0
package/server/services/integrations/registry.js +4 -0
package/server/services/integrations/spotify/provider.js +487 -0
package/server/services/integrations/weather/provider.js +559 -0
package/server/services/integrations/whatsapp/provider.js +6 -2
package/server/services/manager.js +22 -0
package/server/services/messaging/manager.js +29 -7
package/server/services/skills/base_catalog.js +33 -0
package/server/services/tasks/adapters/index.js +2 -0
package/server/services/tasks/adapters/manual.js +12 -0
package/server/services/tasks/adapters/schedule.js +33 -5
package/server/services/tasks/adapters/weather_event.js +84 -0
package/server/services/tasks/integration_runtime.js +85 -0
package/server/services/tasks/runtime.js +2 -2
package/server/services/voice/agentBridge.js +20 -4
package/server/services/voice/message.js +3 -0
package/server/services/voice/runtimeManager.js +136 -1
package/server/services/widgets/service.js +49 -4

package/server/routes/integrations.js CHANGED Viewed

@@ -159,6 +159,9 @@ router.get('/:provider/connect/:sessionId', (req, res) => {
     if (!session) {
       return res.status(404).send('Connection session not found.');
     }
+    const provider = manager.getProvider(req.params.provider);
+    const providerLabel = provider?.label || req.params.provider;
+    const appLabel = provider?.getApp?.(session.appKey)?.label || session.appKey || 'account';
     const trustedOrigin = JSON.stringify(getTrustedPostMessageOrigin(req));
     const statusUrl = `/api/integrations/${encodeURIComponent(req.params.provider)}/connect/${encodeURIComponent(req.params.sessionId)}/status?agentId=${encodeURIComponent(agentId)}`;
     res.send(`
@@ -166,7 +169,7 @@ router.get('/:provider/connect/:sessionId', (req, res) => {
         <head>
           <meta charset="utf-8" />
           <meta name="viewport" content="width=device-width, initial-scale=1" />
-          <title>Connect ${escapeHtml(req.params.provider)}</title>
+          <title>Connect ${escapeHtml(providerLabel)}</title>
           <style>
             body { font-family: ui-sans-serif, system-ui, sans-serif; background: #0b1220; color: #f8fafc; margin: 0; padding: 24px; }
             .card { max-width: 560px; margin: 0 auto; background: #111827; border: 1px solid #1f2937; border-radius: 20px; padding: 24px; }
@@ -179,11 +182,11 @@ router.get('/:provider/connect/:sessionId', (req, res) => {
         <body>
           <div class="card">
             <div class="pill">Official integration</div>
-            <h1>Connect WhatsApp</h1>
-            <p class="muted">This link is isolated from the separate messaging-platform WhatsApp bridge. Scan the QR code with your personal WhatsApp account to finish linking.</p>
+            <h1>Connect ${escapeHtml(providerLabel)}</h1>
+            <p class="muted">Complete connection for ${escapeHtml(appLabel)}. This window closes automatically when linking is finished.</p>
             <div id="status" class="muted">Starting connection…</div>
-            <img id="qr" alt="WhatsApp QR code" style="display:none;" />
-            <p class="muted">When the link finishes, this window will close automatically. If it does not, you can close it manually.</p>
+            <img id="qr" alt="Integration QR code" style="display:none;" />
+            <p class="muted">If this flow needs QR scan approval, the code will appear below.</p>
           </div>
           <script>
             const statusEl = document.getElementById('status');
@@ -207,12 +210,12 @@ router.get('/:provider/connect/:sessionId', (req, res) => {
               const data = await response.json();
               const status = String(data.status || 'connecting');
               if (status === 'awaiting_qr' && data.qr) {
-                statusEl.textContent = 'Scan this QR code with WhatsApp on your phone.';
+                statusEl.textContent = 'Scan this QR code to continue linking.';
                 qrEl.src = 'https://api.qrserver.com/v1/create-qr-code/?data=' + encodeURIComponent(data.qr) + '&size=320x320';
                 qrEl.style.display = 'block';
               } else if (status === 'connected') {
                 qrEl.style.display = 'none';
-                statusEl.textContent = 'Connected as ' + (data.accountEmail || 'your WhatsApp account') + '. Closing…';
+                statusEl.textContent = 'Connected as ' + (data.accountEmail || 'your account') + '. Closing…';
                 notifyOpener({
                   type: 'integration_oauth_success',
                   provider,
@@ -233,7 +236,7 @@ router.get('/:provider/connect/:sessionId', (req, res) => {
                 });
                 return;
               } else {
-                statusEl.textContent = 'Waiting for WhatsApp to generate a QR code…';
+                statusEl.textContent = 'Waiting for the integration to finish linking…';
               }
               setTimeout(refresh, 1500);
             }

package/server/services/ai/engine.js CHANGED Viewed

@@ -686,6 +686,7 @@ class AgentEngine {
     content,
     kind,
     expectsReply = false,
+    deferFollowUp = false,
   } = {}) {
     const runMeta = this.getRunMeta(runId);
     if (!runMeta || runMeta.aborted) {
@@ -715,6 +716,9 @@ class AgentEngine {
       kind: normalizedKind,
       expectsReply,
     });
+    if (deferFollowUp === true) {
+      metadata.defer_follow_up = true;
+    }
     const createdAt = new Date().toISOString();
     if (triggerSource === 'messaging') {
@@ -739,6 +743,7 @@ class AgentEngine {
         content: normalizedContent,
         kind: normalizedKind,
         expectsReply,
+        deferFollowUp,
       });
     } else {
       db.prepare(
@@ -758,6 +763,7 @@ class AgentEngine {
       content: normalizedContent,
       kind: normalizedKind,
       expectsReply: expectsReply === true,
+      deferFollowUp: deferFollowUp === true,
       createdAt,
     });
     runMeta.lastInterimMessage = normalizedContent;
@@ -767,6 +773,7 @@ class AgentEngine {
       content: normalizedContent,
       kind: normalizedKind,
       expectsReply: expectsReply === true,
+      deferFollowUp: deferFollowUp === true,
       triggerSource,
       platform: triggerSource === 'messaging' ? platform : 'web',
     });
@@ -783,6 +790,7 @@ class AgentEngine {
       latestInterim: {
         kind: normalizedKind,
         expectsReply: expectsReply === true,
+        deferFollowUp: deferFollowUp === true,
         content: normalizedContent,
         createdAt,
       },
@@ -795,6 +803,7 @@ class AgentEngine {
       sent: true,
       kind: normalizedKind,
       expectsReply: expectsReply === true,
+      deferFollowUp: deferFollowUp === true,
       content: normalizedContent,
       terminal: terminalInterim,
     };

package/server/services/ai/systemPrompt.js CHANGED Viewed

@@ -138,7 +138,7 @@ When drafting on behalf of the user, match their likely voice from available con
 If the user approves a previously shown draft, send that draft rather than silently rewriting it.
 TASKS
-Use one-time schedule triggers for single reminders or delayed actions, recurring schedule triggers for repeating automation, and official integration triggers when the task should react to connected Gmail, Outlook, Slack, Teams, or WhatsApp Personal events. Make task prompts self-contained: who/what to check, exact action to take, when to notify, and which channel to use if known.
+Use manual triggers for run-on-demand tasks, one-time schedule triggers for single reminders or delayed actions, recurring schedule triggers for repeating automation, and official integration triggers when the task should react to connected Gmail, Outlook, Slack, Teams, or WhatsApp Personal events. When calling task tools, prefer one unified trigger section: trigger={ type, config }. Make task prompts self-contained: who/what to check, exact action to take, when to notify, and which channel to use if known.
 Do not create vague tasks like "check this" when the future run would not know what "this" means. Resolve references into names, links, file paths, IDs, dates, and success criteria before saving the task.
 For notification tasks, distinguish between notifying the user in their current messaging channel, emailing the user, and contacting someone else. Default reminders should notify the user through the active messaging channel unless the user explicitly asks for email, phone, or a third party.
 When creating or updating a task, include whether it should notify every time, only on change, only on errors, or only when a condition is met. If unspecified, choose the least noisy useful behavior and say what you chose.

package/server/services/ai/tools.js CHANGED Viewed

@@ -91,6 +91,103 @@ function compactToolDefinition(tool, options = {}) {
     return compact;
 }
+function normalizeScheduleTriggerConfig(inputConfig = {}) {
+    if (!inputConfig || typeof inputConfig !== 'object' || Array.isArray(inputConfig)) {
+        return inputConfig;
+    }
+    const normalized = { ...inputConfig };
+    const schedule = (normalized.schedule && typeof normalized.schedule === 'object' && !Array.isArray(normalized.schedule))
+        ? normalized.schedule
+        : null;
+    const modeCandidate = String(
+        normalized.mode
+        || normalized.type
+        || normalized.scheduleType
+        || normalized.schedule_type
+        || (normalized.oneTime || normalized.one_time ? 'one_time' : '')
+        || ''
+    ).trim().toLowerCase();
+    if (modeCandidate === 'once') normalized.mode = 'one_time';
+    else if (modeCandidate === 'one_time' || modeCandidate === 'recurring') normalized.mode = modeCandidate;
+    const cronCandidate = normalized.cronExpression
+        || normalized.cron_expression
+        || normalized.cron
+        || schedule?.cronExpression
+        || schedule?.cron_expression
+        || schedule?.cron;
+    if (cronCandidate != null) {
+        normalized.cronExpression = String(cronCandidate).trim();
+    }
+    const runAtCandidate = normalized.runAt
+        || normalized.run_at
+        || normalized.at
+        || normalized.when
+        || schedule?.runAt
+        || schedule?.run_at
+        || schedule?.at
+        || schedule?.when;
+    if (runAtCandidate != null) {
+        normalized.runAt = String(runAtCandidate).trim();
+    }
+    if (!normalized.mode) {
+        normalized.mode = normalized.runAt ? 'one_time' : 'recurring';
+    }
+    delete normalized.schedule;
+    return normalized;
+}
+function normalizeTaskTriggerInput(triggerType, triggerConfig) {
+    if (String(triggerType || '').trim() !== 'schedule') {
+        return triggerConfig;
+    }
+    return normalizeScheduleTriggerConfig(triggerConfig || {});
+}
+function resolveTaskTriggerArgs(args = {}, fallbackTriggerType = null) {
+    let triggerType = args.trigger_type !== undefined
+        ? String(args.trigger_type || '').trim()
+        : String(fallbackTriggerType || '').trim();
+    let triggerConfig = args.trigger_config;
+    let hasType = args.trigger_type !== undefined;
+    let hasConfig = args.trigger_config !== undefined;
+    if (args.trigger && typeof args.trigger === 'object' && !Array.isArray(args.trigger)) {
+        const trigger = args.trigger;
+        const unifiedType = trigger.type ?? trigger.triggerType ?? trigger.trigger_type;
+        const unifiedHasType = unifiedType !== undefined && unifiedType !== null && String(unifiedType).trim().length > 0;
+        if (unifiedHasType) {
+            triggerType = String(unifiedType).trim();
+            hasType = true;
+        }
+        const hasUnifiedConfig = Object.prototype.hasOwnProperty.call(trigger, 'config')
+            || Object.prototype.hasOwnProperty.call(trigger, 'triggerConfig')
+            || Object.prototype.hasOwnProperty.call(trigger, 'trigger_config');
+        if (hasUnifiedConfig) {
+            triggerConfig = trigger.config ?? trigger.triggerConfig ?? trigger.trigger_config;
+            hasConfig = true;
+        }
+    }
+    const normalizedType = String(triggerType || '').trim() || null;
+    const normalizedConfig = hasConfig
+        ? normalizeTaskTriggerInput(normalizedType || fallbackTriggerType || 'schedule', triggerConfig)
+        : undefined;
+    return {
+        triggerType: normalizedType,
+        triggerConfig: normalizedConfig,
+        hasType,
+        hasConfig,
+    };
+}
 function isProactiveTrigger(triggerSource) {
     return triggerSource === 'schedule' || triggerSource === 'tasks';
 }
@@ -876,15 +973,16 @@ function getAvailableTools(app, options = {}) {
                 type: 'object',
                 properties: {
                     name: { type: 'string', description: 'Short descriptive name for the task.' },
-                    trigger_type: { type: 'string', description: 'Trigger type such as schedule, gmail_message_received, outlook_email_received, slack_message_received, teams_message_received, or whatsapp_personal_message_received.' },
-                    trigger_config: { type: 'object', description: 'Trigger-specific configuration object.' },
+                    trigger: { type: 'object', description: 'Unified trigger object. Prefer { type: "manual" | "schedule" | integration_trigger_type, config: {...} }.' },
+                    trigger_type: { type: 'string', description: 'Trigger type such as manual, schedule, gmail_message_received, outlook_email_received, slack_message_received, teams_message_received, weather_event, or whatsapp_personal_message_received.' },
+                    trigger_config: { type: 'object', description: 'Trigger-specific configuration object. For schedule triggers prefer { mode: "recurring", cronExpression: "m h dom mon dow" } or { mode: "one_time", runAt: ISO datetime }. 5-field cron only (seconds unsupported).' },
                     prompt: { type: 'string', description: 'The instructions the agent will run when the trigger fires.' },
                     enabled: { type: 'boolean', description: 'Whether to activate immediately.' },
                     model: { type: 'string', description: 'Optional model override.' },
                     call_to: { type: 'string', description: 'Optional E.164 phone number to call via Telnyx when this task fires.' },
                     call_greeting: { type: 'string', description: 'Optional spoken greeting hint for make_call.' }
                 },
-                required: ['name', 'trigger_type', 'trigger_config', 'prompt']
+                required: ['name', 'prompt']
             }
         },
         {
@@ -911,8 +1009,9 @@ function getAvailableTools(app, options = {}) {
                 properties: {
                     task_id: { type: 'number', description: 'The numeric ID of the task to update.' },
                     name: { type: 'string', description: 'New name for the task.' },
-                    trigger_type: { type: 'string', description: 'Updated trigger type.' },
-                    trigger_config: { type: 'object', description: 'Updated trigger-specific configuration.' },
+                    trigger: { type: 'object', description: 'Unified trigger object. Use { type, config } to update trigger in one section.' },
+                    trigger_type: { type: 'string', description: 'Updated trigger type, e.g. manual, schedule, or integration trigger type.' },
+                    trigger_config: { type: 'object', description: 'Updated trigger-specific configuration. For schedule triggers use mode+cronExpression (recurring) or mode+runAt (one_time).' },
                     prompt: { type: 'string', description: 'Updated task prompt.' },
                     enabled: { type: 'boolean', description: 'Enable or disable the task.' },
                     model: { type: 'string', description: 'Specific AI model ID for this task. Set to empty string to clear the override.' },
@@ -1053,6 +1152,17 @@ function getAvailableTools(app, options = {}) {
                 required: ['image_path']
             }
         },
+        {
+            name: 'ocr_extract',
+            description: 'Extract raw text from an image locally using Tesseract OCR. This is faster and completely offline compared to analyze_image.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    image_path: { type: 'string', description: 'Absolute path to the image file' }
+                },
+                required: ['image_path']
+            }
+        },
         {
             name: 'read_health_data',
             description: 'Read the user\'s synced mobile health data. Omit metric_type for a summary of all available metrics. With metric_type, returns an aggregate summary (total, avg, min, max over all stored data) plus the most recent individual records. Always report the summary figures — avoid listing every raw record.',
@@ -1123,7 +1233,8 @@ function getAvailableTools(app, options = {}) {
                     properties: {
                         content: { type: 'string', description: 'Natural assistant message derived from the current task state.' },
                         kind: { type: 'string', enum: Array.from(INTERIM_KINDS), description: 'ack, progress, question, or blocker' },
-                        expects_reply: { type: 'boolean', description: 'Set true only when the current run should pause for the user to answer.' }
+                        expects_reply: { type: 'boolean', description: 'Set true only when the current run should pause for the user to answer.' },
+                        defer_follow_up: { type: 'boolean', description: 'Set true when you choose to deliver the final result later via the user\'s last connected chat target.' }
                     },
                     required: ['content', 'kind']
                 }
@@ -1847,6 +1958,7 @@ async function executeTool(toolName, args, context, engine) {
             }
             const interimContent = typeof args.content === 'string' ? args.content : '';
             const expectsReply = args.expects_reply === true;
+            const deferFollowUp = args.defer_follow_up === true;
             return engine.publishInterimUpdate({
                 userId,
                 runId,
@@ -1858,6 +1970,7 @@ async function executeTool(toolName, args, context, engine) {
                 content: interimContent,
                 kind: normalizeInterimKind(args.kind),
                 expectsReply,
+                deferFollowUp,
             });
         }
@@ -2205,10 +2318,21 @@ async function executeTool(toolName, args, context, engine) {
             const s = taskRuntime();
             if (!s) return { error: 'Task runtime not available' };
             try {
+                const resolvedTrigger = resolveTaskTriggerArgs(args, 'schedule');
+                if (!resolvedTrigger.hasType || !resolvedTrigger.triggerType) {
+                    return { error: 'Task trigger type is required (use trigger.type or trigger_type).' };
+                }
+                const normalizedTriggerType = String(resolvedTrigger.triggerType || '').trim();
+                const triggerConfig = (!resolvedTrigger.hasConfig || resolvedTrigger.triggerConfig === undefined)
+                    ? (normalizedTriggerType === 'manual' ? {} : undefined)
+                    : resolvedTrigger.triggerConfig;
+                if (triggerConfig === undefined) {
+                    return { error: 'Task trigger config is required (use trigger.config or trigger_config).' };
+                }
                 const task = await s.createTask(userId, {
                     name: args.name,
-                    triggerType: args.trigger_type,
-                    triggerConfig: args.trigger_config,
+                    triggerType: normalizedTriggerType,
+                    triggerConfig,
                     prompt: args.prompt,
                     enabled: args.enabled !== false,
                     model: args.model || null,
@@ -2246,12 +2370,13 @@ async function executeTool(toolName, args, context, engine) {
             const s = taskRuntime();
             if (!s) return { error: 'Task runtime not available' };
             try {
-                const existing = db.prepare('SELECT agent_id FROM scheduled_tasks WHERE id = ? AND user_id = ?').get(args.task_id, userId);
+                const existing = db.prepare('SELECT agent_id, trigger_type FROM scheduled_tasks WHERE id = ? AND user_id = ?').get(args.task_id, userId);
                 if (!existing || existing.agent_id !== agentId) return { error: 'Task not found for this agent.' };
                 const updates = {};
+                const resolvedTrigger = resolveTaskTriggerArgs(args, existing.trigger_type || 'schedule');
                 if (args.name !== undefined) updates.name = args.name;
-                if (args.trigger_type !== undefined) updates.triggerType = args.trigger_type;
-                if (args.trigger_config !== undefined) updates.triggerConfig = args.trigger_config;
+                if (resolvedTrigger.hasType && resolvedTrigger.triggerType) updates.triggerType = resolvedTrigger.triggerType;
+                if (resolvedTrigger.hasConfig && resolvedTrigger.triggerConfig !== undefined) updates.triggerConfig = resolvedTrigger.triggerConfig;
                 if (args.prompt !== undefined) updates.prompt = args.prompt;
                 if (args.enabled !== undefined) updates.enabled = args.enabled;
                 if (args.model !== undefined) updates.model = args.model || null;
@@ -2470,6 +2595,20 @@ async function executeTool(toolName, args, context, engine) {
             }
         }
+        case 'ocr_extract': {
+            try {
+                const fs = require('fs');
+                if (!fs.existsSync(args.image_path)) {
+                    return { error: 'File not found: ' + args.image_path };
+                }
+                const Tesseract = require('tesseract.js');
+                const result = await Tesseract.recognize(args.image_path, 'eng');
+                return { text: result.data.text, confidence: result.data.confidence };
+            } catch (err) {
+                return { error: err.message };
+            }
+        }
         case 'spawn_subagent': {
             try {
                 const task = args.context ? `${args.task}\n\nContext: ${args.context}` : args.task;

package/server/services/browser/controller.js CHANGED Viewed

@@ -6,11 +6,11 @@ const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
 if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
 const USER_AGENTS = [
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
   'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
 ];
 const VIEWPORTS = [
@@ -148,6 +148,50 @@ class BrowserController {
             return arr;
           }
         });
+        // WebGL Spoofing
+        const getParameterProxyHandler = {
+          apply: function(target, ctx, args) {
+            const param = args[0];
+            // UNMASKED_VENDOR_WEBGL
+            if (param === 37445) return 'Google Inc. (Apple)';
+            // UNMASKED_RENDERER_WEBGL
+            if (param === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)';
+            return Reflect.apply(target, ctx, args);
+          }
+        };
+        const getParam = WebGLRenderingContext.prototype.getParameter;
+        WebGLRenderingContext.prototype.getParameter = new Proxy(getParam, getParameterProxyHandler);
+        if (typeof WebGL2RenderingContext !== 'undefined') {
+          const getParam2 = WebGL2RenderingContext.prototype.getParameter;
+          WebGL2RenderingContext.prototype.getParameter = new Proxy(getParam2, getParameterProxyHandler);
+        }
+        // Canvas Spoofing (slight noise)
+        const originalFillText = CanvasRenderingContext2D.prototype.fillText;
+        CanvasRenderingContext2D.prototype.fillText = function(...args) {
+            if (!this._spoofing_applied) {
+                this._spoofing_applied = true;
+                const r = Math.random() * 0.0001;
+                const g = Math.random() * 0.0001;
+                const b = Math.random() * 0.0001;
+                this.fillStyle = \`rgba(\${Math.floor(r * 255)}, \${Math.floor(g * 255)}, \${Math.floor(b * 255)}, 0.01)\`;
+                originalFillText.call(this, "spoof", 0, 0);
+            }
+            return originalFillText.apply(this, args);
+        };
+        // Media Devices Spoofing
+        if (navigator.mediaDevices && navigator.mediaDevices.enumerateDevices) {
+            const originalEnumerateDevices = navigator.mediaDevices.enumerateDevices.bind(navigator.mediaDevices);
+            navigator.mediaDevices.enumerateDevices = async () => {
+                return [
+                    { kind: 'audioinput', deviceId: 'default', groupId: 'a', label: 'MacBook Pro Microphone' },
+                    { kind: 'audiooutput', deviceId: 'default', groupId: 'b', label: 'MacBook Pro Speakers' },
+                    { kind: 'videoinput', deviceId: 'default', groupId: 'c', label: 'FaceTime HD Camera' }
+                ];
+            };
+        }
       })();
     `);
   }

package/server/services/desktop/screenRecorder.js ADDED Viewed

@@ -0,0 +1,126 @@
+'use strict';
+const fs = require('fs/promises');
+const path = require('path');
+const os = require('os');
+const { exec } = require('child_process');
+const { promisify } = require('util');
+const tesseract = require('tesseract.js');
+const db = require('../../db/database');
+const { getErrorMessage } = require('../bootstrap_helpers');
+const execAsync = promisify(exec);
+class ScreenRecorder {
+  constructor() {
+    this.intervalMs = 10000; // 10 seconds
+    this.intervalId = null;
+    this.cleanupIntervalId = null;
+    this.isRecording = false;
+    this.isProcessing = false;
+    this.tempFilePath = path.join(os.tmpdir(), `neoagent-screen-${Date.now()}.png`);
+  }
+  start() {
+    if (process.platform !== 'darwin') {
+      console.log('[ScreenRecorder] Not starting: Screen recording is currently macOS only.');
+      return;
+    }
+    if (this.isRecording) return;
+    this.isRecording = true;
+    console.log('[ScreenRecorder] Starting continuous screen recording (10s interval)');
+    // Start the recording loop
+    this.intervalId = setInterval(() => this.captureAndProcess(), this.intervalMs);
+    // Run an initial capture
+    this.captureAndProcess();
+    // Start daily cleanup of old records (7 days)
+    this.cleanupIntervalId = setInterval(() => this.cleanupOldRecords(), 24 * 60 * 60 * 1000);
+    this.cleanupOldRecords();
+  }
+  stop() {
+    this.isRecording = false;
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+    if (this.cleanupIntervalId) {
+      clearInterval(this.cleanupIntervalId);
+      this.cleanupIntervalId = null;
+    }
+    console.log('[ScreenRecorder] Stopped continuous screen recording');
+  }
+  async captureAndProcess() {
+    if (this.isProcessing || !this.isRecording) return;
+    this.isProcessing = true;
+    try {
+      // Capture screen silently (-x) to file
+      await execAsync(`screencapture -x "${this.tempFilePath}"`);
+      // Verify file exists
+      await fs.access(this.tempFilePath);
+      // Extract text via local OCR
+      const { data } = await tesseract.recognize(this.tempFilePath, 'eng+deu', {
+        logger: () => {} // Silence verbose OCR logs
+      });
+      const textContent = data.text.trim();
+      // Only store if meaningful text was found
+      if (textContent.length > 5) {
+        // We need a user ID. For the local desktop agent, usually user 1 or we query the active user.
+        const userRow = db.prepare('SELECT id FROM users ORDER BY id ASC LIMIT 1').get();
+        if (userRow) {
+          // Identify the active foreground app via AppleScript
+          let appName = 'Unknown';
+          try {
+            const { stdout } = await execAsync(`osascript -e 'tell application "System Events" to get name of first application process whose frontmost is true'`);
+            appName = stdout.trim();
+          } catch (e) {
+            // Ignore AppleScript errors
+          }
+          db.prepare(\`
+            INSERT INTO screen_history (user_id, app_name, text_content)
+            VALUES (?, ?, ?)
+          \`).run(userRow.id, appName, textContent);
+        }
+      }
+    } catch (err) {
+      console.error('[ScreenRecorder] Capture/OCR failed:', getErrorMessage(err));
+    } finally {
+      // Always cleanup the screenshot image immediately
+      try {
+        await fs.unlink(this.tempFilePath);
+      } catch (e) {
+        // Ignore unlink errors if file didn't exist
+      }
+      this.isProcessing = false;
+    }
+  }
+  cleanupOldRecords() {
+    try {
+      const result = db.prepare(\`
+        DELETE FROM screen_history
+        WHERE timestamp < datetime('now', '-7 days')
+      \`).run();
+      if (result.changes > 0) {
+        console.log(\`[ScreenRecorder] Purged \${result.changes} old screen history records.\`);
+      }
+    } catch (err) {
+      console.error('[ScreenRecorder] Cleanup failed:', getErrorMessage(err));
+    }
+  }
+}
+module.exports = { ScreenRecorder };

package/server/services/integrations/env.js CHANGED Viewed

@@ -64,6 +64,10 @@ function resolveHomeAssistantOAuthConfig() {
   };
 }
+function resolveSpotifyOAuthConfig() {
+  return resolveOAuthConfig('SPOTIFY');
+}
 function describeEnvStatus(config, options = {}) {
   const label = String(options.label || 'This integration').trim() || 'This integration';
   if (config.configured) {
@@ -90,5 +94,6 @@ module.exports = {
   resolveOAuthConfig,
   resolveGoogleOAuthConfig,
   resolvePublicBaseUrl,
+  resolveSpotifyOAuthConfig,
   resolveSlackOAuthConfig,
 };

package/server/services/integrations/registry.js CHANGED Viewed

@@ -5,7 +5,9 @@ const { createGoogleWorkspaceProvider } = require('./google/provider');
 const { createHomeAssistantProvider } = require('./home_assistant/provider');
 const { createMicrosoftProvider } = require('./microsoft/provider');
 const { createNotionProvider } = require('./notion/provider');
+const { createSpotifyProvider } = require('./spotify/provider');
 const { createSlackProvider } = require('./slack/provider');
+const { createWeatherProvider } = require('./weather/provider');
 const { createWhatsAppPersonalProvider } = require('./whatsapp');
 function createIntegrationRegistry(options = {}) {
@@ -16,6 +18,8 @@ function createIntegrationRegistry(options = {}) {
     createSlackProvider(),
     createFigmaProvider(),
     createHomeAssistantProvider(),
+    createWeatherProvider(),
+    createSpotifyProvider(),
     createWhatsAppPersonalProvider(options),
   ];
   const byKey = new Map(providers.map((provider) => [provider.key, provider]));