neoagent 2.1.12 → 2.1.14

package/server/routes/android.js

@@ -11,6 +11,7 @@ router.use(requireAuth);
 
 const androidApkUploadDir = path.join(DATA_DIR, 'uploads', 'android-apks');
 fs.mkdirSync(androidApkUploadDir, { recursive: true });
+const INSTALLABLE_ANDROID_PACKAGE_EXTENSIONS = new Set(['.apk', '.apks']);
 
 const androidApkUpload = multer({
   storage: multer.diskStorage({
@@ -28,8 +29,9 @@ const androidApkUpload = multer({
     },
   }),
   fileFilter: (_req, file, cb) => {
-    if (!String(file.originalname || '').toLowerCase().endsWith('.apk')) {
-      cb(new Error('Only .apk files can be installed.'));
+    const extension = path.extname(String(file.originalname || '')).toLowerCase();
+    if (!INSTALLABLE_ANDROID_PACKAGE_EXTENSIONS.has(extension)) {
+      cb(new Error('Only .apk or .apks files can be installed.'));
       return;
     }
     cb(null, true);
@@ -85,6 +87,15 @@ router.post('/screenshot', async (req, res) => {
   }
 });
 
+router.post('/observe', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.observe(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
 router.post('/ui-dump', async (req, res) => {
   try {
     const controller = req.app.locals.androidController;
@@ -183,7 +194,7 @@ router.post('/install-apk', (req, res) => {
       const message =
         uploadError instanceof multer.MulterError &&
           uploadError.code === 'LIMIT_FILE_SIZE'
-        ? 'APK upload is too large. Limit is 512MB.'
+        ? 'Android app upload is too large. Limit is 512MB.'
         : sanitizeError(uploadError);
       res.status(400).json({ error: message });
       return;
@@ -191,7 +202,7 @@ router.post('/install-apk', (req, res) => {
 
     const uploadedApkPath = req.file?.path;
     if (!uploadedApkPath) {
-      res.status(400).json({ error: 'No APK file was uploaded.' });
+      res.status(400).json({ error: 'No APK or APK bundle was uploaded.' });
       return;
     }
 

package/server/services/ai/engine.js

@@ -2,7 +2,12 @@ const { v4: uuidv4 } = require('uuid');
 const fs = require('fs');
 const db = require('../../db/database');
 const { compact } = require('./compaction');
-const { getConversationContext, buildSummaryCarrier, refreshConversationSummary } = require('./history');
+const {
+  getConversationContext,
+  buildSummaryCarrier,
+  refreshConversationSummary,
+  sanitizeConversationMessages
+} = require('./history');
 const { ensureDefaultAiSettings, getAiSettings } = require('./settings');
 const { selectToolsForTask } = require('./toolSelector');
 const { compactToolResult } = require('./toolResult');
@@ -163,6 +168,86 @@ class AgentEngine {
     return this.activeRuns.get(runId) || null;
   }
 
+  findActiveRunForUser(userId, predicate = null) {
+    let candidate = null;
+    for (const [runId, runMeta] of this.activeRuns.entries()) {
+      if (runMeta.userId !== userId || runMeta.aborted) continue;
+      if (typeof predicate === 'function' && !predicate(runMeta, runId)) continue;
+      if (!candidate || (runMeta.startedAt || 0) >= (candidate.startedAt || 0)) {
+        candidate = { runId, ...runMeta };
+      }
+    }
+    return candidate;
+  }
+
+  findSteerableRunForUser(userId, triggerSource = 'web') {
+    return this.findActiveRunForUser(
+      userId,
+      (runMeta) => runMeta.triggerSource === triggerSource && runMeta.triggerType === 'user'
+    );
+  }
+
+  enqueueSteering(runId, content, metadata = {}) {
+    const runMeta = this.getRunMeta(runId);
+    const trimmed = typeof content === 'string' ? content.trim() : '';
+    if (!runMeta || runMeta.aborted || !trimmed) return null;
+
+    const item = {
+      id: uuidv4(),
+      content: trimmed,
+      metadata,
+      createdAt: new Date().toISOString()
+    };
+
+    runMeta.steeringQueue.push(item);
+    this.emit(runMeta.userId, 'run:steer_queued', {
+      runId,
+      content: item.content,
+      pendingCount: runMeta.steeringQueue.length
+    });
+
+    return {
+      runId,
+      pendingCount: runMeta.steeringQueue.length,
+      item
+    };
+  }
+
+  applyQueuedSteering(runId, messages, { userId, conversationId }) {
+    const runMeta = this.getRunMeta(runId);
+    if (!runMeta?.steeringQueue?.length) {
+      return { messages, appliedCount: 0 };
+    }
+
+    const queued = runMeta.steeringQueue.splice(0, runMeta.steeringQueue.length);
+    messages.push({
+      role: 'system',
+      content: [
+        'The user sent follow-up messages while you were already working.',
+        'Treat them as steering or next-up context for the same conversation.',
+        'If a message materially changes the active task, incorporate it now.',
+        'If it is unrelated or better handled after the current task, finish the current work first and then address it.'
+      ].join(' ')
+    });
+
+    for (const entry of queued) {
+      messages.push({ role: 'user', content: entry.content });
+      if (conversationId) {
+        db.prepare('INSERT INTO conversation_messages (conversation_id, role, content) VALUES (?, ?, ?)')
+          .run(conversationId, 'user', entry.content);
+      }
+    }
+
+    this.emit(userId, 'run:steer_applied', {
+      runId,
+      count: queued.length,
+      pendingCount: runMeta.steeringQueue.length,
+      latestContent: queued[queued.length - 1]?.content || ''
+    });
+
+    return { messages, appliedCount: queued.length };
+  }
+
   isRunStopped(runId) {
     return this.getRunMeta(runId)?.aborted === true;
   }
@@ -321,8 +406,12 @@ class AgentEngine {
       status: 'running',
       aborted: false,
       messagingSent: false,
+      triggerType,
+      triggerSource,
+      startedAt: Date.now(),
       lastToolName: null,
       lastToolTarget: null,
+      steeringQueue: [],
       toolPids: new Set()
     });
     this.emit(userId, 'run:start', { runId, title: runTitle, model, triggerType, triggerSource });
@@ -354,6 +443,7 @@ class AgentEngine {
 
     let messages = this.buildContextMessages(systemPrompt, summaryMessage, historyMessages, recallMsg);
     messages.push(this.buildUserMessage(userMessage, options));
+    messages = sanitizeConversationMessages(messages);
 
     if (conversationId) {
       db.prepare('INSERT INTO conversation_messages (conversation_id, role, content) VALUES (?, ?, ?)')
@@ -372,10 +462,18 @@ class AgentEngine {
         if (this.isRunStopped(runId)) break;
         iteration++;
 
+        const steeringAtLoopStart = this.applyQueuedSteering(runId, messages, {
+          userId,
+          conversationId
+        });
+        messages = steeringAtLoopStart.messages;
+        messages = sanitizeConversationMessages(messages);
+
         let metrics = this.estimatePromptMetrics(messages, tools);
         const contextWindow = provider.getContextWindow(model);
         if (metrics.totalEstimatedTokens > contextWindow * 0.7) {
           messages = await compact(messages, provider, model);
+          messages = sanitizeConversationMessages(messages);
           this.emit(userId, 'run:compaction', { runId, iteration });
           metrics = this.estimatePromptMetrics(messages, tools);
         }
@@ -390,9 +488,10 @@ class AgentEngine {
         const callOptions = { model, reasoningEffort: this.getReasoningEffort(providerName, options) };
 
         const tryModelCall = async (retryForFallback = true) => {
+          const requestMessages = sanitizeConversationMessages(messages);
           try {
             if (options.stream !== false) {
-              const gen = provider.stream(messages, tools, callOptions);
+              const gen = provider.stream(requestMessages, tools, callOptions);
               for await (const chunk of gen) {
                 if (chunk.type === 'content') {
                   streamContent += chunk.content;
@@ -418,7 +517,7 @@ class AgentEngine {
                 }
               }
             } else {
-              response = await provider.chat(messages, tools, callOptions);
+              response = await provider.chat(requestMessages, tools, callOptions);
               responseModel = model;
             }
           } catch (err) {
@@ -438,9 +537,10 @@ class AgentEngine {
 
               // Recursive call once
               const retryOptions = { ...callOptions, model, reasoningEffort: this.getReasoningEffort(providerName, options) };
+              const retryMessages = sanitizeConversationMessages(messages);
 
               if (options.stream !== false) {
-                const gen = provider.stream(messages, tools, retryOptions);
+                const gen = provider.stream(retryMessages, tools, retryOptions);
                 for await (const chunk of gen) {
                   if (chunk.type === 'content') {
                     streamContent += chunk.content;
@@ -466,7 +566,7 @@ class AgentEngine {
                   }
                 }
               } else {
-                response = await provider.chat(messages, tools, retryOptions);
+                response = await provider.chat(retryMessages, tools, retryOptions);
                 responseModel = model;
               }
             } else {
@@ -513,7 +613,19 @@ class AgentEngine {
             );
         }
 
-        if (!response.toolCalls || response.toolCalls.length === 0) break;
+        if (!response.toolCalls || response.toolCalls.length === 0) {
+          const steeringAfterResponse = this.applyQueuedSteering(runId, messages, {
+            userId,
+            conversationId
+          });
+          messages = steeringAfterResponse.messages;
+          if (steeringAfterResponse.appliedCount > 0) {
+            iteration = Math.max(0, iteration - 1);
+            lastContent = '';
+            continue;
+          }
+          break;
+        }
 
         for (const toolCall of response.toolCalls) {
           if (this.isRunStopped(runId)) break;
@@ -602,7 +714,7 @@ class AgentEngine {
 
       if ((iteration >= maxIterations && messages[messages.length - 1]?.role === 'tool')
         || (iteration < maxIterations && stepIndex > 0 && !lastContent.trim() && messages[messages.length - 1]?.role !== 'tool')) {
-        const finalResponse = await provider.chat(messages, [], {
+        const finalResponse = await provider.chat(sanitizeConversationMessages(messages), [], {
           model,
           reasoningEffort: this.getReasoningEffort(providerName, options)
         });

package/server/services/ai/history.js

@@ -34,6 +34,67 @@ function normalizeHistoryRows(rows) {
   });
 }
 
+function sanitizeConversationMessages(messages) {
+  const sanitized = [];
+  let pendingToolSequence = null;
+
+  const dropPendingSequence = () => {
+    pendingToolSequence = null;
+  };
+
+  const flushPendingSequence = () => {
+    if (!pendingToolSequence) return;
+    if (pendingToolSequence.pendingIds.size === 0) {
+      sanitized.push(...pendingToolSequence.messages);
+    }
+    pendingToolSequence = null;
+  };
+
+  for (const msg of messages || []) {
+    if (!msg || !msg.role) continue;
+
+    if (msg.role === 'assistant' && Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0) {
+      const toolCallIds = msg.tool_calls
+        .map((toolCall) => toolCall?.id)
+        .filter(Boolean);
+
+      if (toolCallIds.length === 0) {
+        dropPendingSequence();
+        sanitized.push(msg);
+        continue;
+      }
+
+      dropPendingSequence();
+      pendingToolSequence = {
+        messages: [msg],
+        pendingIds: new Set(toolCallIds)
+      };
+      continue;
+    }
+
+    if (msg.role === 'tool') {
+      if (
+        pendingToolSequence
+        && msg.tool_call_id
+        && pendingToolSequence.pendingIds.has(msg.tool_call_id)
+      ) {
+        pendingToolSequence.messages.push(msg);
+        pendingToolSequence.pendingIds.delete(msg.tool_call_id);
+        if (pendingToolSequence.pendingIds.size === 0) {
+          flushPendingSequence();
+        }
+      }
+      continue;
+    }
+
+    dropPendingSequence();
+    sanitized.push(msg);
+  }
+
+  flushPendingSequence();
+  return sanitized;
+}
+
 function serializeHistoryForSummary(messages) {
   return messages.map((msg) => {
     if (msg.role === 'tool') {
@@ -143,7 +204,7 @@ function getConversationContext(conversationId, recentLimit) {
   return {
     summary: convo?.summary || '',
     summaryCount: Number(convo?.summary_message_count || 0),
-    recentMessages: normalizeHistoryRows(recent),
+    recentMessages: sanitizeConversationMessages(normalizeHistoryRows(recent)),
     totalMessages: db.prepare('SELECT COUNT(*) AS count FROM conversation_messages WHERE conversation_id = ?').get(conversationId).count
   };
 }
@@ -184,5 +245,6 @@ module.exports = {
   getWebChatContext,
   refreshConversationSummary,
   refreshWebChatSummary,
+  sanitizeConversationMessages,
   summarizeMessages
 };

package/server/services/ai/providers/base.js

@@ -1,4 +1,9 @@
 class BaseProvider {
+  static readImageAsBase64(imagePath) {
+    const fs = require('fs');
+    return fs.readFileSync(imagePath).toString('base64');
+  }
+
   constructor(config = {}) {
     this.config = config;
     this.name = 'base';
@@ -36,6 +41,18 @@ class BaseProvider {
   getContextWindow(model) {
     return 128000;
   }
+
+  supportsVision() {
+    return false;
+  }
+
+  getDefaultVisionModel() {
+    return null;
+  }
+
+  async analyzeImage(_options = {}) {
+    throw new Error(`Provider '${this.name}' does not support image analysis`);
+  }
 }
 
 module.exports = { BaseProvider };

package/server/services/ai/providers/grok.js

@@ -15,6 +15,14 @@ class GrokProvider extends BaseProvider {
     return 131072; // grok-4 context window
   }
 
+  supportsVision() {
+    return true;
+  }
+
+  getDefaultVisionModel() {
+    return 'grok-4.20-beta-latest-non-reasoning';
+  }
+
   _buildParams(model, messages, tools, options) {
     const params = {
       model,
@@ -116,6 +124,32 @@ class GrokProvider extends BaseProvider {
       }
     }));
   }
+
+  async analyzeImage(options = {}) {
+    const model = options.model || this.getDefaultVisionModel();
+    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
+    const response = await this.client.chat.completions.create({
+      model,
+      max_tokens: options.maxTokens || 4096,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: options.question || 'Describe this image in detail.' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`
+            }
+          }
+        ]
+      }]
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      model: response.model || model,
+    };
+  }
 }
 
 module.exports = { GrokProvider };

package/server/services/ai/providers/openai.js

@@ -48,6 +48,14 @@ class OpenAIProvider extends BaseProvider {
     return 128000;
   }
 
+  supportsVision() {
+    return true;
+  }
+
+  getDefaultVisionModel() {
+    return 'gpt-4.1-mini';
+  }
+
   _buildParams(model, messages, tools, options) {
     const isReasoning = this.isReasoningModel(model);
     // Reasoning models (GPT-5, o-series): use developer role for system messages
@@ -163,6 +171,32 @@ class OpenAIProvider extends BaseProvider {
       }
     }
   }
+
+  async analyzeImage(options = {}) {
+    const model = options.model || this.getDefaultVisionModel();
+    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
+    const response = await this.client.chat.completions.create({
+      model,
+      max_tokens: options.maxTokens || 4096,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: options.question || 'Describe this image in detail.' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`
+            }
+          }
+        ]
+      }]
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      model: response.model || model,
+    };
+  }
 }
 
 module.exports = { OpenAIProvider };

package/server/services/ai/systemPrompt.js

@@ -41,6 +41,7 @@ When prior context makes the goal clear, act on it. Only ask a clarifying questi
 
 REPORT ACTUAL RESULTS
 When a tool returns data, share the relevant parts — summarized if large, direct if short. Never paste raw JSON as the answer. Never narrate what you're about to do at length before doing it.
+Never promise an action in the final answer unless you already took that action in this run. Do not say "I'll check", "I'll fix it", or "I'll send it" and then stop. Either do it first or say you have not done it yet.
 
 DON'T REPEAT YOURSELF
 State a limitation or error once. If the user pushes back, try a different approach before restating the same failure. Repeating the same dead-end across five messages is useless.

package/server/services/ai/toolResult.js

@@ -90,10 +90,12 @@ function compactToolResult(toolName, toolArgs = {}, toolResult, options = {}) {
       break;
 
     case 'android_dump_ui':
+    case 'android_observe':
       envelope = trimObject({
         tool: toolName,
         serial: toolResult?.serial,
         nodeCount: toolResult?.nodeCount,
+        screenshotPath: toolResult?.screenshotPath,
         uiDumpPath: toolResult?.uiDumpPath,
         preview: clampText(JSON.stringify(toolResult?.preview || []).slice(0, Math.floor(softLimit * 0.55)), Math.floor(softLimit * 0.55))
       });

package/server/services/ai/tools.js

@@ -292,6 +292,16 @@ function getAvailableTools(app, options = {}) {
                 }
             }
         },
+        {
+            name: 'android_observe',
+            description: 'Capture the current Android screen end-to-end: fresh screenshot, UI dump path, and a preview of visible UI nodes.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    includeNodes: { type: 'boolean', description: 'Include a preview of parsed UI nodes (default true)' }
+                }
+            }
+        },
         {
             name: 'android_dump_ui',
             description: 'Capture the current Android UIAutomator XML dump and return a preview of the nodes.',
@@ -322,11 +332,11 @@ function getAvailableTools(app, options = {}) {
         },
         {
             name: 'android_install_apk',
-            description: 'Install or replace an APK on the Android emulator.',
+            description: 'Install or replace an APK or universal .apks bundle on the Android emulator.',
             parameters: {
                 type: 'object',
                 properties: {
-                    apkPath: { type: 'string', description: 'Absolute path to the APK file on disk' }
+                    apkPath: { type: 'string', description: 'Absolute path to an .apk file or universal .apks bundle on disk' }
                 },
                 required: ['apkPath']
             }
@@ -763,7 +773,7 @@ function getAvailableTools(app, options = {}) {
         },
         {
             name: 'analyze_image',
-            description: 'Analyze an image file using Grok vision. Use this to describe photos, read QR codes, extract text from screenshots, or answer any visual question about an image.',
+            description: 'Analyze an image file using the best available vision-capable model. Use this to describe photos, read QR codes, extract text from screenshots, or answer visual questions.',
             parameters: {
                 type: 'object',
                 properties: {
@@ -940,6 +950,12 @@ async function executeTool(toolName, args, context, engine) {
             return await controller.waitFor(args || {});
         }
 
+        case 'android_observe': {
+            const controller = ac();
+            if (!controller) return { error: 'Android controller not available' };
+            return await controller.observe(args || {});
+        }
+
         case 'android_dump_ui': {
             const controller = ac();
             if (!controller) return { error: 'Android controller not available' };
@@ -1541,23 +1557,67 @@ async function executeTool(toolName, args, context, engine) {
         case 'analyze_image': {
             try {
                 if (!fs.existsSync(args.image_path)) return { error: `File not found: ${args.image_path}` };
-                const b64 = fs.readFileSync(args.image_path).toString('base64');
                 const ext = path.extname(args.image_path).toLowerCase();
                 const mimeMap = { '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg' };
                 const mime = mimeMap[ext] || 'image/jpeg';
+                const question = args.question || 'Describe this image in detail.';
                 const { getProviderForUser } = require('./engine');
-                const { provider: visionProvider, model: visionModel } = await getProviderForUser(userId);
-                const visionResponse = await visionProvider.chat(
-                    [{
-                        role: 'user', content: [
-                            { type: 'text', text: args.question || 'Describe this image in detail.' },
-                            { type: 'image_url', image_url: { url: `data:${mime};base64,${b64}` } }
-                        ]
-                    }],
-                    [],
-                    { model: visionModel }
-                );
-                return { description: visionResponse.content };
+                const { createProviderInstance, getProviderCatalog } = require('./models');
+
+                const attempted = [];
+                const candidates = [];
+
+                try {
+                    const preferred = await getProviderForUser(userId);
+                    candidates.push({
+                        providerName: preferred.providerName,
+                        provider: preferred.provider,
+                    });
+                } catch (err) {
+                    attempted.push(`default-provider lookup failed: ${err.message}`);
+                }
+
+                for (const providerInfo of getProviderCatalog(userId)) {
+                    if (!providerInfo.available) continue;
+                    if (candidates.some((candidate) => candidate.providerName === providerInfo.id)) continue;
+                    if (!['grok', 'openai'].includes(providerInfo.id)) continue;
+                    try {
+                        candidates.push({
+                            providerName: providerInfo.id,
+                            provider: createProviderInstance(providerInfo.id, userId),
+                        });
+                    } catch (err) {
+                        attempted.push(`${providerInfo.id}: ${err.message}`);
+                    }
+                }
+
+                for (const candidate of candidates) {
+                    if (typeof candidate.provider.supportsVision !== 'function' || candidate.provider.supportsVision() !== true) {
+                        attempted.push(`${candidate.providerName}: image analysis is not supported by this provider integration`);
+                        continue;
+                    }
+
+                    try {
+                        const visionResponse = await candidate.provider.analyzeImage({
+                            imagePath: args.image_path,
+                            mimeType: mime,
+                            question,
+                        });
+                        return {
+                            description: visionResponse.content,
+                            model: visionResponse.model || null,
+                            provider: candidate.providerName,
+                        };
+                    } catch (err) {
+                        attempted.push(`${candidate.providerName}: ${err.message}`);
+                    }
+                }
+
+                return {
+                    error: attempted.length > 0
+                        ? `Image analysis failed. ${attempted.join(' | ')}`
+                        : 'No vision-capable provider is currently available. Configure OpenAI or xAI for image analysis.',
+                };
             } catch (err) {
                 return { error: err.message };
             }