neoagent 2.1.11 → 2.1.13

package/server/routes/android.js

@@ -85,6 +85,15 @@ router.post('/screenshot', async (req, res) => {
   }
 });
 
+router.post('/observe', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.observe(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
 router.post('/ui-dump', async (req, res) => {
   try {
     const controller = req.app.locals.androidController;

package/server/services/ai/engine.js

@@ -7,6 +7,7 @@ const { ensureDefaultAiSettings, getAiSettings } = require('./settings');
 const { selectToolsForTask } = require('./toolSelector');
 const { compactToolResult } = require('./toolResult');
 const { salvageTextToolCalls } = require('./toolCallSalvage');
+const { sanitizeModelOutput } = require('./outputSanitizer');
 
 function generateTitle(task) {
   if (!task || typeof task !== 'string') return 'Untitled';
@@ -162,6 +163,86 @@ class AgentEngine {
     return this.activeRuns.get(runId) || null;
   }
 
+  findActiveRunForUser(userId, predicate = null) {
+    let candidate = null;
+    for (const [runId, runMeta] of this.activeRuns.entries()) {
+      if (runMeta.userId !== userId || runMeta.aborted) continue;
+      if (typeof predicate === 'function' && !predicate(runMeta, runId)) continue;
+      if (!candidate || (runMeta.startedAt || 0) >= (candidate.startedAt || 0)) {
+        candidate = { runId, ...runMeta };
+      }
+    }
+    return candidate;
+  }
+
+  findSteerableRunForUser(userId, triggerSource = 'web') {
+    return this.findActiveRunForUser(
+      userId,
+      (runMeta) => runMeta.triggerSource === triggerSource && runMeta.triggerType === 'user'
+    );
+  }
+
+  enqueueSteering(runId, content, metadata = {}) {
+    const runMeta = this.getRunMeta(runId);
+    const trimmed = typeof content === 'string' ? content.trim() : '';
+    if (!runMeta || runMeta.aborted || !trimmed) return null;
+
+    const item = {
+      id: uuidv4(),
+      content: trimmed,
+      metadata,
+      createdAt: new Date().toISOString()
+    };
+
+    runMeta.steeringQueue.push(item);
+    this.emit(runMeta.userId, 'run:steer_queued', {
+      runId,
+      content: item.content,
+      pendingCount: runMeta.steeringQueue.length
+    });
+
+    return {
+      runId,
+      pendingCount: runMeta.steeringQueue.length,
+      item
+    };
+  }
+
+  applyQueuedSteering(runId, messages, { userId, conversationId }) {
+    const runMeta = this.getRunMeta(runId);
+    if (!runMeta?.steeringQueue?.length) {
+      return { messages, appliedCount: 0 };
+    }
+
+    const queued = runMeta.steeringQueue.splice(0, runMeta.steeringQueue.length);
+    messages.push({
+      role: 'system',
+      content: [
+        'The user sent follow-up messages while you were already working.',
+        'Treat them as steering or next-up context for the same conversation.',
+        'If a message materially changes the active task, incorporate it now.',
+        'If it is unrelated or better handled after the current task, finish the current work first and then address it.'
+      ].join(' ')
+    });
+
+    for (const entry of queued) {
+      messages.push({ role: 'user', content: entry.content });
+      if (conversationId) {
+        db.prepare('INSERT INTO conversation_messages (conversation_id, role, content) VALUES (?, ?, ?)')
+          .run(conversationId, 'user', entry.content);
+      }
+    }
+
+    this.emit(userId, 'run:steer_applied', {
+      runId,
+      count: queued.length,
+      pendingCount: runMeta.steeringQueue.length,
+      latestContent: queued[queued.length - 1]?.content || ''
+    });
+
+    return { messages, appliedCount: queued.length };
+  }
+
   isRunStopped(runId) {
     return this.getRunMeta(runId)?.aborted === true;
   }
@@ -300,13 +381,16 @@ class AgentEngine {
         });
       }
     };
-    const { provider, model, providerName } = await getProviderForUser(
+    const selectedProvider = await getProviderForUser(
       userId,
       userMessage,
       triggerType === 'subagent',
       _modelOverride,
       providerStatusConfig
     );
+    let provider = selectedProvider.provider;
+    let model = selectedProvider.model;
+    let providerName = selectedProvider.providerName;
 
     const runTitle = generateTitle(userMessage);
     db.prepare(`INSERT OR REPLACE INTO agent_runs(id, user_id, title, status, trigger_type, trigger_source, model)
@@ -317,8 +401,12 @@ class AgentEngine {
       status: 'running',
       aborted: false,
       messagingSent: false,
+      triggerType,
+      triggerSource,
+      startedAt: Date.now(),
       lastToolName: null,
       lastToolTarget: null,
+      steeringQueue: [],
       toolPids: new Set()
     });
     this.emit(userId, 'run:start', { runId, title: runTitle, model, triggerType, triggerSource });
@@ -368,6 +456,12 @@ class AgentEngine {
         if (this.isRunStopped(runId)) break;
         iteration++;
 
+        const steeringAtLoopStart = this.applyQueuedSteering(runId, messages, {
+          userId,
+          conversationId
+        });
+        messages = steeringAtLoopStart.messages;
+
         let metrics = this.estimatePromptMetrics(messages, tools);
         const contextWindow = provider.getContextWindow(model);
         if (metrics.totalEstimatedTokens > contextWindow * 0.7) {
@@ -381,6 +475,7 @@ class AgentEngine {
         this.emit(userId, 'run:thinking', { runId, iteration });
 
         let response;
+        let responseModel = model;
         let streamContent = '';
         const callOptions = { model, reasoningEffort: this.getReasoningEffort(providerName, options) };
 
@@ -391,22 +486,30 @@ class AgentEngine {
               for await (const chunk of gen) {
                 if (chunk.type === 'content') {
                   streamContent += chunk.content;
-                  this.emit(userId, 'run:stream', { runId, content: streamContent, iteration });
+                  this.emit(userId, 'run:stream', {
+                    runId,
+                    content: sanitizeModelOutput(streamContent, { model }),
+                    iteration
+                  });
                 }
                 if (chunk.type === 'done') {
                   response = chunk;
+                  responseModel = model;
                 }
                 if (chunk.type === 'tool_calls') {
                   response = {
                     content: chunk.content || streamContent,
                     toolCalls: chunk.toolCalls,
+                    providerContentBlocks: chunk.providerContentBlocks || null,
                     finishReason: 'tool_calls',
                     usage: chunk.usage || null
                   };
+                  responseModel = model;
                 }
               }
             } else {
               response = await provider.chat(messages, tools, callOptions);
+              responseModel = model;
             }
           } catch (err) {
             console.error(`[Engine] Model call failed (${model}):`, err.message);
@@ -419,33 +522,42 @@ class AgentEngine {
                 aiSettings.fallback_model_id,
                 providerStatusConfig
               );
-              // Update local state for the retry
-              const nextProvider = fallback.provider;
-              const nextModel = fallback.model;
-              const nextProviderName = fallback.providerName;
+              provider = fallback.provider;
+              model = fallback.model;
+              providerName = fallback.providerName;
 
               // Recursive call once
-              const retryOptions = { ...callOptions, model: nextModel, reasoningEffort: this.getReasoningEffort(nextProviderName, options) };
+              const retryOptions = { ...callOptions, model, reasoningEffort: this.getReasoningEffort(providerName, options) };
 
               if (options.stream !== false) {
-                const gen = nextProvider.stream(messages, tools, retryOptions);
+                const gen = provider.stream(messages, tools, retryOptions);
                 for await (const chunk of gen) {
                   if (chunk.type === 'content') {
                     streamContent += chunk.content;
-                    this.emit(userId, 'run:stream', { runId, content: streamContent, iteration });
+                    this.emit(userId, 'run:stream', {
+                      runId,
+                      content: sanitizeModelOutput(streamContent, { model }),
+                      iteration
+                    });
+                  }
+                  if (chunk.type === 'done') {
+                    response = chunk;
+                    responseModel = model;
                   }
-                  if (chunk.type === 'done') response = chunk;
                   if (chunk.type === 'tool_calls') {
                     response = {
                       content: chunk.content || streamContent,
                       toolCalls: chunk.toolCalls,
+                      providerContentBlocks: chunk.providerContentBlocks || null,
                       finishReason: 'tool_calls',
                       usage: chunk.usage || null
                     };
+                    responseModel = model;
                   }
                 }
               } else {
-                response = await nextProvider.chat(messages, tools, retryOptions);
+                response = await provider.chat(messages, tools, retryOptions);
+                responseModel = model;
               }
             } else {
               throw err;
@@ -463,7 +575,7 @@ class AgentEngine {
           totalTokens += response.usage.totalTokens || 0;
         }
 
-        lastContent = response.content || streamContent || '';
+        lastContent = sanitizeModelOutput(response.content || streamContent || '', { model: responseModel });
 
         if ((!response.toolCalls || response.toolCalls.length === 0) && lastContent) {
           const salvaged = salvageTextToolCalls(lastContent, tools);
@@ -477,6 +589,7 @@ class AgentEngine {
 
         const assistantMessage = { role: 'assistant', content: lastContent };
         if (response.toolCalls?.length) assistantMessage.tool_calls = response.toolCalls;
+        if (response.providerContentBlocks?.length) assistantMessage.providerContentBlocks = response.providerContentBlocks;
         messages.push(assistantMessage);
 
         if (conversationId) {
@@ -490,7 +603,19 @@ class AgentEngine {
             );
         }
 
-        if (!response.toolCalls || response.toolCalls.length === 0) break;
+        if (!response.toolCalls || response.toolCalls.length === 0) {
+          const steeringAfterResponse = this.applyQueuedSteering(runId, messages, {
+            userId,
+            conversationId
+          });
+          messages = steeringAfterResponse.messages;
+          if (steeringAfterResponse.appliedCount > 0) {
+            iteration = Math.max(0, iteration - 1);
+            lastContent = '';
+            continue;
+          }
+          break;
+        }
 
         for (const toolCall of response.toolCalls) {
           if (this.isRunStopped(runId)) break;
@@ -583,10 +708,14 @@ class AgentEngine {
           model,
           reasoningEffort: this.getReasoningEffort(providerName, options)
         });
-        lastContent = finalResponse.content || '';
+        lastContent = sanitizeModelOutput(finalResponse.content || '', { model });
         forcedFinalResponse = true;
 
-        messages.push({ role: 'assistant', content: lastContent });
+        const finalAssistantMessage = { role: 'assistant', content: lastContent };
+        if (finalResponse.providerContentBlocks?.length) {
+          finalAssistantMessage.providerContentBlocks = finalResponse.providerContentBlocks;
+        }
+        messages.push(finalAssistantMessage);
         if (conversationId) {
           db.prepare('INSERT INTO conversation_messages (conversation_id, role, content, tokens) VALUES (?, ?, ?, ?)')
             .run(conversationId, 'assistant', lastContent, finalResponse.usage?.totalTokens || 0);

package/server/services/ai/outputSanitizer.js

@@ -0,0 +1,67 @@
+const { sanitizeStreamingToolCallText } = require('./toolCallSalvage');
+
+const HAN_CHAR_REGEX = /\p{Script=Han}/gu;
+const LATIN_CHAR_REGEX = /\p{Script=Latin}/gu;
+const LETTER_CHAR_REGEX = /\p{L}/gu;
+const HAN_RUN_REGEX = /[\p{Script=Han}\u3000-\u303F]+/gu;
+const MARKDOWN_CODE_SPAN_REGEX = /(```[\s\S]*?```|`[^`\n]+`)/g;
+
+function countMatches(text, regex) {
+  const matches = text.match(regex);
+  return matches ? matches.length : 0;
+}
+
+function shouldStripIncidentalHan(text, model) {
+  if (model !== 'MiniMax-M2.7') return false;
+
+  const hanCount = countMatches(text, HAN_CHAR_REGEX);
+  if (hanCount === 0) return false;
+  if (hanCount > 24) return false;
+
+  const latinCount = countMatches(text, LATIN_CHAR_REGEX);
+  if (latinCount < 20) return false;
+
+  const letterCount = countMatches(text, LETTER_CHAR_REGEX);
+  if (letterCount > 0 && (hanCount / letterCount) > 0.18) return false;
+
+  return true;
+}
+
+function sanitizePlainText(text) {
+  return text
+    .replace(/([\p{L}\p{N}])[\p{Script=Han}\u3000-\u303F]+([\p{L}\p{N}])/gu, '$1 $2')
+    .replace(HAN_RUN_REGEX, '')
+    .replace(/[ \t]{2,}/g, ' ')
+    .replace(/[ \t]+\n/g, '\n')
+    .replace(/\n[ \t]+/g, '\n')
+    .replace(/[ \t]+([,.;:!?)\]}])/g, '$1')
+    .replace(/([([{])\s+/g, '$1');
+}
+
+function sanitizeMarkdownAware(text) {
+  return text
+    .split(MARKDOWN_CODE_SPAN_REGEX)
+    .map((part) => {
+      if (!part) return part;
+      if (part.startsWith('```') || part.startsWith('`')) return part;
+      return sanitizePlainText(part);
+    })
+    .join('');
+}
+
+function sanitizeModelOutput(text, options = {}) {
+  if (typeof text !== 'string' || text.length === 0) return text;
+
+  let sanitized = text;
+
+  if (options.model === 'MiniMax-M2.7' && (sanitized.includes('<invoke') || sanitized.includes(':tool_call'))) {
+    sanitized = sanitizeStreamingToolCallText(sanitized);
+  }
+
+  if (!shouldStripIncidentalHan(sanitized, options.model)) return sanitized;
+  return sanitizeMarkdownAware(sanitized);
+}
+
+module.exports = {
+  sanitizeModelOutput
+};

package/server/services/ai/providers/anthropic.js

@@ -37,6 +37,50 @@ class AnthropicProvider extends BaseProvider {
     }));
   }
 
+  normalizeContentBlocks(blocks = []) {
+    const normalized = [];
+
+    for (const block of blocks) {
+      if (!block || !block.type) continue;
+
+      if (block.type === 'thinking') {
+        normalized.push({
+          type: 'thinking',
+          thinking: block.thinking || '',
+          ...(block.signature ? { signature: block.signature } : {})
+        });
+        continue;
+      }
+
+      if (block.type === 'redacted_thinking') {
+        normalized.push({
+          type: 'redacted_thinking',
+          data: block.data
+        });
+        continue;
+      }
+
+      if (block.type === 'text') {
+        normalized.push({
+          type: 'text',
+          text: block.text || ''
+        });
+        continue;
+      }
+
+      if (block.type === 'tool_use') {
+        normalized.push({
+          type: 'tool_use',
+          id: block.id,
+          name: block.name,
+          input: block.input || {}
+        });
+      }
+    }
+
+    return normalized;
+  }
+
   convertMessages(messages) {
     let system = '';
     const converted = [];
@@ -60,6 +104,14 @@ class AnthropicProvider extends BaseProvider {
       }
 
       if (msg.role === 'assistant' && msg.tool_calls) {
+        if (Array.isArray(msg.providerContentBlocks) && msg.providerContentBlocks.length > 0) {
+          converted.push({
+            role: 'assistant',
+            content: this.normalizeContentBlocks(msg.providerContentBlocks)
+          });
+          continue;
+        }
+
         const content = [];
         if (msg.content) content.push({ type: 'text', text: msg.content });
         for (const tc of msg.tool_calls) {
@@ -100,6 +152,7 @@ class AnthropicProvider extends BaseProvider {
 
     let content = '';
     const toolCalls = [];
+    const providerContentBlocks = this.normalizeContentBlocks(response.content);
 
     for (const block of response.content) {
       if (block.type === 'text') {
@@ -119,6 +172,7 @@ class AnthropicProvider extends BaseProvider {
     return {
       content,
       toolCalls,
+      providerContentBlocks,
       finishReason: response.stop_reason === 'tool_use' ? 'tool_calls' : 'stop',
       usage: {
         promptTokens: response.usage.input_tokens,
@@ -148,31 +202,106 @@ class AnthropicProvider extends BaseProvider {
     let content = '';
     let currentToolCalls = [];
     let currentToolIndex = -1;
+    const providerContentBlocks = [];
 
     for await (const event of stream) {
       if (event.type === 'content_block_start') {
-        if (event.content_block.type === 'tool_use') {
+        if (event.content_block.type === 'thinking') {
+          providerContentBlocks[event.index] = {
+            type: 'thinking',
+            thinking: event.content_block.thinking || '',
+            signature: event.content_block.signature || ''
+          };
+        } else if (event.content_block.type === 'redacted_thinking') {
+          providerContentBlocks[event.index] = {
+            type: 'redacted_thinking',
+            data: event.content_block.data
+          };
+        } else if (event.content_block.type === 'text') {
+          providerContentBlocks[event.index] = {
+            type: 'text',
+            text: event.content_block.text || ''
+          };
+        } else if (event.content_block.type === 'tool_use') {
           currentToolIndex++;
           currentToolCalls.push({
             id: event.content_block.id,
             type: 'function',
             function: { name: event.content_block.name, arguments: '' }
           });
+          providerContentBlocks[event.index] = {
+            type: 'tool_use',
+            id: event.content_block.id,
+            name: event.content_block.name,
+            input: {}
+          };
         }
       } else if (event.type === 'content_block_delta') {
         if (event.delta.type === 'text_delta') {
           content += event.delta.text;
+          if (providerContentBlocks[event.index]?.type === 'text') {
+            providerContentBlocks[event.index].text += event.delta.text;
+          }
           yield { type: 'content', content: event.delta.text };
+        } else if (event.delta.type === 'thinking_delta') {
+          if (providerContentBlocks[event.index]?.type === 'thinking') {
+            providerContentBlocks[event.index].thinking += event.delta.thinking || '';
+          }
+        } else if (event.delta.type === 'signature_delta') {
+          if (providerContentBlocks[event.index]?.type === 'thinking') {
+            providerContentBlocks[event.index].signature = event.delta.signature || '';
+          }
         } else if (event.delta.type === 'input_json_delta') {
           if (currentToolCalls[currentToolIndex]) {
             currentToolCalls[currentToolIndex].function.arguments += event.delta.partial_json;
           }
+          if (providerContentBlocks[event.index]?.type === 'tool_use') {
+            const currentJson = providerContentBlocks[event.index]._inputJson || '';
+            providerContentBlocks[event.index]._inputJson = currentJson + (event.delta.partial_json || '');
+          }
         }
       } else if (event.type === 'message_stop') {
+        const normalizedBlocks = providerContentBlocks
+          .filter(Boolean)
+          .map((block) => {
+            if (block.type === 'tool_use') {
+              let parsedInput = block.input || {};
+              if (typeof block._inputJson === 'string' && block._inputJson.trim()) {
+                try {
+                  parsedInput = JSON.parse(block._inputJson);
+                } catch { }
+              }
+              return {
+                type: 'tool_use',
+                id: block.id,
+                name: block.name,
+                input: parsedInput
+              };
+            }
+            if (block.type === 'thinking') {
+              return {
+                type: 'thinking',
+                thinking: block.thinking || '',
+                ...(block.signature ? { signature: block.signature } : {})
+              };
+            }
+            if (block.type === 'redacted_thinking') {
+              return {
+                type: 'redacted_thinking',
+                data: block.data
+              };
+            }
+            return {
+              type: 'text',
+              text: block.text || ''
+            };
+          });
+
         yield {
           type: 'done',
           content,
           toolCalls: currentToolCalls,
+          providerContentBlocks: normalizedBlocks,
           finishReason: currentToolCalls.length > 0 ? 'tool_calls' : 'stop',
           usage: null
         };

package/server/services/ai/providers/base.js

@@ -1,4 +1,9 @@
 class BaseProvider {
+  static readImageAsBase64(imagePath) {
+    const fs = require('fs');
+    return fs.readFileSync(imagePath).toString('base64');
+  }
+
   constructor(config = {}) {
     this.config = config;
     this.name = 'base';
@@ -36,6 +41,18 @@ class BaseProvider {
   getContextWindow(model) {
     return 128000;
   }
+
+  supportsVision() {
+    return false;
+  }
+
+  getDefaultVisionModel() {
+    return null;
+  }
+
+  async analyzeImage(_options = {}) {
+    throw new Error(`Provider '${this.name}' does not support image analysis`);
+  }
 }
 
 module.exports = { BaseProvider };

package/server/services/ai/providers/grok.js

@@ -15,6 +15,14 @@ class GrokProvider extends BaseProvider {
     return 131072; // grok-4 context window
   }
 
+  supportsVision() {
+    return true;
+  }
+
+  getDefaultVisionModel() {
+    return 'grok-4.20-beta-latest-non-reasoning';
+  }
+
   _buildParams(model, messages, tools, options) {
     const params = {
       model,
@@ -116,6 +124,32 @@ class GrokProvider extends BaseProvider {
       }
     }));
   }
+
+  async analyzeImage(options = {}) {
+    const model = options.model || this.getDefaultVisionModel();
+    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
+    const response = await this.client.chat.completions.create({
+      model,
+      max_tokens: options.maxTokens || 4096,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: options.question || 'Describe this image in detail.' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`
+            }
+          }
+        ]
+      }]
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      model: response.model || model,
+    };
+  }
 }
 
 module.exports = { GrokProvider };

package/server/services/ai/providers/openai.js

@@ -48,6 +48,14 @@ class OpenAIProvider extends BaseProvider {
     return 128000;
   }
 
+  supportsVision() {
+    return true;
+  }
+
+  getDefaultVisionModel() {
+    return 'gpt-4.1-mini';
+  }
+
   _buildParams(model, messages, tools, options) {
     const isReasoning = this.isReasoningModel(model);
     // Reasoning models (GPT-5, o-series): use developer role for system messages
@@ -163,6 +171,32 @@ class OpenAIProvider extends BaseProvider {
       }
     }
   }
+
+  async analyzeImage(options = {}) {
+    const model = options.model || this.getDefaultVisionModel();
+    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
+    const response = await this.client.chat.completions.create({
+      model,
+      max_tokens: options.maxTokens || 4096,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: options.question || 'Describe this image in detail.' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`
+            }
+          }
+        ]
+      }]
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      model: response.model || model,
+    };
+  }
 }
 
 module.exports = { OpenAIProvider };

package/server/services/ai/systemPrompt.js

@@ -41,6 +41,7 @@ When prior context makes the goal clear, act on it. Only ask a clarifying questi
 
 REPORT ACTUAL RESULTS
 When a tool returns data, share the relevant parts — summarized if large, direct if short. Never paste raw JSON as the answer. Never narrate what you're about to do at length before doing it.
+Never promise an action in the final answer unless you already took that action in this run. Do not say "I'll check", "I'll fix it", or "I'll send it" and then stop. Either do it first or say you have not done it yet.
 
 DON'T REPEAT YOURSELF
 State a limitation or error once. If the user pushes back, try a different approach before restating the same failure. Repeating the same dead-end across five messages is useless.