neoagent 2.1.10 → 2.1.12

package/server/routes/android.js

@@ -1,10 +1,45 @@
 const express = require('express');
+const fs = require('fs');
+const path = require('path');
+const multer = require('multer');
 const router = express.Router();
+const { DATA_DIR } = require('../../runtime/paths');
 const { requireAuth } = require('../middleware/auth');
 const { sanitizeError } = require('../utils/security');
 
 router.use(requireAuth);
 
+const androidApkUploadDir = path.join(DATA_DIR, 'uploads', 'android-apks');
+fs.mkdirSync(androidApkUploadDir, { recursive: true });
+
+const androidApkUpload = multer({
+  storage: multer.diskStorage({
+    destination: (_req, _file, cb) => cb(null, androidApkUploadDir),
+    filename: (_req, file, cb) => {
+      const extension = path.extname(file.originalname || '').toLowerCase();
+      const stem = path.basename(file.originalname || 'upload', extension)
+        .replace(/[^a-z0-9._-]+/gi, '-')
+        .replace(/^-+|-+$/g, '')
+        .slice(0, 64) || 'upload';
+      cb(
+        null,
+        `${Date.now()}-${Math.random().toString(16).slice(2)}-${stem}${extension || '.apk'}`
+      );
+    },
+  }),
+  fileFilter: (_req, file, cb) => {
+    if (!String(file.originalname || '').toLowerCase().endsWith('.apk')) {
+      cb(new Error('Only .apk files can be installed.'));
+      return;
+    }
+    cb(null, true);
+  },
+  limits: {
+    fileSize: 512 * 1024 * 1024,
+    files: 1,
+  },
+});
+
 router.get('/status', async (req, res) => {
   try {
     const controller = req.app.locals.androidController;
@@ -97,6 +132,15 @@ router.post('/tap', async (req, res) => {
   }
 });
 
+router.post('/long-press', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.longPress(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
 router.post('/type', async (req, res) => {
   try {
     const controller = req.app.locals.androidController;
@@ -133,4 +177,47 @@ router.post('/wait-for', async (req, res) => {
   }
 });
 
+router.post('/install-apk', (req, res) => {
+  androidApkUpload.single('apk')(req, res, async (uploadError) => {
+    if (uploadError) {
+      const message =
+        uploadError instanceof multer.MulterError &&
+          uploadError.code === 'LIMIT_FILE_SIZE'
+        ? 'APK upload is too large. Limit is 512MB.'
+        : sanitizeError(uploadError);
+      res.status(400).json({ error: message });
+      return;
+    }
+
+    const uploadedApkPath = req.file?.path;
+    if (!uploadedApkPath) {
+      res.status(400).json({ error: 'No APK file was uploaded.' });
+      return;
+    }
+
+    try {
+      const controller = req.app.locals.androidController;
+      const result = await controller.installApk({ apkPath: uploadedApkPath });
+      res.json({
+        ...result,
+        filename: req.file.originalname,
+        size: req.file.size,
+      });
+    } catch (err) {
+      res.status(500).json({ error: sanitizeError(err) });
+    } finally {
+      fs.promises.unlink(uploadedApkPath).catch(() => {});
+    }
+  });
+});
+
+router.post('/shell', async (req, res) => {
+  try {
+    const controller = req.app.locals.androidController;
+    res.json(await controller.shell(req.body || {}));
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
 module.exports = router;

package/server/services/ai/engine.js

@@ -6,6 +6,8 @@ const { getConversationContext, buildSummaryCarrier, refreshConversationSummary
 const { ensureDefaultAiSettings, getAiSettings } = require('./settings');
 const { selectToolsForTask } = require('./toolSelector');
 const { compactToolResult } = require('./toolResult');
+const { salvageTextToolCalls } = require('./toolCallSalvage');
+const { sanitizeModelOutput } = require('./outputSanitizer');
 
 function generateTitle(task) {
   if (!task || typeof task !== 'string') return 'Untitled';
@@ -299,13 +301,16 @@ class AgentEngine {
         });
       }
     };
-    const { provider, model, providerName } = await getProviderForUser(
+    const selectedProvider = await getProviderForUser(
       userId,
       userMessage,
       triggerType === 'subagent',
       _modelOverride,
       providerStatusConfig
     );
+    let provider = selectedProvider.provider;
+    let model = selectedProvider.model;
+    let providerName = selectedProvider.providerName;
 
     const runTitle = generateTitle(userMessage);
     db.prepare(`INSERT OR REPLACE INTO agent_runs(id, user_id, title, status, trigger_type, trigger_source, model)
@@ -380,6 +385,7 @@ class AgentEngine {
         this.emit(userId, 'run:thinking', { runId, iteration });
 
         let response;
+        let responseModel = model;
         let streamContent = '';
         const callOptions = { model, reasoningEffort: this.getReasoningEffort(providerName, options) };
 
@@ -390,22 +396,30 @@ class AgentEngine {
               for await (const chunk of gen) {
                 if (chunk.type === 'content') {
                   streamContent += chunk.content;
-                  this.emit(userId, 'run:stream', { runId, content: streamContent, iteration });
+                  this.emit(userId, 'run:stream', {
+                    runId,
+                    content: sanitizeModelOutput(streamContent, { model }),
+                    iteration
+                  });
                 }
                 if (chunk.type === 'done') {
                   response = chunk;
+                  responseModel = model;
                 }
                 if (chunk.type === 'tool_calls') {
                   response = {
                     content: chunk.content || streamContent,
                     toolCalls: chunk.toolCalls,
+                    providerContentBlocks: chunk.providerContentBlocks || null,
                     finishReason: 'tool_calls',
                     usage: chunk.usage || null
                   };
+                  responseModel = model;
                 }
               }
             } else {
               response = await provider.chat(messages, tools, callOptions);
+              responseModel = model;
             }
           } catch (err) {
             console.error(`[Engine] Model call failed (${model}):`, err.message);
@@ -418,33 +432,42 @@ class AgentEngine {
                 aiSettings.fallback_model_id,
                 providerStatusConfig
               );
-              // Update local state for the retry
-              const nextProvider = fallback.provider;
-              const nextModel = fallback.model;
-              const nextProviderName = fallback.providerName;
+              provider = fallback.provider;
+              model = fallback.model;
+              providerName = fallback.providerName;
 
               // Recursive call once
-              const retryOptions = { ...callOptions, model: nextModel, reasoningEffort: this.getReasoningEffort(nextProviderName, options) };
+              const retryOptions = { ...callOptions, model, reasoningEffort: this.getReasoningEffort(providerName, options) };
 
               if (options.stream !== false) {
-                const gen = nextProvider.stream(messages, tools, retryOptions);
+                const gen = provider.stream(messages, tools, retryOptions);
                 for await (const chunk of gen) {
                   if (chunk.type === 'content') {
                     streamContent += chunk.content;
-                    this.emit(userId, 'run:stream', { runId, content: streamContent, iteration });
+                    this.emit(userId, 'run:stream', {
+                      runId,
+                      content: sanitizeModelOutput(streamContent, { model }),
+                      iteration
+                    });
+                  }
+                  if (chunk.type === 'done') {
+                    response = chunk;
+                    responseModel = model;
                   }
-                  if (chunk.type === 'done') response = chunk;
                   if (chunk.type === 'tool_calls') {
                     response = {
                       content: chunk.content || streamContent,
                       toolCalls: chunk.toolCalls,
+                      providerContentBlocks: chunk.providerContentBlocks || null,
                       finishReason: 'tool_calls',
                       usage: chunk.usage || null
                     };
+                    responseModel = model;
                   }
                 }
               } else {
-                response = await nextProvider.chat(messages, tools, retryOptions);
+                response = await provider.chat(messages, tools, retryOptions);
+                responseModel = model;
               }
             } else {
               throw err;
@@ -462,10 +485,21 @@ class AgentEngine {
           totalTokens += response.usage.totalTokens || 0;
         }
 
-        lastContent = response.content || streamContent || '';
+        lastContent = sanitizeModelOutput(response.content || streamContent || '', { model: responseModel });
+
+        if ((!response.toolCalls || response.toolCalls.length === 0) && lastContent) {
+          const salvaged = salvageTextToolCalls(lastContent, tools);
+          if (salvaged.toolCalls.length > 0) {
+            response.toolCalls = salvaged.toolCalls;
+            response.finishReason = 'tool_calls';
+            response.content = salvaged.content;
+            lastContent = salvaged.content;
+          }
+        }
 
         const assistantMessage = { role: 'assistant', content: lastContent };
         if (response.toolCalls?.length) assistantMessage.tool_calls = response.toolCalls;
+        if (response.providerContentBlocks?.length) assistantMessage.providerContentBlocks = response.providerContentBlocks;
         messages.push(assistantMessage);
 
         if (conversationId) {
@@ -572,10 +606,14 @@ class AgentEngine {
           model,
           reasoningEffort: this.getReasoningEffort(providerName, options)
         });
-        lastContent = finalResponse.content || '';
+        lastContent = sanitizeModelOutput(finalResponse.content || '', { model });
         forcedFinalResponse = true;
 
-        messages.push({ role: 'assistant', content: lastContent });
+        const finalAssistantMessage = { role: 'assistant', content: lastContent };
+        if (finalResponse.providerContentBlocks?.length) {
+          finalAssistantMessage.providerContentBlocks = finalResponse.providerContentBlocks;
+        }
+        messages.push(finalAssistantMessage);
         if (conversationId) {
           db.prepare('INSERT INTO conversation_messages (conversation_id, role, content, tokens) VALUES (?, ?, ?, ?)')
             .run(conversationId, 'assistant', lastContent, finalResponse.usage?.totalTokens || 0);

package/server/services/ai/outputSanitizer.js

@@ -0,0 +1,67 @@
+const { sanitizeStreamingToolCallText } = require('./toolCallSalvage');
+
+const HAN_CHAR_REGEX = /\p{Script=Han}/gu;
+const LATIN_CHAR_REGEX = /\p{Script=Latin}/gu;
+const LETTER_CHAR_REGEX = /\p{L}/gu;
+const HAN_RUN_REGEX = /[\p{Script=Han}\u3000-\u303F]+/gu;
+const MARKDOWN_CODE_SPAN_REGEX = /(```[\s\S]*?```|`[^`\n]+`)/g;
+
+function countMatches(text, regex) {
+  const matches = text.match(regex);
+  return matches ? matches.length : 0;
+}
+
+function shouldStripIncidentalHan(text, model) {
+  if (model !== 'MiniMax-M2.7') return false;
+
+  const hanCount = countMatches(text, HAN_CHAR_REGEX);
+  if (hanCount === 0) return false;
+  if (hanCount > 24) return false;
+
+  const latinCount = countMatches(text, LATIN_CHAR_REGEX);
+  if (latinCount < 20) return false;
+
+  const letterCount = countMatches(text, LETTER_CHAR_REGEX);
+  if (letterCount > 0 && (hanCount / letterCount) > 0.18) return false;
+
+  return true;
+}
+
+function sanitizePlainText(text) {
+  return text
+    .replace(/([\p{L}\p{N}])[\p{Script=Han}\u3000-\u303F]+([\p{L}\p{N}])/gu, '$1 $2')
+    .replace(HAN_RUN_REGEX, '')
+    .replace(/[ \t]{2,}/g, ' ')
+    .replace(/[ \t]+\n/g, '\n')
+    .replace(/\n[ \t]+/g, '\n')
+    .replace(/[ \t]+([,.;:!?)\]}])/g, '$1')
+    .replace(/([([{])\s+/g, '$1');
+}
+
+function sanitizeMarkdownAware(text) {
+  return text
+    .split(MARKDOWN_CODE_SPAN_REGEX)
+    .map((part) => {
+      if (!part) return part;
+      if (part.startsWith('```') || part.startsWith('`')) return part;
+      return sanitizePlainText(part);
+    })
+    .join('');
+}
+
+function sanitizeModelOutput(text, options = {}) {
+  if (typeof text !== 'string' || text.length === 0) return text;
+
+  let sanitized = text;
+
+  if (options.model === 'MiniMax-M2.7' && (sanitized.includes('<invoke') || sanitized.includes(':tool_call'))) {
+    sanitized = sanitizeStreamingToolCallText(sanitized);
+  }
+
+  if (!shouldStripIncidentalHan(sanitized, options.model)) return sanitized;
+  return sanitizeMarkdownAware(sanitized);
+}
+
+module.exports = {
+  sanitizeModelOutput
+};

package/server/services/ai/providers/anthropic.js

@@ -37,6 +37,50 @@ class AnthropicProvider extends BaseProvider {
     }));
   }
 
+  normalizeContentBlocks(blocks = []) {
+    const normalized = [];
+
+    for (const block of blocks) {
+      if (!block || !block.type) continue;
+
+      if (block.type === 'thinking') {
+        normalized.push({
+          type: 'thinking',
+          thinking: block.thinking || '',
+          ...(block.signature ? { signature: block.signature } : {})
+        });
+        continue;
+      }
+
+      if (block.type === 'redacted_thinking') {
+        normalized.push({
+          type: 'redacted_thinking',
+          data: block.data
+        });
+        continue;
+      }
+
+      if (block.type === 'text') {
+        normalized.push({
+          type: 'text',
+          text: block.text || ''
+        });
+        continue;
+      }
+
+      if (block.type === 'tool_use') {
+        normalized.push({
+          type: 'tool_use',
+          id: block.id,
+          name: block.name,
+          input: block.input || {}
+        });
+      }
+    }
+
+    return normalized;
+  }
+
   convertMessages(messages) {
     let system = '';
     const converted = [];
@@ -60,6 +104,14 @@ class AnthropicProvider extends BaseProvider {
       }
 
       if (msg.role === 'assistant' && msg.tool_calls) {
+        if (Array.isArray(msg.providerContentBlocks) && msg.providerContentBlocks.length > 0) {
+          converted.push({
+            role: 'assistant',
+            content: this.normalizeContentBlocks(msg.providerContentBlocks)
+          });
+          continue;
+        }
+
         const content = [];
         if (msg.content) content.push({ type: 'text', text: msg.content });
         for (const tc of msg.tool_calls) {
@@ -100,6 +152,7 @@ class AnthropicProvider extends BaseProvider {
 
     let content = '';
     const toolCalls = [];
+    const providerContentBlocks = this.normalizeContentBlocks(response.content);
 
     for (const block of response.content) {
       if (block.type === 'text') {
@@ -119,6 +172,7 @@ class AnthropicProvider extends BaseProvider {
     return {
       content,
       toolCalls,
+      providerContentBlocks,
       finishReason: response.stop_reason === 'tool_use' ? 'tool_calls' : 'stop',
       usage: {
         promptTokens: response.usage.input_tokens,
@@ -148,31 +202,106 @@ class AnthropicProvider extends BaseProvider {
     let content = '';
     let currentToolCalls = [];
     let currentToolIndex = -1;
+    const providerContentBlocks = [];
 
     for await (const event of stream) {
       if (event.type === 'content_block_start') {
-        if (event.content_block.type === 'tool_use') {
+        if (event.content_block.type === 'thinking') {
+          providerContentBlocks[event.index] = {
+            type: 'thinking',
+            thinking: event.content_block.thinking || '',
+            signature: event.content_block.signature || ''
+          };
+        } else if (event.content_block.type === 'redacted_thinking') {
+          providerContentBlocks[event.index] = {
+            type: 'redacted_thinking',
+            data: event.content_block.data
+          };
+        } else if (event.content_block.type === 'text') {
+          providerContentBlocks[event.index] = {
+            type: 'text',
+            text: event.content_block.text || ''
+          };
+        } else if (event.content_block.type === 'tool_use') {
           currentToolIndex++;
           currentToolCalls.push({
             id: event.content_block.id,
             type: 'function',
             function: { name: event.content_block.name, arguments: '' }
           });
+          providerContentBlocks[event.index] = {
+            type: 'tool_use',
+            id: event.content_block.id,
+            name: event.content_block.name,
+            input: {}
+          };
         }
       } else if (event.type === 'content_block_delta') {
         if (event.delta.type === 'text_delta') {
           content += event.delta.text;
+          if (providerContentBlocks[event.index]?.type === 'text') {
+            providerContentBlocks[event.index].text += event.delta.text;
+          }
           yield { type: 'content', content: event.delta.text };
+        } else if (event.delta.type === 'thinking_delta') {
+          if (providerContentBlocks[event.index]?.type === 'thinking') {
+            providerContentBlocks[event.index].thinking += event.delta.thinking || '';
+          }
+        } else if (event.delta.type === 'signature_delta') {
+          if (providerContentBlocks[event.index]?.type === 'thinking') {
+            providerContentBlocks[event.index].signature = event.delta.signature || '';
+          }
         } else if (event.delta.type === 'input_json_delta') {
           if (currentToolCalls[currentToolIndex]) {
             currentToolCalls[currentToolIndex].function.arguments += event.delta.partial_json;
           }
+          if (providerContentBlocks[event.index]?.type === 'tool_use') {
+            const currentJson = providerContentBlocks[event.index]._inputJson || '';
+            providerContentBlocks[event.index]._inputJson = currentJson + (event.delta.partial_json || '');
+          }
         }
       } else if (event.type === 'message_stop') {
+        const normalizedBlocks = providerContentBlocks
+          .filter(Boolean)
+          .map((block) => {
+            if (block.type === 'tool_use') {
+              let parsedInput = block.input || {};
+              if (typeof block._inputJson === 'string' && block._inputJson.trim()) {
+                try {
+                  parsedInput = JSON.parse(block._inputJson);
+                } catch { }
+              }
+              return {
+                type: 'tool_use',
+                id: block.id,
+                name: block.name,
+                input: parsedInput
+              };
+            }
+            if (block.type === 'thinking') {
+              return {
+                type: 'thinking',
+                thinking: block.thinking || '',
+                ...(block.signature ? { signature: block.signature } : {})
+              };
+            }
+            if (block.type === 'redacted_thinking') {
+              return {
+                type: 'redacted_thinking',
+                data: block.data
+              };
+            }
+            return {
+              type: 'text',
+              text: block.text || ''
+            };
+          });
+
         yield {
           type: 'done',
           content,
           toolCalls: currentToolCalls,
+          providerContentBlocks: normalizedBlocks,
           finishReason: currentToolCalls.length > 0 ? 'tool_calls' : 'stop',
           usage: null
         };

package/server/services/ai/systemPrompt.js

@@ -56,6 +56,7 @@ The tools listed in this call are exactly what you have. Trust the list. If a to
 
 SHELL COMMANDS
 When you use execute_command, treat timed out or killed commands as unfinished work, not success. For installs, updates, restarts, config changes, or other state-changing shell actions, verify the outcome with a follow-up command before telling the user it is done.
+If you restart or stop the NeoAgent service, this run ends immediately. Warn the user before doing it and say you cannot continue the current run after the restart.
 
 SKILLS
 If a multi-step task produces a reusable pattern, save or improve it as a skill when appropriate.

package/server/services/ai/toolCallSalvage.js

@@ -0,0 +1,142 @@
+const { v4: uuidv4 } = require('uuid')
+
+const INVOKE_OPEN_RE = /(?:[A-Za-z0-9_.-]+:tool_call\s*)?<invoke\s+name="([^"]+)">/g
+const PARAM_OPEN_RE = /<parameter\s+name="([^"]+)">/g
+const PARAM_CLOSED_RE = /<parameter\s+name="([^"]+)">([\s\S]*?)<\/parameter>/g
+const TOOL_WRAPPER_RE = /<\/?[A-Za-z0-9_.-]+:tool_call>/g
+const COMPLETE_INLINE_CALL_RE = /(?:[A-Za-z0-9_.-]+:tool_call\s*)?<invoke\s+name="[^"]+">[\s\S]*?<\/invoke>/g
+const INVOKE_CLOSE = '</invoke>'
+
+function trimLooseControlText(text) {
+  return String(text || '')
+    .replace(TOOL_WRAPPER_RE, '')
+    .replace(/(?:^|\s)[A-Za-z0-9_.-]+:tool_call\s*$/g, '')
+    .trim()
+}
+
+function sanitizeStreamingToolCallText(text) {
+  let visible = String(text || '')
+    .replace(COMPLETE_INLINE_CALL_RE, '')
+    .replace(TOOL_WRAPPER_RE, '')
+
+  const partialStarts = [
+    visible.lastIndexOf('<invoke'),
+    visible.lastIndexOf(':tool_call')
+  ].filter((index) => index >= 0)
+
+  if (partialStarts.length > 0) {
+    const partialStart = Math.max(...partialStarts)
+    const suffix = visible.slice(partialStart)
+    if (!suffix.includes(INVOKE_CLOSE)) {
+      visible = visible.slice(0, partialStart)
+    }
+  }
+
+  return trimLooseControlText(visible).replace(/\n{3,}/g, '\n\n')
+}
+
+function parseParameterMap(body) {
+  const args = {}
+  let sawClosedParam = false
+  let match
+
+  PARAM_CLOSED_RE.lastIndex = 0
+  while ((match = PARAM_CLOSED_RE.exec(body)) !== null) {
+    sawClosedParam = true
+    args[match[1]] = match[2].trim()
+  }
+  if (sawClosedParam) return args
+
+  const markers = []
+  PARAM_OPEN_RE.lastIndex = 0
+  while ((match = PARAM_OPEN_RE.exec(body)) !== null) {
+    markers.push({
+      name: match[1],
+      valueStart: match.index + match[0].length,
+      openIndex: match.index
+    })
+  }
+
+  for (let i = 0; i < markers.length; i++) {
+    const current = markers[i]
+    const next = markers[i + 1]
+    const rawValue = body.slice(current.valueStart, next ? next.openIndex : body.length)
+    args[current.name] = rawValue.trim()
+  }
+
+  return args
+}
+
+function salvageTextToolCalls(content, tools = []) {
+  const text = String(content || '')
+  if (!text.includes('<invoke')) {
+    return { content: text, toolCalls: [] }
+  }
+
+  const allowedToolNames = new Set(
+    Array.isArray(tools) ? tools.map((tool) => tool?.name).filter(Boolean) : []
+  )
+
+  const openings = []
+  let match
+  INVOKE_OPEN_RE.lastIndex = 0
+  while ((match = INVOKE_OPEN_RE.exec(text)) !== null) {
+    openings.push({
+      index: match.index,
+      openText: match[0],
+      name: match[1]
+    })
+  }
+
+  if (openings.length === 0) {
+    return { content: text, toolCalls: [] }
+  }
+
+  const cleanedParts = []
+  const toolCalls = []
+  let cursor = 0
+
+  for (let i = 0; i < openings.length; i++) {
+    const current = openings[i]
+    const next = openings[i + 1]
+    const searchEnd = next ? next.index : text.length
+    const bodyStart = current.index + current.openText.length
+    const closeIndex = text.indexOf(INVOKE_CLOSE, bodyStart)
+    const blockEnd = closeIndex !== -1 && closeIndex < searchEnd
+      ? closeIndex + INVOKE_CLOSE.length
+      : searchEnd
+    const bodyEnd = closeIndex !== -1 && closeIndex < searchEnd
+      ? closeIndex
+      : searchEnd
+
+    cleanedParts.push(text.slice(cursor, current.index))
+    cursor = blockEnd
+
+    if (allowedToolNames.size > 0 && !allowedToolNames.has(current.name)) {
+      cleanedParts.push(text.slice(current.index, blockEnd))
+      continue
+    }
+
+    const args = parseParameterMap(text.slice(bodyStart, bodyEnd))
+    toolCalls.push({
+      id: `salvaged_${uuidv4()}`,
+      type: 'function',
+      function: {
+        name: current.name,
+        arguments: JSON.stringify(args)
+      }
+    })
+  }
+
+  cleanedParts.push(text.slice(cursor))
+
+  return {
+    content: trimLooseControlText(cleanedParts.join('').replace(/\n{3,}/g, '\n\n')),
+    toolCalls
+  }
+}
+
+module.exports = {
+  salvageTextToolCalls,
+  sanitizeStreamingToolCallText
+}

package/server/services/ai/toolResult.js

@@ -108,6 +108,16 @@ function compactToolResult(toolName, toolArgs = {}, toolResult, options = {}) {
       });
       break;
 
+    case 'android_shell':
+      envelope = trimObject({
+        tool: toolName,
+        serial: toolResult?.serial,
+        command: toolArgs.command,
+        screenshotPath: toolResult?.screenshotPath,
+        excerpt: lineExcerpt(toolResult?.stdout || toolResult?.result || toolResult, 18, Math.floor(softLimit * 0.65))
+      });
+      break;
+
     case 'http_request':
       envelope = trimObject({
         tool: toolName,