neoagent 3.2.1-beta.6 → 3.2.1-beta.8

package/server/services/ai/providers/google.js

@@ -1,7 +1,63 @@
-const { GoogleGenerativeAI } = require('@google/generative-ai');
+'use strict';
+
+const { GoogleGenAI } = require('@google/genai');
 const { BaseProvider } = require('./base');
 const { fetchResponseText } = require('../../network/http');
 
+function parseToolArguments(value) {
+  try {
+    const parsed = JSON.parse(value || '{}');
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+      ? parsed
+      : {};
+  } catch {
+    return {};
+  }
+}
+
+function normalizeUsage(usage) {
+  if (!usage) return null;
+  return {
+    inputTokens: usage.promptTokenCount || 0,
+    outputTokens: usage.candidatesTokenCount || 0,
+    reasoningTokens: usage.thoughtsTokenCount || 0,
+    cachedReadTokens: usage.cachedContentTokenCount || 0,
+    cacheWriteTokens: 0,
+    promptTokens: usage.promptTokenCount || 0,
+    completionTokens: usage.candidatesTokenCount || 0,
+    totalTokens: usage.totalTokenCount || 0
+  };
+}
+
+function collectResponseParts(response, toolCalls, seenToolCalls = null) {
+  let content = '';
+  for (const candidate of response?.candidates || []) {
+    for (const part of candidate.content?.parts || []) {
+      if (part.text && part.thought !== true) content += part.text;
+      if (!part.functionCall?.name) continue;
+
+      const args = part.functionCall.args || {};
+      const providerCallId = String(part.functionCall.id || '').trim();
+      const signature = providerCallId
+        || `${part.functionCall.name}:${JSON.stringify(args)}`;
+      if (seenToolCalls?.has(signature)) continue;
+      seenToolCalls?.add(signature);
+
+      toolCalls.push({
+        id: providerCallId
+          || `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+        type: 'function',
+        function: {
+          name: part.functionCall.name,
+          arguments: JSON.stringify(args),
+          thought_signature: part.thoughtSignature
+        }
+      });
+    }
+  }
+  return content;
+}
+
 class GoogleProvider extends BaseProvider {
   constructor(config = {}) {
     super(config);
@@ -29,7 +85,7 @@ class GoogleProvider extends BaseProvider {
       'gemini-1.5-flash': 1048576,
     };
     this.apiKey = config.apiKey || process.env.GOOGLE_AI_KEY;
-    this.genAI = new GoogleGenerativeAI(this.apiKey);
+    this.genAI = new GoogleGenAI({ apiKey: this.apiKey });
   }
 
   async listModels(signal = null) {
@@ -78,11 +134,23 @@ class GoogleProvider extends BaseProvider {
       functionDeclarations: tools.map(tool => ({
         name: tool.name,
         description: tool.description,
-        parameters: tool.parameters || { type: 'object', properties: {} }
+        parametersJsonSchema: tool.parameters || { type: 'object', properties: {} }
       }))
     }];
   }
 
+  buildGenerateConfig(systemInstruction, tools, options) {
+    const config = {};
+    if (systemInstruction) config.systemInstruction = systemInstruction;
+    if (tools.length > 0) config.tools = this.formatTools(tools);
+    if (options.signal) config.abortSignal = options.signal;
+    const maxOutputTokens = Number(options.maxTokens);
+    if (Number.isFinite(maxOutputTokens) && maxOutputTokens > 0) {
+      config.maxOutputTokens = Math.floor(maxOutputTokens);
+    }
+    return config;
+  }
+
   convertMessages(messages) {
     let systemInstruction = '';
     const history = [];
@@ -93,14 +161,16 @@ class GoogleProvider extends BaseProvider {
         continue;
       }
       if (msg.role === 'tool') {
+        const functionResponse = {
+          name: msg.name || 'tool',
+          response: { result: msg.content }
+        };
+        if (msg.tool_call_id) functionResponse.id = msg.tool_call_id;
         history.push({
-          role: 'function',
-          parts: [{
-            functionResponse: {
-              name: msg.name || 'tool',
-              response: { result: msg.content }
-            }
-          }]
+          // The current Gemini API represents tool results as functionResponse
+          // parts on a user turn.
+          role: 'user',
+          parts: [{ functionResponse }]
         });
         continue;
       }
@@ -111,9 +181,10 @@ class GoogleProvider extends BaseProvider {
           const functionCallPart = {
             functionCall: {
               name: tc.function.name,
-              args: JSON.parse(tc.function.arguments || '{}')
+              args: parseToolArguments(tc.function.arguments)
             }
           };
+          if (tc.id) functionCallPart.functionCall.id = tc.id;
           if (tc.function.thought_signature) {
             functionCallPart.thoughtSignature = tc.function.thought_signature;
           }
@@ -147,6 +218,18 @@ class GoogleProvider extends BaseProvider {
       normalizedHistory.push({ role: currentRole, parts: currentParts });
     }
 
+    // Structured helpers can consist entirely of system instructions. Gemini
+    // still requires a user turn to trigger generation.
+    if (
+      normalizedHistory.length === 0
+      || normalizedHistory[normalizedHistory.length - 1].role === 'model'
+    ) {
+      normalizedHistory.push({
+        role: 'user',
+        parts: [{ text: 'Provide the requested response using the system instructions.' }]
+      });
+    }
+
     if (normalizedHistory.length > 0 && normalizedHistory[0].role !== 'user') {
       normalizedHistory.unshift({
         role: 'user',
@@ -160,53 +243,18 @@ class GoogleProvider extends BaseProvider {
   async chat(messages, tools = [], options = {}) {
     const model = options.model || this.config.model || this.getDefaultModel();
     const { systemInstruction, history } = this.convertMessages(messages);
-
-    const genModel = this.genAI.getGenerativeModel({
+    const response = await this.genAI.models.generateContent({
       model,
-      systemInstruction: systemInstruction || undefined,
-      tools: tools.length > 0 ? this.formatTools(tools) : undefined
+      contents: history,
+      config: this.buildGenerateConfig(systemInstruction, tools, options)
     });
-
-    const lastMessage = history.pop();
-    const chat = genModel.startChat({ history });
-    const result = await chat.sendMessage(lastMessage.parts, { signal: options.signal });
-    const response = result.response;
-
-    let content = '';
     const toolCalls = [];
-
-    for (const candidate of response.candidates || []) {
-      for (const part of candidate.content?.parts || []) {
-        if (part.text) content += part.text;
-        if (part.functionCall) {
-          toolCalls.push({
-            id: `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
-            type: 'function',
-            function: {
-              name: part.functionCall.name,
-              arguments: JSON.stringify(part.functionCall.args || {}),
-              thought_signature: part.thoughtSignature
-            }
-          });
-        }
-      }
-    }
-
-    const usage = response.usageMetadata;
+    const content = collectResponseParts(response, toolCalls);
     return {
       content,
       toolCalls,
       finishReason: toolCalls.length > 0 ? 'tool_calls' : 'stop',
-      usage: usage ? {
-        inputTokens: usage.promptTokenCount || 0,
-        outputTokens: usage.candidatesTokenCount || 0,
-        reasoningTokens: usage.thoughtsTokenCount || 0,
-        cachedReadTokens: usage.cachedContentTokenCount || 0,
-        cacheWriteTokens: 0,
-        promptTokens: usage.promptTokenCount || 0,
-        completionTokens: usage.candidatesTokenCount || 0,
-        totalTokens: usage.totalTokenCount || 0
-      } : null,
+      usage: normalizeUsage(response.usageMetadata),
       model
     };
   }
@@ -214,60 +262,32 @@ class GoogleProvider extends BaseProvider {
   async *stream(messages, tools = [], options = {}) {
     const model = options.model || this.config.model || this.getDefaultModel();
     const { systemInstruction, history } = this.convertMessages(messages);
-
-    const genModel = this.genAI.getGenerativeModel({
+    const responseStream = await this.genAI.models.generateContentStream({
       model,
-      systemInstruction: systemInstruction || undefined,
-      tools: tools.length > 0 ? this.formatTools(tools) : undefined
+      contents: history,
+      config: this.buildGenerateConfig(systemInstruction, tools, options)
     });
-
-    const lastMessage = history.pop();
-    const chat = genModel.startChat({ history });
-    const result = await chat.sendMessageStream(lastMessage.parts, { signal: options.signal });
-
     let content = '';
     const toolCalls = [];
+    const seenToolCalls = new Set();
+    let usage = null;
 
-    for await (const chunk of result.stream) {
-      for (const candidate of chunk.candidates || []) {
-        for (const part of candidate.content?.parts || []) {
-          if (part.text) {
-            content += part.text;
-            yield { type: 'content', content: part.text };
-          }
-          if (part.functionCall) {
-            toolCalls.push({
-              id: `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
-              type: 'function',
-              function: {
-                name: part.functionCall.name,
-                arguments: JSON.stringify(part.functionCall.args || {}),
-                thought_signature: part.thoughtSignature
-              }
-            });
-          }
-        }
+    for await (const chunk of responseStream) {
+      const chunkContent = collectResponseParts(chunk, toolCalls, seenToolCalls);
+      if (chunkContent) {
+        content += chunkContent;
+        yield { type: 'content', content: chunkContent };
       }
+      if (chunk.usageMetadata) usage = chunk.usageMetadata;
     }
 
-    const finalResponse = await result.response;
-    const usage = finalResponse.usageMetadata;
-
     yield {
       type: 'done',
       content,
       toolCalls,
       finishReason: toolCalls.length > 0 ? 'tool_calls' : 'stop',
-      usage: usage ? {
-        inputTokens: usage.promptTokenCount || 0,
-        outputTokens: usage.candidatesTokenCount || 0,
-        reasoningTokens: usage.thoughtsTokenCount || 0,
-        cachedReadTokens: usage.cachedContentTokenCount || 0,
-        cacheWriteTokens: 0,
-        promptTokens: usage.promptTokenCount || 0,
-        completionTokens: usage.candidatesTokenCount || 0,
-        totalTokens: usage.totalTokenCount || 0
-      } : null
+      usage: normalizeUsage(usage),
+      model
     };
   }
 }

package/server/services/ai/terminal_reply.js

@@ -10,7 +10,13 @@ function normalizeReply(content) {
 }
 
 function hasExternalBlocker(text) {
-  return /\b(?:blocked|cannot|can't|could not|couldn't|unable to|do not have access|don't have access|missing (?:access|credentials|permission|information)|need you to|requires? your|waiting for your|please (?:provide|send|choose|confirm|authorize|approve))\b/.test(text);
+  return /\b(?:blocked|cannot|can't|could not|couldn't|unable to|do not have access|don't have access|missing (?:access|credentials|permission|information)|need you to|requires? your|waiting for your|please (?:provide|send|choose|confirm|authorize|approve)|blockiert|kann nicht|konnte nicht|mir fehlt|ich brauche von dir|warte auf dein(?:e|en)?|bitte (?:gib|schick|sende|nenn|bestätige|waehle|wähle|genehmige))\b/.test(text);
+}
+
+function isTerminalQuestionOrBlockerReply(content) {
+  const text = normalizeReply(content);
+  if (!text) return false;
+  return /[?？]/.test(text) || hasExternalBlocker(text);
 }
 
 /**
@@ -36,10 +42,16 @@ function isDeferredWorkReply(content) {
     /^(?:working on it|checking now|on it)[.!…]*$/,
     /\bi(?:'ll| will)\s+(?:get back to you|update you|report back|let you know|keep you posted)\b/,
     /\b(?:i(?:'ll| will)\s+follow up|stay tuned)\b/,
+    /^(?:(?:klar|okay|ok|alles klar)[,.!]?\s+)?ich\s+(?:(?:arbeite|pruefe|prüfe|schaue|untersuche|teste|debugge|starte)\s+(?:(?:gerade|aktuell|jetzt|noch)\b|(?:das|dies|die logs?|den code|deine anfrage|deinen auftrag)\b)|(?:kuemmere|kümmere)\s+mich\s+(?:gerade|aktuell|jetzt|noch)\b)/,
+    /^(?:(?:klar|okay|ok|alles klar)[,.!]?\s+)?ich\s+(?:werde|wuerde|würde)\s+(?:jetzt\s+)?(?:pruefen|prüfen|nachsehen|untersuchen|testen|debuggen|starten|fixen|erledigen)\b/,
+    /^(?:(?:klar|okay|ok|alles klar)[,.!]?\s+)?(?:lass|lasst)\s+mich\s+(?:(?:das|dies|die logs?|den code)\s+)?(?:pruefen|prüfen|nachsehen|untersuchen|testen|debuggen)\b/,
+    /^(?:gib mir|gebt mir)\s+(?:einen\s+)?(?:moment|augenblick)|^(?:bin dran|mache ich)[.!…]*$/,
+    /\bich\s+(?:melde mich|gebe dir bescheid|halte dich auf dem laufenden)\b/,
   ];
   return patterns.some((pattern) => pattern.test(text));
 }
 
 module.exports = {
   isDeferredWorkReply,
+  isTerminalQuestionOrBlockerReply,
 };

package/server/services/runtime/docker-vm-manager.js

@@ -306,6 +306,15 @@ class DockerVMManager {
     return Boolean(session && isContainerRunning(session.containerId));
   }
 
+  // A side-effect-free snapshot for request-path capability reporting. Unlike
+  // hasVm(), this deliberately avoids a synchronous `docker inspect`, which can
+  // block the Node event loop. Tool execution still calls ensureVm() and performs
+  // the authoritative liveness check before using a tracked session.
+  hasTrackedVm(userId) {
+    const key = String(userId || '').trim();
+    return Boolean(key && this.instances.has(key));
+  }
+
   // Used by validation.js — cached to avoid docker calls on every status poll.
   getReadiness() {
     const now = Date.now();

package/server/services/runtime/guest_bootstrap.js

@@ -13,6 +13,8 @@ const GUEST_PAYLOAD_PROFILES = Object.freeze({
     { source: 'runtime/paths.js', target: 'runtime/paths.js' },
     { source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
     { source: 'server/services/browser', target: 'server/services/browser' },
+    { source: 'server/services/android/process.js', target: 'server/services/android/process.js' },
+    { source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
   ],
   cli: [
     { source: 'server/guest-agent.cli.package.json', target: 'package.json' },
@@ -28,6 +30,8 @@ const GUEST_PAYLOAD_PROFILES = Object.freeze({
     { source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
     { source: 'server/services/cli', target: 'server/services/cli' },
     { source: 'server/services/browser', target: 'server/services/browser' },
+    { source: 'server/services/android/process.js', target: 'server/services/android/process.js' },
+    { source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
   ],
   android: [
     { source: 'server/guest-agent.android.package.json', target: 'package.json' },
@@ -36,6 +40,9 @@ const GUEST_PAYLOAD_PROFILES = Object.freeze({
     { source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
     { source: 'server/services/cli', target: 'server/services/cli' },
     { source: 'server/services/android', target: 'server/services/android' },
+    { source: 'server/utils/abort.js', target: 'server/utils/abort.js' },
+    { source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
+    { source: 'server/utils/image_payload.js', target: 'server/utils/image_payload.js' },
   ],
 });
 

package/server/services/runtime/guest_image.js

@@ -3,7 +3,7 @@
 const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
-const { spawnSync } = require('child_process');
+const { spawn, spawnSync } = require('child_process');
 const { DATA_DIR } = require('../../../runtime/paths');
 const { stageGuestPayload, normalizeRuntimeProfile } = require('./guest_bootstrap');
 
@@ -105,6 +105,34 @@ function dockerAvailable() {
   return !result.error && result.status === 0;
 }
 
+function runDockerCommand(args, options = {}) {
+  const timeoutMs = Number(options.timeout || BUILD_TIMEOUT_MS);
+  const spawnImpl = options.spawnImpl || spawn;
+  return new Promise((resolve, reject) => {
+    const child = spawnImpl('docker', args, {
+      stdio: options.stdio || ['ignore', 'inherit', 'inherit'],
+      windowsHide: true,
+    });
+    let settled = false;
+    const finish = (error, status = null) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      if (error) reject(error);
+      else resolve(status);
+    };
+    const timeout = setTimeout(() => {
+      const error = new Error(`docker ${args[0]} timed out after ${timeoutMs}ms`);
+      error.code = 'DOCKER_COMMAND_TIMEOUT';
+      try { child.kill('SIGKILL'); } catch {}
+      finish(error);
+    }, timeoutMs);
+    timeout.unref?.();
+    child.once('error', (error) => finish(error));
+    child.once('close', (status) => finish(null, status));
+  });
+}
+
 // Builds and caches the per-profile guest-agent Docker image. The image bakes in
 // the guest agent, its dependencies, and (for browser_cli) the Chromium browser,
 // so containers start the agent directly with no runtime installation step.
@@ -166,15 +194,16 @@ class GuestImageBuilder {
   async #build(tag) {
     console.log(`[GuestImage:${this.profile}] Building guest image ${tag} (one-time; downloads browser + deps)…`);
     const started = Date.now();
-    const result = docker(['build', '-t', tag, this.contextDir], {
-      timeout: BUILD_TIMEOUT_MS,
-      stdio: ['ignore', 'inherit', 'inherit'],
-    });
-    if (result.error) {
-      throw new Error(`Guest image build failed to start: ${result.error.message}`);
+    let status;
+    try {
+      status = await runDockerCommand(['build', '-t', tag, this.contextDir], {
+        timeout: BUILD_TIMEOUT_MS,
+      });
+    } catch (error) {
+      throw new Error(`Guest image build failed: ${error.message}`, { cause: error });
     }
-    if (result.status !== 0) {
-      throw new Error(`Guest image build for ${tag} exited with status ${result.status}`);
+    if (status !== 0) {
+      throw new Error(`Guest image build for ${tag} exited with status ${status}`);
     }
     this.#cachedBuiltAt = Date.now();
     console.log(`[GuestImage:${this.profile}] Built ${tag} in ${Math.round((Date.now() - started) / 1000)}s`);
@@ -185,5 +214,6 @@ class GuestImageBuilder {
 module.exports = {
   GuestImageBuilder,
   dockerAvailable,
+  runDockerCommand,
   BASE_IMAGE,
 };

package/server/services/runtime/manager.js

@@ -87,6 +87,41 @@ class RuntimeManager {
     );
   }
 
+  getCapabilitySnapshot(userId) {
+    const settings = this.getSettings(userId) || {};
+    const key = String(userId || '').trim();
+    const extensionConnected = this.hasActiveExtensionBrowser(userId);
+    const androidController = key && this.androidControllers instanceof Map
+      ? this.androidControllers.get(key)
+      : null;
+    let androidStatus = null;
+    if (androidController && typeof androidController.getStatusSync === 'function') {
+      try {
+        androidStatus = androidController.getStatusSync();
+      } catch {
+        androidStatus = null;
+      }
+    }
+
+    return {
+      browser: {
+        preferredBackend: settings.browser_backend,
+        activeBackend: extensionConnected ? 'extension' : 'vm',
+        extensionConnected,
+        vmInitialized: Boolean(
+          this.browserBackend?.vmManager?.hasTrackedVm?.(userId),
+        ),
+      },
+      android: {
+        initialized: Boolean(androidController),
+        status: androidStatus,
+      },
+      desktop: {
+        connected: this.hasActiveDesktopCompanion(userId),
+      },
+    };
+  }
+
   resolveBackend(userId, requested) {
     void userId;
     return requested === 'browser' ? this.browserBackend : this.cliBackend;