neoagent 2.3.1-beta.85 → 2.3.1-beta.86

package/server/services/runtime/manager.js

@@ -2,14 +2,23 @@ const { LocalVmExecutionBackend } = require('./backends/local-vm');
 const { QemuVmManager } = require('./qemu');
 const { getRuntimeSettings } = require('./settings');
 const { ExtensionBrowserProvider } = require('../browser/extension/provider');
+const { AndroidController } = require('../android/controller');
 
 class RuntimeManager {
   constructor(options = {}) {
     this.browserExtensionRegistry = options.browserExtensionRegistry || null;
-    this.vmBackend = new LocalVmExecutionBackend({
-      vmManager: options.vmManager || new QemuVmManager(),
+    const browserVmManager = options.browserVmManager || new QemuVmManager({
+      runtimeProfile: 'browser_cli',
+      memoryMb: 2048,
+      cpus: 2,
+      warmup: false,
+    });
+    this.browserBackend = new LocalVmExecutionBackend({
+      runtimeProfile: 'browser_cli',
+      vmManager: browserVmManager,
       artifactStore: options.artifactStore,
     });
+    this.androidControllers = new Map();
     this.getExtensionBrowserProvider = options.getExtensionBrowserProvider || ((userId) => new ExtensionBrowserProvider({
       registry: options.browserExtensionRegistry,
       artifactStore: options.artifactStore,
@@ -31,25 +40,27 @@ class RuntimeManager {
 
   resolveBackend(userId, requested) {
     void userId;
-    void requested;
-    return this.vmBackend;
+    return this.browserBackend;
   }
 
   async executeCommand(userId, command, options = {}) {
-    const backend = this.resolveBackend(userId, options.backend);
+    const backend = this.resolveBackend(userId, 'browser_cli');
     return backend.executeCommand(userId, command, options);
   }
 
-  hasVmForUser(userId) {
-    return Boolean(this.vmBackend?.vmManager?.hasVm?.(userId));
+  hasVmForUser(userId, capability = 'browser') {
+    if (capability === 'android') {
+      return Boolean(this.androidControllers.get(String(userId || '').trim()));
+    }
+    return Boolean(this.browserBackend?.vmManager?.hasVm?.(userId));
   }
 
   async killCommand(userId, pid, reason = 'aborted') {
-    return this.vmBackend.killCommand(userId, pid, reason);
+    return this.browserBackend.killCommand(userId, pid, reason);
   }
 
   async getCommandExecutorForUser(userId) {
-    return this.vmBackend.getCommandExecutorForUser(userId);
+    return this.browserBackend.getCommandExecutorForUser(userId);
   }
 
   async getBrowserProviderForUser(userId) {
@@ -57,22 +68,49 @@ class RuntimeManager {
     if (settings.browser_backend === 'extension' && this.hasActiveExtensionBrowser(userId)) {
       return this.getExtensionBrowserProvider(userId);
     }
-    return this.vmBackend.getBrowserProviderForUser(userId);
+    return this.browserBackend.getBrowserProviderForUser(userId);
   }
 
   async getAndroidProviderForUser(userId) {
-    return this.vmBackend.getAndroidProviderForUser(userId);
+    const key = String(userId || '').trim();
+    if (!key) {
+      throw new Error('Android provider requires a user ID.');
+    }
+    if (!this.androidControllers.has(key)) {
+      this.androidControllers.set(key, new AndroidController({
+        userId: key,
+        runtimeBackend: 'host',
+        artifactStore: null,
+      }));
+    }
+    return this.androidControllers.get(key);
   }
 
-  async isGuestAgentReadyForUser(userId, timeoutMs = 1000) {
-    if (typeof this.vmBackend?.isGuestAgentReadyForUser !== 'function') {
+  async isGuestAgentReadyForUser(userId, timeoutMs = 1000, capability = 'browser') {
+    if (capability === 'android') {
+      const controller = this.androidControllers.get(String(userId || '').trim());
+      if (!controller || typeof controller.getStatus !== 'function') {
+        return false;
+      }
+      try {
+        const status = await controller.getStatus();
+        return Boolean(status?.bootstrapped || status?.serial || status?.starting);
+      } catch {
+        return false;
+      }
+    }
+    if (typeof this.browserBackend?.isGuestAgentReadyForUser !== 'function') {
       return false;
     }
-    return this.vmBackend.isGuestAgentReadyForUser(userId, timeoutMs);
+    return this.browserBackend.isGuestAgentReadyForUser(userId, timeoutMs);
   }
 
   async shutdown() {
-    await Promise.allSettled([this.vmBackend.shutdown()]);
+    await Promise.allSettled([
+      this.browserBackend?.shutdown?.(),
+      ...[...this.androidControllers.values()].map((controller) => controller?.stopEmulator?.().catch?.(() => {})),
+    ]);
+    this.androidControllers.clear();
   }
 }
 

package/server/services/runtime/qemu.js

@@ -9,12 +9,9 @@ const { spawn, spawnSync } = require('child_process');
 const { DATA_DIR } = require('../../../runtime/paths');
 const { ensureGuestBootstrapSeed } = require('./guest_bootstrap');
 
+const REPO_ROOT = path.resolve(__dirname, '../../..');
 const VM_ROOT = path.join(DATA_DIR, 'runtime-vms');
-const BASE_IMAGE_CACHE_ROOT = path.join(VM_ROOT, 'base-images');
-const TEMPLATE_ROOT = path.join(VM_ROOT, 'templates');
 fs.mkdirSync(VM_ROOT, { recursive: true });
-fs.mkdirSync(BASE_IMAGE_CACHE_ROOT, { recursive: true });
-fs.mkdirSync(TEMPLATE_ROOT, { recursive: true });
 
 const DEFAULT_UBUNTU_BASE_IMAGE_URLS = Object.freeze({
   x64: 'https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img',
@@ -67,6 +64,47 @@ function generateGuestToken() {
   return crypto.randomBytes(32).toString('hex');
 }
 
+function computeRuntimeTemplateSignature(guestArch, runtimeProfile = 'browser_cli') {
+  const hash = crypto.createHash('sha256');
+  const normalizedProfile = runtimeProfile === 'android' ? 'android' : 'browser_cli';
+  const trackedFiles = normalizedProfile === 'android'
+    ? [
+      'server/guest-agent.android.package.json',
+      'server/guest_agent.js',
+      'server/services/android/controller.js',
+      'server/services/cli/executor.js',
+      'server/services/runtime/guest_bootstrap.js',
+      'runtime/env.js',
+      'runtime/paths.js',
+    ]
+    : [
+      'server/guest-agent.browser.package.json',
+      'server/guest_agent.js',
+      'server/services/browser/controller.js',
+      'server/services/cli/executor.js',
+      'server/services/runtime/guest_bootstrap.js',
+      'runtime/env.js',
+      'runtime/paths.js',
+    ];
+
+  hash.update(String(guestArch || 'x64'));
+  hash.update('\0');
+  hash.update(normalizedProfile);
+  for (const relativePath of trackedFiles) {
+    const filePath = path.join(REPO_ROOT, relativePath);
+    hash.update('\0');
+    hash.update(relativePath);
+    hash.update('\0');
+    try {
+      hash.update(fs.readFileSync(filePath));
+    } catch (error) {
+      hash.update(`missing:${error?.code || 'unknown'}`);
+    }
+  }
+
+  return hash.digest('hex');
+}
+
 function resolveGuestToken(userRoot) {
   const tokenPath = path.join(userRoot, 'guest-token.txt');
   try {
@@ -551,9 +589,22 @@ function formatReadinessIssues(readiness) {
   return issues.length > 0 ? issues : ['VM runtime is unavailable on this host.'];
 }
 
+function readTemplateReadyMetadata(readySentinelPath) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(readySentinelPath, 'utf8'));
+    if (parsed && typeof parsed === 'object') {
+      return parsed;
+    }
+  } catch {}
+  return null;
+}
+
 class QemuVmManager {
   constructor(options = {}) {
-    this.rootDir = path.resolve(options.rootDir || VM_ROOT);
+    this.runtimeProfile = options.runtimeProfile === 'android' ? 'android' : 'browser_cli';
+    this.rootDir = path.resolve(options.rootDir || path.join(VM_ROOT, this.runtimeProfile));
+    this.baseImageCacheRoot = path.resolve(options.baseImageCacheRoot || path.join(this.rootDir, 'base-images'));
+    this.templateRootDir = path.resolve(options.templateRootDir || path.join(this.rootDir, 'templates'));
     this.baseImagePath = options.baseImagePath || process.env.NEOAGENT_VM_BASE_IMAGE || '';
     this.guestArch = options.guestArch || guestArchForHost();
     this.baseImageUrl = normalizeBaseImageUrlForArch(
@@ -565,12 +616,17 @@ class QemuVmManager {
     this.instances = new Map();
     this.baseImagePromise = null;
     this.runtimeTemplatePromise = null;
+    this.warmupEnabled = options.warmup === true;
     fs.mkdirSync(this.rootDir, { recursive: true });
-    setTimeout(() => {
-      this.ensureRuntimeTemplateAvailable().catch((error) => {
-        console.warn(`[VM] Background runtime template warmup failed: ${error.message}`);
-      });
-    }, 0);
+    fs.mkdirSync(this.baseImageCacheRoot, { recursive: true });
+    fs.mkdirSync(this.templateRootDir, { recursive: true });
+    if (this.warmupEnabled) {
+      setTimeout(() => {
+        this.ensureRuntimeTemplateAvailable().catch((error) => {
+          console.warn(`[VM:${this.runtimeProfile}] Background runtime template warmup failed: ${error.message}`);
+        });
+      }, 0);
+    }
   }
 
   getBaseImageCachePath() {
@@ -579,7 +635,7 @@ class QemuVmManager {
     }
     const parsed = new URL(this.baseImageUrl);
     const filename = path.basename(parsed.pathname || '') || `${this.guestArch}-base.img`;
-    return path.join(BASE_IMAGE_CACHE_ROOT, filename);
+    return path.join(this.baseImageCacheRoot, filename);
   }
 
   resolveBaseImagePath() {
@@ -619,7 +675,7 @@ class QemuVmManager {
   }
 
   getRuntimeTemplateRoot() {
-    return path.join(TEMPLATE_ROOT, this.guestArch);
+    return path.join(this.templateRootDir, this.guestArch);
   }
 
   getRuntimeTemplateDiskPath() {
@@ -631,13 +687,24 @@ class QemuVmManager {
   }
 
   getRuntimeTemplateReadyMarker() {
-    return '/var/lib/neoagent/browser-runtime-ready';
+    return this.runtimeProfile === 'android'
+      ? '/var/lib/neoagent/bootstrap-complete'
+      : '/var/lib/neoagent/browser-runtime-ready';
+  }
+
+  getRuntimeTemplateSignature() {
+    return computeRuntimeTemplateSignature(this.guestArch, this.runtimeProfile);
   }
 
   async ensureRuntimeTemplateAvailable() {
     const readyDiskPath = this.getRuntimeTemplateDiskPath();
     const readySentinelPath = path.join(this.getRuntimeTemplateRoot(), '.runtime-template-ready');
-    if (fs.existsSync(readyDiskPath) && fs.existsSync(readySentinelPath)) {
+    const readyMetadata = readTemplateReadyMetadata(readySentinelPath);
+    if (
+      fs.existsSync(readyDiskPath)
+      && readyMetadata
+      && readyMetadata.signature === this.getRuntimeTemplateSignature()
+    ) {
       return readyDiskPath;
     }
     if (!this.runtimeTemplatePromise) {
@@ -653,9 +720,15 @@ class QemuVmManager {
     const readySentinelPath = path.join(this.getRuntimeTemplateRoot(), '.runtime-template-ready');
     const lockDir = this.getRuntimeTemplateLockDir();
     const acquireStartedAt = Date.now();
+    const expectedSignature = this.getRuntimeTemplateSignature();
 
     while (true) {
-      if (fs.existsSync(readyDiskPath) && fs.existsSync(readySentinelPath)) {
+      const readyMetadata = readTemplateReadyMetadata(readySentinelPath);
+      if (
+        fs.existsSync(readyDiskPath)
+        && readyMetadata
+        && readyMetadata.signature === expectedSignature
+      ) {
         return readyDiskPath;
       }
 
@@ -698,8 +771,8 @@ class QemuVmManager {
     const templateDiskPath = this.getRuntimeTemplateDiskPath();
     const readyMarkerPath = this.getRuntimeTemplateReadyMarker();
     const readySentinelPath = path.join(templateRoot, '.runtime-template-ready');
+    const templateSignature = this.getRuntimeTemplateSignature();
 
-    fs.rmSync(templateRoot, { recursive: true, force: true });
     fs.mkdirSync(templateRoot, { recursive: true });
 
     const baseImagePath = await this.ensureBaseImageAvailable();
@@ -709,6 +782,8 @@ class QemuVmManager {
       userRoot: templateRoot,
       guestToken,
       guestArch: this.guestArch,
+      runtimeMode: 'template',
+      runtimeProfile: this.runtimeProfile,
     });
     const consoleLogPath = path.join(templateRoot, 'console.log');
     const firmware = this.guestArch === 'arm64'
@@ -736,7 +811,7 @@ class QemuVmManager {
       firmwareVarsPath,
     });
 
-    console.log(`[VM] Building runtime template for ${this.guestArch}: ${qemuBinaryPath} ${args.join(' ')}`);
+    console.log(`[VM:${this.runtimeProfile}] Building runtime template for ${this.guestArch}: ${qemuBinaryPath} ${args.join(' ')}`);
     const child = spawn(qemuBinaryPath, args, {
       cwd: templateRoot,
       detached: process.platform !== 'win32',
@@ -749,7 +824,7 @@ class QemuVmManager {
     });
 
     const baseUrl = `http://127.0.0.1:${agentPort}`;
-    const checkLiveness = () => !child.killed && child.exitCode === null;
+    const checkLiveness = () => isPidAlive(child.pid);
     try {
       await waitForGuestAgentHealth(baseUrl, guestToken, {
         timeoutMs: 30 * 60 * 1000,
@@ -761,7 +836,31 @@ class QemuVmManager {
         intervalMs: 2000,
         checkLiveness,
       });
-      fs.writeFileSync(readySentinelPath, `${new Date().toISOString()}\n`, 'utf8');
+      try {
+        await requestGuestAgent(baseUrl, guestToken, '/exec', {
+          command: [
+            'cloud-init clean --logs --seed --machine-id || true',
+            'rm -rf /var/lib/cloud/instances/* /var/lib/cloud/seed/* || true',
+            'rm -f /var/lib/neoagent/bootstrap-complete /var/lib/neoagent/browser-runtime-ready || true',
+            'rm -f /var/lib/systemd/random-seed || true',
+            'truncate -s 0 /etc/machine-id || true',
+            'sync',
+          ].join('; '),
+          timeout: 120000,
+        }, { timeoutMs: 120000 });
+      } catch (cleanupError) {
+        console.warn(`[VM:${this.runtimeProfile}] Template cleanup after bootstrap failed: ${cleanupError.message}`);
+      }
+      fs.writeFileSync(
+        readySentinelPath,
+        JSON.stringify({
+          signature: templateSignature,
+          runtimeProfile: this.runtimeProfile,
+          guestArch: this.guestArch,
+          builtAt: new Date().toISOString(),
+        }, null, 2),
+        'utf8',
+      );
     } finally {
       try {
         if (process.platform === 'win32') {
@@ -841,6 +940,8 @@ class QemuVmManager {
       userRoot,
       guestToken,
       guestArch: this.guestArch,
+      runtimeMode: 'user',
+      runtimeProfile: this.runtimeProfile,
     });
     const consoleLogPath = path.join(userRoot, 'console.log');
     const firmware = this.guestArch === 'arm64'
@@ -881,7 +982,7 @@ class QemuVmManager {
       firmwareVarsPath,
     });
 
-    console.log(`[VM] Starting QEMU for user ${key} (${this.guestArch}): ${qemuBinaryPath} ${args.join(' ')}`);
+    console.log(`[VM:${this.runtimeProfile}] Starting QEMU for user ${key} (${this.guestArch}): ${qemuBinaryPath} ${args.join(' ')}`);
     const child = spawn(qemuBinaryPath, args, {
       cwd: userRoot,
       detached: process.platform !== 'win32',
@@ -926,6 +1027,7 @@ class QemuVmManager {
 
     const session = {
       userId: key,
+      runtimeProfile: this.runtimeProfile,
       process: child,
       qemuBinary,
       qemuArgs: args,
@@ -953,12 +1055,36 @@ class QemuVmManager {
     if (!session) return;
 
     try {
+      try {
+        await requestGuestAgent(session.baseUrl, session.guestToken, '/browser/close', {}, { timeoutMs: 10000 });
+      } catch {}
+      try {
+        await requestGuestAgent(session.baseUrl, session.guestToken, '/android/stop', {}, { timeoutMs: 10000 });
+      } catch {}
+      try {
+        await requestGuestAgent(session.baseUrl, session.guestToken, '/exec', {
+          command: 'sync || true',
+          timeout: 15000,
+        }, { timeoutMs: 20000 });
+      } catch {}
+
       if (process.platform === 'win32') {
-        spawnSync('taskkill', ['/F', '/T', '/PID', session.process.pid]);
+        spawnSync('taskkill', ['/T', '/PID', session.process.pid]);
       } else {
-        process.kill(-session.process.pid, 'SIGKILL');
+        process.kill(-session.process.pid, 'SIGTERM');
+      }
+      const shutdownStartedAt = Date.now();
+      while (isPidAlive(session.process.pid) && Date.now() - shutdownStartedAt < 10000) {
+        await sleep(250);
+      }
+      if (isPidAlive(session.process.pid)) {
+        if (process.platform === 'win32') {
+          spawnSync('taskkill', ['/F', '/T', '/PID', session.process.pid]);
+        } else {
+          process.kill(-session.process.pid, 'SIGKILL');
+        }
       }
-    } catch (err) {
+    } catch {
       try {
         session.process.kill('SIGKILL');
       } catch {}
@@ -978,7 +1104,6 @@ class QemuVmManager {
 }
 
 module.exports = {
-  BASE_IMAGE_CACHE_ROOT,
   DEFAULT_UBUNTU_BASE_IMAGE_URLS,
   QemuVmManager,
   VM_ROOT,

package/server/services/runtime/settings.js

@@ -19,8 +19,8 @@ function getEffectiveDefaults() {
 }
 
 const BASE_FALLBACK_SETTINGS = Object.freeze({
-  runtime_profile: 'trusted-host',
-  runtime_backend: 'host',
+  runtime_profile: 'secure-vm',
+  runtime_backend: 'vm',
   browser_backend: 'vm',
   android_backend: 'host',
   mcp_backend: 'host-remote',
@@ -36,15 +36,10 @@ function normalizeChoice(value, allowed, fallback) {
 function deriveDefaultsForProfile(profile) {
   switch (profile) {
     case 'secure-vm':
-      return {
-        runtime_backend: 'vm',
-        browser_backend: 'vm',
-        android_backend: 'vm',
-      };
     case 'trusted-host':
     default:
       return {
-        runtime_backend: 'host',
+        runtime_backend: 'vm',
         browser_backend: 'vm',
         android_backend: 'host',
       };
@@ -56,14 +51,14 @@ function normalizeRuntimeSettings(raw = {}) {
   const defaults = getEffectiveDefaults();
   const profile = normalizeChoice(raw.runtime_profile, ['secure-vm', 'trusted-host'], defaults.runtime_profile);
   const derived = deriveDefaultsForProfile(profile);
-  const runtimeBackend = normalizeChoice(raw.runtime_backend, ['host', 'vm'], derived.runtime_backend);
+  const runtimeBackend = normalizeChoice(raw.runtime_backend, ['vm'], derived.runtime_backend);
   const browserBackend = normalizeChoice(raw.browser_backend, ['vm', 'extension'], derived.browser_backend);
   const androidBackend = normalizeChoice(raw.android_backend, ['host', 'vm'], derived.android_backend);
   return {
-    runtime_profile: profile,
-    runtime_backend: policy.allowHostRuntime ? runtimeBackend : (runtimeBackend === 'host' ? 'vm' : runtimeBackend),
+    runtime_profile: profile === 'trusted-host' ? 'secure-vm' : profile,
+    runtime_backend: runtimeBackend,
     browser_backend: browserBackend === 'extension' ? 'extension' : 'vm',
-    android_backend: policy.allowHostRuntime ? androidBackend : (androidBackend === 'host' ? 'vm' : androidBackend),
+    android_backend: androidBackend === 'vm' ? 'host' : androidBackend,
     mcp_backend: 'host-remote',
   };
 }
@@ -107,8 +102,8 @@ function validateRuntimeSettings(raw = {}) {
     if (settings.browser_backend !== 'vm' && settings.browser_backend !== 'extension') {
       issues.push('This deployment requires the VM browser backend or a paired browser extension backend.');
     }
-    if (settings.android_backend !== 'vm') {
-      issues.push('This deployment requires the VM Android backend.');
+    if (settings.android_backend !== 'host') {
+      issues.push('This deployment requires the host Android backend.');
     }
   }
 

package/server/services/runtime/validation.js

@@ -5,20 +5,19 @@ const { getDeploymentPolicy } = require('../../utils/deployment');
 function getRuntimeValidation(runtimeManager) {
   const policy = getDeploymentPolicy();
   const nodeEnvIsProd = String(process.env.NODE_ENV || '').trim().toLowerCase() === 'prod';
-  const vmReadiness = runtimeManager?.vmBackend?.vmManager?.getReadiness?.() || null;
+  const browserVmReadiness = runtimeManager?.browserBackend?.vmManager?.getReadiness?.() || null;
+  const vmReadiness = browserVmReadiness || null;
   const issues = [];
 
   if (policy.profile === 'prod' || nodeEnvIsProd) {
-    if (!vmReadiness?.ready) {
-      if (!vmReadiness) {
-        issues.push('prod profile requires a working local VM runtime.');
-      } else {
-        if (!vmReadiness.qemuAvailable) {
-          issues.push(`prod profile requires QEMU (${vmReadiness.qemuBinary}) to be installed.`);
-        }
-        if (!vmReadiness.baseImageExists && !vmReadiness.downloadConfigured) {
-          issues.push('prod profile requires a VM base image or a downloadable base image URL.');
-        }
+    if (!browserVmReadiness) {
+      issues.push('prod profile requires a working local VM runtime for browser/CLI.');
+    } else if (!browserVmReadiness.ready) {
+      if (!browserVmReadiness.qemuAvailable) {
+        issues.push(`prod profile requires QEMU (${browserVmReadiness.qemuBinary}) to be installed for browser/CLI.`);
+      }
+      if (!browserVmReadiness.baseImageExists && !browserVmReadiness.downloadConfigured) {
+        issues.push('prod profile requires a VM base image or a downloadable base image URL for browser/CLI.');
       }
     }
   }

package/server/utils/deployment.js

@@ -65,13 +65,13 @@ function getDeploymentPolicy(env = process.env) {
     allowSelfUpdate: mode !== DEPLOYMENT_MODE_MANAGED,
     registrationOpen: isProdProfile,
     runtimeDefaults: {
-      runtime_profile: isProdProfile ? 'secure-vm' : 'trusted-host',
-      runtime_backend: isProdProfile ? 'vm' : 'host',
+      runtime_profile: 'secure-vm',
+      runtime_backend: 'vm',
       browser_backend: 'vm',
-      android_backend: isProdProfile ? 'vm' : 'host',
+      android_backend: 'host',
       mcp_backend: 'host-remote',
     },
-    allowHostRuntime: !isProdProfile,
+    allowHostRuntime: false,
   };
 }
 

package/server/guest-agent.package.json

@@ -1,15 +0,0 @@
-{
-  "name": "neoagent-guest-agent",
-  "private": true,
-  "version": "1.0.0",
-  "description": "Minimal guest runtime for NeoAgent VM browser, CLI, and Android services",
-  "engines": {
-    "node": ">=20"
-  },
-  "dependencies": {
-    "express": "^4.21.2",
-    "playwright": "^1.59.1",
-    "proper-lockfile": "^4.1.2",
-    "puppeteer-core": "^24.40.0"
-  }
-}