neoagent 2.3.1-beta.99 → 2.4.1-beta.10

package/server/db/database.js

@@ -614,6 +614,78 @@ db.exec(`
     FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
   );
 
+  CREATE TABLE IF NOT EXISTS memory_ingestion_jobs (
+    id TEXT PRIMARY KEY,
+    user_id INTEGER NOT NULL,
+    agent_id TEXT,
+    source_type TEXT NOT NULL,
+    provider_key TEXT DEFAULT '',
+    connection_id INTEGER,
+    status TEXT DEFAULT 'pending',
+    freshness_policy_json TEXT DEFAULT '{}',
+    cursor_json TEXT DEFAULT '{}',
+    summary_json TEXT DEFAULT '{}',
+    metadata_json TEXT DEFAULT '{}',
+    document_count INTEGER DEFAULT 0,
+    error_text TEXT,
+    started_at TEXT DEFAULT (datetime('now')),
+    completed_at TEXT,
+    next_sync_at TEXT,
+    created_at TEXT DEFAULT (datetime('now')),
+    updated_at TEXT DEFAULT (datetime('now')),
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL,
+    FOREIGN KEY (connection_id) REFERENCES integration_connections(id) ON DELETE SET NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS memory_ingestion_documents (
+    id TEXT PRIMARY KEY,
+    user_id INTEGER NOT NULL,
+    agent_id TEXT,
+    source_type TEXT NOT NULL,
+    normalized_type TEXT NOT NULL,
+    provider_key TEXT DEFAULT '',
+    connection_id INTEGER NOT NULL DEFAULT 0,
+    external_object_id TEXT NOT NULL,
+    source_account TEXT,
+    title TEXT,
+    content TEXT NOT NULL,
+    summary TEXT,
+    salience INTEGER DEFAULT 5,
+    source_timestamp TEXT,
+    metadata_json TEXT DEFAULT '{}',
+    payload_json TEXT DEFAULT '{}',
+    created_at TEXT DEFAULT (datetime('now')),
+    updated_at TEXT DEFAULT (datetime('now')),
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL,
+    UNIQUE(user_id, agent_id, source_type, provider_key, connection_id, external_object_id)
+  );
+
+  CREATE TABLE IF NOT EXISTS materialized_knowledge_views (
+    id TEXT PRIMARY KEY,
+    user_id INTEGER NOT NULL,
+    agent_id TEXT,
+    view_type TEXT NOT NULL,
+    subject_key TEXT NOT NULL,
+    title TEXT NOT NULL,
+    summary TEXT,
+    markdown_text TEXT,
+    source_memory_ids_json TEXT DEFAULT '[]',
+    source_document_ids_json TEXT DEFAULT '[]',
+    metadata_json TEXT DEFAULT '{}',
+    created_at TEXT DEFAULT (datetime('now')),
+    updated_at TEXT DEFAULT (datetime('now')),
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL,
+    UNIQUE(user_id, agent_id, view_type, subject_key)
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_memory_ingestion_jobs_user ON memory_ingestion_jobs(user_id, agent_id, source_type, updated_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_memory_ingestion_documents_user ON memory_ingestion_documents(user_id, agent_id, source_type, updated_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_memory_ingestion_documents_provider ON memory_ingestion_documents(user_id, agent_id, provider_key, connection_id, updated_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_materialized_knowledge_views_user ON materialized_knowledge_views(user_id, agent_id, view_type, updated_at DESC);
+
   CREATE TABLE IF NOT EXISTS agent_run_events (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     run_id TEXT NOT NULL,
@@ -1698,6 +1770,19 @@ function migrateUsersDisplayName() {
 }
 migrateUsersDisplayName();
 
+function migrateIngestionDocumentsConnectionId() {
+  // connection_id was initially nullable; NULL breaks the UNIQUE dedup constraint.
+  // Coerce any existing NULLs to 0 so the unique index works as intended.
+  try {
+    db.prepare(
+      `UPDATE memory_ingestion_documents SET connection_id = 0 WHERE connection_id IS NULL`
+    ).run();
+  } catch {
+    // Table may not exist yet on first boot; the schema DDL handles that case.
+  }
+}
+migrateIngestionDocumentsConnectionId();
+
 try {
   db.exec(`
     INSERT OR REPLACE INTO conversation_history_fts(rowid, content, role, user_id, agent_id, agent_run_id)

package/server/guest_agent.js

@@ -62,7 +62,18 @@ function isInsideAllowedRoots(targetPath) {
 }
 
 function requireToken(req, res, next) {
-  next();
+  if (!AUTH_TOKEN) {
+    // Token not configured in this environment — allow but unauthenticated.
+    // Pass NEOAGENT_VM_GUEST_TOKEN to the container to enforce auth.
+    return next();
+  }
+  const header = String(req.headers?.authorization || '').trim();
+  const prefix = 'Bearer ';
+  const provided = header.startsWith(prefix) ? header.slice(prefix.length).trim() : '';
+  if (!provided || provided !== AUTH_TOKEN) {
+    return res.status(401).json({ error: 'Unauthorized.' });
+  }
+  return next();
 }
 
 function sanitizeError(err) {
@@ -184,6 +195,13 @@ function requireCapability(controller, name) {
 app.post('/browser/launch', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').launch(req.body || {})));
 app.post('/browser/navigate', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').navigate(req.body?.url, req.body || {})));
 app.post('/browser/screenshot', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').screenshot(req.body || {})));
+app.post('/browser/screenshot-jpeg', async (req, res) => handle(res, async () => {
+  const jpeg = await requireCapability(browserController, 'browser').screenshotJpeg(req.body?.quality, req.body || {});
+  return {
+    contentType: 'image/jpeg',
+    contentBase64: Buffer.from(jpeg).toString('base64'),
+  };
+}));
 app.post('/browser/click', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
 app.post('/browser/click-point', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
 app.post('/browser/fill', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));

package/server/http/routes.js

@@ -25,6 +25,7 @@ const routeRegistry = [
   { basePath: '/api/browser-extension', modulePath: '../routes/browser_extension' },
   { basePath: '/api/android', modulePath: '../routes/android' },
   { basePath: '/api/desktop', modulePath: '../routes/desktop' },
+  { basePath: '/api/stream', modulePath: '../routes/stream' },
   { basePath: '/api/recordings', modulePath: '../routes/recordings' },
   { basePath: '/api/social-video', modulePath: '../routes/social_video' },
   { basePath: '/api/voice-assistant', modulePath: '../routes/voice_assistant' },
@@ -67,6 +68,196 @@ function registerApiRoutes(app) {
     });
   });
 
+  app.get('/api/system/health-check', requireAuth, async (req, res) => {
+    const userId = req.session?.userId;
+    const runtimeManager = req.app?.locals?.runtimeManager;
+    const desktopRegistry = req.app?.locals?.desktopCompanionRegistry;
+    const extensionRegistry = req.app?.locals?.browserExtensionRegistry;
+    const results = [];
+
+    // 1. Backend connectivity — trivially true if we got here.
+    results.push({ id: 'backend', label: 'Backend server', passed: true, detail: 'Reachable' });
+
+    // 2. Cloud VM runtime availability.
+    const runtimeValidation = getRuntimeValidation(runtimeManager);
+    const runtimeReady = Boolean(runtimeValidation?.ready);
+    results.push({
+      id: 'vm_runtime',
+      label: 'Cloud VM runtime',
+      passed: runtimeReady,
+      detail: runtimeReady ? 'Available' : String(runtimeValidation?.issues?.[0] || 'Not configured'),
+    });
+
+    // 3. Cloud VM CLI execution — actually run a command.
+    if (runtimeManager && typeof runtimeManager.executeCommand === 'function') {
+      try {
+        const cmdResult = await runtimeManager.executeCommand(userId, 'echo "health_check_ok"', { timeout: 15000 });
+        const exitOk = cmdResult?.exitCode === 0;
+        const outputOk = String(cmdResult?.stdout || '').includes('health_check_ok');
+        results.push({
+          id: 'vm_cli',
+          label: 'Cloud VM — command execution',
+          passed: exitOk && outputOk,
+          detail: exitOk && outputOk
+            ? 'Commands running'
+            : `Exit ${cmdResult?.exitCode ?? '?'}: ${String(cmdResult?.stderr || cmdResult?.stdout || '').slice(0, 120)}`,
+        });
+      } catch (err) {
+        results.push({ id: 'vm_cli', label: 'Cloud VM — command execution', passed: false, detail: String(err?.message || err).slice(0, 120) });
+      }
+    } else {
+      results.push({ id: 'vm_cli', label: 'Cloud VM — command execution', passed: false, detail: 'VM runtime unavailable' });
+    }
+
+    // 4. Desktop companion (macOS app / remote device) connectivity + permissions.
+    if (desktopRegistry) {
+      try {
+        const desktopStatus = desktopRegistry.getStatus(userId);
+        const connected = Boolean(desktopStatus?.connected);
+        results.push({
+          id: 'desktop_connected',
+          label: 'Desktop companion',
+          passed: connected,
+          detail: connected
+            ? `${desktopStatus.onlineCount} device${desktopStatus.onlineCount !== 1 ? 's' : ''} connected`
+            : 'No device connected — open the desktop app',
+        });
+
+        if (connected && Array.isArray(desktopStatus?.devices)) {
+          const onlineDevice = desktopStatus.devices.find((d) => d.online && !d.revokedAt);
+          const perms = onlineDevice?.permissions || {};
+          const screenOk = Boolean(perms.screenCapture || perms.screen_capture);
+          const inputOk = Boolean(perms.accessibility || perms.inputControl || perms.input_control);
+          results.push({
+            id: 'desktop_screen',
+            label: 'Desktop — screen capture',
+            passed: screenOk,
+            detail: screenOk ? 'Granted' : 'Not granted — open System Settings › Privacy › Screen Recording',
+          });
+          results.push({
+            id: 'desktop_input',
+            label: 'Desktop — input control',
+            passed: inputOk,
+            detail: inputOk ? 'Granted' : 'Not granted — open System Settings › Privacy › Accessibility',
+          });
+        }
+      } catch (err) {
+        results.push({ id: 'desktop_connected', label: 'Desktop companion', passed: false, detail: String(err?.message || err).slice(0, 120) });
+      }
+    }
+
+    // 5. Chrome extension connectivity.
+    if (extensionRegistry) {
+      try {
+        const extStatus = extensionRegistry.getStatus(userId);
+        const extConnected = Boolean(extStatus?.connected);
+        results.push({
+          id: 'chrome_extension',
+          label: 'Chrome extension',
+          passed: extConnected,
+          detail: extConnected ? 'Connected' : 'Not connected — install the NeoAgent extension in Chrome',
+        });
+      } catch (err) {
+        results.push({ id: 'chrome_extension', label: 'Chrome extension', passed: false, detail: String(err?.message || err).slice(0, 120) });
+      }
+    }
+
+    const allPassed = results.every((r) => r.passed);
+    res.json({ passed: allPassed, results });
+  });
+
+  // Targeted runtime self-tests — one check per endpoint so the UI can embed
+  // results inline next to the relevant settings control.
+
+  app.get('/api/system/test/cli', requireAuth, async (req, res) => {
+    const userId = req.session?.userId;
+    const runtimeManager = req.app?.locals?.runtimeManager;
+    if (!runtimeManager || typeof runtimeManager.executeCommand !== 'function') {
+      return res.json({ passed: false, backendUsed: 'vm', detail: 'Runtime not configured on this server.' });
+    }
+    // Note: executeCommand always routes through the VM backend regardless of
+    // the cli_backend setting — desktop CLI routing is not yet implemented.
+    try {
+      const result = await runtimeManager.executeCommand(userId, 'echo "cli_test_ok"', { timeout: 15000 });
+      const exitOk = result?.exitCode === 0;
+      const outputOk = String(result?.stdout || '').includes('cli_test_ok');
+      return res.json({
+        passed: exitOk && outputOk,
+        backendUsed: 'vm',
+        detail: exitOk && outputOk
+          ? 'Command executed successfully'
+          : `Exit ${result?.exitCode ?? '?'}: ${String(result?.stderr || result?.stdout || '').slice(0, 120)}`,
+      });
+    } catch (err) {
+      return res.json({ passed: false, backendUsed: 'vm', detail: String(err?.message || err).slice(0, 120) });
+    }
+  });
+
+  app.get('/api/system/test/extension', requireAuth, (req, res) => {
+    const userId = req.session?.userId;
+    const extensionRegistry = req.app?.locals?.browserExtensionRegistry;
+    if (!extensionRegistry) {
+      return res.json({ passed: false, detail: 'Extension registry not available on this server.' });
+    }
+    try {
+      const status = extensionRegistry.getStatus(userId);
+      const connected = Boolean(status?.connected);
+      return res.json({
+        passed: connected,
+        detail: connected ? 'Extension is connected and live' : 'Extension is not connected',
+        tokenId: status?.activeTokenId || null,
+        meta: status?.connectedMeta || null,
+      });
+    } catch (err) {
+      return res.json({ passed: false, detail: String(err?.message || err).slice(0, 120) });
+    }
+  });
+
+  app.get('/api/system/test/desktop', requireAuth, (req, res) => {
+    const userId = req.session?.userId;
+    const desktopRegistry = req.app?.locals?.desktopCompanionRegistry;
+    if (!desktopRegistry) {
+      return res.json({ passed: false, detail: 'Desktop registry not available on this server.' });
+    }
+    try {
+      const status = desktopRegistry.getStatus(userId);
+      const connected = Boolean(status?.connected);
+      const devices = Array.isArray(status?.devices)
+        ? status.devices.filter((d) => d.online && !d.revokedAt)
+        : [];
+      const selected = status?.selectedDeviceId || null;
+      const activeDevice = selected
+        ? devices.find((d) => d.deviceId === selected)
+        : devices.length === 1 ? devices[0] : null;
+      const perms = activeDevice?.permissions || {};
+      const screenOk = Boolean(perms.screenCapture || perms.screen_capture);
+      const inputOk = Boolean(perms.accessibility || perms.inputControl || perms.input_control);
+      return res.json({
+        passed: connected,
+        connected,
+        onlineCount: devices.length,
+        selectedDeviceId: selected,
+        activeDevice: activeDevice ? {
+          deviceId: activeDevice.deviceId,
+          label: activeDevice.label || activeDevice.hostname || activeDevice.deviceId,
+          platform: activeDevice.platform || null,
+          paused: activeDevice.paused || false,
+          permissions: { screenCapture: screenOk, inputControl: inputOk },
+        } : null,
+        multipleOnline: devices.length > 1 && !activeDevice,
+        detail: !connected
+          ? 'No device connected'
+          : devices.length > 1 && !activeDevice
+            ? `${devices.length} devices online — select one in Desktop settings`
+            : activeDevice?.paused
+              ? `${activeDevice.label || 'Device'} is paused`
+              : `${activeDevice?.label || 'Device'} connected`,
+      });
+    } catch (err) {
+      return res.json({ passed: false, detail: String(err?.message || err).slice(0, 120) });
+    }
+  });
+
   app.get('/api/version', requireAuth, (req, res) => {
     res.json(getVersionInfo());
   });

package/server/http/socket.js

@@ -8,7 +8,7 @@ function createSocketServer(httpServer, { validateOrigin }) {
     pingInterval: Number(process.env.NEOAGENT_SOCKET_PING_INTERVAL_MS || 25000),
     pingTimeout: Number(process.env.NEOAGENT_SOCKET_PING_TIMEOUT_MS || 20000),
     connectTimeout: Number(process.env.NEOAGENT_SOCKET_CONNECT_TIMEOUT_MS || 10000),
-    maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES || 1_000_000),
+    maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES || 8 * 1024 * 1024),
     cors: {
       origin(origin, callback) {
         return validateOrigin(origin, callback, { allowMissingOrigin: true });

package/server/index.js

@@ -34,6 +34,7 @@ const { startServices, stopServices } = require('./services/manager');
 const { bindBrowserExtensionGateway } = require('./services/browser/extension/gateway');
 const { bindDesktopCompanionGateway } = require('./services/desktop/gateway');
 const { bindWearableGateway } = require('./services/wearable/gateway');
+const { StreamHub } = require('./services/streaming/stream-hub');
 
 function parseBooleanFlag(value, fallback = false) {
   const normalized = String(value || '').trim().toLowerCase();
@@ -89,6 +90,8 @@ const app = express();
 app.disable('x-powered-by');
 const httpServer = createServer(app);
 const io = createSocketServer(httpServer, { validateOrigin });
+const streamHub = new StreamHub(io);
+app.locals.streamHub = streamHub;
 app.locals.httpRuntimeConfig = {
   secureCookies: SECURE_COOKIES,
   trustProxy: TRUST_PROXY,
@@ -112,7 +115,7 @@ registerApiRoutes(app);
 registerStaticRoutes(app);
 registerErrorHandler(app);
 bindBrowserExtensionGateway(httpServer, app);
-bindDesktopCompanionGateway(httpServer, app, sessionMiddleware);
+bindDesktopCompanionGateway(httpServer, app, sessionMiddleware, streamHub);
 bindWearableGateway(httpServer, app, sessionMiddleware);
 
 let shuttingDown = false;

package/server/public/.last_build_id

@@ -1 +1 @@
-1f7fbbbdae35e424bcbd5adf22601df0
+523e49e35d448c4b94454f78c7beebb8