neoagent 2.3.1-beta.11 → 2.3.1-beta.111

package/server/services/ai/capabilityHealth.js

@@ -1,7 +1,5 @@
 const db = require('../../db/database');
 const { getProviderHealthCatalog } = require('./models');
-const { resolveBrowserExecutablePath } = require('../browser/controller');
-const { deriveCloudBrowserBackend } = require('../runtime/settings');
 
 function capabilityEntry(overrides = {}) {
   return {
@@ -38,7 +36,6 @@ function summarizeCapabilityHealth(health) {
 
 async function getBrowserHealth(userId, app, engine) {
   const runtimeManager = app?.locals?.runtimeManager || engine?.runtimeManager || null;
-  const executablePath = resolveBrowserExecutablePath();
   const runtimeSettings = typeof runtimeManager?.getSettings === 'function'
     ? runtimeManager.getSettings(userId)
     : null;
@@ -78,26 +75,16 @@ async function getBrowserHealth(userId, app, engine) {
   }
 
   if (!controller) {
-    try {
-      const resolver = app?.locals?.getBrowserControllerForUser;
-      controller = await Promise.resolve(
-        typeof resolver === 'function'
-          ? resolver(userId)
-          : (app?.locals?.browserController || engine?.browserController || null)
-      );
-    } catch (err) {
-      resolutionError = resolutionError || err;
-    }
+    resolutionError = resolutionError || new Error('Browser provider is unavailable. VM runtime is required.');
   }
 
   if (!controller && resolutionError) {
     return capabilityEntry({
-      configured: Boolean(executablePath),
+      configured: true,
       healthy: false,
       degraded: true,
       summary: `Browser controller resolution failed: ${resolutionError.message}`,
       details: {
-        executablePath: executablePath || null,
         error: resolutionError.message,
       },
     });
@@ -117,23 +104,18 @@ async function getBrowserHealth(userId, app, engine) {
 
   return capabilityEntry({
     connected: launched,
-    configured: Boolean(executablePath),
-    healthy: Boolean(executablePath) && !error,
+    configured: true,
+    healthy: !error,
     degraded: Boolean(error) || runtimeSettings?.browser_backend === 'extension',
     summary: error
       ? `Browser runtime error: ${error}`
       : runtimeSettings?.browser_backend === 'extension'
-        ? executablePath
-          ? `No extension device is active. Falling back to the ${deriveCloudBrowserBackend(runtimeSettings)} browser runtime.`
-          : 'No extension device is active and no browser executable was found for Puppeteer.'
-        : executablePath
-          ? (launched ? 'Browser runtime is ready.' : 'Browser executable is available but not launched.')
-          : 'No browser executable was found for Puppeteer.',
+        ? `No extension device is active. Falling back to the vm browser runtime.`
+        : (launched ? 'Browser runtime is ready.' : 'Browser runtime is available but not launched.'),
     details: {
-      executablePath: executablePath || null,
       preferredBackend: runtimeSettings?.browser_backend || null,
       backend: runtimeSettings?.browser_backend === 'extension'
-        ? deriveCloudBrowserBackend(runtimeSettings)
+        ? 'vm'
         : runtimeSettings?.browser_backend || null,
       extensionConnected: extensionStatus?.connected === true,
       activeTokenCount: Array.isArray(extensionStatus?.tokens)
@@ -159,17 +141,7 @@ async function getAndroidHealth(userId, app, engine) {
   }
 
   if (!controller) {
-    try {
-      const resolver = app?.locals?.getAndroidControllerForUser;
-      controller = await Promise.resolve(
-        typeof resolver === 'function'
-          ? resolver(userId)
-          : (app?.locals?.androidController || engine?.androidController || null)
-      );
-    } catch (err) {
-      resolutionError = resolutionError || err;
-      controller = null;
-    }
+    resolutionError = resolutionError || new Error('Android provider is unavailable. VM runtime is required.');
   }
 
   if (!controller || typeof controller.getStatus !== 'function') {
@@ -194,7 +166,7 @@ async function getAndroidHealth(userId, app, engine) {
       summary: status.lastStartError
         ? `Android tooling reported: ${status.lastStartError}`
         : bootstrapped
-          ? 'Android environment is bootstrapped.'
+          ? 'Android environment is ready on this host.'
           : (canBootstrap ? 'Android environment can be bootstrapped on this host.' : 'Android tooling cannot bootstrap on this host.'),
       details: status,
     });
@@ -319,27 +291,27 @@ function getSkillHealth(app, engine) {
   });
 }
 
-function getFileHealth() {
+function getFileHealth(app, engine) {
+  const workspaceManager = app?.locals?.workspaceManager || engine?.workspaceManager || null;
   return capabilityEntry({
-    connected: true,
-    configured: true,
-    healthy: true,
-    summary: 'Filesystem access is available.',
+    connected: Boolean(workspaceManager),
+    configured: Boolean(workspaceManager),
+    healthy: Boolean(workspaceManager),
+    summary: workspaceManager
+      ? 'Per-user workspace access is available.'
+      : 'Per-user workspace service is not available.',
   });
 }
 
 function getCommandHealth(userId, app, engine) {
   const runtimeManager = app?.locals?.runtimeManager || engine?.runtimeManager || null;
-  const backend = runtimeManager?.getSettings?.(userId)?.runtime_backend || 'host';
   return capabilityEntry({
-    connected: Boolean(runtimeManager || engine?.cliExecutor),
-    configured: true,
-    healthy: Boolean(runtimeManager || engine?.cliExecutor),
+    connected: Boolean(runtimeManager),
+    configured: Boolean(runtimeManager),
+    healthy: Boolean(runtimeManager),
     summary: runtimeManager
-      ? `Shell command execution is available through the ${backend} runtime backend.`
-      : engine?.cliExecutor
-        ? 'Shell command execution is available.'
-        : 'Shell executor is not available.',
+      ? 'Shell command execution is available.'
+      : 'Shell command execution is not available in this environment.',
   });
 }
 
@@ -376,7 +348,7 @@ async function getCapabilityHealth({ userId, agentId = null, app, engine }) {
     providers,
     capabilities: {
       command: getCommandHealth(userId, app, engine),
-      files: getFileHealth(),
+      files: getFileHealth(app, engine),
       memory: getMemoryHealth(engine),
       search: getSearchHealth(),
       browser: await getBrowserHealth(userId, app, engine),

package/server/services/ai/compaction.js

@@ -21,7 +21,7 @@ async function compact(messages, provider, model, contextWindow = null) {
   }).join('\n');
 
   const summaryPrompt = [
-    { role: 'system', content: 'Compress conversation context. Preserve goals, constraints, decisions, promised follow-ups, recurring tasks, tool outcomes, errors, and unresolved work. Keep concrete facts (dates/times/names/status) and avoid vague wording.' },
+    { role: 'system', content: 'Compress this conversation into a dense context block. Preserve: active goals and constraints, decisions made, promised actions (sent/created/changed/deleted), tool outcomes and errors, unresolved blockers, task configs, and concrete facts (names, IDs, dates, statuses, file paths). Omit greetings, filler, and tool-call narration. Write in past tense. Be specific — "email sent to alice@example.com at 3pm" beats "a message was sent".' },
     { role: 'user', content: `Summarize this conversation:\n\n${compactionText}` }
   ];
 

package/server/services/ai/deliverables/artifact_helpers.js

@@ -0,0 +1,190 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { normalizeArtifactContract } = require('./contracts');
+
+const FILE_EXTENSION_TO_KIND = {
+  '.ppt': 'slides',
+  '.pptx': 'slides',
+  '.key': 'slides',
+  '.pdf': 'document',
+  '.doc': 'document',
+  '.docx': 'document',
+  '.md': 'document',
+  '.txt': 'document',
+  '.html': 'document',
+  '.htm': 'document',
+  '.csv': 'data',
+  '.tsv': 'data',
+  '.xlsx': 'data',
+  '.xls': 'data',
+  '.json': 'data',
+  '.png': 'image',
+  '.jpg': 'image',
+  '.jpeg': 'image',
+  '.gif': 'image',
+  '.webp': 'image',
+  '.svg': 'image',
+  '.mp4': 'video',
+  '.mov': 'video',
+  '.m4v': 'video',
+  '.webm': 'video',
+};
+
+const FILE_EXTENSION_TO_MIME = {
+  '.ppt': 'application/vnd.ms-powerpoint',
+  '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+  '.pdf': 'application/pdf',
+  '.doc': 'application/msword',
+  '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  '.md': 'text/markdown',
+  '.txt': 'text/plain',
+  '.html': 'text/html',
+  '.htm': 'text/html',
+  '.csv': 'text/csv',
+  '.tsv': 'text/tab-separated-values',
+  '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  '.xls': 'application/vnd.ms-excel',
+  '.json': 'application/json',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.webp': 'image/webp',
+  '.svg': 'image/svg+xml',
+  '.mp4': 'video/mp4',
+  '.mov': 'video/quicktime',
+  '.m4v': 'video/x-m4v',
+  '.webm': 'video/webm',
+};
+
+const CANDIDATE_KEYS = [
+  'path',
+  'paths',
+  'file',
+  'files',
+  'filePath',
+  'filePaths',
+  'fullPath',
+  'fullPaths',
+  'mediaPath',
+  'mediaPaths',
+  'screenshotPath',
+  'uiDumpPath',
+  'url',
+  'urls',
+];
+
+function inferExtension(candidate = '') {
+  return path.extname(String(candidate || '').split('?')[0]).toLowerCase();
+}
+
+function inferArtifactKind(candidate = '', fallback = 'artifact') {
+  const extension = inferExtension(candidate);
+  if (FILE_EXTENSION_TO_KIND[extension]) return FILE_EXTENSION_TO_KIND[extension];
+  const normalized = String(candidate || '').toLowerCase();
+  if (normalized.includes('image')) return 'image';
+  if (normalized.includes('video')) return 'video';
+  if (normalized.includes('slide') || normalized.includes('ppt')) return 'slides';
+  if (normalized.includes('doc') || normalized.includes('pdf')) return 'document';
+  if (normalized.includes('data') || normalized.includes('chart') || normalized.includes('csv')) return 'data';
+  return fallback;
+}
+
+function inferMimeType(candidate = '') {
+  const extension = inferExtension(candidate);
+  return FILE_EXTENSION_TO_MIME[extension] || null;
+}
+
+function normalizePathOrUri(value) {
+  const text = String(value || '').trim();
+  if (!text) return null;
+  if (text.startsWith('/api/artifacts/')) return { uri: text, path: null };
+  if (/^https?:\/\//i.test(text)) return { uri: text, path: null };
+  if (/^[A-Za-z]:\\/.test(text)) return { path: text, uri: null };
+  if (path.isAbsolute(text)) return { path: text, uri: null };
+  return null;
+}
+
+async function buildArtifactFromCandidate(candidate, fallbackKind = 'artifact') {
+  const normalized = normalizePathOrUri(candidate);
+  if (!normalized) return null;
+  const source = normalized.path || normalized.uri || '';
+  const artifact = normalizeArtifactContract({
+    kind: inferArtifactKind(source, fallbackKind),
+    path: normalized.path,
+    uri: normalized.uri,
+    label: path.basename(String(source).split('?')[0]) || null,
+    mimeType: inferMimeType(source),
+  });
+  if (artifact.path) {
+    try {
+      artifact.size = (await fs.promises.stat(artifact.path)).size;
+    } catch (error) {
+      console.warn('[deliverables] Failed to stat artifact candidate:', artifact.path, error?.message || error);
+    }
+  }
+  return artifact.path || artifact.uri ? artifact : null;
+}
+
+function scanStringForCandidates(text) {
+  const input = String(text || '');
+  const matches = [];
+  const regexes = [
+    /\/api\/artifacts\/[A-Za-z0-9%_-]+\/content/g,
+    /\/[^\s"'`]+?\.(?:pptx?|pdf|docx?|md|txt|html?|csv|tsv|xlsx?|json|png|jpe?g|gif|webp|svg|mp4|mov|m4v|webm)\b/g,
+    /[A-Za-z]:\\[^\s"'`]+?\.(?:pptx?|pdf|docx?|md|txt|html?|csv|tsv|xlsx?|json|png|jpe?g|gif|webp|svg|mp4|mov|m4v|webm)\b/g,
+    /https?:\/\/[^\s"'`]+?\.(?:pptx?|pdf|docx?|md|txt|html?|csv|tsv|xlsx?|json|png|jpe?g|gif|webp|svg|mp4|mov|m4v|webm)\b/g,
+  ];
+  for (const regex of regexes) {
+    const found = input.match(regex);
+    if (found) matches.push(...found);
+  }
+  return matches;
+}
+
+async function extractArtifactsFromResult(toolName, result) {
+  const artifacts = [];
+  const seen = new Set();
+  const fallbackKind = inferArtifactKind(toolName, 'artifact');
+
+  async function pushCandidate(candidate) {
+    const artifact = await buildArtifactFromCandidate(candidate, fallbackKind);
+    if (!artifact) return;
+    const key = `${artifact.kind}:${artifact.path || artifact.uri}`;
+    if (seen.has(key)) return;
+    seen.add(key);
+    artifacts.push(artifact);
+  }
+
+  async function visit(value, keyHint = '') {
+    if (value == null) return;
+    if (typeof value === 'string') {
+      if (CANDIDATE_KEYS.includes(keyHint)) await pushCandidate(value);
+      for (const candidate of scanStringForCandidates(value)) {
+        await pushCandidate(candidate);
+      }
+      return;
+    }
+    if (Array.isArray(value)) {
+      for (const item of value) await visit(item, keyHint);
+      return;
+    }
+    if (typeof value === 'object') {
+      for (const [key, nested] of Object.entries(value)) {
+        await visit(nested, key);
+      }
+    }
+  }
+
+  await visit(result);
+  return artifacts;
+}
+
+module.exports = {
+  extractArtifactsFromResult,
+  inferArtifactKind,
+  inferMimeType,
+  normalizePathOrUri,
+};

package/server/services/ai/deliverables/contracts.js

@@ -0,0 +1,113 @@
+'use strict';
+
+const DELIVERABLE_TYPES = [
+  'slides',
+  'document',
+  'research_report',
+  'data_analysis',
+  'image',
+  'video',
+];
+
+const DELIVERABLE_SELECTION_STATUSES = ['selected', 'standard'];
+const DELIVERABLE_VALIDATION_STATUSES = ['passed', 'failed'];
+
+function clampText(value, maxLength = 240) {
+  return String(value || '').trim().slice(0, maxLength);
+}
+
+function normalizeDeliverableType(value) {
+  const normalized = clampText(value, 48).toLowerCase();
+  return DELIVERABLE_TYPES.includes(normalized) ? normalized : null;
+}
+
+function normalizeStringList(value, limit = 8, maxLength = 120) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .map((item) => clampText(item, maxLength))
+    .filter(Boolean)
+    .slice(0, limit);
+}
+
+function normalizeArtifactContract(raw = {}) {
+  const source = raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {};
+  return {
+    kind: clampText(source.kind, 48) || 'artifact',
+    path: clampText(source.path || source.fullPath, 500) || null,
+    uri: clampText(source.uri || source.url, 500) || null,
+    label: clampText(source.label, 120) || null,
+    mimeType: clampText(source.mimeType || source.contentType, 120) || null,
+    size: Number.isFinite(Number(source.size ?? source.byteSize))
+      ? Number(source.size ?? source.byteSize)
+      : null,
+    preview: source.preview && typeof source.preview === 'object' && !Array.isArray(source.preview)
+      ? { ...source.preview }
+      : {},
+  };
+}
+
+function normalizeDeliverableSelection(raw = {}) {
+  const source = raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {};
+  const type = normalizeDeliverableType(source.type);
+  const confidence = Math.max(0, Math.min(1, Number(source.confidence) || 0));
+  const status = DELIVERABLE_SELECTION_STATUSES.includes(String(source.status || '').trim().toLowerCase())
+    ? String(source.status).trim().toLowerCase()
+    : (type ? 'selected' : 'standard');
+
+  return {
+    status,
+    type,
+    confidence,
+    goal: clampText(source.goal, 240),
+    requestedOutputs: normalizeStringList(source.requested_outputs || source.requestedOutputs, 8, 120),
+    supportingCapabilities: normalizeStringList(source.supporting_capabilities || source.supportingCapabilities, 8, 64),
+  };
+}
+
+function normalizeDeliverableValidationResult(raw = {}) {
+  const source = raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {};
+  const status = DELIVERABLE_VALIDATION_STATUSES.includes(String(source.status || '').trim().toLowerCase())
+    ? String(source.status).trim().toLowerCase()
+    : 'failed';
+  const artifacts = Array.isArray(source.artifacts)
+    ? source.artifacts.map(normalizeArtifactContract).filter((artifact) => artifact.path || artifact.uri)
+    : [];
+
+  return {
+    status,
+    summary: clampText(source.summary, 320),
+    errors: normalizeStringList(source.errors, 8, 220),
+    warnings: normalizeStringList(source.warnings, 8, 220),
+    artifacts,
+    metrics: source.metrics && typeof source.metrics === 'object' && !Array.isArray(source.metrics)
+      ? { ...source.metrics }
+      : {},
+  };
+}
+
+function normalizeDeliverableResult(raw = {}) {
+  const source = raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {};
+  return {
+    type: normalizeDeliverableType(source.type),
+    status: clampText(source.status, 32) || 'unknown',
+    summary: clampText(source.summary, 320),
+    artifacts: Array.isArray(source.artifacts)
+      ? source.artifacts.map(normalizeArtifactContract).filter((artifact) => artifact.path || artifact.uri)
+      : [],
+    validation: normalizeDeliverableValidationResult(source.validation),
+    metadata: source.metadata && typeof source.metadata === 'object' && !Array.isArray(source.metadata)
+      ? { ...source.metadata }
+      : {},
+  };
+}
+
+module.exports = {
+  DELIVERABLE_TYPES,
+  clampText,
+  normalizeArtifactContract,
+  normalizeDeliverableResult,
+  normalizeDeliverableSelection,
+  normalizeDeliverableType,
+  normalizeDeliverableValidationResult,
+  normalizeStringList,
+};

package/server/services/ai/deliverables/index.js

@@ -0,0 +1,20 @@
+'use strict';
+
+const { selectDeliverableWorkflow } = require('./selector');
+const {
+  buildDeliverableWorkflowGuidance,
+  getDeliverableWorkflow,
+  listDeliverableWorkflows,
+} = require('./workflows');
+const { extractArtifactsFromResult } = require('./artifact_helpers');
+const { DeliverableValidationError, validateDeliverableExecution } = require('./validator');
+
+module.exports = {
+  buildDeliverableWorkflowGuidance,
+  DeliverableValidationError,
+  extractArtifactsFromResult,
+  getDeliverableWorkflow,
+  listDeliverableWorkflows,
+  selectDeliverableWorkflow,
+  validateDeliverableExecution,
+};

package/server/services/ai/deliverables/selector.js

@@ -0,0 +1,94 @@
+'use strict';
+
+const {
+  clampText,
+  normalizeDeliverableSelection,
+} = require('./contracts');
+const { listDeliverableWorkflows } = require('./workflows');
+
+function buildDeliverableSelectorPrompt({ tools = [] }) {
+  const workflows = listDeliverableWorkflows().map((workflow) => ({
+    type: workflow.type,
+    displayName: workflow.displayName,
+    expectedOutputs: workflow.expectedOutputs,
+  }));
+  const toolNames = tools
+    .map((tool) => String(tool?.name || '').trim())
+    .filter(Boolean)
+    .slice(0, 40);
+
+  return [
+    'Return JSON only.',
+    'Classify whether this request should use a deliverable workflow.',
+    'Choose exactly one type from: slides, document, research_report, data_analysis, image, video.',
+    'Use status="selected" only when a concrete artifact-producing workflow clearly owns the primary outcome.',
+    'Use status="standard" when the request is conversational, operational, ambiguous, or not primarily about producing one of these deliverables.',
+    'Be conservative. If confidence is low, choose status="standard".',
+    'Schema:',
+    JSON.stringify({
+      status: 'selected',
+      type: 'document',
+      confidence: 0.82,
+      goal: 'Create a polished launch memo.',
+      requested_outputs: ['launch memo pdf'],
+      supporting_capabilities: ['web_search'],
+    }, null, 2),
+    `Available workflows:\n${JSON.stringify(workflows, null, 2)}`,
+    toolNames.length ? `Available tools/capabilities: ${toolNames.join(', ')}` : '',
+  ].filter(Boolean).join('\n\n');
+}
+
+async function selectDeliverableWorkflow({
+  engine,
+  provider,
+  providerName,
+  model,
+  messages,
+  tools = [],
+  options = {},
+}) {
+  const response = await engine.requestStructuredJson({
+    provider,
+    providerName,
+    model,
+    messages,
+    prompt: buildDeliverableSelectorPrompt({ tools }),
+    maxTokens: 600,
+    normalize: normalizeDeliverableSelection,
+    fallback: { status: 'standard', confidence: 0 },
+    reasoningEffort: engine.getReasoningEffort(providerName, options),
+  });
+
+  const selection = normalizeDeliverableSelection(response.value);
+  if (
+    selection.status !== 'selected'
+    || !selection.type
+    || Number(selection.confidence || 0) < 0.7
+  ) {
+    return {
+      selection: {
+        status: 'standard',
+        type: null,
+        confidence: Number(selection.confidence || 0),
+        goal: '',
+        requestedOutputs: [],
+        supportingCapabilities: [],
+      },
+      usage: response.usage,
+      raw: response.raw,
+    };
+  }
+
+  return {
+    selection: {
+      ...selection,
+      goal: clampText(selection.goal, 240),
+    },
+    usage: response.usage,
+    raw: response.raw,
+  };
+}
+
+module.exports = {
+  selectDeliverableWorkflow,
+};

package/server/services/ai/deliverables/validator.js

@@ -0,0 +1,63 @@
+'use strict';
+
+const {
+  normalizeDeliverableResult,
+  normalizeDeliverableValidationResult,
+} = require('./contracts');
+
+class DeliverableValidationError extends Error {
+  constructor(message, details = {}) {
+    super(message);
+    this.name = 'DeliverableValidationError';
+    this.deliverableValidation = details.validation || null;
+    this.deliverableResult = details.result || null;
+    this.disableAutonomousRetry = true;
+  }
+}
+
+async function validateDeliverableExecution({
+  workflow,
+  request,
+  plan,
+  finalReply,
+  artifacts,
+  toolExecutions,
+  runId,
+}) {
+  const validation = normalizeDeliverableValidationResult(
+    await workflow.validate({
+      request,
+      plan,
+      finalReply,
+      artifacts,
+      toolExecutions,
+      runId,
+    }),
+  );
+  const result = normalizeDeliverableResult({
+    type: workflow.type,
+    status: validation.status === 'passed' ? 'completed' : 'failed',
+    summary: workflow.summarize({
+      request,
+      plan,
+      finalReply,
+      artifacts: validation.artifacts,
+      validation,
+      toolExecutions,
+      runId,
+    }),
+    artifacts: validation.artifacts,
+    validation,
+    metadata: {
+      requestedOutputs: request?.requestedOutputs || [],
+      supportingCapabilities: request?.supportingCapabilities || [],
+      preferredTools: plan?.preferredTools || [],
+    },
+  });
+  return { validation, result };
+}
+
+module.exports = {
+  DeliverableValidationError,
+  validateDeliverableExecution,
+};