neoagent 3.2.1-beta.1 → 3.2.1-beta.10

package/server/services/ai/loop/messaging_delivery.js

@@ -6,7 +6,12 @@ const {
 const {
   normalizeOutgoingMessage,
 } = require('../messagingFallback');
-const { withProviderRetry, isTransientError } = require('../providerRetry');
+const { withProviderRetry } = require('../providerRetry');
+const {
+  abortableDelay,
+  getErrorCode,
+  getHttpStatus,
+} = require('../../../utils/retry');
 const { shortenRunId, summarizeForLog } = require('../logFormat');
 const {
   TICK_MS,
@@ -16,6 +21,15 @@ const {
   evaluateProgressLiveness,
 } = require('./progress_monitor');
 
+const SAFE_PRE_DELIVERY_NETWORK_CODES = new Set([
+  'EAI_AGAIN',
+  'ENOTFOUND',
+  'ECONNREFUSED',
+  'ENETUNREACH',
+  'EHOSTUNREACH',
+  'UND_ERR_CONNECT_TIMEOUT',
+]);
+
 function isoNow() {
   return new Date().toISOString();
 }
@@ -39,9 +53,26 @@ function requireSuccessfulMessagingDelivery(result, label = 'Messaging delivery'
   const error = new Error(`${label} failed: ${reason}`);
   error.code = 'MESSAGING_DELIVERY_FAILED';
   error.deliveryResult = result || null;
+  error.safeToRetry = result?.safeToRetry === true;
+  error.deliveryAmbiguous = result?.deliveryAmbiguous === true;
   throw error;
 }
 
+function isSafeMessagingDeliveryRetry(error) {
+  if (!error || error.retryable === false || error.deliveryAmbiguous === true) return false;
+  if (error.safeToRetry === true) return true;
+  const result = error.deliveryResult;
+  if (result?.deliveryAmbiguous === true || result?.retryable === false) return false;
+  if (result?.success === false) return true;
+
+  // A rate-limit response explicitly rejected the request. DNS/connect failures
+  // happen before a platform can accept it. Timeouts, resets, broken pipes, and
+  // generic 5xx responses are intentionally excluded because the message may
+  // already have been accepted and retrying would duplicate it.
+  if (getHttpStatus(error) === 429) return true;
+  return SAFE_PRE_DELIVERY_NETWORK_CODES.has(String(getErrorCode(error) || ''));
+}
+
 // Harness-driven progress: the originating chat must see autonomous updates during
 // long runs even when a weak model never calls send_interim_update itself. The
 // supervisor paces this call (FIRST/REPEAT thresholds + a repeat guard). The update
@@ -66,13 +97,30 @@ async function sendRuntimeMessagingHeartbeat(engine, runId, options = {}) {
     && typeof runMeta.composeProgressUpdate === 'function') {
     let statusMsg = '';
     try {
-      const composed = await runMeta.composeProgressUpdate({ stalled: options.stalled === true });
+      const composed = await runMeta.composeProgressUpdate({
+        stalled: options.stalled === true,
+        signal: runMeta.messagingProgressSupervisor?.abortController?.signal || null,
+      });
       if (normalizeOutgoingMessage(composed, platform)) statusMsg = String(composed).trim();
     } catch { /* no dynamic update available this tick */ }
+    const currentRunMeta = engine.getRunMeta(runId);
+    if (
+      currentRunMeta !== runMeta
+      || runMeta.aborted
+      || runMeta.status !== 'running'
+      || runMeta.terminalInterim
+      || runMeta.finalDeliverySent
+      || runMeta.deliveryState?.finalContentDelivered === true
+    ) {
+      return { sent: false, skipped: true, terminal: true };
+    }
     if (statusMsg) try {
       const delivery = await engine.messagingManager.sendMessage(runMeta.userId, platform, chatId, statusMsg, {
         runId,
         agentId: runMeta.agentId,
+        signal: runMeta.messagingProgressSupervisor?.abortController?.signal
+          || runMeta.abortController?.signal
+          || null,
       });
       if (delivery?.success === true && delivery?.suppressed !== true) {
         const nowIso = isoNow();
@@ -157,11 +205,12 @@ async function deliverMessagingFinalFallback(engine, {
   console.info(
     `[Run ${shortenRunId(runId)}] messaging_fallback chunks=${chunks.length} to=${summarizeForLog(chatId, 80)}`
   );
+  const deliveredChunks = [];
   for (let i = 0; i < chunks.length; i++) {
     if (i > 0) {
       const delay = Math.max(1000, Math.min(chunks[i].length * 30, 4000));
       await engine.messagingManager.sendTyping(userId, platform, chatId, true, { agentId }).catch(() => {});
-      await new Promise((resolve) => setTimeout(resolve, delay));
+      await abortableDelay(delay, runMeta.abortController?.signal || null);
     }
     try {
       await withProviderRetry(async () => {
@@ -170,32 +219,34 @@ async function deliverMessagingFinalFallback(engine, {
           platform,
           chatId,
           chunks[i],
-          { runId, agentId },
+          {
+            runId,
+            agentId,
+            idempotencyKey: `${runId}:final:${i}`,
+            signal: runMeta.abortController?.signal || null,
+          },
         );
         return requireSuccessfulMessagingDelivery(deliveryResult, 'Final messaging delivery');
       }, {
         ...engine.messagingDeliveryRetry,
         label: `MessagingDelivery ${platform}`,
-        isRetryable: (error) => (
-          error?.retryable !== false
-          && (
-            error?.code === 'MESSAGING_DELIVERY_FAILED'
-            || isTransientError(error)
-          )
-        ),
+        signal: runMeta.abortController?.signal || null,
+        isRetryable: isSafeMessagingDeliveryRetry,
       });
     } catch (error) {
       error.disableAutonomousRetry = true;
+      error.deliveredChunks = deliveredChunks.slice();
       throw error;
     }
+    deliveredChunks.push(chunks[i]);
+    runMeta.lastSentMessage = chunks[i];
+    if (!Array.isArray(runMeta.sentMessages)) runMeta.sentMessages = [];
+    runMeta.sentMessages.push(chunks[i]);
   }
 
-  runMeta.lastSentMessage = chunks[chunks.length - 1] || cleanedContent;
-  runMeta.sentMessages = Array.isArray(runMeta.sentMessages)
-    ? [...runMeta.sentMessages, ...chunks]
-    : chunks.slice();
+  runMeta.lastSentMessage = deliveredChunks[deliveredChunks.length - 1] || cleanedContent;
   engine.markRunFinalDelivery(runId, runMeta.lastSentMessage);
-  return { sent: true, chunks };
+  return { sent: true, chunks: deliveredChunks };
 }
 
 async function tickMessagingProgressSupervisor(engine, runId) {
@@ -206,54 +257,71 @@ async function tickMessagingProgressSupervisor(engine, runId) {
   if (runMeta.terminalInterim) {
     return { sent: false, skipped: true };
   }
+  if (runMeta.progressSupervisorTickInFlight === true) {
+    return { sent: false, skipped: true, inFlight: true };
+  }
+  runMeta.progressSupervisorTickInFlight = true;
+  let settleTick;
+  const tickPromise = new Promise((resolve) => { settleTick = resolve; });
+  runMeta.progressSupervisorTickPromise = tickPromise;
 
-  const ledger = runMeta.progressLedger || buildInitialProgressLedger({
-    startedAt: runMeta.startedAtIso || isoNow(),
-  });
-  runMeta.progressLedger = ledger;
-  const liveness = evaluateProgressLiveness(runMeta);
-  const stalled = liveness.stalled;
-
-  if (stalled && !ledger.stallNotifiedAt) {
-    engine.updateRunProgress(runId, {
-      stallNotifiedAt: isoNow(),
-      progressState: 'stalled',
+  try {
+    const ledger = runMeta.progressLedger || buildInitialProgressLedger({
+      startedAt: runMeta.startedAtIso || isoNow(),
     });
-    engine.recordRunEvent(runMeta.userId, runId, 'progress_stalled', {
-      currentTool: ledger.currentTool || null,
-      currentStep: ledger.currentStep || null,
-      phase: ledger.currentPhase || 'idle',
-    }, { agentId: runMeta.agentId });
-  }
+    runMeta.progressLedger = ledger;
+    const liveness = evaluateProgressLiveness(runMeta);
+    const stalled = liveness.stalled;
 
-  if (!liveness.shouldNudge) {
-    return { sent: false, skipped: true };
-  }
+    if (stalled && !ledger.stallNotifiedAt) {
+      engine.updateRunProgress(runId, {
+        stallNotifiedAt: isoNow(),
+        progressState: 'stalled',
+      });
+      engine.recordRunEvent(runMeta.userId, runId, 'progress_stalled', {
+        currentTool: ledger.currentTool || null,
+        currentStep: ledger.currentStep || null,
+        phase: ledger.currentPhase || 'idle',
+      }, { agentId: runMeta.agentId });
+    }
 
-  const lastSupervisorNudgeAtMs = timestampMs(runMeta.lastSupervisorNudgeAt, 0);
-  if (lastSupervisorNudgeAtMs > 0 && (Date.now() - lastSupervisorNudgeAtMs) < REPEAT_UPDATE_MS) {
-    return { sent: false, skipped: true };
-  }
+    if (!liveness.shouldNudge) {
+      return { sent: false, skipped: true };
+    }
 
-  if (ledger.currentPhase === 'tool' || ledger.currentPhase === 'model') {
-    return sendRuntimeMessagingHeartbeat(engine, runId, { stalled });
-  }
+    const lastSupervisorNudgeAtMs = timestampMs(runMeta.lastSupervisorNudgeAt, 0);
+    if (lastSupervisorNudgeAtMs > 0 && (Date.now() - lastSupervisorNudgeAtMs) < REPEAT_UPDATE_MS) {
+      return { sent: false, skipped: true };
+    }
 
-  if (ledger.currentPhase !== 'idle') {
-    return { sent: false, skipped: true };
-  }
+    if (ledger.currentPhase === 'tool' || ledger.currentPhase === 'model') {
+      return await sendRuntimeMessagingHeartbeat(engine, runId, { stalled });
+    }
 
-  const queued = engine.enqueueSystemSteering(runId, buildProgressNudge({ stalled }), {
-    reason: stalled ? 'stalled_progress_check' : 'progress_check',
-  });
-  if (!queued) {
-    return { sent: false, skipped: true };
+    if (ledger.currentPhase !== 'idle') {
+      return { sent: false, skipped: true };
+    }
+
+    const queued = engine.enqueueSystemSteering(runId, buildProgressNudge({ stalled }), {
+      reason: stalled ? 'stalled_progress_check' : 'progress_check',
+    });
+    if (!queued) {
+      return { sent: false, skipped: true };
+    }
+    runMeta.lastSupervisorNudgeAt = isoNow();
+    engine.updateRunProgress(runId, {
+      lastUserVisibleUpdateAt: ledger.lastUserVisibleUpdateAt || null,
+    });
+    return { sent: false, queued: true };
+  } finally {
+    settleTick();
+    if (engine.getRunMeta(runId) === runMeta) {
+      runMeta.progressSupervisorTickInFlight = false;
+      if (runMeta.progressSupervisorTickPromise === tickPromise) {
+        runMeta.progressSupervisorTickPromise = null;
+      }
+    }
   }
-  runMeta.lastSupervisorNudgeAt = isoNow();
-  engine.updateRunProgress(runId, {
-    lastUserVisibleUpdateAt: ledger.lastUserVisibleUpdateAt || null,
-  });
-  return { sent: false, queued: true };
 }
 
 function startMessagingProgressSupervisor(engine, runId) {
@@ -264,25 +332,29 @@ function startMessagingProgressSupervisor(engine, runId) {
   if (runMeta.messagingProgressSupervisor?.timer) {
     return true;
   }
+  const abortController = new AbortController();
   const timer = setInterval(() => {
     tickMessagingProgressSupervisor(engine, runId).catch((error) => {
       console.warn('[Engine] Messaging progress supervisor failed:', error?.message || error);
     });
   }, TICK_MS);
   timer.unref?.();
-  runMeta.messagingProgressSupervisor = { timer };
+  runMeta.messagingProgressSupervisor = { timer, abortController };
   return true;
 }
 
 function stopMessagingProgressSupervisor(engine, runId) {
   const runMeta = engine.getRunMeta(runId);
+  const inFlight = runMeta?.progressSupervisorTickPromise || null;
   const timer = runMeta?.messagingProgressSupervisor?.timer || null;
   if (timer) {
     clearInterval(timer);
   }
+  runMeta?.messagingProgressSupervisor?.abortController?.abort('Progress supervisor stopped.');
   if (runMeta?.messagingProgressSupervisor) {
     runMeta.messagingProgressSupervisor = null;
   }
+  return inFlight || Promise.resolve();
 }
 
 module.exports = {

package/server/services/ai/loop/model_call_guard.js

@@ -0,0 +1,91 @@
+'use strict';
+
+const MODEL_CALL_TIMEOUT_MS = 5 * 60 * 1000;
+
+function formatElapsedDuration(durationMs) {
+  const totalSeconds = Math.max(1, Math.floor(Number(durationMs || 0) / 1000));
+  if (totalSeconds < 60) return `${totalSeconds}s`;
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (seconds === 0) return `${minutes}m`;
+  return `${minutes}m ${seconds}s`;
+}
+
+function resolveModelCallTimeoutMs(options = {}) {
+  const requested = Number(options?.modelCallTimeoutMs);
+  if (Number.isFinite(requested) && requested > 0) {
+    return Math.max(10, requested);
+  }
+  return MODEL_CALL_TIMEOUT_MS;
+}
+
+function abortError(reason = null) {
+  if (reason instanceof Error) return reason;
+  const error = new Error(String(reason || 'Model call aborted.'));
+  error.name = 'AbortError';
+  error.code = 'ABORT_ERR';
+  return error;
+}
+
+async function withModelCallTimeout(promise, options = {}, label = 'Model call') {
+  const timeoutMs = resolveModelCallTimeoutMs(options);
+  const signal = options?.modelAbortController?.signal || options?.signal || null;
+  let timer = null;
+  let abortListener = null;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      const error = new Error(`${label} timed out after ${formatElapsedDuration(timeoutMs)}.`);
+      error.code = 'MODEL_CALL_TIMEOUT';
+      options?.modelAbortController?.abort(error);
+      reject(error);
+    }, timeoutMs);
+  });
+  const aborted = new Promise((_, reject) => {
+    if (!signal) return;
+    abortListener = () => reject(abortError(signal.reason));
+    if (signal.aborted) abortListener();
+    else signal.addEventListener('abort', abortListener, { once: true });
+  });
+
+  try {
+    return await Promise.race([Promise.resolve(promise), timeout, aborted]);
+  } finally {
+    if (timer) clearTimeout(timer);
+    if (abortListener) signal?.removeEventListener('abort', abortListener);
+  }
+}
+
+async function runAbortableModelCall(factory, options = {}, label = 'Model call') {
+  const modelAbortController = new AbortController();
+  const parentSignals = [...new Set([
+    options?.signal,
+    ...(Array.isArray(options?.signals) ? options.signals : []),
+  ].filter((signal) => signal && typeof signal.addEventListener === 'function'))];
+  const parentListeners = new Map();
+  for (const parentSignal of parentSignals) {
+    const abortFromParent = () => modelAbortController.abort(parentSignal.reason);
+    parentListeners.set(parentSignal, abortFromParent);
+    if (parentSignal.aborted) abortFromParent();
+    else parentSignal.addEventListener('abort', abortFromParent, { once: true });
+  }
+
+  try {
+    const promise = Promise.resolve().then(() => factory(modelAbortController.signal));
+    return await withModelCallTimeout(
+      promise,
+      { ...options, modelAbortController },
+      label,
+    );
+  } finally {
+    for (const [parentSignal, abortFromParent] of parentListeners) {
+      parentSignal.removeEventListener('abort', abortFromParent);
+    }
+  }
+}
+
+module.exports = {
+  abortError,
+  resolveModelCallTimeoutMs,
+  runAbortableModelCall,
+  withModelCallTimeout,
+};

package/server/services/ai/loop/model_io.js

@@ -5,48 +5,16 @@ const { sanitizeModelOutput } = require('../outputSanitizer');
 const { parseJsonObject } = require('../taskAnalysis');
 const { withProviderRetry, isTransientError } = require('../providerRetry');
 const { normalizeUsage, recordModelUsage } = require('../usage');
-
-const MODEL_CALL_TIMEOUT_MS = 5 * 60 * 1000;
+const {
+  resolveModelCallTimeoutMs,
+  runAbortableModelCall,
+  withModelCallTimeout,
+} = require('./model_call_guard');
 
 function isoNow() {
   return new Date().toISOString();
 }
 
-function formatElapsedDuration(durationMs) {
-  const totalSeconds = Math.max(1, Math.floor(Number(durationMs || 0) / 1000));
-  if (totalSeconds < 60) return `${totalSeconds}s`;
-  const minutes = Math.floor(totalSeconds / 60);
-  const seconds = totalSeconds % 60;
-  if (seconds === 0) return `${minutes}m`;
-  return `${minutes}m ${seconds}s`;
-}
-
-function resolveModelCallTimeoutMs(options = {}) {
-  const requested = Number(options?.modelCallTimeoutMs);
-  if (Number.isFinite(requested) && requested > 0) {
-    return Math.max(10, requested);
-  }
-  return MODEL_CALL_TIMEOUT_MS;
-}
-
-async function withModelCallTimeout(promise, options = {}, label = 'Model call') {
-  const timeoutMs = resolveModelCallTimeoutMs(options);
-  let timer = null;
-  const timeout = new Promise((_, reject) => {
-    timer = setTimeout(() => {
-      options?.modelAbortController?.abort(`${label} timed out.`);
-      const error = new Error(`${label} timed out after ${formatElapsedDuration(timeoutMs)}.`);
-      error.code = 'MODEL_CALL_TIMEOUT';
-      reject(error);
-    }, timeoutMs);
-  });
-  try {
-    return await Promise.race([Promise.resolve(promise), timeout]);
-  } finally {
-    if (timer) clearTimeout(timer);
-  }
-}
-
 async function requestStructuredJson(engine, {
   provider,
   providerName,
@@ -62,7 +30,11 @@ async function requestStructuredJson(engine, {
 }) {
   const startedAt = Date.now();
   const structuredStep = `model:${phase}`;
-  const signal = telemetry?.signal;
+  const modelAbortController = new AbortController();
+  const parentSignal = telemetry?.signal;
+  const abortFromParent = () => modelAbortController.abort(parentSignal?.reason);
+  if (parentSignal?.aborted) abortFromParent();
+  else parentSignal?.addEventListener('abort', abortFromParent, { once: true });
   if (telemetry?.runId) {
     engine.updateRunProgress(telemetry.runId, {
       currentPhase: 'model',
@@ -72,7 +44,6 @@ async function requestStructuredJson(engine, {
     });
   }
 
-  let completed = false;
   try {
     const response = await withProviderRetry(
       () => withModelCallTimeout(
@@ -86,15 +57,18 @@ async function requestStructuredJson(engine, {
             model,
             maxTokens,
             reasoningEffort: reasoningEffort || engine.getReasoningEffort(providerName, {}),
-            signal,
+            signal: modelAbortController.signal,
           }
         ),
-        telemetry || {},
+        { ...(telemetry || {}), modelAbortController },
         `${phase} model call`,
       ),
-      { label: `Engine ${model} (structured)` }
+      {
+        label: `Engine ${model} (structured)`,
+        isRetryable: (err) => !modelAbortController.signal.aborted && isTransientError(err),
+        signal: modelAbortController.signal,
+      }
     );
-    completed = true;
     if (telemetry?.runId && telemetry?.userId) {
       recordModelUsage({
         runId: telemetry.runId,
@@ -117,6 +91,7 @@ async function requestStructuredJson(engine, {
       usage: normalizedUsage?.totalTokens || 0,
     };
   } finally {
+    parentSignal?.removeEventListener('abort', abortFromParent);
     const runMeta = telemetry?.runId ? engine.getRunMeta(telemetry.runId) : null;
     if (runMeta?.progressLedger?.currentStep === structuredStep) {
       engine.updateRunProgress(telemetry.runId, {
@@ -124,8 +99,6 @@ async function requestStructuredJson(engine, {
         currentStep: null,
         currentTool: null,
         currentStepStartedAt: null,
-      }, {
-        verified: completed,
       });
     }
   }
@@ -227,6 +200,7 @@ async function requestModelResponse(engine, {
           phase: 'recovering',
         });
       },
+      signal: modelAbortController.signal,
     }));
   } finally {
     parentSignal?.removeEventListener('abort', abortFromParent);
@@ -271,5 +245,6 @@ module.exports = {
   requestModelResponse,
   requestStructuredJson,
   resolveModelCallTimeoutMs,
+  runAbortableModelCall,
   withModelCallTimeout,
 };

package/server/services/ai/loop/progress_classification.js

@@ -182,7 +182,9 @@ function buildReadOnlyChurnGuidance({ readOnlyCount = 0, alreadyRead = '' } = {}
       : 'Do not re-read or re-search anything already in this conversation.',
     'Decide from the evidence you have now.',
     'If the requested work is already done, no matching target exists, or the available tools cannot make the change, call task_complete with that truthful final answer or blocker.',
+    'If research targets remain uncovered, open a new primary source for one uncovered target now instead of re-querying the same lead.',
     'If exactly one concrete safe action remains, take that action now. Otherwise finish; more poking around is not progress.',
+    'Never invent missing targets, products, people, files, or outcomes to make the answer look complete.',
   ].join(' ');
 }
 

package/server/services/ai/loop/tool_dispatch.js

@@ -4,7 +4,10 @@ const { v4: uuidv4 } = require('uuid');
 const db = require('../../../db/database');
 const { globalHooks } = require('../hooks');
 const { summarizeForLog } = require('../logFormat');
-const { inferToolFailureMessage } = require('../toolEvidence');
+const {
+  inferToolFailureMessage,
+  resolveDeclaredToolAccess,
+} = require('../toolEvidence');
 
 function getAvailableTools(_engine, app, options = {}) {
   const { getAvailableTools: loadAvailableTools } = require('../tools');
@@ -16,8 +19,21 @@ async function executeTool(engine, toolName, args, context) {
   return runTool(toolName, args, context, engine);
 }
 
-function isReadOnlyToolCall(toolCall) {
+function isReadOnlyToolCall(toolCall, toolDefinition = null) {
   const name = String(toolCall?.function?.name || '');
+  let toolArgs = {};
+  try {
+    toolArgs = typeof toolCall?.function?.arguments === 'string'
+      ? JSON.parse(toolCall.function.arguments || '{}')
+      : (toolCall?.function?.arguments || {});
+  } catch {
+    return false;
+  }
+
+  const declaredAccess = resolveDeclaredToolAccess(toolDefinition, toolArgs);
+  if (declaredAccess === 'read') return true;
+  if (declaredAccess === 'write') return false;
+
   const readOnly = new Set([
     'read_file',
     'read_files',
@@ -35,12 +51,7 @@ function isReadOnlyToolCall(toolCall) {
     'read_health_data',
   ]);
   if (name === 'http_request') {
-    try {
-      const args = JSON.parse(toolCall.function.arguments || '{}');
-      return String(args.method || 'GET').toUpperCase() === 'GET';
-    } catch {
-      return false;
-    }
+    return String(toolArgs.method || 'GET').toUpperCase() === 'GET';
   }
   return readOnly.has(name);
 }