neoagent 3.2.1-beta.6 → 3.2.1-beta.8

package/server/services/ai/loop/conversation_loop.js

@@ -59,6 +59,11 @@ const {
   normalizeModelSelections,
   resolveModelSelection,
 } = require('../model_identity');
+const {
+  isModelCoolingDown,
+  recordModelFailure,
+  recordModelSuccess,
+} = require('../model_failure_cache');
 const { getProviderForUser } = require('../provider_selector');
 const { IterationBudget } = require('./iteration_budget');
 const {
@@ -417,11 +422,21 @@ async function getFailureFallbackModelId(
   preferredFallbackId = null,
   failureError = null,
   signal = null,
+  excludedModelIds = [],
 ) {
   const { getSupportedModels } = require('../models');
   const aiSettings = getAiSettings(userId, agentId);
   const models = await getSupportedModels(userId, agentId, { signal });
-  const availableModels = models.filter((model) => model.available !== false);
+  const excluded = new Set(
+    [...excludedModelIds]
+      .map((id) => String(id || '').trim())
+      .filter(Boolean),
+  );
+  const availableModels = models.filter(
+    (model) => model.available !== false
+      && !isModelCoolingDown(userId, agentId, model.id)
+      && !excluded.has(model.id),
+  );
   const configuredEnabledIds = Array.isArray(aiSettings.enabled_models)
     ? aiSettings.enabled_models.map((id) => String(id).trim()).filter(Boolean)
     : [];
@@ -430,15 +445,26 @@ async function getFailureFallbackModelId(
     ? availableModels.filter((model) => enabledIds.includes(model.id))
     : availableModels;
   const fallbackSearchPool = pool;
-  const currentModel = resolveModelSelection(pool, currentModelId)
-    || resolveModelSelection(availableModels, currentModelId);
-
-  // When the failure is a provider-level rate limit, the preferred fallback is
-  // likely on the same provider and will hit the same limit. Skip it and prefer
-  // a fallback from a different provider instead.
-  const isProviderRateLimit = /429|rate.?limit|free-models-per/i.test(String(failureError?.message || ''));
+  const currentModel = resolveModelSelection(models, currentModelId);
+
+  // Provider-wide failures (credentials, throttling, or service outages) are
+  // likely to affect another model on the same provider. Prefer a different
+  // provider before consulting the configured fallback.
+  const failureStatus = Number(
+    failureError?.status
+    ?? failureError?.statusCode
+    ?? failureError?.response?.status,
+  );
+  const isProviderScopedFailure = (
+    failureStatus === 401
+    || failureStatus === 403
+    || failureStatus === 429
+    || (failureStatus >= 500 && failureStatus < 600)
+    || /rate.?limit|free-models-per|service unavailable|provider unavailable|authentication|api key/i
+      .test(String(failureError?.message || ''))
+  );
 
-  if (preferredFallbackId && !isProviderRateLimit) {
+  if (preferredFallbackId && !isProviderScopedFailure) {
     const preferred = resolveModelSelection(fallbackSearchPool, preferredFallbackId)
       || resolveModelSelection(availableModels, preferredFallbackId);
     if (preferred && preferred.id !== currentModel?.id) return preferred.id;
@@ -450,8 +476,7 @@ async function getFailureFallbackModelId(
     if (differentProvider) return differentProvider.id;
   }
 
-  // If no different-provider model exists, still try the preferred fallback
-  // even on rate limits (it's better than nothing).
+  // If no different-provider model exists, still try the preferred fallback.
   if (preferredFallbackId) {
     const preferred = resolveModelSelection(fallbackSearchPool, preferredFallbackId)
       || resolveModelSelection(availableModels, preferredFallbackId);
@@ -483,6 +508,7 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
   let model = null;
   let modelSelectionId = null;
   let providerName = null;
+  const modelTurnFailedModelSelectionIds = new Set();
   let messages = [];
   let iteration = 0;
   let totalTokens = 0;
@@ -598,7 +624,17 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
   }
   db.prepare('UPDATE agent_runs SET model = ?, updated_at = datetime(\'now\') WHERE id = ?')
     .run(modelSelectionId, runId);
-  const switchToFallbackModel = async (failedSelectionId, error, phase) => {
+  const recordFailedModel = (failedSelectionId, error, excludedModels) => {
+    excludedModels.add(failedSelectionId);
+    recordModelFailure(userId, agentId, failedSelectionId, error);
+  };
+  const switchToFallbackModel = async (
+    failedSelectionId,
+    error,
+    phase,
+    excludedModels,
+  ) => {
+    recordFailedModel(failedSelectionId, error, excludedModels);
     const fallbackModelId = await getFailureFallbackModelId(
       userId,
       agentId,
@@ -606,9 +642,21 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
       aiSettings.fallback_model_id,
       error,
       engine.getRunMeta(runId)?.abortController?.signal,
+      excludedModels,
     );
     if (!fallbackModelId || fallbackModelId === failedSelectionId) return false;
-    console.log(`[Engine] ${phase} failed on ${failedSelectionId}; attempting fallback to: ${fallbackModelId}`);
+    const failureSummary = summarizeForLog(error?.message || error, 180);
+    console.log(
+      `[Engine] ${phase} failed on ${failedSelectionId}: ${failureSummary}; attempting fallback to: ${fallbackModelId}`
+    );
+    engine.recordRunEvent(userId, runId, 'model_fallback', {
+      phase,
+      failedModel: failedSelectionId,
+      fallbackModel: fallbackModelId,
+      errorCode: error?.code || null,
+      errorStatus: error?.status || error?.statusCode || error?.response?.status || null,
+      error: failureSummary,
+    }, { agentId });
     engine.emit(userId, 'run:interim', {
       runId,
       message: `Model service failed on ${failedSelectionId}; retrying with ${fallbackModelId}.`,
@@ -638,13 +686,28 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
     return true;
   };
   const runWithModelFallback = async (phase, fn) => {
-    try {
-      return await fn();
-    } catch (err) {
-      const failedSelectionId = modelSelectionId;
-      const switched = await switchToFallbackModel(failedSelectionId, err, phase);
-      if (!switched) throw err;
-      return await fn();
+    let recoveries = 0;
+    const failedModels = new Set();
+    while (true) {
+      try {
+        const result = await fn();
+        recordModelSuccess(userId, agentId, modelSelectionId);
+        return result;
+      } catch (err) {
+        if (recoveries >= loopPolicy.maxModelFailureRecoveries) {
+          recordFailedModel(modelSelectionId, err, failedModels);
+          throw err;
+        }
+        const failedSelectionId = modelSelectionId;
+        const switched = await switchToFallbackModel(
+          failedSelectionId,
+          err,
+          phase,
+          failedModels,
+        );
+        if (!switched) throw err;
+        recoveries += 1;
+      }
     }
   };
 
@@ -1470,66 +1533,59 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
       let responseModel = model;
       let streamContent = '';
 
-      const tryModelCall = async (retryForFallback = true) => {
-        try {
-          const modelCall = await engine.requestModelResponse({
-            provider,
-            providerName,
-            model,
-            messages,
-            tools,
-            options: {
-              ...options,
-              userId,
-              agentId,
+      const tryModelCall = async () => {
+        let requestMessages = messages;
+        let requestPhase = 'model_turn';
+        while (true) {
+          try {
+            const modelCall = await engine.requestModelResponse({
+              provider,
+              providerName,
+              model,
+              messages: requestMessages,
+              tools,
+              options: {
+                ...options,
+                userId,
+                agentId,
+                runId,
+                phase: requestPhase,
+                signal: engine.getRunMeta(runId)?.abortController?.signal,
+              },
               runId,
-              phase: 'model_turn',
-              signal: engine.getRunMeta(runId)?.abortController?.signal,
-            },
-            runId,
-            iteration,
-          });
-          response = modelCall.response;
-          responseModel = modelCall.responseModel;
-          streamContent = modelCall.streamContent;
-        } catch (err) {
-          console.error(`[Engine] Model call failed (${model}):`, err.message);
-          const fallbackModelId = retryForFallback
-            ? await getFailureFallbackModelId(
-              userId,
-              agentId,
+              iteration,
+            });
+            response = modelCall.response;
+            responseModel = modelCall.responseModel;
+            streamContent = modelCall.streamContent;
+            recordModelSuccess(userId, agentId, modelSelectionId);
+            return;
+          } catch (err) {
+            console.error(`[Engine] Model call failed (${model}):`, err.message);
+            const runSignal = engine.getRunMeta(runId)?.abortController?.signal;
+            if (isAbortError(err) || runSignal?.aborted) throw err;
+            if (modelFailureRecoveries >= loopPolicy.maxModelFailureRecoveries) {
+              recordFailedModel(
+                modelSelectionId,
+                err,
+                modelTurnFailedModelSelectionIds,
+              );
+              throw err;
+            }
+
+            const failedModel = model;
+            const switched = await switchToFallbackModel(
               modelSelectionId,
-              aiSettings.fallback_model_id,
               err,
-              engine.getRunMeta(runId)?.abortController?.signal,
-            )
-            : null;
-          if (fallbackModelId) {
-            const failedModel = model;
-            console.log(`[Engine] Attempting fallback to: ${fallbackModelId}`);
-            const fallback = await getProviderForUser(
-              userId,
-              userMessage,
-              triggerType === 'subagent',
-              fallbackModelId,
-              {
-                ...providerStatusConfig,
-                signal: engine.getRunMeta(runId)?.abortController?.signal,
-              }
+              'model turn',
+              modelTurnFailedModelSelectionIds,
             );
-            provider = fallback.provider;
-            model = fallback.model;
-            modelSelectionId = fallback.modelSelectionId;
-            providerName = fallback.providerName;
-            db.prepare('UPDATE agent_runs SET model = ?, updated_at = datetime(\'now\') WHERE id = ?')
-              .run(modelSelectionId, runId);
-            Object.assign(engine.getRunMeta(runId) || {}, {
-              model,
-              modelSelectionId,
-              providerName,
-            });
+            if (!switched) throw err;
 
-            const retryMessages = sanitizeConversationMessages([
+            modelFailureRecoveries += 1;
+            failedStepCount += 1;
+            requestPhase = 'model_turn_fallback';
+            requestMessages = sanitizeConversationMessages([
               ...messages,
               {
                 role: 'system',
@@ -1540,29 +1596,6 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
                 })
               }
             ]);
-
-            const fallbackCall = await engine.requestModelResponse({
-              provider,
-              providerName,
-              model,
-              messages: retryMessages,
-              tools,
-              options: {
-                ...options,
-                userId,
-                agentId,
-                runId,
-                phase: 'model_turn_fallback',
-                signal: engine.getRunMeta(runId)?.abortController?.signal,
-              },
-              runId,
-              iteration,
-            });
-            response = fallbackCall.response;
-            responseModel = fallbackCall.responseModel;
-            streamContent = fallbackCall.streamContent;
-          } else {
-            throw err;
           }
         }
       };
@@ -1585,30 +1618,6 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
           continue;
         }
         if (lifecycleControl?.action === 'stop' || lifecycleControl?.action === 'interrupt') break;
-        const modelError = String(err?.message || 'Model call failed');
-
-        if (modelFailureRecoveries < loopPolicy.maxModelFailureRecoveries) {
-          const failedModel = model;
-          const switched = await switchToFallbackModel(modelSelectionId, err, 'model turn');
-          if (!switched) throw err;
-          modelFailureRecoveries += 1;
-          failedStepCount += 1;
-          messages.push({
-            role: 'system',
-            content: buildModelFailureLoopPrompt({
-              failedModel,
-              nextModel: model,
-              errorMessage: modelError
-            })
-          });
-          engine.emit(userId, 'run:interim', {
-            runId,
-            message: 'Model call failed; adapting and retrying autonomously.',
-            phase: 'recovering'
-          });
-          continue;
-        }
-
         throw err;
       }
 

package/server/services/ai/model_failure_cache.js

@@ -0,0 +1,108 @@
+'use strict';
+
+const { isAbortError } = require('../../utils/abort');
+const {
+  getErrorCode,
+  getHttpStatus,
+  isTransientIoError,
+  retryAfterMilliseconds,
+} = require('../../utils/retry');
+
+const DEFAULT_COOLDOWN_MS = 15 * 60 * 1000;
+const DEFAULT_RECOVERY_COOLDOWN_MS = 60 * 1000;
+const failures = new Map();
+
+function cacheKey(userId, agentId, modelSelectionId) {
+  return [
+    String(userId ?? ''),
+    String(agentId ?? 'main'),
+    String(modelSelectionId || '').trim(),
+  ].join(':');
+}
+
+function readPermanentCooldownMs() {
+  const configured = Number(process.env.NEOAGENT_MODEL_NOT_FOUND_COOLDOWN_MS);
+  if (!Number.isFinite(configured)) return DEFAULT_COOLDOWN_MS;
+  return Math.max(1_000, Math.min(configured, 24 * 60 * 60 * 1000));
+}
+
+function readRecoveryCooldownMs() {
+  const configured = Number(process.env.NEOAGENT_MODEL_RECOVERY_COOLDOWN_MS);
+  if (!Number.isFinite(configured)) return DEFAULT_RECOVERY_COOLDOWN_MS;
+  return Math.max(1_000, Math.min(configured, 15 * 60 * 1000));
+}
+
+function isPermanentModelFailure(error) {
+  const status = getHttpStatus(error);
+  if (status === 404) return true;
+  return /\b404\b.*\b(model|nim|provider|request)\b|\b(model|nim)\b.*\b404\b/i
+    .test(String(error?.message || ''));
+}
+
+function isRecoverableModelFailure(error) {
+  if (!error || isAbortError(error) || isPermanentModelFailure(error)) return false;
+  const status = getHttpStatus(error);
+  if (status === 401 || status === 403 || status === 429 || (status >= 500 && status < 600)) {
+    return true;
+  }
+  const code = String(getErrorCode(error) || '');
+  if (code === 'MODEL_EMPTY_RESPONSE' || code === 'MODEL_CALL_TIMEOUT') return true;
+  if (isTransientIoError(error)) return true;
+  return /\bempty response|temporarily unavailable|overloaded|rate.?limit|timed? ?out|network error\b/i
+    .test(String(error?.message || ''));
+}
+
+function resolveCooldownMs(error, now) {
+  if (isPermanentModelFailure(error)) return readPermanentCooldownMs();
+  if (!isRecoverableModelFailure(error)) return 0;
+
+  const configured = readRecoveryCooldownMs();
+  const retryAfter = retryAfterMilliseconds(
+    error?.headers || error?.response?.headers,
+    now,
+  );
+  if (!Number.isFinite(retryAfter)) return configured;
+  return Math.max(configured, Math.min(retryAfter, 15 * 60 * 1000));
+}
+
+function recordModelFailure(userId, agentId, modelSelectionId, error, now = Date.now()) {
+  const modelId = String(modelSelectionId || '').trim();
+  const cooldownMs = resolveCooldownMs(error, now);
+  if (!modelId || cooldownMs <= 0) return false;
+  failures.set(cacheKey(userId, agentId, modelId), {
+    expiresAt: now + cooldownMs,
+  });
+  return true;
+}
+
+function recordModelSuccess(userId, agentId, modelSelectionId) {
+  const modelId = String(modelSelectionId || '').trim();
+  if (!modelId) return false;
+  return failures.delete(cacheKey(userId, agentId, modelId));
+}
+
+function isModelCoolingDown(userId, agentId, modelSelectionId, now = Date.now()) {
+  const modelId = String(modelSelectionId || '').trim();
+  if (!modelId) return false;
+  const key = cacheKey(userId, agentId, modelId);
+  const entry = failures.get(key);
+  if (!entry) return false;
+  if (entry.expiresAt <= now) {
+    failures.delete(key);
+    return false;
+  }
+  return true;
+}
+
+function clearModelFailureCache() {
+  failures.clear();
+}
+
+module.exports = {
+  clearModelFailureCache,
+  isModelCoolingDown,
+  isPermanentModelFailure,
+  isRecoverableModelFailure,
+  recordModelFailure,
+  recordModelSuccess,
+};

package/server/services/ai/models.js

@@ -264,7 +264,7 @@ async function getProviderHealthCatalog(userId, agentId = null, options = {}) {
         let statusLabel = provider.statusLabel;
         let availabilityReason = provider.availabilityReason;
 
-        if (provider.id === 'ollama' && provider.enabled) {
+        if (provider.id === 'ollama' && provider.enabled && options.probeLocal !== false) {
             const probe = await probeOllama(
                 provider.baseUrl || AI_PROVIDER_DEFINITIONS.ollama.defaultBaseUrl,
                 1500,

package/server/services/ai/provider_selector.js

@@ -7,6 +7,7 @@ const {
   normalizeModelSelections,
   resolveModelSelection,
 } = require('./model_identity');
+const { isModelCoolingDown } = require('./model_failure_cache');
 
 function buildSelection(model, userId, providerConfig) {
   return {
@@ -30,12 +31,29 @@ async function getProviderForUser(
     signal: providerConfig.signal,
   });
   const selectableModels = models.filter((model) => model.available !== false);
+  const readyModels = selectableModels.filter(
+    (model) => !isModelCoolingDown(userId, agentId, model.id),
+  );
 
   if (modelOverride && typeof modelOverride === 'string') {
     const requested = resolveModelSelection(selectableModels, modelOverride);
     if (!requested) {
       throw new Error(`Requested model '${modelOverride.trim()}' is not available.`);
     }
+    if (isModelCoolingDown(userId, agentId, requested.id) && readyModels.length > 0) {
+      const configuredFallback = resolveModelSelection(
+        readyModels,
+        aiSettings.fallback_model_id,
+      );
+      const fallback = configuredFallback
+        || readyModels.find((model) => model.provider !== requested.provider)
+        || readyModels[0];
+      providerConfig.onStatus?.({
+        phase: 'model_fallback',
+        message: `Skipping recently unavailable model ${requested.id}; using ${fallback.id}.`,
+      });
+      return buildSelection(fallback, userId, providerConfig);
+    }
     return buildSelection(requested, userId, providerConfig);
   }
 
@@ -43,9 +61,35 @@ async function getProviderForUser(
     ? aiSettings.enabled_models.map((id) => String(id).trim()).filter(Boolean)
     : [];
   const enabledIds = normalizeModelSelections(selectableModels, configuredEnabledIds);
-  const availableModels = configuredEnabledIds.length > 0
-    ? selectableModels.filter((model) => enabledIds.includes(model.id))
-    : selectableModels;
+  let availableModels = configuredEnabledIds.length > 0
+    ? readyModels.filter((model) => enabledIds.includes(model.id))
+    : readyModels;
+
+  if (
+    availableModels.length === 0
+    && configuredEnabledIds.length > 0
+    && enabledIds.length > 0
+  ) {
+    const configuredFallback = resolveModelSelection(
+      readyModels,
+      aiSettings.fallback_model_id,
+    );
+    if (configuredFallback) {
+      providerConfig.onStatus?.({
+        phase: 'model_fallback',
+        message: `Enabled models are temporarily unavailable; using ${configuredFallback.id}.`,
+      });
+      availableModels = [configuredFallback];
+    }
+  }
+
+  // If every configured model is cooling down and there is no alternative,
+  // allow one retry instead of making the installation permanently unavailable.
+  if (availableModels.length === 0 && readyModels.length === 0) {
+    availableModels = configuredEnabledIds.length > 0
+      ? selectableModels.filter((model) => enabledIds.includes(model.id))
+      : selectableModels;
+  }
 
   if (availableModels.length === 0) {
     const message = configuredEnabledIds.length > 0
@@ -64,6 +108,17 @@ async function getProviderForUser(
   let selectedModel = fallbackModel;
   if (userSelectedDefault !== 'auto') {
     selectedModel = resolveModelSelection(availableModels, userSelectedDefault) || fallbackModel;
+    const configuredDefault = resolveModelSelection(selectableModels, userSelectedDefault);
+    if (
+      configuredDefault
+      && configuredDefault.id !== selectedModel.id
+      && isModelCoolingDown(userId, agentId, configuredDefault.id)
+    ) {
+      providerConfig.onStatus?.({
+        phase: 'model_fallback',
+        message: `Skipping recently unavailable model ${configuredDefault.id}; using ${selectedModel.id}.`,
+      });
+    }
   } else {
     const selectionHint = providerConfig.selectionHint && typeof providerConfig.selectionHint === 'object'
       ? providerConfig.selectionHint