neoagent 2.5.2-beta.8 → 3.0.0

package/server/services/ai/loop/delivery_state.js

@@ -0,0 +1,27 @@
+'use strict';
+
+function createDeliveryState(seed = {}) {
+  return {
+    alreadySent: seed.alreadySent === true,
+    finalResponseSent: seed.finalResponseSent === true,
+    finalContentDelivered: seed.finalContentDelivered === true,
+  };
+}
+
+function markInterimDelivered(state) {
+  if (!state) return;
+  state.alreadySent = true;
+}
+
+function markFinalDelivered(state) {
+  if (!state) return;
+  state.alreadySent = true;
+  state.finalResponseSent = true;
+  state.finalContentDelivered = true;
+}
+
+module.exports = {
+  createDeliveryState,
+  markInterimDelivered,
+  markFinalDelivered,
+};

package/server/services/ai/loop/error_recovery.js

@@ -0,0 +1,38 @@
+'use strict';
+
+function hasTerminalMessagingDelivery(runMeta = null) {
+  return runMeta?.finalDeliverySent === true
+    || runMeta?.finalResponseSent === true
+    || runMeta?.finalContentDelivered === true
+    || runMeta?.deliveryState?.finalResponseSent === true
+    || runMeta?.deliveryState?.finalContentDelivered === true
+    || runMeta?.noResponse === true;
+}
+
+function isRateLimitError(error = null) {
+  return /429|rate.?limit|free-models-per/i.test(String(error?.message || ''));
+}
+
+function shouldRetryMessagingRun() {
+  // Messaging recovery must stay inside the current run. Re-entering
+  // runWithModel starts over from the original task and repeats tool work.
+  return false;
+}
+
+function shouldSendMessagingErrorFallback({
+  triggerSource,
+  options = {},
+  runMeta = null,
+} = {}) {
+  return triggerSource === 'messaging'
+    && Boolean(options.source)
+    && Boolean(options.chatId)
+    && !hasTerminalMessagingDelivery(runMeta);
+}
+
+module.exports = {
+  hasTerminalMessagingDelivery,
+  isRateLimitError,
+  shouldRetryMessagingRun,
+  shouldSendMessagingErrorFallback,
+};

package/server/services/ai/loop/iteration_budget.js

@@ -0,0 +1,24 @@
+'use strict';
+
+class IterationBudget {
+  constructor(maxTotal) {
+    this.maxTotal = Math.max(0, Number(maxTotal) || 0);
+    this.used = 0;
+  }
+
+  consume() {
+    if (this.used >= this.maxTotal) return false;
+    this.used += 1;
+    return true;
+  }
+
+  refund() {
+    if (this.used > 0) this.used -= 1;
+  }
+
+  get remaining() {
+    return Math.max(0, this.maxTotal - this.used);
+  }
+}
+
+module.exports = { IterationBudget };

package/server/services/ai/loop/messaging_delivery.js

@@ -0,0 +1,296 @@
+'use strict';
+
+const {
+  splitOutgoingMessageForPlatform,
+} = require('../../messaging/formatting_guides');
+const {
+  normalizeOutgoingMessage,
+} = require('../messagingFallback');
+const { withProviderRetry, isTransientError } = require('../providerRetry');
+const { shortenRunId, summarizeForLog } = require('../logFormat');
+const {
+  TICK_MS,
+  REPEAT_UPDATE_MS,
+  buildInitialProgressLedger,
+  buildProgressNudge,
+  evaluateProgressLiveness,
+} = require('./progress_monitor');
+
+function isoNow() {
+  return new Date().toISOString();
+}
+
+function timestampMs(value, fallback = 0) {
+  const resolved = value ? Date.parse(value) : NaN;
+  return Number.isFinite(resolved) ? resolved : fallback;
+}
+
+function requireSuccessfulMessagingDelivery(result, label = 'Messaging delivery') {
+  if (result?.success === true && result?.suppressed !== true) {
+    return result;
+  }
+  const reason = String(
+    result?.error
+    || result?.reason
+    || result?.result?.error
+    || result?.result?.reason
+    || 'the platform did not confirm delivery',
+  ).trim();
+  const error = new Error(`${label} failed: ${reason}`);
+  error.code = 'MESSAGING_DELIVERY_FAILED';
+  error.deliveryResult = result || null;
+  throw error;
+}
+
+// Harness-driven progress: the originating chat must see autonomous updates during
+// long runs even when a weak model never calls send_interim_update itself. The
+// supervisor paces this call (FIRST/REPEAT thresholds + a repeat guard). The update
+// text is always authored by the run's own agent loop (composeProgressUpdate); we
+// never emit a hard-coded status line. If the loop can't compose one this tick, we
+// skip the visible send and only nudge the model, then try again next tick.
+async function sendRuntimeMessagingHeartbeat(engine, runId, options = {}) {
+  const runMeta = engine.getRunMeta(runId);
+  if (!runMeta || runMeta.aborted) return { sent: false, skipped: true };
+  if (runMeta.triggerSource !== 'messaging') {
+    return { sent: false, skipped: true };
+  }
+  const createdAt = isoNow();
+  const heartbeatCount = Number(runMeta.progressLedger?.heartbeatCount || 0) + 1;
+  runMeta.lastSupervisorNudgeAt = createdAt;
+  engine.updateRunProgress(runId, { heartbeatCount });
+
+  const ledger = runMeta.progressLedger || {};
+  const { platform, chatId } = runMeta.messagingContext || {};
+
+  if (engine.messagingManager && !runMeta.terminalInterim && platform && chatId
+    && typeof runMeta.composeProgressUpdate === 'function') {
+    let statusMsg = '';
+    try {
+      const composed = await runMeta.composeProgressUpdate({ stalled: options.stalled === true });
+      if (normalizeOutgoingMessage(composed, platform)) statusMsg = String(composed).trim();
+    } catch { /* no dynamic update available this tick */ }
+    if (statusMsg) try {
+      const delivery = await engine.messagingManager.sendMessage(runMeta.userId, platform, chatId, statusMsg, {
+        runId,
+        agentId: runMeta.agentId,
+      });
+      if (delivery?.success === true && delivery?.suppressed !== true) {
+        const nowIso = isoNow();
+        runMeta.progressLedger = { ...ledger, lastUserVisibleUpdateAt: nowIso };
+        runMeta.lastInterimMessage = statusMsg;
+        engine.updateRunProgress(runId, { lastUserVisibleUpdateAt: nowIso });
+        engine.recordRunEvent(runMeta.userId, runId, 'progress_heartbeat_sent', {
+          stalled: options.stalled === true,
+          currentTool: ledger.currentTool || null,
+          currentStep: ledger.currentStep || null,
+          phase: ledger.currentPhase || 'idle',
+          userVisible: true,
+          createdAt,
+        }, { agentId: runMeta.agentId });
+        // Still nudge the model so its next turn can deliver a richer, real update.
+        engine.enqueueSystemSteering(
+          runId,
+          buildProgressNudge({ stalled: options.stalled === true }),
+          { reason: options.stalled === true ? 'stalled_progress_check' : 'progress_check' },
+        );
+        return { sent: true, heartbeat: true };
+      }
+    } catch (err) {
+      console.warn('[Engine] Progress heartbeat send failed:', err?.message || err);
+    }
+  }
+
+  engine.recordRunEvent(runMeta.userId, runId, 'progress_heartbeat_sent', {
+    stalled: options.stalled === true,
+    currentTool: ledger.currentTool || null,
+    currentStep: ledger.currentStep || null,
+    phase: ledger.currentPhase || 'idle',
+    userVisible: false,
+    createdAt,
+  }, { agentId: runMeta.agentId });
+  engine.enqueueSystemSteering(
+    runId,
+    buildProgressNudge({ stalled: options.stalled === true }),
+    { reason: options.stalled === true ? 'stalled_progress_check' : 'progress_check' },
+  );
+  return { sent: false, heartbeat: true, queued: true };
+}
+
+function shouldSendMessagingFinalFallback(_engine, runMeta, content, platform = null) {
+  const cleanedContent = normalizeOutgoingMessage(content || '', platform, {
+    collapseWhitespace: false,
+  });
+  const lastFinalDeliveryMessage = normalizeOutgoingMessage(
+    runMeta?.lastSentMessage
+    || (Array.isArray(runMeta?.sentMessages) ? runMeta.sentMessages[runMeta.sentMessages.length - 1] : '')
+    || '',
+    platform,
+  );
+  return Boolean(
+    cleanedContent
+    && !runMeta?.terminalInterim
+    && runMeta?.explicitMessageSent !== true
+    && runMeta?.finalDeliverySent !== true
+    && runMeta?.deliveryState?.finalContentDelivered !== true
+    && !lastFinalDeliveryMessage
+  );
+}
+
+async function deliverMessagingFinalFallback(engine, {
+  runId,
+  userId,
+  agentId,
+  platform,
+  chatId,
+  content,
+}) {
+  const runMeta = engine.getRunMeta(runId);
+  if (!runMeta || !engine.messagingManager) return { sent: false, skipped: true };
+  const cleanedContent = normalizeOutgoingMessage(content || '', platform, {
+    collapseWhitespace: false,
+  });
+  if (!shouldSendMessagingFinalFallback(engine, runMeta, cleanedContent, platform)) {
+    return { sent: false, skipped: true };
+  }
+
+  const chunks = splitOutgoingMessageForPlatform(platform, cleanedContent);
+  console.info(
+    `[Run ${shortenRunId(runId)}] messaging_fallback chunks=${chunks.length} to=${summarizeForLog(chatId, 80)}`
+  );
+  for (let i = 0; i < chunks.length; i++) {
+    if (i > 0) {
+      const delay = Math.max(1000, Math.min(chunks[i].length * 30, 4000));
+      await engine.messagingManager.sendTyping(userId, platform, chatId, true, { agentId }).catch(() => {});
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+    try {
+      await withProviderRetry(async () => {
+        const deliveryResult = await engine.messagingManager.sendMessage(
+          userId,
+          platform,
+          chatId,
+          chunks[i],
+          { runId, agentId },
+        );
+        return requireSuccessfulMessagingDelivery(deliveryResult, 'Final messaging delivery');
+      }, {
+        ...engine.messagingDeliveryRetry,
+        label: `MessagingDelivery ${platform}`,
+        isRetryable: (error) => (
+          error?.retryable !== false
+          && (
+            error?.code === 'MESSAGING_DELIVERY_FAILED'
+            || isTransientError(error)
+          )
+        ),
+      });
+    } catch (error) {
+      error.disableAutonomousRetry = true;
+      throw error;
+    }
+  }
+
+  runMeta.lastSentMessage = chunks[chunks.length - 1] || cleanedContent;
+  runMeta.sentMessages = Array.isArray(runMeta.sentMessages)
+    ? [...runMeta.sentMessages, ...chunks]
+    : chunks.slice();
+  engine.markRunFinalDelivery(runId, runMeta.lastSentMessage);
+  return { sent: true, chunks };
+}
+
+async function tickMessagingProgressSupervisor(engine, runId) {
+  const runMeta = engine.getRunMeta(runId);
+  if (!runMeta || runMeta.aborted || runMeta.triggerSource !== 'messaging') {
+    return { sent: false, skipped: true };
+  }
+  if (runMeta.terminalInterim) {
+    return { sent: false, skipped: true };
+  }
+
+  const ledger = runMeta.progressLedger || buildInitialProgressLedger({
+    startedAt: runMeta.startedAtIso || isoNow(),
+  });
+  runMeta.progressLedger = ledger;
+  const liveness = evaluateProgressLiveness(runMeta);
+  const stalled = liveness.stalled;
+
+  if (stalled && !ledger.stallNotifiedAt) {
+    engine.updateRunProgress(runId, {
+      stallNotifiedAt: isoNow(),
+      progressState: 'stalled',
+    });
+    engine.recordRunEvent(runMeta.userId, runId, 'progress_stalled', {
+      currentTool: ledger.currentTool || null,
+      currentStep: ledger.currentStep || null,
+      phase: ledger.currentPhase || 'idle',
+    }, { agentId: runMeta.agentId });
+  }
+
+  if (!liveness.shouldNudge) {
+    return { sent: false, skipped: true };
+  }
+
+  const lastSupervisorNudgeAtMs = timestampMs(runMeta.lastSupervisorNudgeAt, 0);
+  if (lastSupervisorNudgeAtMs > 0 && (Date.now() - lastSupervisorNudgeAtMs) < REPEAT_UPDATE_MS) {
+    return { sent: false, skipped: true };
+  }
+
+  if (ledger.currentPhase === 'tool' || ledger.currentPhase === 'model') {
+    return sendRuntimeMessagingHeartbeat(engine, runId, { stalled });
+  }
+
+  if (ledger.currentPhase !== 'idle') {
+    return { sent: false, skipped: true };
+  }
+
+  const queued = engine.enqueueSystemSteering(runId, buildProgressNudge({ stalled }), {
+    reason: stalled ? 'stalled_progress_check' : 'progress_check',
+  });
+  if (!queued) {
+    return { sent: false, skipped: true };
+  }
+  runMeta.lastSupervisorNudgeAt = isoNow();
+  engine.updateRunProgress(runId, {
+    lastUserVisibleUpdateAt: ledger.lastUserVisibleUpdateAt || null,
+  });
+  return { sent: false, queued: true };
+}
+
+function startMessagingProgressSupervisor(engine, runId) {
+  const runMeta = engine.getRunMeta(runId);
+  if (!runMeta || runMeta.triggerSource !== 'messaging' || !runMeta.messagingContext?.platform || !runMeta.messagingContext?.chatId) {
+    return false;
+  }
+  if (runMeta.messagingProgressSupervisor?.timer) {
+    return true;
+  }
+  const timer = setInterval(() => {
+    tickMessagingProgressSupervisor(engine, runId).catch((error) => {
+      console.warn('[Engine] Messaging progress supervisor failed:', error?.message || error);
+    });
+  }, TICK_MS);
+  timer.unref?.();
+  runMeta.messagingProgressSupervisor = { timer };
+  return true;
+}
+
+function stopMessagingProgressSupervisor(engine, runId) {
+  const runMeta = engine.getRunMeta(runId);
+  const timer = runMeta?.messagingProgressSupervisor?.timer || null;
+  if (timer) {
+    clearInterval(timer);
+  }
+  if (runMeta?.messagingProgressSupervisor) {
+    runMeta.messagingProgressSupervisor = null;
+  }
+}
+
+module.exports = {
+  deliverMessagingFinalFallback,
+  requireSuccessfulMessagingDelivery,
+  sendRuntimeMessagingHeartbeat,
+  shouldSendMessagingFinalFallback,
+  startMessagingProgressSupervisor,
+  stopMessagingProgressSupervisor,
+  tickMessagingProgressSupervisor,
+};

package/server/services/ai/loop/model_io.js

@@ -0,0 +1,258 @@
+'use strict';
+
+const { sanitizeConversationMessages } = require('../history');
+const { sanitizeModelOutput } = require('../outputSanitizer');
+const { parseJsonObject } = require('../taskAnalysis');
+const { withProviderRetry, isTransientError } = require('../providerRetry');
+const { normalizeUsage, recordModelUsage } = require('../usage');
+
+const MODEL_CALL_TIMEOUT_MS = 5 * 60 * 1000;
+
+function isoNow() {
+  return new Date().toISOString();
+}
+
+function formatElapsedDuration(durationMs) {
+  const totalSeconds = Math.max(1, Math.floor(Number(durationMs || 0) / 1000));
+  if (totalSeconds < 60) return `${totalSeconds}s`;
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (seconds === 0) return `${minutes}m`;
+  return `${minutes}m ${seconds}s`;
+}
+
+function resolveModelCallTimeoutMs(options = {}) {
+  const requested = Number(options?.modelCallTimeoutMs);
+  if (Number.isFinite(requested) && requested > 0) {
+    return Math.max(10, requested);
+  }
+  return MODEL_CALL_TIMEOUT_MS;
+}
+
+async function withModelCallTimeout(promise, options = {}, label = 'Model call') {
+  const timeoutMs = resolveModelCallTimeoutMs(options);
+  let timer = null;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      const error = new Error(`${label} timed out after ${formatElapsedDuration(timeoutMs)}.`);
+      error.code = 'MODEL_CALL_TIMEOUT';
+      reject(error);
+    }, timeoutMs);
+  });
+  try {
+    return await Promise.race([Promise.resolve(promise), timeout]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
+async function requestStructuredJson(engine, {
+  provider,
+  providerName,
+  model,
+  messages,
+  prompt,
+  maxTokens = 1400,
+  normalize,
+  fallback = {},
+  reasoningEffort,
+  telemetry = null,
+  phase = 'structured',
+}) {
+  const startedAt = Date.now();
+  const structuredStep = `model:${phase}`;
+  if (telemetry?.runId) {
+    engine.updateRunProgress(telemetry.runId, {
+      currentPhase: 'model',
+      currentStep: structuredStep,
+      currentTool: null,
+      currentStepStartedAt: isoNow(),
+    });
+  }
+
+  let completed = false;
+  try {
+    const response = await withProviderRetry(
+      () => withModelCallTimeout(
+        provider.chat(
+          sanitizeConversationMessages([
+            ...messages,
+            { role: 'system', content: prompt },
+          ]),
+          [],
+          {
+            model,
+            maxTokens,
+            reasoningEffort: reasoningEffort || engine.getReasoningEffort(providerName, {}),
+          }
+        ),
+        telemetry || {},
+        `${phase} model call`,
+      ),
+      { label: `Engine ${model} (structured)` }
+    );
+    completed = true;
+    if (telemetry?.runId && telemetry?.userId) {
+      recordModelUsage({
+        runId: telemetry.runId,
+        stepId: telemetry.stepId || null,
+        userId: telemetry.userId,
+        agentId: telemetry.agentId || null,
+        provider: providerName,
+        model,
+        phase,
+        usage: response.usage,
+        latencyMs: Date.now() - startedAt,
+      });
+    }
+
+    const parsed = parseJsonObject(response.content || '');
+    const normalizedUsage = normalizeUsage(response.usage);
+    return {
+      value: normalize(parsed || {}, fallback),
+      raw: response.content || '',
+      usage: normalizedUsage?.totalTokens || 0,
+    };
+  } finally {
+    const runMeta = telemetry?.runId ? engine.getRunMeta(telemetry.runId) : null;
+    if (runMeta?.progressLedger?.currentStep === structuredStep) {
+      engine.updateRunProgress(telemetry.runId, {
+        currentPhase: 'idle',
+        currentStep: null,
+        currentTool: null,
+        currentStepStartedAt: null,
+      }, {
+        verified: completed,
+      });
+    }
+  }
+}
+
+async function requestModelResponse(engine, {
+  provider,
+  providerName,
+  model,
+  messages,
+  tools,
+  options,
+  runId,
+  iteration,
+}) {
+  const startedAt = Date.now();
+  const requestMessages = sanitizeConversationMessages(messages);
+  const callOptions = {
+    model,
+    reasoningEffort: engine.getReasoningEffort(providerName, options),
+  };
+
+  const attemptModelCall = async () => {
+    let response = null;
+    let streamContent = '';
+
+    if (options.stream !== false) {
+      let emittedContent = false;
+      const stream = provider.stream(requestMessages, tools, callOptions);
+      const iterator = stream[Symbol.asyncIterator]();
+      try {
+        while (true) {
+          const next = await withModelCallTimeout(
+            iterator.next(),
+            options,
+            `Model stream iteration ${iteration}`,
+          );
+          if (next.done) break;
+          const chunk = next.value;
+          if (chunk.type === 'content') {
+            emittedContent = true;
+            streamContent += chunk.content;
+            engine.emit(options.userId, 'run:stream', {
+              runId,
+              content: sanitizeModelOutput(streamContent, { model }),
+              iteration,
+            });
+          }
+          if (chunk.type === 'done') {
+            response = chunk;
+          }
+          if (chunk.type === 'tool_calls') {
+            response = {
+              content: chunk.content || streamContent,
+              toolCalls: chunk.toolCalls,
+              providerContentBlocks: chunk.providerContentBlocks || null,
+              finishReason: 'tool_calls',
+              usage: chunk.usage || null,
+            };
+          }
+        }
+      } catch (err) {
+        Promise.resolve(iterator.return?.()).catch(() => {});
+        // Once tokens have streamed to the client a retry would duplicate
+        // output, so only the pre-stream window is safe to replay.
+        if (emittedContent) err.__providerRetryUnsafe = true;
+        throw err;
+      }
+    } else {
+      response = await withModelCallTimeout(
+        provider.chat(requestMessages, tools, callOptions),
+        options,
+        `Model iteration ${iteration}`,
+      );
+    }
+
+    return { response, streamContent };
+  };
+
+  const { response, streamContent } = await withProviderRetry(attemptModelCall, {
+    ...(options.retry || {}),
+    label: `Engine ${model}`,
+    isRetryable: (err) => !err?.__providerRetryUnsafe && isTransientError(err),
+    onRetry: ({ attempt, delayMs }) => {
+      engine.emit(options.userId, 'run:interim', {
+        runId,
+        message: `Model service busy; retrying (attempt ${attempt}) in ${Math.max(1, Math.round(delayMs / 1000))}s.`,
+        phase: 'recovering',
+      });
+    },
+  });
+
+  const resolvedResponse = response || {
+    content: streamContent,
+    toolCalls: [],
+    finishReason: 'stop',
+    usage: null,
+  };
+  const hasContent = Boolean(String(resolvedResponse.content || streamContent || '').trim());
+  const hasToolCalls = Array.isArray(resolvedResponse.toolCalls) && resolvedResponse.toolCalls.length > 0;
+  if (!hasContent && !hasToolCalls) {
+    const error = new Error(`Model ${model} returned an empty response.`);
+    error.code = 'MODEL_EMPTY_RESPONSE';
+    throw error;
+  }
+  if (options.runId && options.userId) {
+    recordModelUsage({
+      runId: options.runId,
+      stepId: options.stepId || null,
+      userId: options.userId,
+      agentId: options.agentId || null,
+      provider: providerName,
+      model,
+      phase: options.phase || 'model_turn',
+      usage: resolvedResponse.usage,
+      latencyMs: Date.now() - startedAt,
+      metadata: { iteration },
+    });
+  }
+
+  return {
+    response: resolvedResponse,
+    responseModel: model,
+    streamContent,
+  };
+}
+
+module.exports = {
+  requestModelResponse,
+  requestStructuredJson,
+  resolveModelCallTimeoutMs,
+  withModelCallTimeout,
+};