neoagent 2.2.0 → 2.2.1-beta.0

package/server/services/scheduler/cron_utils.js

@@ -0,0 +1,216 @@
+const MINUTE_MS = 60 * 1000;
+const MONTH_NAMES = new Map([
+  ['jan', 1],
+  ['feb', 2],
+  ['mar', 3],
+  ['apr', 4],
+  ['may', 5],
+  ['jun', 6],
+  ['jul', 7],
+  ['aug', 8],
+  ['sep', 9],
+  ['oct', 10],
+  ['nov', 11],
+  ['dec', 12],
+]);
+const WEEKDAY_NAMES = new Map([
+  ['sun', 0],
+  ['mon', 1],
+  ['tue', 2],
+  ['wed', 3],
+  ['thu', 4],
+  ['fri', 5],
+  ['sat', 6],
+]);
+
+function normalizeCronValue(raw, names = null) {
+  const value = String(raw || '').trim().toLowerCase();
+  if (names?.has(value)) {
+    return names.get(value);
+  }
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isInteger(parsed)) {
+    throw new Error(`Invalid cron value "${raw}"`);
+  }
+  return parsed;
+}
+
+function addRange(values, start, end, step, min, max, fieldName) {
+  if (!Number.isInteger(start) || !Number.isInteger(end)) {
+    throw new Error(`Invalid ${fieldName} range`);
+  }
+  if (start > end) {
+    throw new Error(`Invalid ${fieldName} range "${start}-${end}"`);
+  }
+  if (start < min || end > max) {
+    throw new Error(`${fieldName} range "${start}-${end}" is out of bounds`);
+  }
+  for (let current = start; current <= end; current += step) {
+    values.add(current);
+  }
+}
+
+function parseCronField(field, { min, max, fieldName, names = null, normalize = null }) {
+  const raw = String(field || '').trim();
+  if (!raw) {
+    throw new Error(`Missing ${fieldName} field`);
+  }
+
+  const values = new Set();
+  const wildcard = raw === '*';
+  const parts = raw.split(',');
+
+  for (const part of parts) {
+    const segment = part.trim();
+    if (!segment) continue;
+
+    const [rangePart, stepPart] = segment.split('/');
+    const step = stepPart == null ? 1 : Number.parseInt(stepPart, 10);
+    if (!Number.isInteger(step) || step <= 0) {
+      throw new Error(`Invalid ${fieldName} step "${stepPart}"`);
+    }
+
+    if (rangePart === '*') {
+      addRange(values, min, max, step, min, max, fieldName);
+      continue;
+    }
+
+    if (rangePart.includes('-')) {
+      const [startRaw, endRaw] = rangePart.split('-', 2);
+      let start = normalizeCronValue(startRaw, names);
+      let end = normalizeCronValue(endRaw, names);
+      if (typeof normalize === 'function') {
+        start = normalize(start);
+        end = normalize(end);
+      }
+      addRange(values, start, end, step, min, max, fieldName);
+      continue;
+    }
+
+    let value = normalizeCronValue(rangePart, names);
+    if (typeof normalize === 'function') {
+      value = normalize(value);
+    }
+    if (value < min || value > max) {
+      throw new Error(`${fieldName} value "${rangePart}" is out of bounds`);
+    }
+    values.add(value);
+  }
+
+  return { wildcard, values };
+}
+
+function parseCronExpression(expression) {
+  const fields = String(expression || '').trim().split(/\s+/);
+  if (fields.length !== 5) {
+    throw new Error(`Invalid cron expression "${expression}"`);
+  }
+
+  return {
+    minute: parseCronField(fields[0], {
+      min: 0,
+      max: 59,
+      fieldName: 'minute',
+    }),
+    hour: parseCronField(fields[1], {
+      min: 0,
+      max: 23,
+      fieldName: 'hour',
+    }),
+    dayOfMonth: parseCronField(fields[2], {
+      min: 1,
+      max: 31,
+      fieldName: 'day-of-month',
+    }),
+    month: parseCronField(fields[3], {
+      min: 1,
+      max: 12,
+      fieldName: 'month',
+      names: MONTH_NAMES,
+    }),
+    dayOfWeek: parseCronField(fields[4], {
+      min: 0,
+      max: 6,
+      fieldName: 'day-of-week',
+      names: WEEKDAY_NAMES,
+      normalize: (value) => (value === 7 ? 0 : value),
+    }),
+  };
+}
+
+function matchesCron(date, schedule) {
+  const minute = date.getUTCMinutes();
+  const hour = date.getUTCHours();
+  const dayOfMonth = date.getUTCDate();
+  const month = date.getUTCMonth() + 1;
+  const dayOfWeek = date.getUTCDay();
+
+  if (!schedule.minute.values.has(minute)) return false;
+  if (!schedule.hour.values.has(hour)) return false;
+  if (!schedule.month.values.has(month)) return false;
+
+  const domMatch = schedule.dayOfMonth.values.has(dayOfMonth);
+  const dowMatch = schedule.dayOfWeek.values.has(dayOfWeek);
+
+  if (schedule.dayOfMonth.wildcard && schedule.dayOfWeek.wildcard) {
+    return true;
+  }
+  if (schedule.dayOfMonth.wildcard) {
+    return dowMatch;
+  }
+  if (schedule.dayOfWeek.wildcard) {
+    return domMatch;
+  }
+  return domMatch || dowMatch;
+}
+
+function floorToMinute(date) {
+  return new Date(Math.floor(date.getTime() / MINUTE_MS) * MINUTE_MS);
+}
+
+function findNextRun(expression, fromDate = new Date(), maxLookaheadMinutes = 366 * 24 * 60) {
+  const schedule = parseCronExpression(expression);
+  const cursor = floorToMinute(fromDate);
+  cursor.setUTCSeconds(0, 0);
+
+  for (let index = 1; index <= maxLookaheadMinutes; index += 1) {
+    const candidate = new Date(cursor.getTime() + (index * MINUTE_MS));
+    if (matchesCron(candidate, schedule)) {
+      return candidate;
+    }
+  }
+  return null;
+}
+
+function getMinimumIntervalMinutes(expression, occurrenceCount = 3) {
+  const matches = [];
+  let cursor = new Date();
+  for (let index = 0; index < occurrenceCount; index += 1) {
+    const next = findNextRun(expression, cursor);
+    if (!next) {
+      break;
+    }
+    matches.push(next);
+    cursor = new Date(next.getTime());
+  }
+  if (matches.length < 2) {
+    return null;
+  }
+
+  let minInterval = Number.POSITIVE_INFINITY;
+  for (let index = 1; index < matches.length; index += 1) {
+    const intervalMinutes = Math.round((matches[index].getTime() - matches[index - 1].getTime()) / MINUTE_MS);
+    if (intervalMinutes < minInterval) {
+      minInterval = intervalMinutes;
+    }
+  }
+
+  return Number.isFinite(minInterval) ? minInterval : null;
+}
+
+module.exports = {
+  findNextRun,
+  getMinimumIntervalMinutes,
+  matchesCron,
+  parseCronExpression,
+};

package/server/services/voice/bufferedLiveRelayAdapter.js

@@ -1,10 +1,11 @@
 'use strict';
 
+const { getProviderRuntimeConfig } = require('../ai/models');
 const { resolveSttModel, transcribeVoiceInput } = require('./providers');
 const { writeTempAudioFile, removeTempFile } = require('./liveAudio');
 
-const DEFAULT_PARTIAL_DEBOUNCE_MS = 1200;
-const DEFAULT_MIN_PARTIAL_BYTES = 12000;
+const DEFAULT_PARTIAL_DEBOUNCE_MS = 700;
+const DEFAULT_MIN_PARTIAL_BYTES = 8000;
 
 class BufferedLiveRelayAdapter {
   constructor({
@@ -32,20 +33,32 @@ class BufferedLiveRelayAdapter {
     }
   }
 
-  async onInputStart(session) {
-    session.resetInput();
+  async onInputStart(session, options = {}) {
+    session.startTurn(options.turnId, options.mimeType);
     this._clearPartialTimer(session.id);
     this._partialInFlight.delete(session.id);
   }
 
   async appendAudioChunk(session, audioBytes, options = {}) {
-    session.appendInputChunk(audioBytes, options.mimeType);
+    const appendResult = session.appendInputChunk(audioBytes, options.mimeType, {
+      turnId: options.turnId,
+      sequence: options.sequence,
+    });
     this._schedulePartialTranscript(session);
+    return appendResult;
   }
 
-  async commitInput(session) {
+  async commitInput(session, options = {}) {
     this._clearPartialTimer(session.id);
-    const audioBytes = session.getInputAudioBuffer();
+    const commitState = session.markCommitPending(options.turnId, options.finalSequence);
+    if (!commitState.ready) {
+      throw new Error(
+        `Voice input is incomplete for commit (${commitState.receivedThrough}/${commitState.finalSequence}).`,
+      );
+    }
+    const audioBytes = session.getInputAudioBuffer({
+      throughSequence: commitState.finalSequence,
+    });
     if (!audioBytes.length) {
       return '';
     }
@@ -54,6 +67,7 @@ class BufferedLiveRelayAdapter {
         model: session.voiceSettings?.liveSttModel,
         userId: session.userId,
         agentId: session.agentId,
+        timeoutMs: 20000,
       });
     } finally {
       // Release buffered audio immediately after commit so completed turns do
@@ -74,11 +88,17 @@ class BufferedLiveRelayAdapter {
       }
       this._partialInFlight.add(session.id);
       try {
-        const snapshot = session.getInputAudioBuffer();
+        const snapshot = session.getInputAudioBuffer({
+          contiguousOnly: true,
+        });
+        if (!snapshot.length) {
+          return;
+        }
         const transcript = await this._transcribeAudioSnapshot(snapshot, session.inputMimeType, {
           model: session.voiceSettings?.liveSttModel,
           userId: session.userId,
           agentId: session.agentId,
+          timeoutMs: 6000,
         });
         if (transcript) {
           await session.publishTranscriptPartial(transcript);
@@ -104,19 +124,67 @@ class BufferedLiveRelayAdapter {
   async _transcribeAudioSnapshot(audioBytes, mimeType, options = {}) {
     const { filePath, mimeType: fileMimeType } = await writeTempAudioFile(audioBytes, mimeType);
     try {
-      const model = resolveSttModel(this.provider, options.model);
-      const transcript = await transcribeVoiceInput(filePath, {
-        provider: this.provider,
-        model,
-        mimeType: fileMimeType,
-        userId: options.userId,
-        agentId: options.agentId,
-      });
-      return String(transcript || '').trim();
+      let lastError = null;
+      for (const attempt of this._buildSttAttempts(options)) {
+        try {
+          const transcript = await transcribeVoiceInput(filePath, {
+            provider: attempt.provider,
+            model: attempt.model,
+            mimeType: fileMimeType,
+            userId: options.userId,
+            agentId: options.agentId,
+            apiKey: attempt.apiKey,
+            baseUrl: attempt.baseUrl,
+            timeoutMs: options.timeoutMs,
+          });
+          return String(transcript || '').trim();
+        } catch (error) {
+          lastError = error;
+        }
+      }
+      throw lastError || new Error('Voice transcription failed.');
     } finally {
       await removeTempFile(filePath);
     }
   }
+
+  _buildSttAttempts(options = {}) {
+    const attempts = [];
+    const providers = [
+      this.provider,
+      ...['openai', 'deepgram', 'gemini'].filter((provider) => provider !== this.provider),
+    ];
+    for (const provider of providers) {
+      const runtime = this._resolveProviderRuntime(provider, options.userId, options.agentId);
+      attempts.push({
+        provider,
+        model: resolveSttModel(provider, provider === this.provider ? options.model : ''),
+        apiKey: runtime.apiKey,
+        baseUrl: runtime.baseUrl,
+      });
+    }
+    return attempts;
+  }
+
+  _resolveProviderRuntime(provider, userId, agentId) {
+    const normalizedProvider = String(provider || '').trim().toLowerCase();
+    if (!normalizedProvider || normalizedProvider === 'deepgram') {
+      return { apiKey: '', baseUrl: '' };
+    }
+    try {
+      const runtime = getProviderRuntimeConfig(
+        userId,
+        normalizedProvider === 'gemini' ? 'google' : normalizedProvider,
+        agentId,
+      );
+      return {
+        apiKey: typeof runtime.apiKey === 'string' ? runtime.apiKey.trim() : '',
+        baseUrl: typeof runtime.baseUrl === 'string' ? runtime.baseUrl.trim() : '',
+      };
+    } catch {
+      return { apiKey: '', baseUrl: '' };
+    }
+  }
 }
 
 module.exports = {

package/server/services/voice/liveSession.js

@@ -21,8 +21,12 @@ class VoiceLiveSession {
     this.currentRunId = null;
     this.interrupted = false;
     this.inputMimeType = 'audio/pcm;rate=16000;channels=1';
-    this.inputChunks = [];
+    this.inputChunks = new Map();
     this.inputBytes = 0;
+    this.activeTurnId = '';
+    this.highestContiguousSequence = -1;
+    this.highestReceivedSequence = -1;
+    this.finalSequence = null;
     this.lastPartialTranscript = '';
     this.lastFinalTranscript = '';
     this.lastAssistantText = '';
@@ -32,8 +36,12 @@ class VoiceLiveSession {
 
   resetInput(mimeType = 'audio/pcm;rate=16000;channels=1') {
     this.inputMimeType = String(mimeType || this.inputMimeType).trim() || 'audio/pcm;rate=16000;channels=1';
-    this.inputChunks = [];
+    this.inputChunks = new Map();
     this.inputBytes = 0;
+    this.activeTurnId = '';
+    this.highestContiguousSequence = -1;
+    this.highestReceivedSequence = -1;
+    this.finalSequence = null;
     this.lastPartialTranscript = '';
   }
 
@@ -45,20 +53,112 @@ class VoiceLiveSession {
     this.interrupted = false;
   }
 
-  appendInputChunk(chunk, mimeType = null) {
+  startTurn(turnId, mimeType = null) {
+    this.resetInput(mimeType || this.inputMimeType);
+    this.activeTurnId = String(turnId || '').trim();
+  }
+
+  appendInputChunk(chunk, mimeType = null, options = {}) {
     if (mimeType) {
       this.inputMimeType = String(mimeType).trim() || this.inputMimeType;
     }
+    const turnId = String(options.turnId || '').trim();
+    if (turnId && this.activeTurnId && turnId !== this.activeTurnId) {
+      throw new Error('Audio chunk turn does not match the active voice turn.');
+    }
+    if (turnId && !this.activeTurnId) {
+      this.activeTurnId = turnId;
+    }
+    const sequence = Number(options.sequence);
+    if (!Number.isInteger(sequence) || sequence < 0) {
+      throw new Error('Audio chunk sequence must be a non-negative integer.');
+    }
     const payload = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk || []);
-    if (payload.length === 0) return;
-    this.inputChunks.push(payload);
+    if (payload.length === 0) {
+      return {
+        duplicate: false,
+        receivedThrough: this.highestContiguousSequence,
+        highestReceived: this.highestReceivedSequence,
+      };
+    }
+    if (this.inputChunks.has(sequence)) {
+      return {
+        duplicate: true,
+        receivedThrough: this.highestContiguousSequence,
+        highestReceived: this.highestReceivedSequence,
+      };
+    }
+    this.inputChunks.set(sequence, payload);
     this.inputBytes += payload.length;
+    if (sequence > this.highestReceivedSequence) {
+      this.highestReceivedSequence = sequence;
+    }
+    while (this.inputChunks.has(this.highestContiguousSequence + 1)) {
+      this.highestContiguousSequence += 1;
+    }
+    return {
+      duplicate: false,
+      receivedThrough: this.highestContiguousSequence,
+      highestReceived: this.highestReceivedSequence,
+    };
+  }
+
+  markCommitPending(turnId, finalSequence) {
+    const normalizedTurnId = String(turnId || '').trim();
+    if (normalizedTurnId && this.activeTurnId && normalizedTurnId !== this.activeTurnId) {
+      throw new Error('Voice commit turn does not match the active voice turn.');
+    }
+    if (normalizedTurnId && !this.activeTurnId) {
+      this.activeTurnId = normalizedTurnId;
+    }
+    const normalizedFinalSequence = Number(finalSequence);
+    if (!Number.isInteger(normalizedFinalSequence) || normalizedFinalSequence < 0) {
+      throw new Error('Voice commit finalSequence must be a non-negative integer.');
+    }
+    this.finalSequence = normalizedFinalSequence;
+    return {
+      finalSequence: this.finalSequence,
+      receivedThrough: this.highestContiguousSequence,
+      ready: this.hasInputThrough(normalizedFinalSequence),
+    };
+  }
+
+  hasInputThrough(sequence) {
+    const normalizedSequence = Number(sequence);
+    if (!Number.isInteger(normalizedSequence) || normalizedSequence < 0) {
+      return false;
+    }
+    return this.highestContiguousSequence >= normalizedSequence;
   }
 
-  getInputAudioBuffer() {
-    return this.inputChunks.length === 1
-      ? Buffer.from(this.inputChunks[0])
-      : Buffer.concat(this.inputChunks);
+  getInputAudioBuffer(options = {}) {
+    const contiguousOnly = options.contiguousOnly !== false;
+    const throughSequence = Number.isInteger(options.throughSequence)
+      ? Number(options.throughSequence)
+      : null;
+    const maxSequence = throughSequence != null
+      ? throughSequence
+      : (contiguousOnly ? this.highestContiguousSequence : this.highestReceivedSequence);
+    if (!Number.isInteger(maxSequence) || maxSequence < 0) {
+      return Buffer.alloc(0);
+    }
+    const ordered = [];
+    for (let sequence = 0; sequence <= maxSequence; sequence += 1) {
+      const chunk = this.inputChunks.get(sequence);
+      if (!chunk) {
+        if (contiguousOnly || throughSequence != null) {
+          break;
+        }
+        continue;
+      }
+      ordered.push(chunk);
+    }
+    if (ordered.length === 0) {
+      return Buffer.alloc(0);
+    }
+    return ordered.length === 1
+      ? Buffer.from(ordered[0])
+      : Buffer.concat(ordered);
   }
 
   async setState(state, extra = {}) {

package/server/services/voice/providers.js

@@ -35,6 +35,25 @@ const GEMINI_API_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/mo
 const DEFAULT_GEMINI_TRANSCRIPTION_PROMPT =
   'Transcribe this audio verbatim. Return only the transcript text.';
 
+function withTimeout(promise, timeoutMs, label) {
+  const normalizedTimeout = Number(timeoutMs);
+  if (!Number.isFinite(normalizedTimeout) || normalizedTimeout <= 0) {
+    return promise;
+  }
+  let timer = null;
+  const timeoutPromise = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      reject(new Error(`${label} timed out after ${normalizedTimeout}ms.`));
+    }, normalizedTimeout);
+    timer.unref?.();
+  });
+  return Promise.race([promise, timeoutPromise]).finally(() => {
+    if (timer) {
+      clearTimeout(timer);
+    }
+  });
+}
+
 function readSharedApiKeys() {
   try {
     const keysPath = path.join(AGENT_DATA_DIR, 'API_KEYS.json');
@@ -287,14 +306,16 @@ async function transcribeWithGemini(filePath, model, mimeType, options = {}) {
 async function transcribeVoiceInput(filePath, options = {}) {
   const provider = normalizeSttProvider(options.provider);
   const model = resolveSttModel(provider, options.model);
+  let request = null;
 
   if (provider === 'openai') {
-    return transcribeWithOpenAi(filePath, model, options);
-  }
-  if (provider === 'deepgram') {
-    return transcribeWithDeepgram(filePath, options.mimeType);
+    request = transcribeWithOpenAi(filePath, model, options);
+  } else if (provider === 'deepgram') {
+    request = transcribeWithDeepgram(filePath, options.mimeType);
+  } else {
+    request = transcribeWithGemini(filePath, model, options.mimeType, options);
   }
-  return transcribeWithGemini(filePath, model, options.mimeType, options);
+  return withTimeout(request, options.timeoutMs, `${provider} STT`);
 }
 
 async function synthesizeWithOpenAi(text, model, voice, options = {}) {
@@ -545,14 +566,16 @@ async function synthesizeVoiceReply(text, options = {}) {
   }
 
   const { provider, model, voice } = normalizeVoiceSynthesisOptions(options);
+  let request = null;
 
   if (provider === 'openai') {
-    return synthesizeWithOpenAi(content, model, voice, options);
-  }
-  if (provider === 'deepgram') {
-    return synthesizeWithDeepgram(content, model);
+    request = synthesizeWithOpenAi(content, model, voice, options);
+  } else if (provider === 'deepgram') {
+    request = synthesizeWithDeepgram(content, model);
+  } else {
+    request = synthesizeWithGemini(content, model, voice, options);
   }
-  return synthesizeWithGemini(content, model, voice, options);
+  return withTimeout(request, options.timeoutMs, `${provider} TTS`);
 }
 
 // Minimum characters before flushing a sentence chunk to TTS to avoid tiny requests.
@@ -590,15 +613,18 @@ async function synthesizeVoiceReplyStream(text, options = {}, onChunk) {
   const { provider, model, voice } = normalizeVoiceSynthesisOptions(options);
   const chunks = splitIntoSentenceChunks(content);
 
-  for (const chunk of chunks) {
-    if (provider === 'openai') {
-      await streamWithOpenAi(chunk, model, voice, options, onChunk);
-    } else if (provider === 'deepgram') {
-      await streamWithDeepgram(chunk, model, onChunk);
-    } else {
-      await streamWithGemini(chunk, model, voice, options, onChunk);
+  const run = (async () => {
+    for (const chunk of chunks) {
+      if (provider === 'openai') {
+        await streamWithOpenAi(chunk, model, voice, options, onChunk);
+      } else if (provider === 'deepgram') {
+        await streamWithDeepgram(chunk, model, onChunk);
+      } else {
+        await streamWithGemini(chunk, model, voice, options, onChunk);
+      }
     }
-  }
+  })();
+  await withTimeout(run, options.timeoutMs, `${provider} TTS stream`);
 }
 
 module.exports = {