neoagent 2.2.1-beta.2 → 2.2.1-beta.3

package/server/services/ai/tools.js

@@ -1160,7 +1160,7 @@ function getAvailableTools(app, options = {}) {
                 properties: {
                     snapshot: {
                         type: 'object',
-                        description: 'Structured widget snapshot payload containing title, optional subtitle/body/metric/trend/rows/chips/iconToken/accentToken/updatedAt/deepLink.'
+                        description: 'Structured widget snapshot payload containing a strong title, optional kicker/subtitle/body, primary and supporting metrics, optional progress, rows, chips, icon/accent/background tokens, optional surfaceColor, updatedAt, and deepLink.'
                     }
                 },
                 required: ['snapshot']

package/server/services/voice/providers.js

@@ -34,6 +34,8 @@ const DEFAULT_TTS_VOICES = Object.freeze({
 const GEMINI_API_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/models';
 const DEFAULT_GEMINI_TRANSCRIPTION_PROMPT =
   'Transcribe this audio verbatim. Return only the transcript text.';
+const EMOJI_SPEECH_REGEX =
+  /[\p{Extended_Pictographic}\p{Emoji_Presentation}\p{Regional_Indicator}\u200D\uFE0F\u20E3]/gu;
 
 function withTimeout(promise, timeoutMs, label) {
   const normalizedTimeout = Number(timeoutMs);
@@ -54,6 +56,19 @@ function withTimeout(promise, timeoutMs, label) {
   });
 }
 
+function sanitizeSpeechText(value) {
+  const text = String(value || '');
+  if (!text) {
+    return '';
+  }
+  return text
+    .replace(EMOJI_SPEECH_REGEX, ' ')
+    .replace(/[ \t]{2,}/g, ' ')
+    .replace(/\s+\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
 function readSharedApiKeys() {
   try {
     const keysPath = path.join(AGENT_DATA_DIR, 'API_KEYS.json');
@@ -641,6 +656,7 @@ module.exports = {
   resolveTtsModel,
   resolveTtsVoice,
   normalizeVoiceSynthesisOptions,
+  sanitizeSpeechText,
   guessExtFromMimeType,
   splitIntoSentenceChunks,
   transcribeVoiceInput,

package/server/services/voice/runtimeManager.js

@@ -7,7 +7,12 @@ const { getVoiceRuntimeSettings } = require('./liveSettings');
 const { VoiceLiveSession } = require('./liveSession');
 const { OpenAiLiveRelayAdapter } = require('./openaiLiveRelayAdapter');
 const { GeminiLiveRelayAdapter } = require('./geminiLiveRelayAdapter');
-const { synthesizeVoiceReply, normalizeVoiceSynthesisOptions, synthesizeVoiceReplyStream } = require('./providers');
+const {
+  synthesizeVoiceReply,
+  normalizeVoiceSynthesisOptions,
+  synthesizeVoiceReplyStream,
+  sanitizeSpeechText,
+} = require('./providers');
 const { VoiceAgentBridge } = require('./agentBridge');
 
 class VoiceRuntimeManager {
@@ -296,17 +301,19 @@ class VoiceRuntimeManager {
       model: session.voiceSettings?.liveTtsModel,
       voice: session.voiceSettings?.liveVoice,
     });
+    const spokenContent = sanitizeSpeechText(content);
 
     let index = 0;
     let streamError = null;
     const ttsAttempts = this.#buildTtsAttemptOrder(session, voiceOptions);
-    try {
+    if (spokenContent) {
+      try {
       for (const attempt of ttsAttempts) {
         index = 0;
         streamError = null;
         try {
           await synthesizeVoiceReplyStream(
-            content,
+            spokenContent,
             attempt,
             async ({ audioBytes, mimeType }) => {
               if (session.closed || session.interrupted) return;
@@ -326,8 +333,9 @@ class VoiceRuntimeManager {
           streamError = String(error?.message || error || 'Voice playback failed.');
         }
       }
-    } catch (error) {
-      streamError = String(error?.message || error || 'Voice playback failed.');
+      } catch (error) {
+        streamError = String(error?.message || error || 'Voice playback failed.');
+      }
     }
 
     if (!streamError && !session.closed && !session.interrupted) {

package/server/services/voice/turnRunner.js

@@ -5,7 +5,11 @@ const { getProviderRuntimeConfig } = require('../ai/models');
 const { buildAgentRunContext } = require('../ai/runContext');
 const { buildDirectVoiceContext } = require('./message');
 const { analyzeVoiceAssistantScreenshot } = require('./screenshotContext');
-const { synthesizeVoiceReply, normalizeVoiceSynthesisOptions } = require('./providers');
+const {
+  synthesizeVoiceReply,
+  normalizeVoiceSynthesisOptions,
+  sanitizeSpeechText,
+} = require('./providers');
 const {
   VOICE_HISTORY_WINDOW,
   buildDirectVoiceRunOptions,
@@ -132,6 +136,14 @@ async function runVoiceTranscriptTurn({
   let modelUsed = voiceOptions.model;
   let voiceUsed = voiceOptions.voice;
   if (synthesize !== false) {
+    const spokenReplyText = sanitizeSpeechText(replyText);
+    if (!spokenReplyText) {
+      synthesized = {
+        mimeType: 'audio/mpeg',
+        audioBytes: Buffer.alloc(0),
+      };
+      ttsError = null;
+    } else {
     const attemptProviders = [
       voiceOptions.provider,
       ...['openai', 'deepgram', 'gemini'].filter((provider) => provider !== voiceOptions.provider),
@@ -145,7 +157,7 @@ async function runVoiceTranscriptTurn({
       });
       const runtime = resolveProviderRuntime(userId, agentId, provider);
       try {
-        synthesized = await synthesizeVoiceReply(replyText, {
+        synthesized = await synthesizeVoiceReply(spokenReplyText, {
           ...normalized,
           apiKey: runtime.apiKey,
           baseUrl: runtime.baseUrl,
@@ -167,6 +179,7 @@ async function runVoiceTranscriptTurn({
         audioBytes: Buffer.alloc(0),
       };
     }
+    }
   } else {
     synthesized = {
       mimeType: 'audio/mpeg',

package/server/services/widgets/service.js

@@ -33,6 +33,15 @@ function normalizeOptionalText(value, maxLength = 4000) {
   return normalized || null;
 }
 
+function normalizeSurfaceColor(value) {
+  const normalized = normalizeOptionalText(value, 16);
+  if (!normalized) return null;
+  const prefixed = normalized.startsWith('#') ? normalized : `#${normalized}`;
+  return /^#(?:[0-9A-Fa-f]{6}|[0-9A-Fa-f]{8})$/.test(prefixed)
+    ? prefixed.toUpperCase()
+    : null;
+}
+
 function buildWidgetRefreshTaskName(name) {
   return `Refresh widget: ${normalizeText(name, 120)}`;
 }
@@ -116,6 +125,28 @@ function normalizeTrend(input) {
   return { label, direction };
 }
 
+function normalizeOptionalNumber(input, { min = null, max = null } = {}) {
+  if (input == null || input === '') return null;
+  const value = Number(input);
+  if (!Number.isFinite(value)) return null;
+  if (min != null && value < min) return min;
+  if (max != null && value > max) return max;
+  return value;
+}
+
+function normalizeProgress(input) {
+  if (!input || typeof input !== 'object' || Array.isArray(input)) return null;
+  const raw = parseJsonObject(input, {});
+  const value = normalizeOptionalNumber(raw.value, { min: 0 });
+  const max = normalizeOptionalNumber(raw.max, { min: 0 });
+  if (value == null || max == null || max <= 0) return null;
+  return {
+    value: Math.min(value, max),
+    max,
+    label: normalizeOptionalText(raw.label, 60),
+  };
+}
+
 function normalizeRows(input) {
   if (!Array.isArray(input)) return [];
   return input
@@ -170,14 +201,23 @@ function validateSnapshotPayload(widget, snapshot = {}) {
     template: widget.template,
     layoutVariant: widget.layoutVariant,
     title,
+    kicker: normalizeOptionalText(payload.kicker, 80),
     subtitle: normalizeOptionalText(payload.subtitle, 160),
     body: normalizeOptionalText(payload.body, 600),
     metric: normalizeOptionalText(payload.metric, 64),
+    metricLabel: normalizeOptionalText(payload.metricLabel, 80),
+    secondaryMetric: normalizeOptionalText(payload.secondaryMetric, 64),
+    secondaryLabel: normalizeOptionalText(payload.secondaryLabel, 80),
+    tertiaryMetric: normalizeOptionalText(payload.tertiaryMetric, 64),
+    tertiaryLabel: normalizeOptionalText(payload.tertiaryLabel, 80),
     trend: normalizeTrend(payload.trend),
+    progress: normalizeProgress(payload.progress),
     rows: normalizeRows(payload.rows),
     chips: normalizeChips(payload.chips),
     iconToken: normalizeOptionalText(payload.iconToken, 40),
     accentToken: normalizeOptionalText(payload.accentToken, 40),
+    backgroundToken: normalizeOptionalText(payload.backgroundToken, 40),
+    surfaceColor: normalizeSurfaceColor(payload.surfaceColor),
     updatedAt: normalizeOptionalText(payload.updatedAt, 80) || new Date().toISOString(),
     deepLink: normalizeOptionalText(payload.deepLink, 200) || `widget:${widget.id}`,
   };
@@ -484,10 +524,20 @@ class WidgetService {
       'You are updating a structured product widget. Keep the layout fixed. Refresh only the content snapshot.',
       'Use fresh tools for time-sensitive claims. Do not rely on stale memory for live data such as weather, markets, incidents, or schedules.',
       'After gathering the latest information, call save_widget_snapshot exactly once with a payload matching this schema:',
-      '{"title":"","subtitle":"","body":"","metric":"","trend":{"label":"","direction":"flat"},"rows":[{"label":"","value":""}],"chips":[""],"iconToken":"","accentToken":"","updatedAt":"","deepLink":""}',
+      '{"title":"","kicker":"","subtitle":"","body":"","metric":"","metricLabel":"","secondaryMetric":"","secondaryLabel":"","tertiaryMetric":"","tertiaryLabel":"","trend":{"label":"","direction":"flat"},"progress":{"value":0,"max":100,"label":""},"rows":[{"label":"","value":""}],"chips":[""],"iconToken":"","accentToken":"","backgroundToken":"","surfaceColor":"","updatedAt":"","deepLink":""}',
       'Rules:',
       '- Do not change the template or layout variant.',
       '- Keep rows to at most 3 and chips to at most 3.',
+      '- Prefer concrete data over generic prose. Use metric + supporting fields whenever live data exists.',
+      '- Make the widget immediately useful at a glance. Avoid filler copy, duplicated labels, or repeating the widget name unless it helps identify the subject.',
+      '- For stat widgets, use title to identify the subject, metric for the main live value, and secondary or tertiary metrics for the next most useful facts.',
+      '- For summary widgets, keep body concise and information-dense. Use kicker or subtitle for the context, not for repeated metadata.',
+      '- For list widgets, rows should be concrete current items with short labels and values. Do not use rows for vague prose.',
+      '- For weather-style widgets, include real temperature/condition/wind/precipitation when available and choose a fitting accent/background token such as sunny, rain, storm, night, or cloud.',
+      '- For vehicle-style widgets, include battery or fuel state, range, odometer or distance, and choose a color token or surfaceColor when the vehicle color is known.',
+      '- Use backgroundToken and accentToken to reflect the actual state of the data, not a default theme.',
+      '- If the subject exposes a progress-like state such as battery charge, tank level, or completion, populate progress with truthful values.',
+      '- Never output placeholders such as "null", "n/a", "---", or invented values.',
       '- If the data source fails, explain the problem briefly in body and still save a truthful degraded snapshot if possible.',
       '- If nothing useful can be produced safely, say so clearly instead of inventing content.',
       '',