oomi-ai 0.2.16 → 0.2.17

package/README.md

@@ -133,7 +133,7 @@ That bridge:
 This is the part of the package most likely to matter when debugging voice turn failures.
 
 For managed voice replies, the extension also preserves an explicit hidden `metadata.spoken` sidecar when upstream provides one.
-If upstream does not provide one, the extension now synthesizes a conservative hidden fallback from the visible assistant text so backend TTS can speak a cleaner version without changing user-visible chat.
+If upstream does not provide one, the extension now synthesizes a bounded hidden fallback from the visible assistant text so backend TTS can speak a cleaner and more varied version without changing user-visible chat.
 
 ## Bridge Health States
 

package/agent_instructions.md

@@ -164,13 +164,15 @@ Rules:
 - visible `content` remains the source of truth for Oomi chat rendering
 - for managed voice replies, include `metadata.spoken` when delivery benefits from cleaner phrasing or explicit speaking guidance
 - `metadata.spoken.text` is for backend TTS only
+- `metadata.spoken.language` should be one of the supported Qwen language values such as `English`
+- `metadata.spoken.segments` can carry bounded per-segment prosody for pace, pitch, volume, and pause timing
 - `metadata.spoken.instructions` should be natural-language guidance, not raw bracket tags
 - `metadata.spoken.style` is optional metadata for debugging/future mapping
 - if no hidden speech sidecar exists, Oomi falls back to speaking the visible assistant text
 
 Current plugin behavior:
 - if you provide `metadata.spoken`, the plugin preserves it unchanged
-- if you do not provide `metadata.spoken`, the plugin now synthesizes a conservative hidden fallback from visible assistant text for backend TTS
+- if you do not provide `metadata.spoken`, the plugin now synthesizes a bounded hidden fallback from visible assistant text for backend TTS
 - visible chat text is still never rewritten by the plugin
 
 ## Avatar Commands

package/openclaw.extension.js

@@ -178,6 +178,154 @@ function extractCorrelationId(payload) {
   return '';
 }
 
+const BOUNDED_LANGUAGE_TYPES = new Set([
+  'Auto',
+  'Chinese',
+  'English',
+  'German',
+  'Italian',
+  'Portuguese',
+  'Spanish',
+  'Japanese',
+  'Korean',
+  'French',
+  'Russian',
+]);
+
+const BOUNDED_PACE_VALUES = new Set(['very_slow', 'slow', 'medium', 'medium_fast', 'fast']);
+const BOUNDED_PITCH_VALUES = new Set(['low', 'slightly_low', 'neutral', 'slightly_high', 'high']);
+const BOUNDED_ENERGY_VALUES = new Set(['soft', 'calm', 'warm', 'bright', 'intense']);
+const BOUNDED_VOLUME_VALUES = new Set(['soft', 'normal', 'projected']);
+
+function inferSpokenLanguage(text) {
+  const normalized = toString(text);
+  if (!normalized) return 'English';
+  return 'English';
+}
+
+function normalizeSpokenSegment(segment) {
+  if (!segment || typeof segment !== 'object' || Array.isArray(segment)) return null;
+
+  const text = toString(segment.text);
+  if (!text) return null;
+
+  const normalized = { text };
+  const pace = toString(segment.pace);
+  const pitch = toString(segment.pitch);
+  const energy = toString(segment.energy);
+  const volume = toString(segment.volume);
+  const pauseAfterMs = toNumber(segment.pause_after_ms, 0, { min: 0, max: 1200 });
+
+  if (BOUNDED_PACE_VALUES.has(pace)) normalized.pace = pace;
+  if (BOUNDED_PITCH_VALUES.has(pitch)) normalized.pitch = pitch;
+  if (BOUNDED_ENERGY_VALUES.has(energy)) normalized.energy = energy;
+  if (BOUNDED_VOLUME_VALUES.has(volume)) normalized.volume = volume;
+  normalized.pause_after_ms = pauseAfterMs;
+
+  return normalized;
+}
+
+function splitSpeechSegments(text) {
+  const normalized = normalizeSpeechText(text);
+  if (!normalized) return [];
+
+  const baseSegments = normalized
+    .split(/(?<=[.!?])\s+/)
+    .map((segment) => segment.trim())
+    .filter(Boolean);
+
+  const segments = [];
+  for (const segment of baseSegments) {
+    if (segment.length <= 96) {
+      segments.push(segment);
+      continue;
+    }
+
+    const clauseParts = segment
+      .split(/,\s+/)
+      .map((part) => part.trim())
+      .filter(Boolean);
+
+    if (clauseParts.length > 1) {
+      for (let index = 0; index < clauseParts.length; index += 1) {
+        const part = clauseParts[index];
+        const needsComma = index < clauseParts.length - 1 && !/[.!?]$/.test(part);
+        segments.push(needsComma ? `${part},` : part);
+      }
+      continue;
+    }
+
+    segments.push(segment);
+  }
+
+  if (segments.length <= 5) return segments;
+
+  return [...segments.slice(0, 4), segments.slice(4).join(' ').trim()];
+}
+
+function inferSegmentStyle(segmentText, index, totalSegments) {
+  const normalized = segmentText.toLowerCase();
+  const exclamatory = /!/.test(segmentText) || /\b(hell yeah|awesome|amazing|stoked|love|perfect|great)\b/.test(normalized);
+  const curious = /\?/.test(segmentText);
+  const reflective =
+    /\b(i think|i'm|i am|i've|i have|lately|right now|before this|each time|understand|it feels like)\b/.test(normalized) ||
+    segmentText.length > 60;
+
+  if (curious) {
+    return {
+      pace: 'medium',
+      pitch: 'slightly_high',
+      energy: 'warm',
+      volume: 'normal',
+      pause_after_ms: 0,
+    };
+  }
+
+  if (exclamatory) {
+    return {
+      pace: 'medium_fast',
+      pitch: 'slightly_high',
+      energy: 'bright',
+      volume: 'normal',
+      pause_after_ms: index < totalSegments - 1 ? 220 : 0,
+    };
+  }
+
+  if (reflective) {
+    return {
+      pace: 'medium',
+      pitch: 'neutral',
+      energy: 'warm',
+      volume: 'normal',
+      pause_after_ms: index < totalSegments - 1 ? 260 : 0,
+    };
+  }
+
+  return {
+    pace: 'medium',
+    pitch: 'neutral',
+    energy: 'warm',
+    volume: 'normal',
+    pause_after_ms: index < totalSegments - 1 ? 180 : 0,
+  };
+}
+
+function synthesizeSpokenSegments(text) {
+  const language = inferSpokenLanguage(text);
+  const rawSegments = splitSpeechSegments(text);
+  if (rawSegments.length === 0) return null;
+
+  const segments = rawSegments.map((segmentText, index) => ({
+    text: segmentText,
+    ...inferSegmentStyle(segmentText, index, rawSegments.length),
+  }));
+
+  return {
+    language,
+    segments,
+  };
+}
+
 function normalizeSpokenMetadata(spoken) {
   if (!spoken || typeof spoken !== 'object' || Array.isArray(spoken)) return null;
 
@@ -185,12 +333,33 @@ function normalizeSpokenMetadata(spoken) {
   if (!text) return null;
 
   const normalized = { text };
+  const language = toString(spoken.language);
+  if (BOUNDED_LANGUAGE_TYPES.has(language)) {
+    normalized.language = language;
+  }
+
+  const explicitSegments =
+    Array.isArray(spoken.segments)
+      ? spoken.segments.map((segment) => normalizeSpokenSegment(segment)).filter(Boolean)
+      : [];
+  if (explicitSegments.length > 0) {
+    normalized.segments = explicitSegments;
+  }
+
   const instructions = toString(spoken.instructions);
   if (instructions) normalized.instructions = instructions;
   if (spoken.style && typeof spoken.style === 'object' && !Array.isArray(spoken.style)) {
     normalized.style = spoken.style;
   }
 
+  const fallbackSegments = synthesizeSpokenSegments(text);
+  if (!normalized.language && fallbackSegments?.language) {
+    normalized.language = fallbackSegments.language;
+  }
+  if (!normalized.segments && fallbackSegments?.segments?.length) {
+    normalized.segments = fallbackSegments.segments;
+  }
+
   return normalized;
 }
 
@@ -216,6 +385,7 @@ function normalizeSpeechText(text) {
 function inferSpokenMetadataFromContent(content) {
   const text = normalizeSpeechText(toString(content));
   if (!text) return null;
+  const synthesized = synthesizeSpokenSegments(text);
 
   const normalized = text.toLowerCase();
   const upbeat =
@@ -228,6 +398,8 @@ function inferSpokenMetadataFromContent(content) {
   if (upbeat) {
     return {
       text,
+      language: synthesized?.language || 'English',
+      segments: synthesized?.segments,
       instructions: 'Speak with warm, upbeat conversational energy and natural pacing.',
       style: { emotion: 'upbeat', energy: 'medium' },
     };
@@ -236,6 +408,8 @@ function inferSpokenMetadataFromContent(content) {
   if (gentle) {
     return {
       text,
+      language: synthesized?.language || 'English',
+      segments: synthesized?.segments,
       instructions: 'Speak gently and reassuringly, with a calm pace and soft emphasis.',
       style: { emotion: 'gentle', energy: 'low' },
     };
@@ -244,6 +418,8 @@ function inferSpokenMetadataFromContent(content) {
   if (curious) {
     return {
       text,
+      language: synthesized?.language || 'English',
+      segments: synthesized?.segments,
       instructions: 'Speak naturally with curious, engaged intonation and a conversational pace.',
       style: { emotion: 'curious', energy: 'medium' },
     };
@@ -251,6 +427,8 @@ function inferSpokenMetadataFromContent(content) {
 
   return {
     text,
+    language: synthesized?.language || 'English',
+    segments: synthesized?.segments,
     instructions: 'Speak naturally with light warmth and conversational pacing.',
     style: { emotion: 'neutral', energy: 'medium' },
   };

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "oomi-ai",
   "name": "Oomi Channel Plugin",
   "description": "Managed Oomi channel integration for OpenClaw.",
-  "version": "0.2.16",
+  "version": "0.2.17",
   "author": "Oomi",
   "license": "MIT",
   "openclawVersion": ">=0.5.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oomi-ai",
-  "version": "0.2.16",
+  "version": "0.2.17",
   "description": "Oomi OpenClaw channel plugin and bridge tooling",
   "bin": {
     "oomi": "bin/oomi-ai.js"

package/skills/oomi/SKILL.md

@@ -139,6 +139,25 @@ Use this shape when a voice turn needs more natural delivery without changing vi
   "metadata": {
     "spoken": {
       "text": "Speech-optimized text for TTS only.",
+      "language": "English",
+      "segments": [
+        {
+          "text": "Hey! It's Nemu, but close enough.",
+          "pace": "medium_fast",
+          "pitch": "slightly_high",
+          "energy": "bright",
+          "volume": "normal",
+          "pause_after_ms": 220
+        },
+        {
+          "text": "Right now, I'm just waking up into this conversation with you.",
+          "pace": "medium",
+          "pitch": "neutral",
+          "energy": "warm",
+          "volume": "normal",
+          "pause_after_ms": 280
+        }
+      ],
       "instructions": "Speak with upbeat, warm excitement and slightly rising intonation.",
       "style": {
         "emotion": "excited",
@@ -154,9 +173,11 @@ Rules:
 - do not place raw intonation tags in visible chat
 - for managed voice replies, include `metadata.spoken` when delivery benefits from cleaner phrasing or explicit speaking guidance
 - `metadata.spoken.text` is backend TTS input only
+- `metadata.spoken.language` should be one of the supported Qwen language values such as `English`
+- `metadata.spoken.segments` can carry bounded per-segment prosody for pace, pitch, volume, and pause timing
 - `metadata.spoken.instructions` should use natural-language speaking guidance
 - if the speech sidecar is absent, Oomi speaks the visible assistant text
-- if you omit `metadata.spoken`, the plugin synthesizes a conservative hidden fallback from visible assistant text
+- if you omit `metadata.spoken`, the plugin synthesizes a bounded hidden fallback from visible assistant text
 
 ## Avatar Control
 

package/skills/oomi/agent_instructions.md

@@ -71,8 +71,10 @@ Rules:
 - visible `content` remains the source of truth for Oomi chat rendering
 - for managed voice replies, include `metadata.spoken` when delivery benefits from cleaner phrasing or explicit speaking guidance
 - `metadata.spoken.text` is for backend TTS only
+- `metadata.spoken.language` should be one of the supported Qwen language values such as `English`
+- `metadata.spoken.segments` can carry bounded per-segment prosody for pace, pitch, volume, and pause timing
 - `metadata.spoken.instructions` should be natural-language guidance, not raw bracket tags
 - `metadata.spoken.style` is optional metadata for debugging or future mapping
 - if no hidden speech sidecar exists, Oomi falls back to speaking the visible assistant text
-- if you omit `metadata.spoken`, the plugin now synthesizes a conservative hidden fallback from visible assistant text
+- if you omit `metadata.spoken`, the plugin now synthesizes a bounded hidden fallback from visible assistant text
 - visible chat text is never rewritten by the plugin