oomi-ai 0.2.14 → 0.2.16

package/README.md

@@ -132,6 +132,9 @@ That bridge:
 
 This is the part of the package most likely to matter when debugging voice turn failures.
 
+For managed voice replies, the extension also preserves an explicit hidden `metadata.spoken` sidecar when upstream provides one.
+If upstream does not provide one, the extension now synthesizes a conservative hidden fallback from the visible assistant text so backend TTS can speak a cleaner version without changing user-visible chat.
+
 ## Bridge Health States
 
 The bridge status file is written locally and should roughly be interpreted as:
@@ -198,6 +201,7 @@ If you are inspecting this package on npm, the main architectural points are:
   - `idempotencyKey` handling
   - bridge status that does not report `connected` before managed subscription is ready
   - runtime fault isolation so local session failures are less likely to crash the whole provider
+  - hidden managed-voice speech metadata forwarding, with a synthesized fallback when upstream does not provide `metadata.spoken`
 
 If you are developing the plugin, test the packaged surface with:
 

package/agent_instructions.md

@@ -138,6 +138,41 @@ That means:
 - STT can succeed even when assistant reply delivery is broken later in the run
 - a `voice_session_*` failure should be investigated, but it should not automatically be treated as proof that all normal Oomi chat is down
 
+### Hidden Speech Payload
+
+For managed voice turns, keep visible assistant chat text natural and user-facing.
+Do not put spoken-style tags like `[happy]`, `[sad]`, or `[excited]` into visible chat text.
+
+When the runtime supports it, voice turns may include a hidden speech sidecar on the assistant message:
+
+```json
+{
+  "metadata": {
+    "spoken": {
+      "text": "Speech-optimized text for TTS only.",
+      "instructions": "Speak with upbeat, warm excitement and slightly rising intonation.",
+      "style": {
+        "emotion": "excited",
+        "energy": "medium_high"
+      }
+    }
+  }
+}
+```
+
+Rules:
+- visible `content` remains the source of truth for Oomi chat rendering
+- for managed voice replies, include `metadata.spoken` when delivery benefits from cleaner phrasing or explicit speaking guidance
+- `metadata.spoken.text` is for backend TTS only
+- `metadata.spoken.instructions` should be natural-language guidance, not raw bracket tags
+- `metadata.spoken.style` is optional metadata for debugging/future mapping
+- if no hidden speech sidecar exists, Oomi falls back to speaking the visible assistant text
+
+Current plugin behavior:
+- if you provide `metadata.spoken`, the plugin preserves it unchanged
+- if you do not provide `metadata.spoken`, the plugin now synthesizes a conservative hidden fallback from visible assistant text for backend TTS
+- visible chat text is still never rewritten by the plugin
+
 ## Avatar Commands
 
 Before using avatar commands, call `get_avatar_capabilities` and prefer canonical values.

package/openclaw.extension.js

@@ -178,6 +178,110 @@ function extractCorrelationId(payload) {
   return '';
 }
 
+function normalizeSpokenMetadata(spoken) {
+  if (!spoken || typeof spoken !== 'object' || Array.isArray(spoken)) return null;
+
+  const text = toString(spoken.text);
+  if (!text) return null;
+
+  const normalized = { text };
+  const instructions = toString(spoken.instructions);
+  if (instructions) normalized.instructions = instructions;
+  if (spoken.style && typeof spoken.style === 'object' && !Array.isArray(spoken.style)) {
+    normalized.style = spoken.style;
+  }
+
+  return normalized;
+}
+
+function stripEmoji(text) {
+  return text.replace(/[\uFE0E\uFE0F]/g, '').replace(/\p{Extended_Pictographic}|\p{Emoji_Presentation}/gu, '');
+}
+
+function normalizeSpeechText(text) {
+  return stripEmoji(text)
+    .replace(/\*\*(.*?)\*\*/g, '$1')
+    .replace(/__(.*?)__/g, '$1')
+    .replace(/`([^`]+)`/g, '$1')
+    .replace(/[–—]/g, ', ')
+    .replace(/…/g, '...')
+    .replace(/\s+/g, ' ')
+    .replace(/\s+([,.;!?])/g, '$1')
+    .replace(/([,.;!?])(?=[^\s])/g, '$1 ')
+    .replace(/,\s*,+/g, ', ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function inferSpokenMetadataFromContent(content) {
+  const text = normalizeSpeechText(toString(content));
+  if (!text) return null;
+
+  const normalized = text.toLowerCase();
+  const upbeat =
+    /!/.test(text) ||
+    /\b(hell yeah|awesome|amazing|great|stoked|love|glad|perfect|nice|cool)\b/.test(normalized);
+  const gentle =
+    /\b(sorry|gentle|softly|careful|reassuring|calm|okay|it'?s okay|i know)\b/.test(normalized);
+  const curious = /\?/.test(text);
+
+  if (upbeat) {
+    return {
+      text,
+      instructions: 'Speak with warm, upbeat conversational energy and natural pacing.',
+      style: { emotion: 'upbeat', energy: 'medium' },
+    };
+  }
+
+  if (gentle) {
+    return {
+      text,
+      instructions: 'Speak gently and reassuringly, with a calm pace and soft emphasis.',
+      style: { emotion: 'gentle', energy: 'low' },
+    };
+  }
+
+  if (curious) {
+    return {
+      text,
+      instructions: 'Speak naturally with curious, engaged intonation and a conversational pace.',
+      style: { emotion: 'curious', energy: 'medium' },
+    };
+  }
+
+  return {
+    text,
+    instructions: 'Speak naturally with light warmth and conversational pacing.',
+    style: { emotion: 'neutral', energy: 'medium' },
+  };
+}
+
+function normalizeOutgoingMetadata(payloadMetadata, { accountId, correlationId, content }) {
+  const metadata =
+    payloadMetadata && typeof payloadMetadata === 'object' && !Array.isArray(payloadMetadata)
+      ? { ...payloadMetadata }
+      : {};
+
+  const explicitSpokenPresent = Object.prototype.hasOwnProperty.call(metadata, 'spoken');
+  const spoken =
+    normalizeSpokenMetadata(metadata.spoken) ||
+    (!explicitSpokenPresent ? inferSpokenMetadataFromContent(content) : null);
+  if (spoken) {
+    metadata.spoken = spoken;
+  } else {
+    delete metadata.spoken;
+  }
+
+  metadata.accountId = accountId;
+  if (correlationId) {
+    metadata.correlationId = correlationId;
+  } else {
+    delete metadata.correlationId;
+  }
+
+  return metadata;
+}
+
 async function postJson({ url, token, body, timeoutMs }) {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), timeoutMs);
@@ -289,10 +393,11 @@ const oomiChannelPlugin = {
           sessionKey,
           content,
           source: 'openclaw.channel',
-          metadata: {
+          metadata: normalizeOutgoingMetadata(payload?.metadata, {
             accountId: resolvedAccountId,
             correlationId,
-          },
+            content,
+          }),
         },
       });
 

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "oomi-ai",
   "name": "Oomi Channel Plugin",
   "description": "Managed Oomi channel integration for OpenClaw.",
-  "version": "0.2.14",
+  "version": "0.2.16",
   "author": "Oomi",
   "license": "MIT",
   "openclawVersion": ">=0.5.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oomi-ai",
-  "version": "0.2.14",
+  "version": "0.2.16",
   "description": "Oomi OpenClaw channel plugin and bridge tooling",
   "bin": {
     "oomi": "bin/oomi-ai.js"

package/skills/oomi/SKILL.md

@@ -128,6 +128,36 @@ Install packaged Oomi operator instructions into an OpenClaw `AGENTS.md` file.
 python3 skills/oomi/scripts/install_agent_instructions.py
 ```
 
+## Hidden Speech Payload
+
+Managed voice can carry a hidden TTS-only speech sidecar alongside the normal assistant message.
+
+Use this shape when a voice turn needs more natural delivery without changing visible chat text:
+
+```json
+{
+  "metadata": {
+    "spoken": {
+      "text": "Speech-optimized text for TTS only.",
+      "instructions": "Speak with upbeat, warm excitement and slightly rising intonation.",
+      "style": {
+        "emotion": "excited",
+        "energy": "medium_high"
+      }
+    }
+  }
+}
+```
+
+Rules:
+- keep visible assistant `content` clean and user-facing
+- do not place raw intonation tags in visible chat
+- for managed voice replies, include `metadata.spoken` when delivery benefits from cleaner phrasing or explicit speaking guidance
+- `metadata.spoken.text` is backend TTS input only
+- `metadata.spoken.instructions` should use natural-language speaking guidance
+- if the speech sidecar is absent, Oomi speaks the visible assistant text
+- if you omit `metadata.spoken`, the plugin synthesizes a conservative hidden fallback from visible assistant text
+
 ## Avatar Control
 
 Before emitting avatar commands, call `get_avatar_capabilities` and prefer canonical values.

package/skills/oomi/agent_instructions.md

@@ -44,3 +44,35 @@ Primary UX requirement:
 - Return `/connect/<invite-token>` auth links only.
 
 Do not ask users to paste gateway IP/token/password when managed connect is available.
+
+## Hidden Speech Payload
+
+For managed voice turns, keep visible assistant chat text natural and user-facing.
+Do not put spoken-style tags like `[happy]`, `[sad]`, or `[excited]` into visible chat text.
+
+When the runtime supports it, voice turns may include a hidden speech sidecar on the assistant message:
+
+```json
+{
+  "metadata": {
+    "spoken": {
+      "text": "Speech-optimized text for TTS only.",
+      "instructions": "Speak with upbeat, warm excitement and slightly rising intonation.",
+      "style": {
+        "emotion": "excited",
+        "energy": "medium_high"
+      }
+    }
+  }
+}
+```
+
+Rules:
+- visible `content` remains the source of truth for Oomi chat rendering
+- for managed voice replies, include `metadata.spoken` when delivery benefits from cleaner phrasing or explicit speaking guidance
+- `metadata.spoken.text` is for backend TTS only
+- `metadata.spoken.instructions` should be natural-language guidance, not raw bracket tags
+- `metadata.spoken.style` is optional metadata for debugging or future mapping
+- if no hidden speech sidecar exists, Oomi falls back to speaking the visible assistant text
+- if you omit `metadata.spoken`, the plugin now synthesizes a conservative hidden fallback from visible assistant text
+- visible chat text is never rewritten by the plugin