crawd 0.8.2 → 0.8.4

package/dist/backend/index.js

@@ -11443,7 +11443,7 @@ var GatewayClient = class _GatewayClient {
 };
 var STARTUP_GRACE_MS = 3e4;
 var SLEEP_CHECK_INTERVAL_MS = 1e4;
-var Coordinator = class {
+var Coordinator = class _Coordinator {
   buffer = [];
   timer = null;
   triggerFn;
@@ -11648,14 +11648,18 @@ var Coordinator = class {
       this._busy = true;
       try {
         const replies = await this.triggerFn(this.config.vibePrompt);
-        if (replies.some((r) => r.trim().toUpperCase() === "NO_REPLY")) {
+        const agentReplies = replies.filter((r) => !this.isApiError(r));
+        if (agentReplies.some((r) => r.trim().toUpperCase() === "NO_REPLY")) {
           noReply = true;
-        } else if (!this.isCompliantReply(replies)) {
-          misaligned = replies.filter((r) => {
+        } else if (!this.isCompliantReply(agentReplies)) {
+          misaligned = agentReplies.filter((r) => {
             const t = r.trim().toUpperCase();
             return t !== "NO_REPLY" && t !== "LIVESTREAM_REPLIED";
           });
         }
+        if (replies.length > agentReplies.length) {
+          this.logger.warn(`[Coordinator] Filtered ${replies.length - agentReplies.length} API error(s) from vibe response`);
+        }
       } catch (err) {
         this.logger.error("[Coordinator] Vibe failed:", err);
       } finally {
@@ -11715,12 +11719,17 @@ var Coordinator = class {
   get busy() {
     return this._busy;
   }
+  /** Detect API/gateway errors surfaced as reply strings (e.g. rate limits) */
+  static API_ERROR_RE = /^\d{3}\s+(status\s+code|error)|^rate\s*limit|^too\s+many\s+requests|^overloaded|^server\s+error/i;
+  isApiError(reply) {
+    return _Coordinator.API_ERROR_RE.test(reply.trim());
+  }
   /** Check if agent replies are compliant (NO_REPLY or LIVESTREAM_REPLIED) */
   isCompliantReply(replies) {
     if (replies.length === 0) return true;
     return replies.every((r) => {
       const t = r.trim().toUpperCase();
-      return t === "NO_REPLY" || t === "LIVESTREAM_REPLIED";
+      return t === "NO_REPLY" || t === "LIVESTREAM_REPLIED" || this.isApiError(r);
     });
   }
   /** Send misalignment correction when agent responds with plaintext */
@@ -11746,8 +11755,12 @@ var Coordinator = class {
       this._busy = true;
       try {
         const replies = await this.triggerFn(batchText);
-        if (!this.isCompliantReply(replies)) {
-          await this.sendMisalignment(replies.filter((r) => {
+        const agentReplies = replies.filter((r) => !this.isApiError(r));
+        if (replies.length > agentReplies.length) {
+          this.logger.warn(`[Coordinator] Filtered ${replies.length - agentReplies.length} API error(s) from chat response`);
+        }
+        if (!this.isCompliantReply(agentReplies)) {
+          await this.sendMisalignment(agentReplies.filter((r) => {
             const t = r.trim().toUpperCase();
             return t !== "NO_REPLY" && t !== "LIVESTREAM_REPLIED";
           }));
@@ -12372,6 +12385,7 @@ async function main() {
           botTTS(response)
         ]);
         const event = {
+          id: randomUUID2(),
           chat: { username, message },
           botMessage: response,
           chatTtsUrl,

package/dist/types.d.ts

@@ -31,6 +31,8 @@ type ChatMessage = {
 type TtsProvider = 'openai' | 'elevenlabs' | 'tiktok';
 /** Turn-based reply: chat message + bot response, each with TTS audio */
 type ReplyTurnEvent = {
+    /** Correlation ID — overlay sends talk:done with this ID when both audios finish */
+    id: string;
     chat: {
         username: string;
         message: string;
@@ -38,6 +40,10 @@ type ReplyTurnEvent = {
     botMessage: string;
     chatTtsUrl: string;
     botTtsUrl: string;
+    /** TTS provider used for the chat audio */
+    chatTtsProvider?: TtsProvider;
+    /** TTS provider used for the bot audio */
+    botTtsProvider?: TtsProvider;
 };
 /** Bot speech bubble with pre-generated TTS (atomic event) */
 type TalkEvent = {
@@ -47,6 +53,8 @@ type TalkEvent = {
     message: string;
     /** Bot TTS audio URL */
     ttsUrl: string;
+    /** TTS provider used for the bot audio */
+    ttsProvider?: TtsProvider;
     /** Optional: chat message being replied to (overlay plays this first) */
     chat?: {
         message: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawd",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "description": "CLI for crawd.bot - AI agent livestreaming platform",
   "type": "module",
   "types": "./dist/types.d.ts",

package/src/backend/coordinator.ts

@@ -765,14 +765,19 @@ export class Coordinator {
       this._busy = true
       try {
         const replies = await this.triggerFn(this.config.vibePrompt)
-        if (replies.some(r => r.trim().toUpperCase() === 'NO_REPLY')) {
+        // Filter out API errors (429s, rate limits) — not agent responses
+        const agentReplies = replies.filter(r => !this.isApiError(r))
+        if (agentReplies.some(r => r.trim().toUpperCase() === 'NO_REPLY')) {
           noReply = true
-        } else if (!this.isCompliantReply(replies)) {
-          misaligned = replies.filter(r => {
+        } else if (!this.isCompliantReply(agentReplies)) {
+          misaligned = agentReplies.filter(r => {
             const t = r.trim().toUpperCase()
             return t !== 'NO_REPLY' && t !== 'LIVESTREAM_REPLIED'
           })
         }
+        if (replies.length > agentReplies.length) {
+          this.logger.warn(`[Coordinator] Filtered ${replies.length - agentReplies.length} API error(s) from vibe response`)
+        }
       } catch (err) {
         this.logger.error('[Coordinator] Vibe failed:', err)
       } finally {
@@ -849,12 +854,19 @@ export class Coordinator {
   /** Whether the coordinator is busy processing a flush or talk */
   get busy(): boolean { return this._busy }
 
+  /** Detect API/gateway errors surfaced as reply strings (e.g. rate limits) */
+  private static readonly API_ERROR_RE = /^\d{3}\s+(status\s+code|error)|^rate\s*limit|^too\s+many\s+requests|^overloaded|^server\s+error/i
+
+  private isApiError(reply: string): boolean {
+    return Coordinator.API_ERROR_RE.test(reply.trim())
+  }
+
   /** Check if agent replies are compliant (NO_REPLY or LIVESTREAM_REPLIED) */
   private isCompliantReply(replies: string[]): boolean {
     if (replies.length === 0) return true
     return replies.every(r => {
       const t = r.trim().toUpperCase()
-      return t === 'NO_REPLY' || t === 'LIVESTREAM_REPLIED'
+      return t === 'NO_REPLY' || t === 'LIVESTREAM_REPLIED' || this.isApiError(r)
     })
   }
 
@@ -889,8 +901,12 @@ export class Coordinator {
       this._busy = true
       try {
         const replies = await this.triggerFn(batchText)
-        if (!this.isCompliantReply(replies)) {
-          await this.sendMisalignment(replies.filter(r => {
+        const agentReplies = replies.filter(r => !this.isApiError(r))
+        if (replies.length > agentReplies.length) {
+          this.logger.warn(`[Coordinator] Filtered ${replies.length - agentReplies.length} API error(s) from chat response`)
+        }
+        if (!this.isCompliantReply(agentReplies)) {
+          await this.sendMisalignment(agentReplies.filter(r => {
             const t = r.trim().toUpperCase()
             return t !== 'NO_REPLY' && t !== 'LIVESTREAM_REPLIED'
           }))

package/src/backend/index.ts

@@ -551,6 +551,7 @@ async function main() {
         ]);
 
         const event: ReplyTurnEvent = {
+          id: randomUUID(),
           chat: { username, message },
           botMessage: response,
           chatTtsUrl,

package/src/backend/server.ts

@@ -96,6 +96,10 @@ export class CrawdBackend {
   private latestMcap: number | null = null
   private mcapInterval: NodeJS.Timeout | null = null
 
+  /** Pending overlay acks — resolves when overlay finishes playing audio for a given event ID */
+  private pendingAcks = new Map<string, { resolve: () => void; timer: ReturnType<typeof setTimeout> }>()
+  private static readonly ACK_TIMEOUT_MS = 60_000
+
   constructor(config: CrawdConfig, logger?: CrawdLogger) {
     this.config = config
     this.logger = logger ?? defaultLogger
@@ -169,7 +173,7 @@ export class CrawdBackend {
   // Public API (used by plugin tool handlers)
   // =========================================================================
 
-  /** Speak on the livestream — emits overlay event + TTS. */
+  /** Speak on the livestream — emits overlay event + TTS. Blocks until overlay finishes playing. */
   async handleTalk(text: string): Promise<{ spoken: boolean }> {
     if (!text || typeof text !== 'string') {
       return { spoken: false }
@@ -179,20 +183,22 @@ export class CrawdBackend {
 
     const id = randomUUID()
     try {
-      const ttsUrl = await this.generateTTSWithFallback(text, this.config.tts.bot)
-      this.logger.info(`TTS generated: ${ttsUrl}`)
-      this.io.emit('crawd:talk', { id, message: text, ttsUrl })
+      const tts = await this.generateTTSWithFallback(text, this.config.tts.bot)
+      this.logger.info(`TTS generated: ${tts.url}`)
+      this.io.emit('crawd:talk', { id, message: text, ttsUrl: tts.url, ttsProvider: tts.provider })
     } catch (e) {
       this.logger.error('Failed to generate TTS, emitting without audio', e)
       this.io.emit('crawd:talk', { id, message: text, ttsUrl: '' })
     }
 
+    await this.waitForAck(id)
     return { spoken: true }
   }
 
   /**
    * Reply to a chat message — reads original aloud (chat voice),
    * then speaks bot reply (bot voice). Emits `crawd:reply-turn`.
+   * Blocks until overlay finishes playing both audios.
    */
   async handleReply(
     text: string,
@@ -204,25 +210,32 @@ export class CrawdBackend {
 
     this.coordinator?.notifySpeech()
 
+    const id = randomUUID()
     try {
-      const [chatTtsUrl, botTtsUrl] = await Promise.all([
+      const [chatTts, botTts] = await Promise.all([
         this.generateTTSWithFallback(`Chat says: ${chat.message}`, this.config.tts.chat),
         this.generateTTSWithFallback(text, this.config.tts.bot),
       ])
       this.io.emit('crawd:reply-turn', {
+        id,
         chat: { username: chat.username, message: chat.message },
         botMessage: text,
-        chatTtsUrl,
-        botTtsUrl,
+        chatTtsUrl: chatTts.url,
+        botTtsUrl: botTts.url,
+        chatTtsProvider: chatTts.provider,
+        botTtsProvider: botTts.provider,
       })
     } catch (e) {
       this.logger.error('Failed to generate reply-turn TTS, falling back to talk', e)
-      const id = randomUUID()
-      this.generateTTSWithFallback(text, this.config.tts.bot)
-        .then((ttsUrl) => this.io.emit('crawd:talk', { id, message: text, ttsUrl }))
-        .catch(() => this.io.emit('crawd:talk', { id, message: text, ttsUrl: '' }))
+      try {
+        const tts = await this.generateTTSWithFallback(text, this.config.tts.bot)
+        this.io.emit('crawd:talk', { id, message: text, ttsUrl: tts.url, ttsProvider: tts.provider })
+      } catch {
+        this.io.emit('crawd:talk', { id, message: text, ttsUrl: '' })
+      }
     }
 
+    await this.waitForAck(id)
     return { spoken: true }
   }
 
@@ -230,23 +243,50 @@ export class CrawdBackend {
     return this.io
   }
 
+  /** Wait for overlay to ack that audio finished playing. Resolves on timeout as fallback. */
+  private waitForAck(id: string): Promise<void> {
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        this.pendingAcks.delete(id)
+        this.logger.warn(`Talk ack timed out (${id}), resolving anyway`)
+        resolve()
+      }, CrawdBackend.ACK_TIMEOUT_MS)
+      this.pendingAcks.set(id, { resolve, timer })
+    })
+  }
+
+  /** Resolve a pending ack (called when overlay sends crawd:talk:done) */
+  private resolveAck(id: string): void {
+    const pending = this.pendingAcks.get(id)
+    if (pending) {
+      clearTimeout(pending.timer)
+      this.pendingAcks.delete(id)
+      pending.resolve()
+    }
+  }
+
   // =========================================================================
   // TTS (with ordered fallback chain)
   // =========================================================================
 
-  async generateTTSWithFallback(text: string, chain: TtsVoiceEntry[]): Promise<string> {
+  async generateTTSWithFallback(text: string, chain: TtsVoiceEntry[]): Promise<{ url: string; provider: TtsVoiceEntry['provider'] }> {
     let lastError: Error | null = null
 
     for (const entry of chain) {
       try {
+        let url: string
         switch (entry.provider) {
           case 'elevenlabs':
-            return await this.generateElevenLabsTTS(text, entry.voice)
+            url = await this.generateElevenLabsTTS(text, entry.voice)
+            break
           case 'openai':
-            return await this.generateOpenAITTS(text, entry.voice)
+            url = await this.generateOpenAITTS(text, entry.voice)
+            break
           case 'tiktok':
-            return await this.generateTikTokTTSFile(text, entry.voice)
+            url = await this.generateTikTokTTSFile(text, entry.voice)
+            break
         }
+        return { url, provider: entry.provider }
       } catch (e) {
         lastError = e instanceof Error ? e : new Error(String(e))
         this.logger.warn(`TTS ${entry.provider}/${entry.voice} failed: ${lastError.message}, trying next...`)
@@ -392,6 +432,13 @@ export class CrawdBackend {
         socket.emit('crawd:mcap', { mcap: this.latestMcap })
       }
 
+      socket.on('crawd:talk:done', (data: { id?: string }) => {
+        if (data?.id) {
+          this.logger.info(`Talk ack received: ${data.id}`)
+          this.resolveAck(data.id)
+        }
+      })
+
       socket.on('crawd:mock-chat', (data: { username?: string; message?: string }) => {
         const { username, message } = data
         if (!username || !message) return
@@ -491,15 +538,18 @@ export class CrawdBackend {
         }
 
         try {
-          const [chatTtsUrl, botTtsUrl] = await Promise.all([
+          const [chatTts, botTts] = await Promise.all([
             this.generateTTSWithFallback(`Chat says: ${message}`, this.config.tts.chat),
             this.generateTTSWithFallback(response, this.config.tts.bot),
           ])
           this.io.emit('crawd:reply-turn', {
+            id: randomUUID(),
             chat: { username, message },
             botMessage: response,
-            chatTtsUrl,
-            botTtsUrl,
+            chatTtsUrl: chatTts.url,
+            botTtsUrl: botTts.url,
+            chatTtsProvider: chatTts.provider,
+            botTtsProvider: botTts.provider,
           })
           return { ok: true }
         } catch (e) {

package/src/types.ts

@@ -20,10 +20,16 @@ export type TtsProvider = 'openai' | 'elevenlabs' | 'tiktok'
 
 /** Turn-based reply: chat message + bot response, each with TTS audio */
 export type ReplyTurnEvent = {
+  /** Correlation ID — overlay sends talk:done with this ID when both audios finish */
+  id: string
   chat: { username: string; message: string }
   botMessage: string
   chatTtsUrl: string
   botTtsUrl: string
+  /** TTS provider used for the chat audio */
+  chatTtsProvider?: TtsProvider
+  /** TTS provider used for the bot audio */
+  botTtsProvider?: TtsProvider
 }
 
 /** Bot speech bubble with pre-generated TTS (atomic event) */
@@ -34,6 +40,8 @@ export type TalkEvent = {
   message: string
   /** Bot TTS audio URL */
   ttsUrl: string
+  /** TTS provider used for the bot audio */
+  ttsProvider?: TtsProvider
   /** Optional: chat message being replied to (overlay plays this first) */
   chat?: {
     message: string