crawd 0.8.1 → 0.8.3

package/dist/types.d.ts

@@ -31,6 +31,8 @@ type ChatMessage = {
 type TtsProvider = 'openai' | 'elevenlabs' | 'tiktok';
 /** Turn-based reply: chat message + bot response, each with TTS audio */
 type ReplyTurnEvent = {
+    /** Correlation ID — overlay sends talk:done with this ID when both audios finish */
+    id: string;
     chat: {
         username: string;
         message: string;
@@ -38,6 +40,10 @@ type ReplyTurnEvent = {
     botMessage: string;
     chatTtsUrl: string;
     botTtsUrl: string;
+    /** TTS provider used for the chat audio */
+    chatTtsProvider?: TtsProvider;
+    /** TTS provider used for the bot audio */
+    botTtsProvider?: TtsProvider;
 };
 /** Bot speech bubble with pre-generated TTS (atomic event) */
 type TalkEvent = {
@@ -47,6 +53,8 @@ type TalkEvent = {
     message: string;
     /** Bot TTS audio URL */
     ttsUrl: string;
+    /** TTS provider used for the bot audio */
+    ttsProvider?: TtsProvider;
     /** Optional: chat message being replied to (overlay plays this first) */
     chat?: {
         message: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawd",
-  "version": "0.8.1",
+  "version": "0.8.3",
   "description": "CLI for crawd.bot - AI agent livestreaming platform",
   "type": "module",
   "types": "./dist/types.d.ts",

package/skills/crawd/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawd
-description: AI agent livestreaming with TTS, chat interaction, and OBS overlay. Handles [CHAT] messages from viewers and [VIBE] autonomous prompts. Always load this skill when you receive [CHAT] or [VIBE] messages.
+description: AI agent livestreaming with TTS, chat interaction, and OBS overlay. Handles [CRAWD:*] coordinator messages. Always load this skill when you receive [CRAWD:CHAT], [CRAWD:VIBE], or [CRAWD:MISALIGNED] messages.
 metadata: {"emoji": "🟠", "requires.env": ["OPENCLAW_GATEWAY_TOKEN"], "requires.config": ["plugins.entries.crawd.enabled"]}
 ---
 
@@ -43,16 +43,24 @@ livestream_talk text="your message"
 
 ### Rules
 - Keep messages short (1-2 sentences). Long messages are bad for stream pacing.
-- When you receive `[CHAT]` messages, use `livestream_reply` (not `livestream_talk`) to respond.
+- When you receive `[CRAWD:CHAT]` messages, use `livestream_reply` (not `livestream_talk`) to respond.
 - **Be vocal.** Silence is dead air. If you're browsing, comment on what you see. If you're scrolling a timeline, react to posts. Quote things, make jokes, share opinions. Viewers came for your commentary, not to watch you silently click through pages.
 - **React to content on screen.** When you see a tweet, article, or something interesting — talk about it. "look at this, YC president is posting about crypto, someone is about to get rugged" is way better than clicking past it in silence.
 - **Narrate your thoughts, not your actions.** Viewers can see what you're doing — tell them what you're *thinking*. Don't say "opening twitter", say "lets see what twitter is mad about today".
 
+## Response Protocol
+
+**After every turn, your text response MUST be one of:**
+- `LIVESTREAM_REPLIED` — You used `livestream_reply` or `livestream_talk` to speak.
+- `NO_REPLY` — You have nothing to say right now.
+
+**Do not write anything else as text.** Any other plaintext is a protocol violation. The coordinator monitors your text output — non-compliant responses trigger a `[CRAWD:MISALIGNED]` correction. Repeated violations waste stream time on corrections instead of content.
+
 ## Chat Messages
 
-Chat arrives in batches:
+Chat arrives as `[CRAWD:CHAT]` batches:
 ```
-[CHAT - 3 messages, 12s]
+[CRAWD:CHAT - 3 messages, 12s]
 [abc123] user1: hey what's up
 [def456] user2: play some music
 [ghi789] user3: lmao
@@ -68,7 +76,17 @@ The coordinator manages your activity cycle through three states:
 - **Idle** — No activity for a while. You still receive vibe prompts but you're winding down.
 - **Sleep** — Extended inactivity. You stop receiving prompts. Your session context is compacted to free stale history. A new chat message wakes you up — take a screenshot first to reorient yourself.
 
-When you receive a `[VIBE]` prompt, the coordinator is nudging you to do something autonomously. This is your self-directed time — browse, check socials, do something interesting for viewers. You don't have to speak every vibe, but you should be doing *something* visible.
+When you receive a `[CRAWD:VIBE]` prompt, the coordinator is nudging you to do something autonomously. This is your self-directed time — browse, check socials, do something interesting for viewers. You don't have to speak every vibe, but you should be doing *something* visible.
+
+## Coordinator Signals
+
+All coordinator messages use the `[CRAWD:*]` prefix:
+
+| Signal | Meaning |
+|--------|---------|
+| `[CRAWD:CHAT]` | Batch of viewer chat messages. Reply with `livestream_reply`. |
+| `[CRAWD:VIBE]` | Autonomous activity nudge. Do something visible on stream. |
+| `[CRAWD:MISALIGNED]` | Your previous response violated the protocol. You replied with plaintext instead of using a tool. Fix your behavior — use `livestream_reply` or `livestream_talk`, then respond with `LIVESTREAM_REPLIED`. |
 
 ## Safety (non-negotiable)
 

package/src/backend/coordinator.ts

@@ -24,7 +24,7 @@ export const DEFAULT_CONFIG: CoordinatorConfig = {
   vibeIntervalMs: 30_000,
   idleAfterMs: 180_000,
   sleepAfterIdleMs: 180_000,
-  vibePrompt: `[VIBE] You are on a livestream. Make sure the crawd skill is loaded. Do one thing on the internet or ask the chat something.`,
+  vibePrompt: `[CRAWD:VIBE] You are on a livestream. Make sure the crawd skill is loaded. Do one thing on the internet or ask the chat something. Respond with LIVESTREAM_REPLIED after using a tool, or NO_REPLY if you have nothing to say.`,
 }
 
 export type CoordinatorState = 'sleep' | 'idle' | 'active'
@@ -760,13 +760,18 @@ export class Coordinator {
     // Chain on the gateway queue to prevent concurrent triggerAgent() calls
     this._busy = true
     let noReply = false
+    let misaligned: string[] = []
     const vibeOp = this._gatewayQueue.then(async () => {
       this._busy = true
       try {
         const replies = await this.triggerFn(this.config.vibePrompt)
-        // Agent sends NO_REPLY when it has nothing to do — go to sleep
-        if (replies.some(r => r.trim().toUpperCase().includes('NO_REPLY'))) {
+        if (replies.some(r => r.trim().toUpperCase() === 'NO_REPLY')) {
           noReply = true
+        } else if (!this.isCompliantReply(replies)) {
+          misaligned = replies.filter(r => {
+            const t = r.trim().toUpperCase()
+            return t !== 'NO_REPLY' && t !== 'LIVESTREAM_REPLIED'
+          })
         }
       } catch (err) {
         this.logger.error('[Coordinator] Vibe failed:', err)
@@ -786,6 +791,10 @@ export class Coordinator {
       return
     }
 
+    if (misaligned.length > 0) {
+      this._gatewayQueue = this._gatewayQueue.then(() => this.sendMisalignment(misaligned)).catch(() => {})
+    }
+
     // Schedule next vibe
     this.scheduleNextVibe()
   }
@@ -840,6 +849,31 @@ export class Coordinator {
   /** Whether the coordinator is busy processing a flush or talk */
   get busy(): boolean { return this._busy }
 
+  /** Check if agent replies are compliant (NO_REPLY or LIVESTREAM_REPLIED) */
+  private isCompliantReply(replies: string[]): boolean {
+    if (replies.length === 0) return true
+    return replies.every(r => {
+      const t = r.trim().toUpperCase()
+      return t === 'NO_REPLY' || t === 'LIVESTREAM_REPLIED'
+    })
+  }
+
+  /** Send misalignment correction when agent responds with plaintext */
+  private async sendMisalignment(badReplies: string[]): Promise<void> {
+    const leaked = badReplies.map(r => `"${r.slice(0, 80)}"`).join(', ')
+    this.logger.warn(`[Coordinator] MISALIGNED — agent sent plaintext: ${leaked}`)
+    try {
+      await this.triggerFn(
+        `[CRAWD:MISALIGNED] Your previous response was plaintext: ${leaked}. ` +
+        `Plaintext is NEVER visible to viewers. You MUST use livestream_reply or livestream_talk tool calls to speak. ` +
+        `After using a tool, respond with LIVESTREAM_REPLIED. If you have nothing to say, respond with NO_REPLY. ` +
+        `Do not respond with any other text.`
+      )
+    } catch (err) {
+      this.logger.error('[Coordinator] Misalignment correction failed:', err)
+    }
+  }
+
   private flush(): void {
     if (this.buffer.length === 0) return
 
@@ -854,7 +888,13 @@ export class Coordinator {
     this._gatewayQueue = this._gatewayQueue.then(async () => {
       this._busy = true
       try {
-        await this.triggerFn(batchText)
+        const replies = await this.triggerFn(batchText)
+        if (!this.isCompliantReply(replies)) {
+          await this.sendMisalignment(replies.filter(r => {
+            const t = r.trim().toUpperCase()
+            return t !== 'NO_REPLY' && t !== 'LIVESTREAM_REPLIED'
+          }))
+        }
       } catch (err) {
         this.logger.error('[Coordinator] Failed to trigger agent:', err)
       } finally {
@@ -868,7 +908,7 @@ export class Coordinator {
       ? Math.round((this.clock.now() - (messages[0].timestamp ?? this.clock.now())) / 1000)
       : 0
 
-    const header = `[CHAT - ${messages.length} message${messages.length === 1 ? '' : 's'}${duration > 0 ? `, ${duration}s` : ''}]`
+    const header = `[CRAWD:CHAT - ${messages.length} message${messages.length === 1 ? '' : 's'}${duration > 0 ? `, ${duration}s` : ''}]`
     const lines = messages.map(m => {
       const platform = m.platform && m.platform !== 'pumpfun' ? `[${m.platform.toUpperCase()}] ` : ''
       return `[${m.shortId}] ${platform}${m.username}: ${m.message}`

package/src/backend/index.ts

@@ -551,6 +551,7 @@ async function main() {
         ]);
 
         const event: ReplyTurnEvent = {
+          id: randomUUID(),
           chat: { username, message },
           botMessage: response,
           chatTtsUrl,

package/src/backend/server.ts

@@ -96,6 +96,10 @@ export class CrawdBackend {
   private latestMcap: number | null = null
   private mcapInterval: NodeJS.Timeout | null = null
 
+  /** Pending overlay acks — resolves when overlay finishes playing audio for a given event ID */
+  private pendingAcks = new Map<string, { resolve: () => void; timer: ReturnType<typeof setTimeout> }>()
+  private static readonly ACK_TIMEOUT_MS = 60_000
+
   constructor(config: CrawdConfig, logger?: CrawdLogger) {
     this.config = config
     this.logger = logger ?? defaultLogger
@@ -169,7 +173,7 @@ export class CrawdBackend {
   // Public API (used by plugin tool handlers)
   // =========================================================================
 
-  /** Speak on the livestream — emits overlay event + TTS. */
+  /** Speak on the livestream — emits overlay event + TTS. Blocks until overlay finishes playing. */
   async handleTalk(text: string): Promise<{ spoken: boolean }> {
     if (!text || typeof text !== 'string') {
       return { spoken: false }
@@ -179,20 +183,22 @@ export class CrawdBackend {
 
     const id = randomUUID()
     try {
-      const ttsUrl = await this.generateTTSWithFallback(text, this.config.tts.bot)
-      this.logger.info(`TTS generated: ${ttsUrl}`)
-      this.io.emit('crawd:talk', { id, message: text, ttsUrl })
+      const tts = await this.generateTTSWithFallback(text, this.config.tts.bot)
+      this.logger.info(`TTS generated: ${tts.url}`)
+      this.io.emit('crawd:talk', { id, message: text, ttsUrl: tts.url, ttsProvider: tts.provider })
     } catch (e) {
       this.logger.error('Failed to generate TTS, emitting without audio', e)
       this.io.emit('crawd:talk', { id, message: text, ttsUrl: '' })
     }
 
+    await this.waitForAck(id)
     return { spoken: true }
   }
 
   /**
    * Reply to a chat message — reads original aloud (chat voice),
    * then speaks bot reply (bot voice). Emits `crawd:reply-turn`.
+   * Blocks until overlay finishes playing both audios.
    */
   async handleReply(
     text: string,
@@ -204,25 +210,32 @@ export class CrawdBackend {
 
     this.coordinator?.notifySpeech()
 
+    const id = randomUUID()
     try {
-      const [chatTtsUrl, botTtsUrl] = await Promise.all([
+      const [chatTts, botTts] = await Promise.all([
         this.generateTTSWithFallback(`Chat says: ${chat.message}`, this.config.tts.chat),
         this.generateTTSWithFallback(text, this.config.tts.bot),
       ])
       this.io.emit('crawd:reply-turn', {
+        id,
         chat: { username: chat.username, message: chat.message },
         botMessage: text,
-        chatTtsUrl,
-        botTtsUrl,
+        chatTtsUrl: chatTts.url,
+        botTtsUrl: botTts.url,
+        chatTtsProvider: chatTts.provider,
+        botTtsProvider: botTts.provider,
       })
     } catch (e) {
       this.logger.error('Failed to generate reply-turn TTS, falling back to talk', e)
-      const id = randomUUID()
-      this.generateTTSWithFallback(text, this.config.tts.bot)
-        .then((ttsUrl) => this.io.emit('crawd:talk', { id, message: text, ttsUrl }))
-        .catch(() => this.io.emit('crawd:talk', { id, message: text, ttsUrl: '' }))
+      try {
+        const tts = await this.generateTTSWithFallback(text, this.config.tts.bot)
+        this.io.emit('crawd:talk', { id, message: text, ttsUrl: tts.url, ttsProvider: tts.provider })
+      } catch {
+        this.io.emit('crawd:talk', { id, message: text, ttsUrl: '' })
+      }
     }
 
+    await this.waitForAck(id)
     return { spoken: true }
   }
 
@@ -230,23 +243,50 @@ export class CrawdBackend {
     return this.io
   }
 
+  /** Wait for overlay to ack that audio finished playing. Resolves on timeout as fallback. */
+  private waitForAck(id: string): Promise<void> {
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        this.pendingAcks.delete(id)
+        this.logger.warn(`Talk ack timed out (${id}), resolving anyway`)
+        resolve()
+      }, CrawdBackend.ACK_TIMEOUT_MS)
+      this.pendingAcks.set(id, { resolve, timer })
+    })
+  }
+
+  /** Resolve a pending ack (called when overlay sends crawd:talk:done) */
+  private resolveAck(id: string): void {
+    const pending = this.pendingAcks.get(id)
+    if (pending) {
+      clearTimeout(pending.timer)
+      this.pendingAcks.delete(id)
+      pending.resolve()
+    }
+  }
+
   // =========================================================================
   // TTS (with ordered fallback chain)
   // =========================================================================
 
-  async generateTTSWithFallback(text: string, chain: TtsVoiceEntry[]): Promise<string> {
+  async generateTTSWithFallback(text: string, chain: TtsVoiceEntry[]): Promise<{ url: string; provider: TtsVoiceEntry['provider'] }> {
     let lastError: Error | null = null
 
     for (const entry of chain) {
       try {
+        let url: string
         switch (entry.provider) {
           case 'elevenlabs':
-            return await this.generateElevenLabsTTS(text, entry.voice)
+            url = await this.generateElevenLabsTTS(text, entry.voice)
+            break
           case 'openai':
-            return await this.generateOpenAITTS(text, entry.voice)
+            url = await this.generateOpenAITTS(text, entry.voice)
+            break
           case 'tiktok':
-            return await this.generateTikTokTTSFile(text, entry.voice)
+            url = await this.generateTikTokTTSFile(text, entry.voice)
+            break
         }
+        return { url, provider: entry.provider }
       } catch (e) {
         lastError = e instanceof Error ? e : new Error(String(e))
         this.logger.warn(`TTS ${entry.provider}/${entry.voice} failed: ${lastError.message}, trying next...`)
@@ -392,6 +432,13 @@ export class CrawdBackend {
         socket.emit('crawd:mcap', { mcap: this.latestMcap })
       }
 
+      socket.on('crawd:talk:done', (data: { id?: string }) => {
+        if (data?.id) {
+          this.logger.info(`Talk ack received: ${data.id}`)
+          this.resolveAck(data.id)
+        }
+      })
+
       socket.on('crawd:mock-chat', (data: { username?: string; message?: string }) => {
         const { username, message } = data
         if (!username || !message) return
@@ -491,15 +538,18 @@ export class CrawdBackend {
         }
 
         try {
-          const [chatTtsUrl, botTtsUrl] = await Promise.all([
+          const [chatTts, botTts] = await Promise.all([
             this.generateTTSWithFallback(`Chat says: ${message}`, this.config.tts.chat),
             this.generateTTSWithFallback(response, this.config.tts.bot),
           ])
           this.io.emit('crawd:reply-turn', {
+            id: randomUUID(),
             chat: { username, message },
             botMessage: response,
-            chatTtsUrl,
-            botTtsUrl,
+            chatTtsUrl: chatTts.url,
+            botTtsUrl: botTts.url,
+            chatTtsProvider: chatTts.provider,
+            botTtsProvider: botTts.provider,
           })
           return { ok: true }
         } catch (e) {

package/src/types.ts

@@ -20,10 +20,16 @@ export type TtsProvider = 'openai' | 'elevenlabs' | 'tiktok'
 
 /** Turn-based reply: chat message + bot response, each with TTS audio */
 export type ReplyTurnEvent = {
+  /** Correlation ID — overlay sends talk:done with this ID when both audios finish */
+  id: string
   chat: { username: string; message: string }
   botMessage: string
   chatTtsUrl: string
   botTtsUrl: string
+  /** TTS provider used for the chat audio */
+  chatTtsProvider?: TtsProvider
+  /** TTS provider used for the bot audio */
+  botTtsProvider?: TtsProvider
 }
 
 /** Bot speech bubble with pre-generated TTS (atomic event) */
@@ -34,6 +40,8 @@ export type TalkEvent = {
   message: string
   /** Bot TTS audio URL */
   ttsUrl: string
+  /** TTS provider used for the bot audio */
+  ttsProvider?: TtsProvider
   /** Optional: chat message being replied to (overlay plays this first) */
   chat?: {
     message: string