fluxy-bot 0.15.10 → 0.15.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fluxy-bot",
-  "version": "0.15.10",
+  "version": "0.15.11",
   "releaseNotes": [
     "1. react router implemented",
     "2. new workspace design",

package/supervisor/channels/manager.ts

@@ -71,6 +71,7 @@ export class ChannelManager {
     const whatsapp = new WhatsAppChannel(
       (sender, senderName, text, fromMe, isSelfChat) => this.handleInboundMessage('whatsapp', sender, senderName, text, fromMe, isSelfChat),
       (status) => this.handleStatusChange(status),
+      (audioBase64) => this.transcribeAudio(audioBase64),
     );
     this.providers.set('whatsapp', whatsapp);
 
@@ -91,6 +92,7 @@ export class ChannelManager {
       const whatsapp = new WhatsAppChannel(
         (sender, senderName, text, fromMe, isSelfChat) => this.handleInboundMessage('whatsapp', sender, senderName, text, fromMe, isSelfChat),
         (status) => this.handleStatusChange(status),
+        (audioBase64) => this.transcribeAudio(audioBase64),
       );
       this.providers.set('whatsapp', whatsapp);
       provider = whatsapp;
@@ -122,6 +124,14 @@ export class ChannelManager {
     await provider.sendMessage(to, text);
   }
 
+  /** Show "typing..." indicator in a chat */
+  startTyping(channel: ChannelType, jid: string): void {
+    const provider = this.providers.get(channel);
+    if (provider && 'startTyping' in provider) {
+      (provider as WhatsAppChannel).startTyping(jid);
+    }
+  }
+
   /** Get status of all channels */
   getStatuses(): ChannelStatus[] {
     return Array.from(this.providers.values()).map((p) => p.getStatus());
@@ -320,6 +330,9 @@ export class ChannelManager {
     // Channel context — tells the agent this is a WhatsApp message, respond naturally
     const channelContext = `[WhatsApp | ${msg.sender} | admin]\n`;
 
+    // Show "typing..." while the agent processes
+    this.startTyping(msg.channel, msg.rawSender);
+
     startFluxyAgentQuery(
       convId,
       channelContext + msg.text,
@@ -416,6 +429,9 @@ export class ChannelManager {
 
     this.activeAgents.set(agentKey, { sender: msg.sender, channel: msg.channel });
 
+    // Show "typing..." while the agent processes
+    this.startTyping(msg.channel, msg.rawSender);
+
     // Build an enriched script prompt with customer memory if available
     let enrichedScript = scriptPrompt;
     if (customerMemory && enrichedScript) {
@@ -453,6 +469,21 @@ export class ChannelManager {
     );
   }
 
+  /** Transcribe audio via the existing whisper endpoint */
+  private async transcribeAudio(audioBase64: string): Promise<string | null> {
+    try {
+      const result = await this.opts.workerApi('/api/whisper/transcribe', 'POST', { audio: audioBase64 });
+      if (result.error) {
+        log.warn(`[channels] Whisper error: ${result.error}`);
+        return null;
+      }
+      return result.transcript || null;
+    } catch (err: any) {
+      log.warn(`[channels] Whisper transcription failed: ${err.message}`);
+      return null;
+    }
+  }
+
   /** Load SCRIPT.md from the active skill configured for this channel */
   private loadActiveScript(channelConfig: ChannelConfig): string | undefined {
     const skillName = channelConfig.skill;

package/supervisor/channels/whatsapp.ts

@@ -7,6 +7,7 @@ import makeWASocket, {
   useMultiFileAuthState,
   makeCacheableSignalKeyStore,
   fetchLatestWaWebVersion,
+  downloadMediaMessage,
   DisconnectReason,
   Browsers,
   type WASocket,
@@ -25,6 +26,9 @@ const AUTH_DIR = path.join(DATA_DIR, 'channels', 'whatsapp', 'auth');
 /** Callback when a new message arrives */
 export type OnWhatsAppMessage = (sender: string, senderName: string | undefined, text: string, fromMe: boolean, isSelfChat: boolean) => void;
 
+/** Callback to transcribe audio via whisper */
+export type TranscribeFn = (audioBase64: string) => Promise<string | null>;
+
 export class WhatsAppChannel implements ChannelProvider {
   readonly type: ChannelType = 'whatsapp';
 
@@ -34,9 +38,16 @@ export class WhatsAppChannel implements ChannelProvider {
   private qrSvg: string | null = null;
   private onMessage: OnWhatsAppMessage;
   private onStatusChange: (status: ChannelStatus) => void;
+  private transcribe: TranscribeFn | null = null;
   private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
   private intentionalDisconnect = false;
 
+  /** IDs of messages we sent — used to prevent echo loops */
+  private sentMessageIds = new Set<string>();
+  private readonly MAX_SENT_IDS = 100;
+  /** Active typing indicator intervals per chat JID */
+  private typingIntervals = new Map<string, ReturnType<typeof setInterval>>();
+
   /** Maps LID JIDs to phone JIDs (WhatsApp uses LIDs internally for self-chat) */
   private lidToPhoneMap = new Map<string, string>();
   /** Our own phone JID (number@s.whatsapp.net) */
@@ -45,9 +56,11 @@ export class WhatsAppChannel implements ChannelProvider {
   constructor(
     onMessage: OnWhatsAppMessage,
     onStatusChange: (status: ChannelStatus) => void,
+    transcribe?: TranscribeFn,
   ) {
     this.onMessage = onMessage;
     this.onStatusChange = onStatusChange;
+    this.transcribe = transcribe || null;
   }
 
   async connect(): Promise<void> {
@@ -65,6 +78,9 @@ export class WhatsAppChannel implements ChannelProvider {
       this.sock.end(undefined);
       this.sock = null;
     }
+    // Clear all typing intervals
+    for (const interval of this.typingIntervals.values()) clearInterval(interval);
+    this.typingIntervals.clear();
     this.connected = false;
     this.qrData = null;
     this.qrSvg = null;
@@ -78,8 +94,52 @@ export class WhatsAppChannel implements ChannelProvider {
     }
     // Normalize: ensure JID format (number@s.whatsapp.net)
     const jid = to.includes('@') ? to : `${to.replace(/[^0-9]/g, '')}@s.whatsapp.net`;
-    await this.sock.sendMessage(jid, { text });
-    log.info(`[whatsapp] Sent message to ${jid}`);
+
+    // Clear typing indicator before sending
+    this.stopTyping(jid);
+
+    const result = await this.sock.sendMessage(jid, { text });
+
+    // Track sent message ID to prevent echo loops
+    if (result?.key?.id) {
+      this.trackSentId(result.key.id);
+    }
+
+    log.info(`[whatsapp] Sent message to ${jid} (id=${result?.key?.id || 'unknown'})`);
+  }
+
+  /** Show "typing..." indicator in a chat. Re-sends every 20s to keep it visible. */
+  startTyping(jid: string): void {
+    if (!this.sock || !this.connected) return;
+    // Clear any existing interval for this chat
+    this.stopTyping(jid);
+
+    const send = () => {
+      this.sock?.sendPresenceUpdate('composing', jid).catch(() => {});
+    };
+
+    send(); // immediate
+    this.typingIntervals.set(jid, setInterval(send, 20_000)); // refresh every 20s
+  }
+
+  /** Clear "typing..." indicator in a chat */
+  stopTyping(jid: string): void {
+    const interval = this.typingIntervals.get(jid);
+    if (interval) {
+      clearInterval(interval);
+      this.typingIntervals.delete(jid);
+    }
+    this.sock?.sendPresenceUpdate('paused', jid).catch(() => {});
+  }
+
+  /** Track a sent message ID, evicting oldest when at capacity */
+  private trackSentId(id: string) {
+    this.sentMessageIds.add(id);
+    if (this.sentMessageIds.size > this.MAX_SENT_IDS) {
+      // Delete the first (oldest) entry
+      const first = this.sentMessageIds.values().next().value;
+      if (first) this.sentMessageIds.delete(first);
+    }
   }
 
   getStatus(): ChannelStatus {
@@ -206,6 +266,8 @@ export class WhatsAppChannel implements ChannelProvider {
         this.qrData = null;
         this.qrSvg = null;
         this.buildLidMap();
+        // Set presence to unavailable so the phone doesn't show "online" constantly
+        sock.sendPresenceUpdate('unavailable').catch(() => {});
         log.ok(`[whatsapp] Connected as ${sock.user?.id}`);
         this.emitStatus();
       }
@@ -244,18 +306,47 @@ export class WhatsAppChannel implements ChannelProvider {
         if (msg.key.remoteJid === 'status@broadcast') continue;
         if (!msg.message) continue;
 
-        // Extract text from various message types
-        const text = this.extractText(msg.message);
-        if (!text) continue;
+        // Echo prevention: skip messages we sent ourselves
+        if (msg.key.id && this.sentMessageIds.has(msg.key.id)) {
+          this.sentMessageIds.delete(msg.key.id);
+          continue;
+        }
+
+        // Extract text — or transcribe audio if it's a voice note
+        let rawText = this.extractText(msg.message);
+
+        if (!rawText && this.isAudioMessage(msg.message)) {
+          // Voice note / audio — download and transcribe
+          if (!this.transcribe) {
+            log.info('[whatsapp] Audio message received but no transcribe function configured — skipping');
+            continue;
+          }
+          try {
+            const buffer = await downloadMediaMessage(msg, 'buffer', {}) as Buffer;
+            const base64 = buffer.toString('base64');
+            log.info(`[whatsapp] Transcribing audio (${Math.round(buffer.length / 1024)}KB)...`);
+            const transcript = await this.transcribe(base64);
+            if (!transcript) {
+              log.warn('[whatsapp] Transcription returned empty — skipping');
+              continue;
+            }
+            rawText = transcript;
+            log.info(`[whatsapp] Transcribed: "${rawText.slice(0, 80)}"`);
+          } catch (err: any) {
+            log.warn(`[whatsapp] Audio transcription failed: ${err.message}`);
+            continue;
+          }
+        }
+
+        if (!rawText) continue;
+
+        // Escape special characters to prevent prompt injection via message content
+        const text = this.escapeMessageText(rawText);
 
         const fromMe = msg.key.fromMe || false;
         const rawSender = msg.key.remoteJid || '';
         const participant = msg.key.participant || '';
 
-        // Debug: log all available sender fields to find the actual sender
-        log.info(`[whatsapp] DEBUG msg.key: remoteJid=${rawSender} fromMe=${fromMe} participant=${participant}`);
-        log.info(`[whatsapp] DEBUG pushName=${msg.pushName || 'none'} verifiedBizName=${(msg as any).verifiedBizName || 'none'}`);
-
         // The actual sender JID — use participant if available (newer protocol), fallback to remoteJid
         const actualSender = participant || rawSender;
 
@@ -302,6 +393,25 @@ export class WhatsAppChannel implements ChannelProvider {
     return null;
   }
 
+  /** Check if a message contains audio (voice note or audio file) */
+  private isAudioMessage(message: any): boolean {
+    if (!message) return false;
+    if (message.audioMessage) return true;
+    // Check inside wrappers
+    if (message.viewOnceMessage?.message?.audioMessage) return true;
+    if (message.viewOnceMessageV2?.message?.audioMessage) return true;
+    if (message.ephemeralMessage?.message?.audioMessage) return true;
+    return false;
+  }
+
+  /** Escape message text to prevent prompt injection via special characters */
+  private escapeMessageText(text: string): string {
+    return text
+      .replace(/\[WhatsApp\s*\|/gi, '(WhatsApp|')  // prevent faking channel context tags
+      .replace(/\[Telegram\s*\|/gi, '(Telegram|')
+      .replace(/\[\s*(admin|customer)\s*\]/gi, '($1)');  // prevent faking role tags
+  }
+
   private emitStatus() {
     this.onStatusChange(this.getStatus());
   }