@shykaruu/jarvis-brain 0.4.0

package/src/comms/streaming.ts

@@ -0,0 +1,149 @@
+import type { LLMStreamEvent } from '../llm/provider.ts';
+import type { WebSocketServer, WSMessage } from './websocket.ts';
+
+export type RelayOptions = {
+  /** Called each time a complete sentence is available during streaming. */
+  onSentence?: (sentence: string) => void;
+  /** Called when all text is done streaming. */
+  onTextDone?: () => void;
+};
+
+// Sentence boundary: period, exclamation, question mark, colon followed by whitespace or end
+const SENTENCE_END_RE = /[.!?:]\s/;
+
+export class StreamRelay {
+  private wsServer: WebSocketServer;
+
+  constructor(wsServer: WebSocketServer) {
+    this.wsServer = wsServer;
+  }
+
+  /**
+   * Relay LLM stream events to all connected WebSocket clients.
+   * Accumulates and returns the complete response text.
+   * Optionally fires onSentence callback as complete sentences arrive.
+   */
+  async relayStream(
+    stream: AsyncIterable<LLMStreamEvent>,
+    requestId: string,
+    options?: RelayOptions,
+  ): Promise<string> {
+    let fullText = '';
+    let sentenceBuffer = '';
+    let streamError: string | null = null;
+
+    try {
+      for await (const event of stream) {
+        if (event.type === 'text') {
+          fullText += event.text;
+
+          // Sentence-level TTS callback
+          if (options?.onSentence) {
+            sentenceBuffer += event.text;
+            // Flush complete sentences from the buffer
+            let match: RegExpExecArray | null;
+            while ((match = SENTENCE_END_RE.exec(sentenceBuffer)) !== null) {
+              const end = match.index + match[0].length;
+              const sentence = sentenceBuffer.slice(0, end).trim();
+              if (sentence) {
+                options.onSentence(sentence);
+              }
+              sentenceBuffer = sentenceBuffer.slice(end);
+            }
+          }
+
+          // Broadcast chunk to all connected clients
+          const message: WSMessage = {
+            type: 'stream',
+            payload: {
+              text: event.text,
+              requestId,
+              accumulated: fullText,
+            },
+            id: requestId,
+            timestamp: Date.now(),
+          };
+
+          this.wsServer.broadcast(message);
+        } else if (event.type === 'tool_call') {
+          // Broadcast tool call notification to clients
+          const toolMessage: WSMessage = {
+            type: 'stream',
+            payload: {
+              tool_call: {
+                name: event.tool_call.name,
+                arguments: event.tool_call.arguments,
+              },
+              requestId,
+            },
+            id: requestId,
+            timestamp: Date.now(),
+          };
+
+          this.wsServer.broadcast(toolMessage);
+        } else if (event.type === 'error') {
+          console.error('[StreamRelay] Stream error:', event.error);
+
+          const errorMessage: WSMessage = {
+            type: 'error',
+            payload: {
+              message: event.error,
+              requestId,
+            },
+            id: requestId,
+            timestamp: Date.now(),
+          };
+
+          this.wsServer.broadcast(errorMessage);
+          streamError = event.error;
+          break;
+        } else if (event.type === 'done') {
+          // Flush remaining sentence buffer
+          if (options?.onSentence && sentenceBuffer.trim()) {
+            options.onSentence(sentenceBuffer.trim());
+            sentenceBuffer = '';
+          }
+          options?.onTextDone?.();
+
+          console.log('[StreamRelay] Stream complete for request:', requestId);
+
+          const doneMessage: WSMessage = {
+            type: 'status',
+            payload: {
+              status: 'done',
+              requestId,
+              fullText,
+              usage: event.response.usage,
+            },
+            id: requestId,
+            timestamp: Date.now(),
+          };
+
+          this.wsServer.broadcast(doneMessage);
+        }
+      }
+
+      if (streamError) {
+        throw new Error(streamError);
+      }
+    } catch (error) {
+      console.error('[StreamRelay] Error relaying stream:', error);
+
+      const errorMessage: WSMessage = {
+        type: 'error',
+        payload: {
+          message: error instanceof Error ? error.message : 'Stream relay error',
+          requestId,
+        },
+        id: requestId,
+        timestamp: Date.now(),
+      };
+
+      this.wsServer.broadcast(errorMessage);
+
+      throw error;
+    }
+
+    return fullText;
+  }
+}

package/src/comms/voice.test.ts

@@ -0,0 +1,504 @@
+import { test, expect, describe, mock, afterEach } from 'bun:test';
+import {
+  createSTTProvider,
+  createTTSProvider,
+  OpenAIWhisperSTT,
+  GroqWhisperSTT,
+  LocalWhisperSTT,
+  EdgeTTSProvider,
+  splitIntoSentences,
+} from './voice.ts';
+import type { STTConfig, TTSConfig } from '../config/types.ts';
+
+/** Build a minimal valid WAV buffer */
+function makeWavBuffer(pcmBytes = 100): Buffer {
+  const dataSize = pcmBytes;
+  const buf = Buffer.alloc(44 + dataSize);
+  buf.write('RIFF', 0, 'ascii');
+  buf.writeUInt32LE(36 + dataSize, 4);
+  buf.write('WAVE', 8, 'ascii');
+  buf.write('fmt ', 12, 'ascii');
+  buf.writeUInt32LE(16, 16);       // subchunk1 size
+  buf.writeUInt16LE(1, 20);        // PCM
+  buf.writeUInt16LE(1, 22);        // mono
+  buf.writeUInt32LE(16000, 24);    // sample rate
+  buf.writeUInt32LE(32000, 28);    // byte rate
+  buf.writeUInt16LE(2, 32);        // block align
+  buf.writeUInt16LE(16, 34);       // bits per sample
+  buf.write('data', 36, 'ascii');
+  buf.writeUInt32LE(dataSize, 40);
+  return buf;
+}
+
+describe('createSTTProvider factory', () => {
+  test('returns OpenAIWhisperSTT when provider=openai and key present', () => {
+    const config: STTConfig = {
+      provider: 'openai',
+      openai: { api_key: 'test-openai-key-not-real' },
+    };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeInstanceOf(OpenAIWhisperSTT);
+  });
+
+  test('passes custom OpenAI-compatible base URL to OpenAIWhisperSTT', () => {
+    const config: STTConfig = {
+      provider: 'openai',
+      openai: {
+        api_key: 'test-openai-key-not-real',
+        base_url: 'https://gateway.example.com/openai',
+      },
+    };
+    const provider = createSTTProvider(config) as any;
+    expect(provider).toBeInstanceOf(OpenAIWhisperSTT);
+    expect(provider.apiUrl).toBe('https://gateway.example.com/openai/audio/transcriptions');
+  });
+
+  test('returns null when provider=openai and no key', () => {
+    const config: STTConfig = { provider: 'openai' };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeNull();
+  });
+
+  test('returns GroqWhisperSTT when provider=groq and key present', () => {
+    const config: STTConfig = {
+      provider: 'groq',
+      groq: { api_key: 'gtest-openai-key-not-real' },
+    };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeInstanceOf(GroqWhisperSTT);
+  });
+
+  test('returns null when provider=groq and no key', () => {
+    const config: STTConfig = { provider: 'groq' };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeNull();
+  });
+
+  test('returns LocalWhisperSTT when provider=local (no key needed)', () => {
+    const config: STTConfig = { provider: 'local' };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeInstanceOf(LocalWhisperSTT);
+  });
+
+  test('returns LocalWhisperSTT with custom endpoint', () => {
+    const config: STTConfig = {
+      provider: 'local',
+      local: { endpoint: 'http://my-server:9000' },
+    };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeInstanceOf(LocalWhisperSTT);
+  });
+
+  test('passes server_type through to LocalWhisperSTT', () => {
+    const config: STTConfig = {
+      provider: 'local',
+      local: { endpoint: 'http://my-server:9000', server_type: 'openai_compatible' },
+    };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeInstanceOf(LocalWhisperSTT);
+  });
+
+  test('returns null for unknown provider', () => {
+    const config = { provider: 'unknown' } as any;
+    const provider = createSTTProvider(config);
+    expect(provider).toBeNull();
+  });
+
+  test('returns OpenAI with custom model', () => {
+    const config: STTConfig = {
+      provider: 'openai',
+      openai: { api_key: 'test-key-not-real', model: 'whisper-large-v3' },
+    };
+    const provider = createSTTProvider(config);
+    expect(provider).toBeInstanceOf(OpenAIWhisperSTT);
+  });
+});
+
+describe('createTTSProvider factory', () => {
+  test('returns null when tts disabled', () => {
+    const config: TTSConfig = { enabled: false };
+    expect(createTTSProvider(config)).toBeNull();
+  });
+
+  test('returns EdgeTTSProvider when enabled', () => {
+    const config: TTSConfig = { enabled: true };
+    const provider = createTTSProvider(config);
+    expect(provider).toBeInstanceOf(EdgeTTSProvider);
+  });
+
+  test('passes voice config to provider', () => {
+    const config: TTSConfig = { enabled: true, voice: 'en-GB-SoniaNeural' };
+    const provider = createTTSProvider(config);
+    expect(provider).toBeInstanceOf(EdgeTTSProvider);
+  });
+
+  test('passes rate and volume config', () => {
+    const config: TTSConfig = { enabled: true, rate: '+20%', volume: '-10%' };
+    const provider = createTTSProvider(config);
+    expect(provider).not.toBeNull();
+  });
+});
+
+describe('EdgeTTSProvider', () => {
+  test('implements TTSProvider interface', () => {
+    const provider = new EdgeTTSProvider();
+    expect(typeof provider.synthesize).toBe('function');
+    expect(typeof provider.synthesizeStream).toBe('function');
+  });
+
+  test('constructor accepts custom voice/rate/volume', () => {
+    const provider = new EdgeTTSProvider('en-GB-SoniaNeural', '+10%', '-5%');
+    expect(provider).toBeInstanceOf(EdgeTTSProvider);
+  });
+});
+
+describe('splitIntoSentences', () => {
+  test('splits on period + capital letter', () => {
+    const result = splitIntoSentences('Hello there. World is great. This works.');
+    expect(result.length).toBe(3);
+    expect(result[0]).toBe('Hello there.');
+    expect(result[1]).toBe('World is great.');
+    expect(result[2]).toBe('This works.');
+  });
+
+  test('splits on exclamation and question marks', () => {
+    const result = splitIntoSentences('Wait! Are you sure? Yes I am.');
+    expect(result.length).toBe(3);
+  });
+
+  test('handles single sentence', () => {
+    const result = splitIntoSentences('Just one sentence.');
+    expect(result).toEqual(['Just one sentence.']);
+  });
+
+  test('handles empty string', () => {
+    const result = splitIntoSentences('');
+    expect(result).toEqual(['']);
+  });
+
+  test('collapses code blocks', () => {
+    const result = splitIntoSentences('Here is code:\n```\nconst x = 1;\n```\nDone.');
+    // Should not split inside code block
+    expect(result.length).toBeLessThanOrEqual(3);
+  });
+
+  test('splits on double newlines (paragraph breaks)', () => {
+    const result = splitIntoSentences('First paragraph\n\nSecond paragraph');
+    expect(result.length).toBe(2);
+  });
+
+  test('handles text with no sentence-ending punctuation', () => {
+    const result = splitIntoSentences('just some words without punctuation');
+    expect(result).toEqual(['just some words without punctuation']);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// LocalWhisperSTT – transcribe() tests
+// ---------------------------------------------------------------------------
+
+describe('LocalWhisperSTT.transcribe', () => {
+  const originalFetch = globalThis.fetch;
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+  });
+
+  // -- Server type defaults ------------------------------------------------
+
+  test('defaults to whisper_cpp server type', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async (_url: string, init: any) => {
+      const body = init.body as FormData;
+      expect(body.has('response_format')).toBe(true);
+      expect(body.has('model')).toBe(false);
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+  });
+
+  // -- whisper_cpp mode ----------------------------------------------------
+
+  test('whisper_cpp: appends /inference to bare-host endpoint', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189');
+    const wav = makeWavBuffer();
+    let calledUrl = '';
+
+    globalThis.fetch = mock(async (url: string) => {
+      calledUrl = url;
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+    expect(calledUrl).toBe('http://localhost:8189/inference');
+  });
+
+  test('whisper_cpp: uses endpoint as-is when it has explicit path', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+    let calledUrl = '';
+
+    globalThis.fetch = mock(async (url: string) => {
+      calledUrl = url;
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+    expect(calledUrl).toBe('http://localhost:8189/inference');
+  });
+
+  test('whisper_cpp: strips trailing slashes from endpoint', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189///');
+    const wav = makeWavBuffer();
+    let calledUrl = '';
+
+    globalThis.fetch = mock(async (url: string) => {
+      calledUrl = url;
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+    expect(calledUrl).toBe('http://localhost:8189/inference');
+  });
+
+  test('whisper_cpp: sends response_format and temperature fields', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference', undefined, 'whisper_cpp');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async (_url: string, init: any) => {
+      const body = init.body as FormData;
+      expect(body.has('response_format')).toBe(true);
+      expect(body.get('response_format')).toBe('json');
+      expect(body.has('temperature')).toBe(true);
+      expect(body.has('model')).toBe(false);
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+  });
+
+  // -- openai_compatible mode ----------------------------------------------
+
+  test('openai_compatible: uses endpoint verbatim', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8080/v1/audio/transcriptions', undefined, 'openai_compatible');
+    const wav = makeWavBuffer();
+    let calledUrl = '';
+
+    globalThis.fetch = mock(async (url: string) => {
+      calledUrl = url;
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+    expect(calledUrl).toBe('http://localhost:8080/v1/audio/transcriptions');
+  });
+
+  test('openai_compatible: sends model and language fields', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8080/v1/audio/transcriptions', 'whisper-1', 'openai_compatible');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async (_url: string, init: any) => {
+      const body = init.body as FormData;
+      expect(body.has('model')).toBe(true);
+      expect(body.get('model')).toBe('whisper-1');
+      expect(body.has('language')).toBe(true);
+      expect(body.has('response_format')).toBe(false);
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+  });
+
+  // -- Response shape parsing ---------------------------------------------
+
+  test('parses JSON response with "text" field', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () =>
+      new Response(JSON.stringify({ text: 'hello world' }), {
+        headers: { 'content-type': 'application/json' },
+      })
+    ) as any;
+
+    expect(await stt.transcribe(wav)).toBe('hello world');
+  });
+
+  test('parses JSON response with "transcript" field', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () =>
+      new Response(JSON.stringify({ transcript: 'transcript field' }), {
+        headers: { 'content-type': 'application/json' },
+      })
+    ) as any;
+
+    expect(await stt.transcribe(wav)).toBe('transcript field');
+  });
+
+  test('parses JSON response with nested "data.text" field', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () =>
+      new Response(JSON.stringify({ data: { text: 'nested text' } }), {
+        headers: { 'content-type': 'application/json' },
+      })
+    ) as any;
+
+    expect(await stt.transcribe(wav)).toBe('nested text');
+  });
+
+  test('parses plain-text response body', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () =>
+      new Response('plain text result', {
+        headers: { 'content-type': 'text/plain' },
+      })
+    ) as any;
+
+    expect(await stt.transcribe(wav)).toBe('plain text result');
+  });
+
+  test('trims whitespace from transcription result', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () =>
+      new Response(JSON.stringify({ text: '  hello  \n' }), {
+        headers: { 'content-type': 'application/json' },
+      })
+    ) as any;
+
+    expect(await stt.transcribe(wav)).toBe('hello');
+  });
+
+  // -- Error handling -----------------------------------------------------
+
+  test('throws on HTTP error with status and body', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () =>
+      new Response('server error', { status: 500 })
+    ) as any;
+
+    try {
+      await stt.transcribe(wav);
+      expect(true).toBe(false);
+    } catch (err: any) {
+      expect(err.message).toContain('Local Whisper STT error');
+      expect(err.message).toContain('500');
+    }
+  });
+
+  test('throws on empty transcription', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () =>
+      new Response(JSON.stringify({ text: '' }), {
+        headers: { 'content-type': 'application/json' },
+      })
+    ) as any;
+
+    try {
+      await stt.transcribe(wav);
+      expect(true).toBe(false);
+    } catch (err: any) {
+      expect(err.message).toContain('empty transcription');
+    }
+  });
+
+  test('propagates network errors', async () => {
+    const stt = new LocalWhisperSTT('http://localhost:8189/inference');
+    const wav = makeWavBuffer();
+
+    globalThis.fetch = mock(async () => {
+      throw new Error('Connection refused');
+    }) as any;
+
+    try {
+      await stt.transcribe(wav);
+      expect(true).toBe(false);
+    } catch (err: any) {
+      expect(err.message).toBe('Connection refused');
+    }
+  });
+});
+
+describe('OpenAIWhisperSTT.transcribe', () => {
+  const originalFetch = globalThis.fetch;
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+  });
+
+  test('defaults to the official OpenAI transcription endpoint', async () => {
+    const stt = new OpenAIWhisperSTT('test-key');
+    const wav = makeWavBuffer();
+    let calledUrl = '';
+
+    globalThis.fetch = mock(async (url: string) => {
+      calledUrl = url;
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+    expect(calledUrl).toBe('https://api.openai.com/v1/audio/transcriptions');
+  });
+
+  test('uses a configured OpenAI-compatible base URL for transcriptions', async () => {
+    const stt = new OpenAIWhisperSTT('test-key', 'whisper-1', 'https://gateway.example.com/openai');
+    const wav = makeWavBuffer();
+    let calledUrl = '';
+
+    globalThis.fetch = mock(async (url: string) => {
+      calledUrl = url;
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+    expect(calledUrl).toBe('https://gateway.example.com/openai/audio/transcriptions');
+  });
+
+  test('accepts non-standard API key formats for compatible gateways', async () => {
+    const stt = new OpenAIWhisperSTT('nvapi-custom-key', 'whisper-1', 'https://gateway.example.com/openai');
+    const wav = makeWavBuffer();
+    let authHeader = '';
+
+    globalThis.fetch = mock(async (_url: string, init?: RequestInit) => {
+      authHeader = String((init?.headers as Record<string, string> | undefined)?.Authorization ?? '');
+      return new Response(JSON.stringify({ text: 'ok' }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }) as any;
+
+    await stt.transcribe(wav);
+    expect(authHeader).toBe('Bearer nvapi-custom-key');
+  });
+});