@elizaos/app-core 2.0.0-alpha.74 → 2.0.0-alpha.76

package/test/avatar/voice-chat.test.ts

@@ -1,391 +0,0 @@
-/**
- * Tests for voice chat logic — SpeechRecognition, SpeechSynthesis, mouth animation.
- *
- * Since useVoiceChat is a React hook and we run in Node (not jsdom),
- * we test the underlying browser APIs and integration patterns directly.
- */
-
-import { nextIdleMouthOpen } from "@elizaos/app-core/hooks";
-import { beforeEach, describe, expect, it, vi } from "vitest";
-
-// ---------------------------------------------------------------------------
-// Mock SpeechRecognition for voice input tests
-// ---------------------------------------------------------------------------
-
-class MockSpeechRecognition {
-  continuous = false;
-  interimResults = false;
-  lang = "";
-  onresult:
-    | ((event: {
-        results: {
-          length: number;
-          [k: number]: {
-            isFinal: boolean;
-            0: { transcript: string; confidence: number };
-          };
-        };
-        resultIndex: number;
-      }) => void)
-    | null = null;
-  onerror: ((event: { error: string }) => void) | null = null;
-  onend: (() => void) | null = null;
-  onstart: (() => void) | null = null;
-  private _running = false;
-
-  start() {
-    this._running = true;
-    this.onstart?.();
-  }
-  stop() {
-    this._running = false;
-    this.onend?.();
-  }
-  abort() {
-    this._running = false;
-  }
-
-  get running() {
-    return this._running;
-  }
-
-  /** Simulate receiving a speech result */
-  simulateResult(transcript: string, isFinal: boolean) {
-    this.onresult?.({
-      results: {
-        length: 1,
-        0: { isFinal, 0: { transcript, confidence: 0.95 } },
-      },
-      resultIndex: 0,
-    });
-  }
-
-  /** Simulate an error */
-  simulateError(error: string) {
-    this.onerror?.({ error });
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Mock SpeechSynthesis for voice output tests
-// ---------------------------------------------------------------------------
-
-class MockSpeechSynthesisUtterance {
-  text: string;
-  rate = 1;
-  pitch = 1;
-  lang = "";
-  onstart: (() => void) | null = null;
-  onend: (() => void) | null = null;
-  onerror: ((e: { error: string }) => void) | null = null;
-
-  constructor(text?: string) {
-    this.text = text ?? "";
-  }
-}
-
-function createMockSynthesis() {
-  const spoken: MockSpeechSynthesisUtterance[] = [];
-  return {
-    speaking: false,
-    pending: false,
-    paused: false,
-    spoken,
-    speak: vi.fn((utterance: MockSpeechSynthesisUtterance) => {
-      spoken.push(utterance);
-    }),
-    cancel: vi.fn(),
-    pause: vi.fn(),
-    resume: vi.fn(),
-    getVoices: vi.fn(() => []),
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-describe("Voice Chat — Speech Recognition", () => {
-  let recognition: MockSpeechRecognition;
-
-  beforeEach(() => {
-    vi.restoreAllMocks();
-    recognition = new MockSpeechRecognition();
-  });
-
-  it("can be configured for continuous recognition", () => {
-    recognition.continuous = true;
-    recognition.interimResults = true;
-    recognition.lang = "en-US";
-
-    expect(recognition.continuous).toBe(true);
-    expect(recognition.interimResults).toBe(true);
-    expect(recognition.lang).toBe("en-US");
-  });
-
-  it("starts and reports running state", () => {
-    recognition.start();
-    expect(recognition.running).toBe(true);
-  });
-
-  it("stops and reports stopped state", () => {
-    recognition.start();
-    recognition.stop();
-    expect(recognition.running).toBe(false);
-  });
-
-  it("delivers interim transcripts", () => {
-    const transcripts: string[] = [];
-    recognition.onresult = (event) => {
-      const result = event.results[0];
-      if (result && !result.isFinal) {
-        transcripts.push(result[0].transcript);
-      }
-    };
-
-    recognition.start();
-    recognition.simulateResult("hel", false);
-    recognition.simulateResult("hello", false);
-
-    expect(transcripts).toEqual(["hel", "hello"]);
-  });
-
-  it("delivers final transcripts", () => {
-    const finals: string[] = [];
-    recognition.onresult = (event) => {
-      const result = event.results[0];
-      if (result?.isFinal) {
-        finals.push(result[0].transcript);
-      }
-    };
-
-    recognition.start();
-    recognition.simulateResult("hello world", true);
-
-    expect(finals).toEqual(["hello world"]);
-  });
-
-  it("handles recognition errors", () => {
-    const errors: string[] = [];
-    recognition.onerror = (event) => {
-      errors.push(event.error);
-    };
-
-    recognition.start();
-    recognition.simulateError("not-allowed");
-
-    expect(errors).toEqual(["not-allowed"]);
-  });
-
-  it("auto-restarts on end when enabled", () => {
-    let restartCount = 0;
-    const orig = recognition.start.bind(recognition);
-    recognition.start = () => {
-      restartCount++;
-      orig();
-    };
-
-    // Simulate the continuous listening pattern from useVoiceChat
-    recognition.onend = () => {
-      if (recognition.running) {
-        // This is what the hook does — restart if still enabled
-        recognition.start();
-      }
-    };
-
-    recognition.start(); // initial start
-    expect(restartCount).toBe(1);
-  });
-});
-
-describe("Voice Chat — Speech Synthesis", () => {
-  let synth: ReturnType<typeof createMockSynthesis>;
-
-  beforeEach(() => {
-    vi.restoreAllMocks();
-    synth = createMockSynthesis();
-  });
-
-  it("speaks text through SpeechSynthesis", () => {
-    const utterance = new MockSpeechSynthesisUtterance("Hello agent");
-    synth.speak(utterance);
-
-    expect(synth.speak).toHaveBeenCalledOnce();
-    expect(synth.spoken).toHaveLength(1);
-    expect(synth.spoken[0]?.text).toBe("Hello agent");
-  });
-
-  it("cancel stops current speech", () => {
-    const utterance = new MockSpeechSynthesisUtterance("Hello");
-    synth.speak(utterance);
-    synth.cancel();
-
-    expect(synth.cancel).toHaveBeenCalledOnce();
-  });
-
-  it("utterance fires onstart and onend callbacks", () => {
-    const events: string[] = [];
-    const utterance = new MockSpeechSynthesisUtterance("Test");
-    utterance.onstart = () => events.push("start");
-    utterance.onend = () => events.push("end");
-
-    // Simulate lifecycle
-    utterance.onstart();
-    utterance.onend();
-
-    expect(events).toEqual(["start", "end"]);
-  });
-
-  it("utterance fires onerror on failure", () => {
-    const errors: string[] = [];
-    const utterance = new MockSpeechSynthesisUtterance("Test");
-    utterance.onerror = (e) => errors.push(e.error);
-
-    utterance.onerror({ error: "synthesis-failed" });
-    expect(errors).toEqual(["synthesis-failed"]);
-  });
-
-  it("cancels previous speech before speaking new text", () => {
-    // This is the pattern used in useVoiceChat
-    const u1 = new MockSpeechSynthesisUtterance("First");
-    synth.speak(u1);
-    synth.cancel(); // cancel first
-    const u2 = new MockSpeechSynthesisUtterance("Second");
-    synth.speak(u2);
-
-    expect(synth.cancel).toHaveBeenCalledOnce();
-    expect(synth.spoken).toHaveLength(2);
-    expect(synth.spoken[1]?.text).toBe("Second");
-  });
-});
-
-describe("Voice Chat — Mouth Animation", () => {
-  it("generates natural-looking mouth values from sine waves", () => {
-    // This tests the animation formula from useVoiceChat
-    const values: number[] = [];
-    const _startTime = Date.now();
-
-    for (let i = 0; i < 20; i++) {
-      const elapsed = i * 0.05; // 50ms intervals
-      const base = Math.sin(elapsed * 12) * 0.3 + 0.4;
-      const detail = Math.sin(elapsed * 18.7) * 0.15;
-      const slow = Math.sin(elapsed * 4.2) * 0.1;
-      const value = Math.max(0, Math.min(1, base + detail + slow));
-      values.push(value);
-    }
-
-    // All values should be in [0, 1]
-    for (const v of values) {
-      expect(v).toBeGreaterThanOrEqual(0);
-      expect(v).toBeLessThanOrEqual(1);
-    }
-
-    // Values should vary (not all the same)
-    const unique = new Set(values.map((v) => v.toFixed(4)));
-    expect(unique.size).toBeGreaterThan(5);
-  });
-
-  it("smooth-closes mouth when speaking stops", () => {
-    // Idle decay should keep closing all the way to silence.
-    let mouthOpen = 0.8;
-    for (let i = 0; i < 40; i++) {
-      mouthOpen = nextIdleMouthOpen(mouthOpen);
-    }
-    expect(mouthOpen).toBe(0);
-  });
-
-  it("does not get stuck at the last stepped mouth-open value", () => {
-    expect(nextIdleMouthOpen(0.06)).toBeCloseTo(0.04);
-    expect(nextIdleMouthOpen(0.04)).toBeCloseTo(0.02);
-    expect(nextIdleMouthOpen(0.02)).toBe(0);
-  });
-});
-
-describe("Voice Chat — Integration patterns", () => {
-  it("voice transcript triggers chat send flow", () => {
-    // Simulates the handleVoiceTranscript callback from ChatAvatar
-    const sentMessages: string[] = [];
-    let chatInput = "";
-    let chatSending = false;
-
-    const handleVoiceTranscript = (text: string) => {
-      if (chatSending) return;
-      chatInput = text;
-      sentMessages.push(text);
-    };
-
-    handleVoiceTranscript("Hello agent");
-    expect(chatInput).toBe("Hello agent");
-    expect(sentMessages).toEqual(["Hello agent"]);
-
-    // Should not send while already sending
-    chatSending = true;
-    handleVoiceTranscript("Another message");
-    expect(sentMessages).toEqual(["Hello agent"]);
-  });
-
-  it("auto-speaks agent responses when voice is active", () => {
-    // Simulates the auto-speak logic from ChatAvatar
-    const spokenTexts: string[] = [];
-    let lastSpokenId: string | null = null;
-    const isListening = true;
-    const chatSending = false;
-
-    const checkAndSpeak = (msg: { id: string; role: string; text: string }) => {
-      if (
-        msg.role === "assistant" &&
-        msg.id !== lastSpokenId &&
-        isListening &&
-        !chatSending
-      ) {
-        lastSpokenId = msg.id;
-        spokenTexts.push(msg.text);
-      }
-    };
-
-    checkAndSpeak({
-      id: "msg-1",
-      role: "assistant",
-      text: "Hello! How can I help?",
-    });
-    expect(spokenTexts).toEqual(["Hello! How can I help?"]);
-
-    // Same message should not be spoken again
-    checkAndSpeak({
-      id: "msg-1",
-      role: "assistant",
-      text: "Hello! How can I help?",
-    });
-    expect(spokenTexts).toEqual(["Hello! How can I help?"]);
-
-    // User messages should not be spoken
-    checkAndSpeak({ id: "msg-2", role: "user", text: "Test" });
-    expect(spokenTexts).toEqual(["Hello! How can I help?"]);
-
-    // New assistant message should be spoken
-    checkAndSpeak({ id: "msg-3", role: "assistant", text: "Sure thing!" });
-    expect(spokenTexts).toEqual(["Hello! How can I help?", "Sure thing!"]);
-  });
-
-  it("does not speak responses when voice is not active", () => {
-    const spokenTexts: string[] = [];
-    let lastSpokenId: string | null = null;
-    const isListening = false; // voice not active
-    const chatSending = false;
-
-    const checkAndSpeak = (msg: { id: string; role: string; text: string }) => {
-      if (
-        msg.role === "assistant" &&
-        msg.id !== lastSpokenId &&
-        isListening &&
-        !chatSending
-      ) {
-        lastSpokenId = msg.id;
-        spokenTexts.push(msg.text);
-      }
-    };
-
-    checkAndSpeak({ id: "msg-1", role: "assistant", text: "Hello!" });
-    expect(spokenTexts).toEqual([]);
-  });
-});

package/test/browser-extension/README.md

@@ -1,138 +0,0 @@
-# Browser Extension Test Harness
-
-This directory contains tools for testing the ComputerUse Bridge Extension.
-
-## Overview
-
-The ComputerUse Bridge Extension is a Chrome MV3 extension that enables JavaScript
-evaluation in browser tabs via a WebSocket connection. This test harness validates
-that the extension correctly:
-
-1. Connects to the WebSocket server at `ws://127.0.0.1:17373`
-2. Receives eval requests with the correct protocol
-3. Evaluates JavaScript in the active tab
-4. Returns results (or errors) with the correct format
-
-## Prerequisites
-
-1. **Chrome/Chromium browser** with Developer Mode enabled
-2. **The extension loaded** from `eliza/packages/computeruse/crates/computeruse/browser-extension/`
-3. **At least one browser tab** open (extension needs a tab to evaluate code in)
-4. **Node.js 22+** with tsx installed
-
-## Loading the Extension
-
-1. Open Chrome and navigate to `chrome://extensions`
-2. Enable "Developer mode" (toggle in top right)
-3. Click "Load unpacked"
-4. Select the `browser-extension/` directory
-5. Verify the extension appears and is enabled
-
-## Running the Test Harness
-
-### Basic Test Run
-
-```bash
-# From milady root
-npx tsx test/browser-extension/test-harness.ts
-```
-
-### Options
-
-```bash
-# Verbose output (show all messages)
-npx tsx test/browser-extension/test-harness.ts --verbose
-
-# Interactive mode (keep server running for manual testing)
-npx tsx test/browser-extension/test-harness.ts --interactive
-
-# Custom port (default: 17373)
-npx tsx test/browser-extension/test-harness.ts --port 17374
-
-# Custom timeout (default: 30000ms)
-npx tsx test/browser-extension/test-harness.ts --timeout 60000
-```
-
-## Protocol Specification
-
-### Request Format
-
-```json
-{
-  "id": "unique-request-id",
-  "action": "eval",
-  "code": "document.title",
-  "awaitPromise": false
-}
-```
-
-### Success Response
-
-```json
-{
-  "id": "unique-request-id",
-  "ok": true,
-  "result": "Page Title"
-}
-```
-
-### Error Response
-
-```json
-{
-  "id": "unique-request-id",
-  "ok": false,
-  "error": "ReferenceError: foo is not defined"
-}
-```
-
-## Test Categories
-
-The harness runs tests in these categories:
-
-1. **Basic JavaScript Evaluation** - Simple expressions, arrays, objects
-2. **DOM Access** - document.title, location.href, querySelectorAll
-3. **Promise Handling** - awaitPromise flag, async operations
-4. **Error Handling** - Syntax errors, runtime errors, rejected promises
-5. **Complex Operations** - Page metadata extraction, finding elements
-
-## Troubleshooting
-
-### Extension doesn't connect
-
-1. Check that the extension is loaded and enabled in `chrome://extensions`
-2. Check that Developer mode is enabled
-3. Reload the extension by clicking the refresh button
-4. Make sure at least one tab is open (not just chrome:// pages)
-
-### Tests timeout
-
-1. The extension may need a moment to connect after loading
-2. Try increasing the timeout: `--timeout 60000`
-3. Check Chrome DevTools for extension errors (go to `chrome://extensions`, click "Inspect views" on the extension)
-
-### "Cannot evaluate in this tab"
-
-Some pages block extension access:
-- `chrome://` pages
-- `edge://` pages
-- Extension pages
-- Some protected sites
-
-Navigate to a regular webpage (like https://example.com) before running tests.
-
-## Manual Testing
-
-For manual testing, start the harness in interactive mode:
-
-```bash
-npx tsx test/browser-extension/test-harness.ts --interactive --verbose
-```
-
-Then send requests manually using a tool like `wscat`:
-
-```bash
-wscat -c ws://127.0.0.1:17373
-> {"id":"1","action":"eval","code":"document.title"}
-< {"id":"1","ok":true,"result":"Page Title"}
-```