@elizaos/plugin-xr 2.0.3-beta.5

package/simulator/src/playwright-fixture.ts

@@ -0,0 +1,169 @@
+/**
+ * Playwright fixture and page wrapper for XR emulator testing.
+ *
+ * Usage:
+ *   import { test, expect } from './fixtures.ts';
+ *
+ *   test('full roundtrip', async ({ xrPage, mockAgent }) => {
+ *     await xrPage.goto('/');
+ *     await mockAgent.waitForConnection();
+ *     await xrPage.injectCameraFrame('./fixtures/desk.jpg');
+ *     const frame = await mockAgent.waitForCameraFrame();
+ *     expect(frame.payload.length).toBeGreaterThan(100);
+ *   });
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { test as base, type Page } from "@playwright/test";
+import { MockAgentServer } from "./mock-agent.ts";
+import type { EmulatorStats, XRPose } from "./types.ts";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const EMULATOR_DIST = resolve(__dirname, "../dist/emulator.js");
+
+// ── XREmulatorPage ────────────────────────────────────────────────────────
+
+export class XREmulatorPage {
+  constructor(readonly page: Page) {}
+
+  /** Inject the emulator script before the page loads. Call before page.goto(). */
+  async inject(): Promise<void> {
+    if (!existsSync(EMULATOR_DIST)) {
+      throw new Error(
+        `Emulator bundle not found at ${EMULATOR_DIST}. Run: cd eliza/plugins/plugin-xr/simulator && bun run build`,
+      );
+    }
+    await this.page.addInitScript({ path: EMULATOR_DIST });
+  }
+
+  /** Navigate and wait for the emulator to be ready. */
+  async goto(url: string): Promise<void> {
+    await this.page.goto(url);
+    // Wait for emulator to install (logs a console message)
+    await this.page.waitForFunction(
+      () => typeof window.__XREmulator !== "undefined",
+      {
+        timeout: 5000,
+      },
+    );
+  }
+
+  /** Set the emulated headset pose. */
+  async setPose(pose: Partial<XRPose>): Promise<void> {
+    await this.page.evaluate((p) => window.__XREmulator.setPose(p), pose);
+  }
+
+  /** Inject a camera frame from a local image file (JPEG or PNG). */
+  async injectCameraFrame(imagePath: string): Promise<void> {
+    const abs = resolve(imagePath);
+    const data = readFileSync(abs);
+    const mime = imagePath.endsWith(".png") ? "image/png" : "image/jpeg";
+    const dataUrl = `data:${mime};base64,${data.toString("base64")}`;
+    await this.page.evaluate(
+      (url) => window.__XREmulator.injectCameraFrame(url),
+      dataUrl,
+    );
+  }
+
+  /** Inject a camera frame from an inline data URL. */
+  async injectCameraDataUrl(dataUrl: string): Promise<void> {
+    await this.page.evaluate(
+      (url) => window.__XREmulator.injectCameraFrame(url),
+      dataUrl,
+    );
+  }
+
+  /** Send a synthetic audio chunk directly to the agent WebSocket. */
+  async sendAudioChunk(
+    base64: string,
+    sampleRate = 48000,
+    encoding = "webm-opus",
+  ): Promise<void> {
+    await this.page.evaluate(
+      ({ b64, sr, enc }) => {
+        if (!window.__xrTestHooks)
+          throw new Error("__xrTestHooks not available — is VITE_TEST=true?");
+        window.__xrTestHooks.sendAudioChunk(b64, sr, enc);
+      },
+      { b64: base64, sr: sampleRate, enc: encoding },
+    );
+  }
+
+  /** Get emulator stats. */
+  async getStats(): Promise<EmulatorStats> {
+    return this.page.evaluate(() => window.__XREmulator.getStats());
+  }
+
+  /** Get WebSocket readyState from test hooks. */
+  async getSocketState(): Promise<string> {
+    return this.page.evaluate(() => {
+      if (!window.__xrTestHooks) return "UNAVAILABLE";
+      return window.__xrTestHooks.getSocketState();
+    });
+  }
+
+  /** Wait for the page's status text to match a pattern. */
+  async waitForStatus(pattern: string | RegExp, timeout = 8000): Promise<void> {
+    await this.page
+      .locator("#status-text")
+      .filter({ hasText: pattern })
+      .waitFor({
+        state: "visible",
+        timeout,
+      });
+  }
+
+  /** Wait for agent response text to appear. */
+  async waitForAgentText(timeout = 10000): Promise<string> {
+    const el = this.page.locator("#agent-response");
+    await el.waitFor({ state: "visible", timeout });
+    await el.filter({ hasNotText: "" }).waitFor({ timeout });
+    return el.innerText();
+  }
+
+  /** Wait for transcript text to appear. */
+  async waitForTranscript(timeout = 10000): Promise<string> {
+    const el = this.page.locator("#transcript");
+    await el.waitFor({ state: "visible", timeout });
+    await el.filter({ hasNotText: "" }).waitFor({ timeout });
+    return el.innerText();
+  }
+
+  /** Force-disconnect the WebSocket (tests reconnect logic). */
+  async simulateDisconnect(): Promise<void> {
+    await this.page.evaluate(() => window.__XREmulator.simulateDisconnect());
+  }
+}
+
+// ── Playwright fixture extensions ─────────────────────────────────────────
+
+interface XRFixtures {
+  mockAgent: MockAgentServer;
+  xrPage: XREmulatorPage;
+}
+
+export const test = base.extend<XRFixtures>({
+  mockAgent: async (_fixtures, use, testInfo) => {
+    // Use a unique port per worker to allow parallel test runs
+    const port = 31338 + testInfo.workerIndex;
+    const server = new MockAgentServer({ port });
+    await server.start();
+    await use(server);
+    await server.stop();
+  },
+
+  xrPage: async ({ page }, use) => {
+    const xrp = new XREmulatorPage(page);
+    await xrp.inject();
+    await use(xrp);
+  },
+});
+
+export { expect } from "@playwright/test";
+
+// ── Node-side exports ─────────────────────────────────────────────────────
+
+export { MockAgentServer } from "./mock-agent.ts";
+export type { EmulatorStats, XRPose } from "./types.ts";

package/simulator/src/types.ts

@@ -0,0 +1,51 @@
+export interface Vec3 {
+  x: number;
+  y: number;
+  z: number;
+}
+
+export interface Quat {
+  x: number;
+  y: number;
+  z: number;
+  w: number;
+}
+
+export interface XRPose {
+  position: Vec3;
+  orientation: Quat;
+}
+
+export interface EmulatorStats {
+  sessionActive: boolean;
+  framesInjected: number;
+  cameraStreamActive: boolean;
+  wsConnected: boolean;
+}
+
+/** window.__XREmulator — set by emulator.ts, consumed by Playwright via page.evaluate() */
+export interface XREmulatorAPI {
+  setPose(pose: Partial<XRPose>): void;
+  injectCameraFrame(jpegDataUrl: string): Promise<void>;
+  getStats(): EmulatorStats;
+  /** Simulate device disconnection (closes WebSocket) */
+  simulateDisconnect(): void;
+  /** Simulate reconnect after a disconnect */
+  simulateReconnect(): void;
+}
+
+declare global {
+  interface Window {
+    __XREmulator: XREmulatorAPI;
+    /** Set by app-xr/src/main.ts in VITE_TEST mode */
+    __xrTestHooks: {
+      sendAudioChunk(
+        base64: string,
+        sampleRate: number,
+        encoding: string,
+      ): void;
+      getSocketState(): "CONNECTING" | "OPEN" | "CLOSING" | "CLOSED";
+      sendPing?(): void;
+    };
+  }
+}

package/simulator/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "noEmit": true,
+    "allowImportingTsExtensions": true,
+    "skipLibCheck": true,
+    "lib": ["ES2022", "DOM"]
+  },
+  "include": ["src/**/*.ts"]
+}

package/simulator/vite.config.ts

@@ -0,0 +1,25 @@
+import { resolve } from "node:path";
+import { defineConfig } from "vite";
+
+// Builds the browser-side emulator as a self-contained IIFE.
+// Output: dist/emulator.js — injected into pages by Playwright via addInitScript.
+export default defineConfig({
+  build: {
+    lib: {
+      entry: resolve(__dirname, "src/emulator.ts"),
+      name: "XREmulator",
+      fileName: "emulator",
+      formats: ["iife"],
+    },
+    outDir: "dist",
+    rollupOptions: {
+      output: {
+        inlineDynamicImports: true,
+        // Force the output filename to emulator.js (not emulator.iife.js)
+        entryFileNames: "[name].js",
+      },
+    },
+    minify: false, // keep readable for debugging
+    sourcemap: true,
+  },
+});

package/src/__tests__/audio-pipeline.test.ts

@@ -0,0 +1,129 @@
+import type { IAgentRuntime } from "@elizaos/core";
+import { ModelType } from "@elizaos/core";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { AudioPipeline } from "../services/audio-pipeline.ts";
+
+function makeRuntime(transcriptResult = "hello world"): IAgentRuntime {
+  return {
+    useModel: vi.fn().mockResolvedValue(transcriptResult),
+  } as unknown as IAgentRuntime;
+}
+
+describe("AudioPipeline", () => {
+  let onTranscript: ReturnType<typeof vi.fn>;
+  let runtime: IAgentRuntime;
+  let pipeline: AudioPipeline;
+
+  beforeEach(() => {
+    onTranscript = vi.fn().mockResolvedValue(undefined);
+    runtime = makeRuntime("test transcript");
+    pipeline = new AudioPipeline(
+      runtime,
+      onTranscript as (id: string, text: string) => Promise<void>,
+    );
+  });
+
+  it("calls onTranscript after flush with enough audio", async () => {
+    const header = {
+      type: "audio" as const,
+      ts: 0,
+      sampleRate: 16000,
+      encoding: "webm-opus" as const,
+    };
+    const chunk = Buffer.alloc(1024, 0x55);
+
+    pipeline.push("conn1", header, chunk);
+    await pipeline.flush("conn1");
+
+    expect(runtime.useModel).toHaveBeenCalledWith(
+      ModelType.TRANSCRIPTION,
+      expect.any(Buffer),
+    );
+    expect(onTranscript).toHaveBeenCalledWith("conn1", "test transcript");
+  });
+
+  it("does not call onTranscript for tiny chunks", async () => {
+    const header = {
+      type: "audio" as const,
+      ts: 0,
+      sampleRate: 16000,
+      encoding: "webm-opus" as const,
+    };
+    pipeline.push("conn1", header, Buffer.alloc(64));
+    await pipeline.flush("conn1");
+
+    expect(runtime.useModel).not.toHaveBeenCalled();
+    expect(onTranscript).not.toHaveBeenCalled();
+  });
+
+  it("ignores empty transcription results", async () => {
+    (runtime.useModel as ReturnType<typeof vi.fn>).mockResolvedValue("   ");
+    const header = {
+      type: "audio" as const,
+      ts: 0,
+      sampleRate: 16000,
+      encoding: "webm-opus" as const,
+    };
+    pipeline.push("conn1", header, Buffer.alloc(1024));
+    await pipeline.flush("conn1");
+
+    expect(onTranscript).not.toHaveBeenCalled();
+  });
+
+  it("clears pending state on clear()", () => {
+    const header = {
+      type: "audio" as const,
+      ts: 0,
+      sampleRate: 16000,
+      encoding: "webm-opus" as const,
+    };
+    pipeline.push("conn1", header, Buffer.alloc(512));
+    pipeline.clear("conn1");
+    // after clear, flush has no pending audio to emit
+    expect(onTranscript).not.toHaveBeenCalled();
+  });
+
+  it("wraps pcm-f32 chunks with a WAV header before transcription", async () => {
+    // pcm-f32: 1024 float32 samples = 4096 bytes of raw audio (> 512-byte floor)
+    const float32Samples = new Float32Array(1024).fill(0.1);
+    const chunk = Buffer.from(float32Samples.buffer);
+
+    const header = {
+      type: "audio" as const,
+      ts: 0,
+      sampleRate: 44100,
+      encoding: "pcm-f32" as const,
+    };
+    pipeline.push("conn1", header, chunk);
+    await pipeline.flush("conn1");
+
+    const callArg = (runtime.useModel as ReturnType<typeof vi.fn>).mock
+      .calls[0]?.[1] as Buffer;
+
+    // WAV header starts with "RIFF"
+    expect(callArg.subarray(0, 4).toString("ascii")).toBe("RIFF");
+    // IEEE_FLOAT format tag = 3 at offset 20 (little-endian uint16)
+    expect(callArg.readUInt16LE(20)).toBe(3);
+    // sample rate at offset 24
+    expect(callArg.readUInt32LE(24)).toBe(44100);
+    // data chunk starts at offset 44 — matches original pcm bytes
+    expect(callArg.subarray(44)).toEqual(chunk);
+  });
+
+  it("auto-flushes after FLUSH_AFTER_MS duration", async () => {
+    const header = {
+      type: "audio" as const,
+      ts: 0,
+      sampleRate: 16000,
+      encoding: "webm-opus" as const,
+    };
+    pipeline.push("conn1", header, Buffer.alloc(1024));
+    // Advance ts by 2100ms to trigger auto-flush
+    const header2 = { ...header, ts: 2100 };
+    pipeline.push("conn1", header2, Buffer.alloc(1024));
+
+    // flush was called internally; useModel should have been called
+    await new Promise((r) => setTimeout(r, 10));
+    expect(runtime.useModel).toHaveBeenCalled();
+  });
+});

package/src/__tests__/protocol.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from "vitest";
+import { decodeBinaryFrame, encodeBinaryFrame } from "../protocol.ts";
+
+describe("binary frame codec", () => {
+  it("round-trips an audio frame", () => {
+    const header = {
+      type: "audio" as const,
+      ts: 1234567890,
+      sampleRate: 48000,
+      encoding: "webm-opus" as const,
+    };
+    const payload = Buffer.from([1, 2, 3, 4, 5]);
+    const frame = encodeBinaryFrame(header, payload);
+    const decoded = decodeBinaryFrame(frame);
+
+    expect(decoded.header).toEqual(header);
+    expect(decoded.payload).toEqual(payload);
+  });
+
+  it("round-trips a frame with pose data", () => {
+    const header = {
+      type: "frame" as const,
+      ts: 999,
+      width: 1280,
+      height: 720,
+      format: "jpeg" as const,
+      pose: {
+        position: { x: 1, y: 2, z: 3 },
+        orientation: { x: 0, y: 0, z: 0, w: 1 },
+      },
+    };
+    const payload = Buffer.alloc(100, 0xab);
+    const { header: h, payload: p } = decodeBinaryFrame(
+      encodeBinaryFrame(header, payload),
+    );
+    expect(h).toEqual(header);
+    expect(p).toEqual(payload);
+  });
+
+  it("handles zero-length payload", () => {
+    const header = {
+      type: "tts_audio" as const,
+      sampleRate: 24000,
+      channels: 1,
+      encoding: "mp3" as const,
+    };
+    const { header: h, payload } = decodeBinaryFrame(
+      encodeBinaryFrame(header, Buffer.alloc(0)),
+    );
+    expect(h).toEqual(header);
+    expect(payload.length).toBe(0);
+  });
+});