@elizaos/plugin-xr 2.0.3-beta.6 → 2.0.3-beta.7

package/simulator/src/emulator.ts

@@ -1,174 +0,0 @@
-/**
- * XR Emulator — browser-side IIFE injected by Playwright via page.addInitScript().
- *
- * What it does:
- *  1. Installs IWER (immersive-web-emulation-runtime) to polyfill navigator.xr
- *     with a controllable Quest 3 device.
- *  2. Overrides navigator.mediaDevices.getUserMedia to return:
- *     - Video: a canvas-captureStream() that Playwright can paint frames onto.
- *     - Audio: a synthetic silence stream (real audio comes via __xrTestHooks).
- *  3. Exposes window.__XREmulator with a programmatic control API.
- *
- * Fork baseline: meta-quest/immersive-web-emulator
- * Additions: camera frame injection, audio stream mock, __XREmulator control API.
- *
- * rawCameraAccess simulation:
- *   The experimental WebXR rawCameraAccess path (XRWebGLBinding.getCameraImage) is
- *   outside IWER's current emulation surface, so app-xr automatically falls back to the getUserMedia
- *   video track (Path 3). Injecting frames via __XREmulator.injectCameraFrame() paints
- *   onto the canvas that feeds getUserMedia, making injected frames reachable by both
- *   the getUserMedia path and any code that reads the canvas directly.
- */
-
-import { metaQuest3, XRDevice } from "iwer";
-import type { EmulatorStats, XREmulatorAPI, XRPose } from "./types.ts";
-
-// ── Camera canvas ─────────────────────────────────────────────────────────
-
-const cameraCanvas = document.createElement("canvas");
-cameraCanvas.width = 640;
-cameraCanvas.height = 480;
-const cameraCtx = cameraCanvas.getContext("2d")!;
-
-// Fill with a recognisable test pattern (grey + crosshair)
-function drawTestPattern(ctx: CanvasRenderingContext2D, w: number, h: number) {
-  ctx.fillStyle = "#333";
-  ctx.fillRect(0, 0, w, h);
-  ctx.strokeStyle = "#0f0";
-  ctx.lineWidth = 2;
-  ctx.beginPath();
-  ctx.moveTo(w / 2, 0);
-  ctx.lineTo(w / 2, h);
-  ctx.moveTo(0, h / 2);
-  ctx.lineTo(w, h / 2);
-  ctx.stroke();
-  ctx.fillStyle = "#0f0";
-  ctx.font = "16px monospace";
-  ctx.fillText("XR SIMULATOR", 12, 24);
-}
-drawTestPattern(cameraCtx, 640, 480);
-
-const cameraStream = cameraCanvas.captureStream(30); // 30 fps canvas stream
-
-// ── Audio stream (silence) ───────────────────────────────────────────────
-
-function createSilentAudioStream(): MediaStream {
-  const ctx = new AudioContext();
-  const dest = ctx.createMediaStreamDestination();
-  // Connect a silent oscillator at 0 gain to keep the stream alive
-  const gain = ctx.createGain();
-  gain.gain.value = 0;
-  const osc = ctx.createOscillator();
-  osc.connect(gain);
-  gain.connect(dest);
-  osc.start();
-  return dest.stream;
-}
-
-let silentAudioStream: MediaStream | null = null;
-
-// ── getUserMedia override ─────────────────────────────────────────────────
-
-const _originalGetUserMedia = navigator.mediaDevices.getUserMedia.bind(
-  navigator.mediaDevices,
-);
-
-navigator.mediaDevices.getUserMedia = async (
-  constraints?: MediaStreamConstraints,
-): Promise<MediaStream> => {
-  const hasVideo = constraints?.video;
-  const hasAudio = constraints?.audio;
-
-  if (hasVideo && !hasAudio) {
-    // Camera-only: return our canvas stream
-    return cameraStream;
-  }
-
-  if (hasAudio && !hasVideo) {
-    // Mic-only: return synthetic silence
-    if (!silentAudioStream) silentAudioStream = createSilentAudioStream();
-    return silentAudioStream;
-  }
-
-  if (hasVideo && hasAudio) {
-    // Combined: merge both tracks into one MediaStream
-    if (!silentAudioStream) silentAudioStream = createSilentAudioStream();
-    const combined = new MediaStream([
-      ...cameraStream.getVideoTracks(),
-      ...silentAudioStream.getAudioTracks(),
-    ]);
-    return combined;
-  }
-
-  // Fallback for other constraint shapes
-  return _originalGetUserMedia(constraints);
-};
-
-// ── IWER XR device ────────────────────────────────────────────────────────
-
-const xrDevice = new XRDevice(metaQuest3);
-xrDevice.installRuntime();
-
-// ── State ─────────────────────────────────────────────────────────────────
-
-let framesInjected = 0;
-
-// ── Control API ───────────────────────────────────────────────────────────
-
-const api: XREmulatorAPI = {
-  setPose(pose: Partial<XRPose>) {
-    if (pose.position) {
-      xrDevice.position.set(pose.position.x, pose.position.y, pose.position.z);
-    }
-    if (pose.orientation) {
-      xrDevice.quaternion.set(
-        pose.orientation.x,
-        pose.orientation.y,
-        pose.orientation.z,
-        pose.orientation.w,
-      );
-    }
-  },
-
-  async injectCameraFrame(jpegDataUrl: string): Promise<void> {
-    // createImageBitmap is more reliable than new Image() in headless contexts
-    const resp = await fetch(jpegDataUrl);
-    const blob = await resp.blob();
-    const bmp = await createImageBitmap(blob);
-    cameraCtx.drawImage(bmp, 0, 0, cameraCanvas.width, cameraCanvas.height);
-    bmp.close();
-    framesInjected++;
-  },
-
-  getStats(): EmulatorStats {
-    const wsConnected =
-      typeof window.__xrTestHooks !== "undefined" &&
-      window.__xrTestHooks.getSocketState() === "OPEN";
-    return {
-      sessionActive: false, // updated below once session is active
-      framesInjected,
-      cameraStreamActive: cameraStream.active,
-      wsConnected,
-    };
-  },
-
-  simulateDisconnect() {
-    // Force-close the WebSocket so the reconnect logic kicks in
-    // The app exposes the socket via __xrTestHooks
-    if (window.__xrTestHooks) {
-      (
-        window as { __xrForceDisconnect?: () => void }
-      ).__xrForceDisconnect?.();
-    }
-  },
-
-  simulateReconnect() {
-    (
-      window as { __xrForceReconnect?: () => void }
-    ).__xrForceReconnect?.();
-  },
-};
-
-window.__XREmulator = api;
-
-console.info("[XR Emulator] installed — navigator.xr:", !!navigator.xr);

package/simulator/src/mock-agent.ts

@@ -1,233 +0,0 @@
-/**
- * Lightweight WebSocket server that simulates plugin-xr's XRSessionService.
- * Used in Playwright tests — starts a real ws server on a configurable port,
- * records all received binary frames, and lets tests script responses.
- */
-
-import { WebSocket, WebSocketServer } from "ws";
-import {
-  decodeBinaryFrame,
-  encodeBinaryFrame,
-  type XRBinaryHeader,
-  type XRClientControl,
-  type XRTTSAudioHeader,
-} from "../../src/protocol.ts";
-
-export interface ReceivedFrame {
-  header: XRBinaryHeader | XRTTSAudioHeader;
-  payload: Buffer;
-  receivedAt: number;
-}
-
-export interface ReceivedControl {
-  message: XRClientControl;
-  receivedAt: number;
-}
-
-export interface MockAgentServerOptions {
-  port?: number;
-  /** Automatically send a transcript response after receiving N audio frames */
-  autoTranscriptAfterFrames?: number;
-  autoTranscriptText?: string;
-}
-
-export class MockAgentServer {
-  private wss: WebSocketServer | null = null;
-  private client: WebSocket | null = null;
-
-  readonly receivedFrames: ReceivedFrame[] = [];
-  readonly receivedControls: ReceivedControl[] = [];
-  private waiters = new Map<string, Array<() => void>>();
-
-  constructor(private readonly options: MockAgentServerOptions = {}) {}
-
-  get port(): number {
-    return this.options.port ?? 31338;
-  }
-
-  async start(): Promise<void> {
-    return new Promise((resolve, reject) => {
-      this.wss = new WebSocketServer({ port: this.port });
-      this.wss.on("listening", resolve);
-      this.wss.on("error", reject);
-      this.wss.on("connection", (ws) => this.onClient(ws));
-    });
-  }
-
-  async stop(): Promise<void> {
-    this.client?.close();
-    this.client = null;
-    return new Promise((resolve) => this.wss?.close(() => resolve()));
-  }
-
-  // ── Sending responses ──────────────────────────────────────────────────
-
-  sendTranscript(text: string, final = true): void {
-    this.sendText({ type: "transcript", text, final });
-  }
-
-  sendAgentText(text: string): void {
-    this.sendText({ type: "agent_text", text });
-  }
-
-  sendTTSAudio(audio: Buffer, sampleRate = 24000): void {
-    if (!this.client || this.client.readyState !== WebSocket.OPEN) return;
-    const header: XRTTSAudioHeader = {
-      type: "tts_audio",
-      sampleRate,
-      channels: 1,
-      encoding: "mp3",
-    };
-    this.client.send(encodeBinaryFrame(header, audio), { binary: true });
-  }
-
-  // ── Waiting helpers ────────────────────────────────────────────────────
-
-  /** Resolves when the device has connected and sent a 'hello' message.
-   *  Resolves immediately if 'hello' was already received (handles the race
-   *  where the connection fires before the waiter is registered). */
-  waitForConnection(timeoutMs = 5000): Promise<void> {
-    if (this.receivedControls.some((c) => c.message.type === "hello")) {
-      return Promise.resolve();
-    }
-    return this.waitFor("connected", timeoutMs);
-  }
-
-  /** Resolves when at least one audio binary frame has been received. */
-  waitForAudioFrame(timeoutMs = 10000): Promise<ReceivedFrame> {
-    return this.waitForFrame("audio", timeoutMs);
-  }
-
-  /** Resolves when at least one camera binary frame has been received. */
-  waitForCameraFrame(timeoutMs = 10000): Promise<ReceivedFrame> {
-    return this.waitForFrame("frame", timeoutMs);
-  }
-
-  audioFrames(): ReceivedFrame[] {
-    return this.receivedFrames.filter((f) => f.header.type === "audio");
-  }
-
-  cameraFrames(): ReceivedFrame[] {
-    return this.receivedFrames.filter((f) => f.header.type === "frame");
-  }
-
-  reset(): void {
-    this.receivedFrames.length = 0;
-    this.receivedControls.length = 0;
-  }
-
-  // ── Private ────────────────────────────────────────────────────────────
-
-  private onClient(ws: WebSocket): void {
-    this.client = ws;
-
-    ws.on("message", (data, isBinary) => {
-      if (isBinary) {
-        this.handleBinary(data as Buffer);
-      } else {
-        this.handleText(ws, data.toString("utf8"));
-      }
-    });
-
-    ws.on("close", () => {
-      if (this.client === ws) this.client = null;
-    });
-  }
-
-  private handleText(ws: WebSocket, raw: string): void {
-    try {
-      const msg = JSON.parse(raw) as XRClientControl;
-      this.receivedControls.push({ message: msg, receivedAt: Date.now() });
-
-      if (msg.type === "hello") {
-        // Send ready
-        ws.send(JSON.stringify({ type: "ready", sessionId: "mock-session" }));
-        this.notify("connected");
-      }
-      if (msg.type === "ping") {
-        ws.send(JSON.stringify({ type: "pong" }));
-      }
-    } catch {}
-  }
-
-  private handleBinary(data: Buffer): void {
-    try {
-      const { header, payload } = decodeBinaryFrame(data);
-      const frame: ReceivedFrame = { header, payload, receivedAt: Date.now() };
-      this.receivedFrames.push(frame);
-      this.notify(`frame:${header.type}`);
-
-      // Auto-transcript after N audio frames
-      const { autoTranscriptAfterFrames, autoTranscriptText } = this.options;
-      if (
-        header.type === "audio" &&
-        autoTranscriptAfterFrames !== undefined &&
-        this.audioFrames().length >= autoTranscriptAfterFrames
-      ) {
-        const text = autoTranscriptText ?? "test transcript";
-        setTimeout(() => {
-          this.sendTranscript(text);
-          this.sendAgentText(`Agent response to: ${text}`);
-        }, 50);
-      }
-    } catch {}
-  }
-
-  private sendText(msg: object): void {
-    if (!this.client || this.client.readyState !== WebSocket.OPEN) return;
-    this.client.send(JSON.stringify(msg));
-  }
-
-  private waitFor(event: string, timeoutMs: number): Promise<void> {
-    return new Promise((resolve, reject) => {
-      const timer = setTimeout(
-        () =>
-          reject(new Error(`MockAgentServer: timeout waiting for "${event}"`)),
-        timeoutMs,
-      );
-      const list = this.waiters.get(event) ?? [];
-      list.push(() => {
-        clearTimeout(timer);
-        resolve();
-      });
-      this.waiters.set(event, list);
-    });
-  }
-
-  private waitForFrame(
-    type: string,
-    timeoutMs: number,
-  ): Promise<ReceivedFrame> {
-    // Check if already received
-    const existing = this.receivedFrames.find((f) => f.header.type === type);
-    if (existing) return Promise.resolve(existing);
-
-    return new Promise((resolve, reject) => {
-      const timer = setTimeout(
-        () =>
-          reject(
-            new Error(
-              `MockAgentServer: timeout waiting for frame type "${type}"`,
-            ),
-          ),
-        timeoutMs,
-      );
-      const event = `frame:${type}`;
-      const list = this.waiters.get(event) ?? [];
-      list.push(() => {
-        clearTimeout(timer);
-        const frame = [...this.receivedFrames].reverse().find(
-          (f: ReceivedFrame) => f.header.type === type,
-        )!;
-        resolve(frame);
-      });
-      this.waiters.set(event, list);
-    });
-  }
-
-  private notify(event: string): void {
-    const list = this.waiters.get(event) ?? [];
-    this.waiters.delete(event);
-    for (const cb of list) cb();
-  }
-}

package/simulator/src/node.ts

@@ -1,9 +0,0 @@
-// Node-side entry point — exports the Playwright fixture and mock server.
-// Do not import this from browser code.
-export {
-  expect,
-  MockAgentServer,
-  test,
-  XREmulatorPage,
-} from "./playwright-fixture.ts";
-export type { EmulatorStats, XRPose } from "./types.ts";

package/simulator/src/playwright-fixture.ts

@@ -1,169 +0,0 @@
-/**
- * Playwright fixture and page wrapper for XR emulator testing.
- *
- * Usage:
- *   import { test, expect } from './fixtures.ts';
- *
- *   test('full roundtrip', async ({ xrPage, mockAgent }) => {
- *     await xrPage.goto('/');
- *     await mockAgent.waitForConnection();
- *     await xrPage.injectCameraFrame('./fixtures/desk.jpg');
- *     const frame = await mockAgent.waitForCameraFrame();
- *     expect(frame.payload.length).toBeGreaterThan(100);
- *   });
- */
-
-import { existsSync, readFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { test as base, type Page } from "@playwright/test";
-import { MockAgentServer } from "./mock-agent.ts";
-import type { EmulatorStats, XRPose } from "./types.ts";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const EMULATOR_DIST = resolve(__dirname, "../dist/emulator.js");
-
-// ── XREmulatorPage ────────────────────────────────────────────────────────
-
-export class XREmulatorPage {
-  constructor(readonly page: Page) {}
-
-  /** Inject the emulator script before the page loads. Call before page.goto(). */
-  async inject(): Promise<void> {
-    if (!existsSync(EMULATOR_DIST)) {
-      throw new Error(
-        `Emulator bundle not found at ${EMULATOR_DIST}. Run: cd eliza/plugins/plugin-xr/simulator && bun run build`,
-      );
-    }
-    await this.page.addInitScript({ path: EMULATOR_DIST });
-  }
-
-  /** Navigate and wait for the emulator to be ready. */
-  async goto(url: string): Promise<void> {
-    await this.page.goto(url);
-    // Wait for emulator to install (logs a console message)
-    await this.page.waitForFunction(
-      () => typeof window.__XREmulator !== "undefined",
-      {
-        timeout: 5000,
-      },
-    );
-  }
-
-  /** Set the emulated headset pose. */
-  async setPose(pose: Partial<XRPose>): Promise<void> {
-    await this.page.evaluate((p) => window.__XREmulator.setPose(p), pose);
-  }
-
-  /** Inject a camera frame from a local image file (JPEG or PNG). */
-  async injectCameraFrame(imagePath: string): Promise<void> {
-    const abs = resolve(imagePath);
-    const data = readFileSync(abs);
-    const mime = imagePath.endsWith(".png") ? "image/png" : "image/jpeg";
-    const dataUrl = `data:${mime};base64,${data.toString("base64")}`;
-    await this.page.evaluate(
-      (url) => window.__XREmulator.injectCameraFrame(url),
-      dataUrl,
-    );
-  }
-
-  /** Inject a camera frame from an inline data URL. */
-  async injectCameraDataUrl(dataUrl: string): Promise<void> {
-    await this.page.evaluate(
-      (url) => window.__XREmulator.injectCameraFrame(url),
-      dataUrl,
-    );
-  }
-
-  /** Send a synthetic audio chunk directly to the agent WebSocket. */
-  async sendAudioChunk(
-    base64: string,
-    sampleRate = 48000,
-    encoding = "webm-opus",
-  ): Promise<void> {
-    await this.page.evaluate(
-      ({ b64, sr, enc }) => {
-        if (!window.__xrTestHooks)
-          throw new Error("__xrTestHooks not available — is VITE_TEST=true?");
-        window.__xrTestHooks.sendAudioChunk(b64, sr, enc);
-      },
-      { b64: base64, sr: sampleRate, enc: encoding },
-    );
-  }
-
-  /** Get emulator stats. */
-  async getStats(): Promise<EmulatorStats> {
-    return this.page.evaluate(() => window.__XREmulator.getStats());
-  }
-
-  /** Get WebSocket readyState from test hooks. */
-  async getSocketState(): Promise<string> {
-    return this.page.evaluate(() => {
-      if (!window.__xrTestHooks) return "UNAVAILABLE";
-      return window.__xrTestHooks.getSocketState();
-    });
-  }
-
-  /** Wait for the page's status text to match a pattern. */
-  async waitForStatus(pattern: string | RegExp, timeout = 8000): Promise<void> {
-    await this.page
-      .locator("#status-text")
-      .filter({ hasText: pattern })
-      .waitFor({
-        state: "visible",
-        timeout,
-      });
-  }
-
-  /** Wait for agent response text to appear. */
-  async waitForAgentText(timeout = 10000): Promise<string> {
-    const el = this.page.locator("#agent-response");
-    await el.waitFor({ state: "visible", timeout });
-    await el.filter({ hasNotText: "" }).waitFor({ timeout });
-    return el.innerText();
-  }
-
-  /** Wait for transcript text to appear. */
-  async waitForTranscript(timeout = 10000): Promise<string> {
-    const el = this.page.locator("#transcript");
-    await el.waitFor({ state: "visible", timeout });
-    await el.filter({ hasNotText: "" }).waitFor({ timeout });
-    return el.innerText();
-  }
-
-  /** Force-disconnect the WebSocket (tests reconnect logic). */
-  async simulateDisconnect(): Promise<void> {
-    await this.page.evaluate(() => window.__XREmulator.simulateDisconnect());
-  }
-}
-
-// ── Playwright fixture extensions ─────────────────────────────────────────
-
-interface XRFixtures {
-  mockAgent: MockAgentServer;
-  xrPage: XREmulatorPage;
-}
-
-export const test = base.extend<XRFixtures>({
-  mockAgent: async (_fixtures, use, testInfo) => {
-    // Use a unique port per worker to allow parallel test runs
-    const port = 31338 + testInfo.workerIndex;
-    const server = new MockAgentServer({ port });
-    await server.start();
-    await use(server);
-    await server.stop();
-  },
-
-  xrPage: async ({ page }, use) => {
-    const xrp = new XREmulatorPage(page);
-    await xrp.inject();
-    await use(xrp);
-  },
-});
-
-export { expect } from "@playwright/test";
-
-// ── Node-side exports ─────────────────────────────────────────────────────
-
-export { MockAgentServer } from "./mock-agent.ts";
-export type { EmulatorStats, XRPose } from "./types.ts";

package/simulator/src/types.ts

@@ -1,51 +0,0 @@
-export interface Vec3 {
-  x: number;
-  y: number;
-  z: number;
-}
-
-export interface Quat {
-  x: number;
-  y: number;
-  z: number;
-  w: number;
-}
-
-export interface XRPose {
-  position: Vec3;
-  orientation: Quat;
-}
-
-export interface EmulatorStats {
-  sessionActive: boolean;
-  framesInjected: number;
-  cameraStreamActive: boolean;
-  wsConnected: boolean;
-}
-
-/** window.__XREmulator — set by emulator.ts, consumed by Playwright via page.evaluate() */
-export interface XREmulatorAPI {
-  setPose(pose: Partial<XRPose>): void;
-  injectCameraFrame(jpegDataUrl: string): Promise<void>;
-  getStats(): EmulatorStats;
-  /** Simulate device disconnection (closes WebSocket) */
-  simulateDisconnect(): void;
-  /** Simulate reconnect after a disconnect */
-  simulateReconnect(): void;
-}
-
-declare global {
-  interface Window {
-    __XREmulator: XREmulatorAPI;
-    /** Set by app-xr/src/main.ts in VITE_TEST mode */
-    __xrTestHooks: {
-      sendAudioChunk(
-        base64: string,
-        sampleRate: number,
-        encoding: string,
-      ): void;
-      getSocketState(): "CONNECTING" | "OPEN" | "CLOSING" | "CLOSED";
-      sendPing?(): void;
-    };
-  }
-}

package/simulator/tsconfig.json

@@ -1,13 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "ESNext",
-    "moduleResolution": "bundler",
-    "strict": true,
-    "noEmit": true,
-    "allowImportingTsExtensions": true,
-    "skipLibCheck": true,
-    "lib": ["ES2022", "DOM"]
-  },
-  "include": ["src/**/*.ts"]
-}

package/simulator/vite.config.ts

@@ -1,25 +0,0 @@
-import { resolve } from "node:path";
-import { defineConfig } from "vite";
-
-// Builds the browser-side emulator as a self-contained IIFE.
-// Output: dist/emulator.js — injected into pages by Playwright via addInitScript.
-export default defineConfig({
-  build: {
-    lib: {
-      entry: resolve(__dirname, "src/emulator.ts"),
-      name: "XREmulator",
-      fileName: "emulator",
-      formats: ["iife"],
-    },
-    outDir: "dist",
-    rollupOptions: {
-      output: {
-        inlineDynamicImports: true,
-        // Force the output filename to emulator.js (not emulator.iife.js)
-        entryFileNames: "[name].js",
-      },
-    },
-    minify: false, // keep readable for debugging
-    sourcemap: true,
-  },
-});