@elizaos/plugin-xr 2.0.3-beta.5 → 2.0.3-beta.7

package/{src/services/xr-session-service.ts → dist/services/xr-session-service.js}

@@ -1,203 +1,132 @@
-import type {
-  Content,
-  HandlerCallback,
-  IAgentRuntime,
-  Memory,
-  UUID,
-} from "@elizaos/core";
 import {
   ChannelType,
   createMessageMemory,
   ModelType,
-  Service,
+  Service
 } from "@elizaos/core";
-import { type WebSocket, WebSocketServer } from "ws";
+import { WebSocketServer } from "ws";
 import {
   decodeBinaryFrame,
-  encodeBinaryFrame,
-  type XRClientControl,
-  type XRDeviceType,
-  type XRPanelConfig,
-  type XRServerControl,
-  type XRTTSAudioHeader,
-} from "../protocol.ts";
-import { AudioPipeline } from "./audio-pipeline.ts";
-import { VisionPipeline } from "./vision-pipeline.ts";
-
-export const XR_SERVICE_TYPE = "xr-session";
-export const XR_WS_PORT_DEFAULT = 31338;
-export const XR_WS_PORT_ENV = "XR_WS_PORT";
-
-export interface XRConnection {
-  id: string;
-  ws: WebSocket;
-  deviceType: XRDeviceType;
-  entityId: UUID;
-  roomId: UUID;
-  connectedAt: Date;
-}
-
-export class XRSessionService extends Service {
-  static override serviceType = XR_SERVICE_TYPE;
-
-  readonly capabilityDescription =
-    "Streams audio and camera from XR headsets (Quest 3, XReal) to the agent and returns voice responses.";
-
-  private wss!: WebSocketServer;
-  private connections = new Map<string, XRConnection>();
-  private audioPipeline!: AudioPipeline;
-  private visionPipeline!: VisionPipeline;
-
-  static override async start(runtime: IAgentRuntime): Promise<Service> {
+  encodeBinaryFrame
+} from "../protocol.js";
+import { AudioPipeline } from "./audio-pipeline.js";
+import { VisionPipeline } from "./vision-pipeline.js";
+const XR_SERVICE_TYPE = "xr-session";
+const XR_WS_PORT_DEFAULT = 31338;
+const XR_WS_PORT_ENV = "XR_WS_PORT";
+class XRSessionService extends Service {
+  static serviceType = XR_SERVICE_TYPE;
+  capabilityDescription = "Streams audio and camera from XR headsets (Quest 3, XReal) to the agent and returns voice responses.";
+  wss;
+  connections = /* @__PURE__ */ new Map();
+  audioPipeline;
+  visionPipeline;
+  static async start(runtime) {
     const svc = new XRSessionService(runtime);
     await svc.initialize(runtime);
     return svc;
   }
-
-  private async initialize(runtime: IAgentRuntime): Promise<void> {
+  async initialize(runtime) {
     this.visionPipeline = new VisionPipeline();
     this.audioPipeline = new AudioPipeline(
       runtime,
-      (connectionId, transcript) =>
-        this.handleTranscript(connectionId, transcript),
+      (connectionId, transcript) => this.handleTranscript(connectionId, transcript)
     );
-
     const port = Number(
-      runtime.getSetting(XR_WS_PORT_ENV) ?? XR_WS_PORT_DEFAULT,
+      runtime.getSetting(XR_WS_PORT_ENV) ?? XR_WS_PORT_DEFAULT
     );
     this.wss = new WebSocketServer({ port });
-
-    this.wss.on("connection", (ws: WebSocket) => this.onConnect(runtime, ws));
-    this.wss.on("error", (err: Error) =>
-      console.error("[plugin-xr] WebSocket server error:", err),
+    this.wss.on("connection", (ws) => this.onConnect(runtime, ws));
+    this.wss.on(
+      "error",
+      (err) => console.error("[plugin-xr] WebSocket server error:", err)
     );
-
     console.info(
-      `[plugin-xr] WebSocket server listening on ws://localhost:${port}`,
+      `[plugin-xr] WebSocket server listening on ws://localhost:${port}`
     );
   }
-
-  override async stop(): Promise<void> {
+  async stop() {
     for (const conn of this.connections.values()) {
       this.audioPipeline.clear(conn.id);
       this.visionPipeline.clear(conn.id);
       conn.ws.close();
     }
     this.connections.clear();
-    await new Promise<void>((resolve) => this.wss.close(() => resolve()));
+    await new Promise((resolve) => this.wss.close(() => resolve()));
   }
-
   // ── Public accessors used by provider / action ──────────────────────────
-
-  getConnections(): XRConnection[] {
+  getConnections() {
     return [...this.connections.values()];
   }
-
-  hasActiveConnections(): boolean {
+  hasActiveConnections() {
     return this.connections.size > 0;
   }
-
-  getVisionPipeline(): VisionPipeline {
+  getVisionPipeline() {
     return this.visionPipeline;
   }
-
-  sendText(connectionId: string, text: string): void {
+  sendText(connectionId, text) {
     const conn = this.connections.get(connectionId);
     if (!conn || conn.ws.readyState !== conn.ws.OPEN) return;
     conn.ws.send(JSON.stringify({ type: "agent_text", text }));
   }
-
-  sendControl(connectionId: string, msg: XRServerControl): void {
+  sendControl(connectionId, msg) {
     const conn = this.connections.get(connectionId);
     if (!conn || conn.ws.readyState !== conn.ws.OPEN) return;
     conn.ws.send(JSON.stringify(msg));
   }
-
-  broadcastControl(msg: XRServerControl): void {
+  broadcastControl(msg) {
     for (const conn of this.connections.values()) {
       if (conn.ws.readyState === conn.ws.OPEN) {
         conn.ws.send(JSON.stringify(msg));
       }
     }
   }
-
-  openView(
-    connectionId: string,
-    viewId: string,
-    agentBaseUrl: string,
-    config?: XRPanelConfig,
-  ): void {
+  openView(connectionId, viewId, agentBaseUrl, config) {
     this.sendControl(connectionId, {
       type: "view_open",
       viewId,
       agentBaseUrl,
-      config,
+      config
     });
   }
-
-  closeView(connectionId: string, viewId: string): void {
+  closeView(connectionId, viewId) {
     this.sendControl(connectionId, { type: "view_close", viewId });
   }
-
-  switchView(connectionId: string, viewId: string): void {
+  switchView(connectionId, viewId) {
     this.sendControl(connectionId, { type: "view_switch", viewId });
   }
-
-  resizeView(
-    connectionId: string,
-    viewId: string,
-    config: XRPanelConfig,
-  ): void {
+  resizeView(connectionId, viewId, config) {
     this.sendControl(connectionId, { type: "view_resize", viewId, config });
   }
-
-  sendViewsCatalog(
-    connectionId: string,
-    views: Array<{
-      id: string;
-      label: string;
-      icon?: string;
-      description?: string;
-    }>,
-  ): void {
+  sendViewsCatalog(connectionId, views) {
     this.sendControl(connectionId, { type: "views_catalog", views });
   }
-
-  sendAudio(connectionId: string, audio: Buffer, sampleRate = 24000): void {
+  sendAudio(connectionId, audio, sampleRate = 24e3) {
     const conn = this.connections.get(connectionId);
     if (!conn || conn.ws.readyState !== conn.ws.OPEN) return;
-    const header: XRTTSAudioHeader = {
+    const header = {
       type: "tts_audio",
       sampleRate,
       channels: 1,
-      encoding: "mp3",
+      encoding: "mp3"
     };
     conn.ws.send(encodeBinaryFrame(header, audio), { binary: true });
   }
-
   // ── WebSocket connection lifecycle ──────────────────────────────────────
-
-  private onConnect(runtime: IAgentRuntime, ws: WebSocket): void {
+  onConnect(runtime, ws) {
     const connId = crypto.randomUUID();
-
-    ws.on("message", (data: unknown, isBinary: boolean) => {
+    ws.on("message", (data, isBinary) => {
       try {
         if (isBinary) {
-          this.handleBinaryMessage(connId, data as Buffer);
+          this.handleBinaryMessage(connId, data);
         } else {
-          const raw =
-            typeof data === "string"
-              ? data
-              : Buffer.isBuffer(data)
-                ? data.toString("utf8")
-                : String(data);
+          const raw = typeof data === "string" ? data : Buffer.isBuffer(data) ? data.toString("utf8") : String(data);
           this.handleTextMessage(runtime, connId, ws, raw);
         }
       } catch (err) {
         console.error("[plugin-xr] message handler error:", err);
       }
     });
-
     ws.on("close", () => {
       this.audioPipeline.flush(connId);
       this.audioPipeline.clear(connId);
@@ -205,30 +134,23 @@ export class XRSessionService extends Service {
       this.connections.delete(connId);
       console.info(`[plugin-xr] device disconnected: ${connId}`);
     });
-
-    ws.on("error", (err: Error) =>
-      console.error(`[plugin-xr] ws error on ${connId}:`, err),
+    ws.on(
+      "error",
+      (err) => console.error(`[plugin-xr] ws error on ${connId}:`, err)
     );
   }
-
-  private handleTextMessage(
-    runtime: IAgentRuntime,
-    connId: string,
-    ws: WebSocket,
-    raw: string,
-  ): void {
-    const msg = JSON.parse(raw) as XRClientControl;
-
+  handleTextMessage(runtime, connId, ws, raw) {
+    const msg = JSON.parse(raw);
     if (msg.type === "hello") {
-      const entityId = crypto.randomUUID() as UUID;
-      const roomId = crypto.randomUUID() as UUID;
-      const conn: XRConnection = {
+      const entityId = crypto.randomUUID();
+      const roomId = crypto.randomUUID();
+      const conn = {
         id: connId,
         ws,
         deviceType: msg.deviceType,
         entityId,
         roomId,
-        connectedAt: new Date(),
+        connectedAt: /* @__PURE__ */ new Date()
       };
       this.connections.set(connId, conn);
       ws.send(JSON.stringify({ type: "ready", sessionId: connId }));
@@ -236,78 +158,58 @@ export class XRSessionService extends Service {
       console.info(`[plugin-xr] ${msg.deviceType} connected: ${connId}`);
       return;
     }
-
     if (msg.type === "ping") {
       ws.send(JSON.stringify({ type: "pong" }));
       return;
     }
-
     if (msg.type === "view_ready") {
       console.info(`[plugin-xr] view ready on ${connId}: ${msg.viewId}`);
       return;
     }
-
     if (msg.type === "view_closed") {
       console.info(`[plugin-xr] view closed on ${connId}: ${msg.viewId}`);
       return;
     }
-
     if (msg.type === "view_event") {
       console.info(
         `[plugin-xr] view event on ${connId}:`,
         msg.viewId,
-        msg.event,
+        msg.event
       );
       return;
     }
   }
-
-  private handleBinaryMessage(connId: string, data: Buffer): void {
+  handleBinaryMessage(connId, data) {
     const conn = this.connections.get(connId);
     if (!conn) return;
-
     const { header, payload } = decodeBinaryFrame(data);
-
     if (header.type === "audio") {
       this.audioPipeline.push(connId, header, payload);
       return;
     }
-
     if (header.type === "frame") {
       this.visionPipeline.storeFrame(connId, header, payload);
       return;
     }
   }
-
   // ── Message routing ─────────────────────────────────────────────────────
-
-  private async handleTranscript(
-    connectionId: string,
-    transcript: string,
-  ): Promise<void> {
+  async handleTranscript(connectionId, transcript) {
     const conn = this.connections.get(connectionId);
     if (!conn) return;
     const runtime = this.runtime;
-
-    // Echo transcript back so the UI can show it
     conn.ws.send(
-      JSON.stringify({ type: "transcript", text: transcript, final: true }),
+      JSON.stringify({ type: "transcript", text: transcript, final: true })
     );
-
-    // Attach latest camera frame as image attachment if available
     const latestFrame = this.visionPipeline.getLatestFrame(connectionId);
-    const attachments: Content["attachments"] = latestFrame
-      ? [
-          {
-            id: crypto.randomUUID(),
-            url: `data:image/${latestFrame.header.format};base64,${latestFrame.data.toString("base64")}`,
-            title: "XR camera frame",
-            contentType: "image",
-            source: "xr-camera",
-          },
-        ]
-      : [];
-
+    const attachments = latestFrame ? [
+      {
+        id: crypto.randomUUID(),
+        url: `data:image/${latestFrame.header.format};base64,${latestFrame.data.toString("base64")}`,
+        title: "XR camera frame",
+        contentType: "image",
+        source: "xr-camera"
+      }
+    ] : [];
     const memory = createMessageMemory({
       entityId: conn.entityId,
       agentId: runtime.agentId,
@@ -315,69 +217,51 @@ export class XRSessionService extends Service {
       content: {
         text: transcript,
         source: `xr-${conn.deviceType}`,
-        attachments,
-      },
+        attachments
+      }
     });
-
     await runtime.createMemory(memory, "messages");
-
-    const callback: HandlerCallback = async (
-      response: Content,
-    ): Promise<Memory[]> => {
+    const callback = async (response) => {
       const text = response.text?.trim() ?? "";
       if (text.length === 0) return [];
-
-      // Send text response immediately
       conn.ws.send(JSON.stringify({ type: "agent_text", text }));
-
-      // Generate TTS and send audio
       try {
         const audio = await runtime.useModel(ModelType.TEXT_TO_SPEECH, text);
-        const audioBuf = Buffer.isBuffer(audio)
-          ? audio
-          : Buffer.from(audio as ArrayBuffer);
+        const audioBuf = Buffer.isBuffer(audio) ? audio : Buffer.from(audio);
         this.sendAudio(connectionId, audioBuf);
       } catch (err) {
         console.error("[plugin-xr] TTS error:", err);
       }
-
-      // Persist agent response as memory
       const responseMemory = createMessageMemory({
         entityId: runtime.agentId,
         agentId: runtime.agentId,
         roomId: conn.roomId,
-        content: { text, source: `xr-agent`, inReplyTo: memory.id },
+        content: { text, source: `xr-agent`, inReplyTo: memory.id }
       });
       await runtime.createMemory(responseMemory, "messages");
       return [responseMemory];
     };
-
     if (runtime.messageService) {
       await runtime.messageService.handleMessage(runtime, memory, callback);
     }
   }
-
   // ── Entity / room bootstrap ─────────────────────────────────────────────
-
-  private async ensureEntities(
-    runtime: IAgentRuntime,
-    conn: XRConnection,
-  ): Promise<void> {
+  async ensureEntities(runtime, conn) {
     try {
       await runtime.createEntity({
         id: conn.entityId,
         agentId: runtime.agentId,
         names: [`XR-${conn.deviceType}`],
-        metadata: { source: `xr-${conn.deviceType}` },
+        metadata: { source: `xr-${conn.deviceType}` }
       });
       await runtime.createRoom({
         id: conn.roomId,
         name: `XR session (${conn.deviceType} ${conn.id.slice(0, 8)})`,
         source: "xr",
         type: ChannelType.GROUP,
-        worldId: undefined,
-        channelId: undefined,
-        messageServerId: undefined,
+        worldId: void 0,
+        channelId: void 0,
+        messageServerId: void 0
       });
       await runtime.addParticipant(conn.entityId, conn.roomId);
       await runtime.addParticipant(runtime.agentId, conn.roomId);
@@ -386,3 +270,10 @@ export class XRSessionService extends Service {
     }
   }
 }
+export {
+  XRSessionService,
+  XR_SERVICE_TYPE,
+  XR_WS_PORT_DEFAULT,
+  XR_WS_PORT_ENV
+};
+//# sourceMappingURL=xr-session-service.js.map

package/dist/services/xr-session-service.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/services/xr-session-service.ts"],"sourcesContent":["import type {\n  Content,\n  HandlerCallback,\n  IAgentRuntime,\n  Memory,\n  UUID,\n} from \"@elizaos/core\";\nimport {\n  ChannelType,\n  createMessageMemory,\n  ModelType,\n  Service,\n} from \"@elizaos/core\";\nimport { type WebSocket, WebSocketServer } from \"ws\";\nimport {\n  decodeBinaryFrame,\n  encodeBinaryFrame,\n  type XRClientControl,\n  type XRDeviceType,\n  type XRPanelConfig,\n  type XRServerControl,\n  type XRTTSAudioHeader,\n} from \"../protocol.js\";\nimport { AudioPipeline } from \"./audio-pipeline.js\";\nimport { VisionPipeline } from \"./vision-pipeline.js\";\n\nexport const XR_SERVICE_TYPE = \"xr-session\";\nexport const XR_WS_PORT_DEFAULT = 31338;\nexport const XR_WS_PORT_ENV = \"XR_WS_PORT\";\n\nexport interface XRConnection {\n  id: string;\n  ws: WebSocket;\n  deviceType: XRDeviceType;\n  entityId: UUID;\n  roomId: UUID;\n  connectedAt: Date;\n}\n\nexport class XRSessionService extends Service {\n  static override serviceType = XR_SERVICE_TYPE;\n\n  readonly capabilityDescription =\n    \"Streams audio and camera from XR headsets (Quest 3, XReal) to the agent and returns voice responses.\";\n\n  private wss!: WebSocketServer;\n  private connections = new Map<string, XRConnection>();\n  private audioPipeline!: AudioPipeline;\n  private visionPipeline!: VisionPipeline;\n\n  static override async start(runtime: IAgentRuntime): Promise<Service> {\n    const svc = new XRSessionService(runtime);\n    await svc.initialize(runtime);\n    return svc;\n  }\n\n  private async initialize(runtime: IAgentRuntime): Promise<void> {\n    this.visionPipeline = new VisionPipeline();\n    this.audioPipeline = new AudioPipeline(\n      runtime,\n      (connectionId, transcript) =>\n        this.handleTranscript(connectionId, transcript),\n    );\n\n    const port = Number(\n      runtime.getSetting(XR_WS_PORT_ENV) ?? XR_WS_PORT_DEFAULT,\n    );\n    this.wss = new WebSocketServer({ port });\n\n    this.wss.on(\"connection\", (ws: WebSocket) => this.onConnect(runtime, ws));\n    this.wss.on(\"error\", (err: Error) =>\n      console.error(\"[plugin-xr] WebSocket server error:\", err),\n    );\n\n    console.info(\n      `[plugin-xr] WebSocket server listening on ws://localhost:${port}`,\n    );\n  }\n\n  override async stop(): Promise<void> {\n    for (const conn of this.connections.values()) {\n      this.audioPipeline.clear(conn.id);\n      this.visionPipeline.clear(conn.id);\n      conn.ws.close();\n    }\n    this.connections.clear();\n    await new Promise<void>((resolve) => this.wss.close(() => resolve()));\n  }\n\n  // ── Public accessors used by provider / action ──────────────────────────\n\n  getConnections(): XRConnection[] {\n    return [...this.connections.values()];\n  }\n\n  hasActiveConnections(): boolean {\n    return this.connections.size > 0;\n  }\n\n  getVisionPipeline(): VisionPipeline {\n    return this.visionPipeline;\n  }\n\n  sendText(connectionId: string, text: string): void {\n    const conn = this.connections.get(connectionId);\n    if (!conn || conn.ws.readyState !== conn.ws.OPEN) return;\n    conn.ws.send(JSON.stringify({ type: \"agent_text\", text }));\n  }\n\n  sendControl(connectionId: string, msg: XRServerControl): void {\n    const conn = this.connections.get(connectionId);\n    if (!conn || conn.ws.readyState !== conn.ws.OPEN) return;\n    conn.ws.send(JSON.stringify(msg));\n  }\n\n  broadcastControl(msg: XRServerControl): void {\n    for (const conn of this.connections.values()) {\n      if (conn.ws.readyState === conn.ws.OPEN) {\n        conn.ws.send(JSON.stringify(msg));\n      }\n    }\n  }\n\n  openView(\n    connectionId: string,\n    viewId: string,\n    agentBaseUrl: string,\n    config?: XRPanelConfig,\n  ): void {\n    this.sendControl(connectionId, {\n      type: \"view_open\",\n      viewId,\n      agentBaseUrl,\n      config,\n    });\n  }\n\n  closeView(connectionId: string, viewId: string): void {\n    this.sendControl(connectionId, { type: \"view_close\", viewId });\n  }\n\n  switchView(connectionId: string, viewId: string): void {\n    this.sendControl(connectionId, { type: \"view_switch\", viewId });\n  }\n\n  resizeView(\n    connectionId: string,\n    viewId: string,\n    config: XRPanelConfig,\n  ): void {\n    this.sendControl(connectionId, { type: \"view_resize\", viewId, config });\n  }\n\n  sendViewsCatalog(\n    connectionId: string,\n    views: Array<{\n      id: string;\n      label: string;\n      icon?: string;\n      description?: string;\n    }>,\n  ): void {\n    this.sendControl(connectionId, { type: \"views_catalog\", views });\n  }\n\n  sendAudio(connectionId: string, audio: Buffer, sampleRate = 24000): void {\n    const conn = this.connections.get(connectionId);\n    if (!conn || conn.ws.readyState !== conn.ws.OPEN) return;\n    const header: XRTTSAudioHeader = {\n      type: \"tts_audio\",\n      sampleRate,\n      channels: 1,\n      encoding: \"mp3\",\n    };\n    conn.ws.send(encodeBinaryFrame(header, audio), { binary: true });\n  }\n\n  // ── WebSocket connection lifecycle ──────────────────────────────────────\n\n  private onConnect(runtime: IAgentRuntime, ws: WebSocket): void {\n    const connId = crypto.randomUUID();\n\n    ws.on(\"message\", (data: unknown, isBinary: boolean) => {\n      try {\n        if (isBinary) {\n          this.handleBinaryMessage(connId, data as Buffer);\n        } else {\n          const raw =\n            typeof data === \"string\"\n              ? data\n              : Buffer.isBuffer(data)\n                ? data.toString(\"utf8\")\n                : String(data);\n          this.handleTextMessage(runtime, connId, ws, raw);\n        }\n      } catch (err) {\n        console.error(\"[plugin-xr] message handler error:\", err);\n      }\n    });\n\n    ws.on(\"close\", () => {\n      this.audioPipeline.flush(connId);\n      this.audioPipeline.clear(connId);\n      this.visionPipeline.clear(connId);\n      this.connections.delete(connId);\n      console.info(`[plugin-xr] device disconnected: ${connId}`);\n    });\n\n    ws.on(\"error\", (err: Error) =>\n      console.error(`[plugin-xr] ws error on ${connId}:`, err),\n    );\n  }\n\n  private handleTextMessage(\n    runtime: IAgentRuntime,\n    connId: string,\n    ws: WebSocket,\n    raw: string,\n  ): void {\n    const msg = JSON.parse(raw) as XRClientControl;\n\n    if (msg.type === \"hello\") {\n      const entityId = crypto.randomUUID() as UUID;\n      const roomId = crypto.randomUUID() as UUID;\n      const conn: XRConnection = {\n        id: connId,\n        ws,\n        deviceType: msg.deviceType,\n        entityId,\n        roomId,\n        connectedAt: new Date(),\n      };\n      this.connections.set(connId, conn);\n      ws.send(JSON.stringify({ type: \"ready\", sessionId: connId }));\n      void this.ensureEntities(runtime, conn);\n      console.info(`[plugin-xr] ${msg.deviceType} connected: ${connId}`);\n      return;\n    }\n\n    if (msg.type === \"ping\") {\n      ws.send(JSON.stringify({ type: \"pong\" }));\n      return;\n    }\n\n    if (msg.type === \"view_ready\") {\n      console.info(`[plugin-xr] view ready on ${connId}: ${msg.viewId}`);\n      return;\n    }\n\n    if (msg.type === \"view_closed\") {\n      console.info(`[plugin-xr] view closed on ${connId}: ${msg.viewId}`);\n      return;\n    }\n\n    if (msg.type === \"view_event\") {\n      console.info(\n        `[plugin-xr] view event on ${connId}:`,\n        msg.viewId,\n        msg.event,\n      );\n      return;\n    }\n  }\n\n  private handleBinaryMessage(connId: string, data: Buffer): void {\n    const conn = this.connections.get(connId);\n    if (!conn) return;\n\n    const { header, payload } = decodeBinaryFrame(data);\n\n    if (header.type === \"audio\") {\n      this.audioPipeline.push(connId, header, payload);\n      return;\n    }\n\n    if (header.type === \"frame\") {\n      this.visionPipeline.storeFrame(connId, header, payload);\n      return;\n    }\n  }\n\n  // ── Message routing ─────────────────────────────────────────────────────\n\n  private async handleTranscript(\n    connectionId: string,\n    transcript: string,\n  ): Promise<void> {\n    const conn = this.connections.get(connectionId);\n    if (!conn) return;\n    const runtime = this.runtime;\n\n    // Echo transcript back so the UI can show it\n    conn.ws.send(\n      JSON.stringify({ type: \"transcript\", text: transcript, final: true }),\n    );\n\n    // Attach latest camera frame as image attachment if available\n    const latestFrame = this.visionPipeline.getLatestFrame(connectionId);\n    const attachments: Content[\"attachments\"] = latestFrame\n      ? [\n          {\n            id: crypto.randomUUID(),\n            url: `data:image/${latestFrame.header.format};base64,${latestFrame.data.toString(\"base64\")}`,\n            title: \"XR camera frame\",\n            contentType: \"image\",\n            source: \"xr-camera\",\n          },\n        ]\n      : [];\n\n    const memory = createMessageMemory({\n      entityId: conn.entityId,\n      agentId: runtime.agentId,\n      roomId: conn.roomId,\n      content: {\n        text: transcript,\n        source: `xr-${conn.deviceType}`,\n        attachments,\n      },\n    });\n\n    await runtime.createMemory(memory, \"messages\");\n\n    const callback: HandlerCallback = async (\n      response: Content,\n    ): Promise<Memory[]> => {\n      const text = response.text?.trim() ?? \"\";\n      if (text.length === 0) return [];\n\n      // Send text response immediately\n      conn.ws.send(JSON.stringify({ type: \"agent_text\", text }));\n\n      // Generate TTS and send audio\n      try {\n        const audio = await runtime.useModel(ModelType.TEXT_TO_SPEECH, text);\n        const audioBuf = Buffer.isBuffer(audio)\n          ? audio\n          : Buffer.from(audio as ArrayBuffer);\n        this.sendAudio(connectionId, audioBuf);\n      } catch (err) {\n        console.error(\"[plugin-xr] TTS error:\", err);\n      }\n\n      // Persist agent response as memory\n      const responseMemory = createMessageMemory({\n        entityId: runtime.agentId,\n        agentId: runtime.agentId,\n        roomId: conn.roomId,\n        content: { text, source: `xr-agent`, inReplyTo: memory.id },\n      });\n      await runtime.createMemory(responseMemory, \"messages\");\n      return [responseMemory];\n    };\n\n    if (runtime.messageService) {\n      await runtime.messageService.handleMessage(runtime, memory, callback);\n    }\n  }\n\n  // ── Entity / room bootstrap ─────────────────────────────────────────────\n\n  private async ensureEntities(\n    runtime: IAgentRuntime,\n    conn: XRConnection,\n  ): Promise<void> {\n    try {\n      await runtime.createEntity({\n        id: conn.entityId,\n        agentId: runtime.agentId,\n        names: [`XR-${conn.deviceType}`],\n        metadata: { source: `xr-${conn.deviceType}` },\n      });\n      await runtime.createRoom({\n        id: conn.roomId,\n        name: `XR session (${conn.deviceType} ${conn.id.slice(0, 8)})`,\n        source: \"xr\",\n        type: ChannelType.GROUP,\n        worldId: undefined,\n        channelId: undefined,\n        messageServerId: undefined,\n      });\n      await runtime.addParticipant(conn.entityId, conn.roomId);\n      await runtime.addParticipant(runtime.agentId, conn.roomId);\n    } catch (err) {\n      console.error(\"[plugin-xr] entity setup error:\", err);\n    }\n  }\n}\n"],"mappings":"AAOA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAyB,uBAAuB;AAChD;AAAA,EACE;AAAA,EACA;AAAA,OAMK;AACP,SAAS,qBAAqB;AAC9B,SAAS,sBAAsB;AAExB,MAAM,kBAAkB;AACxB,MAAM,qBAAqB;AAC3B,MAAM,iBAAiB;AAWvB,MAAM,yBAAyB,QAAQ;AAAA,EAC5C,OAAgB,cAAc;AAAA,EAErB,wBACP;AAAA,EAEM;AAAA,EACA,cAAc,oBAAI,IAA0B;AAAA,EAC5C;AAAA,EACA;AAAA,EAER,aAAsB,MAAM,SAA0C;AACpE,UAAM,MAAM,IAAI,iBAAiB,OAAO;AACxC,UAAM,IAAI,WAAW,OAAO;AAC5B,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,WAAW,SAAuC;AAC9D,SAAK,iBAAiB,IAAI,eAAe;AACzC,SAAK,gBAAgB,IAAI;AAAA,MACvB;AAAA,MACA,CAAC,cAAc,eACb,KAAK,iBAAiB,cAAc,UAAU;AAAA,IAClD;AAEA,UAAM,OAAO;AAAA,MACX,QAAQ,WAAW,cAAc,KAAK;AAAA,IACxC;AACA,SAAK,MAAM,IAAI,gBAAgB,EAAE,KAAK,CAAC;AAEvC,SAAK,IAAI,GAAG,cAAc,CAAC,OAAkB,KAAK,UAAU,SAAS,EAAE,CAAC;AACxE,SAAK,IAAI;AAAA,MAAG;AAAA,MAAS,CAAC,QACpB,QAAQ,MAAM,uCAAuC,GAAG;AAAA,IAC1D;AAEA,YAAQ;AAAA,MACN,4DAA4D,IAAI;AAAA,IAClE;AAAA,EACF;AAAA,EAEA,MAAe,OAAsB;AACnC,eAAW,QAAQ,KAAK,YAAY,OAAO,GAAG;AAC5C,WAAK,cAAc,MAAM,KAAK,EAAE;AAChC,WAAK,eAAe,MAAM,KAAK,EAAE;AACjC,WAAK,GAAG,MAAM;AAAA,IAChB;AACA,SAAK,YAAY,MAAM;AACvB,UAAM,IAAI,QAAc,CAAC,YAAY,KAAK,IAAI,MAAM,MAAM,QAAQ,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAIA,iBAAiC;AAC/B,WAAO,CAAC,GAAG,KAAK,YAAY,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,uBAAgC;AAC9B,WAAO,KAAK,YAAY,OAAO;AAAA,EACjC;AAAA,EAEA,oBAAoC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,SAAS,cAAsB,MAAoB;AACjD,UAAM,OAAO,KAAK,YAAY,IAAI,YAAY;AAC9C,QAAI,CAAC,QAAQ,KAAK,GAAG,eAAe,KAAK,GAAG,KAAM;AAClD,SAAK,GAAG,KAAK,KAAK,UAAU,EAAE,MAAM,cAAc,KAAK,CAAC,CAAC;AAAA,EAC3D;AAAA,EAEA,YAAY,cAAsB,KAA4B;AAC5D,UAAM,OAAO,KAAK,YAAY,IAAI,YAAY;AAC9C,QAAI,CAAC,QAAQ,KAAK,GAAG,eAAe,KAAK,GAAG,KAAM;AAClD,SAAK,GAAG,KAAK,KAAK,UAAU,GAAG,CAAC;AAAA,EAClC;AAAA,EAEA,iBAAiB,KAA4B;AAC3C,eAAW,QAAQ,KAAK,YAAY,OAAO,GAAG;AAC5C,UAAI,KAAK,GAAG,eAAe,KAAK,GAAG,MAAM;AACvC,aAAK,GAAG,KAAK,KAAK,UAAU,GAAG,CAAC;AAAA,MAClC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,SACE,cACA,QACA,cACA,QACM;AACN,SAAK,YAAY,cAAc;AAAA,MAC7B,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,UAAU,cAAsB,QAAsB;AACpD,SAAK,YAAY,cAAc,EAAE,MAAM,cAAc,OAAO,CAAC;AAAA,EAC/D;AAAA,EAEA,WAAW,cAAsB,QAAsB;AACrD,SAAK,YAAY,cAAc,EAAE,MAAM,eAAe,OAAO,CAAC;AAAA,EAChE;AAAA,EAEA,WACE,cACA,QACA,QACM;AACN,SAAK,YAAY,cAAc,EAAE,MAAM,eAAe,QAAQ,OAAO,CAAC;AAAA,EACxE;AAAA,EAEA,iBACE,cACA,OAMM;AACN,SAAK,YAAY,cAAc,EAAE,MAAM,iBAAiB,MAAM,CAAC;AAAA,EACjE;AAAA,EAEA,UAAU,cAAsB,OAAe,aAAa,MAAa;AACvE,UAAM,OAAO,KAAK,YAAY,IAAI,YAAY;AAC9C,QAAI,CAAC,QAAQ,KAAK,GAAG,eAAe,KAAK,GAAG,KAAM;AAClD,UAAM,SAA2B;AAAA,MAC/B,MAAM;AAAA,MACN;AAAA,MACA,UAAU;AAAA,MACV,UAAU;AAAA,IACZ;AACA,SAAK,GAAG,KAAK,kBAAkB,QAAQ,KAAK,GAAG,EAAE,QAAQ,KAAK,CAAC;AAAA,EACjE;AAAA;AAAA,EAIQ,UAAU,SAAwB,IAAqB;AAC7D,UAAM,SAAS,OAAO,WAAW;AAEjC,OAAG,GAAG,WAAW,CAAC,MAAe,aAAsB;AACrD,UAAI;AACF,YAAI,UAAU;AACZ,eAAK,oBAAoB,QAAQ,IAAc;AAAA,QACjD,OAAO;AACL,gBAAM,MACJ,OAAO,SAAS,WACZ,OACA,OAAO,SAAS,IAAI,IAClB,KAAK,SAAS,MAAM,IACpB,OAAO,IAAI;AACnB,eAAK,kBAAkB,SAAS,QAAQ,IAAI,GAAG;AAAA,QACjD;AAAA,MACF,SAAS,KAAK;AACZ,gBAAQ,MAAM,sCAAsC,GAAG;AAAA,MACzD;AAAA,IACF,CAAC;AAED,OAAG,GAAG,SAAS,MAAM;AACnB,WAAK,cAAc,MAAM,MAAM;AAC/B,WAAK,cAAc,MAAM,MAAM;AAC/B,WAAK,eAAe,MAAM,MAAM;AAChC,WAAK,YAAY,OAAO,MAAM;AAC9B,cAAQ,KAAK,oCAAoC,MAAM,EAAE;AAAA,IAC3D,CAAC;AAED,OAAG;AAAA,MAAG;AAAA,MAAS,CAAC,QACd,QAAQ,MAAM,2BAA2B,MAAM,KAAK,GAAG;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,kBACN,SACA,QACA,IACA,KACM;AACN,UAAM,MAAM,KAAK,MAAM,GAAG;AAE1B,QAAI,IAAI,SAAS,SAAS;AACxB,YAAM,WAAW,OAAO,WAAW;AACnC,YAAM,SAAS,OAAO,WAAW;AACjC,YAAM,OAAqB;AAAA,QACzB,IAAI;AAAA,QACJ;AAAA,QACA,YAAY,IAAI;AAAA,QAChB;AAAA,QACA;AAAA,QACA,aAAa,oBAAI,KAAK;AAAA,MACxB;AACA,WAAK,YAAY,IAAI,QAAQ,IAAI;AACjC,SAAG,KAAK,KAAK,UAAU,EAAE,MAAM,SAAS,WAAW,OAAO,CAAC,CAAC;AAC5D,WAAK,KAAK,eAAe,SAAS,IAAI;AACtC,cAAQ,KAAK,eAAe,IAAI,UAAU,eAAe,MAAM,EAAE;AACjE;AAAA,IACF;AAEA,QAAI,IAAI,SAAS,QAAQ;AACvB,SAAG,KAAK,KAAK,UAAU,EAAE,MAAM,OAAO,CAAC,CAAC;AACxC;AAAA,IACF;AAEA,QAAI,IAAI,SAAS,cAAc;AAC7B,cAAQ,KAAK,6BAA6B,MAAM,KAAK,IAAI,MAAM,EAAE;AACjE;AAAA,IACF;AAEA,QAAI,IAAI,SAAS,eAAe;AAC9B,cAAQ,KAAK,8BAA8B,MAAM,KAAK,IAAI,MAAM,EAAE;AAClE;AAAA,IACF;AAEA,QAAI,IAAI,SAAS,cAAc;AAC7B,cAAQ;AAAA,QACN,6BAA6B,MAAM;AAAA,QACnC,IAAI;AAAA,QACJ,IAAI;AAAA,MACN;AACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,QAAgB,MAAoB;AAC9D,UAAM,OAAO,KAAK,YAAY,IAAI,MAAM;AACxC,QAAI,CAAC,KAAM;AAEX,UAAM,EAAE,QAAQ,QAAQ,IAAI,kBAAkB,IAAI;AAElD,QAAI,OAAO,SAAS,SAAS;AAC3B,WAAK,cAAc,KAAK,QAAQ,QAAQ,OAAO;AAC/C;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,SAAS;AAC3B,WAAK,eAAe,WAAW,QAAQ,QAAQ,OAAO;AACtD;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAIA,MAAc,iBACZ,cACA,YACe;AACf,UAAM,OAAO,KAAK,YAAY,IAAI,YAAY;AAC9C,QAAI,CAAC,KAAM;AACX,UAAM,UAAU,KAAK;AAGrB,SAAK,GAAG;AAAA,MACN,KAAK,UAAU,EAAE,MAAM,cAAc,MAAM,YAAY,OAAO,KAAK,CAAC;AAAA,IACtE;AAGA,UAAM,cAAc,KAAK,eAAe,eAAe,YAAY;AACnE,UAAM,cAAsC,cACxC;AAAA,MACE;AAAA,QACE,IAAI,OAAO,WAAW;AAAA,QACtB,KAAK,cAAc,YAAY,OAAO,MAAM,WAAW,YAAY,KAAK,SAAS,QAAQ,CAAC;AAAA,QAC1F,OAAO;AAAA,QACP,aAAa;AAAA,QACb,QAAQ;AAAA,MACV;AAAA,IACF,IACA,CAAC;AAEL,UAAM,SAAS,oBAAoB;AAAA,MACjC,UAAU,KAAK;AAAA,MACf,SAAS,QAAQ;AAAA,MACjB,QAAQ,KAAK;AAAA,MACb,SAAS;AAAA,QACP,MAAM;AAAA,QACN,QAAQ,MAAM,KAAK,UAAU;AAAA,QAC7B;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,QAAQ,aAAa,QAAQ,UAAU;AAE7C,UAAM,WAA4B,OAChC,aACsB;AACtB,YAAM,OAAO,SAAS,MAAM,KAAK,KAAK;AACtC,UAAI,KAAK,WAAW,EAAG,QAAO,CAAC;AAG/B,WAAK,GAAG,KAAK,KAAK,UAAU,EAAE,MAAM,cAAc,KAAK,CAAC,CAAC;AAGzD,UAAI;AACF,cAAM,QAAQ,MAAM,QAAQ,SAAS,UAAU,gBAAgB,IAAI;AACnE,cAAM,WAAW,OAAO,SAAS,KAAK,IAClC,QACA,OAAO,KAAK,KAAoB;AACpC,aAAK,UAAU,cAAc,QAAQ;AAAA,MACvC,SAAS,KAAK;AACZ,gBAAQ,MAAM,0BAA0B,GAAG;AAAA,MAC7C;AAGA,YAAM,iBAAiB,oBAAoB;AAAA,QACzC,UAAU,QAAQ;AAAA,QAClB,SAAS,QAAQ;AAAA,QACjB,QAAQ,KAAK;AAAA,QACb,SAAS,EAAE,MAAM,QAAQ,YAAY,WAAW,OAAO,GAAG;AAAA,MAC5D,CAAC;AACD,YAAM,QAAQ,aAAa,gBAAgB,UAAU;AACrD,aAAO,CAAC,cAAc;AAAA,IACxB;AAEA,QAAI,QAAQ,gBAAgB;AAC1B,YAAM,QAAQ,eAAe,cAAc,SAAS,QAAQ,QAAQ;AAAA,IACtE;AAAA,EACF;AAAA;AAAA,EAIA,MAAc,eACZ,SACA,MACe;AACf,QAAI;AACF,YAAM,QAAQ,aAAa;AAAA,QACzB,IAAI,KAAK;AAAA,QACT,SAAS,QAAQ;AAAA,QACjB,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE;AAAA,QAC/B,UAAU,EAAE,QAAQ,MAAM,KAAK,UAAU,GAAG;AAAA,MAC9C,CAAC;AACD,YAAM,QAAQ,WAAW;AAAA,QACvB,IAAI,KAAK;AAAA,QACT,MAAM,eAAe,KAAK,UAAU,IAAI,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA,QAC3D,QAAQ;AAAA,QACR,MAAM,YAAY;AAAA,QAClB,SAAS;AAAA,QACT,WAAW;AAAA,QACX,iBAAiB;AAAA,MACnB,CAAC;AACD,YAAM,QAAQ,eAAe,KAAK,UAAU,KAAK,MAAM;AACvD,YAAM,QAAQ,eAAe,QAAQ,SAAS,KAAK,MAAM;AAAA,IAC3D,SAAS,KAAK;AACZ,cAAQ,MAAM,mCAAmC,GAAG;AAAA,IACtD;AAAA,EACF;AACF;","names":[]}

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "@elizaos/plugin-xr",
-  "version": "2.0.3-beta.5",
+  "version": "2.0.3-beta.7",
   "type": "module",
   "description": "WebXR audio/video streaming for elizaOS — Quest 3 and XReal glasses",
   "main": "./dist/index.js",
+  "files": [
+    "dist",
+    "README.md"
+  ],
   "exports": {
     "./package.json": "./package.json",
     ".": {
@@ -22,6 +26,7 @@
     "lint": "bunx @biomejs/biome check src",
     "test": "bunx vitest run --config ./vitest.config.ts",
     "build": "bun run build:js && bun run build:types",
+    "prepack": "bun run build",
     "build:all": "bun run build && bun run simulator:build",
     "clean": "node ../../packages/scripts/rm-path-recursive.mjs dist simulator/dist",
     "build:js": "tsup --config ../tsup.plugin-packages.shared.ts",
@@ -30,8 +35,8 @@
     "simulator:watch": "cd simulator && bun run build:watch"
   },
   "dependencies": {
-    "@elizaos/agent": "2.0.3-beta.5",
-    "@elizaos/core": "2.0.3-beta.5",
+    "@elizaos/agent": "2.0.3-beta.7",
+    "@elizaos/core": "2.0.3-beta.7",
     "ws": "^8.18.0",
     "zod": "^4.4.3"
   },
@@ -53,5 +58,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "ff6157011c9459670021cc28a6797592a78b8817"
+  "gitHead": "61094f10458d11055c75b3dd0bae374e3f66bac5"
 }